xref: /titanic_44/usr/src/uts/common/io/ib/adapters/tavor/tavor_qpmod.c (revision de710d24d2fae4468e64da999e1d952a247f142c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * tavor_qpmod.c
29  *    Tavor Queue Pair Modify Routines
30  *
31  *    This contains all the routines necessary to implement the Tavor
32  *    ModifyQP() verb.  This includes all the code for legal transitions to
33  *    and from Reset, Init, RTR, RTS, SQD, SQErr, and Error.
34  */
35 
36 #include <sys/sysmacros.h>
37 #include <sys/types.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/bitmap.h>
43 
44 #include <sys/ib/adapters/tavor/tavor.h>
45 #include <sys/ib/ib_pkt_hdrs.h>
46 
47 static int tavor_qp_reset2init(tavor_state_t *state, tavor_qphdl_t qp,
48     ibt_qp_info_t *info_p);
49 static int tavor_qp_init2init(tavor_state_t *state, tavor_qphdl_t qp,
50     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
51 static int tavor_qp_init2rtr(tavor_state_t *state, tavor_qphdl_t qp,
52     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
53 static int tavor_qp_rtr2rts(tavor_state_t *state, tavor_qphdl_t qp,
54     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
55 static int tavor_qp_rts2rts(tavor_state_t *state, tavor_qphdl_t qp,
56     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
57 static int tavor_qp_rts2sqd(tavor_state_t *state, tavor_qphdl_t qp,
58     ibt_cep_modify_flags_t flags);
59 static int tavor_qp_sqd2rts(tavor_state_t *state, tavor_qphdl_t qp,
60     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
61 static int tavor_qp_sqd2sqd(tavor_state_t *state, tavor_qphdl_t qp,
62     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
63 static int tavor_qp_sqerr2rts(tavor_state_t *state, tavor_qphdl_t qp,
64     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
65 static int tavor_qp_to_error(tavor_state_t *state, tavor_qphdl_t qp);
66 static int tavor_qp_reset2err(tavor_state_t *state, tavor_qphdl_t qp);
67 
68 static uint_t tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,
69     ibt_qp_info_t *info_p, tavor_hw_qpc_t *qpc);
70 static int tavor_qp_validate_resp_rsrc(tavor_state_t *state,
71     ibt_qp_rc_attr_t *rc, uint_t *rra_max);
72 static int tavor_qp_validate_init_depth(tavor_state_t *state,
73     ibt_qp_rc_attr_t *rc, uint_t *sra_max);
74 static int tavor_qp_validate_mtu(tavor_state_t *state, uint_t mtu);
75 
76 /*
77  * tavor_qp_modify()
78  *    Context: Can be called from interrupt or base context.
79  */
80 /* ARGSUSED */
81 int
tavor_qp_modify(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p,ibt_queue_sizes_t * actual_sz)82 tavor_qp_modify(tavor_state_t *state, tavor_qphdl_t qp,
83     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p,
84     ibt_queue_sizes_t *actual_sz)
85 {
86 	ibt_cep_state_t		cur_state, mod_state;
87 	ibt_cep_modify_flags_t	okflags;
88 	int			status;
89 	char			*errormsg;
90 
91 	TAVOR_TNF_ENTER(tavor_qp_modify);
92 
93 	/*
94 	 * Lock the QP so that we can modify it atomically.  After grabbing
95 	 * the lock, get the current QP state.  We will use this current QP
96 	 * state to determine the legal transitions (and the checks that need
97 	 * to be performed.)
98 	 * Below you will find a case for every possible QP state.  In each
99 	 * case we check that no flags are set which are not valid for the
100 	 * possible transitions from that state.  If these tests pass (and if
101 	 * the state transition we are attempting is legal), then we call
102 	 * one of the helper functions.  Each of these functions does some
103 	 * additional setup before posting a Tavor firmware command for the
104 	 * appropriate state transition.
105 	 */
106 	mutex_enter(&qp->qp_lock);
107 
108 	/*
109 	 * Verify that the transport type matches between the serv_type and the
110 	 * qp_trans.  A caller to IBT must specify the qp_trans field as
111 	 * IBT_UD_SRV, IBT_RC_SRV, or IBT_UC_SRV, depending on the QP.  We
112 	 * check here that the correct value was specified, based on our
113 	 * understanding of the QP serv type.
114 	 *
115 	 * Because callers specify part of a 'union' based on what QP type they
116 	 * think they're working with, this ensures that we do not pickup bogus
117 	 * data if the caller thought they were working with a different QP
118 	 * type.
119 	 */
120 	if (!(TAVOR_QP_TYPE_VALID(info_p->qp_trans, qp->qp_serv_type))) {
121 		mutex_exit(&qp->qp_lock);
122 		TNF_PROBE_1(tavor_qp_modify_inv_qp_trans_fail,
123 		    TAVOR_TNF_ERROR, "", tnf_uint, qptrans,
124 		    info_p->qp_trans);
125 		TAVOR_TNF_EXIT(tavor_qp_modify);
126 		return (IBT_QP_SRV_TYPE_INVALID);
127 	}
128 
129 	/*
130 	 * If this is a transition to RTS (which is valid from RTR, RTS,
131 	 * SQError, and SQ Drain) then we should honor the "current QP state"
132 	 * specified by the consumer.  This means converting the IBTF QP state
133 	 * in "info_p->qp_current_state" to a Tavor QP state.  Otherwise, we
134 	 * assume that we already know the current state (i.e. whatever it was
135 	 * last modified to or queried as - in "qp->qp_state").
136 	 */
137 	mod_state = info_p->qp_state;
138 
139 	if (flags & IBT_CEP_SET_RTR_RTS) {
140 		cur_state = TAVOR_QP_RTR;		/* Ready to Receive */
141 
142 	} else if ((flags & IBT_CEP_SET_STATE) &&
143 	    (mod_state == IBT_STATE_RTS)) {
144 
145 		/* Convert the current IBTF QP state to a Tavor QP state */
146 		switch (info_p->qp_current_state) {
147 		case IBT_STATE_RTR:
148 			cur_state = TAVOR_QP_RTR;	/* Ready to Receive */
149 			break;
150 		case IBT_STATE_RTS:
151 			cur_state = TAVOR_QP_RTS;	/* Ready to Send */
152 			break;
153 		case IBT_STATE_SQE:
154 			cur_state = TAVOR_QP_SQERR;	/* Send Queue Error */
155 			break;
156 		case IBT_STATE_SQD:
157 			cur_state = TAVOR_QP_SQD;	/* SQ Drained */
158 			break;
159 		default:
160 			mutex_exit(&qp->qp_lock);
161 			TNF_PROBE_1(tavor_qp_modify_inv_currqpstate_fail,
162 			    TAVOR_TNF_ERROR, "", tnf_uint, qpstate,
163 			    info_p->qp_current_state);
164 			TAVOR_TNF_EXIT(tavor_qp_modify);
165 			return (IBT_QP_STATE_INVALID);
166 		}
167 	} else {
168 		cur_state = qp->qp_state;
169 	}
170 
171 	switch (cur_state) {
172 	case TAVOR_QP_RESET:
173 		okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RESET_INIT |
174 		    IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
175 		    IBT_CEP_SET_ATOMIC | IBT_CEP_SET_PKEY_IX |
176 		    IBT_CEP_SET_PORT | IBT_CEP_SET_QKEY);
177 
178 		/*
179 		 * Check for attempts to modify invalid attributes from the
180 		 * "Reset" state
181 		 */
182 		if (flags & ~okflags) {
183 			mutex_exit(&qp->qp_lock);
184 			/* Set "status" and "errormsg" and goto failure */
185 			TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "reset: invalid flag");
186 			goto qpmod_fail;
187 		}
188 
189 		/*
190 		 * Verify state transition is to either "Init", back to
191 		 * "Reset", or to "Error".
192 		 */
193 		if ((flags & IBT_CEP_SET_RESET_INIT) &&
194 		    (flags & IBT_CEP_SET_STATE) &&
195 		    (mod_state != IBT_STATE_INIT)) {
196 			/* Invalid transition - ambiguous flags */
197 			mutex_exit(&qp->qp_lock);
198 			/* Set "status" and "errormsg" and goto failure */
199 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
200 			    "reset: ambiguous flags");
201 			goto qpmod_fail;
202 
203 		} else if ((flags & IBT_CEP_SET_RESET_INIT) ||
204 		    ((flags & IBT_CEP_SET_STATE) &&
205 		    (mod_state == IBT_STATE_INIT))) {
206 			/*
207 			 * Attempt to transition from "Reset" to "Init"
208 			 */
209 			status = tavor_qp_reset2init(state, qp, info_p);
210 			if (status != DDI_SUCCESS) {
211 				mutex_exit(&qp->qp_lock);
212 				/* Set "status"/"errormsg", goto failure */
213 				TAVOR_TNF_FAIL(status, "reset to init");
214 				goto qpmod_fail;
215 			}
216 			qp->qp_state = TAVOR_QP_INIT;
217 
218 		} else if ((flags & IBT_CEP_SET_STATE) &&
219 		    (mod_state == IBT_STATE_RESET)) {
220 			/*
221 			 * Attempt to transition from "Reset" back to "Reset"
222 			 *    Nothing to do here really... just drop the lock
223 			 *    and return success.  The qp->qp_state should
224 			 *    already be set to TAVOR_QP_RESET.
225 			 *
226 			 * Note: We return here because we do not want to fall
227 			 *    through to the tavor_wrid_from_reset_handling()
228 			 *    routine below (since we are not really moving
229 			 *    _out_ of the "Reset" state.
230 			 */
231 			mutex_exit(&qp->qp_lock);
232 			TNF_PROBE_0_DEBUG(tavor_qp_modify_rst2rst,
233 			    TAVOR_TNF_TRACE, "");
234 			TAVOR_TNF_EXIT(tavor_qp_modify);
235 			return (DDI_SUCCESS);
236 
237 		} else if ((flags & IBT_CEP_SET_STATE) &&
238 		    (mod_state == IBT_STATE_ERROR)) {
239 			/*
240 			 * Attempt to transition from "Reset" to "Error"
241 			 */
242 			status = tavor_qp_reset2err(state, qp);
243 			if (status != DDI_SUCCESS) {
244 				mutex_exit(&qp->qp_lock);
245 				/* Set "status"/"errormsg", goto failure */
246 				TAVOR_TNF_FAIL(status, "reset to error");
247 				goto qpmod_fail;
248 			}
249 			qp->qp_state = TAVOR_QP_ERR;
250 
251 		} else {
252 			/* Invalid transition - return error */
253 			mutex_exit(&qp->qp_lock);
254 			/* Set "status" and "errormsg" and goto failure */
255 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
256 			    "reset: invalid transition");
257 			goto qpmod_fail;
258 		}
259 
260 		/*
261 		 * Do any additional handling necessary here for the transition
262 		 * from the "Reset" state (e.g. re-initialize the workQ WRID
263 		 * lists).  Note: If tavor_wrid_from_reset_handling() fails,
264 		 * then we attempt to transition the QP back to the "Reset"
265 		 * state.  If that fails, then it is an indication of a serious
266 		 * problem (either HW or SW).  So we print out a warning
267 		 * message and return failure.
268 		 */
269 		status = tavor_wrid_from_reset_handling(state, qp);
270 		if (status != DDI_SUCCESS) {
271 			if (tavor_qp_to_reset(state, qp) != DDI_SUCCESS) {
272 				TAVOR_WARNING(state, "failed to reset QP");
273 			}
274 			qp->qp_state = TAVOR_QP_RESET;
275 
276 			mutex_exit(&qp->qp_lock);
277 			/* Set "status" and "errormsg" and goto failure */
278 			TAVOR_TNF_FAIL(status, "reset: wrid_from_reset hdl");
279 			goto qpmod_fail;
280 		}
281 		break;
282 
283 	case TAVOR_QP_INIT:
284 		okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_INIT_RTR |
285 		    IBT_CEP_SET_ADDS_VECT | IBT_CEP_SET_RDMARA_IN |
286 		    IBT_CEP_SET_MIN_RNR_NAK | IBT_CEP_SET_ALT_PATH |
287 		    IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
288 		    IBT_CEP_SET_ATOMIC | IBT_CEP_SET_PKEY_IX |
289 		    IBT_CEP_SET_QKEY | IBT_CEP_SET_PORT);
290 
291 		/*
292 		 * Check for attempts to modify invalid attributes from the
293 		 * "Init" state
294 		 */
295 		if (flags & ~okflags) {
296 			mutex_exit(&qp->qp_lock);
297 			/* Set "status" and "errormsg" and goto failure */
298 			TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "init: invalid flag");
299 			goto qpmod_fail;
300 		}
301 
302 		/*
303 		 * Verify state transition is to either "RTR", back to "Init",
304 		 * to "Reset", or to "Error"
305 		 */
306 		if ((flags & IBT_CEP_SET_INIT_RTR) &&
307 		    (flags & IBT_CEP_SET_STATE) &&
308 		    (mod_state != IBT_STATE_RTR)) {
309 			/* Invalid transition - ambiguous flags */
310 			mutex_exit(&qp->qp_lock);
311 			/* Set "status" and "errormsg" and goto failure */
312 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
313 			    "init: ambiguous flags");
314 			goto qpmod_fail;
315 
316 		} else if ((flags & IBT_CEP_SET_INIT_RTR) ||
317 		    ((flags & IBT_CEP_SET_STATE) &&
318 		    (mod_state == IBT_STATE_RTR))) {
319 			/*
320 			 * Attempt to transition from "Init" to "RTR"
321 			 */
322 			status = tavor_qp_init2rtr(state, qp, flags, info_p);
323 			if (status != DDI_SUCCESS) {
324 				mutex_exit(&qp->qp_lock);
325 				/* Set "status"/"errormsg", goto failure */
326 				TAVOR_TNF_FAIL(status, "init to rtr");
327 				goto qpmod_fail;
328 			}
329 			qp->qp_state = TAVOR_QP_RTR;
330 
331 		} else if ((flags & IBT_CEP_SET_STATE) &&
332 		    (mod_state == IBT_STATE_INIT)) {
333 			/*
334 			 * Attempt to transition from "Init" to "Init"
335 			 */
336 			status = tavor_qp_init2init(state, qp, flags, info_p);
337 			if (status != DDI_SUCCESS) {
338 				mutex_exit(&qp->qp_lock);
339 				/* Set "status"/"errormsg", goto failure */
340 				TAVOR_TNF_FAIL(status, "init to init");
341 				goto qpmod_fail;
342 			}
343 			qp->qp_state = TAVOR_QP_INIT;
344 
345 		} else if ((flags & IBT_CEP_SET_STATE) &&
346 		    (mod_state == IBT_STATE_RESET)) {
347 			/*
348 			 * Attempt to transition from "Init" to "Reset"
349 			 */
350 			status = tavor_qp_to_reset(state, qp);
351 			if (status != DDI_SUCCESS) {
352 				mutex_exit(&qp->qp_lock);
353 				/* Set "status"/"errormsg", goto failure */
354 				TAVOR_TNF_FAIL(status, "init to reset");
355 				goto qpmod_fail;
356 			}
357 			qp->qp_state = TAVOR_QP_RESET;
358 
359 			/*
360 			 * Do any additional handling necessary for the
361 			 * transition _to_ the "Reset" state (e.g. update the
362 			 * workQ WRID lists)
363 			 */
364 			tavor_wrid_to_reset_handling(state, qp);
365 
366 		} else if ((flags & IBT_CEP_SET_STATE) &&
367 		    (mod_state == IBT_STATE_ERROR)) {
368 			/*
369 			 * Attempt to transition from "Init" to "Error"
370 			 */
371 			status = tavor_qp_to_error(state, qp);
372 			if (status != DDI_SUCCESS) {
373 				mutex_exit(&qp->qp_lock);
374 				/* Set "status"/"errormsg", goto failure */
375 				TAVOR_TNF_FAIL(status, "init to error");
376 				goto qpmod_fail;
377 			}
378 			qp->qp_state = TAVOR_QP_ERR;
379 
380 		} else {
381 			/* Invalid transition - return error */
382 			mutex_exit(&qp->qp_lock);
383 			/* Set "status" and "errormsg" and goto failure */
384 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
385 			    "init: invalid transition");
386 			goto qpmod_fail;
387 		}
388 		break;
389 
390 	case TAVOR_QP_RTR:
391 		okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RTR_RTS |
392 		    IBT_CEP_SET_TIMEOUT | IBT_CEP_SET_RETRY |
393 		    IBT_CEP_SET_RNR_NAK_RETRY | IBT_CEP_SET_RDMARA_OUT |
394 		    IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
395 		    IBT_CEP_SET_ATOMIC | IBT_CEP_SET_QKEY |
396 		    IBT_CEP_SET_ALT_PATH | IBT_CEP_SET_MIG |
397 		    IBT_CEP_SET_MIN_RNR_NAK);
398 
399 		/*
400 		 * Check for attempts to modify invalid attributes from the
401 		 * "RTR" state
402 		 */
403 		if (flags & ~okflags) {
404 			mutex_exit(&qp->qp_lock);
405 			/* Set "status" and "errormsg" and goto failure */
406 			TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "rtr: invalid flag");
407 			goto qpmod_fail;
408 		}
409 
410 		/*
411 		 * Verify state transition is to either "RTS", "Reset",
412 		 * or "Error"
413 		 */
414 		if ((flags & IBT_CEP_SET_RTR_RTS) &&
415 		    (flags & IBT_CEP_SET_STATE) &&
416 		    (mod_state != IBT_STATE_RTS)) {
417 			/* Invalid transition - ambiguous flags */
418 			mutex_exit(&qp->qp_lock);
419 			/* Set "status" and "errormsg" and goto failure */
420 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
421 			    "reset: ambiguous flags");
422 			goto qpmod_fail;
423 
424 		} else if ((flags & IBT_CEP_SET_RTR_RTS) ||
425 		    ((flags & IBT_CEP_SET_STATE) &&
426 		    (mod_state == IBT_STATE_RTS))) {
427 			/*
428 			 * Attempt to transition from "RTR" to "RTS"
429 			 */
430 			status = tavor_qp_rtr2rts(state, qp, flags, info_p);
431 			if (status != DDI_SUCCESS) {
432 				mutex_exit(&qp->qp_lock);
433 				/* Set "status"/"errormsg", goto failure */
434 				TAVOR_TNF_FAIL(status, "rtr to rts");
435 				goto qpmod_fail;
436 			}
437 			qp->qp_state = TAVOR_QP_RTS;
438 
439 		} else if ((flags & IBT_CEP_SET_STATE) &&
440 		    (mod_state == IBT_STATE_RESET)) {
441 			/*
442 			 * Attempt to transition from "RTR" to "Reset"
443 			 */
444 			status = tavor_qp_to_reset(state, qp);
445 			if (status != DDI_SUCCESS) {
446 				mutex_exit(&qp->qp_lock);
447 				/* Set "status"/"errormsg", goto failure */
448 				TAVOR_TNF_FAIL(status, "rtr to reset");
449 				goto qpmod_fail;
450 			}
451 			qp->qp_state = TAVOR_QP_RESET;
452 
453 			/*
454 			 * Do any additional handling necessary for the
455 			 * transition _to_ the "Reset" state (e.g. update the
456 			 * workQ WRID lists)
457 			 */
458 			tavor_wrid_to_reset_handling(state, qp);
459 
460 		} else if ((flags & IBT_CEP_SET_STATE) &&
461 		    (mod_state == IBT_STATE_ERROR)) {
462 			/*
463 			 * Attempt to transition from "RTR" to "Error"
464 			 */
465 			status = tavor_qp_to_error(state, qp);
466 			if (status != DDI_SUCCESS) {
467 				mutex_exit(&qp->qp_lock);
468 				/* Set "status"/"errormsg", goto failure */
469 				TAVOR_TNF_FAIL(status, "rtr to error");
470 				goto qpmod_fail;
471 			}
472 			qp->qp_state = TAVOR_QP_ERR;
473 
474 		} else {
475 			/* Invalid transition - return error */
476 			mutex_exit(&qp->qp_lock);
477 			/* Set "status" and "errormsg" and goto failure */
478 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
479 			    "rtr: invalid transition");
480 			goto qpmod_fail;
481 		}
482 		break;
483 
484 	case TAVOR_QP_RTS:
485 		okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RDMA_R |
486 		    IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC |
487 		    IBT_CEP_SET_QKEY | IBT_CEP_SET_ALT_PATH |
488 		    IBT_CEP_SET_MIG | IBT_CEP_SET_MIN_RNR_NAK |
489 		    IBT_CEP_SET_SQD_EVENT);
490 
491 		/*
492 		 * Check for attempts to modify invalid attributes from the
493 		 * "RTS" state
494 		 */
495 		if (flags & ~okflags) {
496 			mutex_exit(&qp->qp_lock);
497 			/* Set "status" and "errormsg" and goto failure */
498 			TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "rts: invalid flag");
499 			goto qpmod_fail;
500 		}
501 
502 		/*
503 		 * Verify state transition is to either "RTS", "SQD", "Reset",
504 		 * or "Error"
505 		 */
506 		if ((flags & IBT_CEP_SET_STATE) &&
507 		    (mod_state == IBT_STATE_RTS)) {
508 			/*
509 			 * Attempt to transition from "RTS" to "RTS"
510 			 */
511 			status = tavor_qp_rts2rts(state, qp, flags, info_p);
512 			if (status != DDI_SUCCESS) {
513 				mutex_exit(&qp->qp_lock);
514 				/* Set "status"/"errormsg", goto failure */
515 				TAVOR_TNF_FAIL(status, "rts to rts");
516 				goto qpmod_fail;
517 			}
518 			/* qp->qp_state = TAVOR_QP_RTS; */
519 
520 		} else if ((flags & IBT_CEP_SET_STATE) &&
521 		    (mod_state == IBT_STATE_SQD)) {
522 			/*
523 			 * Attempt to transition from "RTS" to "SQD"
524 			 */
525 			status = tavor_qp_rts2sqd(state, qp, flags);
526 			if (status != DDI_SUCCESS) {
527 				mutex_exit(&qp->qp_lock);
528 				/* Set "status"/"errormsg", goto failure */
529 				TAVOR_TNF_FAIL(status, "rts to sqd");
530 				goto qpmod_fail;
531 			}
532 			qp->qp_state = TAVOR_QP_SQD;
533 
534 		} else if ((flags & IBT_CEP_SET_STATE) &&
535 		    (mod_state == IBT_STATE_RESET)) {
536 			/*
537 			 * Attempt to transition from "RTS" to "Reset"
538 			 */
539 			status = tavor_qp_to_reset(state, qp);
540 			if (status != DDI_SUCCESS) {
541 				mutex_exit(&qp->qp_lock);
542 				/* Set "status"/"errormsg", goto failure */
543 				TAVOR_TNF_FAIL(status, "rts to reset");
544 				goto qpmod_fail;
545 			}
546 			qp->qp_state = TAVOR_QP_RESET;
547 
548 			/*
549 			 * Do any additional handling necessary for the
550 			 * transition _to_ the "Reset" state (e.g. update the
551 			 * workQ WRID lists)
552 			 */
553 			tavor_wrid_to_reset_handling(state, qp);
554 
555 		} else if ((flags & IBT_CEP_SET_STATE) &&
556 		    (mod_state == IBT_STATE_ERROR)) {
557 			/*
558 			 * Attempt to transition from "RTS" to "Error"
559 			 */
560 			status = tavor_qp_to_error(state, qp);
561 			if (status != DDI_SUCCESS) {
562 				mutex_exit(&qp->qp_lock);
563 				/* Set "status"/"errormsg", goto failure */
564 				TAVOR_TNF_FAIL(status, "rts to error");
565 				goto qpmod_fail;
566 			}
567 			qp->qp_state = TAVOR_QP_ERR;
568 
569 		} else {
570 			/* Invalid transition - return error */
571 			mutex_exit(&qp->qp_lock);
572 			/* Set "status" and "errormsg" and goto failure */
573 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
574 			    "rts: invalid transition");
575 			goto qpmod_fail;
576 		}
577 		break;
578 
579 	case TAVOR_QP_SQERR:
580 		okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RDMA_R |
581 		    IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC |
582 		    IBT_CEP_SET_QKEY | IBT_CEP_SET_MIN_RNR_NAK);
583 
584 		/*
585 		 * Check for attempts to modify invalid attributes from the
586 		 * "SQErr" state
587 		 */
588 		if (flags & ~okflags) {
589 			mutex_exit(&qp->qp_lock);
590 			/* Set "status" and "errormsg" and goto failure */
591 			TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "sqerr: invalid flag");
592 			goto qpmod_fail;
593 		}
594 
595 		/*
596 		 * Verify state transition is to either "RTS", "Reset", or
597 		 * "Error"
598 		 */
599 		if ((flags & IBT_CEP_SET_STATE) &&
600 		    (mod_state == IBT_STATE_RTS)) {
601 			/*
602 			 * Attempt to transition from "SQErr" to "RTS"
603 			 */
604 			status = tavor_qp_sqerr2rts(state, qp, flags, info_p);
605 			if (status != DDI_SUCCESS) {
606 				mutex_exit(&qp->qp_lock);
607 				/* Set "status"/"errormsg", goto failure */
608 				TAVOR_TNF_FAIL(status, "sqerr to rts");
609 				goto qpmod_fail;
610 			}
611 			qp->qp_state = TAVOR_QP_RTS;
612 
613 		} else if ((flags & IBT_CEP_SET_STATE) &&
614 		    (mod_state == IBT_STATE_RESET)) {
615 			/*
616 			 * Attempt to transition from "SQErr" to "Reset"
617 			 */
618 			status = tavor_qp_to_reset(state, qp);
619 			if (status != DDI_SUCCESS) {
620 				mutex_exit(&qp->qp_lock);
621 				/* Set "status"/"errormsg", goto failure */
622 				TAVOR_TNF_FAIL(status, "sqerr to reset");
623 				goto qpmod_fail;
624 			}
625 			qp->qp_state = TAVOR_QP_RESET;
626 
627 			/*
628 			 * Do any additional handling necessary for the
629 			 * transition _to_ the "Reset" state (e.g. update the
630 			 * workQ WRID lists)
631 			 */
632 			tavor_wrid_to_reset_handling(state, qp);
633 
634 		} else if ((flags & IBT_CEP_SET_STATE) &&
635 		    (mod_state == IBT_STATE_ERROR)) {
636 			/*
637 			 * Attempt to transition from "SQErr" to "Error"
638 			 */
639 			status = tavor_qp_to_error(state, qp);
640 			if (status != DDI_SUCCESS) {
641 				mutex_exit(&qp->qp_lock);
642 				/* Set "status"/"errormsg", goto failure */
643 				TAVOR_TNF_FAIL(status, "sqerr to error");
644 				goto qpmod_fail;
645 			}
646 			qp->qp_state = TAVOR_QP_ERR;
647 
648 		} else {
649 			/* Invalid transition - return error */
650 			mutex_exit(&qp->qp_lock);
651 			/* Set "status" and "errormsg" and goto failure */
652 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
653 			    "sqerr: invalid transition");
654 			goto qpmod_fail;
655 		}
656 		break;
657 
658 	case TAVOR_QP_SQD:
659 		okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_ADDS_VECT |
660 		    IBT_CEP_SET_ALT_PATH | IBT_CEP_SET_MIG |
661 		    IBT_CEP_SET_RDMARA_OUT | IBT_CEP_SET_RDMARA_IN |
662 		    IBT_CEP_SET_QKEY | IBT_CEP_SET_PKEY_IX |
663 		    IBT_CEP_SET_TIMEOUT | IBT_CEP_SET_RETRY |
664 		    IBT_CEP_SET_RNR_NAK_RETRY | IBT_CEP_SET_PORT |
665 		    IBT_CEP_SET_MIN_RNR_NAK | IBT_CEP_SET_RDMA_R |
666 		    IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC);
667 
668 		/*
669 		 * Check for attempts to modify invalid attributes from the
670 		 * "SQD" state
671 		 */
672 		if (flags & ~okflags) {
673 			mutex_exit(&qp->qp_lock);
674 			/* Set "status" and "errormsg" and goto failure */
675 			TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "sqd: invalid flag");
676 			goto qpmod_fail;
677 		}
678 
679 		/*
680 		 * Verify state transition is to either "SQD", "RTS", "Reset",
681 		 * or "Error"
682 		 */
683 
684 		if ((flags & IBT_CEP_SET_STATE) &&
685 		    (mod_state == IBT_STATE_SQD)) {
686 			/*
687 			 * Attempt to transition from "SQD" to "SQD"
688 			 */
689 			status = tavor_qp_sqd2sqd(state, qp, flags, info_p);
690 			if (status != DDI_SUCCESS) {
691 				mutex_exit(&qp->qp_lock);
692 				/* Set "status"/"errormsg", goto failure */
693 				TAVOR_TNF_FAIL(status, "sqd to sqd");
694 				goto qpmod_fail;
695 			}
696 			qp->qp_state = TAVOR_QP_SQD;
697 
698 		} else if ((flags & IBT_CEP_SET_STATE) &&
699 		    (mod_state == IBT_STATE_RTS)) {
700 			/*
701 			 * If still draining SQ, then fail transition attempt
702 			 * to RTS.
703 			 */
704 			if (qp->qp_sqd_still_draining) {
705 				mutex_exit(&qp->qp_lock);
706 				/* Set "status"/"errormsg", goto failure */
707 				status = IBT_QP_STATE_INVALID;
708 				TAVOR_TNF_FAIL(status, "sqd to rts; draining");
709 				goto qpmod_fail;
710 			}
711 
712 			/*
713 			 * Attempt to transition from "SQD" to "RTS"
714 			 */
715 			status = tavor_qp_sqd2rts(state, qp, flags, info_p);
716 			if (status != DDI_SUCCESS) {
717 				mutex_exit(&qp->qp_lock);
718 				/* Set "status"/"errormsg", goto failure */
719 				TAVOR_TNF_FAIL(status, "sqd to rts");
720 				goto qpmod_fail;
721 			}
722 			qp->qp_state = TAVOR_QP_RTS;
723 
724 		} else if ((flags & IBT_CEP_SET_STATE) &&
725 		    (mod_state == IBT_STATE_RESET)) {
726 			/*
727 			 * Attempt to transition from "SQD" to "Reset"
728 			 */
729 			status = tavor_qp_to_reset(state, qp);
730 			if (status != DDI_SUCCESS) {
731 				mutex_exit(&qp->qp_lock);
732 				/* Set "status"/"errormsg", goto failure */
733 				TAVOR_TNF_FAIL(status, "sqd to reset");
734 				goto qpmod_fail;
735 			}
736 			qp->qp_state = TAVOR_QP_RESET;
737 
738 			/*
739 			 * Do any additional handling necessary for the
740 			 * transition _to_ the "Reset" state (e.g. update the
741 			 * workQ WRID lists)
742 			 */
743 			tavor_wrid_to_reset_handling(state, qp);
744 
745 		} else if ((flags & IBT_CEP_SET_STATE) &&
746 		    (mod_state == IBT_STATE_ERROR)) {
747 			/*
748 			 * Attempt to transition from "SQD" to "Error"
749 			 */
750 			status = tavor_qp_to_error(state, qp);
751 			if (status != DDI_SUCCESS) {
752 				mutex_exit(&qp->qp_lock);
753 				/* Set "status"/"errormsg", goto failure */
754 				TAVOR_TNF_FAIL(status, "sqd to error");
755 				goto qpmod_fail;
756 			}
757 			qp->qp_state = TAVOR_QP_ERR;
758 
759 		} else {
760 			/* Invalid transition - return error */
761 			mutex_exit(&qp->qp_lock);
762 			/* Set "status" and "errormsg" and goto failure */
763 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
764 			    "sqd: invalid transition");
765 			goto qpmod_fail;
766 		}
767 		break;
768 
769 	case TAVOR_QP_ERR:
770 		/*
771 		 * Verify state transition is to either "Reset" or back to
772 		 * "Error"
773 		 */
774 		if ((flags & IBT_CEP_SET_STATE) &&
775 		    (mod_state == IBT_STATE_RESET)) {
776 			/*
777 			 * Attempt to transition from "Error" to "Reset"
778 			 */
779 			status = tavor_qp_to_reset(state, qp);
780 			if (status != DDI_SUCCESS) {
781 				mutex_exit(&qp->qp_lock);
782 				/* Set "status"/"errormsg", goto failure */
783 				TAVOR_TNF_FAIL(status, "error to reset");
784 				goto qpmod_fail;
785 			}
786 			qp->qp_state = TAVOR_QP_RESET;
787 
788 			/*
789 			 * Do any additional handling necessary for the
790 			 * transition _to_ the "Reset" state (e.g. update the
791 			 * workQ WRID lists)
792 			 */
793 			tavor_wrid_to_reset_handling(state, qp);
794 
795 		} else if ((flags & IBT_CEP_SET_STATE) &&
796 		    (mod_state == IBT_STATE_ERROR)) {
797 			/*
798 			 * Attempt to transition from "Error" back to "Error"
799 			 *    Nothing to do here really... just drop the lock
800 			 *    and return success.  The qp->qp_state should
801 			 *    already be set to TAVOR_QP_ERR.
802 			 *
803 			 */
804 			mutex_exit(&qp->qp_lock);
805 			TNF_PROBE_0_DEBUG(tavor_qp_modify_err2err,
806 			    TAVOR_TNF_TRACE, "");
807 			TAVOR_TNF_EXIT(tavor_qp_modify);
808 			return (DDI_SUCCESS);
809 
810 		} else {
811 			/* Invalid transition - return error */
812 			mutex_exit(&qp->qp_lock);
813 			/* Set "status" and "errormsg" and goto failure */
814 			TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
815 			    "error: invalid transition");
816 			goto qpmod_fail;
817 		}
818 		break;
819 
820 	default:
821 		/*
822 		 * Invalid QP state.  If we got here then it's a warning of
823 		 * a probably serious problem.  So print a message and return
824 		 * failure
825 		 */
826 		mutex_exit(&qp->qp_lock);
827 		TAVOR_WARNING(state, "unknown QP state in modify");
828 		/* Set "status" and "errormsg" and goto failure */
829 		TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID, "invalid curr QP state");
830 		goto qpmod_fail;
831 	}
832 
833 	mutex_exit(&qp->qp_lock);
834 	TAVOR_TNF_EXIT(tavor_qp_modify);
835 	return (DDI_SUCCESS);
836 
837 qpmod_fail:
838 	TNF_PROBE_1(tavor_qp_modify_fail, TAVOR_TNF_ERROR, "",
839 	    tnf_string, msg, errormsg);
840 	TAVOR_TNF_EXIT(tavor_qp_modify);
841 	return (status);
842 }
843 
844 
845 /*
846  * tavor_qp_reset2init()
847  *    Context: Can be called from interrupt or base context.
848  */
849 static int
tavor_qp_reset2init(tavor_state_t * state,tavor_qphdl_t qp,ibt_qp_info_t * info_p)850 tavor_qp_reset2init(tavor_state_t *state, tavor_qphdl_t qp,
851     ibt_qp_info_t *info_p)
852 {
853 	tavor_hw_qpc_t		*qpc;
854 	ibt_qp_rc_attr_t	*rc;
855 	ibt_qp_ud_attr_t	*ud;
856 	ibt_qp_uc_attr_t	*uc;
857 	uint_t			portnum, pkeyindx;
858 	int			status;
859 
860 	TAVOR_TNF_ENTER(tavor_qp_reset2init);
861 
862 	ASSERT(MUTEX_HELD(&qp->qp_lock));
863 
864 	/*
865 	 * Grab the temporary QPC entry from QP software state
866 	 */
867 	qpc = &qp->qpc;
868 
869 	/*
870 	 * Fill in the common and/or Tavor-specific fields in the QPC
871 	 */
872 	if (qp->qp_is_special) {
873 		qpc->serv_type	= TAVOR_QP_MLX;
874 	} else {
875 		qpc->serv_type	= qp->qp_serv_type;
876 	}
877 	qpc->pm_state		= TAVOR_QP_PMSTATE_MIGRATED;
878 	qpc->de			= TAVOR_QP_DESC_EVT_ENABLED;
879 	qpc->sched_q		= TAVOR_QP_SCHEDQ_GET(qp->qp_qpnum);
880 	if (qp->qp_is_umap) {
881 		qpc->usr_page = qp->qp_uarpg;
882 	} else {
883 		qpc->usr_page = 0;
884 	}
885 	qpc->pd			= qp->qp_pdhdl->pd_pdnum;
886 	qpc->wqe_baseaddr	= 0;
887 	qpc->wqe_lkey		= qp->qp_mrhdl->mr_lkey;
888 	qpc->ssc		= qp->qp_sq_sigtype;
889 	qpc->cqn_snd		= qp->qp_sq_cqhdl->cq_cqnum;
890 	qpc->rsc		= TAVOR_QP_RQ_ALL_SIGNALED;
891 	qpc->cqn_rcv		= qp->qp_rq_cqhdl->cq_cqnum;
892 	qpc->srq_en		= qp->qp_srq_en;
893 
894 	if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
895 		qpc->srq_number	= qp->qp_srqhdl->srq_srqnum;
896 	} else {
897 		qpc->srq_number = 0;
898 	}
899 
900 	/*
901 	 * Now fill in the QPC fields which are specific to transport type
902 	 */
903 	if (qp->qp_serv_type == TAVOR_QP_UD) {
904 		ud = &info_p->qp_transport.ud;
905 
906 		/* Set the QKey */
907 		qpc->qkey = ud->ud_qkey;
908 
909 		/* Check for valid port number and fill it in */
910 		portnum = ud->ud_port;
911 		if (tavor_portnum_is_valid(state, portnum)) {
912 			qpc->pri_addr_path.portnum = portnum;
913 		} else {
914 			TNF_PROBE_1(tavor_qp_reset2init_inv_port_fail,
915 			    TAVOR_TNF_ERROR, "", tnf_uint, port, portnum);
916 			TAVOR_TNF_EXIT(tavor_qp_reset2init);
917 			return (IBT_HCA_PORT_INVALID);
918 		}
919 
920 		/* Check for valid PKey index and fill it in */
921 		pkeyindx = ud->ud_pkey_ix;
922 		if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
923 			qpc->pri_addr_path.pkey_indx = pkeyindx;
924 			qp->qp_pkeyindx = pkeyindx;
925 		} else {
926 			TNF_PROBE_1(tavor_qp_reset2init_inv_pkey_fail,
927 			    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx, pkeyindx);
928 			TAVOR_TNF_EXIT(tavor_qp_reset2init);
929 			return (IBT_PKEY_IX_ILLEGAL);
930 		}
931 
932 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
933 		rc = &info_p->qp_transport.rc;
934 
935 		/* Set the RDMA (recv) enable/disable flags */
936 		qpc->rre = (info_p->qp_flags & IBT_CEP_RDMA_RD) ? 1 : 0;
937 		qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
938 		qpc->rae = (info_p->qp_flags & IBT_CEP_ATOMIC)  ? 1 : 0;
939 
940 		/* Check for valid port number and fill it in */
941 		portnum = rc->rc_path.cep_hca_port_num;
942 		if (tavor_portnum_is_valid(state, portnum)) {
943 			qpc->pri_addr_path.portnum = portnum;
944 		} else {
945 			TNF_PROBE_1(tavor_qp_reset2init_inv_port_fail,
946 			    TAVOR_TNF_ERROR, "", tnf_uint, port, portnum);
947 			TAVOR_TNF_EXIT(tavor_qp_reset2init);
948 			return (IBT_HCA_PORT_INVALID);
949 		}
950 
951 		/* Check for valid PKey index and fill it in */
952 		pkeyindx = rc->rc_path.cep_pkey_ix;
953 		if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
954 			qpc->pri_addr_path.pkey_indx = pkeyindx;
955 		} else {
956 			TNF_PROBE_1(tavor_qp_reset2init_inv_pkey_fail,
957 			    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx, pkeyindx);
958 			TAVOR_TNF_EXIT(tavor_qp_reset2init);
959 			return (IBT_PKEY_IX_ILLEGAL);
960 		}
961 
962 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
963 		uc = &info_p->qp_transport.uc;
964 
965 		/*
966 		 * Set the RDMA (recv) enable/disable flags.  Note: RDMA Read
967 		 * and Atomic are ignored by default.
968 		 */
969 		qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
970 
971 		/* Check for valid port number and fill it in */
972 		portnum = uc->uc_path.cep_hca_port_num;
973 		if (tavor_portnum_is_valid(state, portnum)) {
974 			qpc->pri_addr_path.portnum = portnum;
975 		} else {
976 			TNF_PROBE_1(tavor_qp_reset2init_inv_port_fail,
977 			    TAVOR_TNF_ERROR, "", tnf_uint, port, portnum);
978 			TAVOR_TNF_EXIT(tavor_qp_reset2init);
979 			return (IBT_HCA_PORT_INVALID);
980 		}
981 
982 		/* Check for valid PKey index and fill it in */
983 		pkeyindx = uc->uc_path.cep_pkey_ix;
984 		if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
985 			qpc->pri_addr_path.pkey_indx = pkeyindx;
986 		} else {
987 			TNF_PROBE_1(tavor_qp_reset2init_inv_pkey_fail,
988 			    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx, pkeyindx);
989 			TAVOR_TNF_EXIT(tavor_qp_reset2init);
990 			return (IBT_PKEY_IX_ILLEGAL);
991 		}
992 	} else {
993 		/*
994 		 * Invalid QP transport type. If we got here then it's a
995 		 * warning of a probably serious problem.  So print a message
996 		 * and return failure
997 		 */
998 		TAVOR_WARNING(state, "unknown QP transport type in rst2init");
999 		TNF_PROBE_0(tavor_qp_reset2init_inv_transtype_fail,
1000 		    TAVOR_TNF_ERROR, "");
1001 		TAVOR_TNF_EXIT(tavor_qp_reset2init);
1002 		return (ibc_get_ci_failure(0));
1003 	}
1004 
1005 	/*
1006 	 * Post the RST2INIT_QP command to the Tavor firmware
1007 	 *
1008 	 * We do a TAVOR_NOSLEEP here because we are still holding the
1009 	 * "qp_lock".  If we got raised to interrupt level by priority
1010 	 * inversion, we do not want to block in this routine waiting for
1011 	 * success.
1012 	 */
1013 	status = tavor_cmn_qp_cmd_post(state, RST2INIT_QP, qpc, qp->qp_qpnum,
1014 	    0, TAVOR_CMD_NOSLEEP_SPIN);
1015 	if (status != TAVOR_CMD_SUCCESS) {
1016 		cmn_err(CE_CONT, "Tavor: RST2INIT_QP command failed: %08x\n",
1017 		    status);
1018 		TNF_PROBE_1(tavor_qp_reset2init_cmd_fail, TAVOR_TNF_ERROR, "",
1019 		    tnf_uint, status, status);
1020 		TAVOR_TNF_EXIT(tavor_qp_reset2init);
1021 		return (ibc_get_ci_failure(0));
1022 	}
1023 
1024 	TAVOR_TNF_EXIT(tavor_qp_reset2init);
1025 	return (DDI_SUCCESS);
1026 }
1027 
1028 
1029 /*
1030  * tavor_qp_init2init()
1031  *    Context: Can be called from interrupt or base context.
1032  */
1033 static int
tavor_qp_init2init(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)1034 tavor_qp_init2init(tavor_state_t *state, tavor_qphdl_t qp,
1035     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1036 {
1037 	tavor_hw_qpc_t		*qpc;
1038 	ibt_qp_rc_attr_t	*rc;
1039 	ibt_qp_ud_attr_t	*ud;
1040 	ibt_qp_uc_attr_t	*uc;
1041 	uint_t			portnum, pkeyindx;
1042 	uint32_t		opmask = 0;
1043 	int			status;
1044 
1045 	TAVOR_TNF_ENTER(tavor_qp_init2init);
1046 
1047 	ASSERT(MUTEX_HELD(&qp->qp_lock));
1048 
1049 	/*
1050 	 * Grab the temporary QPC entry from QP software state
1051 	 */
1052 	qpc = &qp->qpc;
1053 
1054 	/*
1055 	 * Since there are no common and/or Tavor-specific fields to be filled
1056 	 * in for this command, we begin with the QPC fields which are
1057 	 * specific to transport type.
1058 	 */
1059 	if (qp->qp_serv_type == TAVOR_QP_UD) {
1060 		ud = &info_p->qp_transport.ud;
1061 
1062 		/*
1063 		 * If we are attempting to modify the port for this QP, then
1064 		 * check for valid port number and fill it in.  Also set the
1065 		 * appropriate flag in the "opmask" parameter.
1066 		 */
1067 		if (flags & IBT_CEP_SET_PORT) {
1068 			portnum = ud->ud_port;
1069 			if (tavor_portnum_is_valid(state, portnum)) {
1070 				qpc->pri_addr_path.portnum = portnum;
1071 			} else {
1072 				TNF_PROBE_1(tavor_qp_init2init_inv_port_fail,
1073 				    TAVOR_TNF_ERROR, "", tnf_uint, port,
1074 				    portnum);
1075 				TAVOR_TNF_EXIT(tavor_qp_init2init);
1076 				return (IBT_HCA_PORT_INVALID);
1077 			}
1078 			opmask |= TAVOR_CMD_OP_PRIM_PORT;
1079 		}
1080 
1081 		/*
1082 		 * If we are attempting to modify the PKey index for this QP,
1083 		 * then check for valid PKey index and fill it in.  Also set
1084 		 * the appropriate flag in the "opmask" parameter.
1085 		 */
1086 		if (flags & IBT_CEP_SET_PKEY_IX) {
1087 			pkeyindx = ud->ud_pkey_ix;
1088 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1089 				qpc->pri_addr_path.pkey_indx = pkeyindx;
1090 				opmask |= TAVOR_CMD_OP_PKEYINDX;
1091 				qp->qp_pkeyindx = pkeyindx;
1092 			} else {
1093 				TNF_PROBE_1(tavor_qp_init2init_inv_pkey_fail,
1094 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1095 				    pkeyindx);
1096 				TAVOR_TNF_EXIT(tavor_qp_init2init);
1097 				return (IBT_PKEY_IX_ILLEGAL);
1098 			}
1099 		}
1100 
1101 		/*
1102 		 * If we are attempting to modify the QKey for this QP, then
1103 		 * fill it in and set the appropriate flag in the "opmask"
1104 		 * parameter.
1105 		 */
1106 		if (flags & IBT_CEP_SET_QKEY) {
1107 			qpc->qkey = ud->ud_qkey;
1108 			opmask |= TAVOR_CMD_OP_QKEY;
1109 		}
1110 
1111 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
1112 		rc = &info_p->qp_transport.rc;
1113 
1114 		/*
1115 		 * If we are attempting to modify the port for this QP, then
1116 		 * check for valid port number and fill it in.  Also set the
1117 		 * appropriate flag in the "opmask" parameter.
1118 		 */
1119 		if (flags & IBT_CEP_SET_PORT) {
1120 			portnum = rc->rc_path.cep_hca_port_num;
1121 			if (tavor_portnum_is_valid(state, portnum)) {
1122 				qpc->pri_addr_path.portnum = portnum;
1123 			} else {
1124 				TNF_PROBE_1(tavor_qp_init2init_inv_port_fail,
1125 				    TAVOR_TNF_ERROR, "", tnf_uint, port,
1126 				    portnum);
1127 				TAVOR_TNF_EXIT(tavor_qp_init2init);
1128 				return (IBT_HCA_PORT_INVALID);
1129 			}
1130 			opmask |= TAVOR_CMD_OP_PRIM_PORT;
1131 		}
1132 
1133 		/*
1134 		 * If we are attempting to modify the PKey index for this QP,
1135 		 * then check for valid PKey index and fill it in.  Also set
1136 		 * the appropriate flag in the "opmask" parameter.
1137 		 */
1138 		if (flags & IBT_CEP_SET_PKEY_IX) {
1139 			pkeyindx = rc->rc_path.cep_pkey_ix;
1140 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1141 				qpc->pri_addr_path.pkey_indx = pkeyindx;
1142 				opmask |= TAVOR_CMD_OP_PKEYINDX;
1143 			} else {
1144 				TNF_PROBE_1(tavor_qp_init2init_inv_pkey_fail,
1145 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1146 				    pkeyindx);
1147 				TAVOR_TNF_EXIT(tavor_qp_init2init);
1148 				return (IBT_PKEY_IX_ILLEGAL);
1149 			}
1150 		}
1151 
1152 		/*
1153 		 * Check if any of the flags indicate a change in the RDMA
1154 		 * (recv) enable/disable flags and set the appropriate flag in
1155 		 * the "opmask" parameter
1156 		 */
1157 		opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1158 
1159 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
1160 		uc = &info_p->qp_transport.uc;
1161 
1162 		/*
1163 		 * If we are attempting to modify the port for this QP, then
1164 		 * check for valid port number and fill it in.  Also set the
1165 		 * appropriate flag in the "opmask" parameter.
1166 		 */
1167 		if (flags & IBT_CEP_SET_PORT) {
1168 			portnum = uc->uc_path.cep_hca_port_num;
1169 			if (tavor_portnum_is_valid(state, portnum)) {
1170 				qpc->pri_addr_path.portnum = portnum;
1171 			} else {
1172 				TNF_PROBE_1(tavor_qp_init2init_inv_port_fail,
1173 				    TAVOR_TNF_ERROR, "", tnf_uint, port,
1174 				    portnum);
1175 				TAVOR_TNF_EXIT(tavor_qp_init2init);
1176 				return (IBT_HCA_PORT_INVALID);
1177 			}
1178 			opmask |= TAVOR_CMD_OP_PRIM_PORT;
1179 		}
1180 
1181 		/*
1182 		 * If we are attempting to modify the PKey index for this QP,
1183 		 * then check for valid PKey index and fill it in.  Also set
1184 		 * the appropriate flag in the "opmask" parameter.
1185 		 */
1186 		if (flags & IBT_CEP_SET_PKEY_IX) {
1187 			pkeyindx = uc->uc_path.cep_pkey_ix;
1188 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1189 				qpc->pri_addr_path.pkey_indx = pkeyindx;
1190 				opmask |= TAVOR_CMD_OP_PKEYINDX;
1191 			} else {
1192 				TNF_PROBE_1(tavor_qp_init2init_inv_pkey_fail,
1193 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1194 				    pkeyindx);
1195 				TAVOR_TNF_EXIT(tavor_qp_init2init);
1196 				return (IBT_PKEY_IX_ILLEGAL);
1197 			}
1198 		}
1199 
1200 		/*
1201 		 * Check if any of the flags indicate a change in the RDMA
1202 		 * Write (recv) enable/disable and set the appropriate flag
1203 		 * in the "opmask" parameter. Note: RDMA Read and Atomic are
1204 		 * not valid for UC transport.
1205 		 */
1206 		if (flags & IBT_CEP_SET_RDMA_W) {
1207 			qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1208 			opmask |= TAVOR_CMD_OP_RWE;
1209 		}
1210 	} else {
1211 		/*
1212 		 * Invalid QP transport type. If we got here then it's a
1213 		 * warning of a probably serious problem.  So print a message
1214 		 * and return failure
1215 		 */
1216 		TAVOR_WARNING(state, "unknown QP transport type in init2init");
1217 		TNF_PROBE_0(tavor_qp_init2init_inv_transtype_fail,
1218 		    TAVOR_TNF_ERROR, "");
1219 		TAVOR_TNF_EXIT(tavor_qp_init2init);
1220 		return (ibc_get_ci_failure(0));
1221 	}
1222 
1223 	/*
1224 	 * Post the INIT2INIT_QP command to the Tavor firmware
1225 	 *
1226 	 * We do a TAVOR_NOSLEEP here because we are still holding the
1227 	 * "qp_lock".  If we got raised to interrupt level by priority
1228 	 * inversion, we do not want to block in this routine waiting for
1229 	 * success.
1230 	 */
1231 	status = tavor_cmn_qp_cmd_post(state, INIT2INIT_QP, qpc, qp->qp_qpnum,
1232 	    opmask, TAVOR_CMD_NOSLEEP_SPIN);
1233 	if (status != TAVOR_CMD_SUCCESS) {
1234 		if (status != TAVOR_CMD_BAD_QP_STATE) {
1235 			cmn_err(CE_CONT, "Tavor: INIT2INIT_QP command failed: "
1236 			    "%08x\n", status);
1237 			TNF_PROBE_1(tavor_qp_init2init_cmd_fail,
1238 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1239 			TAVOR_TNF_EXIT(tavor_qp_init2init);
1240 			return (ibc_get_ci_failure(0));
1241 		} else {
1242 			TNF_PROBE_0(tavor_qp_init2init_inv_qpstate_fail,
1243 			    TAVOR_TNF_ERROR, "");
1244 			TAVOR_TNF_EXIT(tavor_qp_init2init);
1245 			return (IBT_QP_STATE_INVALID);
1246 		}
1247 	}
1248 
1249 	TAVOR_TNF_EXIT(tavor_qp_init2init);
1250 	return (DDI_SUCCESS);
1251 }
1252 
1253 
1254 /*
1255  * tavor_qp_init2rtr()
1256  *    Context: Can be called from interrupt or base context.
1257  */
1258 static int
tavor_qp_init2rtr(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)1259 tavor_qp_init2rtr(tavor_state_t *state, tavor_qphdl_t qp,
1260     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1261 {
1262 	tavor_hw_qpc_t		*qpc;
1263 	ibt_qp_rc_attr_t	*rc;
1264 	ibt_qp_ud_attr_t	*ud;
1265 	ibt_qp_uc_attr_t	*uc;
1266 	tavor_hw_addr_path_t	*qpc_path;
1267 	ibt_adds_vect_t		*adds_vect;
1268 	uint_t			portnum, pkeyindx, rdma_ra_in, rra_max;
1269 	uint_t			mtu;
1270 	uint32_t		opmask = 0;
1271 	int			status;
1272 
1273 	TAVOR_TNF_ENTER(tavor_qp_init2rtr);
1274 
1275 	ASSERT(MUTEX_HELD(&qp->qp_lock));
1276 
1277 	/*
1278 	 * Grab the temporary QPC entry from QP software state
1279 	 */
1280 	qpc = &qp->qpc;
1281 
1282 	/*
1283 	 * Since there are no common and/or Tavor-specific fields to be filled
1284 	 * in for this command, we begin with the QPC fields which are
1285 	 * specific to transport type.
1286 	 */
1287 	if (qp->qp_serv_type == TAVOR_QP_UD) {
1288 		ud = &info_p->qp_transport.ud;
1289 
1290 		/*
1291 		 * If this UD QP is also a "special QP" (QP0 or QP1), then
1292 		 * the MTU is 256 bytes.  However, Tavor HW requires us to
1293 		 * set the MTU to 4 (which is the IB code for a 2K MTU).
1294 		 * If this is not a special QP, then we set the MTU to the
1295 		 * configured maximum (which defaults to 2K).  Note: the
1296 		 * QPC "msg_max" must also be set so as to correspond with
1297 		 * the specified MTU value.
1298 		 */
1299 		if (qp->qp_is_special) {
1300 			qpc->mtu = 4;
1301 		} else {
1302 			qpc->mtu = state->ts_cfg_profile->cp_max_mtu;
1303 		}
1304 		qpc->msg_max = qpc->mtu + 7;  /* must equal MTU plus seven */
1305 
1306 		/*
1307 		 * Save away the MTU value.  This is used in future sqd2sqd
1308 		 * transitions, as the MTU must remain the same in future
1309 		 * changes.
1310 		 */
1311 		qp->qp_save_mtu = qpc->mtu;
1312 
1313 		/*
1314 		 * If we are attempting to modify the PKey index for this QP,
1315 		 * then check for valid PKey index and fill it in.  Also set
1316 		 * the appropriate flag in the "opmask" parameter.
1317 		 */
1318 		if (flags & IBT_CEP_SET_PKEY_IX) {
1319 			pkeyindx = ud->ud_pkey_ix;
1320 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1321 				qpc->pri_addr_path.pkey_indx = pkeyindx;
1322 				opmask |= TAVOR_CMD_OP_PKEYINDX;
1323 				qp->qp_pkeyindx = pkeyindx;
1324 			} else {
1325 				TNF_PROBE_1(tavor_qp_init2rtr_inv_pkey_fail,
1326 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1327 				    pkeyindx);
1328 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1329 				return (IBT_PKEY_IX_ILLEGAL);
1330 			}
1331 		}
1332 
1333 		/*
1334 		 * If we are attempting to modify the QKey for this QP, then
1335 		 * fill it in and set the appropriate flag in the "opmask"
1336 		 * parameter.
1337 		 */
1338 		if (flags & IBT_CEP_SET_QKEY) {
1339 			qpc->qkey = ud->ud_qkey;
1340 			opmask |= TAVOR_CMD_OP_QKEY;
1341 		}
1342 
1343 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
1344 		rc = &info_p->qp_transport.rc;
1345 		qpc_path = &qpc->pri_addr_path;
1346 		adds_vect = &rc->rc_path.cep_adds_vect;
1347 
1348 		/*
1349 		 * Set the common primary address path fields
1350 		 */
1351 		status = tavor_set_addr_path(state, adds_vect, qpc_path,
1352 		    TAVOR_ADDRPATH_QP, qp);
1353 		if (status != DDI_SUCCESS) {
1354 			TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1355 			    TAVOR_TNF_ERROR, "");
1356 			TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1357 			return (status);
1358 		}
1359 
1360 		/*
1361 		 * The following values are apparently "required" here (as
1362 		 * they are part of the IBA-defined "Remote Node Address
1363 		 * Vector").  However, they are also going to be "required"
1364 		 * later - at RTR2RTS_QP time.  Not sure why.  But we set
1365 		 * them here anyway.
1366 		 */
1367 		qpc_path->rnr_retry	= rc->rc_rnr_retry_cnt;
1368 		qpc->retry_cnt		= rc->rc_retry_cnt;
1369 		qpc_path->ack_timeout	= rc->rc_path.cep_timeout;
1370 
1371 		/*
1372 		 * Setup the destination QP, recv PSN, MTU, max msg size,etc.
1373 		 * Note max message size is defined to be the maximum IB
1374 		 * allowed message size (which is 2^31 bytes).  Also max
1375 		 * MTU is defined by HCA port properties.
1376 		 */
1377 		qpc->rem_qpn	  = rc->rc_dst_qpn;
1378 		qpc->next_rcv_psn = rc->rc_rq_psn;
1379 		qpc->msg_max	  = TAVOR_QP_LOG_MAX_MSGSZ;
1380 
1381 		/*
1382 		 * If this QP is using an SRQ, 'ric' must be set to 1.
1383 		 */
1384 		qpc->ric = (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) ? 1 : 0;
1385 		mtu = rc->rc_path_mtu;
1386 		if (tavor_qp_validate_mtu(state, mtu) != DDI_SUCCESS) {
1387 			TNF_PROBE_1(tavor_qp_init2rtr_inv_mtu_fail,
1388 			    TAVOR_TNF_ERROR, "", tnf_uint, mtu, mtu);
1389 			TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1390 			return (IBT_HCA_PORT_MTU_EXCEEDED);
1391 		}
1392 		qpc->mtu = mtu;
1393 
1394 		/*
1395 		 * Save away the MTU value.  This is used in future sqd2sqd
1396 		 * transitions, as the MTU must remain the same in future
1397 		 * changes.
1398 		 */
1399 		qp->qp_save_mtu = qpc->mtu;
1400 
1401 		/*
1402 		 * Though it is a "required" parameter, "min_rnr_nak" is
1403 		 * optionally specifiable in Tavor.  So we hardcode the
1404 		 * optional flag here.
1405 		 */
1406 		qpc->min_rnr_nak = rc->rc_min_rnr_nak;
1407 		opmask |= TAVOR_CMD_OP_MINRNRNAK;
1408 
1409 		/*
1410 		 * Check that the number of specified "incoming RDMA resources"
1411 		 * is valid.  And if it is, then setup the "rra_max" and
1412 		 * "ra_buf_index" fields in the QPC to point to the
1413 		 * pre-allocated RDB resources (in DDR)
1414 		 */
1415 		rdma_ra_in = rc->rc_rdma_ra_in;
1416 		if (tavor_qp_validate_resp_rsrc(state, rc, &rra_max) !=
1417 		    DDI_SUCCESS) {
1418 			TNF_PROBE_1(tavor_qp_init2rtr_inv_rdma_in_fail,
1419 			    TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_in,
1420 			    rdma_ra_in);
1421 			TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1422 			return (IBT_INVALID_PARAM);
1423 		}
1424 		qpc->rra_max = rra_max;
1425 		qpc->ra_buff_indx = qp->qp_rdb_ddraddr >> TAVOR_RDB_SIZE_SHIFT;
1426 
1427 		/*
1428 		 * If we are attempting to modify the PKey index for this QP,
1429 		 * then check for valid PKey index and fill it in.  Also set
1430 		 * the appropriate flag in the "opmask" parameter.
1431 		 */
1432 		if (flags & IBT_CEP_SET_PKEY_IX) {
1433 			pkeyindx = rc->rc_path.cep_pkey_ix;
1434 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1435 				qpc->pri_addr_path.pkey_indx = pkeyindx;
1436 				opmask |= TAVOR_CMD_OP_PKEYINDX;
1437 			} else {
1438 				TNF_PROBE_1(tavor_qp_init2rtr_inv_pkey_fail,
1439 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1440 				    pkeyindx);
1441 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1442 				return (IBT_PKEY_IX_ILLEGAL);
1443 			}
1444 		}
1445 
1446 		/*
1447 		 * Check if any of the flags indicate a change in the RDMA
1448 		 * (recv) enable/disable flags and set the appropriate flag in
1449 		 * the "opmask" parameter
1450 		 */
1451 		opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1452 
1453 		/*
1454 		 * Check for optional alternate path and fill in the
1455 		 * appropriate QPC fields if one is specified
1456 		 */
1457 		if (flags & IBT_CEP_SET_ALT_PATH) {
1458 			qpc_path = &qpc->alt_addr_path;
1459 			adds_vect = &rc->rc_alt_path.cep_adds_vect;
1460 
1461 			/* Set the common alternate address path fields */
1462 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
1463 			    TAVOR_ADDRPATH_QP, qp);
1464 			if (status != DDI_SUCCESS) {
1465 				TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1466 				    TAVOR_TNF_ERROR, "");
1467 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1468 				return (status);
1469 			}
1470 			qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
1471 
1472 			/*
1473 			 * Copy the "RNR Retry count" value from the primary
1474 			 * path.  Just as we did above, we need to hardcode
1475 			 * the optional flag here (see below).
1476 			 */
1477 			qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1478 
1479 			/*
1480 			 * Check for valid alternate path port number and fill
1481 			 * it in
1482 			 */
1483 			portnum = rc->rc_alt_path.cep_hca_port_num;
1484 			if (tavor_portnum_is_valid(state, portnum)) {
1485 				qpc->alt_addr_path.portnum = portnum;
1486 			} else {
1487 				TNF_PROBE_1(tavor_qp_init2rtr_inv_altport_fail,
1488 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
1489 				    portnum);
1490 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1491 				return (IBT_HCA_PORT_INVALID);
1492 			}
1493 
1494 			/*
1495 			 * Check for valid alternate path PKey index and fill
1496 			 * it in
1497 			 */
1498 			pkeyindx = rc->rc_alt_path.cep_pkey_ix;
1499 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1500 				qpc->alt_addr_path.pkey_indx = pkeyindx;
1501 			} else {
1502 				TNF_PROBE_1(tavor_qp_init2rtr_inv_altpkey_fail,
1503 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1504 				    pkeyindx);
1505 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1506 				return (IBT_PKEY_IX_ILLEGAL);
1507 			}
1508 			opmask |= (TAVOR_CMD_OP_ALT_PATH |
1509 			    TAVOR_CMD_OP_ALT_RNRRETRY);
1510 		}
1511 
1512 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
1513 		uc = &info_p->qp_transport.uc;
1514 		qpc_path = &qpc->pri_addr_path;
1515 		adds_vect = &uc->uc_path.cep_adds_vect;
1516 
1517 		/*
1518 		 * Set the common primary address path fields
1519 		 */
1520 		status = tavor_set_addr_path(state, adds_vect, qpc_path,
1521 		    TAVOR_ADDRPATH_QP, qp);
1522 		if (status != DDI_SUCCESS) {
1523 			TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1524 			    TAVOR_TNF_ERROR, "");
1525 			TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1526 			return (status);
1527 		}
1528 
1529 		/*
1530 		 * Setup the destination QP, recv PSN, MTU, max msg size,etc.
1531 		 * Note max message size is defined to be the maximum IB
1532 		 * allowed message size (which is 2^31 bytes).  Also max
1533 		 * MTU is defined by HCA port properties.
1534 		 */
1535 		qpc->rem_qpn	  = uc->uc_dst_qpn;
1536 		qpc->next_rcv_psn = uc->uc_rq_psn;
1537 		qpc->msg_max	  = TAVOR_QP_LOG_MAX_MSGSZ;
1538 		mtu = uc->uc_path_mtu;
1539 		if (tavor_qp_validate_mtu(state, mtu) != DDI_SUCCESS) {
1540 			TNF_PROBE_1(tavor_qp_init2rtr_inv_mtu_fail,
1541 			    TAVOR_TNF_ERROR, "", tnf_uint, mtu, mtu);
1542 			TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1543 			return (IBT_HCA_PORT_MTU_EXCEEDED);
1544 		}
1545 		qpc->mtu = mtu;
1546 
1547 		/*
1548 		 * Save away the MTU value.  This is used in future sqd2sqd
1549 		 * transitions, as the MTU must remain the same in future
1550 		 * changes.
1551 		 */
1552 		qp->qp_save_mtu = qpc->mtu;
1553 
1554 		/*
1555 		 * If we are attempting to modify the PKey index for this QP,
1556 		 * then check for valid PKey index and fill it in.  Also set
1557 		 * the appropriate flag in the "opmask" parameter.
1558 		 */
1559 		if (flags & IBT_CEP_SET_PKEY_IX) {
1560 			pkeyindx = uc->uc_path.cep_pkey_ix;
1561 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1562 				qpc->pri_addr_path.pkey_indx = pkeyindx;
1563 				opmask |= TAVOR_CMD_OP_PKEYINDX;
1564 			} else {
1565 				TNF_PROBE_1(tavor_qp_init2rtr_inv_pkey_fail,
1566 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1567 				    pkeyindx);
1568 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1569 				return (IBT_PKEY_IX_ILLEGAL);
1570 			}
1571 		}
1572 
1573 		/*
1574 		 * Check if any of the flags indicate a change in the RDMA
1575 		 * Write (recv) enable/disable and set the appropriate flag
1576 		 * in the "opmask" parameter. Note: RDMA Read and Atomic are
1577 		 * not valid for UC transport.
1578 		 */
1579 		if (flags & IBT_CEP_SET_RDMA_W) {
1580 			qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1581 			opmask |= TAVOR_CMD_OP_RWE;
1582 		}
1583 
1584 		/*
1585 		 * Check for optional alternate path and fill in the
1586 		 * appropriate QPC fields if one is specified
1587 		 */
1588 		if (flags & IBT_CEP_SET_ALT_PATH) {
1589 			qpc_path = &qpc->alt_addr_path;
1590 			adds_vect = &uc->uc_alt_path.cep_adds_vect;
1591 
1592 			/* Set the common alternate address path fields */
1593 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
1594 			    TAVOR_ADDRPATH_QP, qp);
1595 			if (status != DDI_SUCCESS) {
1596 				TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1597 				    TAVOR_TNF_ERROR, "");
1598 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1599 				return (status);
1600 			}
1601 
1602 			/*
1603 			 * Check for valid alternate path port number and fill
1604 			 * it in
1605 			 */
1606 			portnum = uc->uc_alt_path.cep_hca_port_num;
1607 			if (tavor_portnum_is_valid(state, portnum)) {
1608 				qpc->alt_addr_path.portnum = portnum;
1609 			} else {
1610 				TNF_PROBE_1(tavor_qp_init2rtr_inv_altport_fail,
1611 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
1612 				    portnum);
1613 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1614 				return (IBT_HCA_PORT_INVALID);
1615 			}
1616 
1617 			/*
1618 			 * Check for valid alternate path PKey index and fill
1619 			 * it in
1620 			 */
1621 			pkeyindx = uc->uc_alt_path.cep_pkey_ix;
1622 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1623 				qpc->alt_addr_path.pkey_indx = pkeyindx;
1624 			} else {
1625 				TNF_PROBE_1(tavor_qp_init2rtr_inv_altpkey_fail,
1626 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1627 				    pkeyindx);
1628 				TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1629 				return (IBT_PKEY_IX_ILLEGAL);
1630 			}
1631 			opmask |= TAVOR_CMD_OP_ALT_PATH;
1632 		}
1633 	} else {
1634 		/*
1635 		 * Invalid QP transport type. If we got here then it's a
1636 		 * warning of a probably serious problem.  So print a message
1637 		 * and return failure
1638 		 */
1639 		TAVOR_WARNING(state, "unknown QP transport type in init2rtr");
1640 		TNF_PROBE_0(tavor_qp_init2rtr_inv_transtype_fail,
1641 		    TAVOR_TNF_ERROR, "");
1642 		TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1643 		return (ibc_get_ci_failure(0));
1644 	}
1645 
1646 	/*
1647 	 * Post the INIT2RTR_QP command to the Tavor firmware
1648 	 *
1649 	 * We do a TAVOR_NOSLEEP here because we are still holding the
1650 	 * "qp_lock".  If we got raised to interrupt level by priority
1651 	 * inversion, we do not want to block in this routine waiting for
1652 	 * success.
1653 	 */
1654 	status = tavor_cmn_qp_cmd_post(state, INIT2RTR_QP, qpc, qp->qp_qpnum,
1655 	    opmask, TAVOR_CMD_NOSLEEP_SPIN);
1656 	if (status != TAVOR_CMD_SUCCESS) {
1657 		if (status != TAVOR_CMD_BAD_QP_STATE) {
1658 			cmn_err(CE_CONT, "Tavor: INIT2RTR_QP command failed: "
1659 			    "%08x\n", status);
1660 			TNF_PROBE_1(tavor_qp_init2rtr_cmd_fail,
1661 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1662 			TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1663 			return (ibc_get_ci_failure(0));
1664 		} else {
1665 			TNF_PROBE_0(tavor_qp_init2rtr_inv_qpstate_fail,
1666 			    TAVOR_TNF_ERROR, "");
1667 			TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1668 			return (IBT_QP_STATE_INVALID);
1669 		}
1670 	}
1671 
1672 	TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1673 	return (DDI_SUCCESS);
1674 }
1675 
1676 
1677 /*
1678  * tavor_qp_rtr2rts()
1679  *    Context: Can be called from interrupt or base context.
1680  */
1681 static int
tavor_qp_rtr2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)1682 tavor_qp_rtr2rts(tavor_state_t *state, tavor_qphdl_t qp,
1683     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1684 {
1685 	tavor_hw_qpc_t		*qpc;
1686 	ibt_qp_rc_attr_t	*rc;
1687 	ibt_qp_ud_attr_t	*ud;
1688 	ibt_qp_uc_attr_t	*uc;
1689 	tavor_hw_addr_path_t	*qpc_path;
1690 	ibt_adds_vect_t		*adds_vect;
1691 	uint_t			portnum, pkeyindx, rdma_ra_out, sra_max;
1692 	uint32_t		opmask = 0;
1693 	int			status;
1694 
1695 	TAVOR_TNF_ENTER(tavor_qp_rtr2rts);
1696 
1697 	ASSERT(MUTEX_HELD(&qp->qp_lock));
1698 
1699 	/*
1700 	 * Grab the temporary QPC entry from QP software state
1701 	 */
1702 	qpc = &qp->qpc;
1703 
1704 	/*
1705 	 * Fill in the common and/or Tavor-specific fields in the QPC
1706 	 */
1707 	qpc->flight_lim = TAVOR_QP_FLIGHT_LIM_UNLIMITED;
1708 
1709 	/*
1710 	 * Now fill in the QPC fields which are specific to transport type
1711 	 */
1712 	if (qp->qp_serv_type == TAVOR_QP_UD) {
1713 		ud = &info_p->qp_transport.ud;
1714 
1715 		/* Set the send PSN */
1716 		qpc->next_snd_psn = ud->ud_sq_psn;
1717 
1718 		/*
1719 		 * If we are attempting to modify the QKey for this QP, then
1720 		 * fill it in and set the appropriate flag in the "opmask"
1721 		 * parameter.
1722 		 */
1723 		if (flags & IBT_CEP_SET_QKEY) {
1724 			qpc->qkey = ud->ud_qkey;
1725 			opmask |= TAVOR_CMD_OP_QKEY;
1726 		}
1727 
1728 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
1729 		rc = &info_p->qp_transport.rc;
1730 		qpc_path = &qpc->pri_addr_path;
1731 
1732 		/*
1733 		 * Setup the send PSN, ACK timeout, and retry counts
1734 		 */
1735 		qpc->next_snd_psn	= rc->rc_sq_psn;
1736 		qpc_path->ack_timeout	= rc->rc_path.cep_timeout;
1737 		qpc_path->rnr_retry	= rc->rc_rnr_retry_cnt;
1738 		qpc->retry_cnt		= rc->rc_retry_cnt;
1739 
1740 		/*
1741 		 * Set "ack_req_freq" based on the configuration variable
1742 		 */
1743 		qpc->ack_req_freq = state->ts_cfg_profile->cp_ackreq_freq;
1744 
1745 		/*
1746 		 * Check that the number of specified "outgoing RDMA resources"
1747 		 * is valid.  And if it is, then setup the "sra_max"
1748 		 * appropriately
1749 		 */
1750 		rdma_ra_out = rc->rc_rdma_ra_out;
1751 		if (tavor_qp_validate_init_depth(state, rc, &sra_max) !=
1752 		    DDI_SUCCESS) {
1753 			TNF_PROBE_1(tavor_qp_rtr2rts_inv_rdma_out_fail,
1754 			    TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_out,
1755 			    rdma_ra_out);
1756 			TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1757 			return (IBT_INVALID_PARAM);
1758 		}
1759 		qpc->sra_max = sra_max;
1760 
1761 		/*
1762 		 * Configure the QP to allow (sending of) all types of RC
1763 		 * traffic.  Tavor hardware allows these bits to be set to
1764 		 * zero (thereby disabling certain outgoing RDMA types), but
1765 		 * we do not desire to do this.
1766 		 */
1767 		qpc->sre = qpc->swe = qpc->sae = 1;
1768 		qpc->sic = 0;
1769 
1770 		/*
1771 		 * Check if any of the flags indicate a change in the RDMA
1772 		 * (recv) enable/disable flags and set the appropriate flag in
1773 		 * the "opmask" parameter
1774 		 */
1775 		opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1776 
1777 		/*
1778 		 * If we are attempting to modify the path migration state for
1779 		 * this QP, then check for valid state and fill it in.  Also
1780 		 * set the appropriate flag in the "opmask" parameter.
1781 		 */
1782 		if (flags & IBT_CEP_SET_MIG) {
1783 			if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
1784 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1785 			} else if (rc->rc_mig_state == IBT_STATE_REARMED) {
1786 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1787 			} else {
1788 				TNF_PROBE_1(tavor_qp_rtr2rts_inv_mig_state_fail,
1789 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
1790 				    rc->rc_mig_state);
1791 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1792 				return (IBT_QP_APM_STATE_INVALID);
1793 			}
1794 			opmask |= TAVOR_CMD_OP_PM_STATE;
1795 		}
1796 
1797 		/*
1798 		 * If we are attempting to modify the "Minimum RNR NAK" value
1799 		 * for this QP, then fill it in and set the appropriate flag
1800 		 * in the "opmask" parameter.
1801 		 */
1802 		if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
1803 			qpc->min_rnr_nak = rc->rc_min_rnr_nak;
1804 			opmask |= TAVOR_CMD_OP_MINRNRNAK;
1805 		}
1806 
1807 		/*
1808 		 * Check for optional alternate path and fill in the
1809 		 * appropriate QPC fields if one is specified
1810 		 */
1811 		if (flags & IBT_CEP_SET_ALT_PATH) {
1812 			qpc_path = &qpc->alt_addr_path;
1813 			adds_vect = &rc->rc_alt_path.cep_adds_vect;
1814 
1815 			/* Set the common alternate address path fields */
1816 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
1817 			    TAVOR_ADDRPATH_QP, qp);
1818 			if (status != DDI_SUCCESS) {
1819 				TNF_PROBE_0(tavor_qp_rtr2rts_setaddrpath_fail,
1820 				    TAVOR_TNF_ERROR, "");
1821 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1822 				return (status);
1823 			}
1824 
1825 			qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
1826 
1827 			/*
1828 			 * Copy the "RNR Retry count" value from the primary
1829 			 * path.  Just as we did above, we need to hardcode
1830 			 * the optional flag here (see below).
1831 			 */
1832 			qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1833 
1834 			/*
1835 			 * Check for valid alternate path port number and fill
1836 			 * it in
1837 			 */
1838 			portnum = rc->rc_alt_path.cep_hca_port_num;
1839 			if (tavor_portnum_is_valid(state, portnum)) {
1840 				qpc->alt_addr_path.portnum = portnum;
1841 			} else {
1842 				TNF_PROBE_1(tavor_qp_rtr2rts_inv_altport_fail,
1843 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
1844 				    portnum);
1845 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1846 				return (IBT_HCA_PORT_INVALID);
1847 			}
1848 
1849 			/*
1850 			 * Check for valid alternate path PKey index and fill
1851 			 * it in
1852 			 */
1853 			pkeyindx = rc->rc_alt_path.cep_pkey_ix;
1854 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1855 				qpc->alt_addr_path.pkey_indx = pkeyindx;
1856 			} else {
1857 				TNF_PROBE_1(tavor_qp_rtr2rts_inv_altpkey_fail,
1858 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1859 				    pkeyindx);
1860 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1861 				return (IBT_PKEY_IX_ILLEGAL);
1862 			}
1863 			opmask |= (TAVOR_CMD_OP_ALT_PATH |
1864 			    TAVOR_CMD_OP_ALT_RNRRETRY);
1865 		}
1866 
1867 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
1868 		uc = &info_p->qp_transport.uc;
1869 
1870 		/* Set the send PSN */
1871 		qpc->next_snd_psn = uc->uc_sq_psn;
1872 
1873 		/*
1874 		 * Configure the QP to allow (sending of) all types of allowable
1875 		 * UC traffic (i.e. RDMA Write).
1876 		 */
1877 		qpc->swe = 1;
1878 
1879 		/*
1880 		 * Check if any of the flags indicate a change in the RDMA
1881 		 * Write (recv) enable/disable and set the appropriate flag
1882 		 * in the "opmask" parameter. Note: RDMA Read and Atomic are
1883 		 * not valid for UC transport.
1884 		 */
1885 		if (flags & IBT_CEP_SET_RDMA_W) {
1886 			qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1887 			opmask |= TAVOR_CMD_OP_RWE;
1888 		}
1889 
1890 		/*
1891 		 * If we are attempting to modify the path migration state for
1892 		 * this QP, then check for valid state and fill it in.  Also
1893 		 * set the appropriate flag in the "opmask" parameter.
1894 		 */
1895 		if (flags & IBT_CEP_SET_MIG) {
1896 			if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
1897 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1898 			} else if (uc->uc_mig_state == IBT_STATE_REARMED) {
1899 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1900 			} else {
1901 				TNF_PROBE_1(tavor_qp_rtr2rts_inv_mig_state_fail,
1902 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
1903 				    uc->uc_mig_state);
1904 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1905 				return (IBT_QP_APM_STATE_INVALID);
1906 			}
1907 			opmask |= TAVOR_CMD_OP_PM_STATE;
1908 		}
1909 
1910 		/*
1911 		 * Check for optional alternate path and fill in the
1912 		 * appropriate QPC fields if one is specified
1913 		 */
1914 		if (flags & IBT_CEP_SET_ALT_PATH) {
1915 			qpc_path = &qpc->alt_addr_path;
1916 			adds_vect = &uc->uc_alt_path.cep_adds_vect;
1917 
1918 			/* Set the common alternate address path fields */
1919 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
1920 			    TAVOR_ADDRPATH_QP, qp);
1921 			if (status != DDI_SUCCESS) {
1922 				TNF_PROBE_0(tavor_qp_rtr2rts_setaddrpath_fail,
1923 				    TAVOR_TNF_ERROR, "");
1924 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1925 				return (status);
1926 			}
1927 
1928 			/*
1929 			 * Check for valid alternate path port number and fill
1930 			 * it in
1931 			 */
1932 			portnum = uc->uc_alt_path.cep_hca_port_num;
1933 			if (tavor_portnum_is_valid(state, portnum)) {
1934 				qpc->alt_addr_path.portnum = portnum;
1935 			} else {
1936 				TNF_PROBE_1(tavor_qp_rtr2rts_inv_altport_fail,
1937 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
1938 				    portnum);
1939 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1940 				return (IBT_HCA_PORT_INVALID);
1941 			}
1942 
1943 			/*
1944 			 * Check for valid alternate path PKey index and fill
1945 			 * it in
1946 			 */
1947 			pkeyindx = uc->uc_alt_path.cep_pkey_ix;
1948 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1949 				qpc->alt_addr_path.pkey_indx = pkeyindx;
1950 			} else {
1951 				TNF_PROBE_1(tavor_qp_rtr2rts_inv_altpkey_fail,
1952 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1953 				    pkeyindx);
1954 				TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1955 				return (IBT_PKEY_IX_ILLEGAL);
1956 			}
1957 			opmask |= TAVOR_CMD_OP_ALT_PATH;
1958 		}
1959 	} else {
1960 		/*
1961 		 * Invalid QP transport type. If we got here then it's a
1962 		 * warning of a probably serious problem.  So print a message
1963 		 * and return failure
1964 		 */
1965 		TAVOR_WARNING(state, "unknown QP transport type in rtr2rts");
1966 		TNF_PROBE_0(tavor_qp_rtr2rts_inv_transtype_fail,
1967 		    TAVOR_TNF_ERROR, "");
1968 		TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1969 		return (ibc_get_ci_failure(0));
1970 	}
1971 
1972 	/*
1973 	 * Post the RTR2RTS_QP command to the Tavor firmware
1974 	 *
1975 	 * We do a TAVOR_NOSLEEP here because we are still holding the
1976 	 * "qp_lock".  If we got raised to interrupt level by priority
1977 	 * inversion, we do not want to block in this routine waiting for
1978 	 * success.
1979 	 */
1980 	status = tavor_cmn_qp_cmd_post(state, RTR2RTS_QP, qpc, qp->qp_qpnum,
1981 	    opmask, TAVOR_CMD_NOSLEEP_SPIN);
1982 	if (status != TAVOR_CMD_SUCCESS) {
1983 		if (status != TAVOR_CMD_BAD_QP_STATE) {
1984 			cmn_err(CE_CONT, "Tavor: RTR2RTS_QP command failed: "
1985 			    "%08x\n", status);
1986 			TNF_PROBE_1(tavor_qp_rtr2rts_cmd_fail,
1987 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1988 			TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1989 			return (ibc_get_ci_failure(0));
1990 		} else {
1991 			TNF_PROBE_0(tavor_qp_rtr2rts_inv_qpstate_fail,
1992 			    TAVOR_TNF_ERROR, "");
1993 			TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1994 			return (IBT_QP_STATE_INVALID);
1995 		}
1996 	}
1997 
1998 	TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1999 	return (DDI_SUCCESS);
2000 }
2001 
2002 
2003 /*
2004  * tavor_qp_rts2rts()
2005  *    Context: Can be called from interrupt or base context.
2006  */
2007 static int
tavor_qp_rts2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)2008 tavor_qp_rts2rts(tavor_state_t *state, tavor_qphdl_t qp,
2009     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2010 {
2011 	tavor_hw_qpc_t		*qpc;
2012 	ibt_qp_rc_attr_t	*rc;
2013 	ibt_qp_ud_attr_t	*ud;
2014 	ibt_qp_uc_attr_t	*uc;
2015 	tavor_hw_addr_path_t	*qpc_path;
2016 	ibt_adds_vect_t		*adds_vect;
2017 	uint_t			portnum, pkeyindx;
2018 	uint32_t		opmask = 0;
2019 	int			status;
2020 
2021 	TAVOR_TNF_ENTER(tavor_qp_rts2rts);
2022 
2023 	ASSERT(MUTEX_HELD(&qp->qp_lock));
2024 
2025 	/*
2026 	 * Grab the temporary QPC entry from QP software state
2027 	 */
2028 	qpc = &qp->qpc;
2029 
2030 	/*
2031 	 * Since there are no common and/or Tavor-specific fields to be filled
2032 	 * in for this command, we begin with the QPC fields which are
2033 	 * specific to transport type.
2034 	 */
2035 	if (qp->qp_serv_type == TAVOR_QP_UD) {
2036 		ud = &info_p->qp_transport.ud;
2037 
2038 		/*
2039 		 * If we are attempting to modify the QKey for this QP, then
2040 		 * fill it in and set the appropriate flag in the "opmask"
2041 		 * parameter.
2042 		 */
2043 		if (flags & IBT_CEP_SET_QKEY) {
2044 			qpc->qkey = ud->ud_qkey;
2045 			opmask |= TAVOR_CMD_OP_QKEY;
2046 		}
2047 
2048 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
2049 		rc = &info_p->qp_transport.rc;
2050 
2051 		/*
2052 		 * Check if any of the flags indicate a change in the RDMA
2053 		 * (recv) enable/disable flags and set the appropriate flag in
2054 		 * the "opmask" parameter
2055 		 */
2056 		opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2057 
2058 		/*
2059 		 * If we are attempting to modify the path migration state for
2060 		 * this QP, then check for valid state and fill it in.  Also
2061 		 * set the appropriate flag in the "opmask" parameter.
2062 		 */
2063 		if (flags & IBT_CEP_SET_MIG) {
2064 			if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2065 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2066 			} else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2067 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2068 			} else {
2069 				TNF_PROBE_1(tavor_qp_rts2rts_inv_mig_state_fail,
2070 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2071 				    rc->rc_mig_state);
2072 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2073 				return (IBT_QP_APM_STATE_INVALID);
2074 			}
2075 			opmask |= TAVOR_CMD_OP_PM_STATE;
2076 		}
2077 
2078 		/*
2079 		 * If we are attempting to modify the "Minimum RNR NAK" value
2080 		 * for this QP, then fill it in and set the appropriate flag
2081 		 * in the "opmask" parameter.
2082 		 */
2083 		if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2084 			qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2085 			opmask |= TAVOR_CMD_OP_MINRNRNAK;
2086 		}
2087 
2088 		/*
2089 		 * Check for optional alternate path and fill in the
2090 		 * appropriate QPC fields if one is specified
2091 		 */
2092 		if (flags & IBT_CEP_SET_ALT_PATH) {
2093 			qpc_path = &qpc->alt_addr_path;
2094 			adds_vect = &rc->rc_alt_path.cep_adds_vect;
2095 
2096 			/* Set the common alternate address path fields */
2097 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2098 			    TAVOR_ADDRPATH_QP, qp);
2099 			if (status != DDI_SUCCESS) {
2100 				TNF_PROBE_0(tavor_qp_rts2rts_setaddrpath_fail,
2101 				    TAVOR_TNF_ERROR, "");
2102 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2103 				return (status);
2104 			}
2105 			qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2106 
2107 			/*
2108 			 * Check for valid alternate path port number and fill
2109 			 * it in
2110 			 */
2111 			portnum = rc->rc_alt_path.cep_hca_port_num;
2112 			if (tavor_portnum_is_valid(state, portnum)) {
2113 				qpc->alt_addr_path.portnum = portnum;
2114 			} else {
2115 				TNF_PROBE_1(tavor_qp_rts2rts_inv_altport_fail,
2116 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
2117 				    portnum);
2118 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2119 				return (IBT_HCA_PORT_INVALID);
2120 			}
2121 
2122 			/*
2123 			 * Check for valid alternate path PKey index and fill
2124 			 * it in
2125 			 */
2126 			pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2127 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2128 				qpc->alt_addr_path.pkey_indx = pkeyindx;
2129 			} else {
2130 				TNF_PROBE_1(tavor_qp_rts2rts_inv_altpkey_fail,
2131 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2132 				    pkeyindx);
2133 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2134 				return (IBT_PKEY_IX_ILLEGAL);
2135 			}
2136 			opmask |= TAVOR_CMD_OP_ALT_PATH;
2137 		}
2138 
2139 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
2140 		uc = &info_p->qp_transport.uc;
2141 
2142 		/*
2143 		 * Check if any of the flags indicate a change in the RDMA
2144 		 * Write (recv) enable/disable and set the appropriate flag
2145 		 * in the "opmask" parameter. Note: RDMA Read and Atomic are
2146 		 * not valid for UC transport.
2147 		 */
2148 		if (flags & IBT_CEP_SET_RDMA_W) {
2149 			qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2150 			opmask |= TAVOR_CMD_OP_RWE;
2151 		}
2152 
2153 		/*
2154 		 * If we are attempting to modify the path migration state for
2155 		 * this QP, then check for valid state and fill it in.  Also
2156 		 * set the appropriate flag in the "opmask" parameter.
2157 		 */
2158 		if (flags & IBT_CEP_SET_MIG) {
2159 			if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2160 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2161 			} else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2162 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2163 			} else {
2164 				TNF_PROBE_1(tavor_qp_rts2rts_inv_mig_state_fail,
2165 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2166 				    uc->uc_mig_state);
2167 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2168 				return (IBT_QP_APM_STATE_INVALID);
2169 			}
2170 			opmask |= TAVOR_CMD_OP_PM_STATE;
2171 		}
2172 
2173 		/*
2174 		 * Check for optional alternate path and fill in the
2175 		 * appropriate QPC fields if one is specified
2176 		 */
2177 		if (flags & IBT_CEP_SET_ALT_PATH) {
2178 			qpc_path = &qpc->alt_addr_path;
2179 			adds_vect = &uc->uc_alt_path.cep_adds_vect;
2180 
2181 			/* Set the common alternate address path fields */
2182 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2183 			    TAVOR_ADDRPATH_QP, qp);
2184 			if (status != DDI_SUCCESS) {
2185 				TNF_PROBE_0(tavor_qp_rts2rts_setaddrpath_fail,
2186 				    TAVOR_TNF_ERROR, "");
2187 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2188 				return (status);
2189 			}
2190 
2191 			/*
2192 			 * Check for valid alternate path port number and fill
2193 			 * it in
2194 			 */
2195 			portnum = uc->uc_alt_path.cep_hca_port_num;
2196 			if (tavor_portnum_is_valid(state, portnum)) {
2197 				qpc->alt_addr_path.portnum = portnum;
2198 			} else {
2199 				TNF_PROBE_1(tavor_qp_rts2rts_inv_altport_fail,
2200 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
2201 				    portnum);
2202 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2203 				return (IBT_HCA_PORT_INVALID);
2204 			}
2205 
2206 			/*
2207 			 * Check for valid alternate path PKey index and fill
2208 			 * it in
2209 			 */
2210 			pkeyindx = uc->uc_alt_path.cep_pkey_ix;
2211 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2212 				qpc->alt_addr_path.pkey_indx = pkeyindx;
2213 			} else {
2214 				TNF_PROBE_1(tavor_qp_rts2rts_inv_altpkey_fail,
2215 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2216 				    pkeyindx);
2217 				TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2218 				return (IBT_PKEY_IX_ILLEGAL);
2219 			}
2220 			opmask |= TAVOR_CMD_OP_ALT_PATH;
2221 		}
2222 	} else {
2223 		/*
2224 		 * Invalid QP transport type. If we got here then it's a
2225 		 * warning of a probably serious problem.  So print a message
2226 		 * and return failure
2227 		 */
2228 		TAVOR_WARNING(state, "unknown QP transport type in rts2rts");
2229 		TNF_PROBE_0(tavor_qp_rts2rts_inv_transtype_fail,
2230 		    TAVOR_TNF_ERROR, "");
2231 		TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2232 		return (ibc_get_ci_failure(0));
2233 	}
2234 
2235 	/*
2236 	 * Post the RTS2RTS_QP command to the Tavor firmware
2237 	 *
2238 	 * We do a TAVOR_NOSLEEP here because we are still holding the
2239 	 * "qp_lock".  If we got raised to interrupt level by priority
2240 	 * inversion, we do not want to block in this routine waiting for
2241 	 * success.
2242 	 */
2243 	status = tavor_cmn_qp_cmd_post(state, RTS2RTS_QP, qpc, qp->qp_qpnum,
2244 	    opmask, TAVOR_CMD_NOSLEEP_SPIN);
2245 	if (status != TAVOR_CMD_SUCCESS) {
2246 		if (status != TAVOR_CMD_BAD_QP_STATE) {
2247 			cmn_err(CE_CONT, "Tavor: RTS2RTS_QP command failed: "
2248 			    "%08x\n", status);
2249 			TNF_PROBE_1(tavor_qp_rts2rts_cmd_fail,
2250 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
2251 			TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2252 			return (ibc_get_ci_failure(0));
2253 		} else {
2254 			TNF_PROBE_0(tavor_qp_rts2rts_inv_qpstate_fail,
2255 			    TAVOR_TNF_ERROR, "");
2256 			TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2257 			return (IBT_QP_STATE_INVALID);
2258 		}
2259 	}
2260 
2261 	TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2262 	return (DDI_SUCCESS);
2263 }
2264 
2265 
2266 /*
2267  * tavor_qp_rts2sqd()
2268  *    Context: Can be called from interrupt or base context.
2269  */
2270 static int
tavor_qp_rts2sqd(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags)2271 tavor_qp_rts2sqd(tavor_state_t *state, tavor_qphdl_t qp,
2272     ibt_cep_modify_flags_t flags)
2273 {
2274 	int			status;
2275 
2276 	TAVOR_TNF_ENTER(tavor_qp_rts2sqd);
2277 
2278 	ASSERT(MUTEX_HELD(&qp->qp_lock));
2279 
2280 	/*
2281 	 * Set a flag to indicate whether or not the consumer is interested
2282 	 * in receiving the SQ drained event.  Since we are going to always
2283 	 * request hardware generation of the SQD event, we use the value in
2284 	 * "qp_forward_sqd_event" to determine whether or not to pass the event
2285 	 * to the IBTF or to silently consume it.
2286 	 */
2287 	qp->qp_forward_sqd_event = (flags & IBT_CEP_SET_SQD_EVENT) ? 1 : 0;
2288 
2289 	/*
2290 	 * Post the RTS2SQD_QP command to the Tavor firmware
2291 	 *
2292 	 * We do a TAVOR_NOSLEEP here because we are still holding the
2293 	 * "qp_lock".  If we got raised to interrupt level by priority
2294 	 * inversion, we do not want to block in this routine waiting for
2295 	 * success.
2296 	 */
2297 	status = tavor_cmn_qp_cmd_post(state, RTS2SQD_QP, NULL, qp->qp_qpnum,
2298 	    0, TAVOR_CMD_NOSLEEP_SPIN);
2299 	if (status != TAVOR_CMD_SUCCESS) {
2300 		if (status != TAVOR_CMD_BAD_QP_STATE) {
2301 			cmn_err(CE_CONT, "Tavor: RTS2SQD_QP command failed: "
2302 			    "%08x\n", status);
2303 			TNF_PROBE_1(tavor_qp_rts2sqd_cmd_fail,
2304 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
2305 			TAVOR_TNF_EXIT(tavor_qp_rts2sqd);
2306 			return (ibc_get_ci_failure(0));
2307 		} else {
2308 			TNF_PROBE_0(tavor_qp_rts2sqd_inv_qpstate_fail,
2309 			    TAVOR_TNF_ERROR, "");
2310 			TAVOR_TNF_EXIT(tavor_qp_rts2sqd);
2311 			return (IBT_QP_STATE_INVALID);
2312 		}
2313 	}
2314 
2315 	/*
2316 	 * Mark the current QP state as "SQ Draining".  This allows us to
2317 	 * distinguish between the two underlying states in SQD. (see QueryQP()
2318 	 * code in tavor_qp.c)
2319 	 */
2320 	qp->qp_sqd_still_draining = 1;
2321 
2322 	TAVOR_TNF_EXIT(tavor_qp_rts2sqd);
2323 	return (DDI_SUCCESS);
2324 }
2325 
2326 
2327 /*
2328  * tavor_qp_sqd2rts()
2329  *    Context: Can be called from interrupt or base context.
2330  */
2331 static int
tavor_qp_sqd2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)2332 tavor_qp_sqd2rts(tavor_state_t *state, tavor_qphdl_t qp,
2333     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2334 {
2335 	tavor_hw_qpc_t		*qpc;
2336 	ibt_qp_rc_attr_t	*rc;
2337 	ibt_qp_ud_attr_t	*ud;
2338 	ibt_qp_uc_attr_t	*uc;
2339 	tavor_hw_addr_path_t	*qpc_path;
2340 	ibt_adds_vect_t		*adds_vect;
2341 	uint_t			portnum, pkeyindx;
2342 	uint32_t		opmask = 0;
2343 	int			status;
2344 
2345 	TAVOR_TNF_ENTER(tavor_qp_sqd2rts);
2346 
2347 	ASSERT(MUTEX_HELD(&qp->qp_lock));
2348 
2349 	/*
2350 	 * Grab the temporary QPC entry from QP software state
2351 	 */
2352 	qpc = &qp->qpc;
2353 
2354 	/*
2355 	 * Since there are no common and/or Tavor-specific fields to be filled
2356 	 * in for this command, we begin with the QPC fields which are
2357 	 * specific to transport type.
2358 	 */
2359 	if (qp->qp_serv_type == TAVOR_QP_UD) {
2360 		ud = &info_p->qp_transport.ud;
2361 
2362 		/*
2363 		 * If we are attempting to modify the QKey for this QP, then
2364 		 * fill it in and set the appropriate flag in the "opmask"
2365 		 * parameter.
2366 		 */
2367 		if (flags & IBT_CEP_SET_QKEY) {
2368 			qpc->qkey = ud->ud_qkey;
2369 			opmask |= TAVOR_CMD_OP_QKEY;
2370 		}
2371 
2372 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
2373 		rc = &info_p->qp_transport.rc;
2374 
2375 		/*
2376 		 * Check if any of the flags indicate a change in the RDMA
2377 		 * (recv) enable/disable flags and set the appropriate flag in
2378 		 * the "opmask" parameter
2379 		 */
2380 		opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2381 
2382 		/*
2383 		 * If we are attempting to modify the path migration state for
2384 		 * this QP, then check for valid state and fill it in.  Also
2385 		 * set the appropriate flag in the "opmask" parameter.
2386 		 */
2387 		if (flags & IBT_CEP_SET_MIG) {
2388 			if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2389 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2390 			} else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2391 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2392 			} else {
2393 				TNF_PROBE_1(tavor_qp_sqd2rts_inv_mig_state_fail,
2394 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2395 				    rc->rc_mig_state);
2396 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2397 				return (IBT_QP_APM_STATE_INVALID);
2398 			}
2399 			opmask |= TAVOR_CMD_OP_PM_STATE;
2400 		}
2401 
2402 		/*
2403 		 * Check for optional alternate path and fill in the
2404 		 * appropriate QPC fields if one is specified
2405 		 */
2406 		if (flags & IBT_CEP_SET_ALT_PATH) {
2407 			qpc_path = &qpc->alt_addr_path;
2408 			adds_vect = &rc->rc_alt_path.cep_adds_vect;
2409 
2410 			/* Set the common alternate address path fields */
2411 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2412 			    TAVOR_ADDRPATH_QP, qp);
2413 			if (status != DDI_SUCCESS) {
2414 				TNF_PROBE_0(tavor_qp_sqd2rts_setaddrpath_fail,
2415 				    TAVOR_TNF_ERROR, "");
2416 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2417 				return (status);
2418 			}
2419 			qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2420 
2421 			/*
2422 			 * Check for valid alternate path port number and fill
2423 			 * it in
2424 			 */
2425 			portnum = rc->rc_alt_path.cep_hca_port_num;
2426 			if (tavor_portnum_is_valid(state, portnum)) {
2427 				qpc->alt_addr_path.portnum = portnum;
2428 			} else {
2429 				TNF_PROBE_1(tavor_qp_sqd2rts_inv_altport_fail,
2430 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
2431 				    portnum);
2432 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2433 				return (IBT_HCA_PORT_INVALID);
2434 			}
2435 
2436 			/*
2437 			 * Check for valid alternate path PKey index and fill
2438 			 * it in
2439 			 */
2440 			pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2441 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2442 				qpc->alt_addr_path.pkey_indx = pkeyindx;
2443 			} else {
2444 				TNF_PROBE_1(tavor_qp_sqd2rts_inv_altpkey_fail,
2445 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2446 				    pkeyindx);
2447 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2448 				return (IBT_PKEY_IX_ILLEGAL);
2449 			}
2450 			opmask |= TAVOR_CMD_OP_ALT_PATH;
2451 		}
2452 
2453 		/*
2454 		 * If we are attempting to modify the "Minimum RNR NAK" value
2455 		 * for this QP, then fill it in and set the appropriate flag
2456 		 * in the "opmask" parameter.
2457 		 */
2458 		if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2459 			qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2460 			opmask |= TAVOR_CMD_OP_MINRNRNAK;
2461 		}
2462 
2463 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
2464 		uc = &info_p->qp_transport.uc;
2465 
2466 		/*
2467 		 * Check if any of the flags indicate a change in the RDMA
2468 		 * Write (recv) enable/disable and set the appropriate flag
2469 		 * in the "opmask" parameter. Note: RDMA Read and Atomic are
2470 		 * not valid for UC transport.
2471 		 */
2472 		if (flags & IBT_CEP_SET_RDMA_W) {
2473 			qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2474 			opmask |= TAVOR_CMD_OP_RWE;
2475 		}
2476 
2477 		/*
2478 		 * If we are attempting to modify the path migration state for
2479 		 * this QP, then check for valid state and fill it in.  Also
2480 		 * set the appropriate flag in the "opmask" parameter.
2481 		 */
2482 		if (flags & IBT_CEP_SET_MIG) {
2483 			if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2484 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2485 			} else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2486 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2487 			} else {
2488 				TNF_PROBE_1(tavor_qp_sqd2rts_inv_mig_state_fail,
2489 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2490 				    uc->uc_mig_state);
2491 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2492 				return (IBT_QP_APM_STATE_INVALID);
2493 			}
2494 			opmask |= TAVOR_CMD_OP_PM_STATE;
2495 		}
2496 
2497 		/*
2498 		 * Check for optional alternate path and fill in the
2499 		 * appropriate QPC fields if one is specified
2500 		 */
2501 		if (flags & IBT_CEP_SET_ALT_PATH) {
2502 			qpc_path = &qpc->alt_addr_path;
2503 			adds_vect = &uc->uc_alt_path.cep_adds_vect;
2504 
2505 			/* Set the common alternate address path fields */
2506 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2507 			    TAVOR_ADDRPATH_QP, qp);
2508 			if (status != DDI_SUCCESS) {
2509 				TNF_PROBE_0(tavor_qp_sqd2rts_setaddrpath_fail,
2510 				    TAVOR_TNF_ERROR, "");
2511 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2512 				return (status);
2513 			}
2514 
2515 			/*
2516 			 * Check for valid alternate path port number and fill
2517 			 * it in
2518 			 */
2519 			portnum = uc->uc_alt_path.cep_hca_port_num;
2520 			if (tavor_portnum_is_valid(state, portnum)) {
2521 				qpc->alt_addr_path.portnum = portnum;
2522 			} else {
2523 				TNF_PROBE_1(tavor_qp_sqd2rts_inv_altport_fail,
2524 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
2525 				    portnum);
2526 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2527 				return (IBT_HCA_PORT_INVALID);
2528 			}
2529 
2530 			/*
2531 			 * Check for valid alternate path PKey index and fill
2532 			 * it in
2533 			 */
2534 			pkeyindx = uc->uc_alt_path.cep_pkey_ix;
2535 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2536 				qpc->alt_addr_path.pkey_indx = pkeyindx;
2537 			} else {
2538 				TNF_PROBE_1(tavor_qp_sqd2rts_inv_altpkey_fail,
2539 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2540 				    pkeyindx);
2541 				TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2542 				return (IBT_PKEY_IX_ILLEGAL);
2543 			}
2544 			opmask |= TAVOR_CMD_OP_ALT_PATH;
2545 		}
2546 	} else {
2547 		/*
2548 		 * Invalid QP transport type. If we got here then it's a
2549 		 * warning of a probably serious problem.  So print a message
2550 		 * and return failure
2551 		 */
2552 		TAVOR_WARNING(state, "unknown QP transport type in sqd2rts");
2553 		TNF_PROBE_0(tavor_qp_sqd2rts_inv_transtype_fail,
2554 		    TAVOR_TNF_ERROR, "");
2555 		TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2556 		return (ibc_get_ci_failure(0));
2557 	}
2558 
2559 	/*
2560 	 * Post the SQD2RTS_QP command to the Tavor firmware
2561 	 *
2562 	 * We do a TAVOR_NOSLEEP here because we are still holding the
2563 	 * "qp_lock".  If we got raised to interrupt level by priority
2564 	 * inversion, we do not want to block in this routine waiting for
2565 	 * success.
2566 	 */
2567 	status = tavor_cmn_qp_cmd_post(state, SQD2RTS_QP, qpc, qp->qp_qpnum,
2568 	    opmask, TAVOR_CMD_NOSLEEP_SPIN);
2569 	if (status != TAVOR_CMD_SUCCESS) {
2570 		if (status != TAVOR_CMD_BAD_QP_STATE) {
2571 			cmn_err(CE_CONT, "Tavor: SQD2RTS_QP command failed: "
2572 			    "%08x\n", status);
2573 			TNF_PROBE_1(tavor_qp_sqd2rts_cmd_fail,
2574 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
2575 			TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2576 			return (ibc_get_ci_failure(0));
2577 		} else {
2578 			TNF_PROBE_0(tavor_qp_sqd2rts_inv_qpstate_fail,
2579 			    TAVOR_TNF_ERROR, "");
2580 			TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2581 			return (IBT_QP_STATE_INVALID);
2582 		}
2583 	}
2584 
2585 	TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2586 	return (DDI_SUCCESS);
2587 }
2588 
2589 
2590 /*
2591  * tavor_qp_sqd2sqd()
2592  *    Context: Can be called from interrupt or base context.
2593  */
2594 static int
tavor_qp_sqd2sqd(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)2595 tavor_qp_sqd2sqd(tavor_state_t *state, tavor_qphdl_t qp,
2596     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2597 {
2598 	tavor_hw_qpc_t		*qpc;
2599 	ibt_qp_rc_attr_t	*rc;
2600 	ibt_qp_ud_attr_t	*ud;
2601 	ibt_qp_uc_attr_t	*uc;
2602 	tavor_hw_addr_path_t	*qpc_path;
2603 	ibt_adds_vect_t		*adds_vect;
2604 	uint_t			portnum, pkeyindx, rdma_ra_out, rdma_ra_in;
2605 	uint_t			rra_max, sra_max;
2606 	uint32_t		opmask = 0;
2607 	int			status;
2608 
2609 	TAVOR_TNF_ENTER(tavor_qp_sqd2sqd);
2610 
2611 	ASSERT(MUTEX_HELD(&qp->qp_lock));
2612 
2613 	/*
2614 	 * Grab the temporary QPC entry from QP software state
2615 	 */
2616 	qpc = &qp->qpc;
2617 
2618 	/*
2619 	 * Since there are no common and/or Tavor-specific fields to be filled
2620 	 * in for this command, we begin with the QPC fields which are
2621 	 * specific to transport type.
2622 	 */
2623 	if (qp->qp_serv_type == TAVOR_QP_UD) {
2624 		ud = &info_p->qp_transport.ud;
2625 
2626 		/*
2627 		 * If we are attempting to modify the PKey index for this QP,
2628 		 * then check for valid PKey index and fill it in.  Also set
2629 		 * the appropriate flag in the "opmask" parameter.
2630 		 */
2631 		if (flags & IBT_CEP_SET_PKEY_IX) {
2632 			pkeyindx = ud->ud_pkey_ix;
2633 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2634 				qpc->pri_addr_path.pkey_indx = pkeyindx;
2635 				opmask |= TAVOR_CMD_OP_PKEYINDX;
2636 				qp->qp_pkeyindx = pkeyindx;
2637 			} else {
2638 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_pkey_fail,
2639 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
2640 				    pkeyindx);
2641 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2642 				return (IBT_PKEY_IX_ILLEGAL);
2643 			}
2644 		}
2645 
2646 		/*
2647 		 * If we are attempting to modify the QKey for this QP, then
2648 		 * fill it in and set the appropriate flag in the "opmask"
2649 		 * parameter.
2650 		 */
2651 		if (flags & IBT_CEP_SET_QKEY) {
2652 			qpc->qkey = ud->ud_qkey;
2653 			opmask |= TAVOR_CMD_OP_QKEY;
2654 		}
2655 
2656 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
2657 		rc = &info_p->qp_transport.rc;
2658 
2659 		/*
2660 		 * Check if any of the flags indicate a change in the RDMA
2661 		 * (recv) enable/disable flags and set the appropriate flag in
2662 		 * the "opmask" parameter
2663 		 */
2664 		opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2665 
2666 		/*
2667 		 * Check for optional primary path and fill in the
2668 		 * appropriate QPC fields if one is specified
2669 		 */
2670 		if (flags & IBT_CEP_SET_ADDS_VECT) {
2671 			qpc_path = &qpc->pri_addr_path;
2672 			adds_vect = &rc->rc_path.cep_adds_vect;
2673 
2674 			/* Set the common primary address path fields */
2675 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2676 			    TAVOR_ADDRPATH_QP, qp);
2677 			if (status != DDI_SUCCESS) {
2678 				TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2679 				    TAVOR_TNF_ERROR, "");
2680 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2681 				return (status);
2682 			}
2683 			qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
2684 			qpc_path->ack_timeout = rc->rc_path.cep_timeout;
2685 			qpc->retry_cnt = rc->rc_retry_cnt;
2686 
2687 			/*
2688 			 * MTU changes as part of sqd2sqd are not allowed.
2689 			 * Simply keep the same MTU value here, stored in the
2690 			 * qphdl from init2rtr time.
2691 			 */
2692 			qpc->mtu = qp->qp_save_mtu;
2693 
2694 			opmask |= (TAVOR_CMD_OP_PRIM_PATH |
2695 			    TAVOR_CMD_OP_RETRYCNT | TAVOR_CMD_OP_ACKTIMEOUT |
2696 			    TAVOR_CMD_OP_PRIM_RNRRETRY);
2697 		}
2698 
2699 		/*
2700 		 * If we are attempting to modify the path migration state for
2701 		 * this QP, then check for valid state and fill it in.  Also
2702 		 * set the appropriate flag in the "opmask" parameter.
2703 		 */
2704 		if (flags & IBT_CEP_SET_MIG) {
2705 			if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2706 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2707 			} else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2708 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2709 			} else {
2710 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_mig_state_fail,
2711 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2712 				    rc->rc_mig_state);
2713 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2714 				return (IBT_QP_APM_STATE_INVALID);
2715 			}
2716 			opmask |= TAVOR_CMD_OP_PM_STATE;
2717 		}
2718 
2719 		/*
2720 		 * If we are attempting to modify the PKey index for this QP,
2721 		 * then check for valid PKey index and fill it in.  Also set
2722 		 * the appropriate flag in the "opmask" parameter.
2723 		 */
2724 		if (flags & IBT_CEP_SET_PKEY_IX) {
2725 			pkeyindx = rc->rc_path.cep_pkey_ix;
2726 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2727 				qpc->pri_addr_path.pkey_indx = pkeyindx;
2728 				opmask |= TAVOR_CMD_OP_PKEYINDX;
2729 			} else {
2730 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_pkey_fail,
2731 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
2732 				    pkeyindx);
2733 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2734 				return (IBT_PKEY_IX_ILLEGAL);
2735 			}
2736 		}
2737 
2738 		/*
2739 		 * If we are attempting to modify the port for this QP, then
2740 		 * check for valid port number and fill it in.  Also set the
2741 		 * appropriate flag in the "opmask" parameter.
2742 		 */
2743 		if (flags & IBT_CEP_SET_PORT) {
2744 			portnum = rc->rc_path.cep_hca_port_num;
2745 			if (tavor_portnum_is_valid(state, portnum)) {
2746 				qpc->pri_addr_path.portnum = portnum;
2747 			} else {
2748 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_port_fail,
2749 				    TAVOR_TNF_ERROR, "", tnf_uint, port,
2750 				    portnum);
2751 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2752 				return (IBT_HCA_PORT_INVALID);
2753 			}
2754 			opmask |= TAVOR_CMD_OP_PRIM_PORT;
2755 		}
2756 
2757 		/*
2758 		 * Check for optional alternate path and fill in the
2759 		 * appropriate QPC fields if one is specified
2760 		 */
2761 		if (flags & IBT_CEP_SET_ALT_PATH) {
2762 			qpc_path = &qpc->alt_addr_path;
2763 			adds_vect = &rc->rc_alt_path.cep_adds_vect;
2764 
2765 			/* Set the common alternate address path fields */
2766 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2767 			    TAVOR_ADDRPATH_QP, qp);
2768 			if (status != DDI_SUCCESS) {
2769 				TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2770 				    TAVOR_TNF_ERROR, "");
2771 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2772 				return (status);
2773 			}
2774 			qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2775 
2776 			/*
2777 			 * Check for valid alternate path port number and fill
2778 			 * it in
2779 			 */
2780 			portnum = rc->rc_alt_path.cep_hca_port_num;
2781 			if (tavor_portnum_is_valid(state, portnum)) {
2782 				qpc->alt_addr_path.portnum = portnum;
2783 			} else {
2784 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altport_fail,
2785 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
2786 				    portnum);
2787 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2788 				return (IBT_HCA_PORT_INVALID);
2789 			}
2790 
2791 			/*
2792 			 * Check for valid alternate path PKey index and fill
2793 			 * it in
2794 			 */
2795 			pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2796 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2797 				qpc->alt_addr_path.pkey_indx = pkeyindx;
2798 			} else {
2799 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altpkey_fail,
2800 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2801 				    pkeyindx);
2802 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2803 				return (IBT_PKEY_IX_ILLEGAL);
2804 			}
2805 			opmask |= TAVOR_CMD_OP_ALT_PATH;
2806 		}
2807 
2808 		/*
2809 		 * If we are attempting to modify the number of "outgoing
2810 		 * RDMA resources" for this QP, then check for valid value and
2811 		 * fill it in.  Also set the appropriate flag in the "opmask"
2812 		 * parameter.
2813 		 */
2814 		if (flags & IBT_CEP_SET_RDMARA_OUT) {
2815 			rdma_ra_out = rc->rc_rdma_ra_out;
2816 			if (tavor_qp_validate_init_depth(state, rc,
2817 			    &sra_max) != DDI_SUCCESS) {
2818 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_rdma_out_fail,
2819 				    TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_out,
2820 				    rdma_ra_out);
2821 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2822 				return (IBT_INVALID_PARAM);
2823 			}
2824 			qpc->sra_max = sra_max;
2825 			opmask |= TAVOR_CMD_OP_SRA_SET;
2826 		}
2827 
2828 		/*
2829 		 * If we are attempting to modify the number of "incoming
2830 		 * RDMA resources" for this QP, then check for valid value and
2831 		 * update the "rra_max" and "ra_buf_index" fields in the QPC to
2832 		 * point to the pre-allocated RDB resources (in DDR).  Also set
2833 		 * the appropriate flag in the "opmask" parameter.
2834 		 */
2835 		if (flags & IBT_CEP_SET_RDMARA_IN) {
2836 			rdma_ra_in = rc->rc_rdma_ra_in;
2837 			if (tavor_qp_validate_resp_rsrc(state, rc,
2838 			    &rra_max) != DDI_SUCCESS) {
2839 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_rdma_in_fail,
2840 				    TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_in,
2841 				    rdma_ra_in);
2842 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2843 				return (IBT_INVALID_PARAM);
2844 			}
2845 			qpc->rra_max = rra_max;
2846 			qpc->ra_buff_indx = qp->qp_rdb_ddraddr >>
2847 			    TAVOR_RDB_SIZE_SHIFT;
2848 			opmask |= TAVOR_CMD_OP_RRA_SET;
2849 		}
2850 
2851 		/*
2852 		 * If we are attempting to modify the "Local Ack Timeout" value
2853 		 * for this QP, then fill it in and set the appropriate flag in
2854 		 * the "opmask" parameter.
2855 		 */
2856 		if (flags & IBT_CEP_SET_TIMEOUT) {
2857 			qpc_path = &qpc->pri_addr_path;
2858 			qpc_path->ack_timeout = rc->rc_path.cep_timeout;
2859 			opmask |= TAVOR_CMD_OP_ACKTIMEOUT;
2860 		}
2861 
2862 		/*
2863 		 * If we are attempting to modify the "Retry Count" for this QP,
2864 		 * then fill it in and set the appropriate flag in the "opmask"
2865 		 * parameter.
2866 		 */
2867 		if (flags & IBT_CEP_SET_RETRY) {
2868 			qpc->retry_cnt = rc->rc_retry_cnt;
2869 			opmask |= TAVOR_CMD_OP_PRIM_RNRRETRY;
2870 		}
2871 
2872 		/*
2873 		 * If we are attempting to modify the "RNR Retry Count" for this
2874 		 * QP, then fill it in and set the appropriate flag in the
2875 		 * "opmask" parameter.
2876 		 */
2877 		if (flags & IBT_CEP_SET_RNR_NAK_RETRY) {
2878 			qpc_path = &qpc->pri_addr_path;
2879 			qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
2880 			opmask |= TAVOR_CMD_OP_RETRYCNT;
2881 		}
2882 
2883 		/*
2884 		 * If we are attempting to modify the "Minimum RNR NAK" value
2885 		 * for this QP, then fill it in and set the appropriate flag
2886 		 * in the "opmask" parameter.
2887 		 */
2888 		if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2889 			qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2890 			opmask |= TAVOR_CMD_OP_MINRNRNAK;
2891 		}
2892 
2893 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
2894 		uc = &info_p->qp_transport.uc;
2895 
2896 		/*
2897 		 * Check if any of the flags indicate a change in the RDMA
2898 		 * Write (recv) enable/disable and set the appropriate flag
2899 		 * in the "opmask" parameter. Note: RDMA Read and Atomic are
2900 		 * not valid for UC transport.
2901 		 */
2902 		if (flags & IBT_CEP_SET_RDMA_W) {
2903 			qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2904 			opmask |= TAVOR_CMD_OP_RWE;
2905 		}
2906 
2907 		/*
2908 		 * Check for optional primary path and fill in the
2909 		 * appropriate QPC fields if one is specified
2910 		 */
2911 		if (flags & IBT_CEP_SET_ADDS_VECT) {
2912 			qpc_path = &qpc->pri_addr_path;
2913 			adds_vect = &uc->uc_path.cep_adds_vect;
2914 
2915 			/* Set the common primary address path fields */
2916 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2917 			    TAVOR_ADDRPATH_QP, qp);
2918 			if (status != DDI_SUCCESS) {
2919 				TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2920 				    TAVOR_TNF_ERROR, "");
2921 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2922 				return (status);
2923 			}
2924 
2925 			/*
2926 			 * MTU changes as part of sqd2sqd are not allowed.
2927 			 * Simply keep the same MTU value here, stored in the
2928 			 * qphdl from init2rtr time.
2929 			 */
2930 			qpc->mtu = qp->qp_save_mtu;
2931 
2932 			opmask |= TAVOR_CMD_OP_PRIM_PATH;
2933 		}
2934 
2935 		/*
2936 		 * If we are attempting to modify the path migration state for
2937 		 * this QP, then check for valid state and fill it in.  Also
2938 		 * set the appropriate flag in the "opmask" parameter.
2939 		 */
2940 		if (flags & IBT_CEP_SET_MIG) {
2941 			if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2942 				qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2943 			} else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2944 				qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2945 			} else {
2946 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_mig_state_fail,
2947 				    TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2948 				    uc->uc_mig_state);
2949 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2950 				return (IBT_QP_APM_STATE_INVALID);
2951 			}
2952 			opmask |= TAVOR_CMD_OP_PM_STATE;
2953 		}
2954 
2955 		/*
2956 		 * If we are attempting to modify the PKey index for this QP,
2957 		 * then check for valid PKey index and fill it in.  Also set
2958 		 * the appropriate flag in the "opmask" parameter.
2959 		 */
2960 		if (flags & IBT_CEP_SET_PKEY_IX) {
2961 			pkeyindx = uc->uc_path.cep_pkey_ix;
2962 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2963 				qpc->pri_addr_path.pkey_indx = pkeyindx;
2964 				opmask |= TAVOR_CMD_OP_PKEYINDX;
2965 			} else {
2966 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_pkey,
2967 				    TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
2968 				    pkeyindx);
2969 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2970 				return (IBT_PKEY_IX_ILLEGAL);
2971 			}
2972 		}
2973 
2974 		/*
2975 		 * Check for optional alternate path and fill in the
2976 		 * appropriate QPC fields if one is specified
2977 		 */
2978 		if (flags & IBT_CEP_SET_ALT_PATH) {
2979 			qpc_path = &qpc->alt_addr_path;
2980 			adds_vect = &uc->uc_alt_path.cep_adds_vect;
2981 
2982 			/* Set the common alternate address path fields */
2983 			status = tavor_set_addr_path(state, adds_vect, qpc_path,
2984 			    TAVOR_ADDRPATH_QP, qp);
2985 			if (status != DDI_SUCCESS) {
2986 				TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2987 				    TAVOR_TNF_ERROR, "");
2988 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2989 				return (status);
2990 			}
2991 
2992 			/*
2993 			 * Check for valid alternate path port number and fill
2994 			 * it in
2995 			 */
2996 			portnum = uc->uc_alt_path.cep_hca_port_num;
2997 			if (tavor_portnum_is_valid(state, portnum)) {
2998 				qpc->alt_addr_path.portnum = portnum;
2999 			} else {
3000 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altport_fail,
3001 				    TAVOR_TNF_ERROR, "", tnf_uint, altport,
3002 				    portnum);
3003 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3004 				return (IBT_HCA_PORT_INVALID);
3005 			}
3006 
3007 			/*
3008 			 * Check for valid alternate path PKey index and fill
3009 			 * it in
3010 			 */
3011 			pkeyindx = uc->uc_alt_path.cep_pkey_ix;
3012 			if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
3013 				qpc->alt_addr_path.pkey_indx = pkeyindx;
3014 			} else {
3015 				TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altpkey_fail,
3016 				    TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
3017 				    pkeyindx);
3018 				TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3019 				return (IBT_PKEY_IX_ILLEGAL);
3020 			}
3021 			opmask |= TAVOR_CMD_OP_ALT_PATH;
3022 		}
3023 	} else {
3024 		/*
3025 		 * Invalid QP transport type. If we got here then it's a
3026 		 * warning of a probably serious problem.  So print a message
3027 		 * and return failure
3028 		 */
3029 		TAVOR_WARNING(state, "unknown QP transport type in sqd2sqd");
3030 		TNF_PROBE_0(tavor_qp_sqd2sqd_inv_transtype_fail,
3031 		    TAVOR_TNF_ERROR, "");
3032 		TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3033 		return (ibc_get_ci_failure(0));
3034 	}
3035 
3036 	/*
3037 	 * Post the SQD2SQD_QP command to the Tavor firmware
3038 	 *
3039 	 * We do a TAVOR_NOSLEEP here because we are still holding the
3040 	 * "qp_lock".  If we got raised to interrupt level by priority
3041 	 * inversion, we do not want to block in this routine waiting for
3042 	 * success.
3043 	 */
3044 	status = tavor_cmn_qp_cmd_post(state, SQD2SQD_QP, qpc, qp->qp_qpnum,
3045 	    opmask, TAVOR_CMD_NOSLEEP_SPIN);
3046 	if (status != TAVOR_CMD_SUCCESS) {
3047 		if (status != TAVOR_CMD_BAD_QP_STATE) {
3048 			cmn_err(CE_CONT, "Tavor: SQD2SQD_QP command failed: "
3049 			    "%08x\n", status);
3050 			TNF_PROBE_1(tavor_qp_sqd2sqd_cmd_fail,
3051 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3052 			TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3053 			return (ibc_get_ci_failure(0));
3054 		} else {
3055 			TNF_PROBE_0(tavor_qp_sqd2sqd_inv_qpstate_fail,
3056 			    TAVOR_TNF_ERROR, "");
3057 			TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3058 			return (IBT_QP_STATE_INVALID);
3059 		}
3060 	}
3061 
3062 	TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3063 	return (DDI_SUCCESS);
3064 }
3065 
3066 
3067 /*
3068  * tavor_qp_sqerr2rts()
3069  *    Context: Can be called from interrupt or base context.
3070  */
3071 static int
tavor_qp_sqerr2rts(tavor_state_t * state,tavor_qphdl_t qp,ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p)3072 tavor_qp_sqerr2rts(tavor_state_t *state, tavor_qphdl_t qp,
3073     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
3074 {
3075 	tavor_hw_qpc_t		*qpc;
3076 	ibt_qp_ud_attr_t	*ud;
3077 	uint32_t		opmask = 0;
3078 	int			status;
3079 
3080 	TAVOR_TNF_ENTER(tavor_qp_sqerr2rts);
3081 
3082 	ASSERT(MUTEX_HELD(&qp->qp_lock));
3083 
3084 	/*
3085 	 * Grab the temporary QPC entry from QP software state
3086 	 */
3087 	qpc = &qp->qpc;
3088 
3089 	/*
3090 	 * Since there are no common and/or Tavor-specific fields to be filled
3091 	 * in for this command, we begin with the QPC fields which are
3092 	 * specific to transport type.
3093 	 */
3094 	if (qp->qp_serv_type == TAVOR_QP_UD) {
3095 		ud = &info_p->qp_transport.ud;
3096 
3097 		/*
3098 		 * If we are attempting to modify the QKey for this QP, then
3099 		 * fill it in and set the appropriate flag in the "opmask"
3100 		 * parameter.
3101 		 */
3102 		if (flags & IBT_CEP_SET_QKEY) {
3103 			qpc->qkey = ud->ud_qkey;
3104 			opmask |= TAVOR_CMD_OP_QKEY;
3105 		}
3106 
3107 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
3108 
3109 		/*
3110 		 * Check if any of the flags indicate a change in the RDMA
3111 		 * Write (recv) enable/disable and set the appropriate flag
3112 		 * in the "opmask" parameter. Note: RDMA Read and Atomic are
3113 		 * not valid for UC transport.
3114 		 */
3115 		if (flags & IBT_CEP_SET_RDMA_W) {
3116 			qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
3117 			opmask |= TAVOR_CMD_OP_RWE;
3118 		}
3119 	} else {
3120 		/*
3121 		 * Invalid QP transport type. If we got here then it's a
3122 		 * warning of a probably serious problem.  So print a message
3123 		 * and return failure
3124 		 */
3125 		TAVOR_WARNING(state, "unknown QP transport type in sqerr2rts");
3126 		TNF_PROBE_0(tavor_qp_sqerr2rts_inv_transtype_fail,
3127 		    TAVOR_TNF_ERROR, "");
3128 		TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3129 		return (ibc_get_ci_failure(0));
3130 	}
3131 
3132 	/*
3133 	 * Post the SQERR2RTS_QP command to the Tavor firmware
3134 	 *
3135 	 * We do a TAVOR_NOSLEEP here because we are still holding the
3136 	 * "qp_lock".  If we got raised to interrupt level by priority
3137 	 * inversion, we do not want to block in this routine waiting for
3138 	 * success.
3139 	 */
3140 	status = tavor_cmn_qp_cmd_post(state, SQERR2RTS_QP, qpc, qp->qp_qpnum,
3141 	    opmask, TAVOR_CMD_NOSLEEP_SPIN);
3142 	if (status != TAVOR_CMD_SUCCESS) {
3143 		if (status != TAVOR_CMD_BAD_QP_STATE) {
3144 			cmn_err(CE_CONT, "Tavor: SQERR2RTS_QP command failed: "
3145 			    "%08x\n", status);
3146 			TNF_PROBE_1(tavor_qp_sqerr2rts_cmd_fail,
3147 			    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3148 			TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3149 			return (ibc_get_ci_failure(0));
3150 		} else {
3151 			TNF_PROBE_0(tavor_qp_sqerr2rts_inv_qpstate_fail,
3152 			    TAVOR_TNF_ERROR, "");
3153 			TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3154 			return (IBT_QP_STATE_INVALID);
3155 		}
3156 	}
3157 
3158 	TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3159 	return (DDI_SUCCESS);
3160 }
3161 
3162 
3163 /*
3164  * tavor_qp_to_error()
3165  *    Context: Can be called from interrupt or base context.
3166  */
3167 static int
tavor_qp_to_error(tavor_state_t * state,tavor_qphdl_t qp)3168 tavor_qp_to_error(tavor_state_t *state, tavor_qphdl_t qp)
3169 {
3170 	int	status;
3171 
3172 	TAVOR_TNF_ENTER(tavor_qp_to_error);
3173 
3174 	ASSERT(MUTEX_HELD(&qp->qp_lock));
3175 
3176 	/*
3177 	 * Post the TOERR_QP command to the Tavor firmware
3178 	 *
3179 	 * We do a TAVOR_NOSLEEP here because we are still holding the
3180 	 * "qp_lock".  If we got raised to interrupt level by priority
3181 	 * inversion, we do not want to block in this routine waiting for
3182 	 * success.
3183 	 */
3184 	status = tavor_cmn_qp_cmd_post(state, TOERR_QP, NULL, qp->qp_qpnum,
3185 	    0, TAVOR_CMD_NOSLEEP_SPIN);
3186 	if (status != TAVOR_CMD_SUCCESS) {
3187 		cmn_err(CE_CONT, "Tavor: TOERR_QP command failed: %08x\n",
3188 		    status);
3189 		TNF_PROBE_1(tavor_qp_to_error_cmd_fail,
3190 		    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3191 		TAVOR_TNF_EXIT(tavor_qp_to_error);
3192 		return (ibc_get_ci_failure(0));
3193 	}
3194 
3195 	TAVOR_TNF_EXIT(tavor_qp_to_error);
3196 	return (DDI_SUCCESS);
3197 }
3198 
3199 
3200 /*
3201  * tavor_qp_to_reset()
3202  *    Context: Can be called from interrupt or base context.
3203  */
3204 int
tavor_qp_to_reset(tavor_state_t * state,tavor_qphdl_t qp)3205 tavor_qp_to_reset(tavor_state_t *state, tavor_qphdl_t qp)
3206 {
3207 	tavor_hw_qpc_t	*qpc;
3208 	int		status;
3209 
3210 	TAVOR_TNF_ENTER(tavor_qp_to_reset);
3211 
3212 	ASSERT(MUTEX_HELD(&qp->qp_lock));
3213 
3214 	/*
3215 	 * Grab the temporary QPC entry from QP software state
3216 	 */
3217 	qpc = &qp->qpc;
3218 
3219 	/*
3220 	 * Post the TORST_QP command to the Tavor firmware
3221 	 *
3222 	 * We do a TAVOR_NOSLEEP here because we are still holding the
3223 	 * "qp_lock".  If we got raised to interrupt level by priority
3224 	 * inversion, we do not want to block in this routine waiting for
3225 	 * success.
3226 	 */
3227 	status = tavor_cmn_qp_cmd_post(state, TORST_QP, qpc, qp->qp_qpnum,
3228 	    0, TAVOR_CMD_NOSLEEP_SPIN);
3229 	if (status != TAVOR_CMD_SUCCESS) {
3230 		cmn_err(CE_CONT, "Tavor: TORST_QP command failed: %08x\n",
3231 		    status);
3232 		TNF_PROBE_1(tavor_qp_to_reset_cmd_fail,
3233 		    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3234 		TAVOR_TNF_EXIT(tavor_qp_to_reset);
3235 		return (ibc_get_ci_failure(0));
3236 	}
3237 
3238 	TAVOR_TNF_EXIT(tavor_qp_to_reset);
3239 	return (DDI_SUCCESS);
3240 }
3241 
3242 
3243 /*
3244  * tavor_qp_reset2err()
3245  *    Context: Can be called from interrupt or base context.
3246  */
3247 static int
tavor_qp_reset2err(tavor_state_t * state,tavor_qphdl_t qp)3248 tavor_qp_reset2err(tavor_state_t *state, tavor_qphdl_t qp)
3249 {
3250 	tavor_hw_qpc_t	*qpc;
3251 	int		status;
3252 
3253 	TAVOR_TNF_ENTER(tavor_qp_reset2err);
3254 
3255 	ASSERT(MUTEX_HELD(&qp->qp_lock));
3256 
3257 	/*
3258 	 * In order to implement the transition from "Reset" directly to the
3259 	 * "Error" state, it is necessary to first give ownership of the QP
3260 	 * context to the Tavor hardware.  This is accomplished by transitioning
3261 	 * the QP to "Init" as an intermediate step and then, immediately
3262 	 * transitioning to "Error".
3263 	 *
3264 	 * When this function returns success, the QP context will be owned by
3265 	 * the Tavor hardware and will be in the "Error" state.
3266 	 */
3267 
3268 	/*
3269 	 * Grab the temporary QPC entry from QP software state
3270 	 */
3271 	qpc = &qp->qpc;
3272 
3273 	/*
3274 	 * Fill in the common and/or Tavor-specific fields in the QPC
3275 	 */
3276 	if (qp->qp_is_special) {
3277 		qpc->serv_type	= TAVOR_QP_MLX;
3278 	} else {
3279 		qpc->serv_type	= qp->qp_serv_type;
3280 	}
3281 	qpc->pm_state		= TAVOR_QP_PMSTATE_MIGRATED;
3282 	qpc->de			= TAVOR_QP_DESC_EVT_ENABLED;
3283 	qpc->sched_q		= TAVOR_QP_SCHEDQ_GET(qp->qp_qpnum);
3284 	if (qp->qp_is_umap) {
3285 		qpc->usr_page = qp->qp_uarpg;
3286 	} else {
3287 		qpc->usr_page = 0;
3288 	}
3289 	qpc->pd			= qp->qp_pdhdl->pd_pdnum;
3290 	qpc->wqe_baseaddr	= 0;
3291 	qpc->wqe_lkey		= qp->qp_mrhdl->mr_lkey;
3292 	qpc->ssc		= qp->qp_sq_sigtype;
3293 	qpc->cqn_snd		= qp->qp_sq_cqhdl->cq_cqnum;
3294 	qpc->rsc		= TAVOR_QP_RQ_ALL_SIGNALED;
3295 	qpc->cqn_rcv		= qp->qp_rq_cqhdl->cq_cqnum;
3296 	qpc->srq_en		= qp->qp_srq_en;
3297 
3298 	if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
3299 		qpc->srq_number	= qp->qp_srqhdl->srq_srqnum;
3300 	} else {
3301 		qpc->srq_number = 0;
3302 	}
3303 
3304 	/*
3305 	 * Now fill in the QPC fields which are specific to transport type
3306 	 */
3307 	if (qp->qp_serv_type == TAVOR_QP_UD) {
3308 		/* Set the UD parameters to an invalid default */
3309 		qpc->qkey = 0;
3310 		qpc->pri_addr_path.portnum = 1;
3311 		qpc->pri_addr_path.pkey_indx = 0;
3312 
3313 	} else if (qp->qp_serv_type == TAVOR_QP_RC) {
3314 		/* Set the RC parameters to invalid default */
3315 		qpc->rre = 0;
3316 		qpc->rwe = 0;
3317 		qpc->rae = 0;
3318 		qpc->pri_addr_path.portnum = 1;
3319 		qpc->pri_addr_path.pkey_indx = 0;
3320 
3321 	} else if (qp->qp_serv_type == TAVOR_QP_UC) {
3322 		/* Set the UC parameters to invalid default */
3323 		qpc->rwe = 0;
3324 		qpc->pri_addr_path.portnum = 1;
3325 		qpc->pri_addr_path.pkey_indx = 0;
3326 
3327 	} else {
3328 		/*
3329 		 * Invalid QP transport type. If we got here then it's a
3330 		 * warning of a probably serious problem.  So print a message
3331 		 * and return failure
3332 		 */
3333 		TAVOR_WARNING(state, "unknown QP transport type in rst2err");
3334 		TNF_PROBE_0(tavor_qp_reset2err_inv_transtype_fail,
3335 		    TAVOR_TNF_ERROR, "");
3336 		TAVOR_TNF_EXIT(tavor_qp_reset2err);
3337 		return (ibc_get_ci_failure(0));
3338 	}
3339 
3340 	/*
3341 	 * Post the RST2INIT_QP command to the Tavor firmware
3342 	 *
3343 	 * We do a TAVOR_NOSLEEP here because we are still holding the
3344 	 * "qp_lock".  If we got raised to interrupt level by priority
3345 	 * inversion, we do not want to block in this routine waiting for
3346 	 * success.
3347 	 */
3348 	status = tavor_cmn_qp_cmd_post(state, RST2INIT_QP, qpc, qp->qp_qpnum,
3349 	    0, TAVOR_CMD_NOSLEEP_SPIN);
3350 	if (status != TAVOR_CMD_SUCCESS) {
3351 		cmn_err(CE_CONT, "Tavor: RST2INIT_QP command failed: %08x\n",
3352 		    status);
3353 		TNF_PROBE_1(tavor_qp_reset2err_rst2init_cmd_fail,
3354 		    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3355 		TAVOR_TNF_EXIT(tavor_qp_reset2err);
3356 		return (ibc_get_ci_failure(0));
3357 	}
3358 
3359 	/*
3360 	 * Now post the TOERR_QP command to the Tavor firmware
3361 	 *
3362 	 * We still do a TAVOR_NOSLEEP here because we are still holding the
3363 	 * "qp_lock".  Note:  If this fails (which it really never should),
3364 	 * it indicates a serious problem in the HW or SW.  We try to move
3365 	 * the QP back to the "Reset" state if possible and print a warning
3366 	 * message if not.  In any case, we return an error here.
3367 	 */
3368 	status = tavor_cmn_qp_cmd_post(state, TOERR_QP, NULL, qp->qp_qpnum,
3369 	    0, TAVOR_CMD_NOSLEEP_SPIN);
3370 	if (status != TAVOR_CMD_SUCCESS) {
3371 		cmn_err(CE_CONT, "Tavor: TOERR_QP command failed: %08x\n",
3372 		    status);
3373 		if (tavor_qp_to_reset(state, qp) != DDI_SUCCESS) {
3374 			TAVOR_WARNING(state, "failed to reset QP context");
3375 		}
3376 		TNF_PROBE_1(tavor_qp_reset2err_toerr_cmd_fail,
3377 		    TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3378 		TAVOR_TNF_EXIT(tavor_qp_reset2err);
3379 		return (ibc_get_ci_failure(0));
3380 	}
3381 
3382 	TAVOR_TNF_EXIT(tavor_qp_reset2err);
3383 	return (DDI_SUCCESS);
3384 }
3385 
3386 
3387 /*
3388  * tavor_check_rdma_enable_flags()
3389  *    Context: Can be called from interrupt or base context.
3390  */
3391 static uint_t
tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,ibt_qp_info_t * info_p,tavor_hw_qpc_t * qpc)3392 tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,
3393     ibt_qp_info_t *info_p, tavor_hw_qpc_t *qpc)
3394 {
3395 	uint_t	opmask = 0;
3396 
3397 	if (flags & IBT_CEP_SET_RDMA_R) {
3398 		qpc->rre = (info_p->qp_flags & IBT_CEP_RDMA_RD) ? 1 : 0;
3399 		opmask |= TAVOR_CMD_OP_RRE;
3400 	}
3401 
3402 	if (flags & IBT_CEP_SET_RDMA_W) {
3403 		qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
3404 		opmask |= TAVOR_CMD_OP_RWE;
3405 	}
3406 
3407 	if (flags & IBT_CEP_SET_ATOMIC) {
3408 		qpc->rae = (info_p->qp_flags & IBT_CEP_ATOMIC) ? 1 : 0;
3409 		opmask |= TAVOR_CMD_OP_RAE;
3410 	}
3411 
3412 	return (opmask);
3413 }
3414 
3415 /*
3416  * tavor_qp_validate_resp_rsrc()
3417  *    Context: Can be called from interrupt or base context.
3418  */
3419 static int
tavor_qp_validate_resp_rsrc(tavor_state_t * state,ibt_qp_rc_attr_t * rc,uint_t * rra_max)3420 tavor_qp_validate_resp_rsrc(tavor_state_t *state, ibt_qp_rc_attr_t *rc,
3421     uint_t *rra_max)
3422 {
3423 	uint_t	rdma_ra_in;
3424 
3425 	rdma_ra_in = rc->rc_rdma_ra_in;
3426 
3427 	/*
3428 	 * Check if number of responder resources is too large.  Return an
3429 	 * error if it is
3430 	 */
3431 	if (rdma_ra_in > state->ts_cfg_profile->cp_hca_max_rdma_in_qp) {
3432 		return (IBT_INVALID_PARAM);
3433 	}
3434 
3435 	/*
3436 	 * If the number of responder resources is too small, round it up.
3437 	 * Then find the next highest power-of-2
3438 	 */
3439 	if (rdma_ra_in == 0) {
3440 		rdma_ra_in = 1;
3441 	}
3442 	if (ISP2(rdma_ra_in)) {
3443 		*rra_max = highbit(rdma_ra_in) - 1;
3444 	} else {
3445 		*rra_max = highbit(rdma_ra_in);
3446 	}
3447 	return (DDI_SUCCESS);
3448 }
3449 
3450 
3451 /*
3452  * tavor_qp_validate_init_depth()
3453  *    Context: Can be called from interrupt or base context.
3454  */
3455 static int
tavor_qp_validate_init_depth(tavor_state_t * state,ibt_qp_rc_attr_t * rc,uint_t * sra_max)3456 tavor_qp_validate_init_depth(tavor_state_t *state, ibt_qp_rc_attr_t *rc,
3457     uint_t *sra_max)
3458 {
3459 	uint_t	rdma_ra_out;
3460 
3461 	rdma_ra_out = rc->rc_rdma_ra_out;
3462 
3463 	/*
3464 	 * Check if requested initiator depth is too large.  Return an error
3465 	 * if it is
3466 	 */
3467 	if (rdma_ra_out > state->ts_cfg_profile->cp_hca_max_rdma_out_qp) {
3468 		return (IBT_INVALID_PARAM);
3469 	}
3470 
3471 	/*
3472 	 * If the requested initiator depth is too small, round it up.
3473 	 * Then find the next highest power-of-2
3474 	 */
3475 	if (rdma_ra_out == 0) {
3476 		rdma_ra_out = 1;
3477 	}
3478 	if (ISP2(rdma_ra_out)) {
3479 		*sra_max = highbit(rdma_ra_out) - 1;
3480 	} else {
3481 		*sra_max = highbit(rdma_ra_out);
3482 	}
3483 	return (DDI_SUCCESS);
3484 }
3485 
3486 
3487 /*
3488  * tavor_qp_validate_mtu()
3489  *    Context: Can be called from interrupt or base context.
3490  */
3491 static int
tavor_qp_validate_mtu(tavor_state_t * state,uint_t mtu)3492 tavor_qp_validate_mtu(tavor_state_t *state, uint_t mtu)
3493 {
3494 	/*
3495 	 * Check for invalid MTU values (i.e. zero or any value larger than
3496 	 * the HCA's port maximum).
3497 	 */
3498 	if ((mtu == 0) || (mtu > state->ts_cfg_profile->cp_max_mtu)) {
3499 		return (IBT_HCA_PORT_MTU_EXCEEDED);
3500 	}
3501 	return (DDI_SUCCESS);
3502 }
3503