xref: /linux/drivers/infiniband/core/verbs.c (revision 26fbb4c8c7c3ee9a4c3b4de555a8587b5a19154e)
1 /*
2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
9  *
10  * This software is available to you under a choice of one of two
11  * licenses.  You may choose to be licensed under the terms of the GNU
12  * General Public License (GPL) Version 2, available from the file
13  * COPYING in the main directory of this source tree, or the
14  * OpenIB.org BSD license below:
15  *
16  *     Redistribution and use in source and binary forms, with or
17  *     without modification, are permitted provided that the following
18  *     conditions are met:
19  *
20  *      - Redistributions of source code must retain the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer.
23  *
24  *      - Redistributions in binary form must reproduce the above
25  *        copyright notice, this list of conditions and the following
26  *        disclaimer in the documentation and/or other materials
27  *        provided with the distribution.
28  *
29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36  * SOFTWARE.
37  */
38 
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/export.h>
42 #include <linux/string.h>
43 #include <linux/slab.h>
44 #include <linux/in.h>
45 #include <linux/in6.h>
46 #include <net/addrconf.h>
47 #include <linux/security.h>
48 
49 #include <rdma/ib_verbs.h>
50 #include <rdma/ib_cache.h>
51 #include <rdma/ib_addr.h>
52 #include <rdma/rw.h>
53 #include <rdma/lag.h>
54 
55 #include "core_priv.h"
56 #include <trace/events/rdma_core.h>
57 
58 static int ib_resolve_eth_dmac(struct ib_device *device,
59 			       struct rdma_ah_attr *ah_attr);
60 
61 static const char * const ib_events[] = {
62 	[IB_EVENT_CQ_ERR]		= "CQ error",
63 	[IB_EVENT_QP_FATAL]		= "QP fatal error",
64 	[IB_EVENT_QP_REQ_ERR]		= "QP request error",
65 	[IB_EVENT_QP_ACCESS_ERR]	= "QP access error",
66 	[IB_EVENT_COMM_EST]		= "communication established",
67 	[IB_EVENT_SQ_DRAINED]		= "send queue drained",
68 	[IB_EVENT_PATH_MIG]		= "path migration successful",
69 	[IB_EVENT_PATH_MIG_ERR]		= "path migration error",
70 	[IB_EVENT_DEVICE_FATAL]		= "device fatal error",
71 	[IB_EVENT_PORT_ACTIVE]		= "port active",
72 	[IB_EVENT_PORT_ERR]		= "port error",
73 	[IB_EVENT_LID_CHANGE]		= "LID change",
74 	[IB_EVENT_PKEY_CHANGE]		= "P_key change",
75 	[IB_EVENT_SM_CHANGE]		= "SM change",
76 	[IB_EVENT_SRQ_ERR]		= "SRQ error",
77 	[IB_EVENT_SRQ_LIMIT_REACHED]	= "SRQ limit reached",
78 	[IB_EVENT_QP_LAST_WQE_REACHED]	= "last WQE reached",
79 	[IB_EVENT_CLIENT_REREGISTER]	= "client reregister",
80 	[IB_EVENT_GID_CHANGE]		= "GID changed",
81 };
82 
83 const char *__attribute_const__ ib_event_msg(enum ib_event_type event)
84 {
85 	size_t index = event;
86 
87 	return (index < ARRAY_SIZE(ib_events) && ib_events[index]) ?
88 			ib_events[index] : "unrecognized event";
89 }
90 EXPORT_SYMBOL(ib_event_msg);
91 
92 static const char * const wc_statuses[] = {
93 	[IB_WC_SUCCESS]			= "success",
94 	[IB_WC_LOC_LEN_ERR]		= "local length error",
95 	[IB_WC_LOC_QP_OP_ERR]		= "local QP operation error",
96 	[IB_WC_LOC_EEC_OP_ERR]		= "local EE context operation error",
97 	[IB_WC_LOC_PROT_ERR]		= "local protection error",
98 	[IB_WC_WR_FLUSH_ERR]		= "WR flushed",
99 	[IB_WC_MW_BIND_ERR]		= "memory management operation error",
100 	[IB_WC_BAD_RESP_ERR]		= "bad response error",
101 	[IB_WC_LOC_ACCESS_ERR]		= "local access error",
102 	[IB_WC_REM_INV_REQ_ERR]		= "invalid request error",
103 	[IB_WC_REM_ACCESS_ERR]		= "remote access error",
104 	[IB_WC_REM_OP_ERR]		= "remote operation error",
105 	[IB_WC_RETRY_EXC_ERR]		= "transport retry counter exceeded",
106 	[IB_WC_RNR_RETRY_EXC_ERR]	= "RNR retry counter exceeded",
107 	[IB_WC_LOC_RDD_VIOL_ERR]	= "local RDD violation error",
108 	[IB_WC_REM_INV_RD_REQ_ERR]	= "remote invalid RD request",
109 	[IB_WC_REM_ABORT_ERR]		= "operation aborted",
110 	[IB_WC_INV_EECN_ERR]		= "invalid EE context number",
111 	[IB_WC_INV_EEC_STATE_ERR]	= "invalid EE context state",
112 	[IB_WC_FATAL_ERR]		= "fatal error",
113 	[IB_WC_RESP_TIMEOUT_ERR]	= "response timeout error",
114 	[IB_WC_GENERAL_ERR]		= "general error",
115 };
116 
117 const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status)
118 {
119 	size_t index = status;
120 
121 	return (index < ARRAY_SIZE(wc_statuses) && wc_statuses[index]) ?
122 			wc_statuses[index] : "unrecognized status";
123 }
124 EXPORT_SYMBOL(ib_wc_status_msg);
125 
126 __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
127 {
128 	switch (rate) {
129 	case IB_RATE_2_5_GBPS: return   1;
130 	case IB_RATE_5_GBPS:   return   2;
131 	case IB_RATE_10_GBPS:  return   4;
132 	case IB_RATE_20_GBPS:  return   8;
133 	case IB_RATE_30_GBPS:  return  12;
134 	case IB_RATE_40_GBPS:  return  16;
135 	case IB_RATE_60_GBPS:  return  24;
136 	case IB_RATE_80_GBPS:  return  32;
137 	case IB_RATE_120_GBPS: return  48;
138 	case IB_RATE_14_GBPS:  return   6;
139 	case IB_RATE_56_GBPS:  return  22;
140 	case IB_RATE_112_GBPS: return  45;
141 	case IB_RATE_168_GBPS: return  67;
142 	case IB_RATE_25_GBPS:  return  10;
143 	case IB_RATE_100_GBPS: return  40;
144 	case IB_RATE_200_GBPS: return  80;
145 	case IB_RATE_300_GBPS: return 120;
146 	case IB_RATE_28_GBPS:  return  11;
147 	case IB_RATE_50_GBPS:  return  20;
148 	case IB_RATE_400_GBPS: return 160;
149 	case IB_RATE_600_GBPS: return 240;
150 	default:	       return  -1;
151 	}
152 }
153 EXPORT_SYMBOL(ib_rate_to_mult);
154 
155 __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
156 {
157 	switch (mult) {
158 	case 1:   return IB_RATE_2_5_GBPS;
159 	case 2:   return IB_RATE_5_GBPS;
160 	case 4:   return IB_RATE_10_GBPS;
161 	case 8:   return IB_RATE_20_GBPS;
162 	case 12:  return IB_RATE_30_GBPS;
163 	case 16:  return IB_RATE_40_GBPS;
164 	case 24:  return IB_RATE_60_GBPS;
165 	case 32:  return IB_RATE_80_GBPS;
166 	case 48:  return IB_RATE_120_GBPS;
167 	case 6:   return IB_RATE_14_GBPS;
168 	case 22:  return IB_RATE_56_GBPS;
169 	case 45:  return IB_RATE_112_GBPS;
170 	case 67:  return IB_RATE_168_GBPS;
171 	case 10:  return IB_RATE_25_GBPS;
172 	case 40:  return IB_RATE_100_GBPS;
173 	case 80:  return IB_RATE_200_GBPS;
174 	case 120: return IB_RATE_300_GBPS;
175 	case 11:  return IB_RATE_28_GBPS;
176 	case 20:  return IB_RATE_50_GBPS;
177 	case 160: return IB_RATE_400_GBPS;
178 	case 240: return IB_RATE_600_GBPS;
179 	default:  return IB_RATE_PORT_CURRENT;
180 	}
181 }
182 EXPORT_SYMBOL(mult_to_ib_rate);
183 
184 __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
185 {
186 	switch (rate) {
187 	case IB_RATE_2_5_GBPS: return 2500;
188 	case IB_RATE_5_GBPS:   return 5000;
189 	case IB_RATE_10_GBPS:  return 10000;
190 	case IB_RATE_20_GBPS:  return 20000;
191 	case IB_RATE_30_GBPS:  return 30000;
192 	case IB_RATE_40_GBPS:  return 40000;
193 	case IB_RATE_60_GBPS:  return 60000;
194 	case IB_RATE_80_GBPS:  return 80000;
195 	case IB_RATE_120_GBPS: return 120000;
196 	case IB_RATE_14_GBPS:  return 14062;
197 	case IB_RATE_56_GBPS:  return 56250;
198 	case IB_RATE_112_GBPS: return 112500;
199 	case IB_RATE_168_GBPS: return 168750;
200 	case IB_RATE_25_GBPS:  return 25781;
201 	case IB_RATE_100_GBPS: return 103125;
202 	case IB_RATE_200_GBPS: return 206250;
203 	case IB_RATE_300_GBPS: return 309375;
204 	case IB_RATE_28_GBPS:  return 28125;
205 	case IB_RATE_50_GBPS:  return 53125;
206 	case IB_RATE_400_GBPS: return 425000;
207 	case IB_RATE_600_GBPS: return 637500;
208 	default:	       return -1;
209 	}
210 }
211 EXPORT_SYMBOL(ib_rate_to_mbps);
212 
213 __attribute_const__ enum rdma_transport_type
214 rdma_node_get_transport(unsigned int node_type)
215 {
216 
217 	if (node_type == RDMA_NODE_USNIC)
218 		return RDMA_TRANSPORT_USNIC;
219 	if (node_type == RDMA_NODE_USNIC_UDP)
220 		return RDMA_TRANSPORT_USNIC_UDP;
221 	if (node_type == RDMA_NODE_RNIC)
222 		return RDMA_TRANSPORT_IWARP;
223 	if (node_type == RDMA_NODE_UNSPECIFIED)
224 		return RDMA_TRANSPORT_UNSPECIFIED;
225 
226 	return RDMA_TRANSPORT_IB;
227 }
228 EXPORT_SYMBOL(rdma_node_get_transport);
229 
230 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
231 {
232 	enum rdma_transport_type lt;
233 	if (device->ops.get_link_layer)
234 		return device->ops.get_link_layer(device, port_num);
235 
236 	lt = rdma_node_get_transport(device->node_type);
237 	if (lt == RDMA_TRANSPORT_IB)
238 		return IB_LINK_LAYER_INFINIBAND;
239 
240 	return IB_LINK_LAYER_ETHERNET;
241 }
242 EXPORT_SYMBOL(rdma_port_get_link_layer);
243 
244 /* Protection domains */
245 
246 /**
247  * __ib_alloc_pd - Allocates an unused protection domain.
248  * @device: The device on which to allocate the protection domain.
249  * @flags: protection domain flags
250  * @caller: caller's build-time module name
251  *
252  * A protection domain object provides an association between QPs, shared
253  * receive queues, address handles, memory regions, and memory windows.
254  *
255  * Every PD has a local_dma_lkey which can be used as the lkey value for local
256  * memory operations.
257  */
258 struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
259 		const char *caller)
260 {
261 	struct ib_pd *pd;
262 	int mr_access_flags = 0;
263 	int ret;
264 
265 	pd = rdma_zalloc_drv_obj(device, ib_pd);
266 	if (!pd)
267 		return ERR_PTR(-ENOMEM);
268 
269 	pd->device = device;
270 	pd->uobject = NULL;
271 	pd->__internal_mr = NULL;
272 	atomic_set(&pd->usecnt, 0);
273 	pd->flags = flags;
274 
275 	rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
276 	rdma_restrack_set_name(&pd->res, caller);
277 
278 	ret = device->ops.alloc_pd(pd, NULL);
279 	if (ret) {
280 		rdma_restrack_put(&pd->res);
281 		kfree(pd);
282 		return ERR_PTR(ret);
283 	}
284 	rdma_restrack_add(&pd->res);
285 
286 	if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
287 		pd->local_dma_lkey = device->local_dma_lkey;
288 	else
289 		mr_access_flags |= IB_ACCESS_LOCAL_WRITE;
290 
291 	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
292 		pr_warn("%s: enabling unsafe global rkey\n", caller);
293 		mr_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE;
294 	}
295 
296 	if (mr_access_flags) {
297 		struct ib_mr *mr;
298 
299 		mr = pd->device->ops.get_dma_mr(pd, mr_access_flags);
300 		if (IS_ERR(mr)) {
301 			ib_dealloc_pd(pd);
302 			return ERR_CAST(mr);
303 		}
304 
305 		mr->device	= pd->device;
306 		mr->pd		= pd;
307 		mr->type        = IB_MR_TYPE_DMA;
308 		mr->uobject	= NULL;
309 		mr->need_inval	= false;
310 
311 		pd->__internal_mr = mr;
312 
313 		if (!(device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY))
314 			pd->local_dma_lkey = pd->__internal_mr->lkey;
315 
316 		if (flags & IB_PD_UNSAFE_GLOBAL_RKEY)
317 			pd->unsafe_global_rkey = pd->__internal_mr->rkey;
318 	}
319 
320 	return pd;
321 }
322 EXPORT_SYMBOL(__ib_alloc_pd);
323 
324 /**
325  * ib_dealloc_pd_user - Deallocates a protection domain.
326  * @pd: The protection domain to deallocate.
327  * @udata: Valid user data or NULL for kernel object
328  *
329  * It is an error to call this function while any resources in the pd still
330  * exist.  The caller is responsible to synchronously destroy them and
331  * guarantee no new allocations will happen.
332  */
333 int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
334 {
335 	int ret;
336 
337 	if (pd->__internal_mr) {
338 		ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL);
339 		WARN_ON(ret);
340 		pd->__internal_mr = NULL;
341 	}
342 
343 	/* uverbs manipulates usecnt with proper locking, while the kabi
344 	   requires the caller to guarantee we can't race here. */
345 	WARN_ON(atomic_read(&pd->usecnt));
346 
347 	ret = pd->device->ops.dealloc_pd(pd, udata);
348 	if (ret)
349 		return ret;
350 
351 	rdma_restrack_del(&pd->res);
352 	kfree(pd);
353 	return ret;
354 }
355 EXPORT_SYMBOL(ib_dealloc_pd_user);
356 
357 /* Address handles */
358 
359 /**
360  * rdma_copy_ah_attr - Copy rdma ah attribute from source to destination.
361  * @dest:       Pointer to destination ah_attr. Contents of the destination
362  *              pointer is assumed to be invalid and attribute are overwritten.
363  * @src:        Pointer to source ah_attr.
364  */
365 void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
366 		       const struct rdma_ah_attr *src)
367 {
368 	*dest = *src;
369 	if (dest->grh.sgid_attr)
370 		rdma_hold_gid_attr(dest->grh.sgid_attr);
371 }
372 EXPORT_SYMBOL(rdma_copy_ah_attr);
373 
374 /**
375  * rdma_replace_ah_attr - Replace valid ah_attr with new new one.
376  * @old:        Pointer to existing ah_attr which needs to be replaced.
377  *              old is assumed to be valid or zero'd
378  * @new:        Pointer to the new ah_attr.
379  *
380  * rdma_replace_ah_attr() first releases any reference in the old ah_attr if
381  * old the ah_attr is valid; after that it copies the new attribute and holds
382  * the reference to the replaced ah_attr.
383  */
384 void rdma_replace_ah_attr(struct rdma_ah_attr *old,
385 			  const struct rdma_ah_attr *new)
386 {
387 	rdma_destroy_ah_attr(old);
388 	*old = *new;
389 	if (old->grh.sgid_attr)
390 		rdma_hold_gid_attr(old->grh.sgid_attr);
391 }
392 EXPORT_SYMBOL(rdma_replace_ah_attr);
393 
394 /**
395  * rdma_move_ah_attr - Move ah_attr pointed by source to destination.
396  * @dest:       Pointer to destination ah_attr to copy to.
397  *              dest is assumed to be valid or zero'd
398  * @src:        Pointer to the new ah_attr.
399  *
400  * rdma_move_ah_attr() first releases any reference in the destination ah_attr
401  * if it is valid. This also transfers ownership of internal references from
402  * src to dest, making src invalid in the process. No new reference of the src
403  * ah_attr is taken.
404  */
405 void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src)
406 {
407 	rdma_destroy_ah_attr(dest);
408 	*dest = *src;
409 	src->grh.sgid_attr = NULL;
410 }
411 EXPORT_SYMBOL(rdma_move_ah_attr);
412 
413 /*
414  * Validate that the rdma_ah_attr is valid for the device before passing it
415  * off to the driver.
416  */
417 static int rdma_check_ah_attr(struct ib_device *device,
418 			      struct rdma_ah_attr *ah_attr)
419 {
420 	if (!rdma_is_port_valid(device, ah_attr->port_num))
421 		return -EINVAL;
422 
423 	if ((rdma_is_grh_required(device, ah_attr->port_num) ||
424 	     ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) &&
425 	    !(ah_attr->ah_flags & IB_AH_GRH))
426 		return -EINVAL;
427 
428 	if (ah_attr->grh.sgid_attr) {
429 		/*
430 		 * Make sure the passed sgid_attr is consistent with the
431 		 * parameters
432 		 */
433 		if (ah_attr->grh.sgid_attr->index != ah_attr->grh.sgid_index ||
434 		    ah_attr->grh.sgid_attr->port_num != ah_attr->port_num)
435 			return -EINVAL;
436 	}
437 	return 0;
438 }
439 
440 /*
441  * If the ah requires a GRH then ensure that sgid_attr pointer is filled in.
442  * On success the caller is responsible to call rdma_unfill_sgid_attr().
443  */
444 static int rdma_fill_sgid_attr(struct ib_device *device,
445 			       struct rdma_ah_attr *ah_attr,
446 			       const struct ib_gid_attr **old_sgid_attr)
447 {
448 	const struct ib_gid_attr *sgid_attr;
449 	struct ib_global_route *grh;
450 	int ret;
451 
452 	*old_sgid_attr = ah_attr->grh.sgid_attr;
453 
454 	ret = rdma_check_ah_attr(device, ah_attr);
455 	if (ret)
456 		return ret;
457 
458 	if (!(ah_attr->ah_flags & IB_AH_GRH))
459 		return 0;
460 
461 	grh = rdma_ah_retrieve_grh(ah_attr);
462 	if (grh->sgid_attr)
463 		return 0;
464 
465 	sgid_attr =
466 		rdma_get_gid_attr(device, ah_attr->port_num, grh->sgid_index);
467 	if (IS_ERR(sgid_attr))
468 		return PTR_ERR(sgid_attr);
469 
470 	/* Move ownerhip of the kref into the ah_attr */
471 	grh->sgid_attr = sgid_attr;
472 	return 0;
473 }
474 
475 static void rdma_unfill_sgid_attr(struct rdma_ah_attr *ah_attr,
476 				  const struct ib_gid_attr *old_sgid_attr)
477 {
478 	/*
479 	 * Fill didn't change anything, the caller retains ownership of
480 	 * whatever it passed
481 	 */
482 	if (ah_attr->grh.sgid_attr == old_sgid_attr)
483 		return;
484 
485 	/*
486 	 * Otherwise, we need to undo what rdma_fill_sgid_attr so the caller
487 	 * doesn't see any change in the rdma_ah_attr. If we get here
488 	 * old_sgid_attr is NULL.
489 	 */
490 	rdma_destroy_ah_attr(ah_attr);
491 }
492 
493 static const struct ib_gid_attr *
494 rdma_update_sgid_attr(struct rdma_ah_attr *ah_attr,
495 		      const struct ib_gid_attr *old_attr)
496 {
497 	if (old_attr)
498 		rdma_put_gid_attr(old_attr);
499 	if (ah_attr->ah_flags & IB_AH_GRH) {
500 		rdma_hold_gid_attr(ah_attr->grh.sgid_attr);
501 		return ah_attr->grh.sgid_attr;
502 	}
503 	return NULL;
504 }
505 
506 static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
507 				     struct rdma_ah_attr *ah_attr,
508 				     u32 flags,
509 				     struct ib_udata *udata,
510 				     struct net_device *xmit_slave)
511 {
512 	struct rdma_ah_init_attr init_attr = {};
513 	struct ib_device *device = pd->device;
514 	struct ib_ah *ah;
515 	int ret;
516 
517 	might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
518 
519 	if (!udata && !device->ops.create_ah)
520 		return ERR_PTR(-EOPNOTSUPP);
521 
522 	ah = rdma_zalloc_drv_obj_gfp(
523 		device, ib_ah,
524 		(flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
525 	if (!ah)
526 		return ERR_PTR(-ENOMEM);
527 
528 	ah->device = device;
529 	ah->pd = pd;
530 	ah->type = ah_attr->type;
531 	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
532 	init_attr.ah_attr = ah_attr;
533 	init_attr.flags = flags;
534 	init_attr.xmit_slave = xmit_slave;
535 
536 	if (udata)
537 		ret = device->ops.create_user_ah(ah, &init_attr, udata);
538 	else
539 		ret = device->ops.create_ah(ah, &init_attr, NULL);
540 	if (ret) {
541 		kfree(ah);
542 		return ERR_PTR(ret);
543 	}
544 
545 	atomic_inc(&pd->usecnt);
546 	return ah;
547 }
548 
549 /**
550  * rdma_create_ah - Creates an address handle for the
551  * given address vector.
552  * @pd: The protection domain associated with the address handle.
553  * @ah_attr: The attributes of the address vector.
554  * @flags: Create address handle flags (see enum rdma_create_ah_flags).
555  *
556  * It returns 0 on success and returns appropriate error code on error.
557  * The address handle is used to reference a local or global destination
558  * in all UD QP post sends.
559  */
560 struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
561 			     u32 flags)
562 {
563 	const struct ib_gid_attr *old_sgid_attr;
564 	struct net_device *slave;
565 	struct ib_ah *ah;
566 	int ret;
567 
568 	ret = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
569 	if (ret)
570 		return ERR_PTR(ret);
571 	slave = rdma_lag_get_ah_roce_slave(pd->device, ah_attr,
572 					   (flags & RDMA_CREATE_AH_SLEEPABLE) ?
573 					   GFP_KERNEL : GFP_ATOMIC);
574 	if (IS_ERR(slave)) {
575 		rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
576 		return (void *)slave;
577 	}
578 	ah = _rdma_create_ah(pd, ah_attr, flags, NULL, slave);
579 	rdma_lag_put_ah_roce_slave(slave);
580 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
581 	return ah;
582 }
583 EXPORT_SYMBOL(rdma_create_ah);
584 
585 /**
586  * rdma_create_user_ah - Creates an address handle for the
587  * given address vector.
588  * It resolves destination mac address for ah attribute of RoCE type.
589  * @pd: The protection domain associated with the address handle.
590  * @ah_attr: The attributes of the address vector.
591  * @udata: pointer to user's input output buffer information need by
592  *         provider driver.
593  *
594  * It returns 0 on success and returns appropriate error code on error.
595  * The address handle is used to reference a local or global destination
596  * in all UD QP post sends.
597  */
598 struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
599 				  struct rdma_ah_attr *ah_attr,
600 				  struct ib_udata *udata)
601 {
602 	const struct ib_gid_attr *old_sgid_attr;
603 	struct ib_ah *ah;
604 	int err;
605 
606 	err = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
607 	if (err)
608 		return ERR_PTR(err);
609 
610 	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
611 		err = ib_resolve_eth_dmac(pd->device, ah_attr);
612 		if (err) {
613 			ah = ERR_PTR(err);
614 			goto out;
615 		}
616 	}
617 
618 	ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE,
619 			     udata, NULL);
620 
621 out:
622 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
623 	return ah;
624 }
625 EXPORT_SYMBOL(rdma_create_user_ah);
626 
627 int ib_get_rdma_header_version(const union rdma_network_hdr *hdr)
628 {
629 	const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh;
630 	struct iphdr ip4h_checked;
631 	const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh;
632 
633 	/* If it's IPv6, the version must be 6, otherwise, the first
634 	 * 20 bytes (before the IPv4 header) are garbled.
635 	 */
636 	if (ip6h->version != 6)
637 		return (ip4h->version == 4) ? 4 : 0;
638 	/* version may be 6 or 4 because the first 20 bytes could be garbled */
639 
640 	/* RoCE v2 requires no options, thus header length
641 	 * must be 5 words
642 	 */
643 	if (ip4h->ihl != 5)
644 		return 6;
645 
646 	/* Verify checksum.
647 	 * We can't write on scattered buffers so we need to copy to
648 	 * temp buffer.
649 	 */
650 	memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked));
651 	ip4h_checked.check = 0;
652 	ip4h_checked.check = ip_fast_csum((u8 *)&ip4h_checked, 5);
653 	/* if IPv4 header checksum is OK, believe it */
654 	if (ip4h->check == ip4h_checked.check)
655 		return 4;
656 	return 6;
657 }
658 EXPORT_SYMBOL(ib_get_rdma_header_version);
659 
660 static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
661 						     u8 port_num,
662 						     const struct ib_grh *grh)
663 {
664 	int grh_version;
665 
666 	if (rdma_protocol_ib(device, port_num))
667 		return RDMA_NETWORK_IB;
668 
669 	grh_version = ib_get_rdma_header_version((union rdma_network_hdr *)grh);
670 
671 	if (grh_version == 4)
672 		return RDMA_NETWORK_IPV4;
673 
674 	if (grh->next_hdr == IPPROTO_UDP)
675 		return RDMA_NETWORK_IPV6;
676 
677 	return RDMA_NETWORK_ROCE_V1;
678 }
679 
680 struct find_gid_index_context {
681 	u16 vlan_id;
682 	enum ib_gid_type gid_type;
683 };
684 
685 static bool find_gid_index(const union ib_gid *gid,
686 			   const struct ib_gid_attr *gid_attr,
687 			   void *context)
688 {
689 	struct find_gid_index_context *ctx = context;
690 	u16 vlan_id = 0xffff;
691 	int ret;
692 
693 	if (ctx->gid_type != gid_attr->gid_type)
694 		return false;
695 
696 	ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
697 	if (ret)
698 		return false;
699 
700 	return ctx->vlan_id == vlan_id;
701 }
702 
703 static const struct ib_gid_attr *
704 get_sgid_attr_from_eth(struct ib_device *device, u8 port_num,
705 		       u16 vlan_id, const union ib_gid *sgid,
706 		       enum ib_gid_type gid_type)
707 {
708 	struct find_gid_index_context context = {.vlan_id = vlan_id,
709 						 .gid_type = gid_type};
710 
711 	return rdma_find_gid_by_filter(device, sgid, port_num, find_gid_index,
712 				       &context);
713 }
714 
715 int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
716 			      enum rdma_network_type net_type,
717 			      union ib_gid *sgid, union ib_gid *dgid)
718 {
719 	struct sockaddr_in  src_in;
720 	struct sockaddr_in  dst_in;
721 	__be32 src_saddr, dst_saddr;
722 
723 	if (!sgid || !dgid)
724 		return -EINVAL;
725 
726 	if (net_type == RDMA_NETWORK_IPV4) {
727 		memcpy(&src_in.sin_addr.s_addr,
728 		       &hdr->roce4grh.saddr, 4);
729 		memcpy(&dst_in.sin_addr.s_addr,
730 		       &hdr->roce4grh.daddr, 4);
731 		src_saddr = src_in.sin_addr.s_addr;
732 		dst_saddr = dst_in.sin_addr.s_addr;
733 		ipv6_addr_set_v4mapped(src_saddr,
734 				       (struct in6_addr *)sgid);
735 		ipv6_addr_set_v4mapped(dst_saddr,
736 				       (struct in6_addr *)dgid);
737 		return 0;
738 	} else if (net_type == RDMA_NETWORK_IPV6 ||
739 		   net_type == RDMA_NETWORK_IB || RDMA_NETWORK_ROCE_V1) {
740 		*dgid = hdr->ibgrh.dgid;
741 		*sgid = hdr->ibgrh.sgid;
742 		return 0;
743 	} else {
744 		return -EINVAL;
745 	}
746 }
747 EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
748 
749 /* Resolve destination mac address and hop limit for unicast destination
750  * GID entry, considering the source GID entry as well.
751  * ah_attribute must have have valid port_num, sgid_index.
752  */
753 static int ib_resolve_unicast_gid_dmac(struct ib_device *device,
754 				       struct rdma_ah_attr *ah_attr)
755 {
756 	struct ib_global_route *grh = rdma_ah_retrieve_grh(ah_attr);
757 	const struct ib_gid_attr *sgid_attr = grh->sgid_attr;
758 	int hop_limit = 0xff;
759 	int ret = 0;
760 
761 	/* If destination is link local and source GID is RoCEv1,
762 	 * IP stack is not used.
763 	 */
764 	if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) &&
765 	    sgid_attr->gid_type == IB_GID_TYPE_ROCE) {
766 		rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
767 				ah_attr->roce.dmac);
768 		return ret;
769 	}
770 
771 	ret = rdma_addr_find_l2_eth_by_grh(&sgid_attr->gid, &grh->dgid,
772 					   ah_attr->roce.dmac,
773 					   sgid_attr, &hop_limit);
774 
775 	grh->hop_limit = hop_limit;
776 	return ret;
777 }
778 
779 /*
780  * This function initializes address handle attributes from the incoming packet.
781  * Incoming packet has dgid of the receiver node on which this code is
782  * getting executed and, sgid contains the GID of the sender.
783  *
784  * When resolving mac address of destination, the arrived dgid is used
785  * as sgid and, sgid is used as dgid because sgid contains destinations
786  * GID whom to respond to.
787  *
788  * On success the caller is responsible to call rdma_destroy_ah_attr on the
789  * attr.
790  */
791 int ib_init_ah_attr_from_wc(struct ib_device *device, u8 port_num,
792 			    const struct ib_wc *wc, const struct ib_grh *grh,
793 			    struct rdma_ah_attr *ah_attr)
794 {
795 	u32 flow_class;
796 	int ret;
797 	enum rdma_network_type net_type = RDMA_NETWORK_IB;
798 	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
799 	const struct ib_gid_attr *sgid_attr;
800 	int hoplimit = 0xff;
801 	union ib_gid dgid;
802 	union ib_gid sgid;
803 
804 	might_sleep();
805 
806 	memset(ah_attr, 0, sizeof *ah_attr);
807 	ah_attr->type = rdma_ah_find_type(device, port_num);
808 	if (rdma_cap_eth_ah(device, port_num)) {
809 		if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE)
810 			net_type = wc->network_hdr_type;
811 		else
812 			net_type = ib_get_net_type_by_grh(device, port_num, grh);
813 		gid_type = ib_network_to_gid_type(net_type);
814 	}
815 	ret = ib_get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
816 					&sgid, &dgid);
817 	if (ret)
818 		return ret;
819 
820 	rdma_ah_set_sl(ah_attr, wc->sl);
821 	rdma_ah_set_port_num(ah_attr, port_num);
822 
823 	if (rdma_protocol_roce(device, port_num)) {
824 		u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
825 				wc->vlan_id : 0xffff;
826 
827 		if (!(wc->wc_flags & IB_WC_GRH))
828 			return -EPROTOTYPE;
829 
830 		sgid_attr = get_sgid_attr_from_eth(device, port_num,
831 						   vlan_id, &dgid,
832 						   gid_type);
833 		if (IS_ERR(sgid_attr))
834 			return PTR_ERR(sgid_attr);
835 
836 		flow_class = be32_to_cpu(grh->version_tclass_flow);
837 		rdma_move_grh_sgid_attr(ah_attr,
838 					&sgid,
839 					flow_class & 0xFFFFF,
840 					hoplimit,
841 					(flow_class >> 20) & 0xFF,
842 					sgid_attr);
843 
844 		ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
845 		if (ret)
846 			rdma_destroy_ah_attr(ah_attr);
847 
848 		return ret;
849 	} else {
850 		rdma_ah_set_dlid(ah_attr, wc->slid);
851 		rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits);
852 
853 		if ((wc->wc_flags & IB_WC_GRH) == 0)
854 			return 0;
855 
856 		if (dgid.global.interface_id !=
857 					cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) {
858 			sgid_attr = rdma_find_gid_by_port(
859 				device, &dgid, IB_GID_TYPE_IB, port_num, NULL);
860 		} else
861 			sgid_attr = rdma_get_gid_attr(device, port_num, 0);
862 
863 		if (IS_ERR(sgid_attr))
864 			return PTR_ERR(sgid_attr);
865 		flow_class = be32_to_cpu(grh->version_tclass_flow);
866 		rdma_move_grh_sgid_attr(ah_attr,
867 					&sgid,
868 					flow_class & 0xFFFFF,
869 					hoplimit,
870 					(flow_class >> 20) & 0xFF,
871 					sgid_attr);
872 
873 		return 0;
874 	}
875 }
876 EXPORT_SYMBOL(ib_init_ah_attr_from_wc);
877 
878 /**
879  * rdma_move_grh_sgid_attr - Sets the sgid attribute of GRH, taking ownership
880  * of the reference
881  *
882  * @attr:	Pointer to AH attribute structure
883  * @dgid:	Destination GID
884  * @flow_label:	Flow label
885  * @hop_limit:	Hop limit
886  * @traffic_class: traffic class
887  * @sgid_attr:	Pointer to SGID attribute
888  *
889  * This takes ownership of the sgid_attr reference. The caller must ensure
890  * rdma_destroy_ah_attr() is called before destroying the rdma_ah_attr after
891  * calling this function.
892  */
893 void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
894 			     u32 flow_label, u8 hop_limit, u8 traffic_class,
895 			     const struct ib_gid_attr *sgid_attr)
896 {
897 	rdma_ah_set_grh(attr, dgid, flow_label, sgid_attr->index, hop_limit,
898 			traffic_class);
899 	attr->grh.sgid_attr = sgid_attr;
900 }
901 EXPORT_SYMBOL(rdma_move_grh_sgid_attr);
902 
903 /**
904  * rdma_destroy_ah_attr - Release reference to SGID attribute of
905  * ah attribute.
906  * @ah_attr: Pointer to ah attribute
907  *
908  * Release reference to the SGID attribute of the ah attribute if it is
909  * non NULL. It is safe to call this multiple times, and safe to call it on
910  * a zero initialized ah_attr.
911  */
912 void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr)
913 {
914 	if (ah_attr->grh.sgid_attr) {
915 		rdma_put_gid_attr(ah_attr->grh.sgid_attr);
916 		ah_attr->grh.sgid_attr = NULL;
917 	}
918 }
919 EXPORT_SYMBOL(rdma_destroy_ah_attr);
920 
921 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
922 				   const struct ib_grh *grh, u8 port_num)
923 {
924 	struct rdma_ah_attr ah_attr;
925 	struct ib_ah *ah;
926 	int ret;
927 
928 	ret = ib_init_ah_attr_from_wc(pd->device, port_num, wc, grh, &ah_attr);
929 	if (ret)
930 		return ERR_PTR(ret);
931 
932 	ah = rdma_create_ah(pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
933 
934 	rdma_destroy_ah_attr(&ah_attr);
935 	return ah;
936 }
937 EXPORT_SYMBOL(ib_create_ah_from_wc);
938 
939 int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
940 {
941 	const struct ib_gid_attr *old_sgid_attr;
942 	int ret;
943 
944 	if (ah->type != ah_attr->type)
945 		return -EINVAL;
946 
947 	ret = rdma_fill_sgid_attr(ah->device, ah_attr, &old_sgid_attr);
948 	if (ret)
949 		return ret;
950 
951 	ret = ah->device->ops.modify_ah ?
952 		ah->device->ops.modify_ah(ah, ah_attr) :
953 		-EOPNOTSUPP;
954 
955 	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr);
956 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
957 	return ret;
958 }
959 EXPORT_SYMBOL(rdma_modify_ah);
960 
961 int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
962 {
963 	ah_attr->grh.sgid_attr = NULL;
964 
965 	return ah->device->ops.query_ah ?
966 		ah->device->ops.query_ah(ah, ah_attr) :
967 		-EOPNOTSUPP;
968 }
969 EXPORT_SYMBOL(rdma_query_ah);
970 
971 int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)
972 {
973 	const struct ib_gid_attr *sgid_attr = ah->sgid_attr;
974 	struct ib_pd *pd;
975 	int ret;
976 
977 	might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
978 
979 	pd = ah->pd;
980 
981 	ret = ah->device->ops.destroy_ah(ah, flags);
982 	if (ret)
983 		return ret;
984 
985 	atomic_dec(&pd->usecnt);
986 	if (sgid_attr)
987 		rdma_put_gid_attr(sgid_attr);
988 
989 	kfree(ah);
990 	return ret;
991 }
992 EXPORT_SYMBOL(rdma_destroy_ah_user);
993 
994 /* Shared receive queues */
995 
996 /**
997  * ib_create_srq_user - Creates a SRQ associated with the specified protection
998  *   domain.
999  * @pd: The protection domain associated with the SRQ.
1000  * @srq_init_attr: A list of initial attributes required to create the
1001  *   SRQ.  If SRQ creation succeeds, then the attributes are updated to
1002  *   the actual capabilities of the created SRQ.
1003  * @uobject: uobject pointer if this is not a kernel SRQ
1004  * @udata: udata pointer if this is not a kernel SRQ
1005  *
1006  * srq_attr->max_wr and srq_attr->max_sge are read the determine the
1007  * requested size of the SRQ, and set to the actual values allocated
1008  * on return.  If ib_create_srq() succeeds, then max_wr and max_sge
1009  * will always be at least as large as the requested values.
1010  */
1011 struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
1012 				  struct ib_srq_init_attr *srq_init_attr,
1013 				  struct ib_usrq_object *uobject,
1014 				  struct ib_udata *udata)
1015 {
1016 	struct ib_srq *srq;
1017 	int ret;
1018 
1019 	srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
1020 	if (!srq)
1021 		return ERR_PTR(-ENOMEM);
1022 
1023 	srq->device = pd->device;
1024 	srq->pd = pd;
1025 	srq->event_handler = srq_init_attr->event_handler;
1026 	srq->srq_context = srq_init_attr->srq_context;
1027 	srq->srq_type = srq_init_attr->srq_type;
1028 	srq->uobject = uobject;
1029 
1030 	if (ib_srq_has_cq(srq->srq_type)) {
1031 		srq->ext.cq = srq_init_attr->ext.cq;
1032 		atomic_inc(&srq->ext.cq->usecnt);
1033 	}
1034 	if (srq->srq_type == IB_SRQT_XRC) {
1035 		srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
1036 		atomic_inc(&srq->ext.xrc.xrcd->usecnt);
1037 	}
1038 	atomic_inc(&pd->usecnt);
1039 
1040 	ret = pd->device->ops.create_srq(srq, srq_init_attr, udata);
1041 	if (ret) {
1042 		atomic_dec(&srq->pd->usecnt);
1043 		if (srq->srq_type == IB_SRQT_XRC)
1044 			atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1045 		if (ib_srq_has_cq(srq->srq_type))
1046 			atomic_dec(&srq->ext.cq->usecnt);
1047 		kfree(srq);
1048 		return ERR_PTR(ret);
1049 	}
1050 
1051 	return srq;
1052 }
1053 EXPORT_SYMBOL(ib_create_srq_user);
1054 
1055 int ib_modify_srq(struct ib_srq *srq,
1056 		  struct ib_srq_attr *srq_attr,
1057 		  enum ib_srq_attr_mask srq_attr_mask)
1058 {
1059 	return srq->device->ops.modify_srq ?
1060 		srq->device->ops.modify_srq(srq, srq_attr, srq_attr_mask,
1061 					    NULL) : -EOPNOTSUPP;
1062 }
1063 EXPORT_SYMBOL(ib_modify_srq);
1064 
1065 int ib_query_srq(struct ib_srq *srq,
1066 		 struct ib_srq_attr *srq_attr)
1067 {
1068 	return srq->device->ops.query_srq ?
1069 		srq->device->ops.query_srq(srq, srq_attr) : -EOPNOTSUPP;
1070 }
1071 EXPORT_SYMBOL(ib_query_srq);
1072 
1073 int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)
1074 {
1075 	int ret;
1076 
1077 	if (atomic_read(&srq->usecnt))
1078 		return -EBUSY;
1079 
1080 	ret = srq->device->ops.destroy_srq(srq, udata);
1081 	if (ret)
1082 		return ret;
1083 
1084 	atomic_dec(&srq->pd->usecnt);
1085 	if (srq->srq_type == IB_SRQT_XRC)
1086 		atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1087 	if (ib_srq_has_cq(srq->srq_type))
1088 		atomic_dec(&srq->ext.cq->usecnt);
1089 	kfree(srq);
1090 
1091 	return ret;
1092 }
1093 EXPORT_SYMBOL(ib_destroy_srq_user);
1094 
1095 /* Queue pairs */
1096 
1097 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
1098 {
1099 	struct ib_qp *qp = context;
1100 	unsigned long flags;
1101 
1102 	spin_lock_irqsave(&qp->device->qp_open_list_lock, flags);
1103 	list_for_each_entry(event->element.qp, &qp->open_list, open_list)
1104 		if (event->element.qp->event_handler)
1105 			event->element.qp->event_handler(event, event->element.qp->qp_context);
1106 	spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags);
1107 }
1108 
1109 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
1110 				  void (*event_handler)(struct ib_event *, void *),
1111 				  void *qp_context)
1112 {
1113 	struct ib_qp *qp;
1114 	unsigned long flags;
1115 	int err;
1116 
1117 	qp = kzalloc(sizeof *qp, GFP_KERNEL);
1118 	if (!qp)
1119 		return ERR_PTR(-ENOMEM);
1120 
1121 	qp->real_qp = real_qp;
1122 	err = ib_open_shared_qp_security(qp, real_qp->device);
1123 	if (err) {
1124 		kfree(qp);
1125 		return ERR_PTR(err);
1126 	}
1127 
1128 	qp->real_qp = real_qp;
1129 	atomic_inc(&real_qp->usecnt);
1130 	qp->device = real_qp->device;
1131 	qp->event_handler = event_handler;
1132 	qp->qp_context = qp_context;
1133 	qp->qp_num = real_qp->qp_num;
1134 	qp->qp_type = real_qp->qp_type;
1135 
1136 	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1137 	list_add(&qp->open_list, &real_qp->open_list);
1138 	spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
1139 
1140 	return qp;
1141 }
1142 
1143 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
1144 			 struct ib_qp_open_attr *qp_open_attr)
1145 {
1146 	struct ib_qp *qp, *real_qp;
1147 
1148 	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
1149 		return ERR_PTR(-EINVAL);
1150 
1151 	down_read(&xrcd->tgt_qps_rwsem);
1152 	real_qp = xa_load(&xrcd->tgt_qps, qp_open_attr->qp_num);
1153 	if (!real_qp) {
1154 		up_read(&xrcd->tgt_qps_rwsem);
1155 		return ERR_PTR(-EINVAL);
1156 	}
1157 	qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
1158 			  qp_open_attr->qp_context);
1159 	up_read(&xrcd->tgt_qps_rwsem);
1160 	return qp;
1161 }
1162 EXPORT_SYMBOL(ib_open_qp);
1163 
1164 static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
1165 					struct ib_qp_init_attr *qp_init_attr)
1166 {
1167 	struct ib_qp *real_qp = qp;
1168 	int err;
1169 
1170 	qp->event_handler = __ib_shared_qp_event_handler;
1171 	qp->qp_context = qp;
1172 	qp->pd = NULL;
1173 	qp->send_cq = qp->recv_cq = NULL;
1174 	qp->srq = NULL;
1175 	qp->xrcd = qp_init_attr->xrcd;
1176 	atomic_inc(&qp_init_attr->xrcd->usecnt);
1177 	INIT_LIST_HEAD(&qp->open_list);
1178 
1179 	qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
1180 			  qp_init_attr->qp_context);
1181 	if (IS_ERR(qp))
1182 		return qp;
1183 
1184 	err = xa_err(xa_store(&qp_init_attr->xrcd->tgt_qps, real_qp->qp_num,
1185 			      real_qp, GFP_KERNEL));
1186 	if (err) {
1187 		ib_close_qp(qp);
1188 		return ERR_PTR(err);
1189 	}
1190 	return qp;
1191 }
1192 
1193 /**
1194  * ib_create_named_qp - Creates a kernel QP associated with the specified protection
1195  *   domain.
1196  * @pd: The protection domain associated with the QP.
1197  * @qp_init_attr: A list of initial attributes required to create the
1198  *   QP.  If QP creation succeeds, then the attributes are updated to
1199  *   the actual capabilities of the created QP.
1200  * @caller: caller's build-time module name
1201  *
1202  * NOTE: for user qp use ib_create_qp_user with valid udata!
1203  */
1204 struct ib_qp *ib_create_named_qp(struct ib_pd *pd,
1205 				 struct ib_qp_init_attr *qp_init_attr,
1206 				 const char *caller)
1207 {
1208 	struct ib_device *device = pd ? pd->device : qp_init_attr->xrcd->device;
1209 	struct ib_qp *qp;
1210 	int ret;
1211 
1212 	if (qp_init_attr->rwq_ind_tbl &&
1213 	    (qp_init_attr->recv_cq ||
1214 	    qp_init_attr->srq || qp_init_attr->cap.max_recv_wr ||
1215 	    qp_init_attr->cap.max_recv_sge))
1216 		return ERR_PTR(-EINVAL);
1217 
1218 	if ((qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN) &&
1219 	    !(device->attrs.device_cap_flags & IB_DEVICE_INTEGRITY_HANDOVER))
1220 		return ERR_PTR(-EINVAL);
1221 
1222 	/*
1223 	 * If the callers is using the RDMA API calculate the resources
1224 	 * needed for the RDMA READ/WRITE operations.
1225 	 *
1226 	 * Note that these callers need to pass in a port number.
1227 	 */
1228 	if (qp_init_attr->cap.max_rdma_ctxs)
1229 		rdma_rw_init_qp(device, qp_init_attr);
1230 
1231 	qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL, caller);
1232 	if (IS_ERR(qp))
1233 		return qp;
1234 
1235 	ret = ib_create_qp_security(qp, device);
1236 	if (ret)
1237 		goto err;
1238 
1239 	if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
1240 		struct ib_qp *xrc_qp =
1241 			create_xrc_qp_user(qp, qp_init_attr);
1242 
1243 		if (IS_ERR(xrc_qp)) {
1244 			ret = PTR_ERR(xrc_qp);
1245 			goto err;
1246 		}
1247 		return xrc_qp;
1248 	}
1249 
1250 	qp->event_handler = qp_init_attr->event_handler;
1251 	qp->qp_context = qp_init_attr->qp_context;
1252 	if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
1253 		qp->recv_cq = NULL;
1254 		qp->srq = NULL;
1255 	} else {
1256 		qp->recv_cq = qp_init_attr->recv_cq;
1257 		if (qp_init_attr->recv_cq)
1258 			atomic_inc(&qp_init_attr->recv_cq->usecnt);
1259 		qp->srq = qp_init_attr->srq;
1260 		if (qp->srq)
1261 			atomic_inc(&qp_init_attr->srq->usecnt);
1262 	}
1263 
1264 	qp->send_cq = qp_init_attr->send_cq;
1265 	qp->xrcd    = NULL;
1266 
1267 	atomic_inc(&pd->usecnt);
1268 	if (qp_init_attr->send_cq)
1269 		atomic_inc(&qp_init_attr->send_cq->usecnt);
1270 	if (qp_init_attr->rwq_ind_tbl)
1271 		atomic_inc(&qp->rwq_ind_tbl->usecnt);
1272 
1273 	if (qp_init_attr->cap.max_rdma_ctxs) {
1274 		ret = rdma_rw_init_mrs(qp, qp_init_attr);
1275 		if (ret)
1276 			goto err;
1277 	}
1278 
1279 	/*
1280 	 * Note: all hw drivers guarantee that max_send_sge is lower than
1281 	 * the device RDMA WRITE SGE limit but not all hw drivers ensure that
1282 	 * max_send_sge <= max_sge_rd.
1283 	 */
1284 	qp->max_write_sge = qp_init_attr->cap.max_send_sge;
1285 	qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
1286 				 device->attrs.max_sge_rd);
1287 	if (qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN)
1288 		qp->integrity_en = true;
1289 
1290 	return qp;
1291 
1292 err:
1293 	ib_destroy_qp(qp);
1294 	return ERR_PTR(ret);
1295 
1296 }
1297 EXPORT_SYMBOL(ib_create_named_qp);
1298 
1299 static const struct {
1300 	int			valid;
1301 	enum ib_qp_attr_mask	req_param[IB_QPT_MAX];
1302 	enum ib_qp_attr_mask	opt_param[IB_QPT_MAX];
1303 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
1304 	[IB_QPS_RESET] = {
1305 		[IB_QPS_RESET] = { .valid = 1 },
1306 		[IB_QPS_INIT]  = {
1307 			.valid = 1,
1308 			.req_param = {
1309 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1310 						IB_QP_PORT			|
1311 						IB_QP_QKEY),
1312 				[IB_QPT_RAW_PACKET] = IB_QP_PORT,
1313 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
1314 						IB_QP_PORT			|
1315 						IB_QP_ACCESS_FLAGS),
1316 				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
1317 						IB_QP_PORT			|
1318 						IB_QP_ACCESS_FLAGS),
1319 				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
1320 						IB_QP_PORT			|
1321 						IB_QP_ACCESS_FLAGS),
1322 				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
1323 						IB_QP_PORT			|
1324 						IB_QP_ACCESS_FLAGS),
1325 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1326 						IB_QP_QKEY),
1327 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1328 						IB_QP_QKEY),
1329 			}
1330 		},
1331 	},
1332 	[IB_QPS_INIT]  = {
1333 		[IB_QPS_RESET] = { .valid = 1 },
1334 		[IB_QPS_ERR] =   { .valid = 1 },
1335 		[IB_QPS_INIT]  = {
1336 			.valid = 1,
1337 			.opt_param = {
1338 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1339 						IB_QP_PORT			|
1340 						IB_QP_QKEY),
1341 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
1342 						IB_QP_PORT			|
1343 						IB_QP_ACCESS_FLAGS),
1344 				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
1345 						IB_QP_PORT			|
1346 						IB_QP_ACCESS_FLAGS),
1347 				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
1348 						IB_QP_PORT			|
1349 						IB_QP_ACCESS_FLAGS),
1350 				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
1351 						IB_QP_PORT			|
1352 						IB_QP_ACCESS_FLAGS),
1353 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1354 						IB_QP_QKEY),
1355 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1356 						IB_QP_QKEY),
1357 			}
1358 		},
1359 		[IB_QPS_RTR]   = {
1360 			.valid = 1,
1361 			.req_param = {
1362 				[IB_QPT_UC]  = (IB_QP_AV			|
1363 						IB_QP_PATH_MTU			|
1364 						IB_QP_DEST_QPN			|
1365 						IB_QP_RQ_PSN),
1366 				[IB_QPT_RC]  = (IB_QP_AV			|
1367 						IB_QP_PATH_MTU			|
1368 						IB_QP_DEST_QPN			|
1369 						IB_QP_RQ_PSN			|
1370 						IB_QP_MAX_DEST_RD_ATOMIC	|
1371 						IB_QP_MIN_RNR_TIMER),
1372 				[IB_QPT_XRC_INI] = (IB_QP_AV			|
1373 						IB_QP_PATH_MTU			|
1374 						IB_QP_DEST_QPN			|
1375 						IB_QP_RQ_PSN),
1376 				[IB_QPT_XRC_TGT] = (IB_QP_AV			|
1377 						IB_QP_PATH_MTU			|
1378 						IB_QP_DEST_QPN			|
1379 						IB_QP_RQ_PSN			|
1380 						IB_QP_MAX_DEST_RD_ATOMIC	|
1381 						IB_QP_MIN_RNR_TIMER),
1382 			},
1383 			.opt_param = {
1384 				 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1385 						 IB_QP_QKEY),
1386 				 [IB_QPT_UC]  = (IB_QP_ALT_PATH			|
1387 						 IB_QP_ACCESS_FLAGS		|
1388 						 IB_QP_PKEY_INDEX),
1389 				 [IB_QPT_RC]  = (IB_QP_ALT_PATH			|
1390 						 IB_QP_ACCESS_FLAGS		|
1391 						 IB_QP_PKEY_INDEX),
1392 				 [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH		|
1393 						 IB_QP_ACCESS_FLAGS		|
1394 						 IB_QP_PKEY_INDEX),
1395 				 [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH		|
1396 						 IB_QP_ACCESS_FLAGS		|
1397 						 IB_QP_PKEY_INDEX),
1398 				 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1399 						 IB_QP_QKEY),
1400 				 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1401 						 IB_QP_QKEY),
1402 			 },
1403 		},
1404 	},
1405 	[IB_QPS_RTR]   = {
1406 		[IB_QPS_RESET] = { .valid = 1 },
1407 		[IB_QPS_ERR] =   { .valid = 1 },
1408 		[IB_QPS_RTS]   = {
1409 			.valid = 1,
1410 			.req_param = {
1411 				[IB_QPT_UD]  = IB_QP_SQ_PSN,
1412 				[IB_QPT_UC]  = IB_QP_SQ_PSN,
1413 				[IB_QPT_RC]  = (IB_QP_TIMEOUT			|
1414 						IB_QP_RETRY_CNT			|
1415 						IB_QP_RNR_RETRY			|
1416 						IB_QP_SQ_PSN			|
1417 						IB_QP_MAX_QP_RD_ATOMIC),
1418 				[IB_QPT_XRC_INI] = (IB_QP_TIMEOUT		|
1419 						IB_QP_RETRY_CNT			|
1420 						IB_QP_RNR_RETRY			|
1421 						IB_QP_SQ_PSN			|
1422 						IB_QP_MAX_QP_RD_ATOMIC),
1423 				[IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT		|
1424 						IB_QP_SQ_PSN),
1425 				[IB_QPT_SMI] = IB_QP_SQ_PSN,
1426 				[IB_QPT_GSI] = IB_QP_SQ_PSN,
1427 			},
1428 			.opt_param = {
1429 				 [IB_QPT_UD]  = (IB_QP_CUR_STATE		|
1430 						 IB_QP_QKEY),
1431 				 [IB_QPT_UC]  = (IB_QP_CUR_STATE		|
1432 						 IB_QP_ALT_PATH			|
1433 						 IB_QP_ACCESS_FLAGS		|
1434 						 IB_QP_PATH_MIG_STATE),
1435 				 [IB_QPT_RC]  = (IB_QP_CUR_STATE		|
1436 						 IB_QP_ALT_PATH			|
1437 						 IB_QP_ACCESS_FLAGS		|
1438 						 IB_QP_MIN_RNR_TIMER		|
1439 						 IB_QP_PATH_MIG_STATE),
1440 				 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1441 						 IB_QP_ALT_PATH			|
1442 						 IB_QP_ACCESS_FLAGS		|
1443 						 IB_QP_PATH_MIG_STATE),
1444 				 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1445 						 IB_QP_ALT_PATH			|
1446 						 IB_QP_ACCESS_FLAGS		|
1447 						 IB_QP_MIN_RNR_TIMER		|
1448 						 IB_QP_PATH_MIG_STATE),
1449 				 [IB_QPT_SMI] = (IB_QP_CUR_STATE		|
1450 						 IB_QP_QKEY),
1451 				 [IB_QPT_GSI] = (IB_QP_CUR_STATE		|
1452 						 IB_QP_QKEY),
1453 				 [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1454 			 }
1455 		}
1456 	},
1457 	[IB_QPS_RTS]   = {
1458 		[IB_QPS_RESET] = { .valid = 1 },
1459 		[IB_QPS_ERR] =   { .valid = 1 },
1460 		[IB_QPS_RTS]   = {
1461 			.valid = 1,
1462 			.opt_param = {
1463 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1464 						IB_QP_QKEY),
1465 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1466 						IB_QP_ACCESS_FLAGS		|
1467 						IB_QP_ALT_PATH			|
1468 						IB_QP_PATH_MIG_STATE),
1469 				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
1470 						IB_QP_ACCESS_FLAGS		|
1471 						IB_QP_ALT_PATH			|
1472 						IB_QP_PATH_MIG_STATE		|
1473 						IB_QP_MIN_RNR_TIMER),
1474 				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1475 						IB_QP_ACCESS_FLAGS		|
1476 						IB_QP_ALT_PATH			|
1477 						IB_QP_PATH_MIG_STATE),
1478 				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1479 						IB_QP_ACCESS_FLAGS		|
1480 						IB_QP_ALT_PATH			|
1481 						IB_QP_PATH_MIG_STATE		|
1482 						IB_QP_MIN_RNR_TIMER),
1483 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1484 						IB_QP_QKEY),
1485 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1486 						IB_QP_QKEY),
1487 				[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1488 			}
1489 		},
1490 		[IB_QPS_SQD]   = {
1491 			.valid = 1,
1492 			.opt_param = {
1493 				[IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1494 				[IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1495 				[IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1496 				[IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1497 				[IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
1498 				[IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1499 				[IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
1500 			}
1501 		},
1502 	},
1503 	[IB_QPS_SQD]   = {
1504 		[IB_QPS_RESET] = { .valid = 1 },
1505 		[IB_QPS_ERR] =   { .valid = 1 },
1506 		[IB_QPS_RTS]   = {
1507 			.valid = 1,
1508 			.opt_param = {
1509 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1510 						IB_QP_QKEY),
1511 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1512 						IB_QP_ALT_PATH			|
1513 						IB_QP_ACCESS_FLAGS		|
1514 						IB_QP_PATH_MIG_STATE),
1515 				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
1516 						IB_QP_ALT_PATH			|
1517 						IB_QP_ACCESS_FLAGS		|
1518 						IB_QP_MIN_RNR_TIMER		|
1519 						IB_QP_PATH_MIG_STATE),
1520 				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1521 						IB_QP_ALT_PATH			|
1522 						IB_QP_ACCESS_FLAGS		|
1523 						IB_QP_PATH_MIG_STATE),
1524 				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1525 						IB_QP_ALT_PATH			|
1526 						IB_QP_ACCESS_FLAGS		|
1527 						IB_QP_MIN_RNR_TIMER		|
1528 						IB_QP_PATH_MIG_STATE),
1529 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1530 						IB_QP_QKEY),
1531 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1532 						IB_QP_QKEY),
1533 			}
1534 		},
1535 		[IB_QPS_SQD]   = {
1536 			.valid = 1,
1537 			.opt_param = {
1538 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1539 						IB_QP_QKEY),
1540 				[IB_QPT_UC]  = (IB_QP_AV			|
1541 						IB_QP_ALT_PATH			|
1542 						IB_QP_ACCESS_FLAGS		|
1543 						IB_QP_PKEY_INDEX		|
1544 						IB_QP_PATH_MIG_STATE),
1545 				[IB_QPT_RC]  = (IB_QP_PORT			|
1546 						IB_QP_AV			|
1547 						IB_QP_TIMEOUT			|
1548 						IB_QP_RETRY_CNT			|
1549 						IB_QP_RNR_RETRY			|
1550 						IB_QP_MAX_QP_RD_ATOMIC		|
1551 						IB_QP_MAX_DEST_RD_ATOMIC	|
1552 						IB_QP_ALT_PATH			|
1553 						IB_QP_ACCESS_FLAGS		|
1554 						IB_QP_PKEY_INDEX		|
1555 						IB_QP_MIN_RNR_TIMER		|
1556 						IB_QP_PATH_MIG_STATE),
1557 				[IB_QPT_XRC_INI] = (IB_QP_PORT			|
1558 						IB_QP_AV			|
1559 						IB_QP_TIMEOUT			|
1560 						IB_QP_RETRY_CNT			|
1561 						IB_QP_RNR_RETRY			|
1562 						IB_QP_MAX_QP_RD_ATOMIC		|
1563 						IB_QP_ALT_PATH			|
1564 						IB_QP_ACCESS_FLAGS		|
1565 						IB_QP_PKEY_INDEX		|
1566 						IB_QP_PATH_MIG_STATE),
1567 				[IB_QPT_XRC_TGT] = (IB_QP_PORT			|
1568 						IB_QP_AV			|
1569 						IB_QP_TIMEOUT			|
1570 						IB_QP_MAX_DEST_RD_ATOMIC	|
1571 						IB_QP_ALT_PATH			|
1572 						IB_QP_ACCESS_FLAGS		|
1573 						IB_QP_PKEY_INDEX		|
1574 						IB_QP_MIN_RNR_TIMER		|
1575 						IB_QP_PATH_MIG_STATE),
1576 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1577 						IB_QP_QKEY),
1578 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1579 						IB_QP_QKEY),
1580 			}
1581 		}
1582 	},
1583 	[IB_QPS_SQE]   = {
1584 		[IB_QPS_RESET] = { .valid = 1 },
1585 		[IB_QPS_ERR] =   { .valid = 1 },
1586 		[IB_QPS_RTS]   = {
1587 			.valid = 1,
1588 			.opt_param = {
1589 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1590 						IB_QP_QKEY),
1591 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1592 						IB_QP_ACCESS_FLAGS),
1593 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1594 						IB_QP_QKEY),
1595 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1596 						IB_QP_QKEY),
1597 			}
1598 		}
1599 	},
1600 	[IB_QPS_ERR] = {
1601 		[IB_QPS_RESET] = { .valid = 1 },
1602 		[IB_QPS_ERR] =   { .valid = 1 }
1603 	}
1604 };
1605 
1606 bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
1607 			enum ib_qp_type type, enum ib_qp_attr_mask mask)
1608 {
1609 	enum ib_qp_attr_mask req_param, opt_param;
1610 
1611 	if (mask & IB_QP_CUR_STATE  &&
1612 	    cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
1613 	    cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
1614 		return false;
1615 
1616 	if (!qp_state_table[cur_state][next_state].valid)
1617 		return false;
1618 
1619 	req_param = qp_state_table[cur_state][next_state].req_param[type];
1620 	opt_param = qp_state_table[cur_state][next_state].opt_param[type];
1621 
1622 	if ((mask & req_param) != req_param)
1623 		return false;
1624 
1625 	if (mask & ~(req_param | opt_param | IB_QP_STATE))
1626 		return false;
1627 
1628 	return true;
1629 }
1630 EXPORT_SYMBOL(ib_modify_qp_is_ok);
1631 
1632 /**
1633  * ib_resolve_eth_dmac - Resolve destination mac address
1634  * @device:		Device to consider
1635  * @ah_attr:		address handle attribute which describes the
1636  *			source and destination parameters
1637  * ib_resolve_eth_dmac() resolves destination mac address and L3 hop limit It
1638  * returns 0 on success or appropriate error code. It initializes the
1639  * necessary ah_attr fields when call is successful.
1640  */
1641 static int ib_resolve_eth_dmac(struct ib_device *device,
1642 			       struct rdma_ah_attr *ah_attr)
1643 {
1644 	int ret = 0;
1645 
1646 	if (rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1647 		if (ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1648 			__be32 addr = 0;
1649 
1650 			memcpy(&addr, ah_attr->grh.dgid.raw + 12, 4);
1651 			ip_eth_mc_map(addr, (char *)ah_attr->roce.dmac);
1652 		} else {
1653 			ipv6_eth_mc_map((struct in6_addr *)ah_attr->grh.dgid.raw,
1654 					(char *)ah_attr->roce.dmac);
1655 		}
1656 	} else {
1657 		ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
1658 	}
1659 	return ret;
1660 }
1661 
1662 static bool is_qp_type_connected(const struct ib_qp *qp)
1663 {
1664 	return (qp->qp_type == IB_QPT_UC ||
1665 		qp->qp_type == IB_QPT_RC ||
1666 		qp->qp_type == IB_QPT_XRC_INI ||
1667 		qp->qp_type == IB_QPT_XRC_TGT);
1668 }
1669 
1670 /*
1671  * IB core internal function to perform QP attributes modification.
1672  */
1673 static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1674 			 int attr_mask, struct ib_udata *udata)
1675 {
1676 	u8 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1677 	const struct ib_gid_attr *old_sgid_attr_av;
1678 	const struct ib_gid_attr *old_sgid_attr_alt_av;
1679 	int ret;
1680 
1681 	attr->xmit_slave = NULL;
1682 	if (attr_mask & IB_QP_AV) {
1683 		ret = rdma_fill_sgid_attr(qp->device, &attr->ah_attr,
1684 					  &old_sgid_attr_av);
1685 		if (ret)
1686 			return ret;
1687 
1688 		if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
1689 		    is_qp_type_connected(qp)) {
1690 			struct net_device *slave;
1691 
1692 			/*
1693 			 * If the user provided the qp_attr then we have to
1694 			 * resolve it. Kerne users have to provide already
1695 			 * resolved rdma_ah_attr's.
1696 			 */
1697 			if (udata) {
1698 				ret = ib_resolve_eth_dmac(qp->device,
1699 							  &attr->ah_attr);
1700 				if (ret)
1701 					goto out_av;
1702 			}
1703 			slave = rdma_lag_get_ah_roce_slave(qp->device,
1704 							   &attr->ah_attr,
1705 							   GFP_KERNEL);
1706 			if (IS_ERR(slave)) {
1707 				ret = PTR_ERR(slave);
1708 				goto out_av;
1709 			}
1710 			attr->xmit_slave = slave;
1711 		}
1712 	}
1713 	if (attr_mask & IB_QP_ALT_PATH) {
1714 		/*
1715 		 * FIXME: This does not track the migration state, so if the
1716 		 * user loads a new alternate path after the HW has migrated
1717 		 * from primary->alternate we will keep the wrong
1718 		 * references. This is OK for IB because the reference
1719 		 * counting does not serve any functional purpose.
1720 		 */
1721 		ret = rdma_fill_sgid_attr(qp->device, &attr->alt_ah_attr,
1722 					  &old_sgid_attr_alt_av);
1723 		if (ret)
1724 			goto out_av;
1725 
1726 		/*
1727 		 * Today the core code can only handle alternate paths and APM
1728 		 * for IB. Ban them in roce mode.
1729 		 */
1730 		if (!(rdma_protocol_ib(qp->device,
1731 				       attr->alt_ah_attr.port_num) &&
1732 		      rdma_protocol_ib(qp->device, port))) {
1733 			ret = -EINVAL;
1734 			goto out;
1735 		}
1736 	}
1737 
1738 	if (rdma_ib_or_roce(qp->device, port)) {
1739 		if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
1740 			dev_warn(&qp->device->dev,
1741 				 "%s rq_psn overflow, masking to 24 bits\n",
1742 				 __func__);
1743 			attr->rq_psn &= 0xffffff;
1744 		}
1745 
1746 		if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
1747 			dev_warn(&qp->device->dev,
1748 				 " %s sq_psn overflow, masking to 24 bits\n",
1749 				 __func__);
1750 			attr->sq_psn &= 0xffffff;
1751 		}
1752 	}
1753 
1754 	/*
1755 	 * Bind this qp to a counter automatically based on the rdma counter
1756 	 * rules. This only set in RST2INIT with port specified
1757 	 */
1758 	if (!qp->counter && (attr_mask & IB_QP_PORT) &&
1759 	    ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_INIT))
1760 		rdma_counter_bind_qp_auto(qp, attr->port_num);
1761 
1762 	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
1763 	if (ret)
1764 		goto out;
1765 
1766 	if (attr_mask & IB_QP_PORT)
1767 		qp->port = attr->port_num;
1768 	if (attr_mask & IB_QP_AV)
1769 		qp->av_sgid_attr =
1770 			rdma_update_sgid_attr(&attr->ah_attr, qp->av_sgid_attr);
1771 	if (attr_mask & IB_QP_ALT_PATH)
1772 		qp->alt_path_sgid_attr = rdma_update_sgid_attr(
1773 			&attr->alt_ah_attr, qp->alt_path_sgid_attr);
1774 
1775 out:
1776 	if (attr_mask & IB_QP_ALT_PATH)
1777 		rdma_unfill_sgid_attr(&attr->alt_ah_attr, old_sgid_attr_alt_av);
1778 out_av:
1779 	if (attr_mask & IB_QP_AV) {
1780 		rdma_lag_put_ah_roce_slave(attr->xmit_slave);
1781 		rdma_unfill_sgid_attr(&attr->ah_attr, old_sgid_attr_av);
1782 	}
1783 	return ret;
1784 }
1785 
1786 /**
1787  * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
1788  * @ib_qp: The QP to modify.
1789  * @attr: On input, specifies the QP attributes to modify.  On output,
1790  *   the current values of selected QP attributes are returned.
1791  * @attr_mask: A bit-mask used to specify which attributes of the QP
1792  *   are being modified.
1793  * @udata: pointer to user's input output buffer information
1794  *   are being modified.
1795  * It returns 0 on success and returns appropriate error code on error.
1796  */
1797 int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,
1798 			    int attr_mask, struct ib_udata *udata)
1799 {
1800 	return _ib_modify_qp(ib_qp->real_qp, attr, attr_mask, udata);
1801 }
1802 EXPORT_SYMBOL(ib_modify_qp_with_udata);
1803 
1804 int ib_get_eth_speed(struct ib_device *dev, u8 port_num, u16 *speed, u8 *width)
1805 {
1806 	int rc;
1807 	u32 netdev_speed;
1808 	struct net_device *netdev;
1809 	struct ethtool_link_ksettings lksettings;
1810 
1811 	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
1812 		return -EINVAL;
1813 
1814 	netdev = ib_device_get_netdev(dev, port_num);
1815 	if (!netdev)
1816 		return -ENODEV;
1817 
1818 	rtnl_lock();
1819 	rc = __ethtool_get_link_ksettings(netdev, &lksettings);
1820 	rtnl_unlock();
1821 
1822 	dev_put(netdev);
1823 
1824 	if (!rc && lksettings.base.speed != (u32)SPEED_UNKNOWN) {
1825 		netdev_speed = lksettings.base.speed;
1826 	} else {
1827 		netdev_speed = SPEED_1000;
1828 		pr_warn("%s speed is unknown, defaulting to %d\n", netdev->name,
1829 			netdev_speed);
1830 	}
1831 
1832 	if (netdev_speed <= SPEED_1000) {
1833 		*width = IB_WIDTH_1X;
1834 		*speed = IB_SPEED_SDR;
1835 	} else if (netdev_speed <= SPEED_10000) {
1836 		*width = IB_WIDTH_1X;
1837 		*speed = IB_SPEED_FDR10;
1838 	} else if (netdev_speed <= SPEED_20000) {
1839 		*width = IB_WIDTH_4X;
1840 		*speed = IB_SPEED_DDR;
1841 	} else if (netdev_speed <= SPEED_25000) {
1842 		*width = IB_WIDTH_1X;
1843 		*speed = IB_SPEED_EDR;
1844 	} else if (netdev_speed <= SPEED_40000) {
1845 		*width = IB_WIDTH_4X;
1846 		*speed = IB_SPEED_FDR10;
1847 	} else {
1848 		*width = IB_WIDTH_4X;
1849 		*speed = IB_SPEED_EDR;
1850 	}
1851 
1852 	return 0;
1853 }
1854 EXPORT_SYMBOL(ib_get_eth_speed);
1855 
1856 int ib_modify_qp(struct ib_qp *qp,
1857 		 struct ib_qp_attr *qp_attr,
1858 		 int qp_attr_mask)
1859 {
1860 	return _ib_modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
1861 }
1862 EXPORT_SYMBOL(ib_modify_qp);
1863 
1864 int ib_query_qp(struct ib_qp *qp,
1865 		struct ib_qp_attr *qp_attr,
1866 		int qp_attr_mask,
1867 		struct ib_qp_init_attr *qp_init_attr)
1868 {
1869 	qp_attr->ah_attr.grh.sgid_attr = NULL;
1870 	qp_attr->alt_ah_attr.grh.sgid_attr = NULL;
1871 
1872 	return qp->device->ops.query_qp ?
1873 		qp->device->ops.query_qp(qp->real_qp, qp_attr, qp_attr_mask,
1874 					 qp_init_attr) : -EOPNOTSUPP;
1875 }
1876 EXPORT_SYMBOL(ib_query_qp);
1877 
1878 int ib_close_qp(struct ib_qp *qp)
1879 {
1880 	struct ib_qp *real_qp;
1881 	unsigned long flags;
1882 
1883 	real_qp = qp->real_qp;
1884 	if (real_qp == qp)
1885 		return -EINVAL;
1886 
1887 	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1888 	list_del(&qp->open_list);
1889 	spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
1890 
1891 	atomic_dec(&real_qp->usecnt);
1892 	if (qp->qp_sec)
1893 		ib_close_shared_qp_security(qp->qp_sec);
1894 	kfree(qp);
1895 
1896 	return 0;
1897 }
1898 EXPORT_SYMBOL(ib_close_qp);
1899 
1900 static int __ib_destroy_shared_qp(struct ib_qp *qp)
1901 {
1902 	struct ib_xrcd *xrcd;
1903 	struct ib_qp *real_qp;
1904 	int ret;
1905 
1906 	real_qp = qp->real_qp;
1907 	xrcd = real_qp->xrcd;
1908 	down_write(&xrcd->tgt_qps_rwsem);
1909 	ib_close_qp(qp);
1910 	if (atomic_read(&real_qp->usecnt) == 0)
1911 		xa_erase(&xrcd->tgt_qps, real_qp->qp_num);
1912 	else
1913 		real_qp = NULL;
1914 	up_write(&xrcd->tgt_qps_rwsem);
1915 
1916 	if (real_qp) {
1917 		ret = ib_destroy_qp(real_qp);
1918 		if (!ret)
1919 			atomic_dec(&xrcd->usecnt);
1920 	}
1921 
1922 	return 0;
1923 }
1924 
1925 int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
1926 {
1927 	const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr;
1928 	const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr;
1929 	struct ib_pd *pd;
1930 	struct ib_cq *scq, *rcq;
1931 	struct ib_srq *srq;
1932 	struct ib_rwq_ind_table *ind_tbl;
1933 	struct ib_qp_security *sec;
1934 	int ret;
1935 
1936 	WARN_ON_ONCE(qp->mrs_used > 0);
1937 
1938 	if (atomic_read(&qp->usecnt))
1939 		return -EBUSY;
1940 
1941 	if (qp->real_qp != qp)
1942 		return __ib_destroy_shared_qp(qp);
1943 
1944 	pd   = qp->pd;
1945 	scq  = qp->send_cq;
1946 	rcq  = qp->recv_cq;
1947 	srq  = qp->srq;
1948 	ind_tbl = qp->rwq_ind_tbl;
1949 	sec  = qp->qp_sec;
1950 	if (sec)
1951 		ib_destroy_qp_security_begin(sec);
1952 
1953 	if (!qp->uobject)
1954 		rdma_rw_cleanup_mrs(qp);
1955 
1956 	rdma_counter_unbind_qp(qp, true);
1957 	rdma_restrack_del(&qp->res);
1958 	ret = qp->device->ops.destroy_qp(qp, udata);
1959 	if (!ret) {
1960 		if (alt_path_sgid_attr)
1961 			rdma_put_gid_attr(alt_path_sgid_attr);
1962 		if (av_sgid_attr)
1963 			rdma_put_gid_attr(av_sgid_attr);
1964 		if (pd)
1965 			atomic_dec(&pd->usecnt);
1966 		if (scq)
1967 			atomic_dec(&scq->usecnt);
1968 		if (rcq)
1969 			atomic_dec(&rcq->usecnt);
1970 		if (srq)
1971 			atomic_dec(&srq->usecnt);
1972 		if (ind_tbl)
1973 			atomic_dec(&ind_tbl->usecnt);
1974 		if (sec)
1975 			ib_destroy_qp_security_end(sec);
1976 	} else {
1977 		if (sec)
1978 			ib_destroy_qp_security_abort(sec);
1979 	}
1980 
1981 	return ret;
1982 }
1983 EXPORT_SYMBOL(ib_destroy_qp_user);
1984 
1985 /* Completion queues */
1986 
1987 struct ib_cq *__ib_create_cq(struct ib_device *device,
1988 			     ib_comp_handler comp_handler,
1989 			     void (*event_handler)(struct ib_event *, void *),
1990 			     void *cq_context,
1991 			     const struct ib_cq_init_attr *cq_attr,
1992 			     const char *caller)
1993 {
1994 	struct ib_cq *cq;
1995 	int ret;
1996 
1997 	cq = rdma_zalloc_drv_obj(device, ib_cq);
1998 	if (!cq)
1999 		return ERR_PTR(-ENOMEM);
2000 
2001 	cq->device = device;
2002 	cq->uobject = NULL;
2003 	cq->comp_handler = comp_handler;
2004 	cq->event_handler = event_handler;
2005 	cq->cq_context = cq_context;
2006 	atomic_set(&cq->usecnt, 0);
2007 
2008 	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
2009 	rdma_restrack_set_name(&cq->res, caller);
2010 
2011 	ret = device->ops.create_cq(cq, cq_attr, NULL);
2012 	if (ret) {
2013 		rdma_restrack_put(&cq->res);
2014 		kfree(cq);
2015 		return ERR_PTR(ret);
2016 	}
2017 
2018 	rdma_restrack_add(&cq->res);
2019 	return cq;
2020 }
2021 EXPORT_SYMBOL(__ib_create_cq);
2022 
2023 int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
2024 {
2025 	if (cq->shared)
2026 		return -EOPNOTSUPP;
2027 
2028 	return cq->device->ops.modify_cq ?
2029 		cq->device->ops.modify_cq(cq, cq_count,
2030 					  cq_period) : -EOPNOTSUPP;
2031 }
2032 EXPORT_SYMBOL(rdma_set_cq_moderation);
2033 
2034 int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
2035 {
2036 	int ret;
2037 
2038 	if (WARN_ON_ONCE(cq->shared))
2039 		return -EOPNOTSUPP;
2040 
2041 	if (atomic_read(&cq->usecnt))
2042 		return -EBUSY;
2043 
2044 	ret = cq->device->ops.destroy_cq(cq, udata);
2045 	if (ret)
2046 		return ret;
2047 
2048 	rdma_restrack_del(&cq->res);
2049 	kfree(cq);
2050 	return ret;
2051 }
2052 EXPORT_SYMBOL(ib_destroy_cq_user);
2053 
2054 int ib_resize_cq(struct ib_cq *cq, int cqe)
2055 {
2056 	if (cq->shared)
2057 		return -EOPNOTSUPP;
2058 
2059 	return cq->device->ops.resize_cq ?
2060 		cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
2061 }
2062 EXPORT_SYMBOL(ib_resize_cq);
2063 
2064 /* Memory regions */
2065 
2066 struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2067 			     u64 virt_addr, int access_flags)
2068 {
2069 	struct ib_mr *mr;
2070 
2071 	if (access_flags & IB_ACCESS_ON_DEMAND) {
2072 		if (!(pd->device->attrs.device_cap_flags &
2073 		      IB_DEVICE_ON_DEMAND_PAGING)) {
2074 			pr_debug("ODP support not available\n");
2075 			return ERR_PTR(-EINVAL);
2076 		}
2077 	}
2078 
2079 	mr = pd->device->ops.reg_user_mr(pd, start, length, virt_addr,
2080 					 access_flags, NULL);
2081 
2082 	if (IS_ERR(mr))
2083 		return mr;
2084 
2085 	mr->device = pd->device;
2086 	mr->pd = pd;
2087 	mr->dm = NULL;
2088 	atomic_inc(&pd->usecnt);
2089 
2090 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2091 	rdma_restrack_parent_name(&mr->res, &pd->res);
2092 	rdma_restrack_add(&mr->res);
2093 
2094 	return mr;
2095 }
2096 EXPORT_SYMBOL(ib_reg_user_mr);
2097 
2098 int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
2099 		 u32 flags, struct ib_sge *sg_list, u32 num_sge)
2100 {
2101 	if (!pd->device->ops.advise_mr)
2102 		return -EOPNOTSUPP;
2103 
2104 	if (!num_sge)
2105 		return 0;
2106 
2107 	return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,
2108 					 NULL);
2109 }
2110 EXPORT_SYMBOL(ib_advise_mr);
2111 
2112 int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
2113 {
2114 	struct ib_pd *pd = mr->pd;
2115 	struct ib_dm *dm = mr->dm;
2116 	struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
2117 	int ret;
2118 
2119 	trace_mr_dereg(mr);
2120 	rdma_restrack_del(&mr->res);
2121 	ret = mr->device->ops.dereg_mr(mr, udata);
2122 	if (!ret) {
2123 		atomic_dec(&pd->usecnt);
2124 		if (dm)
2125 			atomic_dec(&dm->usecnt);
2126 		kfree(sig_attrs);
2127 	}
2128 
2129 	return ret;
2130 }
2131 EXPORT_SYMBOL(ib_dereg_mr_user);
2132 
2133 /**
2134  * ib_alloc_mr() - Allocates a memory region
2135  * @pd:            protection domain associated with the region
2136  * @mr_type:       memory region type
2137  * @max_num_sg:    maximum sg entries available for registration.
2138  *
2139  * Notes:
2140  * Memory registeration page/sg lists must not exceed max_num_sg.
2141  * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
2142  * max_num_sg * used_page_size.
2143  *
2144  */
2145 struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
2146 			  u32 max_num_sg)
2147 {
2148 	struct ib_mr *mr;
2149 
2150 	if (!pd->device->ops.alloc_mr) {
2151 		mr = ERR_PTR(-EOPNOTSUPP);
2152 		goto out;
2153 	}
2154 
2155 	if (mr_type == IB_MR_TYPE_INTEGRITY) {
2156 		WARN_ON_ONCE(1);
2157 		mr = ERR_PTR(-EINVAL);
2158 		goto out;
2159 	}
2160 
2161 	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
2162 	if (IS_ERR(mr))
2163 		goto out;
2164 
2165 	mr->device = pd->device;
2166 	mr->pd = pd;
2167 	mr->dm = NULL;
2168 	mr->uobject = NULL;
2169 	atomic_inc(&pd->usecnt);
2170 	mr->need_inval = false;
2171 	mr->type = mr_type;
2172 	mr->sig_attrs = NULL;
2173 
2174 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2175 	rdma_restrack_parent_name(&mr->res, &pd->res);
2176 	rdma_restrack_add(&mr->res);
2177 out:
2178 	trace_mr_alloc(pd, mr_type, max_num_sg, mr);
2179 	return mr;
2180 }
2181 EXPORT_SYMBOL(ib_alloc_mr);
2182 
2183 /**
2184  * ib_alloc_mr_integrity() - Allocates an integrity memory region
2185  * @pd:                      protection domain associated with the region
2186  * @max_num_data_sg:         maximum data sg entries available for registration
2187  * @max_num_meta_sg:         maximum metadata sg entries available for
2188  *                           registration
2189  *
2190  * Notes:
2191  * Memory registration page/sg lists must not exceed max_num_sg,
2192  * also the integrity page/sg lists must not exceed max_num_meta_sg.
2193  *
2194  */
2195 struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
2196 				    u32 max_num_data_sg,
2197 				    u32 max_num_meta_sg)
2198 {
2199 	struct ib_mr *mr;
2200 	struct ib_sig_attrs *sig_attrs;
2201 
2202 	if (!pd->device->ops.alloc_mr_integrity ||
2203 	    !pd->device->ops.map_mr_sg_pi) {
2204 		mr = ERR_PTR(-EOPNOTSUPP);
2205 		goto out;
2206 	}
2207 
2208 	if (!max_num_meta_sg) {
2209 		mr = ERR_PTR(-EINVAL);
2210 		goto out;
2211 	}
2212 
2213 	sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
2214 	if (!sig_attrs) {
2215 		mr = ERR_PTR(-ENOMEM);
2216 		goto out;
2217 	}
2218 
2219 	mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
2220 						max_num_meta_sg);
2221 	if (IS_ERR(mr)) {
2222 		kfree(sig_attrs);
2223 		goto out;
2224 	}
2225 
2226 	mr->device = pd->device;
2227 	mr->pd = pd;
2228 	mr->dm = NULL;
2229 	mr->uobject = NULL;
2230 	atomic_inc(&pd->usecnt);
2231 	mr->need_inval = false;
2232 	mr->type = IB_MR_TYPE_INTEGRITY;
2233 	mr->sig_attrs = sig_attrs;
2234 
2235 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2236 	rdma_restrack_parent_name(&mr->res, &pd->res);
2237 	rdma_restrack_add(&mr->res);
2238 out:
2239 	trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
2240 	return mr;
2241 }
2242 EXPORT_SYMBOL(ib_alloc_mr_integrity);
2243 
2244 /* Multicast groups */
2245 
2246 static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
2247 {
2248 	struct ib_qp_init_attr init_attr = {};
2249 	struct ib_qp_attr attr = {};
2250 	int num_eth_ports = 0;
2251 	int port;
2252 
2253 	/* If QP state >= init, it is assigned to a port and we can check this
2254 	 * port only.
2255 	 */
2256 	if (!ib_query_qp(qp, &attr, IB_QP_STATE | IB_QP_PORT, &init_attr)) {
2257 		if (attr.qp_state >= IB_QPS_INIT) {
2258 			if (rdma_port_get_link_layer(qp->device, attr.port_num) !=
2259 			    IB_LINK_LAYER_INFINIBAND)
2260 				return true;
2261 			goto lid_check;
2262 		}
2263 	}
2264 
2265 	/* Can't get a quick answer, iterate over all ports */
2266 	for (port = 0; port < qp->device->phys_port_cnt; port++)
2267 		if (rdma_port_get_link_layer(qp->device, port) !=
2268 		    IB_LINK_LAYER_INFINIBAND)
2269 			num_eth_ports++;
2270 
2271 	/* If we have at lease one Ethernet port, RoCE annex declares that
2272 	 * multicast LID should be ignored. We can't tell at this step if the
2273 	 * QP belongs to an IB or Ethernet port.
2274 	 */
2275 	if (num_eth_ports)
2276 		return true;
2277 
2278 	/* If all the ports are IB, we can check according to IB spec. */
2279 lid_check:
2280 	return !(lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
2281 		 lid == be16_to_cpu(IB_LID_PERMISSIVE));
2282 }
2283 
2284 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2285 {
2286 	int ret;
2287 
2288 	if (!qp->device->ops.attach_mcast)
2289 		return -EOPNOTSUPP;
2290 
2291 	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2292 	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2293 		return -EINVAL;
2294 
2295 	ret = qp->device->ops.attach_mcast(qp, gid, lid);
2296 	if (!ret)
2297 		atomic_inc(&qp->usecnt);
2298 	return ret;
2299 }
2300 EXPORT_SYMBOL(ib_attach_mcast);
2301 
2302 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2303 {
2304 	int ret;
2305 
2306 	if (!qp->device->ops.detach_mcast)
2307 		return -EOPNOTSUPP;
2308 
2309 	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2310 	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2311 		return -EINVAL;
2312 
2313 	ret = qp->device->ops.detach_mcast(qp, gid, lid);
2314 	if (!ret)
2315 		atomic_dec(&qp->usecnt);
2316 	return ret;
2317 }
2318 EXPORT_SYMBOL(ib_detach_mcast);
2319 
2320 /**
2321  * ib_alloc_xrcd_user - Allocates an XRC domain.
2322  * @device: The device on which to allocate the XRC domain.
2323  * @inode: inode to connect XRCD
2324  * @udata: Valid user data or NULL for kernel object
2325  */
2326 struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
2327 				   struct inode *inode, struct ib_udata *udata)
2328 {
2329 	struct ib_xrcd *xrcd;
2330 	int ret;
2331 
2332 	if (!device->ops.alloc_xrcd)
2333 		return ERR_PTR(-EOPNOTSUPP);
2334 
2335 	xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);
2336 	if (!xrcd)
2337 		return ERR_PTR(-ENOMEM);
2338 
2339 	xrcd->device = device;
2340 	xrcd->inode = inode;
2341 	atomic_set(&xrcd->usecnt, 0);
2342 	init_rwsem(&xrcd->tgt_qps_rwsem);
2343 	xa_init(&xrcd->tgt_qps);
2344 
2345 	ret = device->ops.alloc_xrcd(xrcd, udata);
2346 	if (ret)
2347 		goto err;
2348 	return xrcd;
2349 err:
2350 	kfree(xrcd);
2351 	return ERR_PTR(ret);
2352 }
2353 EXPORT_SYMBOL(ib_alloc_xrcd_user);
2354 
2355 /**
2356  * ib_dealloc_xrcd_user - Deallocates an XRC domain.
2357  * @xrcd: The XRC domain to deallocate.
2358  * @udata: Valid user data or NULL for kernel object
2359  */
2360 int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata)
2361 {
2362 	int ret;
2363 
2364 	if (atomic_read(&xrcd->usecnt))
2365 		return -EBUSY;
2366 
2367 	WARN_ON(!xa_empty(&xrcd->tgt_qps));
2368 	ret = xrcd->device->ops.dealloc_xrcd(xrcd, udata);
2369 	if (ret)
2370 		return ret;
2371 	kfree(xrcd);
2372 	return ret;
2373 }
2374 EXPORT_SYMBOL(ib_dealloc_xrcd_user);
2375 
2376 /**
2377  * ib_create_wq - Creates a WQ associated with the specified protection
2378  * domain.
2379  * @pd: The protection domain associated with the WQ.
2380  * @wq_attr: A list of initial attributes required to create the
2381  * WQ. If WQ creation succeeds, then the attributes are updated to
2382  * the actual capabilities of the created WQ.
2383  *
2384  * wq_attr->max_wr and wq_attr->max_sge determine
2385  * the requested size of the WQ, and set to the actual values allocated
2386  * on return.
2387  * If ib_create_wq() succeeds, then max_wr and max_sge will always be
2388  * at least as large as the requested values.
2389  */
2390 struct ib_wq *ib_create_wq(struct ib_pd *pd,
2391 			   struct ib_wq_init_attr *wq_attr)
2392 {
2393 	struct ib_wq *wq;
2394 
2395 	if (!pd->device->ops.create_wq)
2396 		return ERR_PTR(-EOPNOTSUPP);
2397 
2398 	wq = pd->device->ops.create_wq(pd, wq_attr, NULL);
2399 	if (!IS_ERR(wq)) {
2400 		wq->event_handler = wq_attr->event_handler;
2401 		wq->wq_context = wq_attr->wq_context;
2402 		wq->wq_type = wq_attr->wq_type;
2403 		wq->cq = wq_attr->cq;
2404 		wq->device = pd->device;
2405 		wq->pd = pd;
2406 		wq->uobject = NULL;
2407 		atomic_inc(&pd->usecnt);
2408 		atomic_inc(&wq_attr->cq->usecnt);
2409 		atomic_set(&wq->usecnt, 0);
2410 	}
2411 	return wq;
2412 }
2413 EXPORT_SYMBOL(ib_create_wq);
2414 
2415 /**
2416  * ib_destroy_wq_user - Destroys the specified user WQ.
2417  * @wq: The WQ to destroy.
2418  * @udata: Valid user data
2419  */
2420 int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata)
2421 {
2422 	struct ib_cq *cq = wq->cq;
2423 	struct ib_pd *pd = wq->pd;
2424 	int ret;
2425 
2426 	if (atomic_read(&wq->usecnt))
2427 		return -EBUSY;
2428 
2429 	ret = wq->device->ops.destroy_wq(wq, udata);
2430 	if (ret)
2431 		return ret;
2432 
2433 	atomic_dec(&pd->usecnt);
2434 	atomic_dec(&cq->usecnt);
2435 	return ret;
2436 }
2437 EXPORT_SYMBOL(ib_destroy_wq_user);
2438 
2439 /**
2440  * ib_modify_wq - Modifies the specified WQ.
2441  * @wq: The WQ to modify.
2442  * @wq_attr: On input, specifies the WQ attributes to modify.
2443  * @wq_attr_mask: A bit-mask used to specify which attributes of the WQ
2444  *   are being modified.
2445  * On output, the current values of selected WQ attributes are returned.
2446  */
2447 int ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
2448 		 u32 wq_attr_mask)
2449 {
2450 	int err;
2451 
2452 	if (!wq->device->ops.modify_wq)
2453 		return -EOPNOTSUPP;
2454 
2455 	err = wq->device->ops.modify_wq(wq, wq_attr, wq_attr_mask, NULL);
2456 	return err;
2457 }
2458 EXPORT_SYMBOL(ib_modify_wq);
2459 
2460 int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
2461 		       struct ib_mr_status *mr_status)
2462 {
2463 	if (!mr->device->ops.check_mr_status)
2464 		return -EOPNOTSUPP;
2465 
2466 	return mr->device->ops.check_mr_status(mr, check_mask, mr_status);
2467 }
2468 EXPORT_SYMBOL(ib_check_mr_status);
2469 
2470 int ib_set_vf_link_state(struct ib_device *device, int vf, u8 port,
2471 			 int state)
2472 {
2473 	if (!device->ops.set_vf_link_state)
2474 		return -EOPNOTSUPP;
2475 
2476 	return device->ops.set_vf_link_state(device, vf, port, state);
2477 }
2478 EXPORT_SYMBOL(ib_set_vf_link_state);
2479 
2480 int ib_get_vf_config(struct ib_device *device, int vf, u8 port,
2481 		     struct ifla_vf_info *info)
2482 {
2483 	if (!device->ops.get_vf_config)
2484 		return -EOPNOTSUPP;
2485 
2486 	return device->ops.get_vf_config(device, vf, port, info);
2487 }
2488 EXPORT_SYMBOL(ib_get_vf_config);
2489 
2490 int ib_get_vf_stats(struct ib_device *device, int vf, u8 port,
2491 		    struct ifla_vf_stats *stats)
2492 {
2493 	if (!device->ops.get_vf_stats)
2494 		return -EOPNOTSUPP;
2495 
2496 	return device->ops.get_vf_stats(device, vf, port, stats);
2497 }
2498 EXPORT_SYMBOL(ib_get_vf_stats);
2499 
2500 int ib_set_vf_guid(struct ib_device *device, int vf, u8 port, u64 guid,
2501 		   int type)
2502 {
2503 	if (!device->ops.set_vf_guid)
2504 		return -EOPNOTSUPP;
2505 
2506 	return device->ops.set_vf_guid(device, vf, port, guid, type);
2507 }
2508 EXPORT_SYMBOL(ib_set_vf_guid);
2509 
2510 int ib_get_vf_guid(struct ib_device *device, int vf, u8 port,
2511 		   struct ifla_vf_guid *node_guid,
2512 		   struct ifla_vf_guid *port_guid)
2513 {
2514 	if (!device->ops.get_vf_guid)
2515 		return -EOPNOTSUPP;
2516 
2517 	return device->ops.get_vf_guid(device, vf, port, node_guid, port_guid);
2518 }
2519 EXPORT_SYMBOL(ib_get_vf_guid);
2520 /**
2521  * ib_map_mr_sg_pi() - Map the dma mapped SG lists for PI (protection
2522  *     information) and set an appropriate memory region for registration.
2523  * @mr:             memory region
2524  * @data_sg:        dma mapped scatterlist for data
2525  * @data_sg_nents:  number of entries in data_sg
2526  * @data_sg_offset: offset in bytes into data_sg
2527  * @meta_sg:        dma mapped scatterlist for metadata
2528  * @meta_sg_nents:  number of entries in meta_sg
2529  * @meta_sg_offset: offset in bytes into meta_sg
2530  * @page_size:      page vector desired page size
2531  *
2532  * Constraints:
2533  * - The MR must be allocated with type IB_MR_TYPE_INTEGRITY.
2534  *
2535  * Return: 0 on success.
2536  *
2537  * After this completes successfully, the  memory region
2538  * is ready for registration.
2539  */
2540 int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
2541 		    int data_sg_nents, unsigned int *data_sg_offset,
2542 		    struct scatterlist *meta_sg, int meta_sg_nents,
2543 		    unsigned int *meta_sg_offset, unsigned int page_size)
2544 {
2545 	if (unlikely(!mr->device->ops.map_mr_sg_pi ||
2546 		     WARN_ON_ONCE(mr->type != IB_MR_TYPE_INTEGRITY)))
2547 		return -EOPNOTSUPP;
2548 
2549 	mr->page_size = page_size;
2550 
2551 	return mr->device->ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,
2552 					    data_sg_offset, meta_sg,
2553 					    meta_sg_nents, meta_sg_offset);
2554 }
2555 EXPORT_SYMBOL(ib_map_mr_sg_pi);
2556 
2557 /**
2558  * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
2559  *     and set it the memory region.
2560  * @mr:            memory region
2561  * @sg:            dma mapped scatterlist
2562  * @sg_nents:      number of entries in sg
2563  * @sg_offset:     offset in bytes into sg
2564  * @page_size:     page vector desired page size
2565  *
2566  * Constraints:
2567  *
2568  * - The first sg element is allowed to have an offset.
2569  * - Each sg element must either be aligned to page_size or virtually
2570  *   contiguous to the previous element. In case an sg element has a
2571  *   non-contiguous offset, the mapping prefix will not include it.
2572  * - The last sg element is allowed to have length less than page_size.
2573  * - If sg_nents total byte length exceeds the mr max_num_sge * page_size
2574  *   then only max_num_sg entries will be mapped.
2575  * - If the MR was allocated with type IB_MR_TYPE_SG_GAPS, none of these
2576  *   constraints holds and the page_size argument is ignored.
2577  *
2578  * Returns the number of sg elements that were mapped to the memory region.
2579  *
2580  * After this completes successfully, the  memory region
2581  * is ready for registration.
2582  */
2583 int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2584 		 unsigned int *sg_offset, unsigned int page_size)
2585 {
2586 	if (unlikely(!mr->device->ops.map_mr_sg))
2587 		return -EOPNOTSUPP;
2588 
2589 	mr->page_size = page_size;
2590 
2591 	return mr->device->ops.map_mr_sg(mr, sg, sg_nents, sg_offset);
2592 }
2593 EXPORT_SYMBOL(ib_map_mr_sg);
2594 
2595 /**
2596  * ib_sg_to_pages() - Convert the largest prefix of a sg list
2597  *     to a page vector
2598  * @mr:            memory region
2599  * @sgl:           dma mapped scatterlist
2600  * @sg_nents:      number of entries in sg
2601  * @sg_offset_p:   ==== =======================================================
2602  *                 IN   start offset in bytes into sg
2603  *                 OUT  offset in bytes for element n of the sg of the first
2604  *                      byte that has not been processed where n is the return
2605  *                      value of this function.
2606  *                 ==== =======================================================
2607  * @set_page:      driver page assignment function pointer
2608  *
2609  * Core service helper for drivers to convert the largest
2610  * prefix of given sg list to a page vector. The sg list
2611  * prefix converted is the prefix that meet the requirements
2612  * of ib_map_mr_sg.
2613  *
2614  * Returns the number of sg elements that were assigned to
2615  * a page vector.
2616  */
2617 int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
2618 		unsigned int *sg_offset_p, int (*set_page)(struct ib_mr *, u64))
2619 {
2620 	struct scatterlist *sg;
2621 	u64 last_end_dma_addr = 0;
2622 	unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2623 	unsigned int last_page_off = 0;
2624 	u64 page_mask = ~((u64)mr->page_size - 1);
2625 	int i, ret;
2626 
2627 	if (unlikely(sg_nents <= 0 || sg_offset > sg_dma_len(&sgl[0])))
2628 		return -EINVAL;
2629 
2630 	mr->iova = sg_dma_address(&sgl[0]) + sg_offset;
2631 	mr->length = 0;
2632 
2633 	for_each_sg(sgl, sg, sg_nents, i) {
2634 		u64 dma_addr = sg_dma_address(sg) + sg_offset;
2635 		u64 prev_addr = dma_addr;
2636 		unsigned int dma_len = sg_dma_len(sg) - sg_offset;
2637 		u64 end_dma_addr = dma_addr + dma_len;
2638 		u64 page_addr = dma_addr & page_mask;
2639 
2640 		/*
2641 		 * For the second and later elements, check whether either the
2642 		 * end of element i-1 or the start of element i is not aligned
2643 		 * on a page boundary.
2644 		 */
2645 		if (i && (last_page_off != 0 || page_addr != dma_addr)) {
2646 			/* Stop mapping if there is a gap. */
2647 			if (last_end_dma_addr != dma_addr)
2648 				break;
2649 
2650 			/*
2651 			 * Coalesce this element with the last. If it is small
2652 			 * enough just update mr->length. Otherwise start
2653 			 * mapping from the next page.
2654 			 */
2655 			goto next_page;
2656 		}
2657 
2658 		do {
2659 			ret = set_page(mr, page_addr);
2660 			if (unlikely(ret < 0)) {
2661 				sg_offset = prev_addr - sg_dma_address(sg);
2662 				mr->length += prev_addr - dma_addr;
2663 				if (sg_offset_p)
2664 					*sg_offset_p = sg_offset;
2665 				return i || sg_offset ? i : ret;
2666 			}
2667 			prev_addr = page_addr;
2668 next_page:
2669 			page_addr += mr->page_size;
2670 		} while (page_addr < end_dma_addr);
2671 
2672 		mr->length += dma_len;
2673 		last_end_dma_addr = end_dma_addr;
2674 		last_page_off = end_dma_addr & ~page_mask;
2675 
2676 		sg_offset = 0;
2677 	}
2678 
2679 	if (sg_offset_p)
2680 		*sg_offset_p = 0;
2681 	return i;
2682 }
2683 EXPORT_SYMBOL(ib_sg_to_pages);
2684 
2685 struct ib_drain_cqe {
2686 	struct ib_cqe cqe;
2687 	struct completion done;
2688 };
2689 
2690 static void ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
2691 {
2692 	struct ib_drain_cqe *cqe = container_of(wc->wr_cqe, struct ib_drain_cqe,
2693 						cqe);
2694 
2695 	complete(&cqe->done);
2696 }
2697 
2698 /*
2699  * Post a WR and block until its completion is reaped for the SQ.
2700  */
2701 static void __ib_drain_sq(struct ib_qp *qp)
2702 {
2703 	struct ib_cq *cq = qp->send_cq;
2704 	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2705 	struct ib_drain_cqe sdrain;
2706 	struct ib_rdma_wr swr = {
2707 		.wr = {
2708 			.next = NULL,
2709 			{ .wr_cqe	= &sdrain.cqe, },
2710 			.opcode	= IB_WR_RDMA_WRITE,
2711 		},
2712 	};
2713 	int ret;
2714 
2715 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2716 	if (ret) {
2717 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2718 		return;
2719 	}
2720 
2721 	sdrain.cqe.done = ib_drain_qp_done;
2722 	init_completion(&sdrain.done);
2723 
2724 	ret = ib_post_send(qp, &swr.wr, NULL);
2725 	if (ret) {
2726 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2727 		return;
2728 	}
2729 
2730 	if (cq->poll_ctx == IB_POLL_DIRECT)
2731 		while (wait_for_completion_timeout(&sdrain.done, HZ / 10) <= 0)
2732 			ib_process_cq_direct(cq, -1);
2733 	else
2734 		wait_for_completion(&sdrain.done);
2735 }
2736 
2737 /*
2738  * Post a WR and block until its completion is reaped for the RQ.
2739  */
2740 static void __ib_drain_rq(struct ib_qp *qp)
2741 {
2742 	struct ib_cq *cq = qp->recv_cq;
2743 	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2744 	struct ib_drain_cqe rdrain;
2745 	struct ib_recv_wr rwr = {};
2746 	int ret;
2747 
2748 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2749 	if (ret) {
2750 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2751 		return;
2752 	}
2753 
2754 	rwr.wr_cqe = &rdrain.cqe;
2755 	rdrain.cqe.done = ib_drain_qp_done;
2756 	init_completion(&rdrain.done);
2757 
2758 	ret = ib_post_recv(qp, &rwr, NULL);
2759 	if (ret) {
2760 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2761 		return;
2762 	}
2763 
2764 	if (cq->poll_ctx == IB_POLL_DIRECT)
2765 		while (wait_for_completion_timeout(&rdrain.done, HZ / 10) <= 0)
2766 			ib_process_cq_direct(cq, -1);
2767 	else
2768 		wait_for_completion(&rdrain.done);
2769 }
2770 
2771 /**
2772  * ib_drain_sq() - Block until all SQ CQEs have been consumed by the
2773  *		   application.
2774  * @qp:            queue pair to drain
2775  *
2776  * If the device has a provider-specific drain function, then
2777  * call that.  Otherwise call the generic drain function
2778  * __ib_drain_sq().
2779  *
2780  * The caller must:
2781  *
2782  * ensure there is room in the CQ and SQ for the drain work request and
2783  * completion.
2784  *
2785  * allocate the CQ using ib_alloc_cq().
2786  *
2787  * ensure that there are no other contexts that are posting WRs concurrently.
2788  * Otherwise the drain is not guaranteed.
2789  */
2790 void ib_drain_sq(struct ib_qp *qp)
2791 {
2792 	if (qp->device->ops.drain_sq)
2793 		qp->device->ops.drain_sq(qp);
2794 	else
2795 		__ib_drain_sq(qp);
2796 	trace_cq_drain_complete(qp->send_cq);
2797 }
2798 EXPORT_SYMBOL(ib_drain_sq);
2799 
2800 /**
2801  * ib_drain_rq() - Block until all RQ CQEs have been consumed by the
2802  *		   application.
2803  * @qp:            queue pair to drain
2804  *
2805  * If the device has a provider-specific drain function, then
2806  * call that.  Otherwise call the generic drain function
2807  * __ib_drain_rq().
2808  *
2809  * The caller must:
2810  *
2811  * ensure there is room in the CQ and RQ for the drain work request and
2812  * completion.
2813  *
2814  * allocate the CQ using ib_alloc_cq().
2815  *
2816  * ensure that there are no other contexts that are posting WRs concurrently.
2817  * Otherwise the drain is not guaranteed.
2818  */
2819 void ib_drain_rq(struct ib_qp *qp)
2820 {
2821 	if (qp->device->ops.drain_rq)
2822 		qp->device->ops.drain_rq(qp);
2823 	else
2824 		__ib_drain_rq(qp);
2825 	trace_cq_drain_complete(qp->recv_cq);
2826 }
2827 EXPORT_SYMBOL(ib_drain_rq);
2828 
2829 /**
2830  * ib_drain_qp() - Block until all CQEs have been consumed by the
2831  *		   application on both the RQ and SQ.
2832  * @qp:            queue pair to drain
2833  *
2834  * The caller must:
2835  *
2836  * ensure there is room in the CQ(s), SQ, and RQ for drain work requests
2837  * and completions.
2838  *
2839  * allocate the CQs using ib_alloc_cq().
2840  *
2841  * ensure that there are no other contexts that are posting WRs concurrently.
2842  * Otherwise the drain is not guaranteed.
2843  */
2844 void ib_drain_qp(struct ib_qp *qp)
2845 {
2846 	ib_drain_sq(qp);
2847 	if (!qp->srq)
2848 		ib_drain_rq(qp);
2849 }
2850 EXPORT_SYMBOL(ib_drain_qp);
2851 
2852 struct net_device *rdma_alloc_netdev(struct ib_device *device, u8 port_num,
2853 				     enum rdma_netdev_t type, const char *name,
2854 				     unsigned char name_assign_type,
2855 				     void (*setup)(struct net_device *))
2856 {
2857 	struct rdma_netdev_alloc_params params;
2858 	struct net_device *netdev;
2859 	int rc;
2860 
2861 	if (!device->ops.rdma_netdev_get_params)
2862 		return ERR_PTR(-EOPNOTSUPP);
2863 
2864 	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
2865 						&params);
2866 	if (rc)
2867 		return ERR_PTR(rc);
2868 
2869 	netdev = alloc_netdev_mqs(params.sizeof_priv, name, name_assign_type,
2870 				  setup, params.txqs, params.rxqs);
2871 	if (!netdev)
2872 		return ERR_PTR(-ENOMEM);
2873 
2874 	return netdev;
2875 }
2876 EXPORT_SYMBOL(rdma_alloc_netdev);
2877 
2878 int rdma_init_netdev(struct ib_device *device, u8 port_num,
2879 		     enum rdma_netdev_t type, const char *name,
2880 		     unsigned char name_assign_type,
2881 		     void (*setup)(struct net_device *),
2882 		     struct net_device *netdev)
2883 {
2884 	struct rdma_netdev_alloc_params params;
2885 	int rc;
2886 
2887 	if (!device->ops.rdma_netdev_get_params)
2888 		return -EOPNOTSUPP;
2889 
2890 	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
2891 						&params);
2892 	if (rc)
2893 		return rc;
2894 
2895 	return params.initialize_rdma_netdev(device, port_num,
2896 					     netdev, params.param);
2897 }
2898 EXPORT_SYMBOL(rdma_init_netdev);
2899 
2900 void __rdma_block_iter_start(struct ib_block_iter *biter,
2901 			     struct scatterlist *sglist, unsigned int nents,
2902 			     unsigned long pgsz)
2903 {
2904 	memset(biter, 0, sizeof(struct ib_block_iter));
2905 	biter->__sg = sglist;
2906 	biter->__sg_nents = nents;
2907 
2908 	/* Driver provides best block size to use */
2909 	biter->__pg_bit = __fls(pgsz);
2910 }
2911 EXPORT_SYMBOL(__rdma_block_iter_start);
2912 
2913 bool __rdma_block_iter_next(struct ib_block_iter *biter)
2914 {
2915 	unsigned int block_offset;
2916 
2917 	if (!biter->__sg_nents || !biter->__sg)
2918 		return false;
2919 
2920 	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
2921 	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
2922 	biter->__sg_advance += BIT_ULL(biter->__pg_bit) - block_offset;
2923 
2924 	if (biter->__sg_advance >= sg_dma_len(biter->__sg)) {
2925 		biter->__sg_advance = 0;
2926 		biter->__sg = sg_next(biter->__sg);
2927 		biter->__sg_nents--;
2928 	}
2929 
2930 	return true;
2931 }
2932 EXPORT_SYMBOL(__rdma_block_iter_next);
2933