xref: /linux/drivers/infiniband/sw/siw/siw_verbs.c (revision 2c1ed907520c50326b8f604907a8478b27881a2e)
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 
3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4 /* Copyright (c) 2008-2019, IBM Corporation */
5 
6 #include <linux/errno.h>
7 #include <linux/types.h>
8 #include <linux/uaccess.h>
9 #include <linux/vmalloc.h>
10 #include <linux/xarray.h>
11 #include <net/addrconf.h>
12 
13 #include <rdma/iw_cm.h>
14 #include <rdma/ib_verbs.h>
15 #include <rdma/ib_user_verbs.h>
16 #include <rdma/uverbs_ioctl.h>
17 
18 #include "siw.h"
19 #include "siw_verbs.h"
20 #include "siw_mem.h"
21 
22 static int siw_qp_state_to_ib_qp_state[SIW_QP_STATE_COUNT] = {
23 	[SIW_QP_STATE_IDLE] = IB_QPS_INIT,
24 	[SIW_QP_STATE_RTR] = IB_QPS_RTR,
25 	[SIW_QP_STATE_RTS] = IB_QPS_RTS,
26 	[SIW_QP_STATE_CLOSING] = IB_QPS_SQD,
27 	[SIW_QP_STATE_TERMINATE] = IB_QPS_SQE,
28 	[SIW_QP_STATE_ERROR] = IB_QPS_ERR
29 };
30 
31 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
32 	[IB_QPS_RESET] = SIW_QP_STATE_IDLE,
33 	[IB_QPS_INIT] = SIW_QP_STATE_IDLE,
34 	[IB_QPS_RTR] = SIW_QP_STATE_RTR,
35 	[IB_QPS_RTS] = SIW_QP_STATE_RTS,
36 	[IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
37 	[IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
38 	[IB_QPS_ERR] = SIW_QP_STATE_ERROR
39 };
40 
41 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
42 	[IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
43 	[IB_QPS_RTS] = "RTS",     [IB_QPS_SQD] = "SQD",   [IB_QPS_SQE] = "SQE",
44 	[IB_QPS_ERR] = "ERR"
45 };
46 
siw_mmap_free(struct rdma_user_mmap_entry * rdma_entry)47 void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
48 {
49 	struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
50 
51 	kfree(entry);
52 }
53 
siw_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)54 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
55 {
56 	struct siw_ucontext *uctx = to_siw_ctx(ctx);
57 	size_t size = vma->vm_end - vma->vm_start;
58 	struct rdma_user_mmap_entry *rdma_entry;
59 	struct siw_user_mmap_entry *entry;
60 	int rv = -EINVAL;
61 
62 	/*
63 	 * Must be page aligned
64 	 */
65 	if (vma->vm_start & (PAGE_SIZE - 1)) {
66 		pr_warn("siw: mmap not page aligned\n");
67 		return -EINVAL;
68 	}
69 	rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
70 	if (!rdma_entry) {
71 		siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
72 			vma->vm_pgoff, size);
73 		return -EINVAL;
74 	}
75 	entry = to_siw_mmap_entry(rdma_entry);
76 
77 	rv = remap_vmalloc_range(vma, entry->address, 0);
78 	if (rv)
79 		pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
80 			size);
81 	rdma_user_mmap_entry_put(rdma_entry);
82 
83 	return rv;
84 }
85 
siw_alloc_ucontext(struct ib_ucontext * base_ctx,struct ib_udata * udata)86 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
87 {
88 	struct siw_device *sdev = to_siw_dev(base_ctx->device);
89 	struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
90 	struct siw_uresp_alloc_ctx uresp = {};
91 	int rv;
92 
93 	if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
94 		rv = -ENOMEM;
95 		goto err_out;
96 	}
97 	ctx->sdev = sdev;
98 
99 	uresp.dev_id = sdev->vendor_part_id;
100 
101 	if (udata->outlen < sizeof(uresp)) {
102 		rv = -EINVAL;
103 		goto err_out;
104 	}
105 	rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
106 	if (rv)
107 		goto err_out;
108 
109 	siw_dbg(base_ctx->device, "success. now %d context(s)\n",
110 		atomic_read(&sdev->num_ctx));
111 
112 	return 0;
113 
114 err_out:
115 	atomic_dec(&sdev->num_ctx);
116 	siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
117 		atomic_read(&sdev->num_ctx));
118 
119 	return rv;
120 }
121 
siw_dealloc_ucontext(struct ib_ucontext * base_ctx)122 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
123 {
124 	struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
125 
126 	atomic_dec(&uctx->sdev->num_ctx);
127 }
128 
siw_query_device(struct ib_device * base_dev,struct ib_device_attr * attr,struct ib_udata * udata)129 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
130 		     struct ib_udata *udata)
131 {
132 	struct siw_device *sdev = to_siw_dev(base_dev);
133 
134 	if (udata->inlen || udata->outlen)
135 		return -EINVAL;
136 
137 	memset(attr, 0, sizeof(*attr));
138 
139 	/* Revisit atomic caps if RFC 7306 gets supported */
140 	attr->atomic_cap = 0;
141 	attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
142 	attr->kernel_cap_flags = IBK_ALLOW_USER_UNREG;
143 	attr->max_cq = sdev->attrs.max_cq;
144 	attr->max_cqe = sdev->attrs.max_cqe;
145 	attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
146 	attr->max_mr = sdev->attrs.max_mr;
147 	attr->max_mw = sdev->attrs.max_mw;
148 	attr->max_mr_size = ~0ull;
149 	attr->max_pd = sdev->attrs.max_pd;
150 	attr->max_qp = sdev->attrs.max_qp;
151 	attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
152 	attr->max_qp_rd_atom = sdev->attrs.max_ord;
153 	attr->max_qp_wr = sdev->attrs.max_qp_wr;
154 	attr->max_recv_sge = sdev->attrs.max_sge;
155 	attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
156 	attr->max_send_sge = sdev->attrs.max_sge;
157 	attr->max_sge_rd = sdev->attrs.max_sge_rd;
158 	attr->max_srq = sdev->attrs.max_srq;
159 	attr->max_srq_sge = sdev->attrs.max_srq_sge;
160 	attr->max_srq_wr = sdev->attrs.max_srq_wr;
161 	attr->page_size_cap = PAGE_SIZE;
162 	attr->vendor_id = SIW_VENDOR_ID;
163 	attr->vendor_part_id = sdev->vendor_part_id;
164 
165 	addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
166 			    sdev->raw_gid);
167 
168 	return 0;
169 }
170 
siw_query_port(struct ib_device * base_dev,u32 port,struct ib_port_attr * attr)171 int siw_query_port(struct ib_device *base_dev, u32 port,
172 		   struct ib_port_attr *attr)
173 {
174 	struct net_device *ndev;
175 	int rv;
176 
177 	memset(attr, 0, sizeof(*attr));
178 
179 	rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
180 			 &attr->active_width);
181 	if (rv)
182 		return rv;
183 
184 	ndev = ib_device_get_netdev(base_dev, SIW_PORT);
185 	if (!ndev)
186 		return -ENODEV;
187 
188 	attr->gid_tbl_len = 1;
189 	attr->max_msg_sz = -1;
190 	attr->max_mtu = ib_mtu_int_to_enum(ndev->max_mtu);
191 	attr->active_mtu = ib_mtu_int_to_enum(READ_ONCE(ndev->mtu));
192 	attr->state = ib_get_curr_port_state(ndev);
193 	attr->phys_state = attr->state == IB_PORT_ACTIVE ?
194 		IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
195 	attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
196 	/*
197 	 * All zero
198 	 *
199 	 * attr->lid = 0;
200 	 * attr->bad_pkey_cntr = 0;
201 	 * attr->qkey_viol_cntr = 0;
202 	 * attr->sm_lid = 0;
203 	 * attr->lmc = 0;
204 	 * attr->max_vl_num = 0;
205 	 * attr->sm_sl = 0;
206 	 * attr->subnet_timeout = 0;
207 	 * attr->init_type_repy = 0;
208 	 */
209 	dev_put(ndev);
210 	return rv;
211 }
212 
siw_get_port_immutable(struct ib_device * base_dev,u32 port,struct ib_port_immutable * port_immutable)213 int siw_get_port_immutable(struct ib_device *base_dev, u32 port,
214 			   struct ib_port_immutable *port_immutable)
215 {
216 	struct ib_port_attr attr;
217 	int rv = siw_query_port(base_dev, port, &attr);
218 
219 	if (rv)
220 		return rv;
221 
222 	port_immutable->gid_tbl_len = attr.gid_tbl_len;
223 	port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
224 
225 	return 0;
226 }
227 
siw_query_gid(struct ib_device * base_dev,u32 port,int idx,union ib_gid * gid)228 int siw_query_gid(struct ib_device *base_dev, u32 port, int idx,
229 		  union ib_gid *gid)
230 {
231 	struct siw_device *sdev = to_siw_dev(base_dev);
232 
233 	/* subnet_prefix == interface_id == 0; */
234 	memset(gid, 0, sizeof(*gid));
235 	memcpy(gid->raw, sdev->raw_gid, ETH_ALEN);
236 
237 	return 0;
238 }
239 
siw_alloc_pd(struct ib_pd * pd,struct ib_udata * udata)240 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
241 {
242 	struct siw_device *sdev = to_siw_dev(pd->device);
243 
244 	if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
245 		atomic_dec(&sdev->num_pd);
246 		return -ENOMEM;
247 	}
248 	siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
249 
250 	return 0;
251 }
252 
siw_dealloc_pd(struct ib_pd * pd,struct ib_udata * udata)253 int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
254 {
255 	struct siw_device *sdev = to_siw_dev(pd->device);
256 
257 	siw_dbg_pd(pd, "free PD\n");
258 	atomic_dec(&sdev->num_pd);
259 	return 0;
260 }
261 
siw_qp_get_ref(struct ib_qp * base_qp)262 void siw_qp_get_ref(struct ib_qp *base_qp)
263 {
264 	siw_qp_get(to_siw_qp(base_qp));
265 }
266 
siw_qp_put_ref(struct ib_qp * base_qp)267 void siw_qp_put_ref(struct ib_qp *base_qp)
268 {
269 	siw_qp_put(to_siw_qp(base_qp));
270 }
271 
272 static struct rdma_user_mmap_entry *
siw_mmap_entry_insert(struct siw_ucontext * uctx,void * address,size_t length,u64 * offset)273 siw_mmap_entry_insert(struct siw_ucontext *uctx,
274 		      void *address, size_t length,
275 		      u64 *offset)
276 {
277 	struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
278 	int rv;
279 
280 	*offset = SIW_INVAL_UOBJ_KEY;
281 	if (!entry)
282 		return NULL;
283 
284 	entry->address = address;
285 
286 	rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
287 					 &entry->rdma_entry,
288 					 length);
289 	if (rv) {
290 		kfree(entry);
291 		return NULL;
292 	}
293 
294 	*offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
295 
296 	return &entry->rdma_entry;
297 }
298 
299 /*
300  * siw_create_qp()
301  *
302  * Create QP of requested size on given device.
303  *
304  * @qp:		Queue pait
305  * @attrs:	Initial QP attributes.
306  * @udata:	used to provide QP ID, SQ and RQ size back to user.
307  */
308 
siw_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)309 int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
310 		  struct ib_udata *udata)
311 {
312 	struct ib_pd *pd = ibqp->pd;
313 	struct siw_qp *qp = to_siw_qp(ibqp);
314 	struct ib_device *base_dev = pd->device;
315 	struct siw_device *sdev = to_siw_dev(base_dev);
316 	struct siw_ucontext *uctx =
317 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
318 					  base_ucontext);
319 	unsigned long flags;
320 	int num_sqe, num_rqe, rv = 0;
321 	size_t length;
322 
323 	siw_dbg(base_dev, "create new QP\n");
324 
325 	if (attrs->create_flags)
326 		return -EOPNOTSUPP;
327 
328 	if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
329 		siw_dbg(base_dev, "too many QP's\n");
330 		rv = -ENOMEM;
331 		goto err_atomic;
332 	}
333 	if (attrs->qp_type != IB_QPT_RC) {
334 		siw_dbg(base_dev, "only RC QP's supported\n");
335 		rv = -EOPNOTSUPP;
336 		goto err_atomic;
337 	}
338 	if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
339 	    (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
340 	    (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
341 	    (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
342 		siw_dbg(base_dev, "QP size error\n");
343 		rv = -EINVAL;
344 		goto err_atomic;
345 	}
346 	if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
347 		siw_dbg(base_dev, "max inline send: %d > %d\n",
348 			attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
349 		rv = -EINVAL;
350 		goto err_atomic;
351 	}
352 	/*
353 	 * NOTE: we don't allow for a QP unable to hold any SQ WQE
354 	 */
355 	if (attrs->cap.max_send_wr == 0) {
356 		siw_dbg(base_dev, "QP must have send queue\n");
357 		rv = -EINVAL;
358 		goto err_atomic;
359 	}
360 
361 	if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) {
362 		siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
363 		rv = -EINVAL;
364 		goto err_atomic;
365 	}
366 
367 	init_rwsem(&qp->state_lock);
368 	spin_lock_init(&qp->sq_lock);
369 	spin_lock_init(&qp->rq_lock);
370 	spin_lock_init(&qp->orq_lock);
371 
372 	rv = siw_qp_add(sdev, qp);
373 	if (rv)
374 		goto err_atomic;
375 
376 
377 	/* All queue indices are derived from modulo operations
378 	 * on a free running 'get' (consumer) and 'put' (producer)
379 	 * unsigned counter. Having queue sizes at power of two
380 	 * avoids handling counter wrap around.
381 	 */
382 	num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr);
383 	num_rqe = attrs->cap.max_recv_wr;
384 	if (num_rqe)
385 		num_rqe = roundup_pow_of_two(num_rqe);
386 
387 	if (udata)
388 		qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
389 	else
390 		qp->sendq = vcalloc(num_sqe, sizeof(struct siw_sqe));
391 
392 	if (qp->sendq == NULL) {
393 		rv = -ENOMEM;
394 		goto err_out_xa;
395 	}
396 	if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
397 		if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
398 			qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
399 		else {
400 			rv = -EINVAL;
401 			goto err_out_xa;
402 		}
403 	}
404 	qp->pd = pd;
405 	qp->scq = to_siw_cq(attrs->send_cq);
406 	qp->rcq = to_siw_cq(attrs->recv_cq);
407 
408 	if (attrs->srq) {
409 		/*
410 		 * SRQ support.
411 		 * Verbs 6.3.7: ignore RQ size, if SRQ present
412 		 * Verbs 6.3.5: do not check PD of SRQ against PD of QP
413 		 */
414 		qp->srq = to_siw_srq(attrs->srq);
415 		qp->attrs.rq_size = 0;
416 		siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
417 			qp->base_qp.qp_num);
418 	} else if (num_rqe) {
419 		if (udata)
420 			qp->recvq =
421 				vmalloc_user(num_rqe * sizeof(struct siw_rqe));
422 		else
423 			qp->recvq = vcalloc(num_rqe, sizeof(struct siw_rqe));
424 
425 		if (qp->recvq == NULL) {
426 			rv = -ENOMEM;
427 			goto err_out_xa;
428 		}
429 		qp->attrs.rq_size = num_rqe;
430 	}
431 	qp->attrs.sq_size = num_sqe;
432 	qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
433 	qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
434 
435 	/* Make those two tunables fixed for now. */
436 	qp->tx_ctx.gso_seg_limit = 1;
437 	qp->tx_ctx.zcopy_tx = zcopy_tx;
438 
439 	qp->attrs.state = SIW_QP_STATE_IDLE;
440 
441 	if (udata) {
442 		struct siw_uresp_create_qp uresp = {};
443 
444 		uresp.num_sqe = num_sqe;
445 		uresp.num_rqe = num_rqe;
446 		uresp.qp_id = qp_id(qp);
447 
448 		if (qp->sendq) {
449 			length = num_sqe * sizeof(struct siw_sqe);
450 			qp->sq_entry =
451 				siw_mmap_entry_insert(uctx, qp->sendq,
452 						      length, &uresp.sq_key);
453 			if (!qp->sq_entry) {
454 				rv = -ENOMEM;
455 				goto err_out_xa;
456 			}
457 		}
458 
459 		if (qp->recvq) {
460 			length = num_rqe * sizeof(struct siw_rqe);
461 			qp->rq_entry =
462 				siw_mmap_entry_insert(uctx, qp->recvq,
463 						      length, &uresp.rq_key);
464 			if (!qp->rq_entry) {
465 				uresp.sq_key = SIW_INVAL_UOBJ_KEY;
466 				rv = -ENOMEM;
467 				goto err_out_xa;
468 			}
469 		}
470 
471 		if (udata->outlen < sizeof(uresp)) {
472 			rv = -EINVAL;
473 			goto err_out_xa;
474 		}
475 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
476 		if (rv)
477 			goto err_out_xa;
478 	}
479 	qp->tx_cpu = siw_get_tx_cpu(sdev);
480 	if (qp->tx_cpu < 0) {
481 		rv = -EINVAL;
482 		goto err_out_xa;
483 	}
484 	INIT_LIST_HEAD(&qp->devq);
485 	spin_lock_irqsave(&sdev->lock, flags);
486 	list_add_tail(&qp->devq, &sdev->qp_list);
487 	spin_unlock_irqrestore(&sdev->lock, flags);
488 
489 	init_completion(&qp->qp_free);
490 
491 	return 0;
492 
493 err_out_xa:
494 	xa_erase(&sdev->qp_xa, qp_id(qp));
495 	if (uctx) {
496 		rdma_user_mmap_entry_remove(qp->sq_entry);
497 		rdma_user_mmap_entry_remove(qp->rq_entry);
498 	}
499 	vfree(qp->sendq);
500 	vfree(qp->recvq);
501 
502 err_atomic:
503 	atomic_dec(&sdev->num_qp);
504 	return rv;
505 }
506 
507 /*
508  * Minimum siw_query_qp() verb interface.
509  *
510  * @qp_attr_mask is not used but all available information is provided
511  */
siw_query_qp(struct ib_qp * base_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)512 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
513 		 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
514 {
515 	struct siw_qp *qp;
516 	struct net_device *ndev;
517 
518 	if (base_qp && qp_attr && qp_init_attr)
519 		qp = to_siw_qp(base_qp);
520 	else
521 		return -EINVAL;
522 
523 	ndev = ib_device_get_netdev(base_qp->device, SIW_PORT);
524 	if (!ndev)
525 		return -ENODEV;
526 
527 	qp_attr->qp_state = siw_qp_state_to_ib_qp_state[qp->attrs.state];
528 	qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
529 	qp_attr->cap.max_send_wr = qp->attrs.sq_size;
530 	qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
531 	qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
532 	qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
533 	qp_attr->path_mtu = ib_mtu_int_to_enum(READ_ONCE(ndev->mtu));
534 	qp_attr->max_rd_atomic = qp->attrs.irq_size;
535 	qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
536 
537 	qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
538 				   IB_ACCESS_REMOTE_WRITE |
539 				   IB_ACCESS_REMOTE_READ;
540 
541 	qp_init_attr->qp_type = base_qp->qp_type;
542 	qp_init_attr->send_cq = base_qp->send_cq;
543 	qp_init_attr->recv_cq = base_qp->recv_cq;
544 	qp_init_attr->srq = base_qp->srq;
545 
546 	qp_init_attr->cap = qp_attr->cap;
547 
548 	dev_put(ndev);
549 	return 0;
550 }
551 
siw_verbs_modify_qp(struct ib_qp * base_qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)552 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
553 			int attr_mask, struct ib_udata *udata)
554 {
555 	struct siw_qp_attrs new_attrs;
556 	enum siw_qp_attr_mask siw_attr_mask = 0;
557 	struct siw_qp *qp = to_siw_qp(base_qp);
558 	int rv = 0;
559 
560 	if (!attr_mask)
561 		return 0;
562 
563 	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
564 		return -EOPNOTSUPP;
565 
566 	memset(&new_attrs, 0, sizeof(new_attrs));
567 
568 	if (attr_mask & IB_QP_ACCESS_FLAGS) {
569 		siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
570 
571 		if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
572 			new_attrs.flags |= SIW_RDMA_READ_ENABLED;
573 		if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
574 			new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
575 		if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
576 			new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
577 	}
578 	if (attr_mask & IB_QP_STATE) {
579 		siw_dbg_qp(qp, "desired IB QP state: %s\n",
580 			   ib_qp_state_to_string[attr->qp_state]);
581 
582 		new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
583 
584 		if (new_attrs.state > SIW_QP_STATE_RTS)
585 			qp->tx_ctx.tx_suspend = 1;
586 
587 		siw_attr_mask |= SIW_QP_ATTR_STATE;
588 	}
589 	if (!siw_attr_mask)
590 		goto out;
591 
592 	down_write(&qp->state_lock);
593 
594 	rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
595 
596 	up_write(&qp->state_lock);
597 out:
598 	return rv;
599 }
600 
siw_destroy_qp(struct ib_qp * base_qp,struct ib_udata * udata)601 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
602 {
603 	struct siw_qp *qp = to_siw_qp(base_qp);
604 	struct siw_ucontext *uctx =
605 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
606 					  base_ucontext);
607 	struct siw_qp_attrs qp_attrs;
608 
609 	siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
610 
611 	/*
612 	 * Mark QP as in process of destruction to prevent from
613 	 * any async callbacks to RDMA core
614 	 */
615 	qp->attrs.flags |= SIW_QP_IN_DESTROY;
616 	qp->rx_stream.rx_suspend = 1;
617 
618 	if (uctx) {
619 		rdma_user_mmap_entry_remove(qp->sq_entry);
620 		rdma_user_mmap_entry_remove(qp->rq_entry);
621 	}
622 
623 	down_write(&qp->state_lock);
624 
625 	qp_attrs.state = SIW_QP_STATE_ERROR;
626 	siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
627 
628 	if (qp->cep) {
629 		siw_cep_put(qp->cep);
630 		qp->cep = NULL;
631 	}
632 	up_write(&qp->state_lock);
633 
634 	kfree(qp->tx_ctx.mpa_crc_hd);
635 	kfree(qp->rx_stream.mpa_crc_hd);
636 
637 	qp->scq = qp->rcq = NULL;
638 
639 	siw_qp_put(qp);
640 	wait_for_completion(&qp->qp_free);
641 
642 	return 0;
643 }
644 
645 /*
646  * siw_copy_inline_sgl()
647  *
648  * Prepare sgl of inlined data for sending. For userland callers
649  * function checks if given buffer addresses and len's are within
650  * process context bounds.
651  * Data from all provided sge's are copied together into the wqe,
652  * referenced by a single sge.
653  */
siw_copy_inline_sgl(const struct ib_send_wr * core_wr,struct siw_sqe * sqe)654 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
655 			       struct siw_sqe *sqe)
656 {
657 	struct ib_sge *core_sge = core_wr->sg_list;
658 	void *kbuf = &sqe->sge[1];
659 	int num_sge = core_wr->num_sge, bytes = 0;
660 
661 	sqe->sge[0].laddr = (uintptr_t)kbuf;
662 	sqe->sge[0].lkey = 0;
663 
664 	while (num_sge--) {
665 		if (!core_sge->length) {
666 			core_sge++;
667 			continue;
668 		}
669 		bytes += core_sge->length;
670 		if (bytes > SIW_MAX_INLINE) {
671 			bytes = -EINVAL;
672 			break;
673 		}
674 		memcpy(kbuf, ib_virt_dma_to_ptr(core_sge->addr),
675 		       core_sge->length);
676 
677 		kbuf += core_sge->length;
678 		core_sge++;
679 	}
680 	sqe->sge[0].length = max(bytes, 0);
681 	sqe->num_sge = bytes > 0 ? 1 : 0;
682 
683 	return bytes;
684 }
685 
686 /* Complete SQ WR's without processing */
siw_sq_flush_wr(struct siw_qp * qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)687 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
688 			   const struct ib_send_wr **bad_wr)
689 {
690 	int rv = 0;
691 
692 	while (wr) {
693 		struct siw_sqe sqe = {};
694 
695 		switch (wr->opcode) {
696 		case IB_WR_RDMA_WRITE:
697 			sqe.opcode = SIW_OP_WRITE;
698 			break;
699 		case IB_WR_RDMA_READ:
700 			sqe.opcode = SIW_OP_READ;
701 			break;
702 		case IB_WR_RDMA_READ_WITH_INV:
703 			sqe.opcode = SIW_OP_READ_LOCAL_INV;
704 			break;
705 		case IB_WR_SEND:
706 			sqe.opcode = SIW_OP_SEND;
707 			break;
708 		case IB_WR_SEND_WITH_IMM:
709 			sqe.opcode = SIW_OP_SEND_WITH_IMM;
710 			break;
711 		case IB_WR_SEND_WITH_INV:
712 			sqe.opcode = SIW_OP_SEND_REMOTE_INV;
713 			break;
714 		case IB_WR_LOCAL_INV:
715 			sqe.opcode = SIW_OP_INVAL_STAG;
716 			break;
717 		case IB_WR_REG_MR:
718 			sqe.opcode = SIW_OP_REG_MR;
719 			break;
720 		default:
721 			rv = -EINVAL;
722 			break;
723 		}
724 		if (!rv) {
725 			sqe.id = wr->wr_id;
726 			rv = siw_sqe_complete(qp, &sqe, 0,
727 					      SIW_WC_WR_FLUSH_ERR);
728 		}
729 		if (rv) {
730 			if (bad_wr)
731 				*bad_wr = wr;
732 			break;
733 		}
734 		wr = wr->next;
735 	}
736 	return rv;
737 }
738 
739 /* Complete RQ WR's without processing */
siw_rq_flush_wr(struct siw_qp * qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)740 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
741 			   const struct ib_recv_wr **bad_wr)
742 {
743 	struct siw_rqe rqe = {};
744 	int rv = 0;
745 
746 	while (wr) {
747 		rqe.id = wr->wr_id;
748 		rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
749 		if (rv) {
750 			if (bad_wr)
751 				*bad_wr = wr;
752 			break;
753 		}
754 		wr = wr->next;
755 	}
756 	return rv;
757 }
758 
759 /*
760  * siw_post_send()
761  *
762  * Post a list of S-WR's to a SQ.
763  *
764  * @base_qp:	Base QP contained in siw QP
765  * @wr:		Null terminated list of user WR's
766  * @bad_wr:	Points to failing WR in case of synchronous failure.
767  */
siw_post_send(struct ib_qp * base_qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)768 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
769 		  const struct ib_send_wr **bad_wr)
770 {
771 	struct siw_qp *qp = to_siw_qp(base_qp);
772 	struct siw_wqe *wqe = tx_wqe(qp);
773 
774 	unsigned long flags;
775 	int rv = 0;
776 
777 	if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
778 		siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
779 		*bad_wr = wr;
780 		return -EINVAL;
781 	}
782 
783 	/*
784 	 * Try to acquire QP state lock. Must be non-blocking
785 	 * to accommodate kernel clients needs.
786 	 */
787 	if (!down_read_trylock(&qp->state_lock)) {
788 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
789 			/*
790 			 * ERROR state is final, so we can be sure
791 			 * this state will not change as long as the QP
792 			 * exists.
793 			 *
794 			 * This handles an ib_drain_sq() call with
795 			 * a concurrent request to set the QP state
796 			 * to ERROR.
797 			 */
798 			rv = siw_sq_flush_wr(qp, wr, bad_wr);
799 		} else {
800 			siw_dbg_qp(qp, "QP locked, state %d\n",
801 				   qp->attrs.state);
802 			*bad_wr = wr;
803 			rv = -ENOTCONN;
804 		}
805 		return rv;
806 	}
807 	if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
808 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
809 			/*
810 			 * Immediately flush this WR to CQ, if QP
811 			 * is in ERROR state. SQ is guaranteed to
812 			 * be empty, so WR complets in-order.
813 			 *
814 			 * Typically triggered by ib_drain_sq().
815 			 */
816 			rv = siw_sq_flush_wr(qp, wr, bad_wr);
817 		} else {
818 			siw_dbg_qp(qp, "QP out of state %d\n",
819 				   qp->attrs.state);
820 			*bad_wr = wr;
821 			rv = -ENOTCONN;
822 		}
823 		up_read(&qp->state_lock);
824 		return rv;
825 	}
826 	spin_lock_irqsave(&qp->sq_lock, flags);
827 
828 	while (wr) {
829 		u32 idx = qp->sq_put % qp->attrs.sq_size;
830 		struct siw_sqe *sqe = &qp->sendq[idx];
831 
832 		if (sqe->flags) {
833 			siw_dbg_qp(qp, "sq full\n");
834 			rv = -ENOMEM;
835 			break;
836 		}
837 		if (wr->num_sge > qp->attrs.sq_max_sges) {
838 			siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
839 			rv = -EINVAL;
840 			break;
841 		}
842 		sqe->id = wr->wr_id;
843 
844 		if ((wr->send_flags & IB_SEND_SIGNALED) ||
845 		    (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
846 			sqe->flags |= SIW_WQE_SIGNALLED;
847 
848 		if (wr->send_flags & IB_SEND_FENCE)
849 			sqe->flags |= SIW_WQE_READ_FENCE;
850 
851 		switch (wr->opcode) {
852 		case IB_WR_SEND:
853 		case IB_WR_SEND_WITH_INV:
854 			if (wr->send_flags & IB_SEND_SOLICITED)
855 				sqe->flags |= SIW_WQE_SOLICITED;
856 
857 			if (!(wr->send_flags & IB_SEND_INLINE)) {
858 				siw_copy_sgl(wr->sg_list, sqe->sge,
859 					     wr->num_sge);
860 				sqe->num_sge = wr->num_sge;
861 			} else {
862 				rv = siw_copy_inline_sgl(wr, sqe);
863 				if (rv <= 0) {
864 					rv = -EINVAL;
865 					break;
866 				}
867 				sqe->flags |= SIW_WQE_INLINE;
868 				sqe->num_sge = 1;
869 			}
870 			if (wr->opcode == IB_WR_SEND)
871 				sqe->opcode = SIW_OP_SEND;
872 			else {
873 				sqe->opcode = SIW_OP_SEND_REMOTE_INV;
874 				sqe->rkey = wr->ex.invalidate_rkey;
875 			}
876 			break;
877 
878 		case IB_WR_RDMA_READ_WITH_INV:
879 		case IB_WR_RDMA_READ:
880 			/*
881 			 * iWarp restricts RREAD sink to SGL containing
882 			 * 1 SGE only. we could relax to SGL with multiple
883 			 * elements referring the SAME ltag or even sending
884 			 * a private per-rreq tag referring to a checked
885 			 * local sgl with MULTIPLE ltag's.
886 			 */
887 			if (unlikely(wr->num_sge != 1)) {
888 				rv = -EINVAL;
889 				break;
890 			}
891 			siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
892 			/*
893 			 * NOTE: zero length RREAD is allowed!
894 			 */
895 			sqe->raddr = rdma_wr(wr)->remote_addr;
896 			sqe->rkey = rdma_wr(wr)->rkey;
897 			sqe->num_sge = 1;
898 
899 			if (wr->opcode == IB_WR_RDMA_READ)
900 				sqe->opcode = SIW_OP_READ;
901 			else
902 				sqe->opcode = SIW_OP_READ_LOCAL_INV;
903 			break;
904 
905 		case IB_WR_RDMA_WRITE:
906 			if (!(wr->send_flags & IB_SEND_INLINE)) {
907 				siw_copy_sgl(wr->sg_list, &sqe->sge[0],
908 					     wr->num_sge);
909 				sqe->num_sge = wr->num_sge;
910 			} else {
911 				rv = siw_copy_inline_sgl(wr, sqe);
912 				if (unlikely(rv < 0)) {
913 					rv = -EINVAL;
914 					break;
915 				}
916 				sqe->flags |= SIW_WQE_INLINE;
917 				sqe->num_sge = 1;
918 			}
919 			sqe->raddr = rdma_wr(wr)->remote_addr;
920 			sqe->rkey = rdma_wr(wr)->rkey;
921 			sqe->opcode = SIW_OP_WRITE;
922 			break;
923 
924 		case IB_WR_REG_MR:
925 			sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
926 			sqe->rkey = reg_wr(wr)->key;
927 			sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
928 			sqe->opcode = SIW_OP_REG_MR;
929 			break;
930 
931 		case IB_WR_LOCAL_INV:
932 			sqe->rkey = wr->ex.invalidate_rkey;
933 			sqe->opcode = SIW_OP_INVAL_STAG;
934 			break;
935 
936 		default:
937 			siw_dbg_qp(qp, "ib wr type %d unsupported\n",
938 				   wr->opcode);
939 			rv = -EINVAL;
940 			break;
941 		}
942 		siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
943 			   sqe->opcode, sqe->flags,
944 			   (void *)(uintptr_t)sqe->id);
945 
946 		if (unlikely(rv < 0))
947 			break;
948 
949 		/* make SQE only valid after completely written */
950 		smp_wmb();
951 		sqe->flags |= SIW_WQE_VALID;
952 
953 		qp->sq_put++;
954 		wr = wr->next;
955 	}
956 
957 	/*
958 	 * Send directly if SQ processing is not in progress.
959 	 * Eventual immediate errors (rv < 0) do not affect the involved
960 	 * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
961 	 * processing, if new work is already pending. But rv must be passed
962 	 * to caller.
963 	 */
964 	if (wqe->wr_status != SIW_WR_IDLE) {
965 		spin_unlock_irqrestore(&qp->sq_lock, flags);
966 		goto skip_direct_sending;
967 	}
968 	rv = siw_activate_tx(qp);
969 	spin_unlock_irqrestore(&qp->sq_lock, flags);
970 
971 	if (rv <= 0)
972 		goto skip_direct_sending;
973 
974 	if (rdma_is_kernel_res(&qp->base_qp.res)) {
975 		rv = siw_sq_start(qp);
976 	} else {
977 		qp->tx_ctx.in_syscall = 1;
978 
979 		if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
980 			siw_qp_cm_drop(qp, 0);
981 
982 		qp->tx_ctx.in_syscall = 0;
983 	}
984 skip_direct_sending:
985 
986 	up_read(&qp->state_lock);
987 
988 	if (rv >= 0)
989 		return 0;
990 	/*
991 	 * Immediate error
992 	 */
993 	siw_dbg_qp(qp, "error %d\n", rv);
994 
995 	*bad_wr = wr;
996 	return rv;
997 }
998 
999 /*
1000  * siw_post_receive()
1001  *
1002  * Post a list of R-WR's to a RQ.
1003  *
1004  * @base_qp:	Base QP contained in siw QP
1005  * @wr:		Null terminated list of user WR's
1006  * @bad_wr:	Points to failing WR in case of synchronous failure.
1007  */
siw_post_receive(struct ib_qp * base_qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)1008 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
1009 		     const struct ib_recv_wr **bad_wr)
1010 {
1011 	struct siw_qp *qp = to_siw_qp(base_qp);
1012 	unsigned long flags;
1013 	int rv = 0;
1014 
1015 	if (qp->srq || qp->attrs.rq_size == 0) {
1016 		*bad_wr = wr;
1017 		return -EINVAL;
1018 	}
1019 	if (!rdma_is_kernel_res(&qp->base_qp.res)) {
1020 		siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
1021 		*bad_wr = wr;
1022 		return -EINVAL;
1023 	}
1024 
1025 	/*
1026 	 * Try to acquire QP state lock. Must be non-blocking
1027 	 * to accommodate kernel clients needs.
1028 	 */
1029 	if (!down_read_trylock(&qp->state_lock)) {
1030 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1031 			/*
1032 			 * ERROR state is final, so we can be sure
1033 			 * this state will not change as long as the QP
1034 			 * exists.
1035 			 *
1036 			 * This handles an ib_drain_rq() call with
1037 			 * a concurrent request to set the QP state
1038 			 * to ERROR.
1039 			 */
1040 			rv = siw_rq_flush_wr(qp, wr, bad_wr);
1041 		} else {
1042 			siw_dbg_qp(qp, "QP locked, state %d\n",
1043 				   qp->attrs.state);
1044 			*bad_wr = wr;
1045 			rv = -ENOTCONN;
1046 		}
1047 		return rv;
1048 	}
1049 	if (qp->attrs.state > SIW_QP_STATE_RTS) {
1050 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1051 			/*
1052 			 * Immediately flush this WR to CQ, if QP
1053 			 * is in ERROR state. RQ is guaranteed to
1054 			 * be empty, so WR complets in-order.
1055 			 *
1056 			 * Typically triggered by ib_drain_rq().
1057 			 */
1058 			rv = siw_rq_flush_wr(qp, wr, bad_wr);
1059 		} else {
1060 			siw_dbg_qp(qp, "QP out of state %d\n",
1061 				   qp->attrs.state);
1062 			*bad_wr = wr;
1063 			rv = -ENOTCONN;
1064 		}
1065 		up_read(&qp->state_lock);
1066 		return rv;
1067 	}
1068 	/*
1069 	 * Serialize potentially multiple producers.
1070 	 * Not needed for single threaded consumer side.
1071 	 */
1072 	spin_lock_irqsave(&qp->rq_lock, flags);
1073 
1074 	while (wr) {
1075 		u32 idx = qp->rq_put % qp->attrs.rq_size;
1076 		struct siw_rqe *rqe = &qp->recvq[idx];
1077 
1078 		if (rqe->flags) {
1079 			siw_dbg_qp(qp, "RQ full\n");
1080 			rv = -ENOMEM;
1081 			break;
1082 		}
1083 		if (wr->num_sge > qp->attrs.rq_max_sges) {
1084 			siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1085 			rv = -EINVAL;
1086 			break;
1087 		}
1088 		rqe->id = wr->wr_id;
1089 		rqe->num_sge = wr->num_sge;
1090 		siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1091 
1092 		/* make sure RQE is completely written before valid */
1093 		smp_wmb();
1094 
1095 		rqe->flags = SIW_WQE_VALID;
1096 
1097 		qp->rq_put++;
1098 		wr = wr->next;
1099 	}
1100 	spin_unlock_irqrestore(&qp->rq_lock, flags);
1101 
1102 	up_read(&qp->state_lock);
1103 
1104 	if (rv < 0) {
1105 		siw_dbg_qp(qp, "error %d\n", rv);
1106 		*bad_wr = wr;
1107 	}
1108 	return rv > 0 ? 0 : rv;
1109 }
1110 
siw_destroy_cq(struct ib_cq * base_cq,struct ib_udata * udata)1111 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1112 {
1113 	struct siw_cq *cq = to_siw_cq(base_cq);
1114 	struct siw_device *sdev = to_siw_dev(base_cq->device);
1115 	struct siw_ucontext *ctx =
1116 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1117 					  base_ucontext);
1118 
1119 	siw_dbg_cq(cq, "free CQ resources\n");
1120 
1121 	siw_cq_flush(cq);
1122 
1123 	if (ctx)
1124 		rdma_user_mmap_entry_remove(cq->cq_entry);
1125 
1126 	atomic_dec(&sdev->num_cq);
1127 
1128 	vfree(cq->queue);
1129 	return 0;
1130 }
1131 
1132 /*
1133  * siw_create_cq()
1134  *
1135  * Populate CQ of requested size
1136  *
1137  * @base_cq: CQ as allocated by RDMA midlayer
1138  * @attr: Initial CQ attributes
1139  * @attrs: uverbs bundle
1140  */
1141 
siw_create_cq(struct ib_cq * base_cq,const struct ib_cq_init_attr * attr,struct uverbs_attr_bundle * attrs)1142 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1143 		  struct uverbs_attr_bundle *attrs)
1144 {
1145 	struct ib_udata *udata = &attrs->driver_udata;
1146 	struct siw_device *sdev = to_siw_dev(base_cq->device);
1147 	struct siw_cq *cq = to_siw_cq(base_cq);
1148 	int rv, size = attr->cqe;
1149 
1150 	if (attr->flags)
1151 		return -EOPNOTSUPP;
1152 
1153 	if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1154 		siw_dbg(base_cq->device, "too many CQ's\n");
1155 		rv = -ENOMEM;
1156 		goto err_out;
1157 	}
1158 	if (size < 1 || size > sdev->attrs.max_cqe) {
1159 		siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1160 		rv = -EINVAL;
1161 		goto err_out;
1162 	}
1163 	size = roundup_pow_of_two(size);
1164 	cq->base_cq.cqe = size;
1165 	cq->num_cqe = size;
1166 
1167 	if (udata)
1168 		cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1169 					 sizeof(struct siw_cq_ctrl));
1170 	else
1171 		cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1172 				    sizeof(struct siw_cq_ctrl));
1173 
1174 	if (cq->queue == NULL) {
1175 		rv = -ENOMEM;
1176 		goto err_out;
1177 	}
1178 	get_random_bytes(&cq->id, 4);
1179 	siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1180 
1181 	spin_lock_init(&cq->lock);
1182 
1183 	cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1184 
1185 	if (udata) {
1186 		struct siw_uresp_create_cq uresp = {};
1187 		struct siw_ucontext *ctx =
1188 			rdma_udata_to_drv_context(udata, struct siw_ucontext,
1189 						  base_ucontext);
1190 		size_t length = size * sizeof(struct siw_cqe) +
1191 			sizeof(struct siw_cq_ctrl);
1192 
1193 		cq->cq_entry =
1194 			siw_mmap_entry_insert(ctx, cq->queue,
1195 					      length, &uresp.cq_key);
1196 		if (!cq->cq_entry) {
1197 			rv = -ENOMEM;
1198 			goto err_out;
1199 		}
1200 
1201 		uresp.cq_id = cq->id;
1202 		uresp.num_cqe = size;
1203 
1204 		if (udata->outlen < sizeof(uresp)) {
1205 			rv = -EINVAL;
1206 			goto err_out;
1207 		}
1208 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1209 		if (rv)
1210 			goto err_out;
1211 	}
1212 	return 0;
1213 
1214 err_out:
1215 	siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1216 
1217 	if (cq->queue) {
1218 		struct siw_ucontext *ctx =
1219 			rdma_udata_to_drv_context(udata, struct siw_ucontext,
1220 						  base_ucontext);
1221 		if (ctx)
1222 			rdma_user_mmap_entry_remove(cq->cq_entry);
1223 		vfree(cq->queue);
1224 	}
1225 	atomic_dec(&sdev->num_cq);
1226 
1227 	return rv;
1228 }
1229 
1230 /*
1231  * siw_poll_cq()
1232  *
1233  * Reap CQ entries if available and copy work completion status into
1234  * array of WC's provided by caller. Returns number of reaped CQE's.
1235  *
1236  * @base_cq:	Base CQ contained in siw CQ.
1237  * @num_cqe:	Maximum number of CQE's to reap.
1238  * @wc:		Array of work completions to be filled by siw.
1239  */
siw_poll_cq(struct ib_cq * base_cq,int num_cqe,struct ib_wc * wc)1240 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1241 {
1242 	struct siw_cq *cq = to_siw_cq(base_cq);
1243 	int i;
1244 
1245 	for (i = 0; i < num_cqe; i++) {
1246 		if (!siw_reap_cqe(cq, wc))
1247 			break;
1248 		wc++;
1249 	}
1250 	return i;
1251 }
1252 
1253 /*
1254  * siw_req_notify_cq()
1255  *
1256  * Request notification for new CQE's added to that CQ.
1257  * Defined flags:
1258  * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1259  *   event if a WQE with notification flag set enters the CQ
1260  * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1261  *   event if a WQE enters the CQ.
1262  * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1263  *   number of not reaped CQE's regardless of its notification
1264  *   type and current or new CQ notification settings.
1265  *
1266  * @base_cq:	Base CQ contained in siw CQ.
1267  * @flags:	Requested notification flags.
1268  */
siw_req_notify_cq(struct ib_cq * base_cq,enum ib_cq_notify_flags flags)1269 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1270 {
1271 	struct siw_cq *cq = to_siw_cq(base_cq);
1272 
1273 	siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1274 
1275 	if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1276 		/*
1277 		 * Enable CQ event for next solicited completion.
1278 		 * and make it visible to all associated producers.
1279 		 */
1280 		smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1281 	else
1282 		/*
1283 		 * Enable CQ event for any signalled completion.
1284 		 * and make it visible to all associated producers.
1285 		 */
1286 		smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1287 
1288 	if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1289 		return cq->cq_put - cq->cq_get;
1290 
1291 	return 0;
1292 }
1293 
1294 /*
1295  * siw_dereg_mr()
1296  *
1297  * Release Memory Region.
1298  *
1299  * @base_mr: Base MR contained in siw MR.
1300  * @udata: points to user context, unused.
1301  */
siw_dereg_mr(struct ib_mr * base_mr,struct ib_udata * udata)1302 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1303 {
1304 	struct siw_mr *mr = to_siw_mr(base_mr);
1305 	struct siw_device *sdev = to_siw_dev(base_mr->device);
1306 
1307 	siw_dbg_mem(mr->mem, "deregister MR\n");
1308 
1309 	atomic_dec(&sdev->num_mr);
1310 
1311 	siw_mr_drop_mem(mr);
1312 	kfree_rcu(mr, rcu);
1313 
1314 	return 0;
1315 }
1316 
1317 /*
1318  * siw_reg_user_mr()
1319  *
1320  * Register Memory Region.
1321  *
1322  * @pd:		Protection Domain
1323  * @start:	starting address of MR (virtual address)
1324  * @len:	len of MR
1325  * @rnic_va:	not used by siw
1326  * @rights:	MR access rights
1327  * @udata:	user buffer to communicate STag and Key.
1328  */
siw_reg_user_mr(struct ib_pd * pd,u64 start,u64 len,u64 rnic_va,int rights,struct ib_udata * udata)1329 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1330 			      u64 rnic_va, int rights, struct ib_udata *udata)
1331 {
1332 	struct siw_mr *mr = NULL;
1333 	struct siw_umem *umem = NULL;
1334 	struct siw_ureq_reg_mr ureq;
1335 	struct siw_device *sdev = to_siw_dev(pd->device);
1336 	int rv;
1337 
1338 	siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1339 		   (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1340 		   (unsigned long long)len);
1341 
1342 	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1343 		siw_dbg_pd(pd, "too many mr's\n");
1344 		rv = -ENOMEM;
1345 		goto err_out;
1346 	}
1347 	if (!len) {
1348 		rv = -EINVAL;
1349 		goto err_out;
1350 	}
1351 	umem = siw_umem_get(pd->device, start, len, rights);
1352 	if (IS_ERR(umem)) {
1353 		rv = PTR_ERR(umem);
1354 		siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1355 		umem = NULL;
1356 		goto err_out;
1357 	}
1358 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1359 	if (!mr) {
1360 		rv = -ENOMEM;
1361 		goto err_out;
1362 	}
1363 	rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1364 	if (rv)
1365 		goto err_out;
1366 
1367 	if (udata) {
1368 		struct siw_uresp_reg_mr uresp = {};
1369 		struct siw_mem *mem = mr->mem;
1370 
1371 		if (udata->inlen < sizeof(ureq)) {
1372 			rv = -EINVAL;
1373 			goto err_out;
1374 		}
1375 		rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1376 		if (rv)
1377 			goto err_out;
1378 
1379 		mr->base_mr.lkey |= ureq.stag_key;
1380 		mr->base_mr.rkey |= ureq.stag_key;
1381 		mem->stag |= ureq.stag_key;
1382 		uresp.stag = mem->stag;
1383 
1384 		if (udata->outlen < sizeof(uresp)) {
1385 			rv = -EINVAL;
1386 			goto err_out;
1387 		}
1388 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1389 		if (rv)
1390 			goto err_out;
1391 	}
1392 	mr->mem->stag_valid = 1;
1393 
1394 	return &mr->base_mr;
1395 
1396 err_out:
1397 	atomic_dec(&sdev->num_mr);
1398 	if (mr) {
1399 		if (mr->mem)
1400 			siw_mr_drop_mem(mr);
1401 		kfree_rcu(mr, rcu);
1402 	} else {
1403 		if (umem)
1404 			siw_umem_release(umem);
1405 	}
1406 	return ERR_PTR(rv);
1407 }
1408 
siw_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_sge)1409 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1410 			   u32 max_sge)
1411 {
1412 	struct siw_device *sdev = to_siw_dev(pd->device);
1413 	struct siw_mr *mr = NULL;
1414 	struct siw_pbl *pbl = NULL;
1415 	int rv;
1416 
1417 	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1418 		siw_dbg_pd(pd, "too many mr's\n");
1419 		rv = -ENOMEM;
1420 		goto err_out;
1421 	}
1422 	if (mr_type != IB_MR_TYPE_MEM_REG) {
1423 		siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1424 		rv = -EOPNOTSUPP;
1425 		goto err_out;
1426 	}
1427 	if (max_sge > SIW_MAX_SGE_PBL) {
1428 		siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1429 		rv = -ENOMEM;
1430 		goto err_out;
1431 	}
1432 	pbl = siw_pbl_alloc(max_sge);
1433 	if (IS_ERR(pbl)) {
1434 		rv = PTR_ERR(pbl);
1435 		siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1436 		pbl = NULL;
1437 		goto err_out;
1438 	}
1439 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1440 	if (!mr) {
1441 		rv = -ENOMEM;
1442 		goto err_out;
1443 	}
1444 	rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1445 	if (rv)
1446 		goto err_out;
1447 
1448 	mr->mem->is_pbl = 1;
1449 
1450 	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1451 
1452 	return &mr->base_mr;
1453 
1454 err_out:
1455 	atomic_dec(&sdev->num_mr);
1456 
1457 	if (!mr) {
1458 		kfree(pbl);
1459 	} else {
1460 		if (mr->mem)
1461 			siw_mr_drop_mem(mr);
1462 		kfree_rcu(mr, rcu);
1463 	}
1464 	siw_dbg_pd(pd, "failed: %d\n", rv);
1465 
1466 	return ERR_PTR(rv);
1467 }
1468 
1469 /* Just used to count number of pages being mapped */
siw_set_pbl_page(struct ib_mr * base_mr,u64 buf_addr)1470 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1471 {
1472 	return 0;
1473 }
1474 
siw_map_mr_sg(struct ib_mr * base_mr,struct scatterlist * sl,int num_sle,unsigned int * sg_off)1475 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1476 		  unsigned int *sg_off)
1477 {
1478 	struct scatterlist *slp;
1479 	struct siw_mr *mr = to_siw_mr(base_mr);
1480 	struct siw_mem *mem = mr->mem;
1481 	struct siw_pbl *pbl = mem->pbl;
1482 	struct siw_pble *pble;
1483 	unsigned long pbl_size;
1484 	int i, rv;
1485 
1486 	if (!pbl) {
1487 		siw_dbg_mem(mem, "no PBL allocated\n");
1488 		return -EINVAL;
1489 	}
1490 	pble = pbl->pbe;
1491 
1492 	if (pbl->max_buf < num_sle) {
1493 		siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1494 			    num_sle, pbl->max_buf);
1495 		return -ENOMEM;
1496 	}
1497 	for_each_sg(sl, slp, num_sle, i) {
1498 		if (sg_dma_len(slp) == 0) {
1499 			siw_dbg_mem(mem, "empty SGE\n");
1500 			return -EINVAL;
1501 		}
1502 		if (i == 0) {
1503 			pble->addr = sg_dma_address(slp);
1504 			pble->size = sg_dma_len(slp);
1505 			pble->pbl_off = 0;
1506 			pbl_size = pble->size;
1507 			pbl->num_buf = 1;
1508 		} else {
1509 			/* Merge PBL entries if adjacent */
1510 			if (pble->addr + pble->size == sg_dma_address(slp)) {
1511 				pble->size += sg_dma_len(slp);
1512 			} else {
1513 				pble++;
1514 				pbl->num_buf++;
1515 				pble->addr = sg_dma_address(slp);
1516 				pble->size = sg_dma_len(slp);
1517 				pble->pbl_off = pbl_size;
1518 			}
1519 			pbl_size += sg_dma_len(slp);
1520 		}
1521 		siw_dbg_mem(mem,
1522 			"sge[%d], size %u, addr 0x%p, total %lu\n",
1523 			i, pble->size, ib_virt_dma_to_ptr(pble->addr),
1524 			pbl_size);
1525 	}
1526 	rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1527 	if (rv > 0) {
1528 		mem->len = base_mr->length;
1529 		mem->va = base_mr->iova;
1530 		siw_dbg_mem(mem,
1531 			"%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1532 			mem->len, (void *)(uintptr_t)mem->va, num_sle,
1533 			pbl->num_buf);
1534 	}
1535 	return rv;
1536 }
1537 
1538 /*
1539  * siw_get_dma_mr()
1540  *
1541  * Create a (empty) DMA memory region, where no umem is attached.
1542  */
siw_get_dma_mr(struct ib_pd * pd,int rights)1543 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1544 {
1545 	struct siw_device *sdev = to_siw_dev(pd->device);
1546 	struct siw_mr *mr = NULL;
1547 	int rv;
1548 
1549 	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1550 		siw_dbg_pd(pd, "too many mr's\n");
1551 		rv = -ENOMEM;
1552 		goto err_out;
1553 	}
1554 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1555 	if (!mr) {
1556 		rv = -ENOMEM;
1557 		goto err_out;
1558 	}
1559 	rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1560 	if (rv)
1561 		goto err_out;
1562 
1563 	mr->mem->stag_valid = 1;
1564 
1565 	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1566 
1567 	return &mr->base_mr;
1568 
1569 err_out:
1570 	if (rv)
1571 		kfree(mr);
1572 
1573 	atomic_dec(&sdev->num_mr);
1574 
1575 	return ERR_PTR(rv);
1576 }
1577 
1578 /*
1579  * siw_create_srq()
1580  *
1581  * Create Shared Receive Queue of attributes @init_attrs
1582  * within protection domain given by @pd.
1583  *
1584  * @base_srq:	Base SRQ contained in siw SRQ.
1585  * @init_attrs:	SRQ init attributes.
1586  * @udata:	points to user context
1587  */
siw_create_srq(struct ib_srq * base_srq,struct ib_srq_init_attr * init_attrs,struct ib_udata * udata)1588 int siw_create_srq(struct ib_srq *base_srq,
1589 		   struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1590 {
1591 	struct siw_srq *srq = to_siw_srq(base_srq);
1592 	struct ib_srq_attr *attrs = &init_attrs->attr;
1593 	struct siw_device *sdev = to_siw_dev(base_srq->device);
1594 	struct siw_ucontext *ctx =
1595 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1596 					  base_ucontext);
1597 	int rv;
1598 
1599 	if (init_attrs->srq_type != IB_SRQT_BASIC)
1600 		return -EOPNOTSUPP;
1601 
1602 	if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1603 		siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1604 		rv = -ENOMEM;
1605 		goto err_out;
1606 	}
1607 	if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1608 	    attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1609 		rv = -EINVAL;
1610 		goto err_out;
1611 	}
1612 	srq->max_sge = attrs->max_sge;
1613 	srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1614 	srq->limit = attrs->srq_limit;
1615 	if (srq->limit)
1616 		srq->armed = true;
1617 
1618 	srq->is_kernel_res = !udata;
1619 
1620 	if (udata)
1621 		srq->recvq =
1622 			vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1623 	else
1624 		srq->recvq = vcalloc(srq->num_rqe, sizeof(struct siw_rqe));
1625 
1626 	if (srq->recvq == NULL) {
1627 		rv = -ENOMEM;
1628 		goto err_out;
1629 	}
1630 	if (udata) {
1631 		struct siw_uresp_create_srq uresp = {};
1632 		size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1633 
1634 		srq->srq_entry =
1635 			siw_mmap_entry_insert(ctx, srq->recvq,
1636 					      length, &uresp.srq_key);
1637 		if (!srq->srq_entry) {
1638 			rv = -ENOMEM;
1639 			goto err_out;
1640 		}
1641 
1642 		uresp.num_rqe = srq->num_rqe;
1643 
1644 		if (udata->outlen < sizeof(uresp)) {
1645 			rv = -EINVAL;
1646 			goto err_out;
1647 		}
1648 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1649 		if (rv)
1650 			goto err_out;
1651 	}
1652 	spin_lock_init(&srq->lock);
1653 
1654 	siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1655 
1656 	return 0;
1657 
1658 err_out:
1659 	if (srq->recvq) {
1660 		if (ctx)
1661 			rdma_user_mmap_entry_remove(srq->srq_entry);
1662 		vfree(srq->recvq);
1663 	}
1664 	atomic_dec(&sdev->num_srq);
1665 
1666 	return rv;
1667 }
1668 
1669 /*
1670  * siw_modify_srq()
1671  *
1672  * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1673  * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1674  *
1675  * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1676  * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1677  */
siw_modify_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)1678 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1679 		   enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1680 {
1681 	struct siw_srq *srq = to_siw_srq(base_srq);
1682 	unsigned long flags;
1683 	int rv = 0;
1684 
1685 	spin_lock_irqsave(&srq->lock, flags);
1686 
1687 	if (attr_mask & IB_SRQ_MAX_WR) {
1688 		/* resize request not yet supported */
1689 		rv = -EOPNOTSUPP;
1690 		goto out;
1691 	}
1692 	if (attr_mask & IB_SRQ_LIMIT) {
1693 		if (attrs->srq_limit) {
1694 			if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1695 				rv = -EINVAL;
1696 				goto out;
1697 			}
1698 			srq->armed = true;
1699 		} else {
1700 			srq->armed = false;
1701 		}
1702 		srq->limit = attrs->srq_limit;
1703 	}
1704 out:
1705 	spin_unlock_irqrestore(&srq->lock, flags);
1706 
1707 	return rv;
1708 }
1709 
1710 /*
1711  * siw_query_srq()
1712  *
1713  * Query SRQ attributes.
1714  */
siw_query_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs)1715 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1716 {
1717 	struct siw_srq *srq = to_siw_srq(base_srq);
1718 	unsigned long flags;
1719 
1720 	spin_lock_irqsave(&srq->lock, flags);
1721 
1722 	attrs->max_wr = srq->num_rqe;
1723 	attrs->max_sge = srq->max_sge;
1724 	attrs->srq_limit = srq->limit;
1725 
1726 	spin_unlock_irqrestore(&srq->lock, flags);
1727 
1728 	return 0;
1729 }
1730 
1731 /*
1732  * siw_destroy_srq()
1733  *
1734  * Destroy SRQ.
1735  * It is assumed that the SRQ is not referenced by any
1736  * QP anymore - the code trusts the RDMA core environment to keep track
1737  * of QP references.
1738  */
siw_destroy_srq(struct ib_srq * base_srq,struct ib_udata * udata)1739 int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1740 {
1741 	struct siw_srq *srq = to_siw_srq(base_srq);
1742 	struct siw_device *sdev = to_siw_dev(base_srq->device);
1743 	struct siw_ucontext *ctx =
1744 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1745 					  base_ucontext);
1746 
1747 	if (ctx)
1748 		rdma_user_mmap_entry_remove(srq->srq_entry);
1749 	vfree(srq->recvq);
1750 	atomic_dec(&sdev->num_srq);
1751 	return 0;
1752 }
1753 
1754 /*
1755  * siw_post_srq_recv()
1756  *
1757  * Post a list of receive queue elements to SRQ.
1758  * NOTE: The function does not check or lock a certain SRQ state
1759  *       during the post operation. The code simply trusts the
1760  *       RDMA core environment.
1761  *
1762  * @base_srq:	Base SRQ contained in siw SRQ
1763  * @wr:		List of R-WR's
1764  * @bad_wr:	Updated to failing WR if posting fails.
1765  */
siw_post_srq_recv(struct ib_srq * base_srq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)1766 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1767 		      const struct ib_recv_wr **bad_wr)
1768 {
1769 	struct siw_srq *srq = to_siw_srq(base_srq);
1770 	unsigned long flags;
1771 	int rv = 0;
1772 
1773 	if (unlikely(!srq->is_kernel_res)) {
1774 		siw_dbg_pd(base_srq->pd,
1775 			   "[SRQ]: no kernel post_recv for mapped srq\n");
1776 		rv = -EINVAL;
1777 		goto out;
1778 	}
1779 	/*
1780 	 * Serialize potentially multiple producers.
1781 	 * Also needed to serialize potentially multiple
1782 	 * consumers.
1783 	 */
1784 	spin_lock_irqsave(&srq->lock, flags);
1785 
1786 	while (wr) {
1787 		u32 idx = srq->rq_put % srq->num_rqe;
1788 		struct siw_rqe *rqe = &srq->recvq[idx];
1789 
1790 		if (rqe->flags) {
1791 			siw_dbg_pd(base_srq->pd, "SRQ full\n");
1792 			rv = -ENOMEM;
1793 			break;
1794 		}
1795 		if (unlikely(wr->num_sge > srq->max_sge)) {
1796 			siw_dbg_pd(base_srq->pd,
1797 				   "[SRQ]: too many sge's: %d\n", wr->num_sge);
1798 			rv = -EINVAL;
1799 			break;
1800 		}
1801 		rqe->id = wr->wr_id;
1802 		rqe->num_sge = wr->num_sge;
1803 		siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1804 
1805 		/* Make sure S-RQE is completely written before valid */
1806 		smp_wmb();
1807 
1808 		rqe->flags = SIW_WQE_VALID;
1809 
1810 		srq->rq_put++;
1811 		wr = wr->next;
1812 	}
1813 	spin_unlock_irqrestore(&srq->lock, flags);
1814 out:
1815 	if (unlikely(rv < 0)) {
1816 		siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1817 		*bad_wr = wr;
1818 	}
1819 	return rv;
1820 }
1821 
siw_qp_event(struct siw_qp * qp,enum ib_event_type etype)1822 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1823 {
1824 	struct ib_event event;
1825 	struct ib_qp *base_qp = &qp->base_qp;
1826 
1827 	/*
1828 	 * Do not report asynchronous errors on QP which gets
1829 	 * destroyed via verbs interface (siw_destroy_qp())
1830 	 */
1831 	if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1832 		return;
1833 
1834 	event.event = etype;
1835 	event.device = base_qp->device;
1836 	event.element.qp = base_qp;
1837 
1838 	if (base_qp->event_handler) {
1839 		siw_dbg_qp(qp, "reporting event %d\n", etype);
1840 		base_qp->event_handler(&event, base_qp->qp_context);
1841 	}
1842 }
1843 
siw_cq_event(struct siw_cq * cq,enum ib_event_type etype)1844 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1845 {
1846 	struct ib_event event;
1847 	struct ib_cq *base_cq = &cq->base_cq;
1848 
1849 	event.event = etype;
1850 	event.device = base_cq->device;
1851 	event.element.cq = base_cq;
1852 
1853 	if (base_cq->event_handler) {
1854 		siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1855 		base_cq->event_handler(&event, base_cq->cq_context);
1856 	}
1857 }
1858 
siw_srq_event(struct siw_srq * srq,enum ib_event_type etype)1859 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1860 {
1861 	struct ib_event event;
1862 	struct ib_srq *base_srq = &srq->base_srq;
1863 
1864 	event.event = etype;
1865 	event.device = base_srq->device;
1866 	event.element.srq = base_srq;
1867 
1868 	if (base_srq->event_handler) {
1869 		siw_dbg_pd(srq->base_srq.pd,
1870 			   "reporting SRQ event %d\n", etype);
1871 		base_srq->event_handler(&event, base_srq->srq_context);
1872 	}
1873 }
1874 
siw_port_event(struct siw_device * sdev,u32 port,enum ib_event_type etype)1875 void siw_port_event(struct siw_device *sdev, u32 port, enum ib_event_type etype)
1876 {
1877 	struct ib_event event;
1878 
1879 	event.event = etype;
1880 	event.device = &sdev->base_dev;
1881 	event.element.port_num = port;
1882 
1883 	siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1884 
1885 	ib_dispatch_event(&event);
1886 }
1887