xref: /linux/drivers/infiniband/hw/ionic/ionic_controlpath.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */
3 
4 #include <linux/module.h>
5 #include <linux/printk.h>
6 #include <rdma/ib_addr.h>
7 #include <rdma/ib_cache.h>
8 #include <rdma/ib_user_verbs.h>
9 #include <ionic_api.h>
10 
11 #include "ionic_fw.h"
12 #include "ionic_ibdev.h"
13 
14 #define ionic_set_ecn(tos)   (((tos) | 2u) & ~1u)
15 #define ionic_clear_ecn(tos)  ((tos) & ~3u)
16 
17 static int ionic_validate_qdesc(struct ionic_qdesc *q)
18 {
19 	if (!q->addr || !q->size || !q->mask ||
20 	    !q->depth_log2 || !q->stride_log2)
21 		return -EINVAL;
22 
23 	if (q->addr & (PAGE_SIZE - 1))
24 		return -EINVAL;
25 
26 	if (q->mask != BIT(q->depth_log2) - 1)
27 		return -EINVAL;
28 
29 	if (q->size < BIT_ULL(q->depth_log2 + q->stride_log2))
30 		return -EINVAL;
31 
32 	return 0;
33 }
34 
35 static u32 ionic_get_eqid(struct ionic_ibdev *dev, u32 comp_vector, u8 udma_idx)
36 {
37 	/* EQ per vector per udma, and the first eqs reserved for async events.
38 	 * The rest of the vectors can be requested for completions.
39 	 */
40 	u32 comp_vec_count = dev->lif_cfg.eq_count / dev->lif_cfg.udma_count - 1;
41 
42 	return (comp_vector % comp_vec_count + 1) * dev->lif_cfg.udma_count + udma_idx;
43 }
44 
45 static int ionic_get_cqid(struct ionic_ibdev *dev, u32 *cqid, u8 udma_idx)
46 {
47 	unsigned int size, base, bound;
48 	int rc;
49 
50 	size = dev->lif_cfg.cq_count / dev->lif_cfg.udma_count;
51 	base = size * udma_idx;
52 	bound = base + size;
53 
54 	rc = ionic_resid_get_shared(&dev->inuse_cqid, base, bound);
55 	if (rc >= 0) {
56 		/* cq_base is zero or a multiple of two queue groups */
57 		*cqid = dev->lif_cfg.cq_base +
58 			ionic_bitid_to_qid(rc, dev->lif_cfg.udma_qgrp_shift,
59 					   dev->half_cqid_udma_shift);
60 
61 		rc = 0;
62 	}
63 
64 	return rc;
65 }
66 
67 static void ionic_put_cqid(struct ionic_ibdev *dev, u32 cqid)
68 {
69 	u32 bitid = ionic_qid_to_bitid(cqid - dev->lif_cfg.cq_base,
70 				       dev->lif_cfg.udma_qgrp_shift,
71 				       dev->half_cqid_udma_shift);
72 
73 	ionic_resid_put(&dev->inuse_cqid, bitid);
74 }
75 
76 int ionic_create_cq_common(struct ionic_vcq *vcq,
77 			   struct ionic_tbl_buf *buf,
78 			   const struct ib_cq_init_attr *attr,
79 			   struct ionic_ctx *ctx,
80 			   struct ib_udata *udata,
81 			   struct ionic_qdesc *req_cq,
82 			   int udma_idx)
83 {
84 	struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device);
85 	struct ionic_cq *cq = &vcq->cq[udma_idx];
86 	int rc;
87 
88 	cq->vcq = vcq;
89 
90 	if (attr->cqe < 1 || attr->cqe + IONIC_CQ_GRACE > 0xffff) {
91 		rc = -EINVAL;
92 		goto err_args;
93 	}
94 
95 	if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) {
96 		rc = ionic_get_cqid(dev, &cq->cqid, udma_idx);
97 		if (rc)
98 			goto err_args;
99 	}
100 
101 	cq->eqid = ionic_get_eqid(dev, attr->comp_vector, udma_idx);
102 
103 	spin_lock_init(&cq->lock);
104 	INIT_LIST_HEAD(&cq->poll_sq);
105 	INIT_LIST_HEAD(&cq->flush_sq);
106 	INIT_LIST_HEAD(&cq->flush_rq);
107 
108 	if (udata) {
109 		rc = ionic_validate_qdesc(req_cq);
110 		if (rc)
111 			goto err_qdesc;
112 
113 		cq->umem = ib_umem_get_cq_buf_or_va(&dev->ibdev, NULL,
114 						    req_cq->addr, req_cq->size,
115 						    IB_ACCESS_LOCAL_WRITE);
116 		if (IS_ERR(cq->umem)) {
117 			rc = PTR_ERR(cq->umem);
118 			goto err_qdesc;
119 		}
120 
121 		cq->q.ptr = NULL;
122 		cq->q.size = req_cq->size;
123 		cq->q.mask = req_cq->mask;
124 		cq->q.depth_log2 = req_cq->depth_log2;
125 		cq->q.stride_log2 = req_cq->stride_log2;
126 	} else {
127 		rc = ionic_queue_init(&cq->q, dev->lif_cfg.hwdev,
128 				      attr->cqe + IONIC_CQ_GRACE,
129 				      sizeof(struct ionic_v1_cqe));
130 		if (rc)
131 			goto err_q_init;
132 
133 		cq->color = true;
134 		cq->credit = cq->q.mask;
135 	}
136 
137 	rc = ionic_pgtbl_init(dev, buf, cq->umem, cq->q.dma, 1, PAGE_SIZE);
138 	if (rc)
139 		goto err_pgtbl_init;
140 
141 	init_completion(&cq->cq_rel_comp);
142 	kref_init(&cq->cq_kref);
143 
144 	return 0;
145 
146 err_pgtbl_init:
147 	if (!udata)
148 		ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);
149 err_q_init:
150 	if (cq->umem)
151 		ib_umem_release(cq->umem);
152 err_qdesc:
153 	if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ))
154 		ionic_put_cqid(dev, cq->cqid);
155 err_args:
156 	cq->vcq = NULL;
157 
158 	return rc;
159 }
160 
161 int ionic_post_create_cq_cmd(struct ionic_cq *cq,
162 			     struct ib_udata *udata,
163 			     __u32 *resp_cqid)
164 {
165 	struct ionic_ibdev *dev = to_ionic_ibdev(cq->vcq->ibcq.device);
166 
167 	if (udata && resp_cqid)
168 		*resp_cqid = cq->cqid;
169 	else
170 		ionic_queue_dbell_init(&cq->q, cq->cqid);
171 
172 	return xa_insert_irq(&dev->cq_tbl, cq->cqid, cq, GFP_KERNEL);
173 }
174 
175 void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq)
176 {
177 	if (!cq->vcq)
178 		return;
179 
180 	if (!xa_erase_irq(&dev->cq_tbl, cq->cqid))
181 		return;
182 
183 	kref_put(&cq->cq_kref, ionic_cq_complete);
184 	wait_for_completion(&cq->cq_rel_comp);
185 }
186 
187 void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq)
188 {
189 	if (!cq->vcq)
190 		return;
191 
192 	if (cq->umem)
193 		ib_umem_release(cq->umem);
194 	else
195 		ionic_queue_destroy(&cq->q, dev->lif_cfg.hwdev);
196 
197 	if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ))
198 		ionic_put_cqid(dev, cq->cqid);
199 
200 	cq->vcq = NULL;
201 }
202 
203 static int ionic_validate_qdesc_zero(struct ionic_qdesc *q)
204 {
205 	if (q->addr || q->size || q->mask || q->depth_log2 || q->stride_log2)
206 		return -EINVAL;
207 
208 	return 0;
209 }
210 
211 static int ionic_get_pdid(struct ionic_ibdev *dev, u32 *pdid)
212 {
213 	int rc;
214 
215 	rc = ionic_resid_get(&dev->inuse_pdid);
216 	if (rc < 0)
217 		return rc;
218 
219 	*pdid = rc;
220 	return 0;
221 }
222 
223 static int ionic_get_ahid(struct ionic_ibdev *dev, u32 *ahid)
224 {
225 	int rc;
226 
227 	rc = ionic_resid_get(&dev->inuse_ahid);
228 	if (rc < 0)
229 		return rc;
230 
231 	*ahid = rc;
232 	return 0;
233 }
234 
235 static int ionic_get_mrid(struct ionic_ibdev *dev, u32 *mrid)
236 {
237 	int rc;
238 
239 	/* wrap to 1, skip reserved lkey */
240 	rc = ionic_resid_get_shared(&dev->inuse_mrid, 1,
241 				    dev->inuse_mrid.inuse_size);
242 	if (rc < 0)
243 		return rc;
244 
245 	*mrid = ionic_mrid(rc, dev->next_mrkey++);
246 	return 0;
247 }
248 
249 static int ionic_get_gsi_qpid(struct ionic_ibdev *dev, u32 *qpid)
250 {
251 	int rc = 0;
252 
253 	rc = ionic_resid_get_shared(&dev->inuse_qpid, IB_QPT_GSI, IB_QPT_GSI + 1);
254 	if (rc < 0)
255 		return rc;
256 
257 	*qpid = IB_QPT_GSI;
258 	return 0;
259 }
260 
261 static int ionic_get_qpid(struct ionic_ibdev *dev, u32 *qpid,
262 			  u8 *udma_idx, u8 udma_mask)
263 {
264 	unsigned int size, base, bound;
265 	int udma_i, udma_x, udma_ix;
266 	int rc = -EINVAL;
267 
268 	udma_x = dev->next_qpid_udma_idx;
269 
270 	dev->next_qpid_udma_idx ^= dev->lif_cfg.udma_count - 1;
271 
272 	for (udma_i = 0; udma_i < dev->lif_cfg.udma_count; ++udma_i) {
273 		udma_ix = udma_i ^ udma_x;
274 
275 		if (!(udma_mask & BIT(udma_ix)))
276 			continue;
277 
278 		size = dev->lif_cfg.qp_count / dev->lif_cfg.udma_count;
279 		base = size * udma_ix;
280 		bound = base + size;
281 
282 		/* skip reserved SMI and GSI qpids in group zero */
283 		if (!base)
284 			base = 2;
285 
286 		rc = ionic_resid_get_shared(&dev->inuse_qpid, base, bound);
287 		if (rc >= 0) {
288 			*qpid = ionic_bitid_to_qid(rc,
289 						   dev->lif_cfg.udma_qgrp_shift,
290 						   dev->half_qpid_udma_shift);
291 			*udma_idx = udma_ix;
292 
293 			rc = 0;
294 			break;
295 		}
296 	}
297 
298 	return rc;
299 }
300 
301 static int ionic_get_dbid(struct ionic_ibdev *dev, u32 *dbid, phys_addr_t *addr)
302 {
303 	int rc, dbpage_num;
304 
305 	/* wrap to 1, skip kernel reserved */
306 	rc = ionic_resid_get_shared(&dev->inuse_dbid, 1,
307 				    dev->inuse_dbid.inuse_size);
308 	if (rc < 0)
309 		return rc;
310 
311 	dbpage_num = (dev->lif_cfg.lif_hw_index * dev->lif_cfg.dbid_count) + rc;
312 	*addr = dev->lif_cfg.db_phys + ((phys_addr_t)dbpage_num << PAGE_SHIFT);
313 
314 	*dbid = rc;
315 
316 	return 0;
317 }
318 
319 static void ionic_put_pdid(struct ionic_ibdev *dev, u32 pdid)
320 {
321 	ionic_resid_put(&dev->inuse_pdid, pdid);
322 }
323 
324 static void ionic_put_ahid(struct ionic_ibdev *dev, u32 ahid)
325 {
326 	ionic_resid_put(&dev->inuse_ahid, ahid);
327 }
328 
329 static void ionic_put_mrid(struct ionic_ibdev *dev, u32 mrid)
330 {
331 	ionic_resid_put(&dev->inuse_mrid, ionic_mrid_index(mrid));
332 }
333 
334 static void ionic_put_qpid(struct ionic_ibdev *dev, u32 qpid)
335 {
336 	u32 bitid = ionic_qid_to_bitid(qpid,
337 				       dev->lif_cfg.udma_qgrp_shift,
338 				       dev->half_qpid_udma_shift);
339 
340 	ionic_resid_put(&dev->inuse_qpid, bitid);
341 }
342 
343 static void ionic_put_dbid(struct ionic_ibdev *dev, u32 dbid)
344 {
345 	ionic_resid_put(&dev->inuse_dbid, dbid);
346 }
347 
348 static struct rdma_user_mmap_entry*
349 ionic_mmap_entry_insert(struct ionic_ctx *ctx, unsigned long size,
350 			unsigned long pfn, u8 mmap_flags, u64 *offset)
351 {
352 	struct ionic_mmap_entry *entry;
353 	int rc;
354 
355 	entry = kzalloc_obj(*entry);
356 	if (!entry)
357 		return NULL;
358 
359 	entry->size = size;
360 	entry->pfn = pfn;
361 	entry->mmap_flags = mmap_flags;
362 
363 	rc = rdma_user_mmap_entry_insert(&ctx->ibctx, &entry->rdma_entry,
364 					 entry->size);
365 	if (rc) {
366 		kfree(entry);
367 		return NULL;
368 	}
369 
370 	if (offset)
371 		*offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
372 
373 	return &entry->rdma_entry;
374 }
375 
376 int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
377 {
378 	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);
379 	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);
380 	struct ionic_ctx_resp resp = {};
381 	struct ionic_ctx_req req;
382 	phys_addr_t db_phys = 0;
383 	int rc;
384 
385 	rc = ib_copy_validate_udata_in(udata, req, rsvd);
386 	if (rc)
387 		return rc;
388 
389 	/* try to allocate dbid for user ctx */
390 	rc = ionic_get_dbid(dev, &ctx->dbid, &db_phys);
391 	if (rc < 0)
392 		return rc;
393 
394 	ibdev_dbg(&dev->ibdev, "user space dbid %u\n", ctx->dbid);
395 
396 	ctx->mmap_dbell = ionic_mmap_entry_insert(ctx, PAGE_SIZE,
397 						  PHYS_PFN(db_phys), 0, NULL);
398 	if (!ctx->mmap_dbell) {
399 		rc = -ENOMEM;
400 		goto err_mmap_dbell;
401 	}
402 
403 	if (dev->lif_cfg.phc_state) {
404 		ctx->mmap_phc = ionic_mmap_entry_insert(ctx, PAGE_SIZE, 0,
405 							IONIC_MMAP_PHC,
406 							&resp.phc_offset);
407 		if (!ctx->mmap_phc) {
408 			rc = -ENOMEM;
409 			goto err_mmap_phc;
410 		}
411 	}
412 
413 	resp.page_shift = PAGE_SHIFT;
414 
415 	resp.dbell_offset = db_phys & ~PAGE_MASK;
416 
417 	resp.version = dev->lif_cfg.rdma_version;
418 	resp.qp_opcodes = dev->lif_cfg.qp_opcodes;
419 	resp.admin_opcodes = dev->lif_cfg.admin_opcodes;
420 
421 	resp.sq_qtype = dev->lif_cfg.sq_qtype;
422 	resp.rq_qtype = dev->lif_cfg.rq_qtype;
423 	resp.cq_qtype = dev->lif_cfg.cq_qtype;
424 	resp.admin_qtype = dev->lif_cfg.aq_qtype;
425 	resp.max_stride = dev->lif_cfg.max_stride;
426 	resp.max_spec = IONIC_SPEC_HIGH;
427 
428 	resp.udma_count = dev->lif_cfg.udma_count;
429 	resp.expdb_mask = dev->lif_cfg.expdb_mask;
430 	resp.rcq_sign_bit = dev->lif_cfg.rcq_sign_bit;
431 	resp.comp_mask |= IONIC_CTX_CMASK_IONIC_FLAGS;
432 
433 	if (dev->lif_cfg.sq_expdb)
434 		resp.expdb_qtypes |= IONIC_EXPDB_SQ;
435 	if (dev->lif_cfg.rq_expdb)
436 		resp.expdb_qtypes |= IONIC_EXPDB_RQ;
437 
438 	rc = ib_respond_udata(udata, resp);
439 	if (rc)
440 		goto err_resp;
441 
442 	return 0;
443 
444 err_resp:
445 	rdma_user_mmap_entry_remove(ctx->mmap_phc);
446 err_mmap_phc:
447 	rdma_user_mmap_entry_remove(ctx->mmap_dbell);
448 err_mmap_dbell:
449 	ionic_put_dbid(dev, ctx->dbid);
450 
451 	return rc;
452 }
453 
454 void ionic_dealloc_ucontext(struct ib_ucontext *ibctx)
455 {
456 	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);
457 	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);
458 
459 	rdma_user_mmap_entry_remove(ctx->mmap_phc);
460 	rdma_user_mmap_entry_remove(ctx->mmap_dbell);
461 	ionic_put_dbid(dev, ctx->dbid);
462 }
463 
464 static int ionic_mmap_phc_state(struct ionic_ibdev *dev,
465 				struct vm_area_struct *vma)
466 {
467 	if (!(vma->vm_flags & VM_SHARED))
468 		return -EINVAL;
469 
470 	if (vma->vm_flags & (VM_WRITE | VM_EXEC))
471 		return -EPERM;
472 
473 	vm_flags_clear(vma, VM_MAYWRITE);
474 
475 	return vm_insert_page(vma, vma->vm_start,
476 			      virt_to_page(dev->lif_cfg.phc_state));
477 }
478 
479 int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma)
480 {
481 	struct ionic_ibdev *dev = to_ionic_ibdev(ibctx->device);
482 	struct ionic_ctx *ctx = to_ionic_ctx(ibctx);
483 	struct rdma_user_mmap_entry *rdma_entry;
484 	struct ionic_mmap_entry *ionic_entry;
485 	int rc = 0;
486 
487 	rdma_entry = rdma_user_mmap_entry_get(&ctx->ibctx, vma);
488 	if (!rdma_entry) {
489 		ibdev_dbg(&dev->ibdev, "not found %#lx\n",
490 			  vma->vm_pgoff << PAGE_SHIFT);
491 		return -EINVAL;
492 	}
493 
494 	ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry,
495 				   rdma_entry);
496 
497 	if (ionic_entry->mmap_flags & IONIC_MMAP_PHC) {
498 		rc = ionic_mmap_phc_state(dev, vma);
499 		rdma_user_mmap_entry_put(rdma_entry);
500 		return rc;
501 	}
502 
503 	ibdev_dbg(&dev->ibdev, "writecombine? %d\n",
504 		  ionic_entry->mmap_flags & IONIC_MMAP_WC);
505 	if (ionic_entry->mmap_flags & IONIC_MMAP_WC)
506 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
507 	else
508 		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
509 
510 	ibdev_dbg(&dev->ibdev, "remap st %#lx pf %#lx sz %#lx\n",
511 		  vma->vm_start, ionic_entry->pfn, ionic_entry->size);
512 	rc = rdma_user_mmap_io(&ctx->ibctx, vma, ionic_entry->pfn,
513 			       ionic_entry->size, vma->vm_page_prot,
514 			       rdma_entry);
515 	if (rc)
516 		ibdev_dbg(&dev->ibdev, "remap failed %d\n", rc);
517 
518 	rdma_user_mmap_entry_put(rdma_entry);
519 	return rc;
520 }
521 
522 void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
523 {
524 	struct ionic_mmap_entry *ionic_entry;
525 
526 	ionic_entry = container_of(rdma_entry, struct ionic_mmap_entry,
527 				   rdma_entry);
528 	kfree(ionic_entry);
529 }
530 
531 int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
532 {
533 	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
534 	struct ionic_pd *pd = to_ionic_pd(ibpd);
535 	int rc;
536 
537 	rc = ib_no_udata_io(udata);
538 	if (rc)
539 		return rc;
540 
541 	return ionic_get_pdid(dev, &pd->pdid);
542 }
543 
544 int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
545 {
546 	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
547 	struct ionic_pd *pd = to_ionic_pd(ibpd);
548 	int rc;
549 
550 	rc = ib_no_udata_io(udata);
551 	if (rc)
552 		return rc;
553 
554 	ionic_put_pdid(dev, pd->pdid);
555 
556 	return 0;
557 }
558 
559 static int ionic_build_hdr(struct ionic_ibdev *dev,
560 			   struct ib_ud_header *hdr,
561 			   const struct rdma_ah_attr *attr,
562 			   u16 sport, bool want_ecn)
563 {
564 	const struct ib_global_route *grh;
565 	enum rdma_network_type net;
566 	u8 smac[ETH_ALEN];
567 	u16 vlan;
568 	int rc;
569 
570 	if (attr->ah_flags != IB_AH_GRH)
571 		return -EINVAL;
572 	if (attr->type != RDMA_AH_ATTR_TYPE_ROCE)
573 		return -EINVAL;
574 
575 	grh = rdma_ah_read_grh(attr);
576 
577 	rc = rdma_read_gid_l2_fields(grh->sgid_attr, &vlan, smac);
578 	if (rc)
579 		return rc;
580 
581 	net = rdma_gid_attr_network_type(grh->sgid_attr);
582 
583 	rc = ib_ud_header_init(0,	/* no payload */
584 			       0,	/* no lrh */
585 			       1,	/* yes eth */
586 			       vlan != 0xffff,
587 			       0,	/* no grh */
588 			       net == RDMA_NETWORK_IPV4 ? 4 : 6,
589 			       1,	/* yes udp */
590 			       0,	/* no imm */
591 			       hdr);
592 	if (rc)
593 		return rc;
594 
595 	ether_addr_copy(hdr->eth.smac_h, smac);
596 	ether_addr_copy(hdr->eth.dmac_h, attr->roce.dmac);
597 
598 	if (net == RDMA_NETWORK_IPV4) {
599 		hdr->eth.type = cpu_to_be16(ETH_P_IP);
600 		hdr->ip4.frag_off = cpu_to_be16(0x4000); /* don't fragment */
601 		hdr->ip4.ttl = grh->hop_limit;
602 		hdr->ip4.tot_len = cpu_to_be16(0xffff);
603 		hdr->ip4.saddr =
604 			*(const __be32 *)(grh->sgid_attr->gid.raw + 12);
605 		hdr->ip4.daddr = *(const __be32 *)(grh->dgid.raw + 12);
606 
607 		if (want_ecn)
608 			hdr->ip4.tos = ionic_set_ecn(grh->traffic_class);
609 		else
610 			hdr->ip4.tos = ionic_clear_ecn(grh->traffic_class);
611 	} else {
612 		hdr->eth.type = cpu_to_be16(ETH_P_IPV6);
613 		hdr->grh.flow_label = cpu_to_be32(grh->flow_label);
614 		hdr->grh.hop_limit = grh->hop_limit;
615 		hdr->grh.source_gid = grh->sgid_attr->gid;
616 		hdr->grh.destination_gid = grh->dgid;
617 
618 		if (want_ecn)
619 			hdr->grh.traffic_class =
620 				ionic_set_ecn(grh->traffic_class);
621 		else
622 			hdr->grh.traffic_class =
623 				ionic_clear_ecn(grh->traffic_class);
624 	}
625 
626 	if (vlan != 0xffff) {
627 		vlan |= rdma_ah_get_sl(attr) << VLAN_PRIO_SHIFT;
628 		hdr->vlan.tag = cpu_to_be16(vlan);
629 		hdr->vlan.type = hdr->eth.type;
630 		hdr->eth.type = cpu_to_be16(ETH_P_8021Q);
631 	}
632 
633 	hdr->udp.sport = cpu_to_be16(sport);
634 	hdr->udp.dport = cpu_to_be16(ROCE_V2_UDP_DPORT);
635 
636 	return 0;
637 }
638 
639 static void ionic_set_ah_attr(struct ionic_ibdev *dev,
640 			      struct rdma_ah_attr *ah_attr,
641 			      struct ib_ud_header *hdr,
642 			      int sgid_index)
643 {
644 	u32 flow_label;
645 	u16 vlan = 0;
646 	u8  tos, ttl;
647 
648 	if (hdr->vlan_present)
649 		vlan = be16_to_cpu(hdr->vlan.tag);
650 
651 	if (hdr->ipv4_present) {
652 		flow_label = 0;
653 		ttl = hdr->ip4.ttl;
654 		tos = hdr->ip4.tos;
655 		*(__be16 *)(hdr->grh.destination_gid.raw + 10) = cpu_to_be16(0xffff);
656 		*(__be32 *)(hdr->grh.destination_gid.raw + 12) = hdr->ip4.daddr;
657 	} else {
658 		flow_label = be32_to_cpu(hdr->grh.flow_label);
659 		ttl = hdr->grh.hop_limit;
660 		tos = hdr->grh.traffic_class;
661 	}
662 
663 	memset(ah_attr, 0, sizeof(*ah_attr));
664 	ah_attr->type = RDMA_AH_ATTR_TYPE_ROCE;
665 	if (hdr->eth_present)
666 		ether_addr_copy(ah_attr->roce.dmac, hdr->eth.dmac_h);
667 	rdma_ah_set_sl(ah_attr, vlan >> VLAN_PRIO_SHIFT);
668 	rdma_ah_set_port_num(ah_attr, 1);
669 	rdma_ah_set_grh(ah_attr, NULL, flow_label, sgid_index, ttl, tos);
670 	rdma_ah_set_dgid_raw(ah_attr, &hdr->grh.destination_gid);
671 }
672 
673 static int ionic_create_ah_cmd(struct ionic_ibdev *dev,
674 			       struct ionic_ah *ah,
675 			       struct ionic_pd *pd,
676 			       struct rdma_ah_attr *attr,
677 			       u32 flags)
678 {
679 	struct ionic_admin_wr wr = {
680 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
681 		.wqe = {
682 			.op = IONIC_V1_ADMIN_CREATE_AH,
683 			.len = cpu_to_le16(IONIC_ADMIN_CREATE_AH_IN_V1_LEN),
684 			.cmd.create_ah = {
685 				.pd_id = cpu_to_le32(pd->pdid),
686 				.dbid_flags = cpu_to_le16(dev->lif_cfg.dbid),
687 				.id_ver = cpu_to_le32(ah->ahid),
688 			}
689 		}
690 	};
691 	enum ionic_admin_flags admin_flags = 0;
692 	dma_addr_t hdr_dma = 0;
693 	void *hdr_buf;
694 	gfp_t gfp = GFP_ATOMIC;
695 	int rc, hdr_len = 0;
696 
697 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_AH)
698 		return -EBADRQC;
699 
700 	if (flags & RDMA_CREATE_AH_SLEEPABLE)
701 		gfp = GFP_KERNEL;
702 	else
703 		admin_flags |= IONIC_ADMIN_F_BUSYWAIT;
704 
705 	rc = ionic_build_hdr(dev, &ah->hdr, attr, IONIC_ROCE_UDP_SPORT, false);
706 	if (rc)
707 		return rc;
708 
709 	if (ah->hdr.eth.type == cpu_to_be16(ETH_P_8021Q)) {
710 		if (ah->hdr.vlan.type == cpu_to_be16(ETH_P_IP))
711 			wr.wqe.cmd.create_ah.csum_profile =
712 				IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP;
713 		else
714 			wr.wqe.cmd.create_ah.csum_profile =
715 				IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP;
716 	} else {
717 		if (ah->hdr.eth.type == cpu_to_be16(ETH_P_IP))
718 			wr.wqe.cmd.create_ah.csum_profile =
719 				IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP;
720 		else
721 			wr.wqe.cmd.create_ah.csum_profile =
722 				IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP;
723 	}
724 
725 	ah->sgid_index = rdma_ah_read_grh(attr)->sgid_index;
726 
727 	hdr_buf = kmalloc(PAGE_SIZE, gfp);
728 	if (!hdr_buf)
729 		return -ENOMEM;
730 
731 	hdr_len = ib_ud_header_pack(&ah->hdr, hdr_buf);
732 	hdr_len -= IB_BTH_BYTES;
733 	hdr_len -= IB_DETH_BYTES;
734 	ibdev_dbg(&dev->ibdev, "roce packet header template\n");
735 	print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1,
736 			     hdr_buf, hdr_len, true);
737 
738 	hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len,
739 				 DMA_TO_DEVICE);
740 
741 	rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma);
742 	if (rc)
743 		goto err_dma;
744 
745 	wr.wqe.cmd.create_ah.dma_addr = cpu_to_le64(hdr_dma);
746 	wr.wqe.cmd.create_ah.length = cpu_to_le32(hdr_len);
747 
748 	ionic_admin_post(dev, &wr);
749 	rc = ionic_admin_wait(dev, &wr, admin_flags);
750 
751 	dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len,
752 			 DMA_TO_DEVICE);
753 err_dma:
754 	kfree(hdr_buf);
755 
756 	return rc;
757 }
758 
759 static int ionic_destroy_ah_cmd(struct ionic_ibdev *dev, u32 ahid, u32 flags)
760 {
761 	struct ionic_admin_wr wr = {
762 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
763 		.wqe = {
764 			.op = IONIC_V1_ADMIN_DESTROY_AH,
765 			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_AH_IN_V1_LEN),
766 			.cmd.destroy_ah = {
767 				.ah_id = cpu_to_le32(ahid),
768 			},
769 		}
770 	};
771 	enum ionic_admin_flags admin_flags = IONIC_ADMIN_F_TEARDOWN;
772 
773 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_AH)
774 		return -EBADRQC;
775 
776 	if (!(flags & RDMA_CREATE_AH_SLEEPABLE))
777 		admin_flags |= IONIC_ADMIN_F_BUSYWAIT;
778 
779 	ionic_admin_post(dev, &wr);
780 	ionic_admin_wait(dev, &wr, admin_flags);
781 
782 	/* No host-memory resource is associated with ah, so it is ok
783 	 * to "succeed" and complete this destroy ah on the host.
784 	 */
785 	return 0;
786 }
787 
788 int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
789 		    struct ib_udata *udata)
790 {
791 	struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device);
792 	struct rdma_ah_attr *attr = init_attr->ah_attr;
793 	struct ionic_pd *pd = to_ionic_pd(ibah->pd);
794 	struct ionic_ah *ah = to_ionic_ah(ibah);
795 	struct ionic_ah_resp resp = {};
796 	u32 flags = init_attr->flags;
797 	int rc;
798 
799 	rc = ib_is_udata_in_empty(udata);
800 	if (rc)
801 		return rc;
802 
803 	rc = ionic_get_ahid(dev, &ah->ahid);
804 	if (rc)
805 		return rc;
806 
807 	rc = ionic_create_ah_cmd(dev, ah, pd, attr, flags);
808 	if (rc)
809 		goto err_cmd;
810 
811 	if (udata) {
812 		resp.ahid = ah->ahid;
813 
814 		rc = ib_respond_udata(udata, resp);
815 		if (rc)
816 			goto err_resp;
817 	}
818 
819 	return 0;
820 
821 err_resp:
822 	ionic_destroy_ah_cmd(dev, ah->ahid, flags);
823 err_cmd:
824 	ionic_put_ahid(dev, ah->ahid);
825 	return rc;
826 }
827 
828 int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr)
829 {
830 	struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device);
831 	struct ionic_ah *ah = to_ionic_ah(ibah);
832 
833 	ionic_set_ah_attr(dev, ah_attr, &ah->hdr, ah->sgid_index);
834 
835 	return 0;
836 }
837 
838 int ionic_destroy_ah(struct ib_ah *ibah, u32 flags)
839 {
840 	struct ionic_ibdev *dev = to_ionic_ibdev(ibah->device);
841 	struct ionic_ah *ah = to_ionic_ah(ibah);
842 	int rc;
843 
844 	rc = ionic_destroy_ah_cmd(dev, ah->ahid, flags);
845 	if (rc)
846 		return rc;
847 
848 	ionic_put_ahid(dev, ah->ahid);
849 
850 	return 0;
851 }
852 
853 static int ionic_create_mr_cmd(struct ionic_ibdev *dev,
854 			       struct ionic_pd *pd,
855 			       struct ionic_mr *mr,
856 			       u64 addr,
857 			       u64 length)
858 {
859 	struct ionic_admin_wr wr = {
860 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
861 		.wqe = {
862 			.op = IONIC_V1_ADMIN_CREATE_MR,
863 			.len = cpu_to_le16(IONIC_ADMIN_CREATE_MR_IN_V1_LEN),
864 			.cmd.create_mr = {
865 				.va = cpu_to_le64(addr),
866 				.length = cpu_to_le64(length),
867 				.pd_id = cpu_to_le32(pd->pdid),
868 				.page_size_log2 = mr->buf.page_size_log2,
869 				.tbl_index = cpu_to_le32(~0),
870 				.map_count = cpu_to_le32(mr->buf.tbl_pages),
871 				.dma_addr = ionic_pgtbl_dma(&mr->buf, addr),
872 				.dbid_flags = cpu_to_le16(mr->flags),
873 				.id_ver = cpu_to_le32(mr->mrid),
874 			}
875 		}
876 	};
877 	int rc;
878 
879 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_MR)
880 		return -EBADRQC;
881 
882 	ionic_admin_post(dev, &wr);
883 	rc = ionic_admin_wait(dev, &wr, 0);
884 	if (!rc)
885 		mr->created = true;
886 
887 	return rc;
888 }
889 
890 static int ionic_destroy_mr_cmd(struct ionic_ibdev *dev, u32 mrid)
891 {
892 	struct ionic_admin_wr wr = {
893 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
894 		.wqe = {
895 			.op = IONIC_V1_ADMIN_DESTROY_MR,
896 			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_MR_IN_V1_LEN),
897 			.cmd.destroy_mr = {
898 				.mr_id = cpu_to_le32(mrid),
899 			},
900 		}
901 	};
902 
903 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_MR)
904 		return -EBADRQC;
905 
906 	ionic_admin_post(dev, &wr);
907 
908 	return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);
909 }
910 
911 struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access)
912 {
913 	struct ionic_pd *pd = to_ionic_pd(ibpd);
914 	struct ionic_mr *mr;
915 
916 	mr = kzalloc_obj(*mr);
917 	if (!mr)
918 		return ERR_PTR(-ENOMEM);
919 
920 	mr->ibmr.lkey = IONIC_DMA_LKEY;
921 	mr->ibmr.rkey = IONIC_DMA_RKEY;
922 
923 	if (pd)
924 		pd->flags |= IONIC_QPF_PRIVILEGED;
925 
926 	return &mr->ibmr;
927 }
928 
929 struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
930 				u64 addr, int access, struct ib_dmah *dmah,
931 				struct ib_udata *udata)
932 {
933 	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
934 	struct ionic_pd *pd = to_ionic_pd(ibpd);
935 	struct ionic_mr *mr;
936 	unsigned long pg_sz;
937 	int rc;
938 
939 	rc = ib_no_udata_io(udata);
940 	if (rc)
941 		return ERR_PTR(rc);
942 
943 	if (dmah)
944 		return ERR_PTR(-EOPNOTSUPP);
945 
946 	mr = kzalloc_obj(*mr);
947 	if (!mr)
948 		return ERR_PTR(-ENOMEM);
949 
950 	rc = ionic_get_mrid(dev, &mr->mrid);
951 	if (rc)
952 		goto err_mrid;
953 
954 	mr->ibmr.lkey = mr->mrid;
955 	mr->ibmr.rkey = mr->mrid;
956 	mr->ibmr.iova = addr;
957 	mr->ibmr.length = length;
958 
959 	mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);
960 
961 	mr->umem = ib_umem_get_va(&dev->ibdev, start, length, access);
962 	if (IS_ERR(mr->umem)) {
963 		rc = PTR_ERR(mr->umem);
964 		goto err_umem;
965 	}
966 
967 	pg_sz = ib_umem_find_best_pgsz(mr->umem,
968 				       dev->lif_cfg.page_size_supported,
969 				       addr);
970 	if (!pg_sz) {
971 		rc = -EINVAL;
972 		goto err_pgtbl;
973 	}
974 
975 	rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz);
976 	if (rc)
977 		goto err_pgtbl;
978 
979 	rc = ionic_create_mr_cmd(dev, pd, mr, addr, length);
980 	if (rc)
981 		goto err_cmd;
982 
983 	ionic_pgtbl_unbuf(dev, &mr->buf);
984 
985 	return &mr->ibmr;
986 
987 err_cmd:
988 	ionic_pgtbl_unbuf(dev, &mr->buf);
989 err_pgtbl:
990 	ib_umem_release(mr->umem);
991 err_umem:
992 	ionic_put_mrid(dev, mr->mrid);
993 err_mrid:
994 	kfree(mr);
995 	return ERR_PTR(rc);
996 }
997 
998 struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
999 				       u64 length, u64 addr, int fd, int access,
1000 				       struct ib_dmah *dmah,
1001 				       struct uverbs_attr_bundle *attrs)
1002 {
1003 	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
1004 	struct ionic_pd *pd = to_ionic_pd(ibpd);
1005 	struct ib_umem_dmabuf *umem_dmabuf;
1006 	struct ionic_mr *mr;
1007 	u64 pg_sz;
1008 	int rc;
1009 
1010 	if (dmah)
1011 		return ERR_PTR(-EOPNOTSUPP);
1012 
1013 	mr = kzalloc_obj(*mr);
1014 	if (!mr)
1015 		return ERR_PTR(-ENOMEM);
1016 
1017 	rc = ionic_get_mrid(dev, &mr->mrid);
1018 	if (rc)
1019 		goto err_mrid;
1020 
1021 	mr->ibmr.lkey = mr->mrid;
1022 	mr->ibmr.rkey = mr->mrid;
1023 	mr->ibmr.iova = addr;
1024 	mr->ibmr.length = length;
1025 
1026 	mr->flags = IONIC_MRF_USER_MR | to_ionic_mr_flags(access);
1027 
1028 	umem_dmabuf = ib_umem_dmabuf_get_pinned(&dev->ibdev, offset, length,
1029 						fd, access);
1030 	if (IS_ERR(umem_dmabuf)) {
1031 		rc = PTR_ERR(umem_dmabuf);
1032 		goto err_umem;
1033 	}
1034 
1035 	mr->umem = &umem_dmabuf->umem;
1036 
1037 	pg_sz = ib_umem_find_best_pgsz(mr->umem,
1038 				       dev->lif_cfg.page_size_supported,
1039 				       addr);
1040 	if (!pg_sz) {
1041 		rc = -EINVAL;
1042 		goto err_pgtbl;
1043 	}
1044 
1045 	rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, 1, pg_sz);
1046 	if (rc)
1047 		goto err_pgtbl;
1048 
1049 	rc = ionic_create_mr_cmd(dev, pd, mr, addr, length);
1050 	if (rc)
1051 		goto err_cmd;
1052 
1053 	ionic_pgtbl_unbuf(dev, &mr->buf);
1054 
1055 	return &mr->ibmr;
1056 
1057 err_cmd:
1058 	ionic_pgtbl_unbuf(dev, &mr->buf);
1059 err_pgtbl:
1060 	ib_umem_release(mr->umem);
1061 err_umem:
1062 	ionic_put_mrid(dev, mr->mrid);
1063 err_mrid:
1064 	kfree(mr);
1065 	return ERR_PTR(rc);
1066 }
1067 
1068 int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
1069 {
1070 	struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device);
1071 	struct ionic_mr *mr = to_ionic_mr(ibmr);
1072 	int rc;
1073 
1074 	rc = ib_no_udata_io(udata);
1075 	if (rc)
1076 		return rc;
1077 
1078 	if (!mr->ibmr.lkey)
1079 		goto out;
1080 
1081 	if (mr->created) {
1082 		rc = ionic_destroy_mr_cmd(dev, mr->mrid);
1083 		if (rc)
1084 			return rc;
1085 	}
1086 
1087 	ionic_pgtbl_unbuf(dev, &mr->buf);
1088 
1089 	if (mr->umem)
1090 		ib_umem_release(mr->umem);
1091 
1092 	ionic_put_mrid(dev, mr->mrid);
1093 
1094 out:
1095 	kfree(mr);
1096 
1097 	return 0;
1098 }
1099 
1100 struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
1101 			     u32 max_sg)
1102 {
1103 	struct ionic_ibdev *dev = to_ionic_ibdev(ibpd->device);
1104 	struct ionic_pd *pd = to_ionic_pd(ibpd);
1105 	struct ionic_mr *mr;
1106 	int rc;
1107 
1108 	if (type != IB_MR_TYPE_MEM_REG)
1109 		return ERR_PTR(-EINVAL);
1110 
1111 	mr = kzalloc_obj(*mr);
1112 	if (!mr)
1113 		return ERR_PTR(-ENOMEM);
1114 
1115 	rc = ionic_get_mrid(dev, &mr->mrid);
1116 	if (rc)
1117 		goto err_mrid;
1118 
1119 	mr->ibmr.lkey = mr->mrid;
1120 	mr->ibmr.rkey = mr->mrid;
1121 
1122 	mr->flags = IONIC_MRF_PHYS_MR;
1123 
1124 	rc = ionic_pgtbl_init(dev, &mr->buf, mr->umem, 0, max_sg, PAGE_SIZE);
1125 	if (rc)
1126 		goto err_pgtbl;
1127 
1128 	mr->buf.tbl_pages = 0;
1129 
1130 	rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0);
1131 	if (rc)
1132 		goto err_cmd;
1133 
1134 	return &mr->ibmr;
1135 
1136 err_cmd:
1137 	ionic_pgtbl_unbuf(dev, &mr->buf);
1138 err_pgtbl:
1139 	ionic_put_mrid(dev, mr->mrid);
1140 err_mrid:
1141 	kfree(mr);
1142 	return ERR_PTR(rc);
1143 }
1144 
1145 static int ionic_map_mr_page(struct ib_mr *ibmr, u64 dma)
1146 {
1147 	struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device);
1148 	struct ionic_mr *mr = to_ionic_mr(ibmr);
1149 
1150 	ibdev_dbg(&dev->ibdev, "dma %p\n", (void *)dma);
1151 	return ionic_pgtbl_page(&mr->buf, dma);
1152 }
1153 
1154 int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
1155 		    unsigned int *sg_offset)
1156 {
1157 	struct ionic_ibdev *dev = to_ionic_ibdev(ibmr->device);
1158 	struct ionic_mr *mr = to_ionic_mr(ibmr);
1159 	int rc;
1160 
1161 	/* mr must be allocated using ib_alloc_mr() */
1162 	if (unlikely(!mr->buf.tbl_limit))
1163 		return -EINVAL;
1164 
1165 	mr->buf.tbl_pages = 0;
1166 
1167 	if (mr->buf.tbl_buf)
1168 		dma_sync_single_for_cpu(dev->lif_cfg.hwdev, mr->buf.tbl_dma,
1169 					mr->buf.tbl_size, DMA_TO_DEVICE);
1170 
1171 	ibdev_dbg(&dev->ibdev, "sg %p nent %d\n", sg, sg_nents);
1172 	rc = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ionic_map_mr_page);
1173 
1174 	mr->buf.page_size_log2 = order_base_2(ibmr->page_size);
1175 
1176 	if (mr->buf.tbl_buf)
1177 		dma_sync_single_for_device(dev->lif_cfg.hwdev, mr->buf.tbl_dma,
1178 					   mr->buf.tbl_size, DMA_TO_DEVICE);
1179 
1180 	return rc;
1181 }
1182 
1183 int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
1184 {
1185 	struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device);
1186 	struct ionic_pd *pd = to_ionic_pd(ibmw->pd);
1187 	struct ionic_mr *mr = to_ionic_mw(ibmw);
1188 	int rc;
1189 
1190 	rc = ib_no_udata_io(udata);
1191 	if (rc)
1192 		return rc;
1193 
1194 	rc = ionic_get_mrid(dev, &mr->mrid);
1195 	if (rc)
1196 		return rc;
1197 
1198 	mr->ibmw.rkey = mr->mrid;
1199 
1200 	if (mr->ibmw.type == IB_MW_TYPE_1)
1201 		mr->flags = IONIC_MRF_MW_1;
1202 	else
1203 		mr->flags = IONIC_MRF_MW_2;
1204 
1205 	rc = ionic_create_mr_cmd(dev, pd, mr, 0, 0);
1206 	if (rc)
1207 		goto err_cmd;
1208 
1209 	return 0;
1210 
1211 err_cmd:
1212 	ionic_put_mrid(dev, mr->mrid);
1213 	return rc;
1214 }
1215 
1216 int ionic_dealloc_mw(struct ib_mw *ibmw)
1217 {
1218 	struct ionic_ibdev *dev = to_ionic_ibdev(ibmw->device);
1219 	struct ionic_mr *mr = to_ionic_mw(ibmw);
1220 	int rc;
1221 
1222 	rc = ionic_destroy_mr_cmd(dev, mr->mrid);
1223 	if (rc)
1224 		return rc;
1225 
1226 	ionic_put_mrid(dev, mr->mrid);
1227 
1228 	return 0;
1229 }
1230 
1231 static int ionic_create_cq_cmd(struct ionic_ibdev *dev,
1232 			       struct ionic_ctx *ctx,
1233 			       struct ionic_cq *cq,
1234 			       struct ionic_tbl_buf *buf,
1235 			       int udma_idx,
1236 			       int *out_udma_idx)
1237 {
1238 	struct ionic_admin_create_cq_resp *resp_buf;
1239 	const u16 dbid = ionic_ctx_dbid(dev, ctx);
1240 	struct ionic_admin_wr wr = {
1241 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1242 		.wqe = {
1243 			.op = IONIC_V1_ADMIN_CREATE_CQ,
1244 			.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V1_LEN),
1245 			.cmd.create_cq = {
1246 				.eq_id = cpu_to_le32(cq->eqid),
1247 				.depth_log2 = cq->q.depth_log2,
1248 				.stride_log2 = cq->q.stride_log2,
1249 				.page_size_log2 = buf->page_size_log2,
1250 				.tbl_index = cpu_to_le32(~0),
1251 				.map_count = cpu_to_le32(buf->tbl_pages),
1252 				.dma_addr = ionic_pgtbl_dma(buf, 0),
1253 				.dbid_flags = cpu_to_le16(dbid),
1254 				.id_ver = cpu_to_le32(cq->cqid),
1255 			}
1256 		}
1257 	};
1258 	dma_addr_t resp_buf_dma;
1259 	int rc;
1260 
1261 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_CQ)
1262 		return -EBADRQC;
1263 
1264 	if (!ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) {
1265 		ionic_admin_post(dev, &wr);
1266 		return ionic_admin_wait(dev, &wr, 0);
1267 	}
1268 
1269 	resp_buf = kzalloc_obj(*resp_buf);
1270 	if (!resp_buf)
1271 		return -ENOMEM;
1272 
1273 	resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf,
1274 				      sizeof(*resp_buf),
1275 				      DMA_FROM_DEVICE);
1276 
1277 	rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma);
1278 	if (rc) {
1279 		rc = -ENOMEM;
1280 		goto err_dma;
1281 	}
1282 
1283 	wr.wqe.len = cpu_to_le16(IONIC_ADMIN_CREATE_CQ_IN_V2_LEN);
1284 	wr.wqe.cmd.create_cq.udma_idx = udma_idx;
1285 	wr.wqe.cmd.create_cq.resp_dma_addr = cpu_to_le64(resp_buf_dma);
1286 	wr.wqe.cmd.create_cq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN);
1287 
1288 	ionic_admin_post(dev, &wr);
1289 	rc = ionic_admin_wait(dev, &wr, 0);
1290 
1291 	dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf),
1292 			 DMA_FROM_DEVICE);
1293 	if (rc)
1294 		goto err_dma;
1295 
1296 	if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_CQ_OUT_V1_LEN) {
1297 		rc = -EOPNOTSUPP;
1298 		goto err_dma;
1299 	}
1300 
1301 	cq->cqid = le32_to_cpu(resp_buf->id);
1302 	if (out_udma_idx)
1303 		*out_udma_idx = resp_buf->udma_idx;
1304 
1305 err_dma:
1306 	kfree(resp_buf);
1307 
1308 	return rc;
1309 }
1310 
1311 static int ionic_destroy_cq_cmd(struct ionic_ibdev *dev, u32 cqid)
1312 {
1313 	struct ionic_admin_wr wr = {
1314 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1315 		.wqe = {
1316 			.op = IONIC_V1_ADMIN_DESTROY_CQ,
1317 			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_CQ_IN_V1_LEN),
1318 			.cmd.destroy_cq = {
1319 				.cq_id = cpu_to_le32(cqid),
1320 			},
1321 		}
1322 	};
1323 
1324 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_CQ)
1325 		return -EBADRQC;
1326 
1327 	ionic_admin_post(dev, &wr);
1328 
1329 	return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);
1330 }
1331 
1332 int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
1333 		    struct uverbs_attr_bundle *attrs)
1334 {
1335 	struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device);
1336 	struct ib_udata *udata = &attrs->driver_udata;
1337 	struct ionic_ctx *ctx =
1338 		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
1339 	struct ionic_vcq *vcq = to_ionic_vcq(ibcq);
1340 	struct ionic_tbl_buf buf = {};
1341 	struct ionic_cq_resp resp = {};
1342 	struct ionic_cq_req req;
1343 	int udma_idx = 0, rc;
1344 
1345 	if (udata) {
1346 		rc = ib_copy_validate_udata_in(udata, req, rsvd);
1347 		if (rc)
1348 			return rc;
1349 	}
1350 
1351 	vcq->udma_mask = BIT(dev->lif_cfg.udma_count) - 1;
1352 
1353 	if (udata)
1354 		vcq->udma_mask &= req.udma_mask;
1355 
1356 	if (!vcq->udma_mask) {
1357 		rc = -EINVAL;
1358 		goto err_init;
1359 	}
1360 
1361 	for (; udma_idx < dev->lif_cfg.udma_count; ++udma_idx) {
1362 		if (!(vcq->udma_mask & BIT(udma_idx)))
1363 			continue;
1364 
1365 		rc = ionic_create_cq_common(vcq, &buf, attr, ctx, udata,
1366 					    &req.cq[udma_idx],
1367 					    udma_idx);
1368 		if (rc)
1369 			goto err_init;
1370 
1371 		rc = ionic_create_cq_cmd(dev, ctx, &vcq->cq[udma_idx], &buf,
1372 					 udma_idx, NULL);
1373 		if (rc)
1374 			goto err_cmd;
1375 
1376 		ionic_pgtbl_unbuf(dev, &buf);
1377 
1378 		rc = ionic_post_create_cq_cmd(&vcq->cq[udma_idx], udata,
1379 					      &resp.cqid[udma_idx]);
1380 		if (rc)
1381 			goto err_post;
1382 	}
1383 
1384 	vcq->ibcq.cqe = attr->cqe;
1385 
1386 	if (udata) {
1387 		resp.udma_mask = vcq->udma_mask;
1388 
1389 		rc = ib_respond_udata(udata, resp);
1390 		if (rc)
1391 			goto err_resp;
1392 	}
1393 
1394 	return 0;
1395 
1396 err_resp:
1397 	while (udma_idx) {
1398 		--udma_idx;
1399 		if (!(vcq->udma_mask & BIT(udma_idx)))
1400 			continue;
1401 		ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]);
1402 err_post:
1403 		ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid);
1404 err_cmd:
1405 		ionic_pgtbl_unbuf(dev, &buf);
1406 		ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]);
1407 err_init:
1408 		;
1409 	}
1410 
1411 	return rc;
1412 }
1413 
1414 int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1415 {
1416 	struct ionic_ibdev *dev = to_ionic_ibdev(ibcq->device);
1417 	struct ionic_vcq *vcq = to_ionic_vcq(ibcq);
1418 	int udma_idx, rc_tmp, rc = 0;
1419 
1420 	rc = ib_no_udata_io(udata);
1421 	if (rc)
1422 		return rc;
1423 
1424 	for (udma_idx = dev->lif_cfg.udma_count; udma_idx; ) {
1425 		--udma_idx;
1426 
1427 		if (!(vcq->udma_mask & BIT(udma_idx)))
1428 			continue;
1429 
1430 		/*
1431 		 * Untrack the CQ before releasing its hardware ID below, so a
1432 		 * concurrent create that gets the same ID reused by firmware
1433 		 * cannot have its fresh XArray entry erased by this destroy.
1434 		 */
1435 		ionic_pre_destroy_cq_cmd(dev, &vcq->cq[udma_idx]);
1436 
1437 		rc_tmp = ionic_destroy_cq_cmd(dev, vcq->cq[udma_idx].cqid);
1438 		if (rc_tmp) {
1439 			if (!rc)
1440 				rc = rc_tmp;
1441 
1442 			continue;
1443 		}
1444 
1445 		ionic_destroy_cq_common(dev, &vcq->cq[udma_idx]);
1446 	}
1447 
1448 	return rc;
1449 }
1450 
1451 static bool pd_remote_privileged(struct ib_pd *pd)
1452 {
1453 	return pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY;
1454 }
1455 
1456 static int ionic_create_qp_cmd(struct ionic_ibdev *dev,
1457 			       struct ionic_pd *pd,
1458 			       struct ionic_cq *send_cq,
1459 			       struct ionic_cq *recv_cq,
1460 			       struct ionic_qp *qp,
1461 			       struct ionic_tbl_buf *sq_buf,
1462 			       struct ionic_tbl_buf *rq_buf,
1463 			       struct ib_qp_init_attr *attr,
1464 			       u32 ionic_flags)
1465 {
1466 	const u16 dbid = ionic_obj_dbid(dev, pd->ibpd.uobject);
1467 	const u32 flags = to_ionic_qp_flags(0, 0,
1468 					    qp->sq_cmb & IONIC_CMB_ENABLE,
1469 					    qp->rq.cmb & IONIC_CMB_ENABLE,
1470 					    qp->sq_spec, qp->rq.spec,
1471 					    pd->flags & IONIC_QPF_PRIVILEGED,
1472 					    pd_remote_privileged(&pd->ibpd));
1473 	struct ionic_admin_wr wr = {
1474 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1475 		.wqe = {
1476 			.op = IONIC_V1_ADMIN_CREATE_QP,
1477 			.len = cpu_to_le16(IONIC_ADMIN_CREATE_QP_IN_V1_LEN),
1478 			.cmd.create_qp = {
1479 				.pd_id = cpu_to_le32(pd->pdid),
1480 				/* User-supplied ionic_flags are passed through to
1481 				 * firmware, which validates and rejects any
1482 				 * unsupported or unauthorized bits.
1483 				 */
1484 				.priv_flags = cpu_to_be32(flags |
1485 						(ionic_flags & IONIC_QP_USER_FLAGS_MASK)),
1486 				.type_state = to_ionic_qp_type(attr->qp_type),
1487 				.dbid_flags = cpu_to_le16(dbid),
1488 				.id_ver = cpu_to_le32(qp->qpid),
1489 			}
1490 		}
1491 	};
1492 
1493 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_QP)
1494 		return -EBADRQC;
1495 
1496 	if (qp->has_sq) {
1497 		wr.wqe.cmd.create_qp.sq_cq_id = cpu_to_le32(send_cq->cqid);
1498 		wr.wqe.cmd.create_qp.sq_depth_log2 = qp->sq.depth_log2;
1499 		wr.wqe.cmd.create_qp.sq_stride_log2 = qp->sq.stride_log2;
1500 		wr.wqe.cmd.create_qp.sq_page_size_log2 = sq_buf->page_size_log2;
1501 		wr.wqe.cmd.create_qp.sq_tbl_index_xrcd_id = cpu_to_le32(~0);
1502 		wr.wqe.cmd.create_qp.sq_map_count =
1503 			cpu_to_le32(sq_buf->tbl_pages);
1504 		wr.wqe.cmd.create_qp.sq_dma_addr = ionic_pgtbl_dma(sq_buf, 0);
1505 	}
1506 
1507 	if (!qp->srq) {
1508 		wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid);
1509 		wr.wqe.cmd.create_qp.rq_depth_log2 = qp->rq.q.depth_log2;
1510 		wr.wqe.cmd.create_qp.rq_stride_log2 = qp->rq.q.stride_log2;
1511 		wr.wqe.cmd.create_qp.rq_page_size_log2 = rq_buf->page_size_log2;
1512 		wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(~0);
1513 		wr.wqe.cmd.create_qp.rq_map_count =
1514 			cpu_to_le32(rq_buf->tbl_pages);
1515 		wr.wqe.cmd.create_qp.rq_dma_addr = ionic_pgtbl_dma(rq_buf, 0);
1516 	} else {
1517 		wr.wqe.cmd.create_qp.rq_tbl_index_srq_id = cpu_to_le32(qp->srq->rq.qid);
1518 		wr.wqe.cmd.create_qp.rq_cq_id = cpu_to_le32(recv_cq->cqid);
1519 	}
1520 
1521 	ionic_admin_post(dev, &wr);
1522 
1523 	return ionic_admin_wait(dev, &wr, 0);
1524 }
1525 
1526 static int ionic_modify_qp_cmd(struct ionic_ibdev *dev,
1527 			       struct ionic_pd *pd,
1528 			       struct ionic_qp *qp,
1529 			       struct ib_qp_attr *attr,
1530 			       int mask)
1531 {
1532 	const u32 flags = to_ionic_qp_flags(attr->qp_access_flags,
1533 					    attr->en_sqd_async_notify,
1534 					    qp->sq_cmb & IONIC_CMB_ENABLE,
1535 					    qp->rq.cmb & IONIC_CMB_ENABLE,
1536 					    qp->sq_spec, qp->rq.spec,
1537 					    pd->flags & IONIC_QPF_PRIVILEGED,
1538 					    pd_remote_privileged(qp->ibqp.pd));
1539 	const u8 state = to_ionic_qp_modify_state(attr->qp_state,
1540 						  attr->cur_qp_state);
1541 	struct ionic_admin_wr wr = {
1542 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1543 		.wqe = {
1544 			.op = IONIC_V1_ADMIN_MODIFY_QP,
1545 			.len = cpu_to_le16(IONIC_ADMIN_MODIFY_QP_IN_V1_LEN),
1546 			.cmd.mod_qp = {
1547 				.attr_mask = cpu_to_be32(mask),
1548 				.access_flags = cpu_to_be16(flags),
1549 				.rq_psn = cpu_to_le32(attr->rq_psn),
1550 				.sq_psn = cpu_to_le32(attr->sq_psn),
1551 				.rate_limit_kbps =
1552 					cpu_to_le32(attr->rate_limit),
1553 				.pmtu = (attr->path_mtu + 7),
1554 				.retry = (attr->retry_cnt |
1555 					  (attr->rnr_retry << 4)),
1556 				.rnr_timer = attr->min_rnr_timer,
1557 				.retry_timeout = attr->timeout,
1558 				.type_state = state,
1559 				.id_ver = cpu_to_le32(qp->qpid),
1560 			}
1561 		}
1562 	};
1563 	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1564 	void *hdr_buf = NULL;
1565 	dma_addr_t hdr_dma = 0;
1566 	int rc, hdr_len = 0;
1567 	u16 sport;
1568 
1569 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_QP)
1570 		return -EBADRQC;
1571 
1572 	if ((mask & IB_QP_MAX_DEST_RD_ATOMIC) && attr->max_dest_rd_atomic) {
1573 		/* Note, round up/down was already done for allocating
1574 		 * resources on the device. The allocation order is in cache
1575 		 * line size.  We can't use the order of the resource
1576 		 * allocation to determine the order wqes here, because for
1577 		 * queue length <= one cache line it is not distinct.
1578 		 *
1579 		 * Therefore, order wqes is computed again here.
1580 		 *
1581 		 * Account for hole and round up to the next order.
1582 		 */
1583 		wr.wqe.cmd.mod_qp.rsq_depth =
1584 			order_base_2(attr->max_dest_rd_atomic + 1);
1585 		wr.wqe.cmd.mod_qp.rsq_index = cpu_to_le32(~0);
1586 	}
1587 
1588 	if ((mask & IB_QP_MAX_QP_RD_ATOMIC) && attr->max_rd_atomic) {
1589 		/* Account for hole and round down to the next order */
1590 		wr.wqe.cmd.mod_qp.rrq_depth =
1591 			order_base_2(attr->max_rd_atomic + 2) - 1;
1592 		wr.wqe.cmd.mod_qp.rrq_index = cpu_to_le32(~0);
1593 	}
1594 
1595 	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC)
1596 		wr.wqe.cmd.mod_qp.qkey_dest_qpn =
1597 			cpu_to_le32(attr->dest_qp_num);
1598 	else
1599 		wr.wqe.cmd.mod_qp.qkey_dest_qpn = cpu_to_le32(attr->qkey);
1600 
1601 	if (mask & IB_QP_AV) {
1602 		if (!qp->hdr)
1603 			return -ENOMEM;
1604 
1605 		sport = rdma_get_udp_sport(grh->flow_label,
1606 					   qp->qpid,
1607 					   attr->dest_qp_num);
1608 
1609 		rc = ionic_build_hdr(dev, qp->hdr, &attr->ah_attr, sport, true);
1610 		if (rc)
1611 			return rc;
1612 
1613 		qp->sgid_index = grh->sgid_index;
1614 
1615 		hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1616 		if (!hdr_buf)
1617 			return -ENOMEM;
1618 
1619 		hdr_len = ib_ud_header_pack(qp->hdr, hdr_buf);
1620 		hdr_len -= IB_BTH_BYTES;
1621 		hdr_len -= IB_DETH_BYTES;
1622 		ibdev_dbg(&dev->ibdev, "roce packet header template\n");
1623 		print_hex_dump_debug("hdr ", DUMP_PREFIX_OFFSET, 16, 1,
1624 				     hdr_buf, hdr_len, true);
1625 
1626 		hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf, hdr_len,
1627 					 DMA_TO_DEVICE);
1628 
1629 		rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma);
1630 		if (rc)
1631 			goto err_dma;
1632 
1633 		if (qp->hdr->ipv4_present) {
1634 			wr.wqe.cmd.mod_qp.tfp_csum_profile =
1635 				qp->hdr->vlan_present ?
1636 					IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV4_UDP :
1637 					IONIC_TFP_CSUM_PROF_ETH_IPV4_UDP;
1638 		} else {
1639 			wr.wqe.cmd.mod_qp.tfp_csum_profile =
1640 				qp->hdr->vlan_present ?
1641 					IONIC_TFP_CSUM_PROF_ETH_QTAG_IPV6_UDP :
1642 					IONIC_TFP_CSUM_PROF_ETH_IPV6_UDP;
1643 		}
1644 
1645 		wr.wqe.cmd.mod_qp.ah_id_len =
1646 			cpu_to_le32(qp->ahid | (hdr_len << 24));
1647 		wr.wqe.cmd.mod_qp.dma_addr = cpu_to_le64(hdr_dma);
1648 
1649 		wr.wqe.cmd.mod_qp.en_pcp = attr->ah_attr.sl;
1650 		wr.wqe.cmd.mod_qp.ip_dscp = grh->traffic_class >> 2;
1651 	}
1652 
1653 	ionic_admin_post(dev, &wr);
1654 
1655 	rc = ionic_admin_wait(dev, &wr, 0);
1656 
1657 	if (mask & IB_QP_AV)
1658 		dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma, hdr_len,
1659 				 DMA_TO_DEVICE);
1660 err_dma:
1661 	if (mask & IB_QP_AV)
1662 		kfree(hdr_buf);
1663 
1664 	return rc;
1665 }
1666 
1667 static int ionic_query_qp_cmd(struct ionic_ibdev *dev,
1668 			      struct ionic_qp *qp,
1669 			      struct ib_qp_attr *attr,
1670 			      int mask)
1671 {
1672 	struct ionic_admin_wr wr = {
1673 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1674 		.wqe = {
1675 			.op = IONIC_V1_ADMIN_QUERY_QP,
1676 			.len = cpu_to_le16(IONIC_ADMIN_QUERY_QP_IN_V1_LEN),
1677 			.cmd.query_qp = {
1678 				.id_ver = cpu_to_le32(qp->qpid),
1679 			},
1680 		}
1681 	};
1682 	struct ionic_v1_admin_query_qp_sq *query_sqbuf;
1683 	struct ionic_v1_admin_query_qp_rq *query_rqbuf;
1684 	dma_addr_t query_sqdma;
1685 	dma_addr_t query_rqdma;
1686 	dma_addr_t hdr_dma = 0;
1687 	void *hdr_buf = NULL;
1688 	int flags, rc;
1689 
1690 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_QUERY_QP)
1691 		return -EBADRQC;
1692 
1693 	if (qp->has_sq) {
1694 		bool expdb = !!(qp->sq_cmb & IONIC_CMB_EXPDB);
1695 
1696 		attr->cap.max_send_sge =
1697 			ionic_v1_send_wqe_max_sge(qp->sq.stride_log2,
1698 						  qp->sq_spec,
1699 						  expdb);
1700 		attr->cap.max_inline_data =
1701 			ionic_v1_send_wqe_max_data(qp->sq.stride_log2, expdb);
1702 	}
1703 
1704 	if (!qp->srq) {
1705 		attr->cap.max_recv_sge =
1706 			ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2,
1707 						  qp->rq.spec,
1708 						  qp->rq.cmb & IONIC_CMB_EXPDB);
1709 	}
1710 
1711 	query_sqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1712 	if (!query_sqbuf)
1713 		return -ENOMEM;
1714 
1715 	query_rqbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
1716 	if (!query_rqbuf) {
1717 		rc = -ENOMEM;
1718 		goto err_rqbuf;
1719 	}
1720 
1721 	query_sqdma = dma_map_single(dev->lif_cfg.hwdev, query_sqbuf, PAGE_SIZE,
1722 				     DMA_FROM_DEVICE);
1723 	rc = dma_mapping_error(dev->lif_cfg.hwdev, query_sqdma);
1724 	if (rc)
1725 		goto err_sqdma;
1726 
1727 	query_rqdma = dma_map_single(dev->lif_cfg.hwdev, query_rqbuf, PAGE_SIZE,
1728 				     DMA_FROM_DEVICE);
1729 	rc = dma_mapping_error(dev->lif_cfg.hwdev, query_rqdma);
1730 	if (rc)
1731 		goto err_rqdma;
1732 
1733 	if (mask & IB_QP_AV) {
1734 		hdr_buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
1735 		if (!hdr_buf) {
1736 			rc = -ENOMEM;
1737 			goto err_hdrbuf;
1738 		}
1739 
1740 		hdr_dma = dma_map_single(dev->lif_cfg.hwdev, hdr_buf,
1741 					 PAGE_SIZE, DMA_FROM_DEVICE);
1742 		rc = dma_mapping_error(dev->lif_cfg.hwdev, hdr_dma);
1743 		if (rc)
1744 			goto err_hdrdma;
1745 	}
1746 
1747 	wr.wqe.cmd.query_qp.sq_dma_addr = cpu_to_le64(query_sqdma);
1748 	wr.wqe.cmd.query_qp.rq_dma_addr = cpu_to_le64(query_rqdma);
1749 	wr.wqe.cmd.query_qp.hdr_dma_addr = cpu_to_le64(hdr_dma);
1750 	wr.wqe.cmd.query_qp.ah_id = cpu_to_le32(qp->ahid);
1751 
1752 	ionic_admin_post(dev, &wr);
1753 
1754 	rc = ionic_admin_wait(dev, &wr, 0);
1755 
1756 	if (rc)
1757 		goto err_hdrdma;
1758 
1759 	flags = be16_to_cpu(query_sqbuf->access_perms_flags |
1760 			    query_rqbuf->access_perms_flags);
1761 
1762 	print_hex_dump_debug("sqbuf ", DUMP_PREFIX_OFFSET, 16, 1,
1763 			     query_sqbuf, sizeof(*query_sqbuf), true);
1764 	print_hex_dump_debug("rqbuf ", DUMP_PREFIX_OFFSET, 16, 1,
1765 			     query_rqbuf, sizeof(*query_rqbuf), true);
1766 	ibdev_dbg(&dev->ibdev, "query qp %u state_pmtu %#x flags %#x",
1767 		  qp->qpid, query_rqbuf->state_pmtu, flags);
1768 
1769 	attr->qp_state = from_ionic_qp_state(query_rqbuf->state_pmtu >> 4);
1770 	attr->cur_qp_state = attr->qp_state;
1771 	attr->path_mtu = (query_rqbuf->state_pmtu & 0xf) - 7;
1772 	attr->path_mig_state = IB_MIG_MIGRATED;
1773 	attr->qkey = be32_to_cpu(query_sqbuf->qkey_dest_qpn);
1774 	attr->rq_psn = be32_to_cpu(query_sqbuf->rq_psn);
1775 	attr->sq_psn = be32_to_cpu(query_rqbuf->sq_psn);
1776 	attr->dest_qp_num = attr->qkey;
1777 	attr->qp_access_flags = from_ionic_qp_flags(flags);
1778 	attr->pkey_index = 0;
1779 	attr->alt_pkey_index = 0;
1780 	attr->en_sqd_async_notify = !!(flags & IONIC_QPF_SQD_NOTIFY);
1781 	attr->sq_draining = !!(flags & IONIC_QPF_SQ_DRAINING);
1782 	attr->max_rd_atomic = BIT(query_rqbuf->rrq_depth) - 1;
1783 	attr->max_dest_rd_atomic = BIT(query_rqbuf->rsq_depth) - 1;
1784 	attr->min_rnr_timer = query_sqbuf->rnr_timer;
1785 	attr->port_num = 0;
1786 	attr->timeout = query_sqbuf->retry_timeout;
1787 	attr->retry_cnt = query_rqbuf->retry_rnrtry & 0xf;
1788 	attr->rnr_retry = query_rqbuf->retry_rnrtry >> 4;
1789 	attr->alt_port_num = 0;
1790 	attr->alt_timeout = 0;
1791 	attr->rate_limit = be32_to_cpu(query_sqbuf->rate_limit_kbps);
1792 
1793 	if (mask & IB_QP_AV)
1794 		ionic_set_ah_attr(dev, &attr->ah_attr,
1795 				  qp->hdr, qp->sgid_index);
1796 
1797 err_hdrdma:
1798 	if (mask & IB_QP_AV) {
1799 		dma_unmap_single(dev->lif_cfg.hwdev, hdr_dma,
1800 				 PAGE_SIZE, DMA_FROM_DEVICE);
1801 		kfree(hdr_buf);
1802 	}
1803 err_hdrbuf:
1804 	dma_unmap_single(dev->lif_cfg.hwdev, query_rqdma, sizeof(*query_rqbuf),
1805 			 DMA_FROM_DEVICE);
1806 err_rqdma:
1807 	dma_unmap_single(dev->lif_cfg.hwdev, query_sqdma, sizeof(*query_sqbuf),
1808 			 DMA_FROM_DEVICE);
1809 err_sqdma:
1810 	kfree(query_rqbuf);
1811 err_rqbuf:
1812 	kfree(query_sqbuf);
1813 
1814 	return rc;
1815 }
1816 
1817 static int ionic_destroy_qp_cmd(struct ionic_ibdev *dev, u32 qpid)
1818 {
1819 	struct ionic_admin_wr wr = {
1820 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
1821 		.wqe = {
1822 			.op = IONIC_V1_ADMIN_DESTROY_QP,
1823 			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_QP_IN_V1_LEN),
1824 			.cmd.destroy_qp = {
1825 				.qp_id = cpu_to_le32(qpid),
1826 			},
1827 		}
1828 	};
1829 
1830 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_QP)
1831 		return -EBADRQC;
1832 
1833 	ionic_admin_post(dev, &wr);
1834 
1835 	return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);
1836 }
1837 
1838 static bool ionic_expdb_wqe_size_supported(struct ionic_ibdev *dev,
1839 					   uint32_t wqe_size)
1840 {
1841 	switch (wqe_size) {
1842 	case 64: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_64;
1843 	case 128: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_128;
1844 	case 256: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_256;
1845 	case 512: return dev->lif_cfg.expdb_mask & IONIC_EXPDB_512;
1846 	}
1847 
1848 	return false;
1849 }
1850 
1851 static void ionic_qp_sq_init_cmb(struct ionic_ibdev *dev,
1852 				 struct ionic_qp *qp,
1853 				 struct ib_udata *udata,
1854 				 int max_data)
1855 {
1856 	u8 expdb_stride_log2 = 0;
1857 	bool expdb;
1858 	int rc;
1859 
1860 	if (!(qp->sq_cmb & IONIC_CMB_ENABLE))
1861 		goto not_in_cmb;
1862 
1863 	if (qp->sq_cmb & ~IONIC_CMB_SUPPORTED) {
1864 		if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1865 			goto not_in_cmb;
1866 
1867 		qp->sq_cmb &= IONIC_CMB_SUPPORTED;
1868 	}
1869 
1870 	if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.sq_expdb) {
1871 		if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1872 			goto not_in_cmb;
1873 
1874 		qp->sq_cmb &= ~IONIC_CMB_EXPDB;
1875 	}
1876 
1877 	qp->sq_cmb_order = order_base_2(qp->sq.size / PAGE_SIZE);
1878 
1879 	if (qp->sq_cmb_order >= IONIC_SQCMB_ORDER)
1880 		goto not_in_cmb;
1881 
1882 	if (qp->sq_cmb & IONIC_CMB_EXPDB)
1883 		expdb_stride_log2 = qp->sq.stride_log2;
1884 
1885 	rc = ionic_get_cmb(dev->lif_cfg.lif, &qp->sq_cmb_pgid,
1886 			   &qp->sq_cmb_addr, qp->sq_cmb_order,
1887 			   expdb_stride_log2, &expdb);
1888 	if (rc)
1889 		goto not_in_cmb;
1890 
1891 	if ((qp->sq_cmb & IONIC_CMB_EXPDB) && !expdb) {
1892 		if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1893 			goto err_map;
1894 
1895 		qp->sq_cmb &= ~IONIC_CMB_EXPDB;
1896 	}
1897 
1898 	return;
1899 
1900 err_map:
1901 	ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order);
1902 not_in_cmb:
1903 	if (qp->sq_cmb & IONIC_CMB_REQUIRE)
1904 		ibdev_dbg(&dev->ibdev, "could not place sq in cmb as required\n");
1905 
1906 	qp->sq_cmb = 0;
1907 	qp->sq_cmb_order = IONIC_RES_INVALID;
1908 	qp->sq_cmb_pgid = 0;
1909 	qp->sq_cmb_addr = 0;
1910 }
1911 
1912 static void ionic_qp_sq_destroy_cmb(struct ionic_ibdev *dev,
1913 				    struct ionic_ctx *ctx,
1914 				    struct ionic_qp *qp)
1915 {
1916 	if (!(qp->sq_cmb & IONIC_CMB_ENABLE))
1917 		return;
1918 
1919 	if (ctx)
1920 		rdma_user_mmap_entry_remove(qp->mmap_sq_cmb);
1921 
1922 	ionic_put_cmb(dev->lif_cfg.lif, qp->sq_cmb_pgid, qp->sq_cmb_order);
1923 }
1924 
1925 static int ionic_qp_sq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
1926 			    struct ionic_qp *qp, struct ionic_qdesc *sq,
1927 			    struct ionic_tbl_buf *buf, int max_wr, int max_sge,
1928 			    int max_data, int sq_spec, struct ib_udata *udata)
1929 {
1930 	u32 wqe_size;
1931 	int rc = 0;
1932 
1933 	qp->sq_msn_prod = 0;
1934 	qp->sq_msn_cons = 0;
1935 
1936 	if (!qp->has_sq) {
1937 		if (buf) {
1938 			buf->tbl_buf = NULL;
1939 			buf->tbl_limit = 0;
1940 			buf->tbl_pages = 0;
1941 		}
1942 		if (udata)
1943 			rc = ionic_validate_qdesc_zero(sq);
1944 
1945 		return rc;
1946 	}
1947 
1948 	rc = -EINVAL;
1949 
1950 	if (max_wr < 0 || max_wr > 0xffff)
1951 		return rc;
1952 
1953 	if (max_sge < 1)
1954 		return rc;
1955 
1956 	if (max_sge > min(ionic_v1_send_wqe_max_sge(dev->lif_cfg.max_stride, 0,
1957 						    qp->sq_cmb &
1958 						    IONIC_CMB_EXPDB),
1959 			  IONIC_SPEC_HIGH))
1960 		return rc;
1961 
1962 	if (max_data < 0)
1963 		return rc;
1964 
1965 	if (max_data > ionic_v1_send_wqe_max_data(dev->lif_cfg.max_stride,
1966 						  qp->sq_cmb & IONIC_CMB_EXPDB))
1967 		return rc;
1968 
1969 	if (udata) {
1970 		rc = ionic_validate_qdesc(sq);
1971 		if (rc)
1972 			return rc;
1973 
1974 		qp->sq_spec = sq_spec;
1975 
1976 		qp->sq.ptr = NULL;
1977 		qp->sq.size = sq->size;
1978 		qp->sq.mask = sq->mask;
1979 		qp->sq.depth_log2 = sq->depth_log2;
1980 		qp->sq.stride_log2 = sq->stride_log2;
1981 
1982 		qp->sq_meta = NULL;
1983 		qp->sq_msn_idx = NULL;
1984 
1985 		qp->sq_umem = ib_umem_get_va(&dev->ibdev, sq->addr, sq->size, 0);
1986 		if (IS_ERR(qp->sq_umem))
1987 			return PTR_ERR(qp->sq_umem);
1988 	} else {
1989 		qp->sq_umem = NULL;
1990 
1991 		qp->sq_spec = ionic_v1_use_spec_sge(max_sge, sq_spec);
1992 		if (sq_spec && !qp->sq_spec)
1993 			ibdev_dbg(&dev->ibdev,
1994 				  "init sq: max_sge %u disables spec\n",
1995 				  max_sge);
1996 
1997 		if (qp->sq_cmb & IONIC_CMB_EXPDB) {
1998 			wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data,
1999 							      qp->sq_spec,
2000 							      true);
2001 
2002 			if (!ionic_expdb_wqe_size_supported(dev, wqe_size))
2003 				qp->sq_cmb &= ~IONIC_CMB_EXPDB;
2004 		}
2005 
2006 		if (!(qp->sq_cmb & IONIC_CMB_EXPDB))
2007 			wqe_size = ionic_v1_send_wqe_min_size(max_sge, max_data,
2008 							      qp->sq_spec,
2009 							      false);
2010 
2011 		rc = ionic_queue_init(&qp->sq, dev->lif_cfg.hwdev,
2012 				      max_wr, wqe_size);
2013 		if (rc)
2014 			return rc;
2015 
2016 		ionic_queue_dbell_init(&qp->sq, qp->qpid);
2017 
2018 		qp->sq_meta = kmalloc_objs(*qp->sq_meta, (u32)qp->sq.mask + 1);
2019 		if (!qp->sq_meta) {
2020 			rc = -ENOMEM;
2021 			goto err_sq_meta;
2022 		}
2023 
2024 		qp->sq_msn_idx = kmalloc_objs(*qp->sq_msn_idx,
2025 					      (u32)qp->sq.mask + 1);
2026 		if (!qp->sq_msn_idx) {
2027 			rc = -ENOMEM;
2028 			goto err_sq_msn;
2029 		}
2030 	}
2031 
2032 	ionic_qp_sq_init_cmb(dev, qp, udata, max_data);
2033 
2034 	if (qp->sq_cmb & IONIC_CMB_ENABLE)
2035 		rc = ionic_pgtbl_init(dev, buf, NULL,
2036 				      (u64)qp->sq_cmb_pgid << PAGE_SHIFT,
2037 				      1, PAGE_SIZE);
2038 	else
2039 		rc = ionic_pgtbl_init(dev, buf,
2040 				      qp->sq_umem, qp->sq.dma, 1, PAGE_SIZE);
2041 	if (rc)
2042 		goto err_sq_tbl;
2043 
2044 	return 0;
2045 
2046 err_sq_tbl:
2047 	ionic_qp_sq_destroy_cmb(dev, ctx, qp);
2048 	kfree(qp->sq_msn_idx);
2049 err_sq_msn:
2050 	kfree(qp->sq_meta);
2051 err_sq_meta:
2052 	if (qp->sq_umem)
2053 		ib_umem_release(qp->sq_umem);
2054 	else
2055 		ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev);
2056 	return rc;
2057 }
2058 
2059 static void ionic_qp_sq_destroy(struct ionic_ibdev *dev,
2060 				struct ionic_ctx *ctx,
2061 				struct ionic_qp *qp)
2062 {
2063 	if (!qp->has_sq)
2064 		return;
2065 
2066 	ionic_qp_sq_destroy_cmb(dev, ctx, qp);
2067 
2068 	kfree(qp->sq_msn_idx);
2069 	kfree(qp->sq_meta);
2070 
2071 	if (qp->sq_umem)
2072 		ib_umem_release(qp->sq_umem);
2073 	else
2074 		ionic_queue_destroy(&qp->sq, dev->lif_cfg.hwdev);
2075 }
2076 
2077 static void ionic_rq_mmap_cmb(struct ionic_ibdev *dev,
2078 			      struct ionic_ctx *ctx,
2079 			      struct ionic_rq *rq,
2080 			      u64 *cmb_offset)
2081 {
2082 	bool wc;
2083 
2084 	/* set mapping by default to uncached for
2085 	 * expdb (to guarantee writes order) otherwise
2086 	 * writecombine, unless this default is
2087 	 * overridden by userspace
2088 	 */
2089 	if ((rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) ==
2090 		(IONIC_CMB_WC | IONIC_CMB_UC)) {
2091 		ibdev_dbg(&dev->ibdev,
2092 			  "Both rq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC set, using default driver mapping\n");
2093 		rq->cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC);
2094 	}
2095 
2096 	if (rq->cmb & IONIC_CMB_EXPDB)
2097 		wc  = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) == IONIC_CMB_WC;
2098 	else
2099 		wc = (rq->cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) != IONIC_CMB_UC;
2100 
2101 	/* let userspace know the mapping */
2102 	if (wc)
2103 		rq->cmb |= IONIC_CMB_WC;
2104 	else
2105 		rq->cmb |= IONIC_CMB_UC;
2106 
2107 	rq->mmap_cmb = ionic_mmap_entry_insert(ctx, rq->q.size,
2108 					       PHYS_PFN(rq->cmb_addr),
2109 					       wc ? IONIC_MMAP_WC : 0,
2110 					       cmb_offset);
2111 }
2112 
2113 static void ionic_rq_init_cmb(struct ionic_ibdev *dev,
2114 			      struct ionic_rq *rq,
2115 			      struct ib_udata *udata)
2116 {
2117 	u8 expdb_stride_log2 = 0;
2118 	bool expdb;
2119 	int rc;
2120 
2121 	if (!(rq->cmb & IONIC_CMB_ENABLE))
2122 		goto not_in_cmb;
2123 
2124 	if (rq->cmb & ~IONIC_CMB_SUPPORTED) {
2125 		if (rq->cmb & IONIC_CMB_REQUIRE)
2126 			goto not_in_cmb;
2127 
2128 		rq->cmb &= IONIC_CMB_SUPPORTED;
2129 	}
2130 
2131 	if ((rq->cmb & IONIC_CMB_EXPDB) && !dev->lif_cfg.rq_expdb) {
2132 		if (rq->cmb & IONIC_CMB_REQUIRE)
2133 			goto not_in_cmb;
2134 
2135 		rq->cmb &= ~IONIC_CMB_EXPDB;
2136 	}
2137 
2138 	rq->cmb_order = order_base_2(rq->q.size / PAGE_SIZE);
2139 
2140 	if (rq->cmb_order >= IONIC_RQCMB_ORDER)
2141 		goto not_in_cmb;
2142 
2143 	if (rq->cmb & IONIC_CMB_EXPDB)
2144 		expdb_stride_log2 = rq->q.stride_log2;
2145 
2146 	rc = ionic_get_cmb(dev->lif_cfg.lif, &rq->cmb_pgid,
2147 			   &rq->cmb_addr, rq->cmb_order,
2148 			   expdb_stride_log2, &expdb);
2149 	if (rc)
2150 		goto not_in_cmb;
2151 
2152 	if ((rq->cmb & IONIC_CMB_EXPDB) && !expdb) {
2153 		if (rq->cmb & IONIC_CMB_REQUIRE)
2154 			goto err_map;
2155 
2156 		rq->cmb &= ~IONIC_CMB_EXPDB;
2157 	}
2158 
2159 	return;
2160 
2161 err_map:
2162 	ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order);
2163 not_in_cmb:
2164 	if (rq->cmb & IONIC_CMB_REQUIRE)
2165 		ibdev_dbg(&dev->ibdev, "could not place rq in cmb as required\n");
2166 
2167 	rq->cmb = 0;
2168 	rq->cmb_order = IONIC_RES_INVALID;
2169 	rq->cmb_pgid = 0;
2170 	rq->cmb_addr = 0;
2171 }
2172 
2173 static void ionic_rq_destroy_cmb(struct ionic_ibdev *dev,
2174 				 struct ionic_ctx *ctx,
2175 				 struct ionic_rq *rq)
2176 {
2177 	if (!(rq->cmb & IONIC_CMB_ENABLE))
2178 		return;
2179 
2180 	if (ctx)
2181 		rdma_user_mmap_entry_remove(rq->mmap_cmb);
2182 
2183 	ionic_put_cmb(dev->lif_cfg.lif, rq->cmb_pgid, rq->cmb_order);
2184 }
2185 
2186 static int ionic_rq_init(struct ionic_ibdev *dev, struct ionic_ctx *ctx,
2187 			 struct ionic_rq *rq, struct ionic_qdesc *qdesc,
2188 			 struct ionic_tbl_buf *buf, int max_wr, int max_sge,
2189 			 int rq_spec, struct ib_udata *udata)
2190 {
2191 	int rc = 0, i;
2192 	u32 wqe_size;
2193 
2194 	rc = -EINVAL;
2195 
2196 	if (max_wr < 0 || max_wr > 0xffff)
2197 		return rc;
2198 
2199 	if (max_sge < 1)
2200 		return rc;
2201 
2202 	if (max_sge > min(ionic_v1_recv_wqe_max_sge(dev->lif_cfg.max_stride, 0, false),
2203 			  IONIC_SPEC_HIGH))
2204 		return rc;
2205 
2206 	if (udata) {
2207 		rc = ionic_validate_qdesc(qdesc);
2208 		if (rc)
2209 			return rc;
2210 
2211 		rq->spec = rq_spec;
2212 
2213 		rq->q.ptr = NULL;
2214 		rq->q.size = qdesc->size;
2215 		rq->q.mask = qdesc->mask;
2216 		rq->q.depth_log2 = qdesc->depth_log2;
2217 		rq->q.stride_log2 = qdesc->stride_log2;
2218 
2219 		rq->meta = NULL;
2220 
2221 		rq->umem = ib_umem_get_va(&dev->ibdev, qdesc->addr, qdesc->size, 0);
2222 		if (IS_ERR(rq->umem))
2223 			return PTR_ERR(rq->umem);
2224 	} else {
2225 		rq->umem = NULL;
2226 
2227 		rq->spec = ionic_v1_use_spec_sge(max_sge, rq_spec);
2228 		if (rq_spec && !rq->spec)
2229 			ibdev_dbg(&dev->ibdev,
2230 				  "init rq: max_sge %u disables spec\n",
2231 				  max_sge);
2232 
2233 		if (rq->cmb & IONIC_CMB_EXPDB) {
2234 			wqe_size = ionic_v1_recv_wqe_min_size(max_sge,
2235 							      rq->spec,
2236 							      true);
2237 
2238 			if (!ionic_expdb_wqe_size_supported(dev, wqe_size))
2239 				rq->cmb &= ~IONIC_CMB_EXPDB;
2240 		}
2241 
2242 		if (!(rq->cmb & IONIC_CMB_EXPDB))
2243 			wqe_size = ionic_v1_recv_wqe_min_size(max_sge,
2244 							      rq->spec,
2245 							      false);
2246 
2247 		rc = ionic_queue_init(&rq->q, dev->lif_cfg.hwdev,
2248 				      max_wr, wqe_size);
2249 		if (rc)
2250 			return rc;
2251 
2252 		ionic_queue_dbell_init(&rq->q, rq->qid);
2253 
2254 		rq->meta = kmalloc_objs(*rq->meta, (u32)rq->q.mask + 1);
2255 		if (!rq->meta) {
2256 			rc = -ENOMEM;
2257 			goto err_rq_meta;
2258 		}
2259 
2260 		for (i = 0; i < rq->q.mask; ++i)
2261 			rq->meta[i].next = &rq->meta[i + 1];
2262 		rq->meta[i].next = IONIC_META_LAST;
2263 		rq->meta_head = &rq->meta[0];
2264 	}
2265 
2266 	ionic_rq_init_cmb(dev, rq, udata);
2267 
2268 	if (rq->cmb & IONIC_CMB_ENABLE)
2269 		rc = ionic_pgtbl_init(dev, buf, NULL,
2270 				      (u64)rq->cmb_pgid << PAGE_SHIFT,
2271 				      1, PAGE_SIZE);
2272 	else
2273 		rc = ionic_pgtbl_init(dev, buf, rq->umem,
2274 				      rq->q.dma, 1, PAGE_SIZE);
2275 	if (rc)
2276 		goto err_rq_tbl;
2277 
2278 	return 0;
2279 
2280 err_rq_tbl:
2281 	ionic_rq_destroy_cmb(dev, ctx, rq);
2282 	kfree(rq->meta);
2283 err_rq_meta:
2284 	if (rq->umem)
2285 		ib_umem_release(rq->umem);
2286 	else
2287 		ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev);
2288 	return rc;
2289 }
2290 
2291 static void ionic_rq_destroy(struct ionic_ibdev *dev,
2292 			     struct ionic_ctx *ctx,
2293 			     struct ionic_rq *rq)
2294 {
2295 	ionic_rq_destroy_cmb(dev, ctx, rq);
2296 
2297 	kfree(rq->meta);
2298 
2299 	if (rq->umem)
2300 		ib_umem_release(rq->umem);
2301 	else
2302 		ionic_queue_destroy(&rq->q, dev->lif_cfg.hwdev);
2303 }
2304 
2305 int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
2306 		    struct ib_udata *udata)
2307 {
2308 	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2309 	struct ionic_tbl_buf sq_buf = {}, rq_buf = {};
2310 	struct ionic_pd *pd = to_ionic_pd(ibqp->pd);
2311 	struct ionic_qp *qp = to_ionic_qp(ibqp);
2312 	struct ionic_ctx *ctx =
2313 		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
2314 	struct ionic_qp_resp resp = {};
2315 	struct ionic_qp_req req = {};
2316 	struct ionic_cq *cq;
2317 	u8 udma_mask;
2318 	void *entry;
2319 	int rc;
2320 
2321 	if (udata) {
2322 		rc = ib_copy_validate_udata_in(udata, req, rsvd);
2323 		if (rc)
2324 			return rc;
2325 
2326 		if (req.ionic_flags & ~IONIC_QP_USER_FLAGS_MASK)
2327 			return -EINVAL;
2328 
2329 		if (req.rsvd_pad)
2330 			return -EINVAL;
2331 	} else {
2332 		req.sq_spec = IONIC_SPEC_HIGH;
2333 		req.rq_spec = IONIC_SPEC_HIGH;
2334 	}
2335 
2336 	if (attr->qp_type == IB_QPT_SMI || attr->qp_type > IB_QPT_UD)
2337 		return -EOPNOTSUPP;
2338 
2339 	qp->state = IB_QPS_RESET;
2340 
2341 	INIT_LIST_HEAD(&qp->cq_poll_sq);
2342 	INIT_LIST_HEAD(&qp->cq_flush_sq);
2343 	INIT_LIST_HEAD(&qp->rq.cq_flush);
2344 
2345 	spin_lock_init(&qp->sq_lock);
2346 	spin_lock_init(&qp->rq.lock);
2347 
2348 	qp->has_sq = 1;
2349 
2350 	if (attr->srq)
2351 		qp->srq = to_ionic_srq(attr->srq);
2352 	else
2353 		qp->srq = NULL;
2354 
2355 	if (attr->qp_type == IB_QPT_GSI) {
2356 		rc = ionic_get_gsi_qpid(dev, &qp->qpid);
2357 	} else {
2358 		udma_mask = BIT(dev->lif_cfg.udma_count) - 1;
2359 
2360 		if (attr->send_cq)
2361 			udma_mask &= to_ionic_vcq(attr->send_cq)->udma_mask;
2362 
2363 		if (attr->recv_cq)
2364 			udma_mask &= to_ionic_vcq(attr->recv_cq)->udma_mask;
2365 
2366 		if (attr->srq)
2367 			udma_mask &= BIT(to_ionic_srq(attr->srq)->udma_idx);
2368 
2369 		if (udata && req.udma_mask)
2370 			udma_mask &= req.udma_mask;
2371 
2372 		if (!udma_mask)
2373 			return -EINVAL;
2374 
2375 		rc = ionic_get_qpid(dev, &qp->qpid, &qp->udma_idx, udma_mask);
2376 	}
2377 	if (rc)
2378 		return rc;
2379 
2380 	qp->sig_all = attr->sq_sig_type == IB_SIGNAL_ALL_WR;
2381 	qp->has_ah = attr->qp_type == IB_QPT_RC;
2382 
2383 	if (qp->has_ah) {
2384 		qp->hdr = kzalloc_obj(*qp->hdr);
2385 		if (!qp->hdr) {
2386 			rc = -ENOMEM;
2387 			goto err_ah_alloc;
2388 		}
2389 
2390 		rc = ionic_get_ahid(dev, &qp->ahid);
2391 		if (rc)
2392 			goto err_ahid;
2393 	}
2394 
2395 	if (udata && (req.sq_cmb & IONIC_CMB_ENABLE))
2396 		qp->sq_cmb = req.sq_cmb;
2397 
2398 	rc = ionic_qp_sq_init(dev, ctx, qp, &req.sq, &sq_buf,
2399 			      attr->cap.max_send_wr, attr->cap.max_send_sge,
2400 			      attr->cap.max_inline_data, req.sq_spec, udata);
2401 	if (rc)
2402 		goto err_sq;
2403 
2404 	if (qp->srq) {
2405 		rq_buf.tbl_buf = NULL;
2406 		rq_buf.tbl_limit = 0;
2407 		rq_buf.tbl_pages = 0;
2408 
2409 		if (udata)
2410 			rc = ionic_validate_qdesc_zero(&req.rq);
2411 	} else {
2412 		if (udata && (req.rq_cmb & IONIC_CMB_ENABLE))
2413 			qp->rq.cmb = req.rq_cmb;
2414 
2415 		/* for non-srq qps, rq qid is same as qpid */
2416 		qp->rq.qid = qp->qpid;
2417 		rc = ionic_rq_init(dev, ctx, &qp->rq, &req.rq, &rq_buf,
2418 				   attr->cap.max_recv_wr, attr->cap.max_recv_sge,
2419 				   req.rq_spec, udata);
2420 	}
2421 	if (rc)
2422 		goto err_rq;
2423 
2424 	rc = ionic_create_qp_cmd(dev, pd,
2425 				 to_ionic_vcq_cq(attr->send_cq, qp->udma_idx),
2426 				 to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx),
2427 				 qp, &sq_buf, &rq_buf, attr, req.ionic_flags);
2428 	if (rc)
2429 		goto err_cmd;
2430 
2431 	if (udata) {
2432 		resp.qpid = qp->qpid;
2433 		resp.udma_idx = qp->udma_idx;
2434 
2435 		if (qp->sq_cmb & IONIC_CMB_ENABLE) {
2436 			bool wc;
2437 
2438 			if ((qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC)) ==
2439 				(IONIC_CMB_WC | IONIC_CMB_UC)) {
2440 				ibdev_dbg(&dev->ibdev,
2441 					  "Both sq_cmb flags IONIC_CMB_WC and IONIC_CMB_UC are set, using default driver mapping\n");
2442 				qp->sq_cmb &= ~(IONIC_CMB_WC | IONIC_CMB_UC);
2443 			}
2444 
2445 			wc = (qp->sq_cmb & (IONIC_CMB_WC | IONIC_CMB_UC))
2446 					!= IONIC_CMB_UC;
2447 
2448 			/* let userspace know the mapping */
2449 			if (wc)
2450 				qp->sq_cmb |= IONIC_CMB_WC;
2451 			else
2452 				qp->sq_cmb |= IONIC_CMB_UC;
2453 
2454 			qp->mmap_sq_cmb =
2455 			    ionic_mmap_entry_insert(ctx,
2456 						    qp->sq.size,
2457 						    PHYS_PFN(qp->sq_cmb_addr),
2458 						    wc ? IONIC_MMAP_WC : 0,
2459 						    &resp.sq_cmb_offset);
2460 			if (!qp->mmap_sq_cmb) {
2461 				rc = -ENOMEM;
2462 				goto err_qp_cmd;
2463 			}
2464 
2465 			resp.sq_cmb = qp->sq_cmb;
2466 		}
2467 
2468 		if (!qp->srq && (qp->rq.cmb & IONIC_CMB_ENABLE)) {
2469 			ionic_rq_mmap_cmb(dev, ctx, &qp->rq, &resp.rq_cmb_offset);
2470 
2471 			if (!qp->rq.mmap_cmb) {
2472 				rc = -ENOMEM;
2473 				goto err_qp_cmd;
2474 			}
2475 
2476 			resp.rq_cmb = qp->rq.cmb;
2477 		}
2478 
2479 		rc = ib_respond_udata(udata, resp);
2480 		if (rc)
2481 			goto err_qp_cmd;
2482 	}
2483 
2484 	ionic_pgtbl_unbuf(dev, &rq_buf);
2485 	ionic_pgtbl_unbuf(dev, &sq_buf);
2486 
2487 	qp->ibqp.qp_num = qp->qpid;
2488 
2489 	init_completion(&qp->qp_rel_comp);
2490 	kref_init(&qp->qp_kref);
2491 
2492 	entry = xa_store_irq(&dev->qp_tbl, qp->qpid, qp, GFP_KERNEL);
2493 	if (entry) {
2494 		if (!xa_is_err(entry))
2495 			rc = -EINVAL;
2496 		else
2497 			rc = xa_err(entry);
2498 
2499 		goto err_qp_cmd;
2500 	}
2501 
2502 	if (qp->has_sq) {
2503 		cq = to_ionic_vcq_cq(attr->send_cq, qp->udma_idx);
2504 
2505 		attr->cap.max_send_wr = qp->sq.mask;
2506 		attr->cap.max_send_sge =
2507 			ionic_v1_send_wqe_max_sge(qp->sq.stride_log2,
2508 						  qp->sq_spec,
2509 						  qp->sq_cmb & IONIC_CMB_EXPDB);
2510 		attr->cap.max_inline_data =
2511 			ionic_v1_send_wqe_max_data(qp->sq.stride_log2,
2512 						   qp->sq_cmb &
2513 						   IONIC_CMB_EXPDB);
2514 		qp->sq_cqid = cq->cqid;
2515 	}
2516 
2517 	if (!qp->srq) {
2518 		cq = to_ionic_vcq_cq(attr->recv_cq, qp->udma_idx);
2519 
2520 		attr->cap.max_recv_wr = qp->rq.q.mask;
2521 		attr->cap.max_recv_sge =
2522 			ionic_v1_recv_wqe_max_sge(qp->rq.q.stride_log2,
2523 						  qp->rq.spec,
2524 						  qp->rq.cmb & IONIC_CMB_EXPDB);
2525 		qp->rq.cqid = cq->cqid;
2526 	}
2527 
2528 	return 0;
2529 
2530 err_qp_cmd:
2531 	ionic_destroy_qp_cmd(dev, qp->qpid);
2532 err_cmd:
2533 	if (!qp->srq) {
2534 		ionic_pgtbl_unbuf(dev, &rq_buf);
2535 		ionic_rq_destroy(dev, ctx, &qp->rq);
2536 	}
2537 err_rq:
2538 	ionic_pgtbl_unbuf(dev, &sq_buf);
2539 	ionic_qp_sq_destroy(dev, ctx, qp);
2540 err_sq:
2541 	if (qp->has_ah)
2542 		ionic_put_ahid(dev, qp->ahid);
2543 err_ahid:
2544 	kfree(qp->hdr);
2545 err_ah_alloc:
2546 	ionic_put_qpid(dev, qp->qpid);
2547 	return rc;
2548 }
2549 
2550 void ionic_notify_flush_cq(struct ionic_cq *cq)
2551 {
2552 	if (cq->flush && cq->vcq->ibcq.comp_handler)
2553 		cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq,
2554 					   cq->vcq->ibcq.cq_context);
2555 }
2556 
2557 static void ionic_notify_qp_cqs(struct ionic_ibdev *dev, struct ionic_qp *qp)
2558 {
2559 	if (qp->ibqp.send_cq)
2560 		ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.send_cq,
2561 						      qp->udma_idx));
2562 	if (qp->ibqp.recv_cq && qp->ibqp.recv_cq != qp->ibqp.send_cq)
2563 		ionic_notify_flush_cq(to_ionic_vcq_cq(qp->ibqp.recv_cq,
2564 						      qp->udma_idx));
2565 }
2566 
2567 void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)
2568 {
2569 	unsigned long irqflags;
2570 	struct ionic_cq *cq;
2571 
2572 	if (qp->ibqp.send_cq) {
2573 		cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx);
2574 
2575 		/* Hold the CQ lock and QP sq_lock to set up flush */
2576 		spin_lock_irqsave(&cq->lock, irqflags);
2577 		spin_lock(&qp->sq_lock);
2578 		qp->sq_flush = true;
2579 		if (!ionic_queue_empty(&qp->sq)) {
2580 			cq->flush = true;
2581 			list_move_tail(&qp->cq_flush_sq, &cq->flush_sq);
2582 		}
2583 		spin_unlock(&qp->sq_lock);
2584 		spin_unlock_irqrestore(&cq->lock, irqflags);
2585 	}
2586 
2587 	if (qp->ibqp.recv_cq && !qp->srq) {
2588 		cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
2589 
2590 		/* Hold the CQ lock and QP rq.lock to set up flush */
2591 		spin_lock_irqsave(&cq->lock, irqflags);
2592 		spin_lock(&qp->rq.lock);
2593 		qp->rq.flush = true;
2594 		if (!ionic_queue_empty(&qp->rq.q)) {
2595 			cq->flush = true;
2596 			list_move_tail(&qp->rq.cq_flush, &cq->flush_rq);
2597 		}
2598 		spin_unlock(&qp->rq.lock);
2599 		spin_unlock_irqrestore(&cq->lock, irqflags);
2600 	}
2601 }
2602 
2603 static void ionic_clean_cq(struct ionic_cq *cq, u32 qpid)
2604 {
2605 	struct ionic_v1_cqe *qcqe;
2606 	int prod, qtf, qid, type;
2607 	bool color;
2608 
2609 	if (!cq->q.ptr)
2610 		return;
2611 
2612 	color = cq->color;
2613 	prod = cq->q.prod;
2614 	qcqe = ionic_queue_at(&cq->q, prod);
2615 
2616 	while (color == ionic_v1_cqe_color(qcqe)) {
2617 		qtf = ionic_v1_cqe_qtf(qcqe);
2618 		qid = ionic_v1_cqe_qtf_qid(qtf);
2619 		type = ionic_v1_cqe_qtf_type(qtf);
2620 
2621 		if (qid == qpid && type != IONIC_V1_CQE_TYPE_ADMIN)
2622 			ionic_v1_cqe_clean(qcqe);
2623 
2624 		prod = ionic_queue_next(&cq->q, prod);
2625 		qcqe = ionic_queue_at(&cq->q, prod);
2626 		color = ionic_color_wrap(prod, color);
2627 	}
2628 }
2629 
2630 static void ionic_reset_qp(struct ionic_ibdev *dev, struct ionic_qp *qp)
2631 {
2632 	unsigned long irqflags;
2633 	struct ionic_cq *cq;
2634 	int i;
2635 
2636 	local_irq_save(irqflags);
2637 
2638 	if (qp->ibqp.send_cq) {
2639 		cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx);
2640 		spin_lock(&cq->lock);
2641 		ionic_clean_cq(cq, qp->qpid);
2642 		spin_unlock(&cq->lock);
2643 	}
2644 
2645 	if (qp->ibqp.recv_cq) {
2646 		cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
2647 		spin_lock(&cq->lock);
2648 		ionic_clean_cq(cq, qp->qpid);
2649 		spin_unlock(&cq->lock);
2650 	}
2651 
2652 	if (qp->has_sq) {
2653 		spin_lock(&qp->sq_lock);
2654 		qp->sq_flush = false;
2655 		qp->sq_flush_rcvd = false;
2656 		qp->sq_msn_prod = 0;
2657 		qp->sq_msn_cons = 0;
2658 		qp->sq.prod = 0;
2659 		qp->sq.cons = 0;
2660 		spin_unlock(&qp->sq_lock);
2661 	}
2662 
2663 	if (!qp->srq) {
2664 		spin_lock(&qp->rq.lock);
2665 		qp->rq.flush = false;
2666 		qp->rq.q.prod = 0;
2667 		qp->rq.q.cons = 0;
2668 		if (qp->rq.meta) {
2669 			for (i = 0; i < qp->rq.q.mask; ++i)
2670 				qp->rq.meta[i].next = &qp->rq.meta[i + 1];
2671 			qp->rq.meta[i].next = IONIC_META_LAST;
2672 		}
2673 		qp->rq.meta_head = &qp->rq.meta[0];
2674 		spin_unlock(&qp->rq.lock);
2675 	}
2676 
2677 	local_irq_restore(irqflags);
2678 }
2679 
2680 static bool ionic_qp_cur_state_is_ok(enum ib_qp_state q_state,
2681 				     enum ib_qp_state attr_state)
2682 {
2683 	if (q_state == attr_state)
2684 		return true;
2685 
2686 	if (attr_state == IB_QPS_ERR)
2687 		return true;
2688 
2689 	if (attr_state == IB_QPS_SQE)
2690 		return q_state == IB_QPS_RTS || q_state == IB_QPS_SQD;
2691 
2692 	return false;
2693 }
2694 
2695 static bool ionic_is_modify_ok(enum ib_qp_attr_mask ext_mask,
2696 			       enum ib_qp_type type, enum ib_qp_state cur,
2697 			       enum ib_qp_state next)
2698 {
2699 	if (!ext_mask)
2700 		return true;
2701 
2702 	if (ext_mask & ~IB_QP_RATE_LIMIT)
2703 		return false;
2704 
2705 	/* Rate limit is only supported for RC QPs during specific transitions */
2706 	return type == IB_QPT_RC &&
2707 	       ((cur == IB_QPS_INIT && next == IB_QPS_RTR) ||
2708 		(cur == IB_QPS_RTR && next == IB_QPS_RTS) ||
2709 		(cur == IB_QPS_RTS && next == IB_QPS_RTS));
2710 }
2711 
2712 static int ionic_check_modify_qp(struct ionic_qp *qp, struct ib_qp_attr *attr,
2713 				 int mask)
2714 {
2715 	enum ib_qp_state cur_state = (mask & IB_QP_CUR_STATE) ?
2716 		attr->cur_qp_state : qp->state;
2717 	enum ib_qp_state next_state = (mask & IB_QP_STATE) ?
2718 		attr->qp_state : cur_state;
2719 
2720 	if ((mask & IB_QP_CUR_STATE) &&
2721 	    !ionic_qp_cur_state_is_ok(qp->state, attr->cur_qp_state))
2722 		return -EINVAL;
2723 
2724 	if (!ib_modify_qp_is_ok(cur_state, next_state, qp->ibqp.qp_type, mask) ||
2725 	    !ionic_is_modify_ok(mask & ~IB_QP_ATTR_STANDARD_BITS,
2726 				qp->ibqp.qp_type, cur_state, next_state))
2727 		return -EINVAL;
2728 
2729 	/* unprivileged qp not allowed privileged qkey */
2730 	if ((mask & IB_QP_QKEY) && (attr->qkey & 0x80000000) &&
2731 	    qp->ibqp.uobject)
2732 		return -EPERM;
2733 
2734 	return 0;
2735 }
2736 
2737 int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
2738 		    struct ib_udata *udata)
2739 {
2740 	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2741 	struct ionic_pd *pd = to_ionic_pd(ibqp->pd);
2742 	struct ionic_qp *qp = to_ionic_qp(ibqp);
2743 	int rc;
2744 
2745 	rc = ib_no_udata_io(udata);
2746 	if (rc)
2747 		return rc;
2748 
2749 	rc = ionic_check_modify_qp(qp, attr, mask);
2750 	if (rc)
2751 		return rc;
2752 
2753 	if (mask & IB_QP_CAP)
2754 		return -EINVAL;
2755 
2756 	rc = ionic_modify_qp_cmd(dev, pd, qp, attr, mask);
2757 	if (rc)
2758 		return rc;
2759 
2760 	if (mask & IB_QP_STATE) {
2761 		qp->state = attr->qp_state;
2762 
2763 		if (attr->qp_state == IB_QPS_ERR) {
2764 			ionic_flush_qp(dev, qp);
2765 			ionic_notify_qp_cqs(dev, qp);
2766 		} else if (attr->qp_state == IB_QPS_RESET) {
2767 			ionic_reset_qp(dev, qp);
2768 		}
2769 	}
2770 
2771 	return 0;
2772 }
2773 
2774 int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2775 		   int mask, struct ib_qp_init_attr *init_attr)
2776 {
2777 	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2778 	struct ionic_qp *qp = to_ionic_qp(ibqp);
2779 	int rc;
2780 
2781 	memset(attr, 0, sizeof(*attr));
2782 	memset(init_attr, 0, sizeof(*init_attr));
2783 
2784 	rc = ionic_query_qp_cmd(dev, qp, attr, mask);
2785 	if (rc)
2786 		return rc;
2787 
2788 	if (qp->has_sq)
2789 		attr->cap.max_send_wr = qp->sq.mask;
2790 
2791 	if (!qp->srq)
2792 		attr->cap.max_recv_wr = qp->rq.q.mask;
2793 
2794 	init_attr->event_handler = ibqp->event_handler;
2795 	init_attr->qp_context = ibqp->qp_context;
2796 	init_attr->send_cq = ibqp->send_cq;
2797 	init_attr->recv_cq = ibqp->recv_cq;
2798 	init_attr->srq = ibqp->srq;
2799 	init_attr->xrcd = ibqp->xrcd;
2800 	init_attr->cap = attr->cap;
2801 	init_attr->sq_sig_type = qp->sig_all ?
2802 		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
2803 	init_attr->qp_type = ibqp->qp_type;
2804 	init_attr->create_flags = 0;
2805 	init_attr->port_num = 0;
2806 	init_attr->rwq_ind_tbl = ibqp->rwq_ind_tbl;
2807 	init_attr->source_qpn = 0;
2808 
2809 	return rc;
2810 }
2811 
2812 int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2813 {
2814 	struct ionic_ctx *ctx =
2815 		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
2816 	struct ionic_ibdev *dev = to_ionic_ibdev(ibqp->device);
2817 	struct ionic_qp *qp = to_ionic_qp(ibqp);
2818 	unsigned long irqflags;
2819 	struct ionic_cq *cq;
2820 	int rc;
2821 
2822 	rc = ib_no_udata_io(udata);
2823 	if (rc)
2824 		return rc;
2825 
2826 	rc = ionic_destroy_qp_cmd(dev, qp->qpid);
2827 	if (rc)
2828 		return rc;
2829 
2830 	xa_erase_irq(&dev->qp_tbl, qp->qpid);
2831 
2832 	kref_put(&qp->qp_kref, ionic_qp_complete);
2833 	wait_for_completion(&qp->qp_rel_comp);
2834 
2835 	if (qp->ibqp.send_cq) {
2836 		cq = to_ionic_vcq_cq(qp->ibqp.send_cq, qp->udma_idx);
2837 		spin_lock_irqsave(&cq->lock, irqflags);
2838 		ionic_clean_cq(cq, qp->qpid);
2839 		list_del(&qp->cq_poll_sq);
2840 		list_del(&qp->cq_flush_sq);
2841 		spin_unlock_irqrestore(&cq->lock, irqflags);
2842 	}
2843 
2844 	if (qp->ibqp.recv_cq) {
2845 		cq = to_ionic_vcq_cq(qp->ibqp.recv_cq, qp->udma_idx);
2846 		spin_lock_irqsave(&cq->lock, irqflags);
2847 		ionic_clean_cq(cq, qp->qpid);
2848 		list_del(&qp->rq.cq_flush);
2849 		spin_unlock_irqrestore(&cq->lock, irqflags);
2850 	}
2851 
2852 	if (!qp->srq)
2853 		ionic_rq_destroy(dev, ctx, &qp->rq);
2854 
2855 	ionic_qp_sq_destroy(dev, ctx, qp);
2856 	if (qp->has_ah) {
2857 		ionic_put_ahid(dev, qp->ahid);
2858 		kfree(qp->hdr);
2859 	}
2860 	ionic_put_qpid(dev, qp->qpid);
2861 
2862 	return 0;
2863 }
2864 
2865 static int ionic_create_srq_cmd(struct ionic_ibdev *dev,
2866 				struct ionic_ctx *ctx,
2867 				struct ionic_srq *srq,
2868 				struct ionic_pd *pd,
2869 				struct ionic_tbl_buf *buf,
2870 				u8 udma_mask)
2871 {
2872 	const u16 flags = (srq->rq.cmb & IONIC_CMB_ENABLE) ? IONIC_SRQF_CMB : 0;
2873 	struct ionic_admin_create_srq_resp *resp_buf;
2874 	const u16 dbid = ionic_ctx_dbid(dev, ctx);
2875 	struct ionic_admin_wr wr = {
2876 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
2877 		.wqe = {
2878 			.op = IONIC_V1_ADMIN_CREATE_SRQ,
2879 			.len = cpu_to_le16(IONIC_ADMIN_CREATE_SRQ_IN_V1_LEN),
2880 			.cmd.create_srq = {
2881 				.pd_id = cpu_to_le32(pd->pdid),
2882 				.depth_log2 = srq->rq.q.depth_log2,
2883 				.stride_log2 = srq->rq.q.stride_log2,
2884 				.page_size_log2 = buf->page_size_log2,
2885 				.map_count = cpu_to_le32(buf->tbl_pages),
2886 				.dma_addr = ionic_pgtbl_dma(buf, 0),
2887 				.dbid = cpu_to_le16(dbid),
2888 				.qid = cpu_to_le32(srq->rq.qid),
2889 				.low_wqes_limit = cpu_to_le16(srq->srq_limit),
2890 				.udma_mask = udma_mask,
2891 				.flags = cpu_to_le16(flags),
2892 			}
2893 		}
2894 	};
2895 	dma_addr_t resp_buf_dma;
2896 	int rc;
2897 
2898 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_CREATE_SRQ)
2899 		return -EOPNOTSUPP;
2900 
2901 	resp_buf = kzalloc_obj(*resp_buf);
2902 	if (!resp_buf)
2903 		return -ENOMEM;
2904 
2905 	resp_buf_dma = dma_map_single(dev->lif_cfg.hwdev, resp_buf,
2906 				      sizeof(*resp_buf),
2907 				      DMA_FROM_DEVICE);
2908 
2909 	rc = dma_mapping_error(dev->lif_cfg.hwdev, resp_buf_dma);
2910 	if (rc)
2911 		goto err_dma;
2912 
2913 	wr.wqe.cmd.create_srq.resp_dma_addr = cpu_to_le64(resp_buf_dma);
2914 	wr.wqe.cmd.create_srq.resp_buf_len = cpu_to_le32(IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN);
2915 
2916 	ionic_admin_post(dev, &wr);
2917 
2918 	rc = ionic_admin_wait(dev, &wr, 0);
2919 
2920 	dma_unmap_single(dev->lif_cfg.hwdev, resp_buf_dma, sizeof(*resp_buf),
2921 			 DMA_FROM_DEVICE);
2922 	if (rc)
2923 		goto err_dma;
2924 
2925 	if (be32_to_cpu(wr.cqe.status_length) < IONIC_ADMIN_CREATE_SRQ_OUT_V1_LEN) {
2926 		rc = -EOPNOTSUPP;
2927 		goto err_dma;
2928 	}
2929 
2930 	srq->rq.qid = le32_to_cpu(resp_buf->id);
2931 	srq->udma_idx = resp_buf->udma_idx;
2932 
2933 err_dma:
2934 	kfree(resp_buf);
2935 
2936 	return rc;
2937 }
2938 
2939 static int ionic_destroy_srq_cmd(struct ionic_ibdev *dev, u32 srqid)
2940 {
2941 	struct ionic_admin_wr wr = {
2942 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
2943 		.wqe = {
2944 			.op = IONIC_V1_ADMIN_DESTROY_SRQ,
2945 			.len = cpu_to_le16(IONIC_ADMIN_DESTROY_SRQ_IN_V1_LEN),
2946 			.cmd.destroy_srq = {
2947 				.qid = cpu_to_le32(srqid),
2948 			},
2949 		}
2950 	};
2951 
2952 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_DESTROY_SRQ)
2953 		return -EOPNOTSUPP;
2954 
2955 	ionic_admin_post(dev, &wr);
2956 
2957 	return ionic_admin_wait(dev, &wr, IONIC_ADMIN_F_TEARDOWN);
2958 }
2959 
2960 static int ionic_modify_srq_cmd(struct ionic_ibdev *dev, u32 srqid,
2961 				u16 srq_limit)
2962 {
2963 	struct ionic_admin_wr wr = {
2964 		.work = COMPLETION_INITIALIZER_ONSTACK(wr.work),
2965 		.wqe = {
2966 			.op = IONIC_V1_ADMIN_MODIFY_SRQ,
2967 			.len = cpu_to_le16(IONIC_ADMIN_MODIFY_SRQ_IN_V1_LEN),
2968 			.cmd.modify_srq = {
2969 				.qid = cpu_to_le32(srqid),
2970 				.low_wqes_limit = cpu_to_le16(srq_limit),
2971 			},
2972 		}
2973 	};
2974 
2975 	if (dev->lif_cfg.admin_opcodes <= IONIC_V1_ADMIN_MODIFY_SRQ)
2976 		return -EOPNOTSUPP;
2977 
2978 	ionic_admin_post(dev, &wr);
2979 
2980 	return ionic_admin_wait(dev, &wr, 0);
2981 }
2982 
2983 int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr,
2984 		     struct ib_udata *udata)
2985 {
2986 	struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device);
2987 	struct ionic_pd *pd = to_ionic_pd(ibsrq->pd);
2988 	struct ionic_srq *srq = to_ionic_srq(ibsrq);
2989 	struct ionic_ctx *ctx =
2990 		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
2991 	struct ionic_srq_resp resp = {};
2992 	struct ionic_srq_req req = {};
2993 	struct ionic_tbl_buf buf = {};
2994 	u8 udma_mask;
2995 	int rc;
2996 
2997 	if (!ctx)
2998 		return -EOPNOTSUPP;
2999 
3000 	if (attr->srq_type != IB_SRQT_BASIC)
3001 		return -EOPNOTSUPP;
3002 
3003 	if (attr->attr.max_sge > IONIC_MAX_SRQ_SGES)
3004 		return -EINVAL;
3005 
3006 	if (attr->attr.srq_limit >= IONIC_MAX_SRQ_LIMIT)
3007 		return -EINVAL;
3008 
3009 	udma_mask = BIT(dev->lif_cfg.udma_count) - 1;
3010 	if (udata) {
3011 		rc = ib_copy_validate_udata_in(udata, req, rsvd);
3012 		if (rc)
3013 			return rc;
3014 
3015 		if (memchr_inv(req.rsvd, 0, sizeof(req.rsvd)))
3016 			return -EINVAL;
3017 
3018 		udma_mask &= req.udma_mask;
3019 	}
3020 
3021 	if (!udma_mask)
3022 		return -EINVAL;
3023 
3024 	if (udata && (req.rq_cmb & IONIC_CMB_ENABLE))
3025 		srq->rq.cmb = req.rq_cmb;
3026 
3027 	rc = ionic_rq_init(dev, ctx, &srq->rq, &req.rq, &buf,
3028 			   attr->attr.max_wr, attr->attr.max_sge,
3029 			   req.rq_spec, udata);
3030 	if (rc)
3031 		return rc;
3032 
3033 	srq->srq_limit = attr->attr.srq_limit;
3034 
3035 	rc = ionic_create_srq_cmd(dev, ctx, srq, pd, &buf, udma_mask);
3036 	if (rc)
3037 		goto err_cmd;
3038 
3039 	ionic_queue_dbell_init(&srq->rq.q, srq->rq.qid);
3040 
3041 	if (udata) {
3042 		resp.srqid = srq->rq.qid;
3043 		resp.udma_idx = srq->udma_idx;
3044 
3045 		if (srq->rq.cmb & IONIC_CMB_ENABLE) {
3046 			ionic_rq_mmap_cmb(dev, ctx, &srq->rq, &resp.rq_cmb_offset);
3047 			if (!srq->rq.mmap_cmb) {
3048 				rc = -ENOMEM;
3049 				goto err_srq_cmd;
3050 			}
3051 
3052 			resp.rq_cmb = srq->rq.cmb;
3053 		}
3054 
3055 		rc = ib_respond_udata(udata, resp);
3056 		if (rc)
3057 			goto err_srq_cmd;
3058 	}
3059 
3060 	kref_init(&srq->kref);
3061 	init_completion(&srq->rel_comp);
3062 
3063 	rc = xa_insert_irq(&dev->srq_tbl, srq->rq.qid, srq, GFP_KERNEL);
3064 	if (rc)
3065 		goto err_srq_cmd;
3066 
3067 	ionic_pgtbl_unbuf(dev, &buf);
3068 
3069 	attr->attr.max_wr = srq->rq.q.mask;
3070 
3071 	return 0;
3072 
3073 err_srq_cmd:
3074 	ionic_destroy_srq_cmd(dev, srq->rq.qid);
3075 err_cmd:
3076 	ionic_pgtbl_unbuf(dev, &buf);
3077 	ionic_rq_destroy(dev, ctx, &srq->rq);
3078 
3079 	return rc;
3080 }
3081 
3082 int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
3083 {
3084 	struct ionic_ctx *ctx =
3085 		rdma_udata_to_drv_context(udata, struct ionic_ctx, ibctx);
3086 	struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device);
3087 	struct ionic_srq *srq = to_ionic_srq(ibsrq);
3088 	int rc;
3089 
3090 	if (xa_erase_irq(&dev->srq_tbl, srq->rq.qid)) {
3091 		kref_put(&srq->kref, ionic_srq_complete);
3092 		wait_for_completion(&srq->rel_comp);
3093 	}
3094 
3095 	rc = ionic_destroy_srq_cmd(dev, srq->rq.qid);
3096 	if (rc)
3097 		return rc;
3098 
3099 	ionic_rq_destroy(dev, ctx, &srq->rq);
3100 
3101 	return 0;
3102 }
3103 
3104 int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
3105 {
3106 	struct ionic_srq *srq = to_ionic_srq(ibsrq);
3107 
3108 	srq_attr->max_wr = srq->rq.q.mask;
3109 	srq_attr->max_sge =
3110 		ionic_v1_recv_wqe_max_sge(srq->rq.q.stride_log2, srq->rq.spec,
3111 					  srq->rq.cmb & IONIC_CMB_EXPDB);
3112 	srq_attr->srq_limit = srq->srq_limit;
3113 
3114 	return 0;
3115 }
3116 
3117 int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
3118 		     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
3119 {
3120 	struct ionic_ibdev *dev = to_ionic_ibdev(ibsrq->device);
3121 	struct ionic_srq *srq = to_ionic_srq(ibsrq);
3122 	int rc;
3123 
3124 	if (attr_mask & IB_SRQ_MAX_WR)
3125 		return -EINVAL;
3126 
3127 	if (attr_mask & IB_SRQ_LIMIT) {
3128 		if (attr->srq_limit >= IONIC_MAX_SRQ_LIMIT)
3129 			return -EINVAL;
3130 
3131 		rc = ionic_modify_srq_cmd(dev, srq->rq.qid, attr->srq_limit);
3132 		if (rc)
3133 			return rc;
3134 
3135 		srq->srq_limit = attr->srq_limit;
3136 	}
3137 
3138 	return 0;
3139 }
3140