xref: /freebsd/sys/dev/qlnx/qlnxr/qlnxr_verbs.c (revision d485c77f203fb0f4cdc08dea5ff81631b51d8809)
1 /*
2  * Copyright (c) 2018-2019 Cavium, Inc.
3  * All rights reserved.
4  *
5  *  Redistribution and use in source and binary forms, with or without
6  *  modification, are permitted provided that the following conditions
7  *  are met:
8  *
9  *  1. Redistributions of source code must retain the above copyright
10  *     notice, this list of conditions and the following disclaimer.
11  *  2. Redistributions in binary form must reproduce the above copyright
12  *     notice, this list of conditions and the following disclaimer in the
13  *     documentation and/or other materials provided with the distribution.
14  *
15  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16  *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19  *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20  *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21  *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22  *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23  *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24  *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25  *  POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 /*
29  * File: qlnxr_verbs.c
30  */
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "qlnxr_def.h"
35 #include "rdma_common.h"
36 #include "qlnxr_roce.h"
37 #include "qlnxr_cm.h"
38 
39 #define upper_32_bits(x) (uint32_t)(x >> 32)
40 #define lower_32_bits(x) (uint32_t)(x)
41 #define HILO_U64(hi, lo)		((((u64)(hi)) << 32) + (lo))
42 
43 #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr)			\
44 	do {								\
45 		(type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\
46 		(type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\
47 	} while (0)
48 
49 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
50 	do {							\
51 		TYPEPTR_ADDR_SET(sge, addr, vaddr);		\
52 		(sge)->length = cpu_to_le32(vlength);		\
53 		(sge)->flags = cpu_to_le32(vflags);		\
54 	} while (0)
55 
56 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)			\
57 	do {							\
58 		TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id);		\
59 		(hdr)->num_sges = num_sge;			\
60 	} while (0)
61 
62 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)			\
63 	do {							\
64 		TYPEPTR_ADDR_SET(sge, addr, vaddr);		\
65 		(sge)->length = cpu_to_le32(vlength);		\
66 		(sge)->l_key = cpu_to_le32(vlkey);		\
67 	} while (0)
68 
69 #define NIPQUAD(addr) \
70 	((unsigned char *)&addr)[0], \
71 	((unsigned char *)&addr)[1], \
72 	((unsigned char *)&addr)[2], \
73 	((unsigned char *)&addr)[3]
74 
75 static int
76 qlnxr_check_srq_params(struct ib_pd *ibpd,
77 	struct qlnxr_dev *dev,
78 	struct ib_srq_init_attr *attrs);
79 
80 static int
81 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
82 	struct qlnxr_srq *srq,
83 	struct qlnxr_create_srq_ureq *ureq,
84 	int access, int dmasync);
85 
86 static int
87 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
88 	struct qlnxr_dev *dev,
89 	struct ib_srq_init_attr *init_attr);
90 
91 static int
92 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
93 	struct qlnxr_srq *srq,
94 	struct ib_udata *udata);
95 
96 static void
97 qlnxr_free_srq_user_params(struct qlnxr_srq *srq);
98 
99 static void
100 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq);
101 
102 static u32
103 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq);
104 
105 int
106 qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index,
107 	union ib_gid *sgid)
108 {
109 	struct qlnxr_dev	*dev;
110 	qlnx_host_t		*ha;
111 
112 	dev = get_qlnxr_dev(ibdev);
113 	ha = dev->ha;
114 
115 	QL_DPRINT12(ha, "enter\n");
116 
117 	memset(sgid->raw, 0, sizeof(sgid->raw));
118 
119 	memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac));
120 
121 	QL_DPRINT12(ha, "exit\n");
122 
123 	return 0;
124 }
125 
126 int
127 qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index,
128 	union ib_gid *sgid)
129 {
130 	struct qlnxr_dev	*dev;
131 	qlnx_host_t		*ha;
132 
133 	dev = get_qlnxr_dev(ibdev);
134 	ha = dev->ha;
135 	QL_DPRINT12(ha, "enter index: %d\n", index);
136 #if 0
137 	int ret = 0;
138 	/* @@@: if DEFINE_ROCE_GID_TABLE to be used here */
139 	//if (!rdma_cap_roce_gid_table(ibdev, port)) {
140 	if (!(rdma_protocol_roce(ibdev, port) &&
141 		ibdev->add_gid && ibdev->del_gid)) {
142 		QL_DPRINT11(ha, "acquire gid failed\n");
143 		return -ENODEV;
144 	}
145 
146 	ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
147 	if (ret == -EAGAIN) {
148 		memcpy(sgid, &zgid, sizeof(*sgid));
149 		return 0;
150 	}
151 #endif
152 	if ((index >= QLNXR_MAX_SGID) || (index < 0)) {
153 		QL_DPRINT12(ha, "invalid gid index %d\n", index);
154 		memset(sgid, 0, sizeof(*sgid));
155 		return -EINVAL;
156 	}
157 	memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));
158 
159 	QL_DPRINT12(ha, "exit : %p\n", sgid);
160 
161 	return 0;
162 }
163 
164 struct ib_srq *
165 qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
166 	struct ib_udata *udata)
167 {
168 	struct qlnxr_dev	*dev;
169 	qlnx_host_t		*ha;
170 	struct ecore_rdma_destroy_srq_in_params destroy_in_params;
171 	struct ecore_rdma_create_srq_out_params out_params;
172 	struct ecore_rdma_create_srq_in_params in_params;
173 	u64 pbl_base_addr, phy_prod_pair_addr;
174 	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
175 	struct ib_ucontext *ib_ctx = NULL;
176 	struct qlnxr_srq_hwq_info *hw_srq;
177 	struct qlnxr_ucontext *ctx = NULL;
178 	struct qlnxr_create_srq_ureq ureq;
179 	u32 page_cnt, page_size;
180 	struct qlnxr_srq *srq;
181 	int ret = 0;
182 
183 	dev = get_qlnxr_dev((ibpd->device));
184 	ha = dev->ha;
185 
186 	QL_DPRINT12(ha, "enter\n");
187 
188 	ret = qlnxr_check_srq_params(ibpd, dev, init_attr);
189 
190 	srq = kzalloc(sizeof(*srq), GFP_KERNEL);
191 	if (!srq) {
192 		QL_DPRINT11(ha, "cannot allocate memory for srq\n");
193 		return NULL; //@@@ : TODO what to return here?
194 	}
195 
196 	srq->dev = dev;
197 	hw_srq = &srq->hw_srq;
198 	spin_lock_init(&srq->lock);
199 	memset(&in_params, 0, sizeof(in_params));
200 
201 	if (udata && ibpd->uobject && ibpd->uobject->context) {
202 		ib_ctx = ibpd->uobject->context;
203 		ctx = get_qlnxr_ucontext(ib_ctx);
204 
205 		memset(&ureq, 0, sizeof(ureq));
206 		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
207 			udata->inlen))) {
208 			QL_DPRINT11(ha, "problem"
209 				" copying data from user space\n");
210 			goto err0;
211 		}
212 
213 		ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
214 		if (ret)
215 			goto err0;
216 
217 		page_cnt = srq->usrq.pbl_info.num_pbes;
218 		pbl_base_addr = srq->usrq.pbl_tbl->pa;
219 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
220 		// @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT
221 		// page_size = BIT(srq->usrq.umem->page_shift);
222 		// else
223 		page_size = srq->usrq.umem->page_size;
224 	} else {
225 		struct ecore_chain *pbl;
226 		ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr);
227 		if (ret)
228 			goto err0;
229 		pbl = &hw_srq->pbl;
230 
231 		page_cnt = ecore_chain_get_page_cnt(pbl);
232 		pbl_base_addr = ecore_chain_get_pbl_phys(pbl);
233 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
234 		page_size = pbl->elem_per_page << 4;
235 	}
236 
237 	in_params.pd_id = pd->pd_id;
238 	in_params.pbl_base_addr = pbl_base_addr;
239 	in_params.prod_pair_addr = phy_prod_pair_addr;
240 	in_params.num_pages = page_cnt;
241 	in_params.page_size = page_size;
242 
243 	ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
244 	if (ret)
245 		goto err1;
246 
247 	srq->srq_id = out_params.srq_id;
248 
249 	if (udata) {
250 		ret = qlnxr_copy_srq_uresp(dev, srq, udata);
251 		if (ret)
252 			goto err2;
253 	}
254 
255 	QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);
256 	return &srq->ibsrq;
257 err2:
258 	memset(&in_params, 0, sizeof(in_params));
259 	destroy_in_params.srq_id = srq->srq_id;
260 	ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
261 
262 err1:
263 	if (udata)
264 		qlnxr_free_srq_user_params(srq);
265 	else
266 		qlnxr_free_srq_kernel_params(srq);
267 
268 err0:
269 	kfree(srq);
270 	return ERR_PTR(-EFAULT);
271 }
272 
273 int
274 qlnxr_destroy_srq(struct ib_srq *ibsrq)
275 {
276 	struct qlnxr_dev	*dev;
277 	struct qlnxr_srq	*srq;
278 	qlnx_host_t		*ha;
279 	struct ecore_rdma_destroy_srq_in_params in_params;
280 
281 	srq = get_qlnxr_srq(ibsrq);
282 	dev = srq->dev;
283 	ha = dev->ha;
284 
285 	memset(&in_params, 0, sizeof(in_params));
286 	in_params.srq_id = srq->srq_id;
287 
288 	ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params);
289 
290 	if (ibsrq->pd->uobject && ibsrq->pd->uobject->context)
291 		qlnxr_free_srq_user_params(srq);
292 	else
293 		qlnxr_free_srq_kernel_params(srq);
294 
295 	QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);
296 	kfree(srq);
297 	return 0;
298 }
299 
300 int
301 qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
302 	enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
303 {
304 	struct qlnxr_dev	*dev;
305 	struct qlnxr_srq	*srq;
306 	qlnx_host_t		*ha;
307 	struct ecore_rdma_modify_srq_in_params in_params;
308 	int ret = 0;
309 
310 	srq = get_qlnxr_srq(ibsrq);
311 	dev = srq->dev;
312 	ha = dev->ha;
313 
314 	QL_DPRINT12(ha, "enter\n");
315 	if (attr_mask & IB_SRQ_MAX_WR) {
316 		QL_DPRINT12(ha, "invalid attribute mask=0x%x"
317 			" specified for %p\n", attr_mask, srq);
318 		return -EINVAL;
319 	}
320 
321 	if (attr_mask & IB_SRQ_LIMIT) {
322 		if (attr->srq_limit >= srq->hw_srq.max_wr) {
323 			QL_DPRINT12(ha, "invalid srq_limit=0x%x"
324 				" (max_srq_limit = 0x%x)\n",
325 			       attr->srq_limit, srq->hw_srq.max_wr);
326 			return -EINVAL;
327 		}
328 		memset(&in_params, 0, sizeof(in_params));
329 		in_params.srq_id = srq->srq_id;
330 		in_params.wqe_limit = attr->srq_limit;
331 		ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params);
332 		if (ret)
333 			return ret;
334 	}
335 
336 	QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id);
337 	return 0;
338 }
339 
340 int
341 qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
342 {
343 	struct qlnxr_dev	*dev;
344 	struct qlnxr_srq	*srq;
345 	qlnx_host_t		*ha;
346 	struct ecore_rdma_device *qattr;
347 	srq = get_qlnxr_srq(ibsrq);
348 	dev = srq->dev;
349 	ha = dev->ha;
350 	//qattr = &dev->attr;
351 	qattr = ecore_rdma_query_device(dev->rdma_ctx);
352 	QL_DPRINT12(ha, "enter\n");
353 
354 	if (!dev->rdma_ctx) {
355 		QL_DPRINT12(ha, "called with invalid params"
356 			" rdma_ctx is NULL\n");
357 		return -EINVAL;
358 	}
359 
360 	srq_attr->srq_limit = qattr->max_srq;
361 	srq_attr->max_wr = qattr->max_srq_wr;
362 	srq_attr->max_sge = qattr->max_sge;
363 
364 	QL_DPRINT12(ha, "exit\n");
365 	return 0;
366 }
367 
368 /* Increment srq wr producer by one */
369 static
370 void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info)
371 {
372 	info->wr_prod_cnt++;
373 }
374 
375 /* Increment srq wr consumer by one */
376 static
377 void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info)
378 {
379         info->wr_cons_cnt++;
380 }
381 
382 /* get_port_immutable verb is not available in FreeBSD */
383 #if 0
384 int
385 qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num,
386 	struct ib_port_immutable *immutable)
387 {
388 	struct qlnxr_dev                *dev;
389 	qlnx_host_t                     *ha;
390 	dev = get_qlnxr_dev(ibdev);
391 	ha = dev->ha;
392 
393 	QL_DPRINT12(ha, "entered but not implemented!!!\n");
394 }
395 #endif
396 
397 int
398 qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
399 	struct ib_recv_wr **bad_wr)
400 {
401 	struct qlnxr_dev	*dev;
402 	struct qlnxr_srq	*srq;
403 	qlnx_host_t		*ha;
404 	struct qlnxr_srq_hwq_info *hw_srq;
405 	struct ecore_chain *pbl;
406 	unsigned long flags;
407 	int status = 0;
408 	u32 num_sge, offset;
409 
410 	srq = get_qlnxr_srq(ibsrq);
411 	dev = srq->dev;
412 	ha = dev->ha;
413 	hw_srq = &srq->hw_srq;
414 
415 	QL_DPRINT12(ha, "enter\n");
416 	spin_lock_irqsave(&srq->lock, flags);
417 
418 	pbl = &srq->hw_srq.pbl;
419 	while (wr) {
420 		struct rdma_srq_wqe_header *hdr;
421 		int i;
422 
423 		if (!qlnxr_srq_elem_left(hw_srq) ||
424 		    wr->num_sge > srq->hw_srq.max_sges) {
425 			QL_DPRINT11(ha, "WR cannot be posted"
426 			    " (%d, %d) || (%d > %d)\n",
427 			    hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
428 			    wr->num_sge, srq->hw_srq.max_sges);
429 			status = -ENOMEM;
430 			*bad_wr = wr;
431 			break;
432 		}
433 
434 		hdr = ecore_chain_produce(pbl);
435 		num_sge = wr->num_sge;
436 		/* Set number of sge and WR id in header */
437 		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
438 
439                 /* PBL is maintained in case of WR granularity.
440                  * So increment WR producer in case we post a WR.
441                  */
442 		qlnxr_inc_srq_wr_prod(hw_srq);
443 		hw_srq->wqe_prod++;
444 		hw_srq->sge_prod++;
445 
446 		QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n",
447 			wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
448 
449 		for (i = 0; i < wr->num_sge; i++) {
450 			struct rdma_srq_sge *srq_sge =
451 			    ecore_chain_produce(pbl);
452 			/* Set SGE length, lkey and address */
453 			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
454 				wr->sg_list[i].length, wr->sg_list[i].lkey);
455 
456 			QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n",
457 				i, srq_sge->length, srq_sge->l_key,
458 				srq_sge->addr.hi, srq_sge->addr.lo);
459 			hw_srq->sge_prod++;
460 		}
461 		wmb();
462 		/*
463 		 * SRQ prod is 8 bytes. Need to update SGE prod in index
464 		 * in first 4 bytes and need to update WQE prod in next
465 		 * 4 bytes.
466 		 */
467 		*(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod;
468 		offset = offsetof(struct rdma_srq_producers, wqe_prod);
469 		*((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
470 			hw_srq->wqe_prod;
471 		/* Flush prod after updating it */
472 		wmb();
473 		wr = wr->next;
474 	}
475 
476 	QL_DPRINT12(ha, "Elements in SRQ: %d\n",
477 		ecore_chain_get_elem_left(pbl));
478 
479 	spin_unlock_irqrestore(&srq->lock, flags);
480 	QL_DPRINT12(ha, "exit\n");
481 	return status;
482 }
483 
484 int
485 #if __FreeBSD_version < 1102000
486 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr)
487 #else
488 qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,
489 	struct ib_udata *udata)
490 #endif /* #if __FreeBSD_version < 1102000 */
491 
492 {
493 	struct qlnxr_dev		*dev;
494 	struct ecore_rdma_device	*qattr;
495 	qlnx_host_t			*ha;
496 
497 	dev = get_qlnxr_dev(ibdev);
498 	ha = dev->ha;
499 
500 	QL_DPRINT12(ha, "enter\n");
501 
502 #if __FreeBSD_version > 1102000
503 	if (udata->inlen || udata->outlen)
504 		return -EINVAL;
505 #endif /* #if __FreeBSD_version > 1102000 */
506 
507 	if (dev->rdma_ctx == NULL) {
508 		return -EINVAL;
509 	}
510 
511 	qattr = ecore_rdma_query_device(dev->rdma_ctx);
512 
513 	memset(attr, 0, sizeof *attr);
514 
515 	attr->fw_ver = qattr->fw_ver;
516 	attr->sys_image_guid = qattr->sys_image_guid;
517 	attr->max_mr_size = qattr->max_mr_size;
518 	attr->page_size_cap = qattr->page_size_caps;
519 	attr->vendor_id = qattr->vendor_id;
520 	attr->vendor_part_id = qattr->vendor_part_id;
521 	attr->hw_ver = qattr->hw_ver;
522 	attr->max_qp = qattr->max_qp;
523 	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
524 					IB_DEVICE_RC_RNR_NAK_GEN |
525 					IB_DEVICE_LOCAL_DMA_LKEY |
526 					IB_DEVICE_MEM_MGT_EXTENSIONS;
527 
528 	attr->max_sge = qattr->max_sge;
529 	attr->max_sge_rd = qattr->max_sge;
530 	attr->max_cq = qattr->max_cq;
531 	attr->max_cqe = qattr->max_cqe;
532 	attr->max_mr = qattr->max_mr;
533 	attr->max_mw = qattr->max_mw;
534 	attr->max_pd = qattr->max_pd;
535 	attr->atomic_cap = dev->atomic_cap;
536 	attr->max_fmr = qattr->max_fmr;
537 	attr->max_map_per_fmr = 16; /* TBD: FMR */
538 
539 	/* There is an implicit assumption in some of the ib_xxx apps that the
540 	 * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in
541 	 * communication the qp_rd_atom is passed to the other side and used as
542 	 * init_rd_atom without check device capabilities for init_rd_atom.
543 	 * for this reason, we set the qp_rd_atom to be the minimum between the
544 	 * two...There is an additional assumption in mlx4 driver that the
545 	 * values are power of two, fls is performed on the value - 1, which
546 	 * in fact gives a larger power of two for values which are not a power
547 	 * of two. This should be fixed in mlx4 driver, but until then ->
548 	 * we provide a value that is a power of two in our code.
549 	 */
550 	attr->max_qp_init_rd_atom =
551 		1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
552 	attr->max_qp_rd_atom =
553 		min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
554 		    attr->max_qp_init_rd_atom);
555 
556 	attr->max_srq = qattr->max_srq;
557 	attr->max_srq_sge = qattr->max_srq_sge;
558 	attr->max_srq_wr = qattr->max_srq_wr;
559 
560 	/* TODO: R&D to more properly configure the following */
561 	attr->local_ca_ack_delay = qattr->dev_ack_delay;
562 	attr->max_fast_reg_page_list_len = qattr->max_mr/8;
563 	attr->max_pkeys = QLNXR_ROCE_PKEY_MAX;
564 	attr->max_ah = qattr->max_ah;
565 
566 	QL_DPRINT12(ha, "exit\n");
567 	return 0;
568 }
569 
570 static inline void
571 get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width)
572 {
573 	switch (speed) {
574 	case 1000:
575 		*ib_speed = IB_SPEED_SDR;
576 		*ib_width = IB_WIDTH_1X;
577 		break;
578 	case 10000:
579 		*ib_speed = IB_SPEED_QDR;
580 		*ib_width = IB_WIDTH_1X;
581 		break;
582 
583 	case 20000:
584 		*ib_speed = IB_SPEED_DDR;
585 		*ib_width = IB_WIDTH_4X;
586 		break;
587 
588 	case 25000:
589 		*ib_speed = IB_SPEED_EDR;
590 		*ib_width = IB_WIDTH_1X;
591 		break;
592 
593 	case 40000:
594 		*ib_speed = IB_SPEED_QDR;
595 		*ib_width = IB_WIDTH_4X;
596 		break;
597 
598 	case 50000:
599 		*ib_speed = IB_SPEED_QDR;
600 		*ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50...
601 		break;
602 
603 	case 100000:
604 		*ib_speed = IB_SPEED_EDR;
605 		*ib_width = IB_WIDTH_4X;
606 		break;
607 
608 	default:
609 		/* Unsupported */
610 		*ib_speed = IB_SPEED_SDR;
611 		*ib_width = IB_WIDTH_1X;
612 	}
613 	return;
614 }
615 
616 int
617 qlnxr_query_port(struct ib_device *ibdev, uint8_t port,
618 	struct ib_port_attr *attr)
619 {
620 	struct qlnxr_dev	*dev;
621 	struct ecore_rdma_port	*rdma_port;
622 	qlnx_host_t		*ha;
623 
624 	dev = get_qlnxr_dev(ibdev);
625 	ha = dev->ha;
626 
627 	QL_DPRINT12(ha, "enter\n");
628 
629 	if (port > 1) {
630 		QL_DPRINT12(ha, "port [%d] > 1 \n", port);
631 		return -EINVAL;
632 	}
633 
634 	if (dev->rdma_ctx == NULL) {
635 		QL_DPRINT12(ha, "rdma_ctx == NULL\n");
636 		return -EINVAL;
637 	}
638 
639 	rdma_port = ecore_rdma_query_port(dev->rdma_ctx);
640 	memset(attr, 0, sizeof *attr);
641 
642 	if (rdma_port->port_state == ECORE_RDMA_PORT_UP) {
643 		attr->state = IB_PORT_ACTIVE;
644 		attr->phys_state = 5;
645 	} else {
646 		attr->state = IB_PORT_DOWN;
647 		attr->phys_state = 3;
648 	}
649 
650 	attr->max_mtu = IB_MTU_4096;
651 	attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu);
652 	attr->lid = 0;
653 	attr->lmc = 0;
654 	attr->sm_lid = 0;
655 	attr->sm_sl = 0;
656 	attr->port_cap_flags = 0;
657 
658 	if (QLNX_IS_IWARP(dev)) {
659 		attr->gid_tbl_len = 1;
660 		attr->pkey_tbl_len = 1;
661 	} else {
662 		attr->gid_tbl_len = QLNXR_MAX_SGID;
663 		attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN;
664 	}
665 
666 	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
667 	attr->qkey_viol_cntr = 0;
668 
669 	get_link_speed_and_width(rdma_port->link_speed,
670 				 &attr->active_speed, &attr->active_width);
671 
672 	attr->max_msg_sz = rdma_port->max_msg_size;
673 	attr->max_vl_num = 4; /* TODO -> figure this one out... */
674 
675 	QL_DPRINT12(ha, "state = %d phys_state = %d "
676 		" link_speed = %d active_speed = %d active_width = %d"
677 		" attr->gid_tbl_len = %d attr->pkey_tbl_len = %d"
678 		" max_msg_sz = 0x%x max_vl_num = 0x%x \n",
679 		attr->state, attr->phys_state,
680 		rdma_port->link_speed, attr->active_speed,
681 		attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len,
682 		attr->max_msg_sz, attr->max_vl_num);
683 
684 	QL_DPRINT12(ha, "exit\n");
685 	return 0;
686 }
687 
688 int
689 qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask,
690 	struct ib_port_modify *props)
691 {
692 	struct qlnxr_dev	*dev;
693 	qlnx_host_t		*ha;
694 
695 	dev = get_qlnxr_dev(ibdev);
696 	ha = dev->ha;
697 
698 	QL_DPRINT12(ha, "enter\n");
699 
700 	if (port > 1) {
701 		QL_DPRINT12(ha, "port (%d) > 1\n", port);
702 		return -EINVAL;
703 	}
704 
705 	QL_DPRINT12(ha, "exit\n");
706 	return 0;
707 }
708 
709 enum rdma_link_layer
710 qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num)
711 {
712 	struct qlnxr_dev	*dev;
713 	qlnx_host_t		*ha;
714 
715 	dev = get_qlnxr_dev(ibdev);
716 	ha = dev->ha;
717 
718 	QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num);
719 
720         return IB_LINK_LAYER_ETHERNET;
721 }
722 
723 struct ib_pd *
724 qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
725 	struct ib_udata *udata)
726 {
727 	struct qlnxr_pd		*pd = NULL;
728 	u16			pd_id;
729 	int			rc;
730 	struct qlnxr_dev	*dev;
731 	qlnx_host_t		*ha;
732 
733 	dev = get_qlnxr_dev(ibdev);
734 	ha = dev->ha;
735 
736 	QL_DPRINT12(ha, "ibdev = %p context = %p"
737 		" udata = %p enter\n", ibdev, context, udata);
738 
739 	if (dev->rdma_ctx == NULL) {
740 		QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");
741 		rc = -1;
742 		goto err;
743 	}
744 
745 	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
746 	if (!pd) {
747 		rc = -ENOMEM;
748 		QL_DPRINT11(ha, "kzalloc(pd) = NULL\n");
749 		goto err;
750 	}
751 
752 	rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);
753 	if (rc)	{
754 		QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n");
755 		goto err;
756 	}
757 
758 	pd->pd_id = pd_id;
759 
760 	if (udata && context) {
761 		rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));
762 		if (rc) {
763 			QL_DPRINT11(ha, "ib_copy_to_udata failed\n");
764 			ecore_rdma_free_pd(dev->rdma_ctx, pd_id);
765 			goto err;
766 		}
767 
768 		pd->uctx = get_qlnxr_ucontext(context);
769 		pd->uctx->pd = pd;
770 	}
771 
772 	atomic_add_rel_32(&dev->pd_count, 1);
773 	QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
774 		pd, pd_id, dev->pd_count);
775 
776 	return &pd->ibpd;
777 
778 err:
779 	kfree(pd);
780 	QL_DPRINT12(ha, "exit -1\n");
781 	return ERR_PTR(rc);
782 }
783 
784 int
785 qlnxr_dealloc_pd(struct ib_pd *ibpd)
786 {
787 	struct qlnxr_pd		*pd;
788 	struct qlnxr_dev	*dev;
789 	qlnx_host_t		*ha;
790 
791 	pd = get_qlnxr_pd(ibpd);
792 	dev = get_qlnxr_dev((ibpd->device));
793 	ha = dev->ha;
794 
795 	QL_DPRINT12(ha, "enter\n");
796 
797 	if (pd == NULL) {
798 		QL_DPRINT11(ha, "pd = NULL\n");
799 	} else {
800 		ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);
801 		kfree(pd);
802 		atomic_subtract_rel_32(&dev->pd_count, 1);
803 		QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
804 			pd, pd->pd_id, dev->pd_count);
805 	}
806 
807 	QL_DPRINT12(ha, "exit\n");
808 	return 0;
809 }
810 
811 #define ROCE_WQE_ELEM_SIZE	sizeof(struct rdma_sq_sge)
812 #define	RDMA_MAX_SGE_PER_SRQ	(4) /* Should be part of HSI */
813 /* Should be part of HSI */
814 #define RDMA_MAX_SRQ_WQE_SIZE	(RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */
815 #define DB_ADDR_SHIFT(addr)		((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
816 
817 static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *);
818 static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *);
819 
820 int
821 qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
822 {
823 	struct qlnxr_dev	*dev;
824 	qlnx_host_t		*ha;
825 
826 	dev = get_qlnxr_dev(ibdev);
827 	ha = dev->ha;
828 
829 	QL_DPRINT12(ha, "enter index = 0x%x\n", index);
830 
831 	if (index > QLNXR_ROCE_PKEY_TABLE_LEN)
832 		return -EINVAL;
833 
834 	*pkey = QLNXR_ROCE_PKEY_DEFAULT;
835 
836 	QL_DPRINT12(ha, "exit\n");
837 	return 0;
838 }
839 
840 static inline bool
841 qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask,
842        u16 *vlan_id)
843 {
844 	bool ret = false;
845 
846 	QL_DPRINT12(ha, "enter \n");
847 
848 	*vlan_id = 0;
849 
850 #if __FreeBSD_version >= 1100000
851 	u16 tmp_vlan_id;
852 
853 #if __FreeBSD_version >= 1102000
854 	union ib_gid *dgid;
855 
856 	dgid = &attr->ah_attr.grh.dgid;
857 	tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];
858 
859 	if (!(tmp_vlan_id & ~EVL_VLID_MASK)) {
860 		*vlan_id = tmp_vlan_id;
861 		ret = true;
862 	}
863 #else
864 	tmp_vlan_id = attr->vlan_id;
865 
866 	if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) {
867 		*vlan_id = tmp_vlan_id;
868 		ret = true;
869 	}
870 
871 #endif /* #if __FreeBSD_version > 1102000 */
872 
873 #else
874 	ret = true;
875 
876 #endif /* #if __FreeBSD_version >= 1100000 */
877 
878 	QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret);
879 
880 	return (ret);
881 }
882 
883 static inline void
884 get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr,
885 	int attr_mask,
886 	struct qlnxr_dev *dev,
887 	struct qlnxr_qp *qp,
888 	struct ecore_rdma_modify_qp_in_params *qp_params)
889 {
890 	int		i;
891 	qlnx_host_t	*ha;
892 
893 	ha = dev->ha;
894 
895 	QL_DPRINT12(ha, "enter\n");
896 
897 	memcpy(&qp_params->sgid.bytes[0],
898 	       &dev->sgid_tbl[qp->sgid_idx].raw[0],
899 	       sizeof(qp_params->sgid.bytes));
900 	memcpy(&qp_params->dgid.bytes[0],
901 	       &attr->ah_attr.grh.dgid.raw[0],
902 	       sizeof(qp_params->dgid));
903 
904 	qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id);
905 
906 	for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) {
907 		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
908 		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
909 	}
910 
911 	QL_DPRINT12(ha, "exit\n");
912 	return;
913 }
914 
915 static int
916 qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
917 {
918 	struct qlnxr_mm	*mm;
919 	qlnx_host_t	*ha;
920 
921 	ha = uctx->dev->ha;
922 
923 	QL_DPRINT12(ha, "enter\n");
924 
925 	mm = kzalloc(sizeof(*mm), GFP_KERNEL);
926 	if (mm == NULL) {
927 		QL_DPRINT11(ha, "mm = NULL\n");
928 		return -ENOMEM;
929 	}
930 
931 	mm->key.phy_addr = phy_addr;
932 
933 	/* This function might be called with a length which is not a multiple
934 	 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
935 	 * forces this granularity by increasing the requested size if needed.
936 	 * When qedr_mmap is called, it will search the list with the updated
937 	 * length as a key. To prevent search failures, the length is rounded up
938 	 * in advance to PAGE_SIZE.
939 	 */
940 	mm->key.len = roundup(len, PAGE_SIZE);
941 	INIT_LIST_HEAD(&mm->entry);
942 
943 	mutex_lock(&uctx->mm_list_lock);
944 	list_add(&mm->entry, &uctx->mm_head);
945 	mutex_unlock(&uctx->mm_list_lock);
946 
947 	QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
948 		(unsigned long long)mm->key.phy_addr,
949 		(unsigned long)mm->key.len, uctx);
950 
951 	return 0;
952 }
953 
954 static bool
955 qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
956 {
957 	bool		found = false;
958 	struct qlnxr_mm	*mm;
959 	qlnx_host_t	*ha;
960 
961 	ha = uctx->dev->ha;
962 
963 	QL_DPRINT12(ha, "enter\n");
964 
965 	mutex_lock(&uctx->mm_list_lock);
966 	list_for_each_entry(mm, &uctx->mm_head, entry) {
967 		if (len != mm->key.len || phy_addr != mm->key.phy_addr)
968 			continue;
969 
970 		found = true;
971 		break;
972 	}
973 	mutex_unlock(&uctx->mm_list_lock);
974 
975 	QL_DPRINT12(ha,
976 		"searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n",
977 		mm->key.phy_addr, mm->key.len, uctx, found);
978 
979 	return found;
980 }
981 
982 struct
983 ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
984                 struct ib_udata *udata)
985 {
986         int rc;
987         struct qlnxr_ucontext *ctx;
988         struct qlnxr_alloc_ucontext_resp uresp;
989         struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
990         qlnx_host_t *ha = dev->ha;
991         struct ecore_rdma_add_user_out_params oparams;
992 
993         if (!udata) {
994                 return ERR_PTR(-EFAULT);
995         }
996 
997 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
998 	if (!ctx)
999 		return ERR_PTR(-ENOMEM);
1000 
1001 	rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);
1002 	if (rc) {
1003 		QL_DPRINT12(ha,
1004 			"Failed to allocate a DPI for a new RoCE application "
1005 			",rc = %d. To overcome this, consider to increase "
1006 			"the number of DPIs, increase the doorbell BAR size "
1007 			"or just close unnecessary RoCE applications. In "
1008 			"order to increase the number of DPIs consult the "
1009 			"README\n", rc);
1010 		goto err;
1011 	}
1012 
1013 	ctx->dpi = oparams.dpi;
1014 	ctx->dpi_addr = oparams.dpi_addr;
1015 	ctx->dpi_phys_addr = oparams.dpi_phys_addr;
1016 	ctx->dpi_size = oparams.dpi_size;
1017 	INIT_LIST_HEAD(&ctx->mm_head);
1018 	mutex_init(&ctx->mm_list_lock);
1019 
1020 	memset(&uresp, 0, sizeof(uresp));
1021 	uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled)
1022 				< udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out
1023 	uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled)
1024 				< udata->outlen ? 1 : 0; //TODO: figure this out
1025 	uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count)
1026 				< udata->outlen ? oparams.wid_count : 0; //TODO: figure this out
1027         uresp.db_pa = ctx->dpi_phys_addr;
1028         uresp.db_size = ctx->dpi_size;
1029         uresp.max_send_wr = dev->attr.max_sqe;
1030         uresp.max_recv_wr = dev->attr.max_rqe;
1031         uresp.max_srq_wr = dev->attr.max_srq_wr;
1032         uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
1033         uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
1034         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
1035         uresp.max_cqes = QLNXR_MAX_CQES;
1036 
1037 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1038 	if (rc)
1039 		goto err;
1040 
1041 	ctx->dev = dev;
1042 
1043 	rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
1044 	if (rc)
1045 		goto err;
1046 	QL_DPRINT12(ha, "Allocated user context %p\n",
1047 		&ctx->ibucontext);
1048 
1049 	return &ctx->ibucontext;
1050 err:
1051 	kfree(ctx);
1052 	return ERR_PTR(rc);
1053 }
1054 
1055 int
1056 qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
1057 {
1058         struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx);
1059         struct qlnxr_dev *dev = uctx->dev;
1060         qlnx_host_t *ha = dev->ha;
1061         struct qlnxr_mm *mm, *tmp;
1062         int status = 0;
1063 
1064         QL_DPRINT12(ha, "Deallocating user context %p\n",
1065                         uctx);
1066 
1067         if (dev) {
1068                 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
1069         }
1070 
1071         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
1072                 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for"
1073                                 " ctx=%p\n",
1074                                 mm->key.phy_addr, mm->key.len, uctx);
1075                 list_del(&mm->entry);
1076                 kfree(mm);
1077         }
1078         kfree(uctx);
1079         return status;
1080 }
1081 
1082 int
1083 qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
1084 {
1085 	struct qlnxr_ucontext	*ucontext = get_qlnxr_ucontext(context);
1086 	struct qlnxr_dev	*dev = get_qlnxr_dev((context->device));
1087 	unsigned long		vm_page = vma->vm_pgoff << PAGE_SHIFT;
1088 	u64 			unmapped_db;
1089 	unsigned long 		len = (vma->vm_end - vma->vm_start);
1090 	int 			rc = 0;
1091 	bool 			found;
1092 	qlnx_host_t		*ha;
1093 
1094 	ha = dev->ha;
1095 
1096 #if __FreeBSD_version > 1102000
1097 	unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
1098 #else
1099 	unmapped_db = dev->db_phys_addr;
1100 #endif /* #if __FreeBSD_version > 1102000 */
1101 
1102 	QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx"
1103 		" vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
1104 		vm_page, vma->vm_pgoff, unmapped_db,
1105 		dev->db_size, len);
1106 
1107 	if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
1108 		QL_DPRINT11(ha, "Vma_start not page aligned "
1109 			"vm_start = %ld vma_end = %ld\n", vma->vm_start,
1110 			vma->vm_end);
1111 		return -EINVAL;
1112 	}
1113 
1114 	found = qlnxr_search_mmap(ucontext, vm_page, len);
1115 	if (!found) {
1116 		QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n",
1117 			vma->vm_pgoff);
1118 		return -EINVAL;
1119 	}
1120 
1121 	QL_DPRINT12(ha, "Mapping doorbell bar\n");
1122 
1123 #if __FreeBSD_version > 1102000
1124 
1125 	if ((vm_page < unmapped_db) ||
1126 		((vm_page + len) > (unmapped_db + ucontext->dpi_size))) {
1127 		QL_DPRINT11(ha, "failed pages are outside of dpi;"
1128 			"page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n",
1129 			vm_page, unmapped_db, ucontext->dpi_size);
1130 		return -EINVAL;
1131 	}
1132 
1133 	if (vma->vm_flags & VM_READ) {
1134 		QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n");
1135 		return -EINVAL;
1136 	}
1137 
1138 	vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1139 	rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
1140 			vma->vm_page_prot);
1141 
1142 #else
1143 
1144 	if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
1145 		dev->db_size))) {
1146 		QL_DPRINT12(ha, "Mapping doorbell bar\n");
1147 
1148 		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1149 
1150 		rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1151 					    PAGE_SIZE, vma->vm_page_prot);
1152 	} else {
1153 		QL_DPRINT12(ha, "Mapping chains\n");
1154 		rc = io_remap_pfn_range(vma, vma->vm_start,
1155 					 vma->vm_pgoff, len, vma->vm_page_prot);
1156 	}
1157 
1158 #endif /* #if __FreeBSD_version > 1102000 */
1159 
1160 	QL_DPRINT12(ha, "exit [%d]\n", rc);
1161 	return rc;
1162 }
1163 
1164 struct ib_mr *
1165 qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc)
1166 {
1167 	struct qlnxr_mr		*mr;
1168 	struct qlnxr_dev	*dev = get_qlnxr_dev((ibpd->device));
1169 	struct qlnxr_pd		*pd = get_qlnxr_pd(ibpd);
1170 	int			rc;
1171 	qlnx_host_t		*ha;
1172 
1173 	ha = dev->ha;
1174 
1175 	QL_DPRINT12(ha, "enter\n");
1176 
1177 	if (acc & IB_ACCESS_MW_BIND) {
1178 		QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n");
1179 	}
1180 
1181 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1182 	if (!mr) {
1183 		rc = -ENOMEM;
1184 		QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc);
1185 		goto err0;
1186 	}
1187 
1188 	mr->type = QLNXR_MR_DMA;
1189 
1190 	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1191 	if (rc) {
1192 		QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc);
1193 		goto err1;
1194 	}
1195 
1196 	/* index only, 18 bit long, lkey = itid << 8 | key */
1197 	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1198 	mr->hw_mr.pd = pd->pd_id;
1199 	mr->hw_mr.local_read = 1;
1200 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1201 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1202 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1203 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1204 	mr->hw_mr.dma_mr = true;
1205 
1206 	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1207 	if (rc) {
1208 		QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc);
1209 		goto err2;
1210 	}
1211 
1212 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1213 
1214 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1215 		mr->hw_mr.remote_atomic) {
1216 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1217 	}
1218 
1219 	QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey);
1220 
1221 	return &mr->ibmr;
1222 
1223 err2:
1224 	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1225 err1:
1226 	kfree(mr);
1227 err0:
1228 	QL_DPRINT12(ha, "exit [%d]\n", rc);
1229 
1230 	return ERR_PTR(rc);
1231 }
1232 
1233 static void
1234 qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info,
1235 	struct qlnxr_pbl *pbl)
1236 {
1237 	int		i;
1238 	qlnx_host_t	*ha;
1239 
1240 	ha = dev->ha;
1241 
1242 	QL_DPRINT12(ha, "enter\n");
1243 
1244 	for (i = 0; i < pbl_info->num_pbls; i++) {
1245 		if (!pbl[i].va)
1246 			continue;
1247 		qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa,
1248 			pbl_info->pbl_size);
1249 	}
1250 	kfree(pbl);
1251 
1252 	QL_DPRINT12(ha, "exit\n");
1253 	return;
1254 }
1255 
1256 #define MIN_FW_PBL_PAGE_SIZE (4*1024)
1257 #define MAX_FW_PBL_PAGE_SIZE (64*1024)
1258 
1259 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
1260 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
1261 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE)
1262 
1263 static struct qlnxr_pbl *
1264 qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev,
1265 	struct qlnxr_pbl_info *pbl_info, gfp_t flags)
1266 {
1267 	void			*va;
1268 	dma_addr_t		pa;
1269 	dma_addr_t		*pbl_main_tbl;
1270 	struct qlnxr_pbl	*pbl_table;
1271 	int			i, rc = 0;
1272 	qlnx_host_t		*ha;
1273 
1274 	ha = dev->ha;
1275 
1276 	QL_DPRINT12(ha, "enter\n");
1277 
1278 	pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags);
1279 
1280 	if (!pbl_table) {
1281 		QL_DPRINT12(ha, "pbl_table = NULL\n");
1282 		return NULL;
1283 	}
1284 
1285 	for (i = 0; i < pbl_info->num_pbls; i++) {
1286 		va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size);
1287 		if (!va) {
1288 			QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i);
1289 			rc = -ENOMEM;
1290 			goto err;
1291 		}
1292 		memset(va, 0, pbl_info->pbl_size);
1293 		pbl_table[i].va = va;
1294 		pbl_table[i].pa = pa;
1295 	}
1296 
1297 	/* Two-Layer PBLs, if we have more than one pbl we need to initialize
1298 	 * the first one with physical pointers to all of the rest
1299 	 */
1300 	pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
1301 	for (i = 0; i < pbl_info->num_pbls - 1; i++)
1302 		pbl_main_tbl[i] = pbl_table[i + 1].pa;
1303 
1304 	QL_DPRINT12(ha, "exit\n");
1305 	return pbl_table;
1306 
1307 err:
1308 	qlnxr_free_pbl(dev, pbl_info, pbl_table);
1309 
1310 	QL_DPRINT12(ha, "exit with error\n");
1311 	return NULL;
1312 }
1313 
1314 static int
1315 qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev,
1316 	struct qlnxr_pbl_info *pbl_info,
1317 	u32 num_pbes,
1318 	int two_layer_capable)
1319 {
1320 	u32		pbl_capacity;
1321 	u32		pbl_size;
1322 	u32		num_pbls;
1323 	qlnx_host_t	*ha;
1324 
1325 	ha = dev->ha;
1326 
1327 	QL_DPRINT12(ha, "enter\n");
1328 
1329 	if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
1330 		if (num_pbes > MAX_PBES_TWO_LAYER) {
1331 			QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n",
1332 				num_pbes);
1333 			return -EINVAL;
1334 		}
1335 
1336 		/* calculate required pbl page size */
1337 		pbl_size = MIN_FW_PBL_PAGE_SIZE;
1338 		pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
1339 			NUM_PBES_ON_PAGE(pbl_size);
1340 
1341 		while (pbl_capacity < num_pbes) {
1342 			pbl_size *= 2;
1343 			pbl_capacity = pbl_size / sizeof(u64);
1344 			pbl_capacity = pbl_capacity * pbl_capacity;
1345 		}
1346 
1347 		num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
1348 		num_pbls++; /* One for the layer0 ( points to the pbls) */
1349 		pbl_info->two_layered = true;
1350 	} else {
1351 		/* One layered PBL */
1352 		num_pbls = 1;
1353 		pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \
1354 				roundup_pow_of_two((num_pbes * sizeof(u64))));
1355 		pbl_info->two_layered = false;
1356 	}
1357 
1358 	pbl_info->num_pbls = num_pbls;
1359 	pbl_info->pbl_size = pbl_size;
1360 	pbl_info->num_pbes = num_pbes;
1361 
1362 	QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n",
1363 		pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
1364 
1365 	return 0;
1366 }
1367 
1368 #define upper_32_bits(x) (uint32_t)(x >> 32)
1369 #define lower_32_bits(x) (uint32_t)(x)
1370 
1371 static void
1372 qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem,
1373 	struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
1374 {
1375 	struct regpair		*pbe;
1376 	struct qlnxr_pbl	*pbl_tbl;
1377 	struct scatterlist	*sg;
1378 	int			shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
1379 	qlnx_host_t		*ha;
1380 
1381 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1382         int                     i;
1383         struct                  ib_umem_chunk *chunk = NULL;
1384 #else
1385         int                     entry;
1386 #endif
1387 
1388 	ha = dev->ha;
1389 
1390 	QL_DPRINT12(ha, "enter\n");
1391 
1392 	if (!pbl_info) {
1393 		QL_DPRINT11(ha, "PBL_INFO not initialized\n");
1394 		return;
1395 	}
1396 
1397 	if (!pbl_info->num_pbes) {
1398 		QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
1399 		return;
1400 	}
1401 
1402 	/* If we have a two layered pbl, the first pbl points to the rest
1403 	 * of the pbls and the first entry lays on the second pbl in the table
1404 	 */
1405 	if (pbl_info->two_layered)
1406 		pbl_tbl = &pbl[1];
1407 	else
1408 		pbl_tbl = pbl;
1409 
1410 	pbe = (struct regpair *)pbl_tbl->va;
1411 	if (!pbe) {
1412 		QL_DPRINT12(ha, "pbe is NULL\n");
1413 		return;
1414 	}
1415 
1416 	pbe_cnt = 0;
1417 
1418 	shift = ilog2(umem->page_size);
1419 
1420 #ifndef DEFINE_IB_UMEM_WITH_CHUNK
1421 
1422 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
1423 #else
1424 	list_for_each_entry(chunk, &umem->chunk_list, list) {
1425 		/* get all the dma regions from the chunk. */
1426 		for (i = 0; i < chunk->nmap; i++) {
1427 			sg = &chunk->page_list[i];
1428 #endif
1429 			pages = sg_dma_len(sg) >> shift;
1430 			for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
1431 				/* store the page address in pbe */
1432 				pbe->lo =
1433 				    cpu_to_le32(sg_dma_address(sg) +
1434 						(umem->page_size * pg_cnt));
1435 				pbe->hi =
1436 				    cpu_to_le32(upper_32_bits
1437 						((sg_dma_address(sg) +
1438 						  umem->page_size * pg_cnt)));
1439 
1440 				QL_DPRINT12(ha,
1441 					"Populate pbl table:"
1442 					" pbe->addr=0x%x:0x%x "
1443 					" pbe_cnt = %d total_num_pbes=%d"
1444 					" pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
1445 					total_num_pbes, pbe);
1446 
1447 				pbe_cnt ++;
1448 				total_num_pbes ++;
1449 				pbe++;
1450 
1451 				if (total_num_pbes == pbl_info->num_pbes)
1452 					return;
1453 
1454 				/* if the given pbl is full storing the pbes,
1455 				 * move to next pbl.
1456 				 */
1457 				if (pbe_cnt ==
1458 					(pbl_info->pbl_size / sizeof(u64))) {
1459 					pbl_tbl++;
1460 					pbe = (struct regpair *)pbl_tbl->va;
1461 					pbe_cnt = 0;
1462 				}
1463 			}
1464 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
1465 		}
1466 #endif
1467 	}
1468 	QL_DPRINT12(ha, "exit\n");
1469 	return;
1470 }
1471 
1472 static void
1473 free_mr_info(struct qlnxr_dev *dev, struct mr_info *info)
1474 {
1475 	struct qlnxr_pbl *pbl, *tmp;
1476 	qlnx_host_t		*ha;
1477 
1478 	ha = dev->ha;
1479 
1480 	QL_DPRINT12(ha, "enter\n");
1481 
1482 	if (info->pbl_table)
1483 		list_add_tail(&info->pbl_table->list_entry,
1484 			      &info->free_pbl_list);
1485 
1486 	if (!list_empty(&info->inuse_pbl_list))
1487 		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
1488 
1489 	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
1490 		list_del(&pbl->list_entry);
1491 		qlnxr_free_pbl(dev, &info->pbl_info, pbl);
1492 	}
1493 	QL_DPRINT12(ha, "exit\n");
1494 
1495 	return;
1496 }
1497 
1498 static int
1499 qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info,
1500 	size_t page_list_len, bool two_layered)
1501 {
1502 	int			rc;
1503 	struct qlnxr_pbl	*tmp;
1504 	qlnx_host_t		*ha;
1505 
1506 	ha = dev->ha;
1507 
1508 	QL_DPRINT12(ha, "enter\n");
1509 
1510 	INIT_LIST_HEAD(&info->free_pbl_list);
1511 	INIT_LIST_HEAD(&info->inuse_pbl_list);
1512 
1513 	rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info,
1514 				  page_list_len, two_layered);
1515 	if (rc) {
1516 		QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc);
1517 		goto done;
1518 	}
1519 
1520 	info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1521 
1522 	if (!info->pbl_table) {
1523 		rc = -ENOMEM;
1524 		QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n");
1525 		goto done;
1526 	}
1527 
1528 	QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa);
1529 
1530 	/* in usual case we use 2 PBLs, so we add one to free
1531 	 * list and allocating another one
1532 	 */
1533 	tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
1534 
1535 	if (!tmp) {
1536 		QL_DPRINT11(ha, "Extra PBL is not allocated\n");
1537 		goto done; /* it's OK if second allocation fails, so rc = 0*/
1538 	}
1539 
1540 	list_add_tail(&tmp->list_entry, &info->free_pbl_list);
1541 
1542 	QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa);
1543 
1544 done:
1545 	if (rc)
1546 		free_mr_info(dev, info);
1547 
1548 	QL_DPRINT12(ha, "exit [%d]\n", rc);
1549 
1550 	return rc;
1551 }
1552 
1553 struct ib_mr *
1554 #if __FreeBSD_version >= 1102000
1555 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1556 	u64 usr_addr, int acc, struct ib_udata *udata)
1557 #else
1558 qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
1559 	u64 usr_addr, int acc, struct ib_udata *udata, int mr_id)
1560 #endif /* #if __FreeBSD_version >= 1102000 */
1561 {
1562 	int		rc = -ENOMEM;
1563 	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
1564 	struct qlnxr_mr *mr;
1565 	struct qlnxr_pd *pd;
1566 	qlnx_host_t	*ha;
1567 
1568 	ha = dev->ha;
1569 
1570 	QL_DPRINT12(ha, "enter\n");
1571 
1572 	pd = get_qlnxr_pd(ibpd);
1573 
1574 	QL_DPRINT12(ha, "qedr_register user mr pd = %d"
1575 		" start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
1576 		pd->pd_id, start, len, usr_addr, acc);
1577 
1578 	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
1579 		QL_DPRINT11(ha,
1580 			"(acc & IB_ACCESS_REMOTE_WRITE &&"
1581 			" !(acc & IB_ACCESS_LOCAL_WRITE))\n");
1582 		return ERR_PTR(-EINVAL);
1583 	}
1584 
1585 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1586 	if (!mr) {
1587 		QL_DPRINT11(ha, "kzalloc(mr) failed\n");
1588 		return ERR_PTR(rc);
1589 	}
1590 
1591 	mr->type = QLNXR_MR_USER;
1592 
1593 	mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
1594 	if (IS_ERR(mr->umem)) {
1595 		rc = -EFAULT;
1596 		QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem);
1597 		goto err0;
1598 	}
1599 
1600 	rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
1601 	if (rc) {
1602 		QL_DPRINT11(ha,
1603 			"qlnxr_init_mr_info failed [%d]\n", rc);
1604 		goto err1;
1605 	}
1606 
1607 	qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
1608 			   &mr->info.pbl_info);
1609 
1610 	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
1611 
1612 	if (rc) {
1613 		QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
1614 		goto err1;
1615 	}
1616 
1617 	/* index only, 18 bit long, lkey = itid << 8 | key */
1618 	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
1619 	mr->hw_mr.key = 0;
1620 	mr->hw_mr.pd = pd->pd_id;
1621 	mr->hw_mr.local_read = 1;
1622 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
1623 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
1624 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1625 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
1626 	mr->hw_mr.mw_bind = false; /* TBD MW BIND */
1627 	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
1628 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
1629 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
1630 	mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */
1631 
1632 #if __FreeBSD_version >= 1102000
1633 	mr->hw_mr.fbo = ib_umem_offset(mr->umem);
1634 #else
1635 	mr->hw_mr.fbo = mr->umem->offset;
1636 #endif
1637 	mr->hw_mr.length = len;
1638 	mr->hw_mr.vaddr = usr_addr;
1639 	mr->hw_mr.zbva = false; /* TBD figure when this should be true */
1640 	mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
1641 	mr->hw_mr.dma_mr = false;
1642 
1643 	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
1644 	if (rc) {
1645 		QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
1646 		goto err2;
1647 	}
1648 
1649 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1650 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
1651 		mr->hw_mr.remote_atomic)
1652 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
1653 
1654 	QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey);
1655 
1656 	return (&mr->ibmr);
1657 
1658 err2:
1659 	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
1660 err1:
1661 	qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1662 err0:
1663 	kfree(mr);
1664 
1665 	QL_DPRINT12(ha, "exit [%d]\n", rc);
1666 	return (ERR_PTR(rc));
1667 }
1668 
1669 int
1670 qlnxr_dereg_mr(struct ib_mr *ib_mr)
1671 {
1672 	struct qlnxr_mr	*mr = get_qlnxr_mr(ib_mr);
1673 	struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));
1674 	int		rc = 0;
1675 	qlnx_host_t	*ha;
1676 
1677 	ha = dev->ha;
1678 
1679 	QL_DPRINT12(ha, "enter\n");
1680 
1681 	if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR))
1682 		qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
1683 
1684 	/* it could be user registered memory. */
1685 	if (mr->umem)
1686 		ib_umem_release(mr->umem);
1687 
1688 	kfree(mr->pages);
1689 
1690 	kfree(mr);
1691 
1692 	QL_DPRINT12(ha, "exit\n");
1693 	return rc;
1694 }
1695 
1696 static int
1697 qlnxr_copy_cq_uresp(struct qlnxr_dev *dev,
1698 	struct qlnxr_cq *cq, struct ib_udata *udata)
1699 {
1700 	struct qlnxr_create_cq_uresp	uresp;
1701 	int				rc;
1702 	qlnx_host_t			*ha;
1703 
1704 	ha = dev->ha;
1705 
1706 	QL_DPRINT12(ha, "enter\n");
1707 
1708 	memset(&uresp, 0, sizeof(uresp));
1709 
1710 	uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1711 	uresp.icid = cq->icid;
1712 
1713 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1714 
1715 	if (rc) {
1716 		QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n",
1717 			cq->icid, rc);
1718 	}
1719 
1720 	QL_DPRINT12(ha, "exit [%d]\n", rc);
1721 	return rc;
1722 }
1723 
1724 static void
1725 consume_cqe(struct qlnxr_cq *cq)
1726 {
1727 
1728 	if (cq->latest_cqe == cq->toggle_cqe)
1729 		cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1730 
1731 	cq->latest_cqe = ecore_chain_consume(&cq->pbl);
1732 }
1733 
1734 static inline int
1735 qlnxr_align_cq_entries(int entries)
1736 {
1737 	u64 size, aligned_size;
1738 
1739 	/* We allocate an extra entry that we don't report to the FW.
1740 	 * Why?
1741 	 * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes
1742 	 * (for performance purposes). Allocating an extra entry and telling
1743 	 * the FW we have less prevents overwriting the first entry in case of
1744 	 * a wrap i.e. when the FW writes the last entry and the application
1745 	 * hasn't read the first one.
1746 	 */
1747 	size = (entries + 1) * QLNXR_CQE_SIZE;
1748 
1749 	/* We align to PAGE_SIZE.
1750 	 * Why?
1751 	 * Since the CQ is going to be mapped and the mapping is anyhow in whole
1752 	 * kernel pages we benefit from the possibly extra CQEs.
1753 	 */
1754 	aligned_size = ALIGN(size, PAGE_SIZE);
1755 
1756 	/* note: for CQs created in user space the result of this function
1757 	 * should match the size mapped in user space
1758 	 */
1759 	return (aligned_size / QLNXR_CQE_SIZE);
1760 }
1761 
1762 static inline int
1763 qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev,
1764 	struct qlnxr_userq *q, u64 buf_addr, size_t buf_len,
1765 	int access, int dmasync, int alloc_and_init)
1766 {
1767 	int		page_cnt;
1768 	int		rc;
1769 	qlnx_host_t	*ha;
1770 
1771 	ha = dev->ha;
1772 
1773 	QL_DPRINT12(ha, "enter\n");
1774 
1775 	q->buf_addr = buf_addr;
1776 	q->buf_len = buf_len;
1777 
1778 	QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x"
1779 	      " dmasync : %x\n", q->buf_addr, q->buf_len,
1780 		access, dmasync);
1781 
1782 	q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
1783 
1784 	if (IS_ERR(q->umem)) {
1785 		QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem));
1786 		return PTR_ERR(q->umem);
1787 	}
1788 
1789 	page_cnt = ib_umem_page_count(q->umem);
1790 	rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt,
1791 				  0 /* SQ and RQ don't support dual layer pbl.
1792 				     * CQ may, but this is yet uncoded.
1793 				     */);
1794 	if (rc) {
1795 		QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc);
1796 		goto err;
1797 	}
1798 
1799 	if (alloc_and_init) {
1800 		q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
1801 
1802 		if (!q->pbl_tbl) {
1803 			QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1804 			rc = -ENOMEM;
1805 			goto err;
1806 		}
1807 
1808 		qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
1809 	} else {
1810 		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
1811 
1812 		if (!q->pbl_tbl) {
1813 			QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n");
1814 			rc = -ENOMEM;
1815 			goto err;
1816 		}
1817 	}
1818 
1819 	QL_DPRINT12(ha, "exit\n");
1820 	return 0;
1821 
1822 err:
1823 	ib_umem_release(q->umem);
1824 	q->umem = NULL;
1825 
1826 	QL_DPRINT12(ha, "exit [%d]\n", rc);
1827 	return rc;
1828 }
1829 
1830 #if __FreeBSD_version >= 1102000
1831 
1832 struct ib_cq *
1833 qlnxr_create_cq(struct ib_device *ibdev,
1834 	const struct ib_cq_init_attr *attr,
1835 	struct ib_ucontext *ib_ctx,
1836 	struct ib_udata *udata)
1837 
1838 #else
1839 
1840 #if __FreeBSD_version >= 1100000
1841 
1842 struct ib_cq *
1843 qlnxr_create_cq(struct ib_device *ibdev,
1844 	struct ib_cq_init_attr *attr,
1845 	struct ib_ucontext *ib_ctx,
1846 	struct ib_udata *udata)
1847 
1848 #else
1849 
1850 struct ib_cq *
1851 qlnxr_create_cq(struct ib_device *ibdev,
1852 	int entries,
1853 	int vector,
1854 	struct ib_ucontext *ib_ctx,
1855 	struct ib_udata *udata)
1856 #endif /* #if __FreeBSD_version >= 1100000 */
1857 
1858 #endif /* #if __FreeBSD_version >= 1102000 */
1859 {
1860 	struct qlnxr_ucontext			*ctx;
1861 	struct ecore_rdma_destroy_cq_out_params destroy_oparams;
1862 	struct ecore_rdma_destroy_cq_in_params	destroy_iparams;
1863 	struct qlnxr_dev			*dev;
1864 	struct ecore_rdma_create_cq_in_params	params;
1865 	struct qlnxr_create_cq_ureq		ureq;
1866 
1867 #if __FreeBSD_version >= 1100000
1868 	int					vector = attr->comp_vector;
1869 	int					entries = attr->cqe;
1870 #endif
1871 	struct qlnxr_cq				*cq;
1872 	int					chain_entries, rc, page_cnt;
1873 	u64					pbl_ptr;
1874 	u16					icid;
1875 	qlnx_host_t				*ha;
1876 
1877 	dev = get_qlnxr_dev(ibdev);
1878 	ha = dev->ha;
1879 
1880 	QL_DPRINT12(ha, "called from %s. entries = %d, "
1881 		"vector = %d\n",
1882 		(udata ? "User Lib" : "Kernel"), entries, vector);
1883 
1884         memset(&params, 0, sizeof(struct ecore_rdma_create_cq_in_params));
1885         memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params));
1886         memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params));
1887 
1888 	if (entries > QLNXR_MAX_CQES) {
1889 		QL_DPRINT11(ha,
1890 			"the number of entries %d is too high. "
1891 			"Must be equal or below %d.\n",
1892 			entries, QLNXR_MAX_CQES);
1893 		return ERR_PTR(-EINVAL);
1894 	}
1895 	chain_entries = qlnxr_align_cq_entries(entries);
1896 	chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES);
1897 
1898 	cq = qlnx_zalloc((sizeof(struct qlnxr_cq)));
1899 
1900 	if (!cq)
1901 		return ERR_PTR(-ENOMEM);
1902 
1903 	if (udata) {
1904 		memset(&ureq, 0, sizeof(ureq));
1905 
1906 		if (ib_copy_from_udata(&ureq, udata,
1907 			min(sizeof(ureq), udata->inlen))) {
1908 			QL_DPRINT11(ha, "ib_copy_from_udata failed\n");
1909 			goto err0;
1910 		}
1911 
1912 		if (!ureq.len) {
1913 			QL_DPRINT11(ha, "ureq.len == 0\n");
1914 			goto err0;
1915 		}
1916 
1917 		cq->cq_type = QLNXR_CQ_TYPE_USER;
1918 
1919 		qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len,
1920 				     IB_ACCESS_LOCAL_WRITE, 1, 1);
1921 
1922 		pbl_ptr = cq->q.pbl_tbl->pa;
1923 		page_cnt = cq->q.pbl_info.num_pbes;
1924 		cq->ibcq.cqe = chain_entries;
1925 	} else {
1926 		cq->cq_type = QLNXR_CQ_TYPE_KERNEL;
1927 
1928                 rc = ecore_chain_alloc(&dev->ha->cdev,
1929                            ECORE_CHAIN_USE_TO_CONSUME,
1930                            ECORE_CHAIN_MODE_PBL,
1931                            ECORE_CHAIN_CNT_TYPE_U32,
1932                            chain_entries,
1933                            sizeof(union roce_cqe),
1934                            &cq->pbl, NULL);
1935 
1936 		if (rc)
1937 			goto err1;
1938 
1939 		page_cnt = ecore_chain_get_page_cnt(&cq->pbl);
1940 		pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl);
1941 		cq->ibcq.cqe = cq->pbl.capacity;
1942 	}
1943 
1944         params.cq_handle_hi = upper_32_bits((uintptr_t)cq);
1945         params.cq_handle_lo = lower_32_bits((uintptr_t)cq);
1946         params.cnq_id = vector;
1947         params.cq_size = chain_entries - 1;
1948         params.pbl_num_pages = page_cnt;
1949         params.pbl_ptr = pbl_ptr;
1950         params.pbl_two_level = 0;
1951 
1952 	if (ib_ctx != NULL) {
1953 		ctx = get_qlnxr_ucontext(ib_ctx);
1954         	params.dpi = ctx->dpi;
1955 	} else {
1956         	params.dpi = dev->dpi;
1957 	}
1958 
1959 	rc = ecore_rdma_create_cq(dev->rdma_ctx, &params, &icid);
1960 	if (rc)
1961 		goto err2;
1962 
1963 	cq->icid = icid;
1964 	cq->sig = QLNXR_CQ_MAGIC_NUMBER;
1965 	spin_lock_init(&cq->cq_lock);
1966 
1967 	if (ib_ctx) {
1968 		rc = qlnxr_copy_cq_uresp(dev, cq, udata);
1969 		if (rc)
1970 			goto err3;
1971 	} else {
1972 		/* Generate doorbell address.
1973 		 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT.
1974 		 * TODO: consider moving to device scope as it is a function of
1975 		 *       the device.
1976 		 * TODO: add ifdef if plan to support 16 bit.
1977 		 */
1978 		cq->db_addr = dev->db_addr +
1979 			DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
1980 		cq->db.data.icid = cq->icid;
1981 		cq->db.data.params = DB_AGG_CMD_SET <<
1982 				     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1983 
1984 		/* point to the very last element, passing it we will toggle */
1985 		cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl);
1986 		cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;
1987 
1988 		/* must be different from pbl_toggle */
1989 		cq->latest_cqe = NULL;
1990 		consume_cqe(cq);
1991 		cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
1992 	}
1993 
1994 	QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p,"
1995 		" number of entries = 0x%x\n",
1996 		cq->icid, cq, params.cq_size);
1997 	QL_DPRINT12(ha,"cq_addr = %p\n", cq);
1998 	return &cq->ibcq;
1999 
2000 err3:
2001 	destroy_iparams.icid = cq->icid;
2002 	ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams);
2003 err2:
2004 	if (udata)
2005 		qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2006 	else
2007 		ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2008 err1:
2009 	if (udata)
2010 		ib_umem_release(cq->q.umem);
2011 err0:
2012 	kfree(cq);
2013 
2014 	QL_DPRINT12(ha, "exit error\n");
2015 
2016 	return ERR_PTR(-EINVAL);
2017 }
2018 
2019 int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
2020 {
2021 	int			status = 0;
2022 	struct qlnxr_dev	*dev = get_qlnxr_dev((ibcq->device));
2023 	qlnx_host_t		*ha;
2024 
2025 	ha = dev->ha;
2026 
2027 	QL_DPRINT12(ha, "enter/exit\n");
2028 
2029 	return status;
2030 }
2031 
2032 int
2033 qlnxr_destroy_cq(struct ib_cq *ibcq)
2034 {
2035 	struct qlnxr_dev			*dev = get_qlnxr_dev((ibcq->device));
2036 	struct ecore_rdma_destroy_cq_out_params oparams;
2037 	struct ecore_rdma_destroy_cq_in_params	iparams;
2038 	struct qlnxr_cq				*cq = get_qlnxr_cq(ibcq);
2039 	int					rc = 0;
2040 	qlnx_host_t				*ha;
2041 
2042 	ha = dev->ha;
2043 
2044 	QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid);
2045 
2046 	cq->destroyed = 1;
2047 
2048 	/* TODO: Syncronize irq of the CNQ the CQ belongs to for validation
2049 	 * that all completions with notification are dealt with. The rest
2050 	 * of the completions are not interesting
2051 	 */
2052 
2053 	/* GSIs CQs are handled by driver, so they don't exist in the FW */
2054 
2055 	if (cq->cq_type != QLNXR_CQ_TYPE_GSI) {
2056 		iparams.icid = cq->icid;
2057 
2058 		rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
2059 
2060 		if (rc) {
2061 			QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n",
2062 				cq->icid);
2063 			return rc;
2064 		}
2065 
2066 		QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);
2067 		ecore_chain_free(&dev->ha->cdev, &cq->pbl);
2068 	}
2069 
2070 	if (ibcq->uobject && ibcq->uobject->context) {
2071 		qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
2072 		ib_umem_release(cq->q.umem);
2073 	}
2074 
2075 	cq->sig = ~cq->sig;
2076 
2077 	kfree(cq);
2078 
2079 	QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);
2080 
2081 	return rc;
2082 }
2083 
2084 static int
2085 qlnxr_check_qp_attrs(struct ib_pd *ibpd,
2086 	struct qlnxr_dev *dev,
2087 	struct ib_qp_init_attr *attrs,
2088 	struct ib_udata *udata)
2089 {
2090 	struct ecore_rdma_device	*qattr;
2091 	qlnx_host_t			*ha;
2092 
2093 	qattr = ecore_rdma_query_device(dev->rdma_ctx);
2094 	ha = dev->ha;
2095 
2096 	QL_DPRINT12(ha, "enter\n");
2097 
2098 	QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type);
2099 	QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type);
2100 	QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags);
2101 
2102 #if __FreeBSD_version < 1102000
2103 	QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type);
2104 #endif
2105 
2106 	QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num);
2107 	QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr);
2108 	QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr);
2109 	QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge);
2110 	QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge);
2111 	QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n",
2112 		attrs->cap.max_inline_data);
2113 
2114 #if __FreeBSD_version < 1102000
2115 	QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n",
2116 		attrs->cap.qpg_tss_mask_sz);
2117 #endif
2118 
2119 	QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id);
2120 	QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id);
2121 	QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver);
2122 	QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver);
2123 	QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid);
2124 	QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n",
2125 		(void *)qattr->sys_image_guid);
2126 	QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq);
2127 	QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge);
2128 	QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge);
2129 	QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline);
2130 	QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe);
2131 	QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe);
2132 	QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n",
2133 		qattr->max_qp_resp_rd_atomic_resc);
2134 	QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n",
2135 		qattr->max_qp_req_rd_atomic_resc);
2136 	QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n",
2137 		qattr->max_dev_resp_rd_atomic_resc);
2138 	QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq);
2139 	QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp);
2140 	QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq);
2141 	QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr);
2142 	QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size);
2143 	QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe);
2144 	QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw);
2145 	QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr);
2146 	QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n",
2147 		qattr->max_mr_mw_fmr_pbl);
2148 	QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n",
2149 		(void *)qattr->max_mr_mw_fmr_size);
2150 	QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd);
2151 	QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah);
2152 	QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey);
2153 	QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr);
2154 	QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n",
2155 		qattr->max_stats_queues);
2156 	//QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps);
2157 	QL_DPRINT12(ha, "qattr->page_size_caps = %p\n",
2158 		(void *)qattr->page_size_caps);
2159 	QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay);
2160 	QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey);
2161 	QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n",
2162 		qattr->bad_pkey_counter);
2163 
2164 	if ((attrs->qp_type == IB_QPT_GSI) && udata) {
2165 		QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n");
2166 		return -EINVAL;
2167 	}
2168 
2169 	if (udata && !(ibpd->uobject && ibpd->uobject->context)) {
2170 		QL_DPRINT12(ha, "called from user without context\n");
2171 		return -EINVAL;
2172 	}
2173 
2174 	/* QP0... attrs->qp_type == IB_QPT_GSI */
2175 	if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
2176 		QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n",
2177 			   attrs->qp_type);
2178 		return -EINVAL;
2179 	}
2180 	if (attrs->qp_type == IB_QPT_GSI && attrs->srq) {
2181 		QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n");
2182 		return -EINVAL;
2183 	}
2184 	/* Skip the check for QP1 to support CM size of 128 */
2185 	if (attrs->cap.max_send_wr > qattr->max_wqe) {
2186 		QL_DPRINT12(ha, "cannot create a SQ with %d elements "
2187 			" (max_send_wr=0x%x)\n",
2188 			attrs->cap.max_send_wr, qattr->max_wqe);
2189 		return -EINVAL;
2190 	}
2191 	if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) {
2192 		QL_DPRINT12(ha, "cannot create a RQ with %d elements"
2193 			" (max_recv_wr=0x%x)\n",
2194 			attrs->cap.max_recv_wr, qattr->max_wqe);
2195 		return -EINVAL;
2196 	}
2197 	if (attrs->cap.max_inline_data > qattr->max_inline) {
2198 		QL_DPRINT12(ha,
2199 			"unsupported inline data size=0x%x "
2200 			"requested (max_inline=0x%x)\n",
2201 			attrs->cap.max_inline_data, qattr->max_inline);
2202 		return -EINVAL;
2203 	}
2204 	if (attrs->cap.max_send_sge > qattr->max_sge) {
2205 		QL_DPRINT12(ha,
2206 			"unsupported send_sge=0x%x "
2207 			"requested (max_send_sge=0x%x)\n",
2208 			attrs->cap.max_send_sge, qattr->max_sge);
2209 		return -EINVAL;
2210 	}
2211 	if (attrs->cap.max_recv_sge > qattr->max_sge) {
2212 		QL_DPRINT12(ha,
2213 			"unsupported recv_sge=0x%x requested "
2214 			" (max_recv_sge=0x%x)\n",
2215 			attrs->cap.max_recv_sge, qattr->max_sge);
2216 		return -EINVAL;
2217 	}
2218 	/* unprivileged user space cannot create special QP */
2219 	if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
2220 		QL_DPRINT12(ha,
2221 			"userspace can't create special QPs of type=0x%x\n",
2222 			attrs->qp_type);
2223 		return -EINVAL;
2224 	}
2225 	/* allow creating only one GSI type of QP */
2226 	if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {
2227 		QL_DPRINT12(ha,
2228 			"create qp: GSI special QPs already created.\n");
2229 		return -EINVAL;
2230 	}
2231 
2232 	/* verify consumer QPs are not trying to use GSI QP's CQ */
2233 	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {
2234 		struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq);
2235 		struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq);
2236 
2237 		if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) ||
2238 		    (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) {
2239 			QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n");
2240 			return -EINVAL;
2241 		}
2242 	}
2243 	QL_DPRINT12(ha, "exit\n");
2244 	return 0;
2245 }
2246 
2247 static int
2248 qlnxr_copy_srq_uresp(struct qlnxr_dev *dev,
2249 	struct qlnxr_srq *srq,
2250 	struct ib_udata *udata)
2251 {
2252 	struct qlnxr_create_srq_uresp	uresp;
2253 	qlnx_host_t			*ha;
2254 	int				rc;
2255 
2256 	ha = dev->ha;
2257 
2258 	QL_DPRINT12(ha, "enter\n");
2259 
2260 	memset(&uresp, 0, sizeof(uresp));
2261 
2262 	uresp.srq_id = srq->srq_id;
2263 
2264 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2265 
2266 	QL_DPRINT12(ha, "exit [%d]\n", rc);
2267 	return rc;
2268 }
2269 
2270 static void
2271 qlnxr_copy_rq_uresp(struct qlnxr_dev *dev,
2272 	struct qlnxr_create_qp_uresp *uresp,
2273 	struct qlnxr_qp *qp)
2274 {
2275 	qlnx_host_t	*ha;
2276 
2277 	ha = dev->ha;
2278 
2279 	/* Return if QP is associated with SRQ instead of RQ */
2280 	QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq);
2281 
2282 	if (qp->srq)
2283 		return;
2284 
2285 	/* iWARP requires two doorbells per RQ. */
2286 	if (QLNX_IS_IWARP(dev)) {
2287 		uresp->rq_db_offset =
2288 			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2289 		uresp->rq_db2_offset =
2290 			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2291 
2292 		QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x "
2293 			"uresp->rq_db2_offset = 0x%x\n",
2294 			uresp->rq_db_offset, uresp->rq_db2_offset);
2295 	} else {
2296 		uresp->rq_db_offset =
2297 			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2298 	}
2299 	uresp->rq_icid = qp->icid;
2300 
2301 	QL_DPRINT12(ha, "exit\n");
2302 	return;
2303 }
2304 
2305 static void
2306 qlnxr_copy_sq_uresp(struct qlnxr_dev *dev,
2307 	struct qlnxr_create_qp_uresp *uresp,
2308 	struct qlnxr_qp *qp)
2309 {
2310 	qlnx_host_t	*ha;
2311 
2312 	ha = dev->ha;
2313 
2314 	QL_DPRINT12(ha, "enter\n");
2315 
2316 	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2317 
2318 	/* iWARP uses the same cid for rq and sq*/
2319 	if (QLNX_IS_IWARP(dev)) {
2320 		uresp->sq_icid = qp->icid;
2321 		QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid);
2322 	} else
2323 		uresp->sq_icid = qp->icid + 1;
2324 
2325 	QL_DPRINT12(ha, "exit\n");
2326 	return;
2327 }
2328 
2329 static int
2330 qlnxr_copy_qp_uresp(struct qlnxr_dev *dev,
2331 	struct qlnxr_qp *qp,
2332 	struct ib_udata *udata)
2333 {
2334 	int				rc;
2335 	struct qlnxr_create_qp_uresp	uresp;
2336 	qlnx_host_t			*ha;
2337 
2338 	ha = dev->ha;
2339 
2340 	QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid);
2341 
2342 	memset(&uresp, 0, sizeof(uresp));
2343 	qlnxr_copy_sq_uresp(dev, &uresp, qp);
2344 	qlnxr_copy_rq_uresp(dev, &uresp, qp);
2345 
2346 	uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
2347 	uresp.qp_id = qp->qp_id;
2348 
2349 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
2350 
2351 	QL_DPRINT12(ha, "exit [%d]\n", rc);
2352 	return rc;
2353 }
2354 
2355 static void
2356 qlnxr_set_common_qp_params(struct qlnxr_dev *dev,
2357 	struct qlnxr_qp *qp,
2358 	struct qlnxr_pd *pd,
2359 	struct ib_qp_init_attr *attrs)
2360 {
2361 	qlnx_host_t			*ha;
2362 
2363 	ha = dev->ha;
2364 
2365 	QL_DPRINT12(ha, "enter\n");
2366 
2367 	spin_lock_init(&qp->q_lock);
2368 
2369 	atomic_set(&qp->refcnt, 1);
2370 	qp->pd = pd;
2371 	qp->sig = QLNXR_QP_MAGIC_NUMBER;
2372 	qp->qp_type = attrs->qp_type;
2373 	qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2374 	qp->sq.max_sges = attrs->cap.max_send_sge;
2375 	qp->state = ECORE_ROCE_QP_STATE_RESET;
2376 	qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
2377 	qp->sq_cq = get_qlnxr_cq(attrs->send_cq);
2378 	qp->rq_cq = get_qlnxr_cq(attrs->recv_cq);
2379 	qp->dev = dev;
2380 
2381 	if (!attrs->srq) {
2382 		/* QP is associated with RQ instead of SRQ */
2383 		qp->rq.max_sges = attrs->cap.max_recv_sge;
2384 		QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
2385 			qp->rq.max_sges, qp->rq_cq->icid);
2386 	} else {
2387 		qp->srq = get_qlnxr_srq(attrs->srq);
2388 	}
2389 
2390 	QL_DPRINT12(ha,
2391 		"QP params:\tpd = %d, qp_type = %d, max_inline_data = %d,"
2392 		" state = %d, signaled = %d, use_srq=%d\n",
2393 		pd->pd_id, qp->qp_type, qp->max_inline_data,
2394 		qp->state, qp->signaled, ((attrs->srq) ? 1 : 0));
2395 	QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
2396 		qp->sq.max_sges, qp->sq_cq->icid);
2397 	return;
2398 }
2399 
2400 static int
2401 qlnxr_check_srq_params(struct ib_pd *ibpd,
2402 	struct qlnxr_dev *dev,
2403 	struct ib_srq_init_attr *attrs)
2404 {
2405 	struct ecore_rdma_device *qattr;
2406 	qlnx_host_t		*ha;
2407 
2408 	ha = dev->ha;
2409 	qattr = ecore_rdma_query_device(dev->rdma_ctx);
2410 
2411 	QL_DPRINT12(ha, "enter\n");
2412 
2413 	if (attrs->attr.max_wr > qattr->max_srq_wqe) {
2414 		QL_DPRINT12(ha, "unsupported srq_wr=0x%x"
2415 			" requested (max_srq_wr=0x%x)\n",
2416 			attrs->attr.max_wr, qattr->max_srq_wr);
2417 		return -EINVAL;
2418 	}
2419 
2420 	if (attrs->attr.max_sge > qattr->max_sge) {
2421 		QL_DPRINT12(ha,
2422 			"unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
2423 			attrs->attr.max_sge, qattr->max_sge);
2424 		return -EINVAL;
2425 	}
2426 
2427 	if (attrs->attr.srq_limit > attrs->attr.max_wr) {
2428 		QL_DPRINT12(ha,
2429 		       "unsupported srq_limit=0x%x requested"
2430 			" (max_srq_limit=0x%x)\n",
2431 			attrs->attr.srq_limit, attrs->attr.srq_limit);
2432 		return -EINVAL;
2433 	}
2434 
2435 	QL_DPRINT12(ha, "exit\n");
2436 	return 0;
2437 }
2438 
2439 static void
2440 qlnxr_free_srq_user_params(struct qlnxr_srq *srq)
2441 {
2442 	struct qlnxr_dev	*dev = srq->dev;
2443 	qlnx_host_t		*ha;
2444 
2445 	ha = dev->ha;
2446 
2447 	QL_DPRINT12(ha, "enter\n");
2448 
2449 	qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2450 	ib_umem_release(srq->usrq.umem);
2451 	ib_umem_release(srq->prod_umem);
2452 
2453 	QL_DPRINT12(ha, "exit\n");
2454 	return;
2455 }
2456 
2457 static void
2458 qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq)
2459 {
2460 	struct qlnxr_srq_hwq_info *hw_srq  = &srq->hw_srq;
2461 	struct qlnxr_dev	*dev = srq->dev;
2462 	qlnx_host_t		*ha;
2463 
2464 	ha = dev->ha;
2465 
2466 	QL_DPRINT12(ha, "enter\n");
2467 
2468 	ecore_chain_free(dev->cdev, &hw_srq->pbl);
2469 
2470 	qlnx_dma_free_coherent(&dev->cdev,
2471 		hw_srq->virt_prod_pair_addr,
2472 		hw_srq->phy_prod_pair_addr,
2473 		sizeof(struct rdma_srq_producers));
2474 
2475 	QL_DPRINT12(ha, "exit\n");
2476 
2477 	return;
2478 }
2479 
2480 static int
2481 qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx,
2482 	struct qlnxr_srq *srq,
2483 	struct qlnxr_create_srq_ureq *ureq,
2484 	int access, int dmasync)
2485 {
2486 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2487 	struct ib_umem_chunk	*chunk;
2488 #endif
2489 	struct scatterlist	*sg;
2490 	int			rc;
2491 	struct qlnxr_dev	*dev = srq->dev;
2492 	qlnx_host_t		*ha;
2493 
2494 	ha = dev->ha;
2495 
2496 	QL_DPRINT12(ha, "enter\n");
2497 
2498 	rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
2499 				  ureq->srq_len, access, dmasync, 1);
2500 	if (rc)
2501 		return rc;
2502 
2503 	srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
2504 				     sizeof(struct rdma_srq_producers),
2505 				     access, dmasync);
2506 	if (IS_ERR(srq->prod_umem)) {
2507 		qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
2508 		ib_umem_release(srq->usrq.umem);
2509 
2510 		QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n",
2511 			PTR_ERR(srq->prod_umem));
2512 
2513 		return PTR_ERR(srq->prod_umem);
2514 	}
2515 
2516 #ifdef DEFINE_IB_UMEM_WITH_CHUNK
2517 	chunk = container_of((&srq->prod_umem->chunk_list)->next,
2518 			     typeof(*chunk), list);
2519 	sg = &chunk->page_list[0];
2520 #else
2521 	sg = srq->prod_umem->sg_head.sgl;
2522 #endif
2523 	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
2524 
2525 	QL_DPRINT12(ha, "exit\n");
2526 	return 0;
2527 }
2528 
2529 static int
2530 qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq,
2531 	struct qlnxr_dev *dev,
2532 	struct ib_srq_init_attr *init_attr)
2533 {
2534 	struct qlnxr_srq_hwq_info	*hw_srq  = &srq->hw_srq;
2535 	dma_addr_t			phy_prod_pair_addr;
2536 	u32				num_elems, max_wr;
2537 	void				*va;
2538 	int				rc;
2539 	qlnx_host_t			*ha;
2540 
2541 	ha = dev->ha;
2542 
2543 	QL_DPRINT12(ha, "enter\n");
2544 
2545 	va = qlnx_dma_alloc_coherent(&dev->cdev,
2546 			&phy_prod_pair_addr,
2547 			sizeof(struct rdma_srq_producers));
2548 	if (!va) {
2549 		QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n");
2550 		return -ENOMEM;
2551 	}
2552 
2553 	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
2554 	hw_srq->virt_prod_pair_addr = va;
2555 
2556 	max_wr = init_attr->attr.max_wr;
2557 
2558 	num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE;
2559 
2560         rc = ecore_chain_alloc(dev->cdev,
2561                    ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2562                    ECORE_CHAIN_MODE_PBL,
2563                    ECORE_CHAIN_CNT_TYPE_U32,
2564                    num_elems,
2565                    ECORE_RDMA_SRQ_WQE_ELEM_SIZE,
2566                    &hw_srq->pbl, NULL);
2567 
2568 	if (rc) {
2569 		QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc);
2570 		goto err0;
2571 	}
2572 
2573 	hw_srq->max_wr = max_wr;
2574 	hw_srq->num_elems = num_elems;
2575 	hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ;
2576 
2577 	QL_DPRINT12(ha, "exit\n");
2578 	return 0;
2579 
2580 err0:
2581 	qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr,
2582 		sizeof(struct rdma_srq_producers));
2583 
2584 	QL_DPRINT12(ha, "exit [%d]\n", rc);
2585 	return rc;
2586 }
2587 
2588 static inline void
2589 qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev,
2590 	struct qlnxr_pd *pd,
2591 	struct qlnxr_qp *qp,
2592 	struct ib_qp_init_attr *attrs,
2593 	bool fmr_and_reserved_lkey,
2594 	struct ecore_rdma_create_qp_in_params *params)
2595 {
2596 	qlnx_host_t	*ha;
2597 
2598 	ha = dev->ha;
2599 
2600 	QL_DPRINT12(ha, "enter\n");
2601 
2602 	/* QP handle to be written in an async event */
2603 	params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
2604 	params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
2605 
2606 	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
2607 	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
2608 	params->pd = pd->pd_id;
2609 	params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
2610 	params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
2611 	params->stats_queue = 0;
2612 
2613 	params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
2614 
2615 	if (qp->srq) {
2616 		/* QP is associated with SRQ instead of RQ */
2617 		params->srq_id = qp->srq->srq_id;
2618 		params->use_srq = true;
2619 		QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
2620 			params->srq_id, params->use_srq);
2621 		return;
2622 	}
2623 
2624 	params->srq_id = 0;
2625 	params->use_srq = false;
2626 
2627 	QL_DPRINT12(ha, "exit\n");
2628 	return;
2629 }
2630 
2631 static inline void
2632 qlnxr_qp_user_print( struct qlnxr_dev *dev,
2633 	struct qlnxr_qp *qp)
2634 {
2635 	QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, "
2636 		"rq_addr=0x%llx, rq_len=%zd\n",
2637 		qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
2638 		qp->urq.buf_len);
2639 	return;
2640 }
2641 
2642 static int
2643 qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id)
2644 {
2645 	u32		newid;
2646 	int		rc;
2647 	qlnx_host_t	*ha;
2648 
2649 	ha = dev->ha;
2650 
2651 	QL_DPRINT12(ha, "enter\n");
2652 
2653 	if (!QLNX_IS_IWARP(dev))
2654 		return 0;
2655 
2656 	do {
2657 		if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) {
2658 			QL_DPRINT11(ha, "idr_pre_get failed\n");
2659 			return -ENOMEM;
2660 		}
2661 
2662 		mtx_lock(&dev->idr_lock);
2663 
2664 		rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid);
2665 
2666 		mtx_unlock(&dev->idr_lock);
2667 
2668 	} while (rc == -EAGAIN);
2669 
2670 	QL_DPRINT12(ha, "exit [%d]\n", rc);
2671 
2672 	return rc;
2673 }
2674 
2675 static void
2676 qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id)
2677 {
2678 	qlnx_host_t	*ha;
2679 
2680 	ha = dev->ha;
2681 
2682 	QL_DPRINT12(ha, "enter\n");
2683 
2684 	if (!QLNX_IS_IWARP(dev))
2685 		return;
2686 
2687 	mtx_lock(&dev->idr_lock);
2688 	idr_remove(&dev->qpidr, id);
2689 	mtx_unlock(&dev->idr_lock);
2690 
2691 	QL_DPRINT12(ha, "exit \n");
2692 
2693 	return;
2694 }
2695 
2696 static inline void
2697 qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev,
2698 	struct qlnxr_qp *qp,
2699 	struct ecore_rdma_create_qp_out_params *out_params)
2700 {
2701 	qlnx_host_t	*ha;
2702 
2703 	ha = dev->ha;
2704 
2705 	QL_DPRINT12(ha, "enter\n");
2706 
2707 	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
2708 	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
2709 
2710 	qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
2711 			   &qp->usq.pbl_info);
2712 
2713 	if (qp->srq) {
2714 		QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq);
2715 		return;
2716 	}
2717 
2718 	qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
2719 	qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
2720 
2721 	qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
2722 			   &qp->urq.pbl_info);
2723 
2724 	QL_DPRINT12(ha, "exit\n");
2725 	return;
2726 }
2727 
2728 static int
2729 qlnxr_create_user_qp(struct qlnxr_dev *dev,
2730 	struct qlnxr_qp *qp,
2731 	struct ib_pd *ibpd,
2732 	struct ib_udata *udata,
2733 	struct ib_qp_init_attr *attrs)
2734 {
2735 	struct ecore_rdma_destroy_qp_out_params d_out_params;
2736 	struct ecore_rdma_create_qp_in_params in_params;
2737 	struct ecore_rdma_create_qp_out_params out_params;
2738 	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
2739 	struct ib_ucontext *ib_ctx = NULL;
2740 	struct qlnxr_ucontext *ctx = NULL;
2741 	struct qlnxr_create_qp_ureq ureq;
2742 	int alloc_and_init = QLNX_IS_ROCE(dev);
2743 	int rc = -EINVAL;
2744 	qlnx_host_t	*ha;
2745 
2746 	ha = dev->ha;
2747 
2748 	QL_DPRINT12(ha, "enter\n");
2749 
2750 	ib_ctx = ibpd->uobject->context;
2751 	ctx = get_qlnxr_ucontext(ib_ctx);
2752 
2753 	memset(&ureq, 0, sizeof(ureq));
2754 	rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
2755 
2756 	if (rc) {
2757 		QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc);
2758 		return rc;
2759 	}
2760 
2761 	/* SQ - read access only (0), dma sync not required (0) */
2762 	rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
2763 				  ureq.sq_len, 0, 0,
2764 				  alloc_and_init);
2765 	if (rc) {
2766 		QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2767 		return rc;
2768 	}
2769 
2770 	if (!qp->srq) {
2771 		/* RQ - read access only (0), dma sync not required (0) */
2772 		rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
2773 					  ureq.rq_len, 0, 0,
2774 					  alloc_and_init);
2775 
2776 		if (rc) {
2777 			QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);
2778 			return rc;
2779 		}
2780 	}
2781 
2782 	memset(&in_params, 0, sizeof(in_params));
2783 	qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
2784 	in_params.qp_handle_lo = ureq.qp_handle_lo;
2785 	in_params.qp_handle_hi = ureq.qp_handle_hi;
2786 	in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
2787 	in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
2788 
2789 	if (!qp->srq) {
2790 		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
2791 		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
2792 	}
2793 
2794 	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params);
2795 
2796 	if (!qp->ecore_qp) {
2797 		rc = -ENOMEM;
2798 		QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
2799 		goto err1;
2800 	}
2801 
2802 	if (QLNX_IS_IWARP(dev))
2803 		qlnxr_iwarp_populate_user_qp(dev, qp, &out_params);
2804 
2805 	qp->qp_id = out_params.qp_id;
2806 	qp->icid = out_params.icid;
2807 
2808 	rc = qlnxr_copy_qp_uresp(dev, qp, udata);
2809 
2810 	if (rc) {
2811 		QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n");
2812 		goto err;
2813 	}
2814 
2815 	qlnxr_qp_user_print(dev, qp);
2816 
2817 	QL_DPRINT12(ha, "exit\n");
2818 	return 0;
2819 err:
2820 	rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
2821 
2822 	if (rc)
2823 		QL_DPRINT12(ha, "fatal fault\n");
2824 
2825 err1:
2826 	qlnxr_cleanup_user(dev, qp);
2827 
2828 	QL_DPRINT12(ha, "exit[%d]\n", rc);
2829 	return rc;
2830 }
2831 
2832 static void
2833 qlnxr_set_roce_db_info(struct qlnxr_dev *dev,
2834 	struct qlnxr_qp *qp)
2835 {
2836 	qlnx_host_t	*ha;
2837 
2838 	ha = dev->ha;
2839 
2840 	QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2841 
2842 	qp->sq.db = dev->db_addr +
2843 		DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2844 	qp->sq.db_data.data.icid = qp->icid + 1;
2845 
2846 	if (!qp->srq) {
2847 		qp->rq.db = dev->db_addr +
2848 			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
2849 		qp->rq.db_data.data.icid = qp->icid;
2850 	}
2851 
2852 	QL_DPRINT12(ha, "exit\n");
2853 	return;
2854 }
2855 
2856 static void
2857 qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev,
2858 	struct qlnxr_qp *qp)
2859 
2860 {
2861 	qlnx_host_t	*ha;
2862 
2863 	ha = dev->ha;
2864 
2865 	QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);
2866 
2867 	qp->sq.db = dev->db_addr +
2868 		DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
2869 	qp->sq.db_data.data.icid = qp->icid;
2870 
2871 	if (!qp->srq) {
2872 		qp->rq.db = dev->db_addr +
2873 			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2874 		qp->rq.db_data.data.icid = qp->icid;
2875 
2876 		qp->rq.iwarp_db2 = dev->db_addr +
2877 			DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2878 		qp->rq.iwarp_db2_data.data.icid = qp->icid;
2879 		qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2880 	}
2881 
2882 	QL_DPRINT12(ha,
2883 		"qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n"
2884 		"\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n"
2885 		"\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x"
2886 		" qp->rq.iwarp_db2.data.prod_val =0x%x\n",
2887 		qp->sq.db, qp->sq.db_data.data.icid,
2888 		qp->rq.db, qp->rq.db_data.data.icid,
2889 		qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid,
2890 		qp->rq.iwarp_db2_data.data.value);
2891 
2892 	QL_DPRINT12(ha, "exit\n");
2893 	return;
2894 }
2895 
2896 static int
2897 qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev,
2898 	struct qlnxr_qp *qp,
2899 	struct ecore_rdma_create_qp_in_params *in_params,
2900 	u32 n_sq_elems,
2901 	u32 n_rq_elems)
2902 {
2903 	struct ecore_rdma_create_qp_out_params out_params;
2904 	int		rc;
2905 	qlnx_host_t	*ha;
2906 
2907 	ha = dev->ha;
2908 
2909 	QL_DPRINT12(ha, "enter\n");
2910 
2911         rc = ecore_chain_alloc(
2912                 dev->cdev,
2913                 ECORE_CHAIN_USE_TO_PRODUCE,
2914                 ECORE_CHAIN_MODE_PBL,
2915                 ECORE_CHAIN_CNT_TYPE_U32,
2916                 n_sq_elems,
2917                 QLNXR_SQE_ELEMENT_SIZE,
2918                 &qp->sq.pbl,
2919                 NULL);
2920 
2921 	if (rc) {
2922 		QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc);
2923 		return rc;
2924 	}
2925 
2926 	in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl);
2927 	in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl);
2928 
2929 	if (!qp->srq) {
2930                 rc = ecore_chain_alloc(
2931                         dev->cdev,
2932                         ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
2933                         ECORE_CHAIN_MODE_PBL,
2934                         ECORE_CHAIN_CNT_TYPE_U32,
2935                         n_rq_elems,
2936                         QLNXR_RQE_ELEMENT_SIZE,
2937                         &qp->rq.pbl,
2938                         NULL);
2939 
2940 		if (rc) {
2941 			QL_DPRINT11(ha,
2942 				"ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc);
2943 			return rc;
2944 		}
2945 
2946 		in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl);
2947 		in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl);
2948 	}
2949 
2950 	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
2951 
2952 	if (!qp->ecore_qp) {
2953 		QL_DPRINT11(ha, "qp->ecore_qp == NULL\n");
2954 		return -EINVAL;
2955 	}
2956 
2957 	qp->qp_id = out_params.qp_id;
2958 	qp->icid = out_params.icid;
2959 
2960 	qlnxr_set_roce_db_info(dev, qp);
2961 
2962 	QL_DPRINT12(ha, "exit\n");
2963 	return 0;
2964 }
2965 
2966 static int
2967 qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev,
2968 	struct qlnxr_qp *qp,
2969 	struct ecore_rdma_create_qp_in_params *in_params,
2970 	u32 n_sq_elems,
2971 	u32 n_rq_elems)
2972 {
2973 	struct ecore_rdma_destroy_qp_out_params d_out_params;
2974 	struct ecore_rdma_create_qp_out_params out_params;
2975 	struct ecore_chain_ext_pbl ext_pbl;
2976 	int rc;
2977 	qlnx_host_t	*ha;
2978 
2979 	ha = dev->ha;
2980 
2981 	QL_DPRINT12(ha, "enter\n");
2982 
2983 	in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems,
2984 						     QLNXR_SQE_ELEMENT_SIZE,
2985 						     ECORE_CHAIN_MODE_PBL);
2986 	in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems,
2987 						     QLNXR_RQE_ELEMENT_SIZE,
2988 						     ECORE_CHAIN_MODE_PBL);
2989 
2990 	QL_DPRINT12(ha, "n_sq_elems = 0x%x"
2991 		" n_rq_elems = 0x%x in_params\n"
2992 		"\t\t\tqp_handle_lo\t\t= 0x%08x\n"
2993 		"\t\t\tqp_handle_hi\t\t= 0x%08x\n"
2994 		"\t\t\tqp_handle_async_lo\t\t= 0x%08x\n"
2995 		"\t\t\tqp_handle_async_hi\t\t= 0x%08x\n"
2996 		"\t\t\tuse_srq\t\t\t= 0x%x\n"
2997 		"\t\t\tsignal_all\t\t= 0x%x\n"
2998 		"\t\t\tfmr_and_reserved_lkey\t= 0x%x\n"
2999 		"\t\t\tpd\t\t\t= 0x%x\n"
3000 		"\t\t\tdpi\t\t\t= 0x%x\n"
3001 		"\t\t\tsq_cq_id\t\t\t= 0x%x\n"
3002 		"\t\t\tsq_num_pages\t\t= 0x%x\n"
3003 		"\t\t\tsq_pbl_ptr\t\t= %p\n"
3004 		"\t\t\tmax_sq_sges\t\t= 0x%x\n"
3005 		"\t\t\trq_cq_id\t\t\t= 0x%x\n"
3006 		"\t\t\trq_num_pages\t\t= 0x%x\n"
3007 		"\t\t\trq_pbl_ptr\t\t= %p\n"
3008 		"\t\t\tsrq_id\t\t\t= 0x%x\n"
3009 		"\t\t\tstats_queue\t\t= 0x%x\n",
3010 		n_sq_elems, n_rq_elems,
3011 		in_params->qp_handle_lo,
3012 		in_params->qp_handle_hi,
3013 		in_params->qp_handle_async_lo,
3014 		in_params->qp_handle_async_hi,
3015 		in_params->use_srq,
3016 		in_params->signal_all,
3017 		in_params->fmr_and_reserved_lkey,
3018 		in_params->pd,
3019 		in_params->dpi,
3020 		in_params->sq_cq_id,
3021 		in_params->sq_num_pages,
3022 		(void *)in_params->sq_pbl_ptr,
3023 		in_params->max_sq_sges,
3024 		in_params->rq_cq_id,
3025 		in_params->rq_num_pages,
3026 		(void *)in_params->rq_pbl_ptr,
3027 		in_params->srq_id,
3028 		in_params->stats_queue );
3029 
3030 	memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params));
3031 	memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl));
3032 
3033 	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);
3034 
3035 	if (!qp->ecore_qp) {
3036 		QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n");
3037 		return -EINVAL;
3038 	}
3039 
3040 	/* Now we allocate the chain */
3041 	ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
3042 	ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
3043 
3044 	QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3045 		"ext_pbl.p_pbl_phys = %p\n",
3046 		ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3047 
3048         rc = ecore_chain_alloc(
3049                 dev->cdev,
3050                 ECORE_CHAIN_USE_TO_PRODUCE,
3051                 ECORE_CHAIN_MODE_PBL,
3052                 ECORE_CHAIN_CNT_TYPE_U32,
3053                 n_sq_elems,
3054                 QLNXR_SQE_ELEMENT_SIZE,
3055                 &qp->sq.pbl,
3056                 &ext_pbl);
3057 
3058 	if (rc) {
3059 		QL_DPRINT11(ha,
3060 			"ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc);
3061 		goto err;
3062 	}
3063 
3064 	ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
3065 	ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
3066 
3067 	QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p "
3068 		"ext_pbl.p_pbl_phys = %p\n",
3069 		ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys);
3070 
3071 	if (!qp->srq) {
3072                 rc = ecore_chain_alloc(
3073                         dev->cdev,
3074                         ECORE_CHAIN_USE_TO_CONSUME_PRODUCE,
3075                         ECORE_CHAIN_MODE_PBL,
3076                         ECORE_CHAIN_CNT_TYPE_U32,
3077                         n_rq_elems,
3078                         QLNXR_RQE_ELEMENT_SIZE,
3079                         &qp->rq.pbl,
3080                         &ext_pbl);
3081 
3082 		if (rc) {
3083 			QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl"
3084 				" failed rc = %d\n", rc);
3085 			goto err;
3086 		}
3087 	}
3088 
3089 	QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n",
3090 		out_params.qp_id, out_params.icid);
3091 
3092 	qp->qp_id = out_params.qp_id;
3093 	qp->icid = out_params.icid;
3094 
3095 	qlnxr_set_iwarp_db_info(dev, qp);
3096 
3097 	QL_DPRINT12(ha, "exit\n");
3098 	return 0;
3099 
3100 err:
3101 	ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);
3102 
3103 	QL_DPRINT12(ha, "exit rc = %d\n", rc);
3104 	return rc;
3105 }
3106 
3107 static int
3108 qlnxr_create_kernel_qp(struct qlnxr_dev *dev,
3109 	struct qlnxr_qp *qp,
3110 	struct ib_pd *ibpd,
3111 	struct ib_qp_init_attr *attrs)
3112 {
3113 	struct ecore_rdma_create_qp_in_params in_params;
3114 	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3115 	int rc = -EINVAL;
3116 	u32 n_rq_elems;
3117 	u32 n_sq_elems;
3118 	u32 n_sq_entries;
3119 	struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3120 	qlnx_host_t	*ha;
3121 
3122 	ha = dev->ha;
3123 
3124 	QL_DPRINT12(ha, "enter\n");
3125 
3126 	memset(&in_params, 0, sizeof(in_params));
3127 
3128 	/* A single work request may take up to MAX_SQ_WQE_SIZE elements in
3129 	 * the ring. The ring should allow at least a single WR, even if the
3130 	 * user requested none, due to allocation issues.
3131 	 * We should add an extra WR since the prod and cons indices of
3132 	 * wqe_wr_id are managed in such a way that the WQ is considered full
3133 	 * when (prod+1)%max_wr==cons. We currently don't do that because we
3134 	 * double the number of entries due an iSER issue that pushes far more
3135 	 * WRs than indicated. If we decline its ib_post_send() then we get
3136 	 * error prints in the dmesg we'd like to avoid.
3137 	 */
3138 	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
3139 			      qattr->max_wqe);
3140 
3141 	qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),
3142 			GFP_KERNEL);
3143 	if (!qp->wqe_wr_id) {
3144 		QL_DPRINT11(ha, "failed SQ shadow memory allocation\n");
3145 		return -ENOMEM;
3146 	}
3147 
3148 	/* QP handle to be written in CQE */
3149 	in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp);
3150 	in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp);
3151 
3152 	/* A single work request may take up to MAX_RQ_WQE_SIZE elements in
3153 	 * the ring. There ring should allow at least a single WR, even if the
3154 	 * user requested none, due to allocation issues.
3155 	 */
3156 	qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1);
3157 
3158 	/* Allocate driver internal RQ array */
3159 	if (!qp->srq) {
3160 		qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),
3161 					GFP_KERNEL);
3162 		if (!qp->rqe_wr_id) {
3163 			QL_DPRINT11(ha, "failed RQ shadow memory allocation\n");
3164 			kfree(qp->wqe_wr_id);
3165 			return -ENOMEM;
3166 		}
3167 	}
3168 
3169 	//qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
3170 
3171         in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
3172         in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
3173 
3174         in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
3175         in_params.fmr_and_reserved_lkey = true;
3176         in_params.pd = pd->pd_id;
3177         in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
3178         in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;
3179         in_params.stats_queue = 0;
3180 
3181         in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;
3182 
3183         if (qp->srq) {
3184                 /* QP is associated with SRQ instead of RQ */
3185                 in_params.srq_id = qp->srq->srq_id;
3186                 in_params.use_srq = true;
3187                 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",
3188                         in_params.srq_id, in_params.use_srq);
3189         } else {
3190         	in_params.srq_id = 0;
3191 		in_params.use_srq = false;
3192 	}
3193 
3194 	n_sq_entries = attrs->cap.max_send_wr;
3195 	n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe);
3196 	n_sq_entries = max_t(u32, n_sq_entries, 1);
3197 	n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
3198 
3199 	n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE;
3200 
3201 	if (QLNX_IS_ROCE(dev)) {
3202 		rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params,
3203 						n_sq_elems, n_rq_elems);
3204 	} else {
3205 		rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params,
3206 						 n_sq_elems, n_rq_elems);
3207 	}
3208 
3209 	if (rc)
3210 		qlnxr_cleanup_kernel(dev, qp);
3211 
3212 	QL_DPRINT12(ha, "exit [%d]\n", rc);
3213 	return rc;
3214 }
3215 
3216 struct ib_qp *
3217 qlnxr_create_qp(struct ib_pd *ibpd,
3218 		struct ib_qp_init_attr *attrs,
3219 		struct ib_udata *udata)
3220 {
3221 	struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device);
3222 	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
3223 	struct qlnxr_qp *qp;
3224 	int rc = 0;
3225 	qlnx_host_t	*ha;
3226 
3227 	ha = dev->ha;
3228 
3229 	QL_DPRINT12(ha, "enter\n");
3230 
3231 	rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata);
3232 	if (rc) {
3233 		QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc);
3234 		return ERR_PTR(rc);
3235 	}
3236 
3237 	QL_DPRINT12(ha, "called from %s, event_handle=%p,"
3238 		" eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
3239 		(udata ? "user library" : "kernel"),
3240 		attrs->event_handler, pd,
3241 		get_qlnxr_cq(attrs->send_cq),
3242 		get_qlnxr_cq(attrs->send_cq)->icid,
3243 		get_qlnxr_cq(attrs->recv_cq),
3244 		get_qlnxr_cq(attrs->recv_cq)->icid);
3245 
3246 	qp = qlnx_zalloc(sizeof(struct qlnxr_qp));
3247 
3248 	if (!qp) {
3249 		QL_DPRINT11(ha, "kzalloc(qp) failed\n");
3250 		return ERR_PTR(-ENOMEM);
3251 	}
3252 
3253 	qlnxr_set_common_qp_params(dev, qp, pd, attrs);
3254 
3255 	if (attrs->qp_type == IB_QPT_GSI) {
3256 		QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n");
3257 		return qlnxr_create_gsi_qp(dev, attrs, qp);
3258 	}
3259 
3260 	if (udata) {
3261 		rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs);
3262 
3263 		if (rc) {
3264 			QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n");
3265 			goto err;
3266 		}
3267 	} else {
3268 		rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs);
3269 
3270 		if (rc) {
3271 			QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n");
3272 			goto err;
3273 		}
3274 	}
3275 
3276 	qp->ibqp.qp_num = qp->qp_id;
3277 
3278 	rc = qlnxr_idr_add(dev, qp, qp->qp_id);
3279 
3280 	if (rc) {
3281 		QL_DPRINT11(ha, "qlnxr_idr_add failed\n");
3282 		goto err;
3283 	}
3284 
3285 	QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp);
3286 
3287 	return &qp->ibqp;
3288 err:
3289 	kfree(qp);
3290 
3291 	QL_DPRINT12(ha, "failed exit\n");
3292 	return ERR_PTR(-EFAULT);
3293 }
3294 
3295 static enum ib_qp_state
3296 qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state)
3297 {
3298 	enum ib_qp_state state = IB_QPS_ERR;
3299 
3300 	switch (qp_state) {
3301 	case ECORE_ROCE_QP_STATE_RESET:
3302 		state = IB_QPS_RESET;
3303 		break;
3304 
3305 	case ECORE_ROCE_QP_STATE_INIT:
3306 		state = IB_QPS_INIT;
3307 		break;
3308 
3309 	case ECORE_ROCE_QP_STATE_RTR:
3310 		state = IB_QPS_RTR;
3311 		break;
3312 
3313 	case ECORE_ROCE_QP_STATE_RTS:
3314 		state = IB_QPS_RTS;
3315 		break;
3316 
3317 	case ECORE_ROCE_QP_STATE_SQD:
3318 		state = IB_QPS_SQD;
3319 		break;
3320 
3321 	case ECORE_ROCE_QP_STATE_ERR:
3322 		state = IB_QPS_ERR;
3323 		break;
3324 
3325 	case ECORE_ROCE_QP_STATE_SQE:
3326 		state = IB_QPS_SQE;
3327 		break;
3328 	}
3329 	return state;
3330 }
3331 
3332 static enum ecore_roce_qp_state
3333 qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state)
3334 {
3335 	enum ecore_roce_qp_state ecore_qp_state;
3336 
3337 	ecore_qp_state = ECORE_ROCE_QP_STATE_ERR;
3338 
3339 	switch (qp_state) {
3340 	case IB_QPS_RESET:
3341 		ecore_qp_state =  ECORE_ROCE_QP_STATE_RESET;
3342 		break;
3343 
3344 	case IB_QPS_INIT:
3345 		ecore_qp_state =  ECORE_ROCE_QP_STATE_INIT;
3346 		break;
3347 
3348 	case IB_QPS_RTR:
3349 		ecore_qp_state =  ECORE_ROCE_QP_STATE_RTR;
3350 		break;
3351 
3352 	case IB_QPS_RTS:
3353 		ecore_qp_state =  ECORE_ROCE_QP_STATE_RTS;
3354 		break;
3355 
3356 	case IB_QPS_SQD:
3357 		ecore_qp_state =  ECORE_ROCE_QP_STATE_SQD;
3358 		break;
3359 
3360 	case IB_QPS_ERR:
3361 		ecore_qp_state =  ECORE_ROCE_QP_STATE_ERR;
3362 		break;
3363 
3364 	default:
3365 		ecore_qp_state =  ECORE_ROCE_QP_STATE_ERR;
3366 		break;
3367 	}
3368 
3369 	return (ecore_qp_state);
3370 }
3371 
3372 static void
3373 qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph)
3374 {
3375 	ecore_chain_reset(&qph->pbl);
3376 	qph->prod = qph->cons = 0;
3377 	qph->wqe_cons = 0;
3378 	qph->db_data.data.value = cpu_to_le16(0);
3379 
3380 	return;
3381 }
3382 
3383 static int
3384 qlnxr_update_qp_state(struct qlnxr_dev *dev,
3385 	struct qlnxr_qp *qp,
3386 	enum ecore_roce_qp_state new_state)
3387 {
3388 	int		status = 0;
3389 	uint32_t	reg_addr;
3390 	struct ecore_dev *cdev;
3391 	qlnx_host_t	*ha;
3392 
3393 	ha = dev->ha;
3394 	cdev = &ha->cdev;
3395 
3396 	QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n",
3397 		qp, new_state, qp->state);
3398 
3399 	if (new_state == qp->state) {
3400 		return 0;
3401 	}
3402 
3403 	switch (qp->state) {
3404 	case ECORE_ROCE_QP_STATE_RESET:
3405 		switch (new_state) {
3406 		case ECORE_ROCE_QP_STATE_INIT:
3407 			qp->prev_wqe_size = 0;
3408 			qlnxr_reset_qp_hwq_info(&qp->sq);
3409 			if (!(qp->srq))
3410 				qlnxr_reset_qp_hwq_info(&qp->rq);
3411 			break;
3412 		default:
3413 			status = -EINVAL;
3414 			break;
3415 		};
3416 		break;
3417 	case ECORE_ROCE_QP_STATE_INIT:
3418 		/* INIT->XXX */
3419 		switch (new_state) {
3420 		case ECORE_ROCE_QP_STATE_RTR:
3421 		/* Update doorbell (in case post_recv was done before move to RTR) */
3422 			if (qp->srq)
3423 				break;
3424 			wmb();
3425 			//writel(qp->rq.db_data.raw, qp->rq.db);
3426 			//if (QLNX_IS_IWARP(dev))
3427 			//	writel(qp->rq.iwarp_db2_data.raw,
3428 			//	       qp->rq.iwarp_db2);
3429 
3430 			reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
3431 					(uint8_t *)cdev->doorbells);
3432 
3433 			bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
3434 			bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
3435 
3436 			if (QLNX_IS_IWARP(dev)) {
3437 				reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
3438 					(uint8_t *)cdev->doorbells);
3439 				bus_write_4(ha->pci_dbells, reg_addr,\
3440 					qp->rq.iwarp_db2_data.raw);
3441 				bus_barrier(ha->pci_dbells,  0, 0,\
3442 					BUS_SPACE_BARRIER_READ);
3443 			}
3444 
3445 
3446 			mmiowb();
3447 			break;
3448 		case ECORE_ROCE_QP_STATE_ERR:
3449 			/* TBD:flush qps... */
3450 			break;
3451 		default:
3452 			/* invalid state change. */
3453 			status = -EINVAL;
3454 			break;
3455 		};
3456 		break;
3457 	case ECORE_ROCE_QP_STATE_RTR:
3458 		/* RTR->XXX */
3459 		switch (new_state) {
3460 		case ECORE_ROCE_QP_STATE_RTS:
3461 			break;
3462 		case ECORE_ROCE_QP_STATE_ERR:
3463 			break;
3464 		default:
3465 			/* invalid state change. */
3466 			status = -EINVAL;
3467 			break;
3468 		};
3469 		break;
3470 	case ECORE_ROCE_QP_STATE_RTS:
3471 		/* RTS->XXX */
3472 		switch (new_state) {
3473 		case ECORE_ROCE_QP_STATE_SQD:
3474 			break;
3475 		case ECORE_ROCE_QP_STATE_ERR:
3476 			break;
3477 		default:
3478 			/* invalid state change. */
3479 			status = -EINVAL;
3480 			break;
3481 		};
3482 		break;
3483 	case ECORE_ROCE_QP_STATE_SQD:
3484 		/* SQD->XXX */
3485 		switch (new_state) {
3486 		case ECORE_ROCE_QP_STATE_RTS:
3487 		case ECORE_ROCE_QP_STATE_ERR:
3488 			break;
3489 		default:
3490 			/* invalid state change. */
3491 			status = -EINVAL;
3492 			break;
3493 		};
3494 		break;
3495 	case ECORE_ROCE_QP_STATE_ERR:
3496 		/* ERR->XXX */
3497 		switch (new_state) {
3498 		case ECORE_ROCE_QP_STATE_RESET:
3499 			if ((qp->rq.prod != qp->rq.cons) ||
3500 			    (qp->sq.prod != qp->sq.cons)) {
3501 				QL_DPRINT11(ha,
3502 					"Error->Reset with rq/sq "
3503 					"not empty rq.prod=0x%x rq.cons=0x%x"
3504 					" sq.prod=0x%x sq.cons=0x%x\n",
3505 					qp->rq.prod, qp->rq.cons,
3506 					qp->sq.prod, qp->sq.cons);
3507 				status = -EINVAL;
3508 			}
3509 			break;
3510 		default:
3511 			status = -EINVAL;
3512 			break;
3513 		};
3514 		break;
3515 	default:
3516 		status = -EINVAL;
3517 		break;
3518 	};
3519 
3520 	QL_DPRINT12(ha, "exit\n");
3521 	return status;
3522 }
3523 
3524 int
3525 qlnxr_modify_qp(struct ib_qp	*ibqp,
3526 	struct ib_qp_attr	*attr,
3527 	int			attr_mask,
3528 	struct ib_udata		*udata)
3529 {
3530 	int rc = 0;
3531 	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3532 	struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev);
3533 	struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
3534 	enum ib_qp_state old_qp_state, new_qp_state;
3535 	struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);
3536 	qlnx_host_t	*ha;
3537 
3538 	ha = dev->ha;
3539 
3540 	QL_DPRINT12(ha,
3541 		"enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n",
3542 		qp, attr_mask, attr->qp_state, udata);
3543 
3544 	old_qp_state = qlnxr_get_ibqp_state(qp->state);
3545 	if (attr_mask & IB_QP_STATE)
3546 		new_qp_state = attr->qp_state;
3547 	else
3548 		new_qp_state = old_qp_state;
3549 
3550 	if (QLNX_IS_ROCE(dev)) {
3551 #if __FreeBSD_version >= 1100000
3552 		if (!ib_modify_qp_is_ok(old_qp_state,
3553 					new_qp_state,
3554 					ibqp->qp_type,
3555 					attr_mask,
3556 					IB_LINK_LAYER_ETHERNET)) {
3557 			QL_DPRINT12(ha,
3558 				"invalid attribute mask=0x%x"
3559 				" specified for qpn=0x%x of type=0x%x \n"
3560 				" old_qp_state=0x%x, new_qp_state=0x%x\n",
3561 				attr_mask, qp->qp_id, ibqp->qp_type,
3562 				old_qp_state, new_qp_state);
3563 			rc = -EINVAL;
3564 			goto err;
3565 		}
3566 #else
3567 		if (!ib_modify_qp_is_ok(old_qp_state,
3568 					new_qp_state,
3569 					ibqp->qp_type,
3570 					attr_mask )) {
3571 			QL_DPRINT12(ha,
3572 				"invalid attribute mask=0x%x"
3573 				" specified for qpn=0x%x of type=0x%x \n"
3574 				" old_qp_state=0x%x, new_qp_state=0x%x\n",
3575 				attr_mask, qp->qp_id, ibqp->qp_type,
3576 				old_qp_state, new_qp_state);
3577 			rc = -EINVAL;
3578 			goto err;
3579 		}
3580 
3581 #endif /* #if __FreeBSD_version >= 1100000 */
3582 	}
3583 	/* translate the masks... */
3584 	if (attr_mask & IB_QP_STATE) {
3585 		SET_FIELD(qp_params.modify_flags,
3586 			  ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
3587 		qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state);
3588 	}
3589 
3590 	// TBD consider changing ecore to be a flag as well...
3591 	if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
3592 		qp_params.sqd_async = true;
3593 
3594 	if (attr_mask & IB_QP_PKEY_INDEX) {
3595 		SET_FIELD(qp_params.modify_flags,
3596 			  ECORE_ROCE_MODIFY_QP_VALID_PKEY,
3597 			  1);
3598 		if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) {
3599 			rc = -EINVAL;
3600 			goto err;
3601 		}
3602 
3603 		qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT;
3604 	}
3605 
3606 	if (attr_mask & IB_QP_QKEY) {
3607 		qp->qkey = attr->qkey;
3608 	}
3609 
3610 	/* tbd consider splitting in ecore.. */
3611 	if (attr_mask & IB_QP_ACCESS_FLAGS) {
3612 		SET_FIELD(qp_params.modify_flags,
3613 			  ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
3614 		qp_params.incoming_rdma_read_en =
3615 			attr->qp_access_flags & IB_ACCESS_REMOTE_READ;
3616 		qp_params.incoming_rdma_write_en =
3617 			attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE;
3618 		qp_params.incoming_atomic_en =
3619 			attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC;
3620 	}
3621 
3622 	if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
3623 		if (attr_mask & IB_QP_PATH_MTU) {
3624 			if (attr->path_mtu < IB_MTU_256 ||
3625 			    attr->path_mtu > IB_MTU_4096) {
3626 				QL_DPRINT12(ha,
3627 					"Only MTU sizes of 256, 512, 1024,"
3628 					" 2048 and 4096 are supported "
3629 					" attr->path_mtu = [%d]\n",
3630 					attr->path_mtu);
3631 
3632 				rc = -EINVAL;
3633 				goto err;
3634 			}
3635 			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
3636 				      ib_mtu_enum_to_int(
3637 						iboe_get_mtu(dev->ha->ifp->if_mtu)));
3638 		}
3639 
3640 		if (qp->mtu == 0) {
3641 			qp->mtu = ib_mtu_enum_to_int(
3642 					iboe_get_mtu(dev->ha->ifp->if_mtu));
3643 			QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n",
3644 				qp->mtu);
3645 		}
3646 
3647 		SET_FIELD(qp_params.modify_flags,
3648 			  ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR,
3649 			  1);
3650 
3651 		qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
3652 		qp_params.flow_label = attr->ah_attr.grh.flow_label;
3653 		qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
3654 
3655 		qp->sgid_idx = attr->ah_attr.grh.sgid_index;
3656 
3657 		get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params);
3658 
3659 		rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr);
3660 		if (rc)
3661 			return rc;
3662 
3663 		qp_params.use_local_mac = true;
3664 		memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
3665 
3666 		QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n",
3667 		       qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
3668 		       qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
3669 		QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n",
3670 		       qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
3671 		       qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
3672 		QL_DPRINT12(ha,
3673 			"remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n",
3674 			qp_params.remote_mac_addr[0],
3675 			qp_params.remote_mac_addr[1],
3676 			qp_params.remote_mac_addr[2],
3677 			qp_params.remote_mac_addr[3],
3678 			qp_params.remote_mac_addr[4],
3679 			qp_params.remote_mac_addr[5]);
3680 
3681 		qp_params.mtu = qp->mtu;
3682 	}
3683 
3684 	if (qp_params.mtu == 0) {
3685 		/* stay with current MTU */
3686 		if (qp->mtu) {
3687 			qp_params.mtu = qp->mtu;
3688 		} else {
3689 			qp_params.mtu = ib_mtu_enum_to_int(
3690 						iboe_get_mtu(dev->ha->ifp->if_mtu));
3691 		}
3692 	}
3693 
3694 	if (attr_mask & IB_QP_TIMEOUT) {
3695 		SET_FIELD(qp_params.modify_flags, \
3696 			ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
3697 
3698 		qp_params.ack_timeout = attr->timeout;
3699 		if (attr->timeout) {
3700 			u32 temp;
3701 
3702 			/* 12.7.34 LOCAL ACK TIMEOUT
3703 			 * Value representing the transport (ACK) timeout for
3704 			 * use by the remote, expressed as (4.096 μS*2Local ACK
3705 			 * Timeout)
3706 			 */
3707 			/* We use 1UL since the temporal value may be  overflow
3708 			 * 32 bits
3709 			 */
3710 			temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
3711 			qp_params.ack_timeout = temp; /* FW requires [msec] */
3712 		}
3713 		else
3714 			qp_params.ack_timeout = 0; /* infinite */
3715 	}
3716 	if (attr_mask & IB_QP_RETRY_CNT) {
3717 		SET_FIELD(qp_params.modify_flags,\
3718 			 ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
3719 		qp_params.retry_cnt = attr->retry_cnt;
3720 	}
3721 
3722 	if (attr_mask & IB_QP_RNR_RETRY) {
3723 		SET_FIELD(qp_params.modify_flags,
3724 			  ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT,
3725 			  1);
3726 		qp_params.rnr_retry_cnt = attr->rnr_retry;
3727 	}
3728 
3729 	if (attr_mask & IB_QP_RQ_PSN) {
3730 		SET_FIELD(qp_params.modify_flags,
3731 			  ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN,
3732 			  1);
3733 		qp_params.rq_psn = attr->rq_psn;
3734 		qp->rq_psn = attr->rq_psn;
3735 	}
3736 
3737 	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
3738 		if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) {
3739 			rc = -EINVAL;
3740 			QL_DPRINT12(ha,
3741 				"unsupported  max_rd_atomic=%d, supported=%d\n",
3742 				attr->max_rd_atomic,
3743 				qattr->max_qp_req_rd_atomic_resc);
3744 			goto err;
3745 		}
3746 
3747 		SET_FIELD(qp_params.modify_flags,
3748 			  ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ,
3749 			  1);
3750 		qp_params.max_rd_atomic_req = attr->max_rd_atomic;
3751 	}
3752 
3753 	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
3754 		SET_FIELD(qp_params.modify_flags,
3755 			  ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER,
3756 			  1);
3757 		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
3758 	}
3759 
3760 	if (attr_mask & IB_QP_SQ_PSN) {
3761 		SET_FIELD(qp_params.modify_flags,
3762 			  ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN,
3763 			  1);
3764 		qp_params.sq_psn = attr->sq_psn;
3765 		qp->sq_psn = attr->sq_psn;
3766 	}
3767 
3768 	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
3769 		if (attr->max_dest_rd_atomic >
3770 		    qattr->max_qp_resp_rd_atomic_resc) {
3771 			QL_DPRINT12(ha,
3772 				"unsupported max_dest_rd_atomic=%d, "
3773 				"supported=%d\n",
3774 				attr->max_dest_rd_atomic,
3775 				qattr->max_qp_resp_rd_atomic_resc);
3776 
3777 			rc = -EINVAL;
3778 			goto err;
3779 		}
3780 
3781 		SET_FIELD(qp_params.modify_flags,
3782 			  ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP,
3783 			  1);
3784 		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
3785 	}
3786 
3787  	if (attr_mask & IB_QP_DEST_QPN) {
3788 		SET_FIELD(qp_params.modify_flags,
3789 			  ECORE_ROCE_MODIFY_QP_VALID_DEST_QP,
3790 			  1);
3791 
3792 		qp_params.dest_qp = attr->dest_qp_num;
3793 		qp->dest_qp_num = attr->dest_qp_num;
3794 	}
3795 
3796 	/*
3797 	 * Update the QP state before the actual ramrod to prevent a race with
3798 	 * fast path. Modifying the QP state to error will cause the device to
3799 	 * flush the CQEs and while polling the flushed CQEs will considered as
3800 	 * a potential issue if the QP isn't in error state.
3801 	 */
3802 	if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) &&
3803 		(!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR))
3804 		qp->state = ECORE_ROCE_QP_STATE_ERR;
3805 
3806 	if (qp->qp_type != IB_QPT_GSI)
3807 		rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
3808 
3809 	if (attr_mask & IB_QP_STATE) {
3810 		if ((qp->qp_type != IB_QPT_GSI) && (!udata))
3811 			rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state);
3812 		qp->state = qp_params.new_state;
3813 	}
3814 
3815 err:
3816 	QL_DPRINT12(ha, "exit\n");
3817 	return rc;
3818 }
3819 
3820 static int
3821 qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params)
3822 {
3823 	int ib_qp_acc_flags = 0;
3824 
3825 	if (params->incoming_rdma_write_en)
3826 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
3827 	if (params->incoming_rdma_read_en)
3828 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
3829 	if (params->incoming_atomic_en)
3830 		ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
3831 	if (true) /* FIXME -> local write ?? */
3832 		ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
3833 
3834 	return ib_qp_acc_flags;
3835 }
3836 
3837 static enum ib_mtu
3838 qlnxr_mtu_int_to_enum(u16 mtu)
3839 {
3840 	enum ib_mtu ib_mtu_size;
3841 
3842 	switch (mtu) {
3843 	case 256:
3844 		ib_mtu_size = IB_MTU_256;
3845 		break;
3846 
3847 	case 512:
3848 		ib_mtu_size = IB_MTU_512;
3849 		break;
3850 
3851 	case 1024:
3852 		ib_mtu_size = IB_MTU_1024;
3853 		break;
3854 
3855 	case 2048:
3856 		ib_mtu_size = IB_MTU_2048;
3857 		break;
3858 
3859 	case 4096:
3860 		ib_mtu_size = IB_MTU_4096;
3861 		break;
3862 
3863 	default:
3864 		ib_mtu_size = IB_MTU_1024;
3865 		break;
3866 	}
3867 	return (ib_mtu_size);
3868 }
3869 
3870 int
3871 qlnxr_query_qp(struct ib_qp *ibqp,
3872 	struct ib_qp_attr *qp_attr,
3873 	int attr_mask,
3874 	struct ib_qp_init_attr *qp_init_attr)
3875 {
3876 	int rc = 0;
3877 	struct ecore_rdma_query_qp_out_params params;
3878 	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
3879 	struct qlnxr_dev *dev = qp->dev;
3880 	qlnx_host_t	*ha;
3881 
3882 	ha = dev->ha;
3883 
3884 	QL_DPRINT12(ha, "enter\n");
3885 
3886 	memset(&params, 0, sizeof(params));
3887 
3888 	rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, &params);
3889 	if (rc)
3890 		goto err;
3891 
3892 	memset(qp_attr, 0, sizeof(*qp_attr));
3893 	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
3894 
3895 	qp_attr->qp_state = qlnxr_get_ibqp_state(params.state);
3896 	qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state);
3897 
3898 	/* In some cases in iWARP qelr will ask for the state only */
3899 	if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) {
3900 		QL_DPRINT11(ha, "only state requested\n");
3901 		return 0;
3902 	}
3903 
3904 	qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu);
3905 	qp_attr->path_mig_state = IB_MIG_MIGRATED;
3906 	qp_attr->rq_psn = params.rq_psn;
3907 	qp_attr->sq_psn = params.sq_psn;
3908 	qp_attr->dest_qp_num = params.dest_qp;
3909 
3910 	qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(&params);
3911 
3912 	QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x "
3913 		"path_mtu = %d qp_access_flags = 0x%x\n",
3914 		qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu,
3915 		qp_attr->qp_access_flags);
3916 
3917 	qp_attr->cap.max_send_wr = qp->sq.max_wr;
3918 	qp_attr->cap.max_recv_wr = qp->rq.max_wr;
3919 	qp_attr->cap.max_send_sge = qp->sq.max_sges;
3920 	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
3921 	qp_attr->cap.max_inline_data = qp->max_inline_data;
3922 	qp_init_attr->cap = qp_attr->cap;
3923 
3924 	memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],
3925 	       sizeof(qp_attr->ah_attr.grh.dgid.raw));
3926 
3927 	qp_attr->ah_attr.grh.flow_label = params.flow_label;
3928 	qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
3929 	qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
3930 	qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
3931 
3932 	qp_attr->ah_attr.ah_flags = IB_AH_GRH;
3933 	qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */
3934 	qp_attr->ah_attr.sl = 0;/* FIXME -> check this */
3935 	qp_attr->timeout = params.timeout;
3936 	qp_attr->rnr_retry = params.rnr_retry;
3937 	qp_attr->retry_cnt = params.retry_cnt;
3938 	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
3939 	qp_attr->pkey_index = params.pkey_index;
3940 	qp_attr->port_num = 1; /* FIXME -> check this */
3941 	qp_attr->ah_attr.src_path_bits = 0;
3942 	qp_attr->ah_attr.static_rate = 0;
3943 	qp_attr->alt_pkey_index = 0;
3944 	qp_attr->alt_port_num = 0;
3945 	qp_attr->alt_timeout = 0;
3946 	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
3947 
3948 	qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0;
3949 	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
3950 	qp_attr->max_rd_atomic = params.max_rd_atomic;
3951 	qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0;
3952 
3953 	QL_DPRINT12(ha, "max_inline_data=%d\n",
3954 		qp_attr->cap.max_inline_data);
3955 
3956 err:
3957 	QL_DPRINT12(ha, "exit\n");
3958 	return rc;
3959 }
3960 
3961 static void
3962 qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3963 {
3964 	qlnx_host_t	*ha;
3965 
3966 	ha = dev->ha;
3967 
3968 	QL_DPRINT12(ha, "enter\n");
3969 
3970 	if (qp->usq.umem)
3971 		ib_umem_release(qp->usq.umem);
3972 
3973 	qp->usq.umem = NULL;
3974 
3975 	if (qp->urq.umem)
3976 		ib_umem_release(qp->urq.umem);
3977 
3978 	qp->urq.umem = NULL;
3979 
3980 	QL_DPRINT12(ha, "exit\n");
3981 	return;
3982 }
3983 
3984 static void
3985 qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
3986 {
3987 	qlnx_host_t	*ha;
3988 
3989 	ha = dev->ha;
3990 
3991 	QL_DPRINT12(ha, "enter\n");
3992 
3993 	if (qlnxr_qp_has_sq(qp)) {
3994 		QL_DPRINT12(ha, "freeing SQ\n");
3995 		ha->qlnxr_debug = 1;
3996 //		ecore_chain_free(dev->cdev, &qp->sq.pbl);
3997 		ha->qlnxr_debug = 0;
3998 		kfree(qp->wqe_wr_id);
3999 	}
4000 
4001 	if (qlnxr_qp_has_rq(qp)) {
4002 		QL_DPRINT12(ha, "freeing RQ\n");
4003 		ha->qlnxr_debug = 1;
4004 	//	ecore_chain_free(dev->cdev, &qp->rq.pbl);
4005 		ha->qlnxr_debug = 0;
4006 		kfree(qp->rqe_wr_id);
4007 	}
4008 
4009 	QL_DPRINT12(ha, "exit\n");
4010 	return;
4011 }
4012 
4013 int
4014 qlnxr_free_qp_resources(struct qlnxr_dev *dev,
4015 	struct qlnxr_qp *qp)
4016 {
4017 	int		rc = 0;
4018 	qlnx_host_t	*ha;
4019 	struct ecore_rdma_destroy_qp_out_params d_out_params;
4020 
4021 	ha = dev->ha;
4022 
4023 	QL_DPRINT12(ha, "enter\n");
4024 
4025 #if 0
4026 	if (qp->qp_type != IB_QPT_GSI) {
4027 		rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4028 				&d_out_params);
4029 		if (rc)
4030 			return rc;
4031 	}
4032 
4033 	if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4034 		qlnxr_cleanup_user(dev, qp);
4035 	else
4036 		qlnxr_cleanup_kernel(dev, qp);
4037 #endif
4038 
4039 	if (qp->ibqp.uobject && qp->ibqp.uobject->context)
4040 		qlnxr_cleanup_user(dev, qp);
4041 	else
4042 		qlnxr_cleanup_kernel(dev, qp);
4043 
4044 	if (qp->qp_type != IB_QPT_GSI) {
4045 		rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,
4046 				&d_out_params);
4047 		if (rc)
4048 			return rc;
4049 	}
4050 
4051 	QL_DPRINT12(ha, "exit\n");
4052 	return 0;
4053 }
4054 
4055 int
4056 qlnxr_destroy_qp(struct ib_qp *ibqp)
4057 {
4058 	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
4059 	struct qlnxr_dev *dev = qp->dev;
4060 	int rc = 0;
4061 	struct ib_qp_attr attr;
4062 	int attr_mask = 0;
4063 	qlnx_host_t	*ha;
4064 
4065 	ha = dev->ha;
4066 
4067 	QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type);
4068 
4069 	qp->destroyed = 1;
4070 
4071 	if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET |
4072 				  ECORE_ROCE_QP_STATE_ERR |
4073 				  ECORE_ROCE_QP_STATE_INIT))) {
4074 		attr.qp_state = IB_QPS_ERR;
4075 		attr_mask |= IB_QP_STATE;
4076 
4077 		/* change the QP state to ERROR */
4078 		qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL);
4079 	}
4080 
4081 	if (qp->qp_type == IB_QPT_GSI)
4082 		qlnxr_destroy_gsi_qp(dev);
4083 
4084 	qp->sig = ~qp->sig;
4085 
4086 	qlnxr_free_qp_resources(dev, qp);
4087 
4088 	if (atomic_dec_and_test(&qp->refcnt)) {
4089 		/* TODO: only for iWARP? */
4090 		qlnxr_idr_remove(dev, qp->qp_id);
4091 		kfree(qp);
4092 	}
4093 
4094 	QL_DPRINT12(ha, "exit\n");
4095 	return rc;
4096 }
4097 
4098 static inline int
4099 qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq)
4100 {
4101 	return (((wq->prod + 1) % wq->max_wr) == wq->cons);
4102 }
4103 
4104 static int
4105 sge_data_len(struct ib_sge *sg_list, int num_sge)
4106 {
4107 	int i, len = 0;
4108 	for (i = 0; i < num_sge; i++)
4109 		len += sg_list[i].length;
4110 	return len;
4111 }
4112 
4113 static void
4114 swap_wqe_data64(u64 *p)
4115 {
4116 	int i;
4117 
4118 	for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
4119 		*p = cpu_to_be64(cpu_to_le64(*p));
4120 }
4121 
4122 static u32
4123 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev,
4124 	struct qlnxr_qp		*qp,
4125 	u8			*wqe_size,
4126 	struct ib_send_wr	*wr,
4127 	struct ib_send_wr	**bad_wr,
4128 	u8			*bits,
4129 	u8			bit)
4130 {
4131 	int i, seg_siz;
4132 	char *seg_prt, *wqe;
4133 	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
4134 	qlnx_host_t	*ha;
4135 
4136 	ha = dev->ha;
4137 
4138 	QL_DPRINT12(ha, "enter[%d]\n", data_size);
4139 
4140 	if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
4141 		QL_DPRINT12(ha,
4142 			"Too much inline data in WR:[%d, %d]\n",
4143 			data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE);
4144 		*bad_wr = wr;
4145 		return 0;
4146 	}
4147 
4148 	if (!data_size)
4149 		return data_size;
4150 
4151 	/* set the bit */
4152 	*bits |= bit;
4153 
4154 	seg_prt = wqe = NULL;
4155 	seg_siz = 0;
4156 
4157 	/* copy data inline */
4158 	for (i = 0; i < wr->num_sge; i++) {
4159 		u32 len = wr->sg_list[i].length;
4160 		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
4161 
4162 		while (len > 0) {
4163 			u32 cur;
4164 
4165 			/* new segment required */
4166 			if (!seg_siz) {
4167 				wqe = (char *)ecore_chain_produce(&qp->sq.pbl);
4168 				seg_prt = wqe;
4169 				seg_siz = sizeof(struct rdma_sq_common_wqe);
4170 				(*wqe_size)++;
4171 			}
4172 
4173 			/* calculate currently allowed length */
4174 			cur = MIN(len, seg_siz);
4175 
4176 			memcpy(seg_prt, src, cur);
4177 
4178 			/* update segment variables */
4179 			seg_prt += cur;
4180 			seg_siz -= cur;
4181 			/* update sge variables */
4182 			src += cur;
4183 			len -= cur;
4184 
4185 			/* swap fully-completed segments */
4186 			if (!seg_siz)
4187 				swap_wqe_data64((u64 *)wqe);
4188 		}
4189 	}
4190 
4191 	/* swap last not completed segment */
4192 	if (seg_siz)
4193 		swap_wqe_data64((u64 *)wqe);
4194 
4195 	QL_DPRINT12(ha, "exit\n");
4196 	return data_size;
4197 }
4198 
4199 static u32
4200 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp,
4201 	u8 *wqe_size, struct ib_send_wr *wr)
4202 {
4203 	int i;
4204 	u32 data_size = 0;
4205 	qlnx_host_t	*ha;
4206 
4207 	ha = dev->ha;
4208 
4209 	QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge);
4210 
4211 	for (i = 0; i < wr->num_sge; i++) {
4212 		struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl);
4213 
4214 		TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);
4215 		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
4216 		sge->length = cpu_to_le32(wr->sg_list[i].length);
4217 		data_size += wr->sg_list[i].length;
4218 	}
4219 
4220 	if (wqe_size)
4221 		*wqe_size += wr->num_sge;
4222 
4223 	QL_DPRINT12(ha, "exit data_size = %d\n", data_size);
4224 	return data_size;
4225 }
4226 
4227 static u32
4228 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev,
4229 	struct qlnxr_qp *qp,
4230 	struct rdma_sq_rdma_wqe_1st *rwqe,
4231 	struct rdma_sq_rdma_wqe_2nd *rwqe2,
4232 	struct ib_send_wr *wr,
4233 	struct ib_send_wr **bad_wr)
4234 {
4235 	qlnx_host_t	*ha;
4236 	u32             ret = 0;
4237 
4238 	ha = dev->ha;
4239 
4240 	QL_DPRINT12(ha, "enter\n");
4241 
4242 	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
4243 	TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr);
4244 
4245 	if (wr->send_flags & IB_SEND_INLINE) {
4246 		u8 flags = 0;
4247 		SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
4248 		return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size,
4249 				wr, bad_wr, &rwqe->flags, flags);
4250 	}
4251 
4252 	ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr);
4253 
4254 	QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4255 
4256 	return (ret);
4257 }
4258 
4259 static u32
4260 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev,
4261 	struct qlnxr_qp *qp,
4262 	struct rdma_sq_send_wqe *swqe,
4263 	struct rdma_sq_send_wqe *swqe2,
4264 	struct ib_send_wr *wr,
4265 	struct ib_send_wr **bad_wr)
4266 {
4267 	qlnx_host_t	*ha;
4268 	u32             ret = 0;
4269 
4270 	ha = dev->ha;
4271 
4272 	QL_DPRINT12(ha, "enter\n");
4273 
4274 	memset(swqe2, 0, sizeof(*swqe2));
4275 
4276 	if (wr->send_flags & IB_SEND_INLINE) {
4277 		u8 flags = 0;
4278 		SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
4279 		return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size,
4280 				wr, bad_wr, &swqe->flags, flags);
4281 	}
4282 
4283 	ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr);
4284 
4285 	QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);
4286 
4287 	return (ret);
4288 }
4289 
4290 static void
4291 qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info)
4292 {
4293 	qlnx_host_t	*ha;
4294 
4295 	ha = dev->ha;
4296 
4297 	int work = info->completed - info->completed_handled - 1;
4298 
4299 	QL_DPRINT12(ha, "enter [%d]\n", work);
4300 
4301 	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
4302 		struct qlnxr_pbl *pbl;
4303 
4304 		/* Free all the page list that are possible to be freed
4305 		 * (all the ones that were invalidated), under the assumption
4306 		 * that if an FMR was completed successfully that means that
4307 		 * if there was an invalidate operation before it also ended
4308 		 */
4309 		pbl = list_first_entry(&info->inuse_pbl_list,
4310 				       struct qlnxr_pbl,
4311 				       list_entry);
4312 		list_del(&pbl->list_entry);
4313 		list_add_tail(&pbl->list_entry, &info->free_pbl_list);
4314 		info->completed_handled++;
4315 	}
4316 
4317 	QL_DPRINT12(ha, "exit\n");
4318 	return;
4319 }
4320 
4321 #if __FreeBSD_version >= 1102000
4322 
4323 static int qlnxr_prepare_reg(struct qlnxr_qp *qp,
4324 		struct rdma_sq_fmr_wqe_1st *fwqe1,
4325 		struct ib_reg_wr *wr)
4326 {
4327 	struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr);
4328 	struct rdma_sq_fmr_wqe_2nd *fwqe2;
4329 
4330 	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl);
4331 	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
4332 	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
4333 	fwqe1->l_key = wr->key;
4334 
4335 	fwqe2->access_ctrl = 0;
4336 
4337 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
4338 		!!(wr->access & IB_ACCESS_REMOTE_READ));
4339 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
4340 		!!(wr->access & IB_ACCESS_REMOTE_WRITE));
4341 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
4342 		!!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
4343 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
4344 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
4345 		!!(wr->access & IB_ACCESS_LOCAL_WRITE));
4346 	fwqe2->fmr_ctrl = 0;
4347 
4348 	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4349 		ilog2(mr->ibmr.page_size) - 12);
4350 
4351 	fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */
4352 	fwqe2->length_lo = mr->ibmr.length;
4353 	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
4354 	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
4355 
4356 	qp->wqe_wr_id[qp->sq.prod].mr = mr;
4357 
4358 	return 0;
4359 }
4360 
4361 #else
4362 
4363 static void
4364 build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr,
4365 	struct mr_info *info)
4366 {
4367 	int i;
4368 	u64 buf_addr = 0;
4369 	int num_pbes, total_num_pbes = 0;
4370 	struct regpair *pbe;
4371 	struct qlnxr_pbl *pbl_tbl = info->pbl_table;
4372 	struct qlnxr_pbl_info *pbl_info = &info->pbl_info;
4373 	qlnx_host_t	*ha;
4374 
4375 	ha = dev->ha;
4376 
4377 	QL_DPRINT12(ha, "enter\n");
4378 
4379 	pbe = (struct regpair *)pbl_tbl->va;
4380 	num_pbes = 0;
4381 
4382 	for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {
4383 		buf_addr = wr->wr.fast_reg.page_list->page_list[i];
4384 		pbe->lo = cpu_to_le32((u32)buf_addr);
4385 		pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr));
4386 
4387 		num_pbes += 1;
4388 		pbe++;
4389 		total_num_pbes++;
4390 
4391 		if (total_num_pbes == pbl_info->num_pbes)
4392 			return;
4393 
4394 		/* if the given pbl is full storing the pbes,
4395 		 * move to next pbl.
4396 		 */
4397 		if (num_pbes ==
4398 		    (pbl_info->pbl_size / sizeof(u64))) {
4399 			pbl_tbl++;
4400 			pbe = (struct regpair *)pbl_tbl->va;
4401 			num_pbes = 0;
4402 		}
4403 	}
4404 	QL_DPRINT12(ha, "exit\n");
4405 
4406 	return;
4407 }
4408 
4409 static int
4410 qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info)
4411 {
4412 	int rc = 0;
4413 	qlnx_host_t	*ha;
4414 
4415 	ha = dev->ha;
4416 
4417 	QL_DPRINT12(ha, "enter\n");
4418 
4419 	if (info->completed == 0) {
4420 		//DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n");
4421 		/* first fmr */
4422 		return 0;
4423 	}
4424 
4425 	qlnx_handle_completed_mrs(dev, info);
4426 
4427 	list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list);
4428 
4429 	if (list_empty(&info->free_pbl_list)) {
4430 		info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info,
4431 							  GFP_ATOMIC);
4432 	} else {
4433 		info->pbl_table = list_first_entry(&info->free_pbl_list,
4434 					struct qlnxr_pbl,
4435 					list_entry);
4436 		list_del(&info->pbl_table->list_entry);
4437 	}
4438 
4439 	if (!info->pbl_table)
4440 		rc = -ENOMEM;
4441 
4442 	QL_DPRINT12(ha, "exit\n");
4443 	return rc;
4444 }
4445 
4446 static inline int
4447 qlnxr_prepare_fmr(struct qlnxr_qp *qp,
4448 	struct rdma_sq_fmr_wqe_1st *fwqe1,
4449 	struct ib_send_wr *wr)
4450 {
4451 	struct qlnxr_dev *dev = qp->dev;
4452 	u64 fbo;
4453 	struct qlnxr_fast_reg_page_list *frmr_list =
4454 		get_qlnxr_frmr_list(wr->wr.fast_reg.page_list);
4455 	struct rdma_sq_fmr_wqe *fwqe2 =
4456 		(struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl);
4457 	int rc = 0;
4458 	qlnx_host_t	*ha;
4459 
4460 	ha = dev->ha;
4461 
4462 	QL_DPRINT12(ha, "enter\n");
4463 
4464 	if (wr->wr.fast_reg.page_list_len == 0)
4465 		BUG();
4466 
4467 	rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info);
4468 	if (rc)
4469 		return rc;
4470 
4471 	fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start);
4472 	fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start);
4473 	fwqe1->l_key = wr->wr.fast_reg.rkey;
4474 
4475 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ,
4476 		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ));
4477 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE,
4478 		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE));
4479 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC,
4480 		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC));
4481 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1);
4482 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE,
4483 		   !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE));
4484 
4485 	fwqe2->fmr_ctrl = 0;
4486 
4487 	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
4488 		   ilog2(1 << wr->wr.fast_reg.page_shift) - 12);
4489 	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0);
4490 
4491 	fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */
4492 	fwqe2->length_lo = wr->wr.fast_reg.length;
4493 	fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa);
4494 	fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa);
4495 
4496 	/* produce another wqe for fwqe3 */
4497 	ecore_chain_produce(&qp->sq.pbl);
4498 
4499 	fbo = wr->wr.fast_reg.iova_start -
4500 	    (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);
4501 
4502 	QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x"
4503 		" length = %x pbl_addr %x:%x\n",
4504 		wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey,
4505 		fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo,
4506 		fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo);
4507 
4508 	build_frmr_pbes(dev, wr, &frmr_list->info);
4509 
4510 	qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list;
4511 
4512 	QL_DPRINT12(ha, "exit\n");
4513 	return 0;
4514 }
4515 
4516 #endif /* #if __FreeBSD_version >= 1102000 */
4517 
4518 static enum ib_wc_opcode
4519 qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
4520 {
4521 	switch (opcode) {
4522 	case IB_WR_RDMA_WRITE:
4523 	case IB_WR_RDMA_WRITE_WITH_IMM:
4524 		return IB_WC_RDMA_WRITE;
4525 	case IB_WR_SEND_WITH_IMM:
4526 	case IB_WR_SEND:
4527 	case IB_WR_SEND_WITH_INV:
4528 		return IB_WC_SEND;
4529 	case IB_WR_RDMA_READ:
4530 		return IB_WC_RDMA_READ;
4531 	case IB_WR_ATOMIC_CMP_AND_SWP:
4532 		return IB_WC_COMP_SWAP;
4533 	case IB_WR_ATOMIC_FETCH_AND_ADD:
4534 		return IB_WC_FETCH_ADD;
4535 
4536 #if __FreeBSD_version >= 1102000
4537 	case IB_WR_REG_MR:
4538 		return IB_WC_REG_MR;
4539 #else
4540 	case IB_WR_FAST_REG_MR:
4541 		return IB_WC_FAST_REG_MR;
4542 #endif /* #if __FreeBSD_version >= 1102000 */
4543 
4544 	case IB_WR_LOCAL_INV:
4545 		return IB_WC_LOCAL_INV;
4546 	default:
4547 		return IB_WC_SEND;
4548 	}
4549 }
4550 static inline bool
4551 qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr)
4552 {
4553 	int wq_is_full, err_wr, pbl_is_full;
4554 	struct qlnxr_dev *dev = qp->dev;
4555 	qlnx_host_t	*ha;
4556 
4557 	ha = dev->ha;
4558 
4559 	QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr);
4560 
4561 	/* prevent SQ overflow and/or processing of a bad WR */
4562 	err_wr = wr->num_sge > qp->sq.max_sges;
4563 	wq_is_full = qlnxr_wq_is_full(&qp->sq);
4564 	pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) <
4565 		      QLNXR_MAX_SQE_ELEMENTS_PER_SQE;
4566 	if (wq_is_full || err_wr || pbl_is_full) {
4567 		if (wq_is_full &&
4568 		    !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) {
4569 			qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL;
4570 
4571 			QL_DPRINT12(ha,
4572 				"error: WQ is full. Post send on QP failed"
4573 				" (this error appears only once) "
4574 				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4575 				qp, wr, qp->err_bitmap);
4576 		}
4577 
4578 		if (err_wr &&
4579 		    !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) {
4580 			qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR;
4581 
4582 			QL_DPRINT12(ha,
4583 				"error: WQ is bad. Post send on QP failed"
4584 				" (this error appears only once) "
4585 				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4586 				qp, wr, qp->err_bitmap);
4587 		}
4588 
4589 		if (pbl_is_full &&
4590 		    !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) {
4591 			qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL;
4592 
4593 			QL_DPRINT12(ha,
4594 				"error: WQ PBL is full. Post send on QP failed"
4595 				" (this error appears only once) "
4596 				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",
4597 				qp, wr, qp->err_bitmap);
4598 		}
4599 		return false;
4600 	}
4601 	QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr);
4602 	return true;
4603 }
4604 
4605 int
4606 qlnxr_post_send(struct ib_qp *ibqp,
4607 	struct ib_send_wr *wr,
4608 	struct ib_send_wr **bad_wr)
4609 {
4610 	struct qlnxr_dev	*dev = get_qlnxr_dev(ibqp->device);
4611 	struct qlnxr_qp		*qp = get_qlnxr_qp(ibqp);
4612 	unsigned long 		flags;
4613 	int 			status = 0, rc = 0;
4614 	bool			comp;
4615 	qlnx_host_t		*ha;
4616 	uint32_t		reg_addr;
4617 
4618 	*bad_wr = NULL;
4619 	ha = dev->ha;
4620 
4621 	QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4622 		ibqp, wr, bad_wr);
4623 
4624 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
4625 		return -EINVAL;
4626 
4627 	if (qp->qp_type == IB_QPT_GSI)
4628 		return qlnxr_gsi_post_send(ibqp, wr, bad_wr);
4629 
4630 	spin_lock_irqsave(&qp->q_lock, flags);
4631 
4632 	if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) &&
4633 	    (qp->state != ECORE_ROCE_QP_STATE_ERR) &&
4634 	    (qp->state != ECORE_ROCE_QP_STATE_SQD)) {
4635 		spin_unlock_irqrestore(&qp->q_lock, flags);
4636 		*bad_wr = wr;
4637 		QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n",
4638 			qp->icid, qp->state);
4639 		return -EINVAL;
4640 	}
4641 
4642 	if (!wr) {
4643 		QL_DPRINT11(ha, "Got an empty post send???\n");
4644 	}
4645 
4646 	while (wr) {
4647 		struct rdma_sq_common_wqe	*wqe;
4648 		struct rdma_sq_send_wqe		*swqe;
4649 		struct rdma_sq_send_wqe		*swqe2;
4650 		struct rdma_sq_rdma_wqe_1st	*rwqe;
4651 		struct rdma_sq_rdma_wqe_2nd	*rwqe2;
4652 		struct rdma_sq_local_inv_wqe	*iwqe;
4653 		struct rdma_sq_atomic_wqe	*awqe1;
4654 		struct rdma_sq_atomic_wqe	*awqe2;
4655 		struct rdma_sq_atomic_wqe	*awqe3;
4656 		struct rdma_sq_fmr_wqe_1st	*fwqe1;
4657 
4658 		if (!qlnxr_can_post_send(qp, wr)) {
4659 			status = -ENOMEM;
4660 			*bad_wr = wr;
4661 			break;
4662 		}
4663 
4664 		wqe = ecore_chain_produce(&qp->sq.pbl);
4665 
4666 		qp->wqe_wr_id[qp->sq.prod].signaled =
4667 			!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
4668 
4669 		/* common fields */
4670 		wqe->flags = 0;
4671 		wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK <<
4672 				RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT);
4673 
4674 		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \
4675 			!!(wr->send_flags & IB_SEND_SOLICITED));
4676 
4677 		comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) ||
4678 				(qp->signaled);
4679 
4680 		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
4681 		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,  \
4682 			!!(wr->send_flags & IB_SEND_FENCE));
4683 
4684 		wqe->prev_wqe_size = qp->prev_wqe_size;
4685 
4686 		qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode);
4687 
4688 		switch (wr->opcode) {
4689 		case IB_WR_SEND_WITH_IMM:
4690 
4691 			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
4692 			swqe = (struct rdma_sq_send_wqe *)wqe;
4693 			swqe->wqe_size = 2;
4694 			swqe2 = (struct rdma_sq_send_wqe *)
4695 					ecore_chain_produce(&qp->sq.pbl);
4696 			swqe->inv_key_or_imm_data =
4697 				cpu_to_le32(wr->ex.imm_data);
4698 			swqe->length = cpu_to_le32(
4699 						qlnxr_prepare_sq_send_data(dev,
4700 							qp, swqe, swqe2, wr,
4701 							bad_wr));
4702 
4703 			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4704 			qp->prev_wqe_size = swqe->wqe_size;
4705 			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4706 
4707 			QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n",
4708 				swqe->length, wr->ex.imm_data);
4709 
4710 			break;
4711 
4712 		case IB_WR_SEND:
4713 
4714 			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
4715 			swqe = (struct rdma_sq_send_wqe *)wqe;
4716 
4717 			swqe->wqe_size = 2;
4718 			swqe2 = (struct rdma_sq_send_wqe *)
4719 					ecore_chain_produce(&qp->sq.pbl);
4720 			swqe->length = cpu_to_le32(
4721 						qlnxr_prepare_sq_send_data(dev,
4722 							qp, swqe, swqe2, wr,
4723 							bad_wr));
4724 			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4725 			qp->prev_wqe_size = swqe->wqe_size;
4726 			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4727 
4728 			QL_DPRINT12(ha, "SEND w/o IMM length = %d\n",
4729 				swqe->length);
4730 
4731 			break;
4732 
4733 		case IB_WR_SEND_WITH_INV:
4734 
4735 			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
4736 			swqe = (struct rdma_sq_send_wqe *)wqe;
4737 			swqe2 = (struct rdma_sq_send_wqe *)
4738 					ecore_chain_produce(&qp->sq.pbl);
4739 			swqe->wqe_size = 2;
4740 			swqe->inv_key_or_imm_data =
4741 				cpu_to_le32(wr->ex.invalidate_rkey);
4742 			swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev,
4743 						qp, swqe, swqe2, wr, bad_wr));
4744 			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
4745 			qp->prev_wqe_size = swqe->wqe_size;
4746 			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
4747 
4748 			QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n",
4749 				swqe->length);
4750 			break;
4751 
4752 		case IB_WR_RDMA_WRITE_WITH_IMM:
4753 
4754 			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
4755 			rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4756 
4757 			rwqe->wqe_size = 2;
4758 			rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
4759 			rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4760 					ecore_chain_produce(&qp->sq.pbl);
4761 			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4762 						qp, rwqe, rwqe2, wr, bad_wr));
4763 			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4764 			qp->prev_wqe_size = rwqe->wqe_size;
4765 			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4766 
4767 			QL_DPRINT12(ha,
4768 				"RDMA WRITE w/ IMM length = %d imm data=%x\n",
4769 				rwqe->length, rwqe->imm_data);
4770 
4771 			break;
4772 
4773 		case IB_WR_RDMA_WRITE:
4774 
4775 			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
4776 			rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4777 
4778 			rwqe->wqe_size = 2;
4779 			rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4780 					ecore_chain_produce(&qp->sq.pbl);
4781 			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4782 						qp, rwqe, rwqe2, wr, bad_wr));
4783 			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4784 			qp->prev_wqe_size = rwqe->wqe_size;
4785 			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4786 
4787 			QL_DPRINT12(ha,
4788 				"RDMA WRITE w/o IMM length = %d\n",
4789 				rwqe->length);
4790 
4791 			break;
4792 
4793 		case IB_WR_RDMA_READ_WITH_INV:
4794 
4795 			QL_DPRINT12(ha,
4796 				"RDMA READ WITH INVALIDATE not supported\n");
4797 
4798 			*bad_wr = wr;
4799 			rc = -EINVAL;
4800 
4801 			break;
4802 
4803 		case IB_WR_RDMA_READ:
4804 
4805 			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
4806 			rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
4807 
4808 			rwqe->wqe_size = 2;
4809 			rwqe2 = (struct rdma_sq_rdma_wqe_2nd *)
4810 					ecore_chain_produce(&qp->sq.pbl);
4811 			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,
4812 						qp, rwqe, rwqe2, wr, bad_wr));
4813 
4814 			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
4815 			qp->prev_wqe_size = rwqe->wqe_size;
4816 			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
4817 
4818 			QL_DPRINT12(ha, "RDMA READ length = %d\n",
4819 				rwqe->length);
4820 
4821 			break;
4822 
4823 		case IB_WR_ATOMIC_CMP_AND_SWP:
4824 		case IB_WR_ATOMIC_FETCH_AND_ADD:
4825 
4826 			QL_DPRINT12(ha,
4827 				"ATOMIC operation = %s\n",
4828 				((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ?
4829 					"IB_WR_ATOMIC_CMP_AND_SWP" :
4830 					"IB_WR_ATOMIC_FETCH_AND_ADD"));
4831 
4832 			awqe1 = (struct rdma_sq_atomic_wqe *)wqe;
4833 			awqe1->prev_wqe_size = 4;
4834 
4835 			awqe2 = (struct rdma_sq_atomic_wqe *)
4836 					ecore_chain_produce(&qp->sq.pbl);
4837 
4838 			TYPEPTR_ADDR_SET(awqe2, remote_va, \
4839 				atomic_wr(wr)->remote_addr);
4840 
4841 			awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
4842 
4843 			awqe3 = (struct rdma_sq_atomic_wqe *)
4844 					ecore_chain_produce(&qp->sq.pbl);
4845 
4846 			if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
4847 				wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
4848 				TYPEPTR_ADDR_SET(awqe3, swap_data,
4849 						 atomic_wr(wr)->compare_add);
4850 			} else {
4851 				wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
4852 				TYPEPTR_ADDR_SET(awqe3, swap_data,
4853 						 atomic_wr(wr)->swap);
4854 				TYPEPTR_ADDR_SET(awqe3, cmp_data,
4855 						 atomic_wr(wr)->compare_add);
4856 			}
4857 
4858 			qlnxr_prepare_sq_sges(dev, qp, NULL, wr);
4859 
4860 			qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size;
4861 			qp->prev_wqe_size = awqe1->prev_wqe_size;
4862 
4863 			break;
4864 
4865 		case IB_WR_LOCAL_INV:
4866 
4867 			QL_DPRINT12(ha,
4868 				"INVALIDATE length (IB_WR_LOCAL_INV)\n");
4869 
4870 			iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
4871 			iwqe->prev_wqe_size = 1;
4872 
4873 			iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
4874 			iwqe->inv_l_key = wr->ex.invalidate_rkey;
4875 			qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size;
4876 			qp->prev_wqe_size = iwqe->prev_wqe_size;
4877 
4878 			break;
4879 
4880 #if __FreeBSD_version >= 1102000
4881 
4882 		case IB_WR_REG_MR:
4883 
4884 			QL_DPRINT12(ha, "IB_WR_REG_MR\n");
4885 
4886 			wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4887 			fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4888 			fwqe1->wqe_size = 2;
4889 
4890 			rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr));
4891 			if (rc) {
4892 				QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc);
4893 				*bad_wr = wr;
4894 				break;
4895 			}
4896 
4897 			qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
4898 			qp->prev_wqe_size = fwqe1->wqe_size;
4899 
4900 			break;
4901 #else
4902 		case IB_WR_FAST_REG_MR:
4903 
4904 			QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n");
4905 
4906 			wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
4907 			fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
4908 			fwqe1->prev_wqe_size = 3;
4909 
4910 			rc = qlnxr_prepare_fmr(qp, fwqe1, wr);
4911 
4912 			if (rc) {
4913 				QL_DPRINT12(ha,
4914 					"FAST_MR (IB_WR_FAST_REG_MR) failed"
4915 					" rc = %d\n", rc);
4916 				*bad_wr = wr;
4917 				break;
4918 			}
4919 
4920 			qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size;
4921 			qp->prev_wqe_size = fwqe1->prev_wqe_size;
4922 
4923 			break;
4924 #endif /* #if __FreeBSD_version >= 1102000 */
4925 
4926 		default:
4927 
4928 			QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode);
4929 
4930 			rc = -EINVAL;
4931 			*bad_wr = wr;
4932 			break;
4933 		}
4934 
4935 		if (*bad_wr) {
4936 			/*
4937 			 * restore prod to its position before this WR was processed
4938 			 */
4939 			ecore_chain_set_prod(&qp->sq.pbl,
4940 			     le16_to_cpu(qp->sq.db_data.data.value),
4941 			     wqe);
4942 			/* restore prev_wqe_size */
4943 			qp->prev_wqe_size = wqe->prev_wqe_size;
4944 			status = rc;
4945 
4946 			QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr);
4947 			break; /* out of the loop */
4948 		}
4949 
4950 		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
4951 
4952 		qlnxr_inc_sw_prod(&qp->sq);
4953 
4954 		qp->sq.db_data.data.value++;
4955 
4956 		wr = wr->next;
4957 	}
4958 
4959 	/* Trigger doorbell
4960 	 * If there was a failure in the first WR then it will be triggered in
4961 	 * vane. However this is not harmful (as long as the producer value is
4962 	 * unchanged). For performance reasons we avoid checking for this
4963 	 * redundant doorbell.
4964 	 */
4965 	wmb();
4966 	//writel(qp->sq.db_data.raw, qp->sq.db);
4967 
4968 	reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells);
4969         bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw);
4970         bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
4971 
4972 	mmiowb();
4973 
4974 	spin_unlock_irqrestore(&qp->q_lock, flags);
4975 
4976 	QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n",
4977 		ibqp, wr, bad_wr);
4978 
4979 	return status;
4980 }
4981 
4982 static u32
4983 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq)
4984 {
4985 	u32 used;
4986 
4987 	/* Calculate number of elements used based on producer
4988 	 * count and consumer count and subtract it from max
4989 	 * work request supported so that we get elements left.
4990 	 */
4991 	used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
4992 
4993 	return hw_srq->max_wr - used;
4994 }
4995 
4996 int
4997 qlnxr_post_recv(struct ib_qp *ibqp,
4998 	struct ib_recv_wr *wr,
4999 	struct ib_recv_wr **bad_wr)
5000 {
5001  	struct qlnxr_qp		*qp = get_qlnxr_qp(ibqp);
5002 	struct qlnxr_dev	*dev = qp->dev;
5003 	unsigned long		flags;
5004 	int			status = 0;
5005 	qlnx_host_t		*ha;
5006 	uint32_t		reg_addr;
5007 
5008 	ha = dev->ha;
5009 
5010 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5011 		return -EINVAL;
5012 
5013 	QL_DPRINT12(ha, "enter\n");
5014 
5015 	if (qp->qp_type == IB_QPT_GSI) {
5016 		QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n");
5017 		return qlnxr_gsi_post_recv(ibqp, wr, bad_wr);
5018 	}
5019 
5020 	if (qp->srq) {
5021 		QL_DPRINT11(ha, "qp->srq [%p]"
5022 			" QP is associated with SRQ, cannot post RQ buffers\n",
5023 			qp->srq);
5024 		return -EINVAL;
5025 	}
5026 
5027 	spin_lock_irqsave(&qp->q_lock, flags);
5028 
5029 	if (qp->state == ECORE_ROCE_QP_STATE_RESET) {
5030 		spin_unlock_irqrestore(&qp->q_lock, flags);
5031 		*bad_wr = wr;
5032 
5033 		QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n");
5034 
5035 		return -EINVAL;
5036 	}
5037 
5038 	while (wr) {
5039 		int i;
5040 
5041 		if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) <
5042 			QLNXR_MAX_RQE_ELEMENTS_PER_RQE) ||
5043 			(wr->num_sge > qp->rq.max_sges)) {
5044 			status = -ENOMEM;
5045 			*bad_wr = wr;
5046 			break;
5047 		}
5048 		for (i = 0; i < wr->num_sge; i++) {
5049 			u32 flags = 0;
5050 			struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5051 
5052 			/* first one must include the number of SGE in the list */
5053 			if (!i)
5054 				SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge);
5055 
5056 			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey);
5057 
5058 			RQ_SGE_SET(rqe, wr->sg_list[i].addr, \
5059 				wr->sg_list[i].length, flags);
5060 		}
5061 		/* Special case of no sges. FW requires between 1-4 sges...
5062 		 * in this case we need to post 1 sge with length zero. this is
5063 		 * because rdma write with immediate consumes an RQ. */
5064 		if (!wr->num_sge) {
5065 			u32 flags = 0;
5066 			struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);
5067 
5068 			/* first one must include the number of SGE in the list */
5069 			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
5070 			SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
5071 
5072 			//RQ_SGE_SET(rqe, 0, 0, flags);
5073 			rqe->addr.hi = 0;
5074 			rqe->addr.lo = 0;
5075 
5076 			rqe->length = 0;
5077 			rqe->flags = cpu_to_le32(flags);
5078 
5079 			i = 1;
5080 		}
5081 
5082 		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
5083 		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
5084 
5085 		qlnxr_inc_sw_prod(&qp->rq);
5086 
5087 		wmb();
5088 
5089 		qp->rq.db_data.data.value++;
5090 
5091 	//	writel(qp->rq.db_data.raw, qp->rq.db);
5092 		mmiowb();
5093 	//	if (QLNX_IS_IWARP(dev)) {
5094 	//		writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
5095 	//		mmiowb(); /* for second doorbell */
5096 	//	}
5097 
5098 		reg_addr = (uint32_t)((uint8_t *)qp->rq.db -
5099 				(uint8_t *)ha->cdev.doorbells);
5100 
5101 		bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);
5102 		bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
5103 
5104 		if (QLNX_IS_IWARP(dev)) {
5105 			reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -
5106 						(uint8_t *)ha->cdev.doorbells);
5107 			bus_write_4(ha->pci_dbells, reg_addr, \
5108 				qp->rq.iwarp_db2_data.raw);
5109 			bus_barrier(ha->pci_dbells,  0, 0, \
5110 				BUS_SPACE_BARRIER_READ);
5111 		}
5112 
5113 		wr = wr->next;
5114 	}
5115 
5116 	spin_unlock_irqrestore(&qp->q_lock, flags);
5117 
5118 	QL_DPRINT12(ha, "exit status = 0x%x\n", status);
5119 
5120 	return status;
5121 }
5122 
5123 /* In fmr we need to increase the number of fmr completed counter for the fmr
5124  * algorithm determining whether we can free a pbl or not.
5125  * we need to perform this whether the work request was signaled or not. for
5126  * this purpose we call this function from the condition that checks if a wr
5127  * should be skipped, to make sure we don't miss it ( possibly this fmr
5128  * operation was not signalted)
5129  */
5130 static inline void
5131 qlnxr_chk_if_fmr(struct qlnxr_qp *qp)
5132 {
5133 #if __FreeBSD_version >= 1102000
5134 
5135 	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
5136 		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5137 #else
5138 	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR)
5139 		qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5140 
5141 #endif /* #if __FreeBSD_version >= 1102000 */
5142 }
5143 
5144 static int
5145 process_req(struct qlnxr_dev *dev,
5146 	struct qlnxr_qp *qp,
5147 	struct qlnxr_cq *cq,
5148 	int num_entries,
5149 	struct ib_wc *wc,
5150 	u16 hw_cons,
5151 	enum ib_wc_status status,
5152 	int force)
5153 {
5154 	u16		cnt = 0;
5155 	qlnx_host_t	*ha = dev->ha;
5156 
5157 	QL_DPRINT12(ha, "enter\n");
5158 
5159 	while (num_entries && qp->sq.wqe_cons != hw_cons) {
5160 		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
5161 			qlnxr_chk_if_fmr(qp);
5162 			/* skip WC */
5163 			goto next_cqe;
5164 		}
5165 
5166 		/* fill WC */
5167 		wc->status = status;
5168 		wc->vendor_err = 0;
5169 		wc->wc_flags = 0;
5170 		wc->src_qp = qp->id;
5171 		wc->qp = &qp->ibqp;
5172 
5173 		// common section
5174 		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
5175 		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
5176 
5177 		switch (wc->opcode) {
5178 		case IB_WC_RDMA_WRITE:
5179 
5180 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
5181 
5182 			QL_DPRINT12(ha,
5183 				"opcode = IB_WC_RDMA_WRITE bytes = %d\n",
5184 				qp->wqe_wr_id[qp->sq.cons].bytes_len);
5185 			break;
5186 
5187 		case IB_WC_COMP_SWAP:
5188 		case IB_WC_FETCH_ADD:
5189 			wc->byte_len = 8;
5190 			break;
5191 
5192 #if __FreeBSD_version >= 1102000
5193 		case IB_WC_REG_MR:
5194 			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
5195 			break;
5196 #else
5197 		case IB_WC_FAST_REG_MR:
5198 			qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++;
5199 			break;
5200 #endif /* #if __FreeBSD_version >= 1102000 */
5201 
5202 		case IB_WC_RDMA_READ:
5203 		case IB_WC_SEND:
5204 
5205 			QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode);
5206 			break;
5207 		default:
5208 			;//DP_ERR("TBD ERROR");
5209 		}
5210 
5211 		num_entries--;
5212 		wc++;
5213 		cnt++;
5214 next_cqe:
5215 		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
5216 			ecore_chain_consume(&qp->sq.pbl);
5217 		qlnxr_inc_sw_cons(&qp->sq);
5218 	}
5219 
5220 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5221 	return cnt;
5222 }
5223 
5224 static int
5225 qlnxr_poll_cq_req(struct qlnxr_dev *dev,
5226 	struct qlnxr_qp *qp,
5227 	struct qlnxr_cq *cq,
5228 	int num_entries,
5229 	struct ib_wc *wc,
5230 	struct rdma_cqe_requester *req)
5231 {
5232 	int		cnt = 0;
5233 	qlnx_host_t	*ha = dev->ha;
5234 
5235 	QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status);
5236 
5237 	switch (req->status) {
5238 	case RDMA_CQE_REQ_STS_OK:
5239 
5240 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5241 			IB_WC_SUCCESS, 0);
5242 		break;
5243 
5244 	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
5245 
5246 		if (qp->state != ECORE_ROCE_QP_STATE_ERR)
5247 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
5248 				  IB_WC_WR_FLUSH_ERR, 1);
5249 		break;
5250 
5251 	default: /* other errors case */
5252 
5253 		/* process all WQE before the cosumer */
5254 		qp->state = ECORE_ROCE_QP_STATE_ERR;
5255 		cnt = process_req(dev, qp, cq, num_entries, wc,
5256 				req->sq_cons - 1, IB_WC_SUCCESS, 0);
5257 		wc += cnt;
5258 		/* if we have extra WC fill it with actual error info */
5259 
5260 		if (cnt < num_entries) {
5261 			enum ib_wc_status wc_status;
5262 
5263 			switch (req->status) {
5264 			case 	RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
5265 				wc_status = IB_WC_BAD_RESP_ERR;
5266 				break;
5267 			case 	RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
5268 				wc_status = IB_WC_LOC_LEN_ERR;
5269 				break;
5270 			case    RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
5271 				wc_status = IB_WC_LOC_QP_OP_ERR;
5272 				break;
5273 			case    RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
5274 				wc_status = IB_WC_LOC_PROT_ERR;
5275 				break;
5276 			case    RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
5277 				wc_status = IB_WC_MW_BIND_ERR;
5278 				break;
5279 			case    RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
5280 				wc_status = IB_WC_REM_INV_REQ_ERR;
5281 				break;
5282 			case    RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
5283 				wc_status = IB_WC_REM_ACCESS_ERR;
5284 				break;
5285 			case    RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
5286 				wc_status = IB_WC_REM_OP_ERR;
5287 				break;
5288 			case    RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
5289 				wc_status = IB_WC_RNR_RETRY_EXC_ERR;
5290 				break;
5291 			case    RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
5292 				wc_status = IB_WC_RETRY_EXC_ERR;
5293 				break;
5294 			default:
5295 				wc_status = IB_WC_GENERAL_ERR;
5296 			}
5297 
5298 			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
5299 					wc_status, 1 /* force use of WC */);
5300 		}
5301 	}
5302 
5303 	QL_DPRINT12(ha, "exit cnt = %d\n", cnt);
5304 	return cnt;
5305 }
5306 
5307 static void
5308 __process_resp_one(struct qlnxr_dev *dev,
5309 	struct qlnxr_qp *qp,
5310 	struct qlnxr_cq *cq,
5311 	struct ib_wc *wc,
5312 	struct rdma_cqe_responder *resp,
5313 	u64 wr_id)
5314 {
5315 	enum ib_wc_status	wc_status = IB_WC_SUCCESS;
5316 #if __FreeBSD_version < 1102000
5317 	u8			flags;
5318 #endif
5319 	qlnx_host_t		*ha = dev->ha;
5320 
5321 	QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n",
5322 		qp, resp->status);
5323 
5324 	wc->opcode = IB_WC_RECV;
5325 	wc->wc_flags = 0;
5326 
5327 	switch (resp->status) {
5328 	case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
5329 		wc_status = IB_WC_LOC_ACCESS_ERR;
5330 		break;
5331 
5332 	case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
5333 		wc_status = IB_WC_LOC_LEN_ERR;
5334 		break;
5335 
5336 	case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
5337 		wc_status = IB_WC_LOC_QP_OP_ERR;
5338 		break;
5339 
5340 	case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
5341 		wc_status = IB_WC_LOC_PROT_ERR;
5342 		break;
5343 
5344 	case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
5345 		wc_status = IB_WC_MW_BIND_ERR;
5346 		break;
5347 
5348 	case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
5349 		wc_status = IB_WC_REM_INV_RD_REQ_ERR;
5350 		break;
5351 
5352 	case RDMA_CQE_RESP_STS_OK:
5353 
5354 #if __FreeBSD_version >= 1102000
5355 		if (resp->flags & QLNXR_RESP_IMM) {
5356 			wc->ex.imm_data =
5357 				le32_to_cpu(resp->imm_data_or_inv_r_Key);
5358 			wc->wc_flags |= IB_WC_WITH_IMM;
5359 
5360 			if (resp->flags & QLNXR_RESP_RDMA)
5361 				wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5362 
5363 			if (resp->flags & QLNXR_RESP_INV) {
5364 				QL_DPRINT11(ha,
5365 					"Invalid flags QLNXR_RESP_INV [0x%x]"
5366 					"qp = %p qp->id = 0x%x cq = %p"
5367 					" cq->icid = 0x%x\n",
5368 					resp->flags, qp, qp->id, cq, cq->icid );
5369 			}
5370 		} else if (resp->flags & QLNXR_RESP_INV) {
5371 			wc->ex.imm_data =
5372 				le32_to_cpu(resp->imm_data_or_inv_r_Key);
5373 			wc->wc_flags |= IB_WC_WITH_INVALIDATE;
5374 
5375 			if (resp->flags & QLNXR_RESP_RDMA) {
5376 				QL_DPRINT11(ha,
5377 					"Invalid flags QLNXR_RESP_RDMA [0x%x]"
5378 					"qp = %p qp->id = 0x%x cq = %p"
5379 					" cq->icid = 0x%x\n",
5380 					resp->flags, qp, qp->id, cq, cq->icid );
5381 			}
5382 		} else if (resp->flags & QLNXR_RESP_RDMA) {
5383 			QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5384 				"qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5385 				resp->flags, qp, qp->id, cq, cq->icid );
5386 		}
5387 #else
5388 		wc_status = IB_WC_SUCCESS;
5389 		wc->byte_len = le32_to_cpu(resp->length);
5390 
5391 		flags = resp->flags & QLNXR_RESP_RDMA_IMM;
5392 
5393 		switch (flags) {
5394 		case QLNXR_RESP_RDMA_IMM:
5395 			/* update opcode */
5396 			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
5397 			/* fall to set imm data */
5398 		case QLNXR_RESP_IMM:
5399 			wc->ex.imm_data =
5400 				le32_to_cpu(resp->imm_data_or_inv_r_Key);
5401 			wc->wc_flags |= IB_WC_WITH_IMM;
5402 			break;
5403 		case QLNXR_RESP_RDMA:
5404 			QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"
5405 				"qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",
5406 				resp->flags, qp, qp->id, cq, cq->icid );
5407 			break;
5408 		default:
5409 			/* valid configuration, but nothing todo here */
5410 			;
5411 		}
5412 #endif /* #if __FreeBSD_version >= 1102000 */
5413 
5414 		break;
5415 	default:
5416 		wc_status = IB_WC_GENERAL_ERR;
5417 	}
5418 
5419 	/* fill WC */
5420 	wc->status = wc_status;
5421 	wc->vendor_err = 0;
5422 	wc->src_qp = qp->id;
5423 	wc->qp = &qp->ibqp;
5424 	wc->wr_id = wr_id;
5425 
5426 	QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status);
5427 
5428 	return;
5429 }
5430 
5431 static int
5432 process_resp_one_srq(struct qlnxr_dev *dev,
5433 	struct qlnxr_qp *qp,
5434 	struct qlnxr_cq *cq,
5435 	struct ib_wc *wc,
5436 	struct rdma_cqe_responder *resp)
5437 {
5438 	struct qlnxr_srq	*srq = qp->srq;
5439 	u64			wr_id;
5440 	qlnx_host_t		*ha = dev->ha;
5441 
5442 	QL_DPRINT12(ha, "enter\n");
5443 
5444 	wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo);
5445 
5446 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5447 		wc->status = IB_WC_WR_FLUSH_ERR;
5448 		wc->vendor_err = 0;
5449 		wc->wr_id = wr_id;
5450 		wc->byte_len = 0;
5451 		wc->src_qp = qp->id;
5452 		wc->qp = &qp->ibqp;
5453 		wc->wr_id = wr_id;
5454 	} else {
5455 		__process_resp_one(dev, qp, cq, wc, resp, wr_id);
5456 	}
5457 
5458 	/* PBL is maintained in case of WR granularity.
5459 	 * So increment WR consumer after consuming WR
5460 	 */
5461 	srq->hw_srq.wr_cons_cnt++;
5462 
5463 	QL_DPRINT12(ha, "exit\n");
5464 	return 1;
5465 }
5466 
5467 static int
5468 process_resp_one(struct qlnxr_dev *dev,
5469 	struct qlnxr_qp *qp,
5470 	struct qlnxr_cq *cq,
5471 	struct ib_wc *wc,
5472 	struct rdma_cqe_responder *resp)
5473 {
5474 	qlnx_host_t	*ha = dev->ha;
5475 	u64		wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5476 
5477 	QL_DPRINT12(ha, "enter\n");
5478 
5479 	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
5480 
5481 	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5482 		ecore_chain_consume(&qp->rq.pbl);
5483 	qlnxr_inc_sw_cons(&qp->rq);
5484 
5485 	QL_DPRINT12(ha, "exit\n");
5486 	return 1;
5487 }
5488 
5489 static int
5490 process_resp_flush(struct qlnxr_qp *qp,
5491 	int num_entries,
5492 	struct ib_wc *wc,
5493 	u16 hw_cons)
5494 {
5495 	u16		cnt = 0;
5496 	qlnx_host_t	*ha = qp->dev->ha;
5497 
5498 	QL_DPRINT12(ha, "enter\n");
5499 
5500 	while (num_entries && qp->rq.wqe_cons != hw_cons) {
5501 		/* fill WC */
5502 		wc->status = IB_WC_WR_FLUSH_ERR;
5503 		wc->vendor_err = 0;
5504 		wc->wc_flags = 0;
5505 		wc->src_qp = qp->id;
5506 		wc->byte_len = 0;
5507 		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
5508 		wc->qp = &qp->ibqp;
5509 		num_entries--;
5510 		wc++;
5511 		cnt++;
5512 		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
5513 			ecore_chain_consume(&qp->rq.pbl);
5514 		qlnxr_inc_sw_cons(&qp->rq);
5515 	}
5516 
5517 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5518 	return cnt;
5519 }
5520 
5521 static void
5522 try_consume_resp_cqe(struct qlnxr_cq *cq,
5523 	struct qlnxr_qp *qp,
5524 	struct rdma_cqe_responder *resp,
5525 	int *update)
5526 {
5527 	if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
5528 		consume_cqe(cq);
5529 		*update |= 1;
5530 	}
5531 }
5532 
5533 static int
5534 qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev,
5535 	struct qlnxr_qp *qp,
5536 	struct qlnxr_cq *cq,
5537 	int num_entries,
5538 	struct ib_wc *wc,
5539 	struct rdma_cqe_responder *resp,
5540 	int *update)
5541 {
5542 	int		cnt;
5543 	qlnx_host_t	*ha = dev->ha;
5544 
5545 	QL_DPRINT12(ha, "enter\n");
5546 
5547 	cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
5548 	consume_cqe(cq);
5549 	*update |= 1;
5550 
5551 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5552 	return cnt;
5553 }
5554 
5555 static int
5556 qlnxr_poll_cq_resp(struct qlnxr_dev *dev,
5557 	struct qlnxr_qp *qp,
5558 	struct qlnxr_cq *cq,
5559 	int num_entries,
5560 	struct ib_wc *wc,
5561 	struct rdma_cqe_responder *resp,
5562 	int *update)
5563 {
5564 	int		cnt;
5565 	qlnx_host_t	*ha = dev->ha;
5566 
5567 	QL_DPRINT12(ha, "enter\n");
5568 
5569 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
5570 		cnt = process_resp_flush(qp, num_entries, wc,
5571 				resp->rq_cons);
5572 		try_consume_resp_cqe(cq, qp, resp, update);
5573 	} else {
5574 		cnt = process_resp_one(dev, qp, cq, wc, resp);
5575 		consume_cqe(cq);
5576 		*update |= 1;
5577 	}
5578 
5579 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);
5580 	return cnt;
5581 }
5582 
5583 static void
5584 try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp,
5585 	struct rdma_cqe_requester *req, int *update)
5586 {
5587 	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
5588 		consume_cqe(cq);
5589 		*update |= 1;
5590 	}
5591 }
5592 
5593 static void
5594 doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags)
5595 {
5596 	uint64_t	reg_addr;
5597 	qlnx_host_t	*ha = dev->ha;
5598 
5599 	QL_DPRINT12(ha, "enter\n");
5600 
5601 	wmb();
5602 	cq->db.data.agg_flags = flags;
5603 	cq->db.data.value = cpu_to_le32(cons);
5604 
5605 	reg_addr = (uint64_t)((uint8_t *)cq->db_addr -
5606 				(uint8_t *)(ha->cdev.doorbells));
5607 
5608 	bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw);
5609 	bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);
5610 
5611 	QL_DPRINT12(ha, "exit\n");
5612 	return;
5613 
5614 //#ifdef __LP64__
5615 //	writeq(cq->db.raw, cq->db_addr);
5616 //#else
5617 	/* Note that since the FW allows 64 bit write only, in 32bit systems
5618 	 * the value of db_addr must be low enough. This is currently not
5619 	 * enforced.
5620 	 */
5621 //	writel(cq->db.raw & 0xffffffff, cq->db_addr);
5622 //	mmiowb();
5623 //#endif
5624 }
5625 
5626 static int
5627 is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe)
5628 {
5629 	struct rdma_cqe_requester *resp_cqe = &cqe->req;
5630 	return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) ==
5631 			cq->pbl_toggle;
5632 }
5633 
5634 int
5635 qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
5636 {
5637 	struct qlnxr_cq	*cq = get_qlnxr_cq(ibcq);
5638 	struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
5639 	int		done = 0;
5640 	union rdma_cqe	*cqe = cq->latest_cqe;
5641 	int 		update = 0;
5642 	u32		old_cons, new_cons;
5643 	unsigned long	flags;
5644 	qlnx_host_t	*ha = dev->ha;
5645 
5646 	QL_DPRINT12(ha, "enter\n");
5647 
5648 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5649 		return -EINVAL;
5650 
5651 	if (cq->destroyed) {
5652 		QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n",
5653 			cq, cq->icid);
5654 		return 0;
5655 	}
5656 
5657 	if (cq->cq_type == QLNXR_CQ_TYPE_GSI)
5658 		return qlnxr_gsi_poll_cq(ibcq, num_entries, wc);
5659 
5660 	spin_lock_irqsave(&cq->cq_lock, flags);
5661 
5662 	old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5663 
5664 	while (num_entries && is_valid_cqe(cq, cqe)) {
5665 		int cnt = 0;
5666 		struct qlnxr_qp *qp;
5667 		struct rdma_cqe_requester *resp_cqe;
5668 		enum rdma_cqe_type cqe_type;
5669 
5670 		/* prevent speculative reads of any field of CQE */
5671 		rmb();
5672 
5673 		resp_cqe = &cqe->req;
5674 		qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi,
5675 						resp_cqe->qp_handle.lo);
5676 
5677 		if (!qp) {
5678 			QL_DPRINT11(ha, "qp = NULL\n");
5679 			break;
5680 		}
5681 
5682 		wc->qp = &qp->ibqp;
5683 
5684 		cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
5685 
5686 		switch (cqe_type) {
5687 		case RDMA_CQE_TYPE_REQUESTER:
5688 			cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries,
5689 					wc, &cqe->req);
5690 			try_consume_req_cqe(cq, qp, &cqe->req, &update);
5691 			break;
5692 		case RDMA_CQE_TYPE_RESPONDER_RQ:
5693 			cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries,
5694 					wc, &cqe->resp, &update);
5695 			break;
5696 		case RDMA_CQE_TYPE_RESPONDER_SRQ:
5697 			cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries,
5698 					wc, &cqe->resp, &update);
5699 			break;
5700 		case RDMA_CQE_TYPE_INVALID:
5701 		default:
5702 			QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type);
5703 			break;
5704 		}
5705 		num_entries -= cnt;
5706 		wc += cnt;
5707 		done += cnt;
5708 
5709 		cqe = cq->latest_cqe;
5710 	}
5711 	new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);
5712 
5713 	cq->cq_cons += new_cons - old_cons;
5714 
5715 	if (update) {
5716 		/* doorbell notifies abount latest VALID entry,
5717 		 * but chain already point to the next INVALID one
5718 		 */
5719 		doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags);
5720 		QL_DPRINT12(ha, "cq = %p cons = 0x%x "
5721 			"arm_flags = 0x%x db.icid = 0x%x\n", cq,
5722 			(cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid);
5723 	}
5724 
5725 	spin_unlock_irqrestore(&cq->cq_lock, flags);
5726 
5727 	QL_DPRINT12(ha, "exit\n");
5728 
5729 	return done;
5730 }
5731 
5732 int
5733 qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
5734 {
5735         struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
5736         unsigned long sflags;
5737         struct qlnxr_dev *dev;
5738 	qlnx_host_t	*ha;
5739 
5740 	dev = get_qlnxr_dev((ibcq->device));
5741 	ha = dev->ha;
5742 
5743 	QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x "
5744 		"cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq,
5745 		flags, cq, cq->cq_cons, cq->cq_type);
5746 
5747 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
5748 		return -EINVAL;
5749 
5750 	if (cq->destroyed) {
5751 		QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n",
5752 			cq, cq->icid);
5753 		return -EINVAL;
5754 	}
5755 
5756         if (cq->cq_type == QLNXR_CQ_TYPE_GSI) {
5757                 return 0;
5758         }
5759 
5760         spin_lock_irqsave(&cq->cq_lock, sflags);
5761 
5762         cq->arm_flags = 0;
5763 
5764         if (flags & IB_CQ_SOLICITED) {
5765                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
5766         }
5767         if (flags & IB_CQ_NEXT_COMP) {
5768                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
5769         }
5770 
5771         doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags);
5772 
5773         spin_unlock_irqrestore(&cq->cq_lock, sflags);
5774 
5775 	QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags);
5776         return 0;
5777 }
5778 
5779 static struct qlnxr_mr *
5780 __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len)
5781 {
5782 	struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
5783 	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
5784 	struct qlnxr_mr *mr;
5785 	int		rc = -ENOMEM;
5786 	qlnx_host_t	*ha;
5787 
5788 	ha = dev->ha;
5789 
5790 	QL_DPRINT12(ha, "enter ibpd = %p pd = %p "
5791 		" pd_id = %d max_page_list_len = %d\n",
5792 		ibpd, pd, pd->pd_id, max_page_list_len);
5793 
5794 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
5795 	if (!mr) {
5796 		QL_DPRINT11(ha, "kzalloc(mr) failed\n");
5797 		return ERR_PTR(rc);
5798 	}
5799 
5800 	mr->dev = dev;
5801 	mr->type = QLNXR_MR_FRMR;
5802 
5803 	rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len,
5804 				  1 /* allow dual layer pbl */);
5805 	if (rc) {
5806 		QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n");
5807 		goto err0;
5808 	}
5809 
5810 	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
5811 	if (rc) {
5812 		QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n");
5813 		goto err0;
5814 	}
5815 
5816 	/* index only, 18 bit long, lkey = itid << 8 | key */
5817 	mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR;
5818 	mr->hw_mr.key = 0;
5819 	mr->hw_mr.pd = pd->pd_id;
5820 	mr->hw_mr.local_read = 1;
5821 	mr->hw_mr.local_write = 0;
5822 	mr->hw_mr.remote_read = 0;
5823 	mr->hw_mr.remote_write = 0;
5824 	mr->hw_mr.remote_atomic = 0;
5825 	mr->hw_mr.mw_bind = false; /* TBD MW BIND */
5826 	mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */
5827 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
5828 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
5829 	mr->hw_mr.fbo = 0;
5830 	mr->hw_mr.length = 0;
5831 	mr->hw_mr.vaddr = 0;
5832 	mr->hw_mr.zbva = false; /* TBD figure when this should be true */
5833 	mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */
5834 	mr->hw_mr.dma_mr = false;
5835 
5836 	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
5837 	if (rc) {
5838 		QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n");
5839 		goto err1;
5840 	}
5841 
5842 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
5843 	mr->ibmr.rkey = mr->ibmr.lkey;
5844 
5845 	QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n",
5846 		mr, mr->ibmr.lkey);
5847 
5848 	return mr;
5849 
5850 err1:
5851 	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
5852 err0:
5853 	kfree(mr);
5854 
5855 	QL_DPRINT12(ha, "exit\n");
5856 
5857 	return ERR_PTR(rc);
5858 }
5859 
5860 #if __FreeBSD_version >= 1102000
5861 
5862 struct ib_mr *
5863 qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg)
5864 {
5865 	struct qlnxr_dev *dev;
5866 	struct qlnxr_mr *mr;
5867 	qlnx_host_t     *ha;
5868 
5869 	dev = get_qlnxr_dev(ibpd->device);
5870 	ha = dev->ha;
5871 
5872 	QL_DPRINT12(ha, "enter\n");
5873 
5874 	if (mr_type != IB_MR_TYPE_MEM_REG)
5875 		return ERR_PTR(-EINVAL);
5876 
5877 	mr = __qlnxr_alloc_mr(ibpd, max_num_sg);
5878 
5879 	if (IS_ERR(mr))
5880 		return ERR_PTR(-EINVAL);
5881 
5882 	QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr);
5883 
5884 	return &mr->ibmr;
5885 }
5886 
5887 static int
5888 qlnxr_set_page(struct ib_mr *ibmr, u64 addr)
5889 {
5890 	struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5891 	struct qlnxr_pbl *pbl_table;
5892 	struct regpair *pbe;
5893 	struct qlnxr_dev *dev;
5894 	qlnx_host_t     *ha;
5895 	u32 pbes_in_page;
5896 
5897 	dev = mr->dev;
5898 	ha = dev->ha;
5899 
5900 	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
5901 		QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages);
5902 		return -ENOMEM;
5903 	}
5904 
5905 	QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages,
5906 		((void *)addr));
5907 
5908 	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
5909 	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
5910 	pbe = (struct regpair *)pbl_table->va;
5911 	pbe +=  mr->npages % pbes_in_page;
5912 	pbe->lo = cpu_to_le32((u32)addr);
5913 	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
5914 
5915 	mr->npages++;
5916 
5917 	QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages,
5918 		((void *)addr));
5919 	return 0;
5920 }
5921 
5922 int
5923 qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
5924 	int sg_nents, unsigned int *sg_offset)
5925 {
5926 	int             ret;
5927 	struct qlnxr_mr *mr = get_qlnxr_mr(ibmr);
5928 	qlnx_host_t     *ha;
5929 
5930 	if (mr == NULL)
5931 		return (-1);
5932 
5933 	if (mr->dev == NULL)
5934 		return (-1);
5935 
5936 	ha = mr->dev->ha;
5937 
5938 	QL_DPRINT12(ha, "enter\n");
5939 
5940 	mr->npages = 0;
5941 	qlnx_handle_completed_mrs(mr->dev, &mr->info);
5942 
5943 	ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page);
5944 
5945 	QL_DPRINT12(ha, "exit ret = %d\n", ret);
5946 
5947 	return (ret);
5948 }
5949 
5950 #else
5951 
5952 struct ib_mr *
5953 qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len)
5954 {
5955 	struct qlnxr_dev *dev;
5956 	struct qlnxr_mr *mr;
5957 	qlnx_host_t	*ha;
5958 	struct ib_mr *ibmr = NULL;
5959 
5960 	dev = get_qlnxr_dev((ibpd->device));
5961 	ha = dev->ha;
5962 
5963 	QL_DPRINT12(ha, "enter\n");
5964 
5965 	mr = __qlnxr_alloc_mr(ibpd, max_page_list_len);
5966 
5967 	if (IS_ERR(mr)) {
5968 		ibmr = ERR_PTR(-EINVAL);
5969 	} else {
5970 		ibmr = &mr->ibmr;
5971 	}
5972 
5973 	QL_DPRINT12(ha, "exit %p\n", ibmr);
5974 	return (ibmr);
5975 }
5976 
5977 void
5978 qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list)
5979 {
5980 	struct qlnxr_fast_reg_page_list *frmr_list;
5981 
5982 	frmr_list = get_qlnxr_frmr_list(page_list);
5983 
5984 	free_mr_info(frmr_list->dev, &frmr_list->info);
5985 
5986 	kfree(frmr_list->ibfrpl.page_list);
5987 	kfree(frmr_list);
5988 
5989 	return;
5990 }
5991 
5992 struct ib_fast_reg_page_list *
5993 qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len)
5994 {
5995 	struct qlnxr_fast_reg_page_list *frmr_list = NULL;
5996 	struct qlnxr_dev		*dev;
5997 	int				size = page_list_len * sizeof(u64);
5998 	int				rc = -ENOMEM;
5999 	qlnx_host_t			*ha;
6000 
6001 	dev = get_qlnxr_dev(ibdev);
6002 	ha = dev->ha;
6003 
6004 	QL_DPRINT12(ha, "enter\n");
6005 
6006 	frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL);
6007 	if (!frmr_list) {
6008 		QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n");
6009 		goto err;
6010 	}
6011 
6012 	frmr_list->dev = dev;
6013 	frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL);
6014 	if (!frmr_list->ibfrpl.page_list) {
6015 		QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n");
6016 		goto err0;
6017 	}
6018 
6019 	rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len,
6020 			  1 /* allow dual layer pbl */);
6021 	if (rc)
6022 		goto err1;
6023 
6024 	QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl);
6025 
6026 	return &frmr_list->ibfrpl;
6027 
6028 err1:
6029 	kfree(frmr_list->ibfrpl.page_list);
6030 err0:
6031 	kfree(frmr_list);
6032 err:
6033 	QL_DPRINT12(ha, "exit with error\n");
6034 
6035 	return ERR_PTR(rc);
6036 }
6037 
6038 static int
6039 qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6040 	int buf_cnt, uint64_t *total_size)
6041 {
6042 	u64 size = 0;
6043 
6044 	*total_size = 0;
6045 
6046 	if (!buf_cnt || buf_list == NULL) {
6047 		QL_DPRINT11(ha,
6048 			"failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt);
6049 		return (-1);
6050 	}
6051 
6052 	size = buf_list->size;
6053 
6054 	if (!size) {
6055 		QL_DPRINT11(ha,
6056 			"failed buf_list = %p buf_cnt = %d"
6057 			" buf_list->size = 0\n", buf_list, buf_cnt);
6058 		return (-1);
6059 	}
6060 
6061 	while (buf_cnt) {
6062 		*total_size += buf_list->size;
6063 
6064 		if (buf_list->size != size) {
6065 			QL_DPRINT11(ha,
6066 				"failed buf_list = %p buf_cnt = %d"
6067 				" all buffers should have same size\n",
6068 				buf_list, buf_cnt);
6069 			return (-1);
6070 		}
6071 
6072 		buf_list++;
6073 		buf_cnt--;
6074 	}
6075 	return (0);
6076 }
6077 
6078 static size_t
6079 qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list,
6080 	int buf_cnt)
6081 {
6082 	int	i;
6083 	size_t	num_pages = 0;
6084 	u64	size;
6085 
6086 	for (i = 0; i < buf_cnt; i++) {
6087 		size = 0;
6088 		while (size < buf_list->size) {
6089 			size += PAGE_SIZE;
6090 			num_pages++;
6091 		}
6092 		buf_list++;
6093 	}
6094 	return (num_pages);
6095 }
6096 
6097 static void
6098 qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev,
6099 	struct ib_phys_buf *buf_list, int buf_cnt,
6100 	struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info)
6101 {
6102 	struct regpair		*pbe;
6103 	struct qlnxr_pbl	*pbl_tbl;
6104 	int			pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
6105 	qlnx_host_t		*ha;
6106         int                     i;
6107 	u64			pbe_addr;
6108 
6109 	ha = dev->ha;
6110 
6111 	QL_DPRINT12(ha, "enter\n");
6112 
6113 	if (!pbl_info) {
6114 		QL_DPRINT11(ha, "PBL_INFO not initialized\n");
6115 		return;
6116 	}
6117 
6118 	if (!pbl_info->num_pbes) {
6119 		QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");
6120 		return;
6121 	}
6122 
6123 	/* If we have a two layered pbl, the first pbl points to the rest
6124 	 * of the pbls and the first entry lays on the second pbl in the table
6125 	 */
6126 	if (pbl_info->two_layered)
6127 		pbl_tbl = &pbl[1];
6128 	else
6129 		pbl_tbl = pbl;
6130 
6131 	pbe = (struct regpair *)pbl_tbl->va;
6132 	if (!pbe) {
6133 		QL_DPRINT12(ha, "pbe is NULL\n");
6134 		return;
6135 	}
6136 
6137 	pbe_cnt = 0;
6138 
6139 	for (i = 0; i < buf_cnt; i++) {
6140 		pages = buf_list->size >> PAGE_SHIFT;
6141 
6142 		for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
6143 			/* store the page address in pbe */
6144 
6145 			pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt);
6146 
6147 			pbe->lo = cpu_to_le32((u32)pbe_addr);
6148 			pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32)));
6149 
6150 			QL_DPRINT12(ha, "Populate pbl table:"
6151 				" pbe->addr=0x%x:0x%x "
6152 				" pbe_cnt = %d total_num_pbes=%d"
6153 				" pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,
6154 				total_num_pbes, pbe);
6155 
6156 			pbe_cnt ++;
6157 			total_num_pbes ++;
6158 			pbe++;
6159 
6160 			if (total_num_pbes == pbl_info->num_pbes)
6161 				return;
6162 
6163 			/* if the given pbl is full storing the pbes,
6164 			 * move to next pbl.  */
6165 
6166 			if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
6167 				pbl_tbl++;
6168 				pbe = (struct regpair *)pbl_tbl->va;
6169 				pbe_cnt = 0;
6170 			}
6171 		}
6172 		buf_list++;
6173 	}
6174 	QL_DPRINT12(ha, "exit\n");
6175 	return;
6176 }
6177 
6178 struct ib_mr *
6179 qlnxr_reg_kernel_mr(struct ib_pd *ibpd,
6180 	struct ib_phys_buf *buf_list,
6181 	int buf_cnt, int acc, u64 *iova_start)
6182 {
6183 	int		rc = -ENOMEM;
6184 	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));
6185 	struct qlnxr_mr *mr;
6186 	struct qlnxr_pd *pd;
6187 	qlnx_host_t	*ha;
6188 	size_t		num_pages = 0;
6189 	uint64_t	length;
6190 
6191 	ha = dev->ha;
6192 
6193 	QL_DPRINT12(ha, "enter\n");
6194 
6195 	pd = get_qlnxr_pd(ibpd);
6196 
6197 	QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d,"
6198 		" iova_start = %p, acc = %d\n",
6199 		pd->pd_id, buf_list, buf_cnt, iova_start, acc);
6200 
6201 	//if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {
6202 	//	QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&"
6203 	//		" !(acc & IB_ACCESS_LOCAL_WRITE))\n");
6204 	//	return ERR_PTR(-EINVAL);
6205 	//}
6206 
6207 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
6208 	if (!mr) {
6209 		QL_DPRINT11(ha, "kzalloc(mr) failed\n");
6210 		return ERR_PTR(rc);
6211 	}
6212 
6213 	mr->type = QLNXR_MR_KERNEL;
6214 	mr->iova_start = iova_start;
6215 
6216 	rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length);
6217 	if (rc)
6218 		goto err0;
6219 
6220 	num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt);
6221 	if (!num_pages)
6222 		goto err0;
6223 
6224 	rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1);
6225 	if (rc) {
6226 		QL_DPRINT11(ha,
6227 			"qlnxr_init_mr_info failed [%d]\n", rc);
6228 		goto err1;
6229 	}
6230 
6231 	qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table,
6232 		   &mr->info.pbl_info);
6233 
6234 	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
6235 
6236 	if (rc) {
6237 		QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);
6238 		goto err1;
6239 	}
6240 
6241 	/* index only, 18 bit long, lkey = itid << 8 | key */
6242 	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;
6243 	mr->hw_mr.key = 0;
6244 	mr->hw_mr.pd = pd->pd_id;
6245 	mr->hw_mr.local_read = 1;
6246 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
6247 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
6248 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
6249 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
6250 	mr->hw_mr.mw_bind = false; /* TBD MW BIND */
6251 	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
6252 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
6253 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
6254 	mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */
6255 
6256 	mr->hw_mr.fbo = 0;
6257 
6258 	mr->hw_mr.length = length;
6259 	mr->hw_mr.vaddr = (uint64_t)iova_start;
6260 	mr->hw_mr.zbva = false; /* TBD figure when this should be true */
6261 	mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */
6262 	mr->hw_mr.dma_mr = false;
6263 
6264 	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
6265 	if (rc) {
6266 		QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);
6267 		goto err2;
6268 	}
6269 
6270 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6271 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
6272 		mr->hw_mr.remote_atomic)
6273 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
6274 
6275 	QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey);
6276 
6277 	return (&mr->ibmr);
6278 
6279 err2:
6280 	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
6281 err1:
6282 	qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
6283 err0:
6284 	kfree(mr);
6285 
6286 	QL_DPRINT12(ha, "exit [%d]\n", rc);
6287 	return (ERR_PTR(rc));
6288 }
6289 
6290 #endif /* #if __FreeBSD_version >= 1102000 */
6291 
6292 struct ib_ah *
6293 #if __FreeBSD_version >= 1102000
6294 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
6295 	struct ib_udata *udata)
6296 #else
6297 qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
6298 #endif /* #if __FreeBSD_version >= 1102000 */
6299 {
6300 	struct qlnxr_dev *dev;
6301 	qlnx_host_t	*ha;
6302 	struct qlnxr_ah *ah;
6303 
6304 	dev = get_qlnxr_dev((ibpd->device));
6305 	ha = dev->ha;
6306 
6307 	QL_DPRINT12(ha, "in create_ah\n");
6308 
6309 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
6310 	if (!ah) {
6311 		QL_DPRINT12(ha, "no address handle can be allocated\n");
6312 		return ERR_PTR(-ENOMEM);
6313 	}
6314 
6315 	ah->attr = *attr;
6316 
6317 	return &ah->ibah;
6318 }
6319 
6320 int
6321 qlnxr_destroy_ah(struct ib_ah *ibah)
6322 {
6323 	struct qlnxr_dev *dev;
6324 	qlnx_host_t     *ha;
6325 	struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
6326 
6327 	dev = get_qlnxr_dev((ibah->device));
6328 	ha = dev->ha;
6329 
6330 	QL_DPRINT12(ha, "in destroy_ah\n");
6331 
6332 	kfree(ah);
6333 	return 0;
6334 }
6335 
6336 int
6337 qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6338 {
6339 	struct qlnxr_dev *dev;
6340 	qlnx_host_t     *ha;
6341 
6342 	dev = get_qlnxr_dev((ibah->device));
6343 	ha = dev->ha;
6344 	QL_DPRINT12(ha, "Query AH not supported\n");
6345 	return -EINVAL;
6346 }
6347 
6348 int
6349 qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr)
6350 {
6351 	struct qlnxr_dev *dev;
6352 	qlnx_host_t     *ha;
6353 
6354 	dev = get_qlnxr_dev((ibah->device));
6355 	ha = dev->ha;
6356 	QL_DPRINT12(ha, "Modify AH not supported\n");
6357 	return -ENOSYS;
6358 }
6359 
6360 #if __FreeBSD_version >= 1102000
6361 int
6362 qlnxr_process_mad(struct ib_device *ibdev,
6363 		int process_mad_flags,
6364 		u8 port_num,
6365 		const struct ib_wc *in_wc,
6366 		const struct ib_grh *in_grh,
6367 		const struct ib_mad_hdr *mad_hdr,
6368 		size_t in_mad_size,
6369 		struct ib_mad_hdr *out_mad,
6370 		size_t *out_mad_size,
6371 		u16 *out_mad_pkey_index)
6372 
6373 #else
6374 
6375 int
6376 qlnxr_process_mad(struct ib_device *ibdev,
6377                         int process_mad_flags,
6378                         u8 port_num,
6379                         struct ib_wc *in_wc,
6380                         struct ib_grh *in_grh,
6381                         struct ib_mad *in_mad,
6382                         struct ib_mad *out_mad)
6383 
6384 #endif /* #if __FreeBSD_version >= 1102000 */
6385 {
6386 	struct qlnxr_dev *dev;
6387 	qlnx_host_t	*ha;
6388 
6389 	dev = get_qlnxr_dev(ibdev);
6390 	ha = dev->ha;
6391 	QL_DPRINT12(ha, "process mad not supported\n");
6392 
6393 	return -ENOSYS;
6394 //	QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n",
6395 //               in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version,
6396 //               in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific,
6397 //               in_mad->mad_hdr.class_version, in_mad->mad_hdr.method,
6398 //               in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status);
6399 
6400 //	return IB_MAD_RESULT_SUCCESS;
6401 }
6402 
6403 #if __FreeBSD_version >= 1102000
6404 int
6405 qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num,
6406 	struct ib_port_immutable *immutable)
6407 {
6408 	struct qlnxr_dev        *dev;
6409 	qlnx_host_t             *ha;
6410 	struct ib_port_attr     attr;
6411 	int                     err;
6412 
6413 	dev = get_qlnxr_dev(ibdev);
6414 	ha = dev->ha;
6415 
6416 	QL_DPRINT12(ha, "enter\n");
6417 
6418 	err = qlnxr_query_port(ibdev, port_num, &attr);
6419 	if (err)
6420 		return err;
6421 
6422 	if (QLNX_IS_IWARP(dev)) {
6423 		immutable->pkey_tbl_len = 1;
6424 		immutable->gid_tbl_len = 1;
6425 		immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
6426 		immutable->max_mad_size = 0;
6427 	} else {
6428 		immutable->pkey_tbl_len = attr.pkey_tbl_len;
6429 		immutable->gid_tbl_len = attr.gid_tbl_len;
6430 		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
6431 		immutable->max_mad_size = IB_MGMT_MAD_SIZE;
6432 	}
6433 
6434 	QL_DPRINT12(ha, "exit\n");
6435 	return 0;
6436 }
6437 #endif /* #if __FreeBSD_version > 1102000 */
6438 
6439 /***** iWARP related functions *************/
6440 
6441 static void
6442 qlnxr_iw_mpa_request(void *context,
6443 	struct ecore_iwarp_cm_event_params *params)
6444 {
6445 	struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context;
6446 	struct qlnxr_dev *dev = listener->dev;
6447 	struct qlnxr_iw_ep *ep;
6448 	struct iw_cm_event event;
6449 	struct sockaddr_in *laddr;
6450 	struct sockaddr_in *raddr;
6451 	qlnx_host_t	*ha;
6452 
6453 	ha = dev->ha;
6454 
6455 	QL_DPRINT12(ha, "enter\n");
6456 
6457 	if (params->cm_info->ip_version != ECORE_TCP_IPV4) {
6458 		QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n",
6459 			params->cm_info->ip_version);
6460 		return;
6461 	}
6462 
6463 	ep = kzalloc(sizeof(*ep), GFP_ATOMIC);
6464 
6465 	if (!ep) {
6466 		QL_DPRINT11(ha, "kzalloc{ep) failed\n");
6467 		return;
6468 	}
6469 
6470 	ep->dev = dev;
6471 	ep->ecore_context = params->ep_context;
6472 
6473 	memset(&event, 0, sizeof(event));
6474 
6475 	event.event = IW_CM_EVENT_CONNECT_REQUEST;
6476 	event.status = params->status;
6477 
6478 	laddr = (struct sockaddr_in *)&event.local_addr;
6479 	raddr = (struct sockaddr_in *)&event.remote_addr;
6480 
6481 	laddr->sin_family = AF_INET;
6482 	raddr->sin_family = AF_INET;
6483 
6484 	laddr->sin_port = htons(params->cm_info->local_port);
6485 	raddr->sin_port = htons(params->cm_info->remote_port);
6486 
6487 	laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]);
6488 	raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]);
6489 
6490 	event.provider_data = (void *)ep;
6491 	event.private_data = (void *)params->cm_info->private_data;
6492 	event.private_data_len = (u8)params->cm_info->private_data_len;
6493 
6494 #if __FreeBSD_version >= 1100000
6495 	event.ord = params->cm_info->ord;
6496 	event.ird = params->cm_info->ird;
6497 #endif /* #if __FreeBSD_version >= 1100000 */
6498 
6499 	listener->cm_id->event_handler(listener->cm_id, &event);
6500 
6501 	QL_DPRINT12(ha, "exit\n");
6502 
6503 	return;
6504 }
6505 
6506 static void
6507 qlnxr_iw_issue_event(void *context,
6508 	 struct ecore_iwarp_cm_event_params *params,
6509 	 enum iw_cm_event_type event_type,
6510 	 char *str)
6511 {
6512 	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6513 	struct qlnxr_dev *dev = ep->dev;
6514 	struct iw_cm_event event;
6515 	qlnx_host_t	*ha;
6516 
6517 	ha = dev->ha;
6518 
6519 	QL_DPRINT12(ha, "enter\n");
6520 
6521 	memset(&event, 0, sizeof(event));
6522 	event.status = params->status;
6523 	event.event = event_type;
6524 
6525 	if (params->cm_info != NULL) {
6526 #if __FreeBSD_version >= 1100000
6527 		event.ird = params->cm_info->ird;
6528 		event.ord = params->cm_info->ord;
6529 		QL_DPRINT12(ha, "ord=[%d] \n", event.ord);
6530 		QL_DPRINT12(ha, "ird=[%d] \n", event.ird);
6531 #endif /* #if __FreeBSD_version >= 1100000 */
6532 
6533 		event.private_data_len = params->cm_info->private_data_len;
6534 		event.private_data = (void *)params->cm_info->private_data;
6535 		QL_DPRINT12(ha, "private_data_len=[%d] \n",
6536 			event.private_data_len);
6537 	}
6538 
6539 	QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str);
6540 	QL_DPRINT12(ha, "status=[%d] \n", event.status);
6541 
6542 	if (ep) {
6543 		if (ep->cm_id)
6544 			ep->cm_id->event_handler(ep->cm_id, &event);
6545 		else
6546 			QL_DPRINT11(ha, "ep->cm_id == NULL \n");
6547 	} else {
6548 		QL_DPRINT11(ha, "ep == NULL \n");
6549 	}
6550 
6551 	QL_DPRINT12(ha, "exit\n");
6552 
6553 	return;
6554 }
6555 
6556 static void
6557 qlnxr_iw_close_event(void *context,
6558 	 struct ecore_iwarp_cm_event_params *params)
6559 {
6560 	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6561 	struct qlnxr_dev *dev = ep->dev;
6562 	qlnx_host_t	*ha;
6563 
6564 	ha = dev->ha;
6565 
6566 	QL_DPRINT12(ha, "enter\n");
6567 
6568 	if (ep->cm_id) {
6569 		qlnxr_iw_issue_event(context,
6570 				    params,
6571 				    IW_CM_EVENT_CLOSE,
6572 				    "IW_CM_EVENT_EVENT_CLOSE");
6573 		ep->cm_id->rem_ref(ep->cm_id);
6574 		ep->cm_id = NULL;
6575 	}
6576 
6577 	QL_DPRINT12(ha, "exit\n");
6578 
6579 	return;
6580 }
6581 
6582 #if __FreeBSD_version >= 1102000
6583 
6584 static void
6585 qlnxr_iw_passive_complete(void *context,
6586         struct ecore_iwarp_cm_event_params *params)
6587 {
6588         struct qlnxr_iw_ep      *ep = (struct qlnxr_iw_ep *)context;
6589         struct qlnxr_dev        *dev = ep->dev;
6590         qlnx_host_t             *ha;
6591 
6592         ha = dev->ha;
6593 
6594         /* We will only reach the following state if MPA_REJECT was called on
6595          * passive. In this case there will be no associated QP.
6596          */
6597         if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) {
6598                 QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n");
6599                 kfree(ep);
6600                 return;
6601         }
6602 
6603         /* We always issue an established event, however, ofed does not look
6604          * at event code for established. So if there was a failure, we follow
6605          * with close...
6606          */
6607         qlnxr_iw_issue_event(context,
6608                 params,
6609                 IW_CM_EVENT_ESTABLISHED,
6610                 "IW_CM_EVENT_ESTABLISHED");
6611 
6612         if (params->status < 0) {
6613                 qlnxr_iw_close_event(context, params);
6614         }
6615 
6616         return;
6617 }
6618 
6619 struct qlnxr_discon_work {
6620         struct work_struct work;
6621         struct qlnxr_iw_ep *ep;
6622         enum ecore_iwarp_event_type event;
6623         int status;
6624 };
6625 
6626 static void
6627 qlnxr_iw_disconnect_worker(struct work_struct *work)
6628 {
6629         struct qlnxr_discon_work *dwork =
6630                 container_of(work, struct qlnxr_discon_work, work);
6631         struct ecore_rdma_modify_qp_in_params qp_params = { 0 };
6632         struct qlnxr_iw_ep *ep = dwork->ep;
6633         struct qlnxr_dev *dev = ep->dev;
6634         struct qlnxr_qp *qp = ep->qp;
6635         struct iw_cm_event event;
6636 
6637         if (qp->destroyed) {
6638                 kfree(dwork);
6639                 qlnxr_iw_qp_rem_ref(&qp->ibqp);
6640                 return;
6641         }
6642 
6643         memset(&event, 0, sizeof(event));
6644         event.status = dwork->status;
6645         event.event = IW_CM_EVENT_DISCONNECT;
6646 
6647         /* Success means graceful disconnect was requested. modifying
6648          * to SQD is translated to graceful disconnect. O/w reset is sent
6649          */
6650         if (dwork->status)
6651                 qp_params.new_state = ECORE_ROCE_QP_STATE_ERR;
6652         else
6653                 qp_params.new_state = ECORE_ROCE_QP_STATE_SQD;
6654 
6655         kfree(dwork);
6656 
6657         if (ep->cm_id)
6658                 ep->cm_id->event_handler(ep->cm_id, &event);
6659 
6660         SET_FIELD(qp_params.modify_flags,
6661                   ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
6662 
6663         ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);
6664 
6665         qlnxr_iw_qp_rem_ref(&qp->ibqp);
6666 
6667         return;
6668 }
6669 
6670 void
6671 qlnxr_iw_disconnect_event(void *context,
6672         struct ecore_iwarp_cm_event_params *params)
6673 {
6674         struct qlnxr_discon_work *work;
6675         struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6676         struct qlnxr_dev *dev = ep->dev;
6677         struct qlnxr_qp *qp = ep->qp;
6678 
6679         work = kzalloc(sizeof(*work), GFP_ATOMIC);
6680         if (!work)
6681                 return;
6682 
6683         qlnxr_iw_qp_add_ref(&qp->ibqp);
6684         work->ep = ep;
6685         work->event = params->event;
6686         work->status = params->status;
6687 
6688         INIT_WORK(&work->work, qlnxr_iw_disconnect_worker);
6689         queue_work(dev->iwarp_wq, &work->work);
6690 
6691         return;
6692 }
6693 
6694 #endif /* #if __FreeBSD_version >= 1102000 */
6695 
6696 static int
6697 qlnxr_iw_mpa_reply(void *context,
6698 	struct ecore_iwarp_cm_event_params *params)
6699 {
6700         struct qlnxr_iw_ep	*ep = (struct qlnxr_iw_ep *)context;
6701         struct qlnxr_dev	*dev = ep->dev;
6702         struct ecore_iwarp_send_rtr_in rtr_in;
6703         int			rc;
6704 	qlnx_host_t		*ha;
6705 
6706 	ha = dev->ha;
6707 
6708 	QL_DPRINT12(ha, "enter\n");
6709 
6710 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6711 		return -EINVAL;
6712 
6713 	bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in));
6714         rtr_in.ep_context = params->ep_context;
6715 
6716         rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in);
6717 
6718 	QL_DPRINT12(ha, "exit rc = %d\n", rc);
6719         return rc;
6720 }
6721 
6722 void
6723 qlnxr_iw_qp_event(void *context,
6724 	struct ecore_iwarp_cm_event_params *params,
6725 	enum ib_event_type ib_event,
6726 	char *str)
6727 {
6728         struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6729         struct qlnxr_dev *dev = ep->dev;
6730         struct ib_qp *ibqp = &(ep->qp->ibqp);
6731         struct ib_event event;
6732 	qlnx_host_t	*ha;
6733 
6734 	ha = dev->ha;
6735 
6736 	QL_DPRINT12(ha,
6737 		"[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n",
6738 		context, params->event, str, ibqp->event_handler);
6739 
6740         if (ibqp->event_handler) {
6741                 event.event = ib_event;
6742                 event.device = ibqp->device;
6743                 event.element.qp = ibqp;
6744                 ibqp->event_handler(&event, ibqp->qp_context);
6745         }
6746 
6747 	return;
6748 }
6749 
6750 int
6751 qlnxr_iw_event_handler(void *context,
6752 	struct ecore_iwarp_cm_event_params *params)
6753 {
6754 	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6755 	struct qlnxr_dev *dev = ep->dev;
6756 	qlnx_host_t	*ha;
6757 
6758 	ha = dev->ha;
6759 
6760 	QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6761 		"enter\n", context, params->event);
6762 
6763 	switch (params->event) {
6764 	/* Passive side request received */
6765 	case ECORE_IWARP_EVENT_MPA_REQUEST:
6766 		qlnxr_iw_mpa_request(context, params);
6767 		break;
6768 
6769         case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY:
6770                 qlnxr_iw_mpa_reply(context, params);
6771                 break;
6772 
6773 	/* Passive side established ( ack on mpa response ) */
6774 	case ECORE_IWARP_EVENT_PASSIVE_COMPLETE:
6775 
6776 #if __FreeBSD_version >= 1102000
6777 
6778 		ep->during_connect = 0;
6779 		qlnxr_iw_passive_complete(context, params);
6780 
6781 #else
6782 		qlnxr_iw_issue_event(context,
6783 				    params,
6784 				    IW_CM_EVENT_ESTABLISHED,
6785 				    "IW_CM_EVENT_ESTABLISHED");
6786 #endif /* #if __FreeBSD_version >= 1102000 */
6787 		break;
6788 
6789 	/* Active side reply received */
6790 	case ECORE_IWARP_EVENT_ACTIVE_COMPLETE:
6791 		ep->during_connect = 0;
6792 		qlnxr_iw_issue_event(context,
6793 				    params,
6794 				    IW_CM_EVENT_CONNECT_REPLY,
6795 				    "IW_CM_EVENT_CONNECT_REPLY");
6796 		if (params->status < 0) {
6797 			struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context;
6798 
6799 			ep->cm_id->rem_ref(ep->cm_id);
6800 			ep->cm_id = NULL;
6801 		}
6802 		break;
6803 
6804 	case ECORE_IWARP_EVENT_DISCONNECT:
6805 
6806 #if __FreeBSD_version >= 1102000
6807 		qlnxr_iw_disconnect_event(context, params);
6808 #else
6809 		qlnxr_iw_issue_event(context,
6810 				    params,
6811 				    IW_CM_EVENT_DISCONNECT,
6812 				    "IW_CM_EVENT_DISCONNECT");
6813 		qlnxr_iw_close_event(context, params);
6814 #endif /* #if __FreeBSD_version >= 1102000 */
6815 		break;
6816 
6817 	case ECORE_IWARP_EVENT_CLOSE:
6818 		ep->during_connect = 0;
6819 		qlnxr_iw_close_event(context, params);
6820 		break;
6821 
6822         case ECORE_IWARP_EVENT_RQ_EMPTY:
6823                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6824                                  "IWARP_EVENT_RQ_EMPTY");
6825                 break;
6826 
6827         case ECORE_IWARP_EVENT_IRQ_FULL:
6828                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6829                                  "IWARP_EVENT_IRQ_FULL");
6830                 break;
6831 
6832         case ECORE_IWARP_EVENT_LLP_TIMEOUT:
6833                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6834                                  "IWARP_EVENT_LLP_TIMEOUT");
6835                 break;
6836 
6837         case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR:
6838                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6839                                  "IWARP_EVENT_REMOTE_PROTECTION_ERROR");
6840                 break;
6841 
6842         case ECORE_IWARP_EVENT_CQ_OVERFLOW:
6843                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6844                                  "QED_IWARP_EVENT_CQ_OVERFLOW");
6845                 break;
6846 
6847         case ECORE_IWARP_EVENT_QP_CATASTROPHIC:
6848                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6849                                  "QED_IWARP_EVENT_QP_CATASTROPHIC");
6850                 break;
6851 
6852         case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR:
6853                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR,
6854                                  "IWARP_EVENT_LOCAL_ACCESS_ERROR");
6855                 break;
6856 
6857         case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR:
6858                 qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL,
6859                                  "IWARP_EVENT_REMOTE_OPERATION_ERROR");
6860                 break;
6861 
6862         case ECORE_IWARP_EVENT_TERMINATE_RECEIVED:
6863 		QL_DPRINT12(ha, "Got terminate message"
6864 			" ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n");
6865                 break;
6866 
6867 	default:
6868 		QL_DPRINT12(ha,
6869 			"Unknown event [0x%x] received \n", params->event);
6870 		break;
6871 	};
6872 
6873 	QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "
6874 		"exit\n", context, params->event);
6875 	return 0;
6876 }
6877 
6878 static int
6879 qlnxr_addr4_resolve(struct qlnxr_dev *dev,
6880 			      struct sockaddr_in *src_in,
6881 			      struct sockaddr_in *dst_in,
6882 			      u8 *dst_mac)
6883 {
6884 	int rc;
6885 
6886 #if __FreeBSD_version >= 1100000
6887 	rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in,
6888 			dst_mac, NULL, NULL);
6889 #else
6890 	struct llentry *lle;
6891 
6892 	rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in,
6893 			dst_mac, &lle);
6894 #endif
6895 
6896 	QL_DPRINT12(dev->ha, "rc = %d "
6897 		"sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d "
6898 		"Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc,
6899 		dst_in->sin_len, dst_in->sin_family,
6900 		NIPQUAD((dst_in->sin_addr.s_addr)),
6901 		dst_mac[0], dst_mac[1], dst_mac[2],
6902 		dst_mac[3], dst_mac[4], dst_mac[5]);
6903 
6904 	return rc;
6905 }
6906 
6907 int
6908 qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
6909 {
6910 	struct qlnxr_dev *dev;
6911 	struct ecore_iwarp_connect_out out_params;
6912 	struct ecore_iwarp_connect_in in_params;
6913 	struct qlnxr_iw_ep *ep;
6914 	struct qlnxr_qp *qp;
6915 	struct sockaddr_in *laddr;
6916 	struct sockaddr_in *raddr;
6917 	int rc = 0;
6918 	qlnx_host_t	*ha;
6919 
6920 	dev = get_qlnxr_dev((cm_id->device));
6921 	ha = dev->ha;
6922 
6923 	QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] "
6924 		"enter \n", cm_id, conn_param);
6925 
6926 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
6927 		return -EINVAL;
6928 
6929 	qp = idr_find(&dev->qpidr, conn_param->qpn);
6930 
6931 	laddr = (struct sockaddr_in *)&cm_id->local_addr;
6932 	raddr = (struct sockaddr_in *)&cm_id->remote_addr;
6933 
6934 	QL_DPRINT12(ha,
6935 		"local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n",
6936 		NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port,
6937 		NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port);
6938 
6939 	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
6940 	if (!ep) {
6941 		QL_DPRINT11(ha, "struct qlnxr_iw_ep "
6942 			"alloc memory failed\n");
6943 		return -ENOMEM;
6944 	}
6945 
6946 	ep->dev = dev;
6947 	ep->qp = qp;
6948 	cm_id->add_ref(cm_id);
6949 	ep->cm_id = cm_id;
6950 
6951 	memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in));
6952 	memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out));
6953 
6954 	in_params.event_cb = qlnxr_iw_event_handler;
6955 	in_params.cb_context = ep;
6956 
6957 	in_params.cm_info.ip_version = ECORE_TCP_IPV4;
6958 
6959 	in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr);
6960 	in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr);
6961 	in_params.cm_info.remote_port = ntohs(raddr->sin_port);
6962 	in_params.cm_info.local_port = ntohs(laddr->sin_port);
6963 	in_params.cm_info.vlan = 0;
6964 	in_params.mss = dev->ha->ifp->if_mtu - 40;
6965 
6966 	QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] "
6967 		"local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d "
6968 		"vlan = %d\n",
6969 		NIPQUAD((in_params.cm_info.remote_ip[0])),
6970 		NIPQUAD((in_params.cm_info.local_ip[0])),
6971 		in_params.cm_info.remote_port, in_params.cm_info.local_port,
6972 		in_params.cm_info.vlan);
6973 
6974 	rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr);
6975 
6976 	if (rc) {
6977 		QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n");
6978 		goto err;
6979 	}
6980 
6981 	QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p"
6982 		" private_data_len=%d rq_psn=%d\n",
6983 		conn_param->ord, conn_param->ird, conn_param->private_data,
6984 		conn_param->private_data_len, qp->rq_psn);
6985 
6986 	in_params.cm_info.ord = conn_param->ord;
6987 	in_params.cm_info.ird = conn_param->ird;
6988 	in_params.cm_info.private_data = conn_param->private_data;
6989 	in_params.cm_info.private_data_len = conn_param->private_data_len;
6990 	in_params.qp = qp->ecore_qp;
6991 
6992 	memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);
6993 
6994 	rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params);
6995 
6996 	if (rc) {
6997 		QL_DPRINT12(ha, "ecore_iwarp_connect failed\n");
6998 		goto err;
6999 	}
7000 
7001 	QL_DPRINT12(ha, "exit\n");
7002 
7003 	return rc;
7004 
7005 err:
7006 	cm_id->rem_ref(cm_id);
7007 	kfree(ep);
7008 
7009 	QL_DPRINT12(ha, "exit [%d]\n", rc);
7010 	return rc;
7011 }
7012 
7013 int
7014 qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog)
7015 {
7016 	struct qlnxr_dev *dev;
7017 	struct qlnxr_iw_listener *listener;
7018 	struct ecore_iwarp_listen_in iparams;
7019 	struct ecore_iwarp_listen_out oparams;
7020 	struct sockaddr_in *laddr;
7021 	qlnx_host_t	*ha;
7022 	int rc;
7023 
7024 	dev = get_qlnxr_dev((cm_id->device));
7025 	ha = dev->ha;
7026 
7027 	QL_DPRINT12(ha, "enter\n");
7028 
7029 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7030 		return -EINVAL;
7031 
7032 	laddr = (struct sockaddr_in *)&cm_id->local_addr;
7033 
7034 	listener = kzalloc(sizeof(*listener), GFP_KERNEL);
7035 
7036 	if (listener == NULL) {
7037 		QL_DPRINT11(ha, "listener memory alloc failed\n");
7038 		return -ENOMEM;
7039 	}
7040 
7041 	listener->dev = dev;
7042 	cm_id->add_ref(cm_id);
7043 	listener->cm_id = cm_id;
7044 	listener->backlog = backlog;
7045 
7046 	memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in));
7047 	memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out));
7048 
7049 	iparams.cb_context = listener;
7050 	iparams.event_cb = qlnxr_iw_event_handler;
7051 	iparams.max_backlog = backlog;
7052 
7053 	iparams.ip_version = ECORE_TCP_IPV4;
7054 
7055 	iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr);
7056 	iparams.port = ntohs(laddr->sin_port);
7057 	iparams.vlan = 0;
7058 
7059 	QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n",
7060 		NIPQUAD((laddr->sin_addr.s_addr)),
7061 		laddr->sin_port, iparams.port);
7062 
7063 	rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams);
7064 	if (rc) {
7065 		QL_DPRINT11(ha,
7066 			"ecore_iwarp_create_listen failed rc = %d\n", rc);
7067 		goto err;
7068 	}
7069 
7070 	listener->ecore_handle = oparams.handle;
7071 	cm_id->provider_data = listener;
7072 
7073 	QL_DPRINT12(ha, "exit\n");
7074 	return rc;
7075 
7076 err:
7077 	cm_id->rem_ref(cm_id);
7078 	kfree(listener);
7079 
7080 	QL_DPRINT12(ha, "exit [%d]\n", rc);
7081 	return rc;
7082 }
7083 
7084 void
7085 qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id)
7086 {
7087 	struct qlnxr_iw_listener *listener = cm_id->provider_data;
7088 	struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device));
7089 	int rc = 0;
7090 	qlnx_host_t	*ha;
7091 
7092 	ha = dev->ha;
7093 
7094 	QL_DPRINT12(ha, "enter\n");
7095 
7096 	if (listener->ecore_handle)
7097 		rc = ecore_iwarp_destroy_listen(dev->rdma_ctx,
7098 				listener->ecore_handle);
7099 
7100 	cm_id->rem_ref(cm_id);
7101 
7102 	QL_DPRINT12(ha, "exit [%d]\n", rc);
7103 	return;
7104 }
7105 
7106 int
7107 qlnxr_iw_accept(struct iw_cm_id *cm_id,
7108 	struct iw_cm_conn_param *conn_param)
7109 {
7110 	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7111 	struct qlnxr_dev *dev = ep->dev;
7112 	struct qlnxr_qp *qp;
7113 	struct ecore_iwarp_accept_in params;
7114 	int rc;
7115 	qlnx_host_t	*ha;
7116 
7117 	ha = dev->ha;
7118 
7119 	QL_DPRINT12(ha, "enter  qpid=%d\n", conn_param->qpn);
7120 
7121 	if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING))
7122 		return -EINVAL;
7123 
7124 	qp = idr_find(&dev->qpidr, conn_param->qpn);
7125 	if (!qp) {
7126 		QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n",
7127 			conn_param->qpn);
7128 		return -EINVAL;
7129 	}
7130 	ep->qp = qp;
7131 	qp->ep = ep;
7132 	cm_id->add_ref(cm_id);
7133 	ep->cm_id = cm_id;
7134 
7135 	params.ep_context = ep->ecore_context;
7136 	params.cb_context = ep;
7137 	params.qp = ep->qp->ecore_qp;
7138 	params.private_data = conn_param->private_data;
7139 	params.private_data_len = conn_param->private_data_len;
7140 	params.ird = conn_param->ird;
7141 	params.ord = conn_param->ord;
7142 
7143 	rc = ecore_iwarp_accept(dev->rdma_ctx, &params);
7144 	if (rc) {
7145 		QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc);
7146 		goto err;
7147 	}
7148 
7149 	QL_DPRINT12(ha, "exit\n");
7150 	return 0;
7151 err:
7152 	cm_id->rem_ref(cm_id);
7153 	QL_DPRINT12(ha, "exit rc = %d\n", rc);
7154 	return rc;
7155 }
7156 
7157 int
7158 qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
7159 {
7160 #if __FreeBSD_version >= 1102000
7161 
7162         struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;
7163         struct qlnxr_dev *dev = ep->dev;
7164         struct ecore_iwarp_reject_in params;
7165         int rc;
7166 
7167         params.ep_context = ep->ecore_context;
7168         params.cb_context = ep;
7169         params.private_data = pdata;
7170         params.private_data_len = pdata_len;
7171         ep->qp = NULL;
7172 
7173         rc = ecore_iwarp_reject(dev->rdma_ctx, &params);
7174 
7175         return rc;
7176 
7177 #else
7178 
7179 	printf("iWARP reject_cr not implemented\n");
7180 	return -EINVAL;
7181 
7182 #endif /* #if __FreeBSD_version >= 1102000 */
7183 }
7184 
7185 void
7186 qlnxr_iw_qp_add_ref(struct ib_qp *ibqp)
7187 {
7188 	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7189 	qlnx_host_t	*ha;
7190 
7191 	ha = qp->dev->ha;
7192 
7193 	QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp);
7194 
7195 	atomic_inc(&qp->refcnt);
7196 
7197 	QL_DPRINT12(ha, "exit \n");
7198 	return;
7199 }
7200 
7201 void
7202 qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp)
7203 {
7204 	struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
7205 	qlnx_host_t	*ha;
7206 
7207 	ha = qp->dev->ha;
7208 
7209 	QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp);
7210 
7211 	if (atomic_dec_and_test(&qp->refcnt)) {
7212 		qlnxr_idr_remove(qp->dev, qp->qp_id);
7213 		kfree(qp);
7214 	}
7215 
7216 	QL_DPRINT12(ha, "exit \n");
7217 	return;
7218 }
7219 
7220 struct ib_qp *
7221 qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn)
7222 {
7223 	struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
7224 	struct ib_qp *qp;
7225 	qlnx_host_t	*ha;
7226 
7227 	ha = dev->ha;
7228 
7229 	QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn);
7230 
7231 	qp = idr_find(&dev->qpidr, qpn);
7232 
7233 	QL_DPRINT12(ha, "exit qp = %p\n", qp);
7234 
7235 	return (qp);
7236 }
7237