2  * Copyright (c) 2018-2019 Cavium, Inc.
41 		(type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\
42 		(type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\
48 		(sge)->length = cpu_to_le32(vlength);		\
49 		(sge)->flags = cpu_to_le32(vflags);		\
55 		(hdr)->num_sges = num_sge;			\
61 		(sge)->length = cpu_to_le32(vlength);		\
62 		(sge)->l_key = cpu_to_le32(vlkey);		\
108 	ha = dev->ha;  in qlnxr_iw_query_gid()
112 	memset(sgid->raw, 0, sizeof(sgid->raw));  in qlnxr_iw_query_gid()
114 	memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac));  in qlnxr_iw_query_gid()
129 	ha = dev->ha;  in qlnxr_query_gid()
136 		ibdev->add_gid && ibdev->del_gid)) {  in qlnxr_query_gid()
138 		return -ENODEV;  in qlnxr_query_gid()
142 	if (ret == -EAGAIN) {  in qlnxr_query_gid()
150 		return -EINVAL;  in qlnxr_query_gid()
152 	memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid));  in qlnxr_query_gid()
177 	dev = get_qlnxr_dev(ibsrq->device);  in qlnxr_create_srq()
178 	ha = dev->ha;  in qlnxr_create_srq()
184 	srq->dev = dev;  in qlnxr_create_srq()
185 	hw_srq = &srq->hw_srq;  in qlnxr_create_srq()
186 	spin_lock_init(&srq->lock);  in qlnxr_create_srq()
195 			udata->inlen))) {  in qlnxr_create_srq()
201 		ret = qlnxr_init_srq_user_params(&ctx->ibucontext, srq, &ureq, 0, 0);  in qlnxr_create_srq()
205 		page_cnt = srq->usrq.pbl_info.num_pbes;  in qlnxr_create_srq()
206 		pbl_base_addr = srq->usrq.pbl_tbl->pa;  in qlnxr_create_srq()
207 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;  in qlnxr_create_srq()
209 		// page_size = BIT(srq->usrq.umem->page_shift);  in qlnxr_create_srq()
211 		page_size = srq->usrq.umem->page_size;  in qlnxr_create_srq()
217 		pbl = &hw_srq->pbl;  in qlnxr_create_srq()
221 		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;  in qlnxr_create_srq()
222 		page_size = pbl->elem_per_page << 4;  in qlnxr_create_srq()
225 	in_params.pd_id = get_qlnxr_pd(ibsrq->pd)->pd_id;  in qlnxr_create_srq()
231 	ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);  in qlnxr_create_srq()
235 	srq->srq_id = out_params.srq_id;  in qlnxr_create_srq()
243 	QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);  in qlnxr_create_srq()
247 	destroy_in_params.srq_id = srq->srq_id;  in qlnxr_create_srq()
248 	ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);  in qlnxr_create_srq()
257 	return (-EFAULT);  in qlnxr_create_srq()
269 	dev = srq->dev;  in qlnxr_destroy_srq()
270 	ha = dev->ha;  in qlnxr_destroy_srq()
273 	in_params.srq_id = srq->srq_id;  in qlnxr_destroy_srq()
275 	ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params);  in qlnxr_destroy_srq()
277 	if (ibsrq->pd->uobject && ibsrq->pd->uobject->context)  in qlnxr_destroy_srq()
282 	QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);  in qlnxr_destroy_srq()
296 	dev = srq->dev;  in qlnxr_modify_srq()
297 	ha = dev->ha;  in qlnxr_modify_srq()
301 		QL_DPRINT12(ha, "invalid attribute mask=0x%x"  in qlnxr_modify_srq()
303 		return -EINVAL;  in qlnxr_modify_srq()
307 		if (attr->srq_limit >= srq->hw_srq.max_wr) {  in qlnxr_modify_srq()
308 			QL_DPRINT12(ha, "invalid srq_limit=0x%x"  in qlnxr_modify_srq()
309 				" (max_srq_limit = 0x%x)\n",  in qlnxr_modify_srq()
310 			       attr->srq_limit, srq->hw_srq.max_wr);  in qlnxr_modify_srq()
311 			return -EINVAL;	  in qlnxr_modify_srq()
314 		in_params.srq_id = srq->srq_id;  in qlnxr_modify_srq()
315 		in_params.wqe_limit = attr->srq_limit;  in qlnxr_modify_srq()
316 		ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params);  in qlnxr_modify_srq()
321 	QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id);  in qlnxr_modify_srq()
333 	dev = srq->dev;  in qlnxr_query_srq()
334 	ha = dev->ha;  in qlnxr_query_srq()
335 	//qattr = &dev->attr;  in qlnxr_query_srq()
336 	qattr = ecore_rdma_query_device(dev->rdma_ctx);  in qlnxr_query_srq()
339 	if (!dev->rdma_ctx) {  in qlnxr_query_srq()
342 		return -EINVAL;  in qlnxr_query_srq()
345 	srq_attr->srq_limit = qattr->max_srq;  in qlnxr_query_srq()
346 	srq_attr->max_wr = qattr->max_srq_wr;  in qlnxr_query_srq()
347 	srq_attr->max_sge = qattr->max_sge;  in qlnxr_query_srq()
357 	info->wr_prod_cnt++;  in qlnxr_inc_srq_wr_prod()
364         info->wr_cons_cnt++;  in qlnxr_inc_srq_wr_cons()
376 	ha = dev->ha;
396 	dev = srq->dev;  in qlnxr_post_srq_recv()
397 	ha = dev->ha;  in qlnxr_post_srq_recv()
398 	hw_srq = &srq->hw_srq;  in qlnxr_post_srq_recv()
401 	spin_lock_irqsave(&srq->lock, flags);  in qlnxr_post_srq_recv()
403 	pbl = &srq->hw_srq.pbl;  in qlnxr_post_srq_recv()
409 		    wr->num_sge > srq->hw_srq.max_sges) {  in qlnxr_post_srq_recv()
412 			    hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,  in qlnxr_post_srq_recv()
413 			    wr->num_sge, srq->hw_srq.max_sges);  in qlnxr_post_srq_recv()
414 			status = -ENOMEM;  in qlnxr_post_srq_recv()
420 		num_sge = wr->num_sge;  in qlnxr_post_srq_recv()
422 		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);  in qlnxr_post_srq_recv()
428 		hw_srq->wqe_prod++;  in qlnxr_post_srq_recv()
429 		hw_srq->sge_prod++;  in qlnxr_post_srq_recv()
432 			wr->num_sge, hw_srq->wqe_prod, wr->wr_id);  in qlnxr_post_srq_recv()
434 		for (i = 0; i < wr->num_sge; i++) {  in qlnxr_post_srq_recv()
438 			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,  in qlnxr_post_srq_recv()
439 				wr->sg_list[i].length, wr->sg_list[i].lkey);  in qlnxr_post_srq_recv()
441 			QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n",  in qlnxr_post_srq_recv()
442 				i, srq_sge->length, srq_sge->l_key,  in qlnxr_post_srq_recv()
443 				srq_sge->addr.hi, srq_sge->addr.lo);  in qlnxr_post_srq_recv()
444 			hw_srq->sge_prod++;  in qlnxr_post_srq_recv()
452 		*(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod;  in qlnxr_post_srq_recv()
454 		*((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =  in qlnxr_post_srq_recv()
455 			hw_srq->wqe_prod;  in qlnxr_post_srq_recv()
458 		wr = wr->next;  in qlnxr_post_srq_recv()
464 	spin_unlock_irqrestore(&srq->lock, flags);	  in qlnxr_post_srq_recv()
478 	ha = dev->ha;  in qlnxr_query_device()
482 	if (udata->inlen || udata->outlen)  in qlnxr_query_device()
483 		return -EINVAL;  in qlnxr_query_device()
485 	if (dev->rdma_ctx == NULL) {  in qlnxr_query_device()
486 		return -EINVAL;  in qlnxr_query_device()
489 	qattr = ecore_rdma_query_device(dev->rdma_ctx);  in qlnxr_query_device()
493 	attr->fw_ver = qattr->fw_ver;  in qlnxr_query_device()
494 	attr->sys_image_guid = qattr->sys_image_guid;  in qlnxr_query_device()
495 	attr->max_mr_size = qattr->max_mr_size;  in qlnxr_query_device()
496 	attr->page_size_cap = qattr->page_size_caps;  in qlnxr_query_device()
497 	attr->vendor_id = qattr->vendor_id;  in qlnxr_query_device()
498 	attr->vendor_part_id = qattr->vendor_part_id;  in qlnxr_query_device()
499 	attr->hw_ver = qattr->hw_ver;  in qlnxr_query_device()
500 	attr->max_qp = qattr->max_qp;  in qlnxr_query_device()
501 	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |  in qlnxr_query_device()
506 	attr->max_sge = qattr->max_sge;  in qlnxr_query_device()
507 	attr->max_sge_rd = qattr->max_sge;  in qlnxr_query_device()
508 	attr->max_cq = qattr->max_cq;  in qlnxr_query_device()
509 	attr->max_cqe = qattr->max_cqe;  in qlnxr_query_device()
510 	attr->max_mr = qattr->max_mr;  in qlnxr_query_device()
511 	attr->max_mw = qattr->max_mw;  in qlnxr_query_device()
512 	attr->max_pd = qattr->max_pd;  in qlnxr_query_device()
513 	attr->atomic_cap = dev->atomic_cap;  in qlnxr_query_device()
514 	attr->max_fmr = qattr->max_fmr;  in qlnxr_query_device()
515 	attr->max_map_per_fmr = 16; /* TBD: FMR */  in qlnxr_query_device()
523 	 * values are power of two, fls is performed on the value - 1, which  in qlnxr_query_device()
525 	 * of two. This should be fixed in mlx4 driver, but until then ->  in qlnxr_query_device()
528 	attr->max_qp_init_rd_atom =  in qlnxr_query_device()
529 		1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);  in qlnxr_query_device()
530 	attr->max_qp_rd_atom =  in qlnxr_query_device()
531 		min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),  in qlnxr_query_device()
532 		    attr->max_qp_init_rd_atom);  in qlnxr_query_device()
534 	attr->max_srq = qattr->max_srq;  in qlnxr_query_device()
535 	attr->max_srq_sge = qattr->max_srq_sge;  in qlnxr_query_device()
536 	attr->max_srq_wr = qattr->max_srq_wr;  in qlnxr_query_device()
539 	attr->local_ca_ack_delay = qattr->dev_ack_delay;  in qlnxr_query_device()
540 	attr->max_fast_reg_page_list_len = qattr->max_mr/8;  in qlnxr_query_device()
541 	attr->max_pkeys = QLNXR_ROCE_PKEY_MAX;  in qlnxr_query_device()
542 	attr->max_ah = qattr->max_ah;  in qlnxr_query_device()
603 	ha = dev->ha;  in qlnxr_query_port()
609 		return -EINVAL;  in qlnxr_query_port()
612 	if (dev->rdma_ctx == NULL) {  in qlnxr_query_port()
614 		return -EINVAL;  in qlnxr_query_port()
617 	rdma_port = ecore_rdma_query_port(dev->rdma_ctx);  in qlnxr_query_port()
620 	if (rdma_port->port_state == ECORE_RDMA_PORT_UP) {  in qlnxr_query_port()
621 		attr->state = IB_PORT_ACTIVE;  in qlnxr_query_port()
622 		attr->phys_state = 5;  in qlnxr_query_port()
624 		attr->state = IB_PORT_DOWN;  in qlnxr_query_port()
625 		attr->phys_state = 3;  in qlnxr_query_port()
628 	attr->max_mtu = IB_MTU_4096;  in qlnxr_query_port()
629 	attr->active_mtu = iboe_get_mtu(if_getmtu(dev->ha->ifp));  in qlnxr_query_port()
630 	attr->lid = 0;  in qlnxr_query_port()
631 	attr->lmc = 0;  in qlnxr_query_port()
632 	attr->sm_lid = 0;  in qlnxr_query_port()
633 	attr->sm_sl = 0;  in qlnxr_query_port()
634 	attr->port_cap_flags = 0;  in qlnxr_query_port()
637 		attr->gid_tbl_len = 1;  in qlnxr_query_port()
638 		attr->pkey_tbl_len = 1;  in qlnxr_query_port()
640 		attr->gid_tbl_len = QLNXR_MAX_SGID;  in qlnxr_query_port()
641 		attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN;  in qlnxr_query_port()
644 	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;  in qlnxr_query_port()
645 	attr->qkey_viol_cntr = 0;  in qlnxr_query_port()
647 	get_link_speed_and_width(rdma_port->link_speed,  in qlnxr_query_port()
648 				 &attr->active_speed, &attr->active_width);  in qlnxr_query_port()
650 	attr->max_msg_sz = rdma_port->max_msg_size;  in qlnxr_query_port()
651 	attr->max_vl_num = 4; /* TODO -> figure this one out... */  in qlnxr_query_port()
655 		" attr->gid_tbl_len = %d attr->pkey_tbl_len = %d"  in qlnxr_query_port()
656 		" max_msg_sz = 0x%x max_vl_num = 0x%x \n",  in qlnxr_query_port()
657 		attr->state, attr->phys_state,  in qlnxr_query_port()
658 		rdma_port->link_speed, attr->active_speed,  in qlnxr_query_port()
659 		attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len,  in qlnxr_query_port()
660 		attr->max_msg_sz, attr->max_vl_num);  in qlnxr_query_port()
674 	ha = dev->ha;  in qlnxr_modify_port()
680 		return -EINVAL;  in qlnxr_modify_port()
694 	ha = dev->ha;  in qlnxr_link_layer()
696 	QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num);  in qlnxr_link_layer()
704 	struct ib_device *ibdev = ibpd->device;  in qlnxr_alloc_pd()
707 	int			rc;  in qlnxr_alloc_pd()  local
712 	ha = dev->ha;  in qlnxr_alloc_pd()
716 	if (dev->rdma_ctx == NULL) {  in qlnxr_alloc_pd()
717 		QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");  in qlnxr_alloc_pd()
718 		rc = -1;  in qlnxr_alloc_pd()
722 	rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);  in qlnxr_alloc_pd()
723 	if (rc)	{  in qlnxr_alloc_pd()
728 	pd->pd_id = pd_id;  in qlnxr_alloc_pd()
731 		rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));  in qlnxr_alloc_pd()
732 		if (rc) {  in qlnxr_alloc_pd()
734 			ecore_rdma_free_pd(dev->rdma_ctx, pd_id);  in qlnxr_alloc_pd()
738 		pd->uctx = rdma_udata_to_drv_context(  in qlnxr_alloc_pd()
740 		pd->uctx->pd = pd;  in qlnxr_alloc_pd()
743 	atomic_add_rel_32(&dev->pd_count, 1);  in qlnxr_alloc_pd()
744 	QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",  in qlnxr_alloc_pd()
745 		pd, pd_id, dev->pd_count);  in qlnxr_alloc_pd()
750 	QL_DPRINT12(ha, "exit -1\n");  in qlnxr_alloc_pd()
751 	return (rc);  in qlnxr_alloc_pd()
762 	dev = get_qlnxr_dev((ibpd->device));  in qlnxr_dealloc_pd()
763 	ha = dev->ha;  in qlnxr_dealloc_pd()
770 		ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);  in qlnxr_dealloc_pd()
771 		atomic_subtract_rel_32(&dev->pd_count, 1);  in qlnxr_dealloc_pd()
772 		QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",  in qlnxr_dealloc_pd()
773 			pd, pd->pd_id, dev->pd_count);  in qlnxr_dealloc_pd()
795 	ha = dev->ha;  in qlnxr_query_pkey()
797 	QL_DPRINT12(ha, "enter index = 0x%x\n", index);  in qlnxr_query_pkey()
800 		return -EINVAL;  in qlnxr_query_pkey()
822 	dgid = &attr->ah_attr.grh.dgid;  in qlnxr_get_vlan_id_qp()
823 	tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12];  in qlnxr_get_vlan_id_qp()
830 	QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret);  in qlnxr_get_vlan_id_qp()
845 	ha = dev->ha;  in get_gid_info()
849 	memcpy(&qp_params->sgid.bytes[0],  in get_gid_info()
850 	       &dev->sgid_tbl[qp->sgid_idx].raw[0],  in get_gid_info()
851 	       sizeof(qp_params->sgid.bytes));  in get_gid_info()
852 	memcpy(&qp_params->dgid.bytes[0],  in get_gid_info()
853 	       &attr->ah_attr.grh.dgid.raw[0],  in get_gid_info()
854 	       sizeof(qp_params->dgid));  in get_gid_info()
856 	qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id);  in get_gid_info()
858 	for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) {  in get_gid_info()
859 		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);  in get_gid_info()
860 		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);  in get_gid_info()
873 	ha = uctx->dev->ha;  in qlnxr_add_mmap()
880 		return -ENOMEM;  in qlnxr_add_mmap()
883 	mm->key.phy_addr = phy_addr;  in qlnxr_add_mmap()
892 	mm->key.len = roundup(len, PAGE_SIZE);  in qlnxr_add_mmap()
893 	INIT_LIST_HEAD(&mm->entry);  in qlnxr_add_mmap()
895 	mutex_lock(&uctx->mm_list_lock);  in qlnxr_add_mmap()
896 	list_add(&mm->entry, &uctx->mm_head);  in qlnxr_add_mmap()
897 	mutex_unlock(&uctx->mm_list_lock);  in qlnxr_add_mmap()
899 	QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",  in qlnxr_add_mmap()
900 		(unsigned long long)mm->key.phy_addr,  in qlnxr_add_mmap()
901 		(unsigned long)mm->key.len, uctx);  in qlnxr_add_mmap()
913 	ha = uctx->dev->ha;  in qlnxr_search_mmap()
917 	mutex_lock(&uctx->mm_list_lock);  in qlnxr_search_mmap()
918 	list_for_each_entry(mm, &uctx->mm_head, entry) {  in qlnxr_search_mmap()
919 		if (len != mm->key.len || phy_addr != mm->key.phy_addr)  in qlnxr_search_mmap()
925 	mutex_unlock(&uctx->mm_list_lock);  in qlnxr_search_mmap()
928 		"searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n",  in qlnxr_search_mmap()
929 		mm->key.phy_addr, mm->key.len, uctx, found);  in qlnxr_search_mmap()
937         int rc;  in qlnxr_alloc_ucontext()  local
940         struct qlnxr_dev *dev = get_qlnxr_dev(uctx->device);  in qlnxr_alloc_ucontext()
941         qlnx_host_t *ha = dev->ha;  in qlnxr_alloc_ucontext()
945                 return -EFAULT;  in qlnxr_alloc_ucontext()
947 	rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);  in qlnxr_alloc_ucontext()
948 	if (rc) {  in qlnxr_alloc_ucontext()
951 			",rc = %d. To overcome this, consider to increase "  in qlnxr_alloc_ucontext()
955 			"README\n", rc);  in qlnxr_alloc_ucontext()
959 	ctx->dpi = oparams.dpi;  in qlnxr_alloc_ucontext()
960 	ctx->dpi_addr = oparams.dpi_addr;  in qlnxr_alloc_ucontext()
961 	ctx->dpi_phys_addr = oparams.dpi_phys_addr;  in qlnxr_alloc_ucontext()
962 	ctx->dpi_size = oparams.dpi_size;  in qlnxr_alloc_ucontext()
963 	INIT_LIST_HEAD(&ctx->mm_head);  in qlnxr_alloc_ucontext()
964 	mutex_init(&ctx->mm_list_lock);  in qlnxr_alloc_ucontext()
968 				< udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out  in qlnxr_alloc_ucontext()
970 				< udata->outlen ? 1 : 0; //TODO: figure this out  in qlnxr_alloc_ucontext()
972 				< udata->outlen ? oparams.wid_count : 0; //TODO: figure this out   in qlnxr_alloc_ucontext()
973         uresp.db_pa = ctx->dpi_phys_addr;  in qlnxr_alloc_ucontext()
974         uresp.db_size = ctx->dpi_size;  in qlnxr_alloc_ucontext()
975         uresp.max_send_wr = dev->attr.max_sqe;  in qlnxr_alloc_ucontext()
976         uresp.max_recv_wr = dev->attr.max_rqe;  in qlnxr_alloc_ucontext()
977         uresp.max_srq_wr = dev->attr.max_srq_wr;  in qlnxr_alloc_ucontext()
980         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;  in qlnxr_alloc_ucontext()
983 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));  in qlnxr_alloc_ucontext()
984 	if (rc)  in qlnxr_alloc_ucontext()
987 	ctx->dev = dev;  in qlnxr_alloc_ucontext()
989 	rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);  in qlnxr_alloc_ucontext()
990 	if (rc)  in qlnxr_alloc_ucontext()
993 		&ctx->ibucontext);  in qlnxr_alloc_ucontext()
997 	return (rc);  in qlnxr_alloc_ucontext()
1004         struct qlnxr_dev *dev = uctx->dev;  in qlnxr_dealloc_ucontext()
1005         qlnx_host_t *ha = dev->ha;  in qlnxr_dealloc_ucontext()
1012                 ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);  in qlnxr_dealloc_ucontext()
1015         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {  in qlnxr_dealloc_ucontext()
1016                 QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for"  in qlnxr_dealloc_ucontext()
1018                                 mm->key.phy_addr, mm->key.len, uctx);  in qlnxr_dealloc_ucontext()
1019                 list_del(&mm->entry);  in qlnxr_dealloc_ucontext()
1028 	struct qlnxr_dev	*dev = get_qlnxr_dev((context->device));  in qlnxr_mmap()
1029 	unsigned long		vm_page = vma->vm_pgoff << PAGE_SHIFT;  in qlnxr_mmap()
1031 	unsigned long 		len = (vma->vm_end - vma->vm_start);  in qlnxr_mmap()
1032 	int 			rc = 0;  in qlnxr_mmap()  local
1036 	ha = dev->ha;  in qlnxr_mmap()
1038 	unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);  in qlnxr_mmap()
1040 	QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx"  in qlnxr_mmap()
1041 		" vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",  in qlnxr_mmap()
1042 		vm_page, vma->vm_pgoff, unmapped_db,  in qlnxr_mmap()
1043 		dev->db_size, len);  in qlnxr_mmap()
1045 	if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {  in qlnxr_mmap()
1047 			"vm_start = %ld vma_end = %ld\n", vma->vm_start,  in qlnxr_mmap()
1048 			vma->vm_end);  in qlnxr_mmap()
1049 		return -EINVAL;  in qlnxr_mmap()
1055 			vma->vm_pgoff);  in qlnxr_mmap()
1056 		return -EINVAL;  in qlnxr_mmap()
1062 		((vm_page + len) > (unmapped_db + ucontext->dpi_size))) {  in qlnxr_mmap()
1064 			"page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n",  in qlnxr_mmap()
1065 			vm_page, unmapped_db, ucontext->dpi_size);  in qlnxr_mmap()
1066 		return -EINVAL;  in qlnxr_mmap()
1069 	if (vma->vm_flags & VM_READ) {  in qlnxr_mmap()
1071 		return -EINVAL;  in qlnxr_mmap()
1074 	vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);  in qlnxr_mmap()
1075 	rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,  in qlnxr_mmap()
1076 			vma->vm_page_prot);  in qlnxr_mmap()
1078 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_mmap()
1079 	return rc;  in qlnxr_mmap()
1086 	struct qlnxr_dev	*dev = get_qlnxr_dev((ibpd->device));  in qlnxr_get_dma_mr()
1088 	int			rc;  in qlnxr_get_dma_mr()  local
1091 	ha = dev->ha;  in qlnxr_get_dma_mr()
1101 		rc = -ENOMEM;  in qlnxr_get_dma_mr()
1102 		QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc);  in qlnxr_get_dma_mr()
1106 	mr->type = QLNXR_MR_DMA;  in qlnxr_get_dma_mr()
1108 	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);  in qlnxr_get_dma_mr()
1109 	if (rc) {  in qlnxr_get_dma_mr()
1110 		QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc);  in qlnxr_get_dma_mr()
1115 	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;  in qlnxr_get_dma_mr()
1116 	mr->hw_mr.pd = pd->pd_id;  in qlnxr_get_dma_mr()
1117 	mr->hw_mr.local_read = 1;  in qlnxr_get_dma_mr()
1118 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in qlnxr_get_dma_mr()
1119 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in qlnxr_get_dma_mr()
1120 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in qlnxr_get_dma_mr()
1121 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in qlnxr_get_dma_mr()
1122 	mr->hw_mr.dma_mr = true;  in qlnxr_get_dma_mr()
1124 	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);  in qlnxr_get_dma_mr()
1125 	if (rc) {  in qlnxr_get_dma_mr()
1126 		QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc);  in qlnxr_get_dma_mr()
1130 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qlnxr_get_dma_mr()
1132 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||  in qlnxr_get_dma_mr()
1133 		mr->hw_mr.remote_atomic) {  in qlnxr_get_dma_mr()
1134 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qlnxr_get_dma_mr()
1137 	QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey);  in qlnxr_get_dma_mr()
1139 	return &mr->ibmr;  in qlnxr_get_dma_mr()
1142 	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);  in qlnxr_get_dma_mr()
1146 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_get_dma_mr()
1148 	return ERR_PTR(rc);  in qlnxr_get_dma_mr()
1158 	ha = dev->ha;  in qlnxr_free_pbl()
1162 	for (i = 0; i < pbl_info->num_pbls; i++) {  in qlnxr_free_pbl()
1165 		qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa,  in qlnxr_free_pbl()
1166 			pbl_info->pbl_size);  in qlnxr_free_pbl()
1192 	ha = dev->ha;  in qlnxr_alloc_pbl_tbl()
1196 	pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags);  in qlnxr_alloc_pbl_tbl()
1203 	for (i = 0; i < pbl_info->num_pbls; i++) {  in qlnxr_alloc_pbl_tbl()
1204 		va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size);  in qlnxr_alloc_pbl_tbl()
1209 		memset(va, 0, pbl_info->pbl_size);  in qlnxr_alloc_pbl_tbl()
1214 	/* Two-Layer PBLs, if we have more than one pbl we need to initialize  in qlnxr_alloc_pbl_tbl()
1218 	for (i = 0; i < pbl_info->num_pbls - 1; i++)  in qlnxr_alloc_pbl_tbl()
1242 	ha = dev->ha;  in qlnxr_prepare_pbl_tbl()
1250 			return -EINVAL;  in qlnxr_prepare_pbl_tbl()
1266 		pbl_info->two_layered = true;  in qlnxr_prepare_pbl_tbl()
1272 		pbl_info->two_layered = false;  in qlnxr_prepare_pbl_tbl()
1275 	pbl_info->num_pbls = num_pbls;  in qlnxr_prepare_pbl_tbl()
1276 	pbl_info->pbl_size = pbl_size;  in qlnxr_prepare_pbl_tbl()
1277 	pbl_info->num_pbes = num_pbes;  in qlnxr_prepare_pbl_tbl()
1280 		pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);  in qlnxr_prepare_pbl_tbl()
1296 	ha = dev->ha;  in qlnxr_populate_pbls()
1305 	if (!pbl_info->num_pbes) {  in qlnxr_populate_pbls()
1306 		QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n");  in qlnxr_populate_pbls()
1313 	if (pbl_info->two_layered)  in qlnxr_populate_pbls()
1318 	pbe = (struct regpair *)pbl_tbl->va;  in qlnxr_populate_pbls()
1326 	shift = ilog2(umem->page_size);  in qlnxr_populate_pbls()
1328 	for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {  in qlnxr_populate_pbls()
1332 			pbe->lo =  in qlnxr_populate_pbls()
1334 					(umem->page_size * pg_cnt));  in qlnxr_populate_pbls()
1335 			pbe->hi =  in qlnxr_populate_pbls()
1338 					  umem->page_size * pg_cnt)));  in qlnxr_populate_pbls()
1342 				" pbe->addr=0x%x:0x%x "  in qlnxr_populate_pbls()
1344 				" pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt,  in qlnxr_populate_pbls()
1351 			if (total_num_pbes == pbl_info->num_pbes)  in qlnxr_populate_pbls()
1358 				(pbl_info->pbl_size / sizeof(u64))) {  in qlnxr_populate_pbls()
1360 				pbe = (struct regpair *)pbl_tbl->va;  in qlnxr_populate_pbls()
1375 	ha = dev->ha;  in free_mr_info()
1379 	if (info->pbl_table)  in free_mr_info()
1380 		list_add_tail(&info->pbl_table->list_entry,  in free_mr_info()
1381 			      &info->free_pbl_list);  in free_mr_info()
1383 	if (!list_empty(&info->inuse_pbl_list))  in free_mr_info()
1384 		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);  in free_mr_info()
1386 	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {  in free_mr_info()
1387 		list_del(&pbl->list_entry);  in free_mr_info()
1388 		qlnxr_free_pbl(dev, &info->pbl_info, pbl);  in free_mr_info()
1399 	int			rc;  in qlnxr_init_mr_info()  local
1403 	ha = dev->ha;  in qlnxr_init_mr_info()
1407 	INIT_LIST_HEAD(&info->free_pbl_list);  in qlnxr_init_mr_info()
1408 	INIT_LIST_HEAD(&info->inuse_pbl_list);  in qlnxr_init_mr_info()
1410 	rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info,  in qlnxr_init_mr_info()
1412 	if (rc) {  in qlnxr_init_mr_info()
1413 		QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc);  in qlnxr_init_mr_info()
1417 	info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);  in qlnxr_init_mr_info()
1419 	if (!info->pbl_table) {  in qlnxr_init_mr_info()
1420 		rc = -ENOMEM;  in qlnxr_init_mr_info()
1425 	QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa);  in qlnxr_init_mr_info()
1430 	tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);  in qlnxr_init_mr_info()
1434 		goto done; /* it's OK if second allocation fails, so rc = 0*/  in qlnxr_init_mr_info()
1437 	list_add_tail(&tmp->list_entry, &info->free_pbl_list);  in qlnxr_init_mr_info()
1439 	QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa);  in qlnxr_init_mr_info()
1442 	if (rc)  in qlnxr_init_mr_info()
1445 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_init_mr_info()
1447 	return rc;  in qlnxr_init_mr_info()
1454 	int		rc = -ENOMEM;  in qlnxr_reg_user_mr()  local
1455 	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));  in qlnxr_reg_user_mr()
1460 	ha = dev->ha;  in qlnxr_reg_user_mr()
1468 		pd->pd_id, start, len, usr_addr, acc);  in qlnxr_reg_user_mr()
1474 		return ERR_PTR(-EINVAL);  in qlnxr_reg_user_mr()
1480 		return ERR_PTR(rc);  in qlnxr_reg_user_mr()
1483 	mr->type = QLNXR_MR_USER;  in qlnxr_reg_user_mr()
1485 	mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);  in qlnxr_reg_user_mr()
1486 	if (IS_ERR(mr->umem)) {  in qlnxr_reg_user_mr()
1487 		rc = -EFAULT;  in qlnxr_reg_user_mr()
1488 		QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem);  in qlnxr_reg_user_mr()
1492 	rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);  in qlnxr_reg_user_mr()
1493 	if (rc) {  in qlnxr_reg_user_mr()
1495 			"qlnxr_init_mr_info failed [%d]\n", rc);  in qlnxr_reg_user_mr()
1499 	qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table,  in qlnxr_reg_user_mr()
1500 			   &mr->info.pbl_info);  in qlnxr_reg_user_mr()
1502 	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);  in qlnxr_reg_user_mr()
1504 	if (rc) {  in qlnxr_reg_user_mr()
1505 		QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc);  in qlnxr_reg_user_mr()
1510 	mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR;  in qlnxr_reg_user_mr()
1511 	mr->hw_mr.key = 0;  in qlnxr_reg_user_mr()
1512 	mr->hw_mr.pd = pd->pd_id;  in qlnxr_reg_user_mr()
1513 	mr->hw_mr.local_read = 1;  in qlnxr_reg_user_mr()
1514 	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in qlnxr_reg_user_mr()
1515 	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in qlnxr_reg_user_mr()
1516 	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in qlnxr_reg_user_mr()
1517 	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in qlnxr_reg_user_mr()
1518 	mr->hw_mr.mw_bind = false; /* TBD MW BIND */  in qlnxr_reg_user_mr()
1519 	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;  in qlnxr_reg_user_mr()
1520 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;  in qlnxr_reg_user_mr()
1521 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);  in qlnxr_reg_user_mr()
1522 	mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */  in qlnxr_reg_user_mr()
1524 	mr->hw_mr.fbo = ib_umem_offset(mr->umem);  in qlnxr_reg_user_mr()
1525 	mr->hw_mr.length = len;  in qlnxr_reg_user_mr()
1526 	mr->hw_mr.vaddr = usr_addr;  in qlnxr_reg_user_mr()
1527 	mr->hw_mr.zbva = false; /* TBD figure when this should be true */  in qlnxr_reg_user_mr()
1528 	mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */  in qlnxr_reg_user_mr()
1529 	mr->hw_mr.dma_mr = false;  in qlnxr_reg_user_mr()
1531 	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);  in qlnxr_reg_user_mr()
1532 	if (rc) {  in qlnxr_reg_user_mr()
1533 		QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc);  in qlnxr_reg_user_mr()
1537 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qlnxr_reg_user_mr()
1538 	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||  in qlnxr_reg_user_mr()
1539 		mr->hw_mr.remote_atomic)  in qlnxr_reg_user_mr()
1540 		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in qlnxr_reg_user_mr()
1542 	QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey);  in qlnxr_reg_user_mr()
1544 	return (&mr->ibmr);  in qlnxr_reg_user_mr()
1547 	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);  in qlnxr_reg_user_mr()
1549 	qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);  in qlnxr_reg_user_mr()
1553 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_reg_user_mr()
1554 	return (ERR_PTR(rc));  in qlnxr_reg_user_mr()
1561 	struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));  in qlnxr_dereg_mr()
1562 	int		rc = 0;  in qlnxr_dereg_mr()  local
1565 	ha = dev->ha;  in qlnxr_dereg_mr()
1569 	if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR))  in qlnxr_dereg_mr()
1570 		qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);  in qlnxr_dereg_mr()
1573 	if (mr->umem)  in qlnxr_dereg_mr()
1574 		ib_umem_release(mr->umem);  in qlnxr_dereg_mr()
1576 	kfree(mr->pages);  in qlnxr_dereg_mr()
1581 	return rc;  in qlnxr_dereg_mr()
1589 	int				rc;  in qlnxr_copy_cq_uresp()  local
1592 	ha = dev->ha;  in qlnxr_copy_cq_uresp()
1599 	uresp.icid = cq->icid;  in qlnxr_copy_cq_uresp()
1601 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));  in qlnxr_copy_cq_uresp()
1603 	if (rc) {  in qlnxr_copy_cq_uresp()
1604 		QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n",  in qlnxr_copy_cq_uresp()
1605 			cq->icid, rc);  in qlnxr_copy_cq_uresp()
1608 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_copy_cq_uresp()
1609 	return rc;  in qlnxr_copy_cq_uresp()
1616 	if (cq->latest_cqe == cq->toggle_cqe)  in consume_cqe()
1617 		cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;  in consume_cqe()
1619 	cq->latest_cqe = ecore_chain_consume(&cq->pbl);  in consume_cqe()
1656 	int		rc;  in qlnxr_init_user_queue()  local
1659 	ha = dev->ha;  in qlnxr_init_user_queue()
1663 	q->buf_addr = buf_addr;  in qlnxr_init_user_queue()
1664 	q->buf_len = buf_len;  in qlnxr_init_user_queue()
1666 	QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x"  in qlnxr_init_user_queue()
1667 	      " dmasync : %x\n", q->buf_addr, q->buf_len,  in qlnxr_init_user_queue()
1670 	q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);  in qlnxr_init_user_queue()
1672 	if (IS_ERR(q->umem)) {  in qlnxr_init_user_queue()
1673 		QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem));  in qlnxr_init_user_queue()
1674 		return PTR_ERR(q->umem);  in qlnxr_init_user_queue()
1677 	page_cnt = ib_umem_page_count(q->umem);  in qlnxr_init_user_queue()
1678 	rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt,  in qlnxr_init_user_queue()
1682 	if (rc) {  in qlnxr_init_user_queue()
1683 		QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc);  in qlnxr_init_user_queue()
1688 		q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);  in qlnxr_init_user_queue()
1690 		if (!q->pbl_tbl) {  in qlnxr_init_user_queue()
1692 			rc = -ENOMEM;  in qlnxr_init_user_queue()
1696 		qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);  in qlnxr_init_user_queue()
1698 		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);  in qlnxr_init_user_queue()
1700 		if (!q->pbl_tbl) {  in qlnxr_init_user_queue()
1702 			rc = -ENOMEM;  in qlnxr_init_user_queue()
1711 	ib_umem_release(q->umem);  in qlnxr_init_user_queue()
1712 	q->umem = NULL;  in qlnxr_init_user_queue()
1714 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_init_user_queue()
1715 	return rc;  in qlnxr_init_user_queue()
1730 	int					vector = attr->comp_vector;  in qlnxr_create_cq()
1731 	int					entries = attr->cqe;  in qlnxr_create_cq()
1733 	int					chain_entries, rc, page_cnt;  in qlnxr_create_cq()  local
1738 	dev = get_qlnxr_dev(ibcq->device);  in qlnxr_create_cq()
1739 	ha = dev->ha;  in qlnxr_create_cq()
1754 		return -EINVAL;  in qlnxr_create_cq()
1766 			min(sizeof(ureq), udata->inlen))) {  in qlnxr_create_cq()
1776 		cq->cq_type = QLNXR_CQ_TYPE_USER;  in qlnxr_create_cq()
1778 		qlnxr_init_user_queue(&ctx->ibucontext, dev, &cq->q, ureq.addr, ureq.len,  in qlnxr_create_cq()
1781 		pbl_ptr = cq->q.pbl_tbl->pa;  in qlnxr_create_cq()
1782 		page_cnt = cq->q.pbl_info.num_pbes;  in qlnxr_create_cq()
1783 		cq->ibcq.cqe = chain_entries;  in qlnxr_create_cq()
1787 		cq->cq_type = QLNXR_CQ_TYPE_KERNEL;  in qlnxr_create_cq()
1789                 rc = ecore_chain_alloc(&dev->ha->cdev,  in qlnxr_create_cq()
1795                            &cq->pbl, NULL);  in qlnxr_create_cq()
1797 		if (rc)  in qlnxr_create_cq()
1800 		page_cnt = ecore_chain_get_page_cnt(&cq->pbl);  in qlnxr_create_cq()
1801 		pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl);  in qlnxr_create_cq()
1802 		cq->ibcq.cqe = cq->pbl.capacity;  in qlnxr_create_cq()
1808         params.cq_size = chain_entries - 1;  in qlnxr_create_cq()
1814         	params.dpi = ctx->dpi;  in qlnxr_create_cq()
1816         	params.dpi = dev->dpi;  in qlnxr_create_cq()
1819 	rc = ecore_rdma_create_cq(dev->rdma_ctx, &params, &icid);  in qlnxr_create_cq()
1820 	if (rc)  in qlnxr_create_cq()
1823 	cq->icid = icid;  in qlnxr_create_cq()
1824 	cq->sig = QLNXR_CQ_MAGIC_NUMBER;  in qlnxr_create_cq()
1825 	spin_lock_init(&cq->cq_lock);  in qlnxr_create_cq()
1828 		rc = qlnxr_copy_cq_uresp(dev, cq, udata);  in qlnxr_create_cq()
1829 		if (rc)  in qlnxr_create_cq()
1833 		 * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT.  in qlnxr_create_cq()
1838 		cq->db_addr = dev->db_addr +  in qlnxr_create_cq()
1840 		cq->db.data.icid = cq->icid;  in qlnxr_create_cq()
1841 		cq->db.data.params = DB_AGG_CMD_SET <<  in qlnxr_create_cq()
1845 		cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl);  in qlnxr_create_cq()
1846 		cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK;  in qlnxr_create_cq()
1849 		cq->latest_cqe = NULL;  in qlnxr_create_cq()
1851 		cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);  in qlnxr_create_cq()
1854 	QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p,"  in qlnxr_create_cq()
1855 		" number of entries = 0x%x\n",  in qlnxr_create_cq()
1856 		cq->icid, cq, params.cq_size);  in qlnxr_create_cq()
1861 	destroy_iparams.icid = cq->icid;  in qlnxr_create_cq()
1862 	ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams);  in qlnxr_create_cq()
1865 		qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);  in qlnxr_create_cq()
1867 		ecore_chain_free(&dev->ha->cdev, &cq->pbl);  in qlnxr_create_cq()
1870 		ib_umem_release(cq->q.umem);  in qlnxr_create_cq()
1874 	return (-EINVAL);  in qlnxr_create_cq()
1880 	struct qlnxr_dev	*dev = get_qlnxr_dev((ibcq->device));  in qlnxr_resize_cq()
1883 	ha = dev->ha;  in qlnxr_resize_cq()
1893 	struct qlnxr_dev			*dev = get_qlnxr_dev((ibcq->device));  in qlnxr_destroy_cq()
1897 	int					rc = 0;  in qlnxr_destroy_cq()  local
1900 	ha = dev->ha;  in qlnxr_destroy_cq()
1902 	QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid);  in qlnxr_destroy_cq()
1904 	cq->destroyed = 1;  in qlnxr_destroy_cq()
1913 	if (cq->cq_type != QLNXR_CQ_TYPE_GSI) {  in qlnxr_destroy_cq()
1914 		iparams.icid = cq->icid;  in qlnxr_destroy_cq()
1916 		rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);  in qlnxr_destroy_cq()
1918 		if (rc) {  in qlnxr_destroy_cq()
1920 				cq->icid);  in qlnxr_destroy_cq()
1924 		QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);  in qlnxr_destroy_cq()
1925 		ecore_chain_free(&dev->ha->cdev, &cq->pbl);  in qlnxr_destroy_cq()
1929 		qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);  in qlnxr_destroy_cq()
1930 		ib_umem_release(cq->q.umem);  in qlnxr_destroy_cq()
1933 	cq->sig = ~cq->sig;  in qlnxr_destroy_cq()
1935 	QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);  in qlnxr_destroy_cq()
1947 	qattr = ecore_rdma_query_device(dev->rdma_ctx);  in qlnxr_check_qp_attrs()
1948 	ha = dev->ha;  in qlnxr_check_qp_attrs()
1952 	QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type);  in qlnxr_check_qp_attrs()
1953 	QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type);  in qlnxr_check_qp_attrs()
1954 	QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags);  in qlnxr_check_qp_attrs()
1956 	QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num);  in qlnxr_check_qp_attrs()
1957 	QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr);  in qlnxr_check_qp_attrs()
1958 	QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr);  in qlnxr_check_qp_attrs()
1959 	QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge);  in qlnxr_check_qp_attrs()
1960 	QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge);  in qlnxr_check_qp_attrs()
1961 	QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n",  in qlnxr_check_qp_attrs()
1962 		attrs->cap.max_inline_data);  in qlnxr_check_qp_attrs()
1964 	QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id);  in qlnxr_check_qp_attrs()
1965 	QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id);  in qlnxr_check_qp_attrs()
1966 	QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver);  in qlnxr_check_qp_attrs()
1967 	QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver);  in qlnxr_check_qp_attrs()
1968 	QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid);  in qlnxr_check_qp_attrs()
1969 	QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n",  in qlnxr_check_qp_attrs()
1970 		(void *)qattr->sys_image_guid);  in qlnxr_check_qp_attrs()
1971 	QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq);  in qlnxr_check_qp_attrs()
1972 	QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge);  in qlnxr_check_qp_attrs()
1973 	QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge);  in qlnxr_check_qp_attrs()
1974 	QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline);  in qlnxr_check_qp_attrs()
1975 	QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe);  in qlnxr_check_qp_attrs()
1976 	QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe);  in qlnxr_check_qp_attrs()
1977 	QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n",  in qlnxr_check_qp_attrs()
1978 		qattr->max_qp_resp_rd_atomic_resc);  in qlnxr_check_qp_attrs()
1979 	QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n",  in qlnxr_check_qp_attrs()
1980 		qattr->max_qp_req_rd_atomic_resc);  in qlnxr_check_qp_attrs()
1981 	QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n",  in qlnxr_check_qp_attrs()
1982 		qattr->max_dev_resp_rd_atomic_resc);  in qlnxr_check_qp_attrs()
1983 	QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq);  in qlnxr_check_qp_attrs()
1984 	QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp);  in qlnxr_check_qp_attrs()
1985 	QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq);  in qlnxr_check_qp_attrs()
1986 	QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr);  in qlnxr_check_qp_attrs()
1987 	QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size);  in qlnxr_check_qp_attrs()
1988 	QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe);  in qlnxr_check_qp_attrs()
1989 	QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw);  in qlnxr_check_qp_attrs()
1990 	QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr);  in qlnxr_check_qp_attrs()
1991 	QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n",  in qlnxr_check_qp_attrs()
1992 		qattr->max_mr_mw_fmr_pbl);  in qlnxr_check_qp_attrs()
1993 	QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n",  in qlnxr_check_qp_attrs()
1994 		(void *)qattr->max_mr_mw_fmr_size);  in qlnxr_check_qp_attrs()
1995 	QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd);  in qlnxr_check_qp_attrs()
1996 	QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah);  in qlnxr_check_qp_attrs()
1997 	QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey);  in qlnxr_check_qp_attrs()
1998 	QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr);  in qlnxr_check_qp_attrs()
1999 	QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n",  in qlnxr_check_qp_attrs()
2000 		qattr->max_stats_queues);  in qlnxr_check_qp_attrs()
2001 	//QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps);  in qlnxr_check_qp_attrs()
2002 	QL_DPRINT12(ha, "qattr->page_size_caps = %p\n",  in qlnxr_check_qp_attrs()
2003 		(void *)qattr->page_size_caps);  in qlnxr_check_qp_attrs()
2004 	QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay);  in qlnxr_check_qp_attrs()
2005 	QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey);  in qlnxr_check_qp_attrs()
2006 	QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n",  in qlnxr_check_qp_attrs()
2007 		qattr->bad_pkey_counter);  in qlnxr_check_qp_attrs()
2009 	if ((attrs->qp_type == IB_QPT_GSI) && udata) {  in qlnxr_check_qp_attrs()
2011 		return -EINVAL;  in qlnxr_check_qp_attrs()
2014 	if (udata && !(ibpd->uobject && ibpd->uobject->context)) {  in qlnxr_check_qp_attrs()
2016 		return -EINVAL;  in qlnxr_check_qp_attrs()
2019 	/* QP0... attrs->qp_type == IB_QPT_GSI */  in qlnxr_check_qp_attrs()
2020 	if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {  in qlnxr_check_qp_attrs()
2021 		QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n",   in qlnxr_check_qp_attrs()
2022 			   attrs->qp_type);  in qlnxr_check_qp_attrs()
2023 		return -EINVAL;  in qlnxr_check_qp_attrs()
2025 	if (attrs->qp_type == IB_QPT_GSI && attrs->srq) {  in qlnxr_check_qp_attrs()
2027 		return -EINVAL;  in qlnxr_check_qp_attrs()
2030 	if (attrs->cap.max_send_wr > qattr->max_wqe) {  in qlnxr_check_qp_attrs()
2032 			" (max_send_wr=0x%x)\n",  in qlnxr_check_qp_attrs()
2033 			attrs->cap.max_send_wr, qattr->max_wqe);  in qlnxr_check_qp_attrs()
2034 		return -EINVAL;  in qlnxr_check_qp_attrs()
2036 	if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) {  in qlnxr_check_qp_attrs()
2038 			" (max_recv_wr=0x%x)\n",  in qlnxr_check_qp_attrs()
2039 			attrs->cap.max_recv_wr, qattr->max_wqe);  in qlnxr_check_qp_attrs()
2040 		return -EINVAL;  in qlnxr_check_qp_attrs()
2042 	if (attrs->cap.max_inline_data > qattr->max_inline) {  in qlnxr_check_qp_attrs()
2044 			"unsupported inline data size=0x%x "  in qlnxr_check_qp_attrs()
2045 			"requested (max_inline=0x%x)\n",  in qlnxr_check_qp_attrs()
2046 			attrs->cap.max_inline_data, qattr->max_inline);  in qlnxr_check_qp_attrs()
2047 		return -EINVAL;  in qlnxr_check_qp_attrs()
2049 	if (attrs->cap.max_send_sge > qattr->max_sge) {  in qlnxr_check_qp_attrs()
2051 			"unsupported send_sge=0x%x "  in qlnxr_check_qp_attrs()
2052 			"requested (max_send_sge=0x%x)\n",  in qlnxr_check_qp_attrs()
2053 			attrs->cap.max_send_sge, qattr->max_sge);  in qlnxr_check_qp_attrs()
2054 		return -EINVAL;  in qlnxr_check_qp_attrs()
2056 	if (attrs->cap.max_recv_sge > qattr->max_sge) {  in qlnxr_check_qp_attrs()
2058 			"unsupported recv_sge=0x%x requested "  in qlnxr_check_qp_attrs()
2059 			" (max_recv_sge=0x%x)\n",  in qlnxr_check_qp_attrs()
2060 			attrs->cap.max_recv_sge, qattr->max_sge);  in qlnxr_check_qp_attrs()
2061 		return -EINVAL;  in qlnxr_check_qp_attrs()
2064 	if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {  in qlnxr_check_qp_attrs()
2066 			"userspace can't create special QPs of type=0x%x\n",  in qlnxr_check_qp_attrs()
2067 			attrs->qp_type);  in qlnxr_check_qp_attrs()
2068 		return -EINVAL;  in qlnxr_check_qp_attrs()
2071 	if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {  in qlnxr_check_qp_attrs()
2074 		return -EINVAL;  in qlnxr_check_qp_attrs()
2078 	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {  in qlnxr_check_qp_attrs()
2079 		struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq);  in qlnxr_check_qp_attrs()
2080 		struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq);  in qlnxr_check_qp_attrs()
2082 		if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) ||  in qlnxr_check_qp_attrs()
2083 		    (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) {  in qlnxr_check_qp_attrs()
2085 			return -EINVAL;  in qlnxr_check_qp_attrs()
2099 	int				rc;  in qlnxr_copy_srq_uresp()  local
2101 	ha = dev->ha;  in qlnxr_copy_srq_uresp()
2107 	uresp.srq_id = srq->srq_id;  in qlnxr_copy_srq_uresp()
2109 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));  in qlnxr_copy_srq_uresp()
2111 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_copy_srq_uresp()
2112 	return rc;  in qlnxr_copy_srq_uresp()
2122 	ha = dev->ha;  in qlnxr_copy_rq_uresp()
2125 	QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq);  in qlnxr_copy_rq_uresp()
2127 	if (qp->srq)  in qlnxr_copy_rq_uresp()
2132 		uresp->rq_db_offset =  in qlnxr_copy_rq_uresp()
2134 		uresp->rq_db2_offset =  in qlnxr_copy_rq_uresp()
2137 		QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x "  in qlnxr_copy_rq_uresp()
2138 			"uresp->rq_db2_offset = 0x%x\n",  in qlnxr_copy_rq_uresp()
2139 			uresp->rq_db_offset, uresp->rq_db2_offset);  in qlnxr_copy_rq_uresp()
2141 		uresp->rq_db_offset =  in qlnxr_copy_rq_uresp()
2144 	uresp->rq_icid = qp->icid;  in qlnxr_copy_rq_uresp()
2157 	ha = dev->ha;  in qlnxr_copy_sq_uresp()
2161 	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);  in qlnxr_copy_sq_uresp()
2165 		uresp->sq_icid = qp->icid;  in qlnxr_copy_sq_uresp()
2166 		QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid);  in qlnxr_copy_sq_uresp()
2168 		uresp->sq_icid = qp->icid + 1;  in qlnxr_copy_sq_uresp()
2179 	int				rc;  in qlnxr_copy_qp_uresp()  local
2183 	ha = dev->ha;  in qlnxr_copy_qp_uresp()
2185 	QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid);  in qlnxr_copy_qp_uresp()
2191 	uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;  in qlnxr_copy_qp_uresp()
2192 	uresp.qp_id = qp->qp_id;  in qlnxr_copy_qp_uresp()
2194 	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));  in qlnxr_copy_qp_uresp()
2196 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_copy_qp_uresp()
2197 	return rc;  in qlnxr_copy_qp_uresp()
2208 	ha = dev->ha;  in qlnxr_set_common_qp_params()
2212 	spin_lock_init(&qp->q_lock);  in qlnxr_set_common_qp_params()
2214 	atomic_set(&qp->refcnt, 1);  in qlnxr_set_common_qp_params()
2215 	qp->pd = pd;  in qlnxr_set_common_qp_params()
2216 	qp->sig = QLNXR_QP_MAGIC_NUMBER;  in qlnxr_set_common_qp_params()
2217 	qp->qp_type = attrs->qp_type;  in qlnxr_set_common_qp_params()
2218 	qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;  in qlnxr_set_common_qp_params()
2219 	qp->sq.max_sges = attrs->cap.max_send_sge;  in qlnxr_set_common_qp_params()
2220 	qp->state = ECORE_ROCE_QP_STATE_RESET;  in qlnxr_set_common_qp_params()
2221 	qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;  in qlnxr_set_common_qp_params()
2222 	qp->sq_cq = get_qlnxr_cq(attrs->send_cq);  in qlnxr_set_common_qp_params()
2223 	qp->rq_cq = get_qlnxr_cq(attrs->recv_cq);  in qlnxr_set_common_qp_params()
2224 	qp->dev = dev;  in qlnxr_set_common_qp_params()
2226 	if (!attrs->srq) {  in qlnxr_set_common_qp_params()
2228 		qp->rq.max_sges = attrs->cap.max_recv_sge;  in qlnxr_set_common_qp_params()
2230 			qp->rq.max_sges, qp->rq_cq->icid);  in qlnxr_set_common_qp_params()
2232 		qp->srq = get_qlnxr_srq(attrs->srq);  in qlnxr_set_common_qp_params()
2238 		pd->pd_id, qp->qp_type, qp->max_inline_data,  in qlnxr_set_common_qp_params()
2239 		qp->state, qp->signaled, ((attrs->srq) ? 1 : 0));  in qlnxr_set_common_qp_params()
2241 		qp->sq.max_sges, qp->sq_cq->icid);  in qlnxr_set_common_qp_params()
2252 	ha = dev->ha;  in qlnxr_check_srq_params()
2253 	qattr = ecore_rdma_query_device(dev->rdma_ctx);  in qlnxr_check_srq_params()
2257 	if (attrs->attr.max_wr > qattr->max_srq_wqe) {  in qlnxr_check_srq_params()
2258 		QL_DPRINT12(ha, "unsupported srq_wr=0x%x"  in qlnxr_check_srq_params()
2259 			" requested (max_srq_wr=0x%x)\n",  in qlnxr_check_srq_params()
2260 			attrs->attr.max_wr, qattr->max_srq_wr);  in qlnxr_check_srq_params()
2261 		return -EINVAL;  in qlnxr_check_srq_params()
2264 	if (attrs->attr.max_sge > qattr->max_sge) {  in qlnxr_check_srq_params()
2266 			"unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",  in qlnxr_check_srq_params()
2267 			attrs->attr.max_sge, qattr->max_sge);  in qlnxr_check_srq_params()
2268 		return -EINVAL;  in qlnxr_check_srq_params()
2271 	if (attrs->attr.srq_limit > attrs->attr.max_wr) {  in qlnxr_check_srq_params()
2273 		       "unsupported srq_limit=0x%x requested"  in qlnxr_check_srq_params()
2274 			" (max_srq_limit=0x%x)\n",  in qlnxr_check_srq_params()
2275 			attrs->attr.srq_limit, attrs->attr.srq_limit);  in qlnxr_check_srq_params()
2276 		return -EINVAL;  in qlnxr_check_srq_params()
2286 	struct qlnxr_dev	*dev = srq->dev;  in qlnxr_free_srq_user_params()
2289 	ha = dev->ha;  in qlnxr_free_srq_user_params()
2293 	qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);  in qlnxr_free_srq_user_params()
2294 	ib_umem_release(srq->usrq.umem);  in qlnxr_free_srq_user_params()
2295 	ib_umem_release(srq->prod_umem);  in qlnxr_free_srq_user_params()
2304 	struct qlnxr_srq_hwq_info *hw_srq  = &srq->hw_srq;  in qlnxr_free_srq_kernel_params()
2305 	struct qlnxr_dev	*dev = srq->dev;  in qlnxr_free_srq_kernel_params()
2308 	ha = dev->ha;  in qlnxr_free_srq_kernel_params()
2312 	ecore_chain_free(dev->cdev, &hw_srq->pbl);  in qlnxr_free_srq_kernel_params()
2314 	qlnx_dma_free_coherent(&dev->cdev,  in qlnxr_free_srq_kernel_params()
2315 		hw_srq->virt_prod_pair_addr,  in qlnxr_free_srq_kernel_params()
2316 		hw_srq->phy_prod_pair_addr,  in qlnxr_free_srq_kernel_params()
2331 	int			rc;  in qlnxr_init_srq_user_params()  local
2332 	struct qlnxr_dev	*dev = srq->dev;  in qlnxr_init_srq_user_params()
2335 	ha = dev->ha;  in qlnxr_init_srq_user_params()
2339 	rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,  in qlnxr_init_srq_user_params()
2340 				  ureq->srq_len, access, dmasync, 1);  in qlnxr_init_srq_user_params()
2341 	if (rc)  in qlnxr_init_srq_user_params()
2342 		return rc;  in qlnxr_init_srq_user_params()
2344 	srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,  in qlnxr_init_srq_user_params()
2347 	if (IS_ERR(srq->prod_umem)) {  in qlnxr_init_srq_user_params()
2348 		qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);  in qlnxr_init_srq_user_params()
2349 		ib_umem_release(srq->usrq.umem);  in qlnxr_init_srq_user_params()
2352 			PTR_ERR(srq->prod_umem));  in qlnxr_init_srq_user_params()
2354 		return PTR_ERR(srq->prod_umem);  in qlnxr_init_srq_user_params()
2357 	sg = srq->prod_umem->sg_head.sgl;  in qlnxr_init_srq_user_params()
2358 	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);  in qlnxr_init_srq_user_params()
2369 	struct qlnxr_srq_hwq_info	*hw_srq  = &srq->hw_srq;  in qlnxr_alloc_srq_kernel_params()
2373 	int				rc;  in qlnxr_alloc_srq_kernel_params()  local
2376 	ha = dev->ha;  in qlnxr_alloc_srq_kernel_params()
2380 	va = qlnx_dma_alloc_coherent(&dev->cdev,  in qlnxr_alloc_srq_kernel_params()
2385 		return -ENOMEM;  in qlnxr_alloc_srq_kernel_params()
2388 	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;  in qlnxr_alloc_srq_kernel_params()
2389 	hw_srq->virt_prod_pair_addr = va;  in qlnxr_alloc_srq_kernel_params()
2391 	max_wr = init_attr->attr.max_wr;  in qlnxr_alloc_srq_kernel_params()
2395         rc = ecore_chain_alloc(dev->cdev,  in qlnxr_alloc_srq_kernel_params()
2401                    &hw_srq->pbl, NULL);  in qlnxr_alloc_srq_kernel_params()
2403 	if (rc) {  in qlnxr_alloc_srq_kernel_params()
2404 		QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc);  in qlnxr_alloc_srq_kernel_params()
2408 	hw_srq->max_wr = max_wr;  in qlnxr_alloc_srq_kernel_params()
2409 	hw_srq->num_elems = num_elems;  in qlnxr_alloc_srq_kernel_params()
2410 	hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ;  in qlnxr_alloc_srq_kernel_params()
2416 	qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr,  in qlnxr_alloc_srq_kernel_params()
2419 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_alloc_srq_kernel_params()
2420 	return rc;  in qlnxr_alloc_srq_kernel_params()
2433 	ha = dev->ha;  in qlnxr_init_common_qp_in_params()
2438 	params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);  in qlnxr_init_common_qp_in_params()
2439 	params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);  in qlnxr_init_common_qp_in_params()
2441 	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);  in qlnxr_init_common_qp_in_params()
2442 	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;  in qlnxr_init_common_qp_in_params()
2443 	params->pd = pd->pd_id;  in qlnxr_init_common_qp_in_params()
2444 	params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;  in qlnxr_init_common_qp_in_params()
2445 	params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;  in qlnxr_init_common_qp_in_params()
2446 	params->stats_queue = 0;  in qlnxr_init_common_qp_in_params()
2448 	params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;  in qlnxr_init_common_qp_in_params()
2450 	if (qp->srq) {  in qlnxr_init_common_qp_in_params()
2452 		params->srq_id = qp->srq->srq_id;  in qlnxr_init_common_qp_in_params()
2453 		params->use_srq = true;  in qlnxr_init_common_qp_in_params()
2454 		QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",  in qlnxr_init_common_qp_in_params()
2455 			params->srq_id, params->use_srq);  in qlnxr_init_common_qp_in_params()
2459 	params->srq_id = 0;  in qlnxr_init_common_qp_in_params()
2460 	params->use_srq = false;  in qlnxr_init_common_qp_in_params()
2470 	QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, "  in qlnxr_qp_user_print()
2471 		"rq_addr=0x%llx, rq_len=%zd\n",  in qlnxr_qp_user_print()
2472 		qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,  in qlnxr_qp_user_print()
2473 		qp->urq.buf_len);  in qlnxr_qp_user_print()
2481 	int		rc;  in qlnxr_idr_add()  local
2484 	ha = dev->ha;  in qlnxr_idr_add()
2492 		if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) {  in qlnxr_idr_add()
2494 			return -ENOMEM;  in qlnxr_idr_add()
2497 		mtx_lock(&dev->idr_lock);  in qlnxr_idr_add()
2499 		rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid);  in qlnxr_idr_add()
2501 		mtx_unlock(&dev->idr_lock);  in qlnxr_idr_add()
2503 	} while (rc == -EAGAIN);  in qlnxr_idr_add()
2505 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_idr_add()
2507 	return rc;  in qlnxr_idr_add()
2515 	ha = dev->ha;  in qlnxr_idr_remove()
2522 	mtx_lock(&dev->idr_lock);  in qlnxr_idr_remove()
2523 	idr_remove(&dev->qpidr, id);  in qlnxr_idr_remove()
2524 	mtx_unlock(&dev->idr_lock);  in qlnxr_idr_remove()
2538 	ha = dev->ha;  in qlnxr_iwarp_populate_user_qp()
2542 	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;  in qlnxr_iwarp_populate_user_qp()
2543 	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;  in qlnxr_iwarp_populate_user_qp()
2545 	qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,  in qlnxr_iwarp_populate_user_qp()
2546 			   &qp->usq.pbl_info);  in qlnxr_iwarp_populate_user_qp()
2548 	if (qp->srq) {  in qlnxr_iwarp_populate_user_qp()
2549 		QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq);  in qlnxr_iwarp_populate_user_qp()
2553 	qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;  in qlnxr_iwarp_populate_user_qp()
2554 	qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;  in qlnxr_iwarp_populate_user_qp()
2556 	qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,  in qlnxr_iwarp_populate_user_qp()
2557 			   &qp->urq.pbl_info);  in qlnxr_iwarp_populate_user_qp()
2577 	int rc = -EINVAL;  in qlnxr_create_user_qp()  local
2580 	ha = dev->ha;  in qlnxr_create_user_qp()
2584 	ib_ctx = ibpd->uobject->context;  in qlnxr_create_user_qp()
2587 	rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));  in qlnxr_create_user_qp()
2589 	if (rc) {  in qlnxr_create_user_qp()
2590 		QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc);  in qlnxr_create_user_qp()
2591 		return rc;  in qlnxr_create_user_qp()
2594 	/* SQ - read access only (0), dma sync not required (0) */  in qlnxr_create_user_qp()
2595 	rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,  in qlnxr_create_user_qp()
2598 	if (rc) {  in qlnxr_create_user_qp()
2599 		QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);  in qlnxr_create_user_qp()
2600 		return rc;  in qlnxr_create_user_qp()
2603 	if (!qp->srq) {  in qlnxr_create_user_qp()
2604 		/* RQ - read access only (0), dma sync not required (0) */  in qlnxr_create_user_qp()
2605 		rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,  in qlnxr_create_user_qp()
2609 		if (rc) {  in qlnxr_create_user_qp()
2610 			QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc);  in qlnxr_create_user_qp()
2611 			return rc;  in qlnxr_create_user_qp()
2619 	in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;  in qlnxr_create_user_qp()
2620 	in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;  in qlnxr_create_user_qp()
2622 	if (!qp->srq) {  in qlnxr_create_user_qp()
2623 		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;  in qlnxr_create_user_qp()
2624 		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;  in qlnxr_create_user_qp()
2627 	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params);  in qlnxr_create_user_qp()
2629 	if (!qp->ecore_qp) {  in qlnxr_create_user_qp()
2630 		rc = -ENOMEM;  in qlnxr_create_user_qp()
2638 	qp->qp_id = out_params.qp_id;  in qlnxr_create_user_qp()
2639 	qp->icid = out_params.icid;  in qlnxr_create_user_qp()
2641 	rc = qlnxr_copy_qp_uresp(dev, qp, udata);  in qlnxr_create_user_qp()
2643 	if (rc) {  in qlnxr_create_user_qp()
2653 	rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);  in qlnxr_create_user_qp()
2655 	if (rc)  in qlnxr_create_user_qp()
2661 	QL_DPRINT12(ha, "exit[%d]\n", rc);  in qlnxr_create_user_qp()
2662 	return rc;  in qlnxr_create_user_qp()
2671 	ha = dev->ha;  in qlnxr_set_roce_db_info()
2673 	QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);  in qlnxr_set_roce_db_info()
2675 	qp->sq.db = dev->db_addr +  in qlnxr_set_roce_db_info()
2677 	qp->sq.db_data.data.icid = qp->icid + 1;  in qlnxr_set_roce_db_info()
2679 	if (!qp->srq) {  in qlnxr_set_roce_db_info()
2680 		qp->rq.db = dev->db_addr +  in qlnxr_set_roce_db_info()
2682 		qp->rq.db_data.data.icid = qp->icid;  in qlnxr_set_roce_db_info()
2696 	ha = dev->ha;  in qlnxr_set_iwarp_db_info()
2698 	QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq);  in qlnxr_set_iwarp_db_info()
2700 	qp->sq.db = dev->db_addr +  in qlnxr_set_iwarp_db_info()
2702 	qp->sq.db_data.data.icid = qp->icid;  in qlnxr_set_iwarp_db_info()
2704 	if (!qp->srq) {  in qlnxr_set_iwarp_db_info()
2705 		qp->rq.db = dev->db_addr +  in qlnxr_set_iwarp_db_info()
2707 		qp->rq.db_data.data.icid = qp->icid;  in qlnxr_set_iwarp_db_info()
2709 		qp->rq.iwarp_db2 = dev->db_addr +  in qlnxr_set_iwarp_db_info()
2711 		qp->rq.iwarp_db2_data.data.icid = qp->icid;  in qlnxr_set_iwarp_db_info()
2712 		qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;  in qlnxr_set_iwarp_db_info()
2716 		"qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n"  in qlnxr_set_iwarp_db_info()
2717 		"\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n"  in qlnxr_set_iwarp_db_info()
2718 		"\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x"  in qlnxr_set_iwarp_db_info()
2719 		" qp->rq.iwarp_db2.data.prod_val =0x%x\n",  in qlnxr_set_iwarp_db_info()
2720 		qp->sq.db, qp->sq.db_data.data.icid,  in qlnxr_set_iwarp_db_info()
2721 		qp->rq.db, qp->rq.db_data.data.icid,  in qlnxr_set_iwarp_db_info()
2722 		qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid,  in qlnxr_set_iwarp_db_info()
2723 		qp->rq.iwarp_db2_data.data.value);  in qlnxr_set_iwarp_db_info()
2737 	int		rc;  in qlnxr_roce_create_kernel_qp()  local
2740 	ha = dev->ha;  in qlnxr_roce_create_kernel_qp()
2744         rc = ecore_chain_alloc(  in qlnxr_roce_create_kernel_qp()
2745                 dev->cdev,  in qlnxr_roce_create_kernel_qp()
2751                 &qp->sq.pbl,  in qlnxr_roce_create_kernel_qp()
2754 	if (rc) {  in qlnxr_roce_create_kernel_qp()
2755 		QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc);  in qlnxr_roce_create_kernel_qp()
2756 		return rc;  in qlnxr_roce_create_kernel_qp()
2759 	in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl);  in qlnxr_roce_create_kernel_qp()
2760 	in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl);  in qlnxr_roce_create_kernel_qp()
2762 	if (!qp->srq) {  in qlnxr_roce_create_kernel_qp()
2763                 rc = ecore_chain_alloc(  in qlnxr_roce_create_kernel_qp()
2764                         dev->cdev,  in qlnxr_roce_create_kernel_qp()
2770                         &qp->rq.pbl,  in qlnxr_roce_create_kernel_qp()
2773 		if (rc) {  in qlnxr_roce_create_kernel_qp()
2775 				"ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc);  in qlnxr_roce_create_kernel_qp()
2776 			return rc;  in qlnxr_roce_create_kernel_qp()
2779 		in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl);  in qlnxr_roce_create_kernel_qp()
2780 		in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl);  in qlnxr_roce_create_kernel_qp()
2783 	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);  in qlnxr_roce_create_kernel_qp()
2785 	if (!qp->ecore_qp) {  in qlnxr_roce_create_kernel_qp()
2786 		QL_DPRINT11(ha, "qp->ecore_qp == NULL\n");  in qlnxr_roce_create_kernel_qp()
2787 		return -EINVAL;  in qlnxr_roce_create_kernel_qp()
2790 	qp->qp_id = out_params.qp_id;  in qlnxr_roce_create_kernel_qp()
2791 	qp->icid = out_params.icid;  in qlnxr_roce_create_kernel_qp()
2809 	int rc;  in qlnxr_iwarp_create_kernel_qp()  local
2812 	ha = dev->ha;  in qlnxr_iwarp_create_kernel_qp()
2816 	in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems,  in qlnxr_iwarp_create_kernel_qp()
2819 	in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems,  in qlnxr_iwarp_create_kernel_qp()
2823 	QL_DPRINT12(ha, "n_sq_elems = 0x%x"  in qlnxr_iwarp_create_kernel_qp()
2824 		" n_rq_elems = 0x%x in_params\n"  in qlnxr_iwarp_create_kernel_qp()
2825 		"\t\t\tqp_handle_lo\t\t= 0x%08x\n"  in qlnxr_iwarp_create_kernel_qp()
2826 		"\t\t\tqp_handle_hi\t\t= 0x%08x\n"  in qlnxr_iwarp_create_kernel_qp()
2827 		"\t\t\tqp_handle_async_lo\t\t= 0x%08x\n"  in qlnxr_iwarp_create_kernel_qp()
2828 		"\t\t\tqp_handle_async_hi\t\t= 0x%08x\n"  in qlnxr_iwarp_create_kernel_qp()
2829 		"\t\t\tuse_srq\t\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2830 		"\t\t\tsignal_all\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2831 		"\t\t\tfmr_and_reserved_lkey\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2832 		"\t\t\tpd\t\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2833 		"\t\t\tdpi\t\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2834 		"\t\t\tsq_cq_id\t\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2835 		"\t\t\tsq_num_pages\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2837 		"\t\t\tmax_sq_sges\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2838 		"\t\t\trq_cq_id\t\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2839 		"\t\t\trq_num_pages\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2841 		"\t\t\tsrq_id\t\t\t= 0x%x\n"  in qlnxr_iwarp_create_kernel_qp()
2842 		"\t\t\tstats_queue\t\t= 0x%x\n",  in qlnxr_iwarp_create_kernel_qp()
2844 		in_params->qp_handle_lo,  in qlnxr_iwarp_create_kernel_qp()
2845 		in_params->qp_handle_hi,  in qlnxr_iwarp_create_kernel_qp()
2846 		in_params->qp_handle_async_lo,  in qlnxr_iwarp_create_kernel_qp()
2847 		in_params->qp_handle_async_hi,  in qlnxr_iwarp_create_kernel_qp()
2848 		in_params->use_srq,  in qlnxr_iwarp_create_kernel_qp()
2849 		in_params->signal_all,  in qlnxr_iwarp_create_kernel_qp()
2850 		in_params->fmr_and_reserved_lkey,  in qlnxr_iwarp_create_kernel_qp()
2851 		in_params->pd,  in qlnxr_iwarp_create_kernel_qp()
2852 		in_params->dpi,  in qlnxr_iwarp_create_kernel_qp()
2853 		in_params->sq_cq_id,  in qlnxr_iwarp_create_kernel_qp()
2854 		in_params->sq_num_pages,  in qlnxr_iwarp_create_kernel_qp()
2855 		(void *)in_params->sq_pbl_ptr,  in qlnxr_iwarp_create_kernel_qp()
2856 		in_params->max_sq_sges,  in qlnxr_iwarp_create_kernel_qp()
2857 		in_params->rq_cq_id,  in qlnxr_iwarp_create_kernel_qp()
2858 		in_params->rq_num_pages,  in qlnxr_iwarp_create_kernel_qp()
2859 		(void *)in_params->rq_pbl_ptr,  in qlnxr_iwarp_create_kernel_qp()
2860 		in_params->srq_id,  in qlnxr_iwarp_create_kernel_qp()
2861 		in_params->stats_queue );  in qlnxr_iwarp_create_kernel_qp()
2866 	qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params);  in qlnxr_iwarp_create_kernel_qp()
2868 	if (!qp->ecore_qp) {  in qlnxr_iwarp_create_kernel_qp()
2870 		return -EINVAL;  in qlnxr_iwarp_create_kernel_qp()
2881         rc = ecore_chain_alloc(  in qlnxr_iwarp_create_kernel_qp()
2882                 dev->cdev,  in qlnxr_iwarp_create_kernel_qp()
2888                 &qp->sq.pbl,  in qlnxr_iwarp_create_kernel_qp()
2891 	if (rc) {  in qlnxr_iwarp_create_kernel_qp()
2893 			"ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc);  in qlnxr_iwarp_create_kernel_qp()
2904 	if (!qp->srq) {  in qlnxr_iwarp_create_kernel_qp()
2905                 rc = ecore_chain_alloc(  in qlnxr_iwarp_create_kernel_qp()
2906                         dev->cdev,  in qlnxr_iwarp_create_kernel_qp()
2912                         &qp->rq.pbl,  in qlnxr_iwarp_create_kernel_qp()
2915 		if (rc) {  in qlnxr_iwarp_create_kernel_qp()
2916 			QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl"  in qlnxr_iwarp_create_kernel_qp()
2917 				" failed rc = %d\n", rc);  in qlnxr_iwarp_create_kernel_qp()
2922 	QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n",  in qlnxr_iwarp_create_kernel_qp()
2925 	qp->qp_id = out_params.qp_id;  in qlnxr_iwarp_create_kernel_qp()
2926 	qp->icid = out_params.icid;  in qlnxr_iwarp_create_kernel_qp()
2934 	ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params);  in qlnxr_iwarp_create_kernel_qp()
2936 	QL_DPRINT12(ha, "exit rc = %d\n", rc);  in qlnxr_iwarp_create_kernel_qp()
2937 	return rc;  in qlnxr_iwarp_create_kernel_qp()
2948 	int rc = -EINVAL;  in qlnxr_create_kernel_qp()  local
2952 	struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);  in qlnxr_create_kernel_qp()
2955 	ha = dev->ha;  in qlnxr_create_kernel_qp()
2971 	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,  in qlnxr_create_kernel_qp()
2972 			      qattr->max_wqe);  in qlnxr_create_kernel_qp()
2974 	qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id),  in qlnxr_create_kernel_qp()
2976 	if (!qp->wqe_wr_id) {  in qlnxr_create_kernel_qp()
2978 		return -ENOMEM;  in qlnxr_create_kernel_qp()
2989 	qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1);  in qlnxr_create_kernel_qp()
2992 	if (!qp->srq) {  in qlnxr_create_kernel_qp()
2993 		qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id),  in qlnxr_create_kernel_qp()
2995 		if (!qp->rqe_wr_id) {  in qlnxr_create_kernel_qp()
2997 			kfree(qp->wqe_wr_id);  in qlnxr_create_kernel_qp()
2998 			return -ENOMEM;  in qlnxr_create_kernel_qp()
3007         in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);  in qlnxr_create_kernel_qp()
3009         in_params.pd = pd->pd_id;  in qlnxr_create_kernel_qp()
3010         in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;  in qlnxr_create_kernel_qp()
3011         in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid;  in qlnxr_create_kernel_qp()
3014         in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid;  in qlnxr_create_kernel_qp()
3016         if (qp->srq) {  in qlnxr_create_kernel_qp()
3018                 in_params.srq_id = qp->srq->srq_id;  in qlnxr_create_kernel_qp()
3020                 QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n",  in qlnxr_create_kernel_qp()
3027 	n_sq_entries = attrs->cap.max_send_wr;  in qlnxr_create_kernel_qp()
3028 	n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe);  in qlnxr_create_kernel_qp()
3032 	n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE;  in qlnxr_create_kernel_qp()
3035 		rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params,  in qlnxr_create_kernel_qp()
3038 		rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params,  in qlnxr_create_kernel_qp()
3042 	if (rc)  in qlnxr_create_kernel_qp()
3045 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_create_kernel_qp()
3046 	return rc;  in qlnxr_create_kernel_qp()
3054 	struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device);  in qlnxr_create_qp()
3057 	int rc = 0;  in qlnxr_create_qp()  local
3060 	ha = dev->ha;  in qlnxr_create_qp()
3064 	rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata);  in qlnxr_create_qp()
3065 	if (rc) {  in qlnxr_create_qp()
3066 		QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc);  in qlnxr_create_qp()
3067 		return ERR_PTR(rc);  in qlnxr_create_qp()
3073 		attrs->event_handler, pd,  in qlnxr_create_qp()
3074 		get_qlnxr_cq(attrs->send_cq),  in qlnxr_create_qp()
3075 		get_qlnxr_cq(attrs->send_cq)->icid,  in qlnxr_create_qp()
3076 		get_qlnxr_cq(attrs->recv_cq),  in qlnxr_create_qp()
3077 		get_qlnxr_cq(attrs->recv_cq)->icid);  in qlnxr_create_qp()
3083 		return ERR_PTR(-ENOMEM);  in qlnxr_create_qp()
3088 	if (attrs->qp_type == IB_QPT_GSI) {  in qlnxr_create_qp()
3094 		rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs);  in qlnxr_create_qp()
3096 		if (rc) {  in qlnxr_create_qp()
3101 		rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs);  in qlnxr_create_qp()
3103 		if (rc) {  in qlnxr_create_qp()
3109 	qp->ibqp.qp_num = qp->qp_id;  in qlnxr_create_qp()
3111 	rc = qlnxr_idr_add(dev, qp, qp->qp_id);  in qlnxr_create_qp()
3113 	if (rc) {  in qlnxr_create_qp()
3118 	QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp);  in qlnxr_create_qp()
3120 	return &qp->ibqp;  in qlnxr_create_qp()
3123 	return ERR_PTR(-EFAULT);  in qlnxr_create_qp()
3206 	ecore_chain_reset(&qph->pbl);  in qlnxr_reset_qp_hwq_info()
3207 	qph->prod = qph->cons = 0;  in qlnxr_reset_qp_hwq_info()
3208 	qph->wqe_cons = 0;  in qlnxr_reset_qp_hwq_info()
3209 	qph->db_data.data.value = cpu_to_le16(0);  in qlnxr_reset_qp_hwq_info()
3224 	ha = dev->ha;  in qlnxr_update_qp_state()
3225 	cdev = &ha->cdev;  in qlnxr_update_qp_state()
3227 	QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n",  in qlnxr_update_qp_state()
3228 		qp, new_state, qp->state);  in qlnxr_update_qp_state()
3230 	if (new_state == qp->state) {  in qlnxr_update_qp_state()
3234 	switch (qp->state) {  in qlnxr_update_qp_state()
3238 			qp->prev_wqe_size = 0;  in qlnxr_update_qp_state()
3239 			qlnxr_reset_qp_hwq_info(&qp->sq);  in qlnxr_update_qp_state()
3240 			if (!(qp->srq))  in qlnxr_update_qp_state()
3241 				qlnxr_reset_qp_hwq_info(&qp->rq);  in qlnxr_update_qp_state()
3244 			status = -EINVAL;  in qlnxr_update_qp_state()
3249 		/* INIT->XXX */  in qlnxr_update_qp_state()
3253 			if (qp->srq)  in qlnxr_update_qp_state()
3256 			//writel(qp->rq.db_data.raw, qp->rq.db);  in qlnxr_update_qp_state()
3258 			//	writel(qp->rq.iwarp_db2_data.raw,  in qlnxr_update_qp_state()
3259 			//	       qp->rq.iwarp_db2);  in qlnxr_update_qp_state()
3261 			reg_addr = (uint32_t)((uint8_t *)qp->rq.db -  in qlnxr_update_qp_state()
3262 					(uint8_t *)cdev->doorbells);  in qlnxr_update_qp_state()
3264 			bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);  in qlnxr_update_qp_state()
3265 			bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);  in qlnxr_update_qp_state()
3268 				reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -  in qlnxr_update_qp_state()
3269 					(uint8_t *)cdev->doorbells);  in qlnxr_update_qp_state()
3270 				bus_write_4(ha->pci_dbells, reg_addr,\  in qlnxr_update_qp_state()
3271 					qp->rq.iwarp_db2_data.raw);  in qlnxr_update_qp_state()
3272 				bus_barrier(ha->pci_dbells,  0, 0,\  in qlnxr_update_qp_state()
3284 			status = -EINVAL;  in qlnxr_update_qp_state()
3289 		/* RTR->XXX */  in qlnxr_update_qp_state()
3297 			status = -EINVAL;  in qlnxr_update_qp_state()
3302 		/* RTS->XXX */  in qlnxr_update_qp_state()
3310 			status = -EINVAL;  in qlnxr_update_qp_state()
3315 		/* SQD->XXX */  in qlnxr_update_qp_state()
3322 			status = -EINVAL;  in qlnxr_update_qp_state()
3327 		/* ERR->XXX */  in qlnxr_update_qp_state()
3330 			if ((qp->rq.prod != qp->rq.cons) ||  in qlnxr_update_qp_state()
3331 			    (qp->sq.prod != qp->sq.cons)) {  in qlnxr_update_qp_state()
3333 					"Error->Reset with rq/sq "  in qlnxr_update_qp_state()
3334 					"not empty rq.prod=0x%x rq.cons=0x%x"  in qlnxr_update_qp_state()
3335 					" sq.prod=0x%x sq.cons=0x%x\n",  in qlnxr_update_qp_state()
3336 					qp->rq.prod, qp->rq.cons,  in qlnxr_update_qp_state()
3337 					qp->sq.prod, qp->sq.cons);  in qlnxr_update_qp_state()
3338 				status = -EINVAL;  in qlnxr_update_qp_state()
3342 			status = -EINVAL;  in qlnxr_update_qp_state()
3347 		status = -EINVAL;  in qlnxr_update_qp_state()
3361 	int rc = 0;  in qlnxr_modify_qp()  local
3363 	struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev);  in qlnxr_modify_qp()
3366 	struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx);  in qlnxr_modify_qp()
3369 	ha = dev->ha;  in qlnxr_modify_qp()
3372 		"enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n",  in qlnxr_modify_qp()
3373 		qp, attr_mask, attr->qp_state, udata);  in qlnxr_modify_qp()
3375 	old_qp_state = qlnxr_get_ibqp_state(qp->state);  in qlnxr_modify_qp()
3377 		new_qp_state = attr->qp_state;  in qlnxr_modify_qp()
3384 					ibqp->qp_type,  in qlnxr_modify_qp()
3387 				"invalid attribute mask=0x%x"  in qlnxr_modify_qp()
3388 				" specified for qpn=0x%x of type=0x%x \n"  in qlnxr_modify_qp()
3389 				" old_qp_state=0x%x, new_qp_state=0x%x\n",  in qlnxr_modify_qp()
3390 				attr_mask, qp->qp_id, ibqp->qp_type,  in qlnxr_modify_qp()
3392 			rc = -EINVAL;  in qlnxr_modify_qp()
3400 		qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state);  in qlnxr_modify_qp()
3411 		if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) {  in qlnxr_modify_qp()
3412 			rc = -EINVAL;  in qlnxr_modify_qp()
3420 		qp->qkey = attr->qkey;  in qlnxr_modify_qp()
3428 			attr->qp_access_flags & IB_ACCESS_REMOTE_READ;  in qlnxr_modify_qp()
3430 			attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE;  in qlnxr_modify_qp()
3432 			attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC;  in qlnxr_modify_qp()
3437 			if (attr->path_mtu < IB_MTU_256 ||  in qlnxr_modify_qp()
3438 			    attr->path_mtu > IB_MTU_4096) {  in qlnxr_modify_qp()
3442 					" attr->path_mtu = [%d]\n",  in qlnxr_modify_qp()
3443 					attr->path_mtu);  in qlnxr_modify_qp()
3445 				rc = -EINVAL;  in qlnxr_modify_qp()
3448 			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),  in qlnxr_modify_qp()
3450 						iboe_get_mtu(if_getmtu(dev->ha->ifp))));  in qlnxr_modify_qp()
3453 		if (qp->mtu == 0) {  in qlnxr_modify_qp()
3454 			qp->mtu = ib_mtu_enum_to_int(  in qlnxr_modify_qp()
3455 					iboe_get_mtu(if_getmtu(dev->ha->ifp)));  in qlnxr_modify_qp()
3456 			QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n",  in qlnxr_modify_qp()
3457 				qp->mtu);  in qlnxr_modify_qp()
3464 		qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;  in qlnxr_modify_qp()
3465 		qp_params.flow_label = attr->ah_attr.grh.flow_label;  in qlnxr_modify_qp()
3466 		qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;  in qlnxr_modify_qp()
3468 		qp->sgid_idx = attr->ah_attr.grh.sgid_index;  in qlnxr_modify_qp()
3472 		rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr);  in qlnxr_modify_qp()
3473 		if (rc)  in qlnxr_modify_qp()
3474 			return rc;  in qlnxr_modify_qp()
3477 		memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);  in qlnxr_modify_qp()
3479 		QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n",  in qlnxr_modify_qp()
3482 		QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n",  in qlnxr_modify_qp()
3486 			"remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n",  in qlnxr_modify_qp()
3494 		qp_params.mtu = qp->mtu;  in qlnxr_modify_qp()
3499 		if (qp->mtu) {  in qlnxr_modify_qp()
3500 			qp_params.mtu = qp->mtu;  in qlnxr_modify_qp()
3503 						iboe_get_mtu(if_getmtu(dev->ha->ifp)));  in qlnxr_modify_qp()
3511 		qp_params.ack_timeout = attr->timeout;  in qlnxr_modify_qp()
3512 		if (attr->timeout) {  in qlnxr_modify_qp()
3523 			temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;  in qlnxr_modify_qp()
3532 		qp_params.retry_cnt = attr->retry_cnt;  in qlnxr_modify_qp()
3539 		qp_params.rnr_retry_cnt = attr->rnr_retry;  in qlnxr_modify_qp()
3546 		qp_params.rq_psn = attr->rq_psn;  in qlnxr_modify_qp()
3547 		qp->rq_psn = attr->rq_psn;  in qlnxr_modify_qp()
3551 		if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) {  in qlnxr_modify_qp()
3552 			rc = -EINVAL;  in qlnxr_modify_qp()
3555 				attr->max_rd_atomic,  in qlnxr_modify_qp()
3556 				qattr->max_qp_req_rd_atomic_resc);  in qlnxr_modify_qp()
3563 		qp_params.max_rd_atomic_req = attr->max_rd_atomic;  in qlnxr_modify_qp()
3570 		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;  in qlnxr_modify_qp()
3577 		qp_params.sq_psn = attr->sq_psn;  in qlnxr_modify_qp()
3578 		qp->sq_psn = attr->sq_psn;  in qlnxr_modify_qp()
3582 		if (attr->max_dest_rd_atomic >  in qlnxr_modify_qp()
3583 		    qattr->max_qp_resp_rd_atomic_resc) {  in qlnxr_modify_qp()
3587 				attr->max_dest_rd_atomic,  in qlnxr_modify_qp()
3588 				qattr->max_qp_resp_rd_atomic_resc);  in qlnxr_modify_qp()
3590 			rc = -EINVAL;  in qlnxr_modify_qp()
3597 		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;  in qlnxr_modify_qp()
3605 		qp_params.dest_qp = attr->dest_qp_num;  in qlnxr_modify_qp()
3606 		qp->dest_qp_num = attr->dest_qp_num;  in qlnxr_modify_qp()
3615 	if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) &&  in qlnxr_modify_qp()
3617 		qp->state = ECORE_ROCE_QP_STATE_ERR;  in qlnxr_modify_qp()
3619 	if (qp->qp_type != IB_QPT_GSI)  in qlnxr_modify_qp()
3620 		rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);  in qlnxr_modify_qp()
3623 		if ((qp->qp_type != IB_QPT_GSI) && (!udata))  in qlnxr_modify_qp()
3624 			rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state);  in qlnxr_modify_qp()
3625 		qp->state = qp_params.new_state;  in qlnxr_modify_qp()
3630 	return rc;  in qlnxr_modify_qp()
3638 	if (params->incoming_rdma_write_en)  in qlnxr_to_ib_qp_acc_flags()
3640 	if (params->incoming_rdma_read_en)  in qlnxr_to_ib_qp_acc_flags()
3642 	if (params->incoming_atomic_en)  in qlnxr_to_ib_qp_acc_flags()
3644 	if (true) /* FIXME -> local write ?? */  in qlnxr_to_ib_qp_acc_flags()
3689 	int rc = 0;  in qlnxr_query_qp()  local
3692 	struct qlnxr_dev *dev = qp->dev;  in qlnxr_query_qp()
3695 	ha = dev->ha;  in qlnxr_query_qp()
3701 	rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, &params);  in qlnxr_query_qp()
3702 	if (rc)  in qlnxr_query_qp()
3708 	qp_attr->qp_state = qlnxr_get_ibqp_state(params.state);  in qlnxr_query_qp()
3709 	qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state);  in qlnxr_query_qp()
3717 	qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu);  in qlnxr_query_qp()
3718 	qp_attr->path_mig_state = IB_MIG_MIGRATED;  in qlnxr_query_qp()
3719 	qp_attr->rq_psn = params.rq_psn;  in qlnxr_query_qp()
3720 	qp_attr->sq_psn = params.sq_psn;  in qlnxr_query_qp()
3721 	qp_attr->dest_qp_num = params.dest_qp;  in qlnxr_query_qp()
3723 	qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(&params);  in qlnxr_query_qp()
3725 	QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x "  in qlnxr_query_qp()
3726 		"path_mtu = %d qp_access_flags = 0x%x\n",  in qlnxr_query_qp()
3727 		qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu,  in qlnxr_query_qp()
3728 		qp_attr->qp_access_flags);  in qlnxr_query_qp()
3730 	qp_attr->cap.max_send_wr = qp->sq.max_wr;  in qlnxr_query_qp()
3731 	qp_attr->cap.max_recv_wr = qp->rq.max_wr;  in qlnxr_query_qp()
3732 	qp_attr->cap.max_send_sge = qp->sq.max_sges;  in qlnxr_query_qp()
3733 	qp_attr->cap.max_recv_sge = qp->rq.max_sges;  in qlnxr_query_qp()
3734 	qp_attr->cap.max_inline_data = qp->max_inline_data;  in qlnxr_query_qp()
3735 	qp_init_attr->cap = qp_attr->cap;  in qlnxr_query_qp()
3737 	memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], &params.dgid.bytes[0],  in qlnxr_query_qp()
3738 	       sizeof(qp_attr->ah_attr.grh.dgid.raw));  in qlnxr_query_qp()
3740 	qp_attr->ah_attr.grh.flow_label = params.flow_label;  in qlnxr_query_qp()
3741 	qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;  in qlnxr_query_qp()
3742 	qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;  in qlnxr_query_qp()
3743 	qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;  in qlnxr_query_qp()
3745 	qp_attr->ah_attr.ah_flags = IB_AH_GRH;  in qlnxr_query_qp()
3746 	qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */  in qlnxr_query_qp()
3747 	qp_attr->ah_attr.sl = 0;/* FIXME -> check this */  in qlnxr_query_qp()
3748 	qp_attr->timeout = params.timeout;  in qlnxr_query_qp()
3749 	qp_attr->rnr_retry = params.rnr_retry;  in qlnxr_query_qp()
3750 	qp_attr->retry_cnt = params.retry_cnt;  in qlnxr_query_qp()
3751 	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;  in qlnxr_query_qp()
3752 	qp_attr->pkey_index = params.pkey_index;  in qlnxr_query_qp()
3753 	qp_attr->port_num = 1; /* FIXME -> check this */  in qlnxr_query_qp()
3754 	qp_attr->ah_attr.src_path_bits = 0;  in qlnxr_query_qp()
3755 	qp_attr->ah_attr.static_rate = 0;  in qlnxr_query_qp()
3756 	qp_attr->alt_pkey_index = 0;  in qlnxr_query_qp()
3757 	qp_attr->alt_port_num = 0;  in qlnxr_query_qp()
3758 	qp_attr->alt_timeout = 0;  in qlnxr_query_qp()
3759 	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));  in qlnxr_query_qp()
3761 	qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0;  in qlnxr_query_qp()
3762 	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;  in qlnxr_query_qp()
3763 	qp_attr->max_rd_atomic = params.max_rd_atomic;  in qlnxr_query_qp()
3764 	qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0;  in qlnxr_query_qp()
3767 		qp_attr->cap.max_inline_data);  in qlnxr_query_qp()
3771 	return rc;  in qlnxr_query_qp()
3779 	ha = dev->ha;  in qlnxr_cleanup_user()
3783 	if (qp->usq.umem)  in qlnxr_cleanup_user()
3784 		ib_umem_release(qp->usq.umem);  in qlnxr_cleanup_user()
3786 	qp->usq.umem = NULL;  in qlnxr_cleanup_user()
3788 	if (qp->urq.umem)  in qlnxr_cleanup_user()
3789 		ib_umem_release(qp->urq.umem);  in qlnxr_cleanup_user()
3791 	qp->urq.umem = NULL;  in qlnxr_cleanup_user()
3802 	ha = dev->ha;  in qlnxr_cleanup_kernel()
3808 		ha->qlnxr_debug = 1;  in qlnxr_cleanup_kernel()
3809 //		ecore_chain_free(dev->cdev, &qp->sq.pbl);  in qlnxr_cleanup_kernel()
3810 		ha->qlnxr_debug = 0;  in qlnxr_cleanup_kernel()
3811 		kfree(qp->wqe_wr_id);  in qlnxr_cleanup_kernel()
3816 		ha->qlnxr_debug = 1;  in qlnxr_cleanup_kernel()
3817 	//	ecore_chain_free(dev->cdev, &qp->rq.pbl);  in qlnxr_cleanup_kernel()
3818 		ha->qlnxr_debug = 0;  in qlnxr_cleanup_kernel()
3819 		kfree(qp->rqe_wr_id);  in qlnxr_cleanup_kernel()
3830 	int		rc = 0;  in qlnxr_free_qp_resources()  local
3834 	ha = dev->ha;  in qlnxr_free_qp_resources()
3839 	if (qp->qp_type != IB_QPT_GSI) {  in qlnxr_free_qp_resources()
3840 		rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,  in qlnxr_free_qp_resources()
3842 		if (rc)  in qlnxr_free_qp_resources()
3843 			return rc;  in qlnxr_free_qp_resources()
3857 	if (qp->qp_type != IB_QPT_GSI) {  in qlnxr_free_qp_resources()
3858 		rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp,  in qlnxr_free_qp_resources()
3860 		if (rc)  in qlnxr_free_qp_resources()
3861 			return rc;  in qlnxr_free_qp_resources()
3872 	struct qlnxr_dev *dev = qp->dev;  in qlnxr_destroy_qp()
3873 	int rc = 0;  in qlnxr_destroy_qp()  local
3878 	ha = dev->ha;  in qlnxr_destroy_qp()
3880 	QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type);  in qlnxr_destroy_qp()
3882 	qp->destroyed = 1;  in qlnxr_destroy_qp()
3884 	if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET |  in qlnxr_destroy_qp()
3894 	if (qp->qp_type == IB_QPT_GSI)  in qlnxr_destroy_qp()
3897 	qp->sig = ~qp->sig;  in qlnxr_destroy_qp()
3901 	if (atomic_dec_and_test(&qp->refcnt)) {  in qlnxr_destroy_qp()
3903 		qlnxr_idr_remove(dev, qp->qp_id);  in qlnxr_destroy_qp()
3907 	return rc;  in qlnxr_destroy_qp()
3913 	return (((wq->prod + 1) % wq->max_wr) == wq->cons);  in qlnxr_wq_is_full()
3945 	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);  in qlnxr_prepare_sq_inline_data()
3948 	ha = dev->ha;  in qlnxr_prepare_sq_inline_data()
3970 	for (i = 0; i < wr->num_sge; i++) {  in qlnxr_prepare_sq_inline_data()
3971 		u32 len = wr->sg_list[i].length;  in qlnxr_prepare_sq_inline_data()
3972 		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;  in qlnxr_prepare_sq_inline_data()
3979 				wqe = (char *)ecore_chain_produce(&qp->sq.pbl);  in qlnxr_prepare_sq_inline_data()
3992 			seg_siz -= cur;  in qlnxr_prepare_sq_inline_data()
3995 			len -= cur;  in qlnxr_prepare_sq_inline_data()
3997 			/* swap fully-completed segments */  in qlnxr_prepare_sq_inline_data()
4019 	ha = dev->ha;  in qlnxr_prepare_sq_sges()
4021 	QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge);  in qlnxr_prepare_sq_sges()
4023 	for (i = 0; i < wr->num_sge; i++) {  in qlnxr_prepare_sq_sges()
4024 		struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl);  in qlnxr_prepare_sq_sges()
4026 		TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr);  in qlnxr_prepare_sq_sges()
4027 		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);  in qlnxr_prepare_sq_sges()
4028 		sge->length = cpu_to_le32(wr->sg_list[i].length);  in qlnxr_prepare_sq_sges()
4029 		data_size += wr->sg_list[i].length;  in qlnxr_prepare_sq_sges()
4033 		*wqe_size += wr->num_sge;  in qlnxr_prepare_sq_sges()
4050 	ha = dev->ha;  in qlnxr_prepare_sq_rdma_data()
4054 	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);  in qlnxr_prepare_sq_rdma_data()
4055 	TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr);  in qlnxr_prepare_sq_rdma_data()
4057 	if (wr->send_flags & IB_SEND_INLINE) {  in qlnxr_prepare_sq_rdma_data()
4060 		return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size,  in qlnxr_prepare_sq_rdma_data()
4061 				wr, bad_wr, &rwqe->flags, flags);  in qlnxr_prepare_sq_rdma_data()
4064 	ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr);  in qlnxr_prepare_sq_rdma_data()
4066 	QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);  in qlnxr_prepare_sq_rdma_data()
4082 	ha = dev->ha;  in qlnxr_prepare_sq_send_data()
4088 	if (wr->send_flags & IB_SEND_INLINE) {  in qlnxr_prepare_sq_send_data()
4091 		return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size,  in qlnxr_prepare_sq_send_data()
4092 				wr, bad_wr, &swqe->flags, flags);  in qlnxr_prepare_sq_send_data()
4095 	ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr);  in qlnxr_prepare_sq_send_data()
4097 	QL_DPRINT12(ha, "exit ret = 0x%x\n", ret);  in qlnxr_prepare_sq_send_data()
4107 	ha = dev->ha;  in qlnx_handle_completed_mrs()
4109 	int work = info->completed - info->completed_handled - 1;  in qlnx_handle_completed_mrs()
4113 	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {  in qlnx_handle_completed_mrs()
4121 		pbl = list_first_entry(&info->inuse_pbl_list,  in qlnx_handle_completed_mrs()
4124 		list_del(&pbl->list_entry);  in qlnx_handle_completed_mrs()
4125 		list_add_tail(&pbl->list_entry, &info->free_pbl_list);  in qlnx_handle_completed_mrs()
4126 		info->completed_handled++;  in qlnx_handle_completed_mrs()
4137 	struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr);  in qlnxr_prepare_reg()
4140 	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl);  in qlnxr_prepare_reg()
4141 	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);  in qlnxr_prepare_reg()
4142 	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);  in qlnxr_prepare_reg()
4143 	fwqe1->l_key = wr->key;  in qlnxr_prepare_reg()
4145 	fwqe2->access_ctrl = 0;  in qlnxr_prepare_reg()
4147 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,  in qlnxr_prepare_reg()
4148 		!!(wr->access & IB_ACCESS_REMOTE_READ));  in qlnxr_prepare_reg()
4149 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,  in qlnxr_prepare_reg()
4150 		!!(wr->access & IB_ACCESS_REMOTE_WRITE));  in qlnxr_prepare_reg()
4151 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,  in qlnxr_prepare_reg()
4152 		!!(wr->access & IB_ACCESS_REMOTE_ATOMIC));  in qlnxr_prepare_reg()
4153 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);  in qlnxr_prepare_reg()
4154 	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,  in qlnxr_prepare_reg()
4155 		!!(wr->access & IB_ACCESS_LOCAL_WRITE));  in qlnxr_prepare_reg()
4156 	fwqe2->fmr_ctrl = 0;  in qlnxr_prepare_reg()
4158 	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,  in qlnxr_prepare_reg()
4159 		ilog2(mr->ibmr.page_size) - 12);  in qlnxr_prepare_reg()
4161 	fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */  in qlnxr_prepare_reg()
4162 	fwqe2->length_lo = mr->ibmr.length;  in qlnxr_prepare_reg()
4163 	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);  in qlnxr_prepare_reg()
4164 	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);  in qlnxr_prepare_reg()
4166 	qp->wqe_wr_id[qp->sq.prod].mr = mr;  in qlnxr_prepare_reg()
4200 	struct qlnxr_dev *dev = qp->dev;  in qlnxr_can_post_send()
4203 	ha = dev->ha;  in qlnxr_can_post_send()
4208 	err_wr = wr->num_sge > qp->sq.max_sges;  in qlnxr_can_post_send()
4209 	wq_is_full = qlnxr_wq_is_full(&qp->sq);  in qlnxr_can_post_send()
4210 	pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) <  in qlnxr_can_post_send()
4214 		    !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) {  in qlnxr_can_post_send()
4215 			qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL;  in qlnxr_can_post_send()
4220 				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",  in qlnxr_can_post_send()
4221 				qp, wr, qp->err_bitmap);  in qlnxr_can_post_send()
4225 		    !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) {  in qlnxr_can_post_send()
4226 			qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR;  in qlnxr_can_post_send()
4231 				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",  in qlnxr_can_post_send()
4232 				qp, wr, qp->err_bitmap);  in qlnxr_can_post_send()
4236 		    !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) {  in qlnxr_can_post_send()
4237 			qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL;  in qlnxr_can_post_send()
4242 				"[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n",  in qlnxr_can_post_send()
4243 				qp, wr, qp->err_bitmap);  in qlnxr_can_post_send()
4256 	struct qlnxr_dev	*dev = get_qlnxr_dev(ibqp->device);  in qlnxr_post_send()
4259 	int 			status = 0, rc = 0;  in qlnxr_post_send()  local
4265 	ha = dev->ha;  in qlnxr_post_send()
4270 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_post_send()
4271 		return -EINVAL;  in qlnxr_post_send()
4273 	if (qp->qp_type == IB_QPT_GSI)  in qlnxr_post_send()
4276 	spin_lock_irqsave(&qp->q_lock, flags);  in qlnxr_post_send()
4278 	if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) &&  in qlnxr_post_send()
4279 	    (qp->state != ECORE_ROCE_QP_STATE_ERR) &&  in qlnxr_post_send()
4280 	    (qp->state != ECORE_ROCE_QP_STATE_SQD)) {  in qlnxr_post_send()
4281 		spin_unlock_irqrestore(&qp->q_lock, flags);  in qlnxr_post_send()
4283 		QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n",  in qlnxr_post_send()
4284 			qp->icid, qp->state);  in qlnxr_post_send()
4285 		return -EINVAL;  in qlnxr_post_send()
4305 			status = -ENOMEM;  in qlnxr_post_send()
4310 		wqe = ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4312 		qp->wqe_wr_id[qp->sq.prod].signaled =  in qlnxr_post_send()
4313 			!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;  in qlnxr_post_send()
4316 		wqe->flags = 0;  in qlnxr_post_send()
4317 		wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK <<  in qlnxr_post_send()
4320 		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \  in qlnxr_post_send()
4321 			!!(wr->send_flags & IB_SEND_SOLICITED));  in qlnxr_post_send()
4323 		comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) ||  in qlnxr_post_send()
4324 				(qp->signaled);  in qlnxr_post_send()
4326 		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);  in qlnxr_post_send()
4327 		SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,  \  in qlnxr_post_send()
4328 			!!(wr->send_flags & IB_SEND_FENCE));  in qlnxr_post_send()
4330 		wqe->prev_wqe_size = qp->prev_wqe_size;  in qlnxr_post_send()
4332 		qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode);  in qlnxr_post_send()
4334 		switch (wr->opcode) {  in qlnxr_post_send()
4337 			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;  in qlnxr_post_send()
4339 			swqe->wqe_size = 2;  in qlnxr_post_send()
4341 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4342 			swqe->inv_key_or_imm_data =  in qlnxr_post_send()
4343 				cpu_to_le32(wr->ex.imm_data);  in qlnxr_post_send()
4344 			swqe->length = cpu_to_le32(  in qlnxr_post_send()
4349 			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;  in qlnxr_post_send()
4350 			qp->prev_wqe_size = swqe->wqe_size;  in qlnxr_post_send()
4351 			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;  in qlnxr_post_send()
4353 			QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n",  in qlnxr_post_send()
4354 				swqe->length, wr->ex.imm_data);  in qlnxr_post_send()
4360 			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;  in qlnxr_post_send()
4363 			swqe->wqe_size = 2;  in qlnxr_post_send()
4365 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4366 			swqe->length = cpu_to_le32(  in qlnxr_post_send()
4370 			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;  in qlnxr_post_send()
4371 			qp->prev_wqe_size = swqe->wqe_size;  in qlnxr_post_send()
4372 			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;  in qlnxr_post_send()
4375 				swqe->length);  in qlnxr_post_send()
4381 			wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;  in qlnxr_post_send()
4384 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4385 			swqe->wqe_size = 2;  in qlnxr_post_send()
4386 			swqe->inv_key_or_imm_data =  in qlnxr_post_send()
4387 				cpu_to_le32(wr->ex.invalidate_rkey);  in qlnxr_post_send()
4388 			swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev,  in qlnxr_post_send()
4390 			qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;  in qlnxr_post_send()
4391 			qp->prev_wqe_size = swqe->wqe_size;  in qlnxr_post_send()
4392 			qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;  in qlnxr_post_send()
4395 				swqe->length);  in qlnxr_post_send()
4400 			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;  in qlnxr_post_send()
4403 			rwqe->wqe_size = 2;  in qlnxr_post_send()
4404 			rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));  in qlnxr_post_send()
4406 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4407 			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,  in qlnxr_post_send()
4409 			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;  in qlnxr_post_send()
4410 			qp->prev_wqe_size = rwqe->wqe_size;  in qlnxr_post_send()
4411 			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;  in qlnxr_post_send()
4414 				"RDMA WRITE w/ IMM length = %d imm data=%x\n",  in qlnxr_post_send()
4415 				rwqe->length, rwqe->imm_data);  in qlnxr_post_send()
4421 			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;  in qlnxr_post_send()
4424 			rwqe->wqe_size = 2;  in qlnxr_post_send()
4426 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4427 			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,  in qlnxr_post_send()
4429 			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;  in qlnxr_post_send()
4430 			qp->prev_wqe_size = rwqe->wqe_size;  in qlnxr_post_send()
4431 			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;  in qlnxr_post_send()
4435 				rwqe->length);  in qlnxr_post_send()
4445 			rc = -EINVAL;  in qlnxr_post_send()
4451 			wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;  in qlnxr_post_send()
4454 			rwqe->wqe_size = 2;  in qlnxr_post_send()
4456 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4457 			rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev,  in qlnxr_post_send()
4460 			qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;  in qlnxr_post_send()
4461 			qp->prev_wqe_size = rwqe->wqe_size;  in qlnxr_post_send()
4462 			qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;  in qlnxr_post_send()
4465 				rwqe->length);  in qlnxr_post_send()
4474 				((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ?  in qlnxr_post_send()
4479 			awqe1->prev_wqe_size = 4;  in qlnxr_post_send()
4482 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4485 				atomic_wr(wr)->remote_addr);  in qlnxr_post_send()
4487 			awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);  in qlnxr_post_send()
4490 					ecore_chain_produce(&qp->sq.pbl);  in qlnxr_post_send()
4492 			if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {  in qlnxr_post_send()
4493 				wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;  in qlnxr_post_send()
4495 						 atomic_wr(wr)->compare_add);  in qlnxr_post_send()
4497 				wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;  in qlnxr_post_send()
4499 						 atomic_wr(wr)->swap);  in qlnxr_post_send()
4501 						 atomic_wr(wr)->compare_add);  in qlnxr_post_send()
4506 			qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size;  in qlnxr_post_send()
4507 			qp->prev_wqe_size = awqe1->prev_wqe_size;  in qlnxr_post_send()
4517 			iwqe->prev_wqe_size = 1;  in qlnxr_post_send()
4519 			iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;  in qlnxr_post_send()
4520 			iwqe->inv_l_key = wr->ex.invalidate_rkey;  in qlnxr_post_send()
4521 			qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size;  in qlnxr_post_send()
4522 			qp->prev_wqe_size = iwqe->prev_wqe_size;  in qlnxr_post_send()
4530 			wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;  in qlnxr_post_send()
4532 			fwqe1->wqe_size = 2;  in qlnxr_post_send()
4534 			rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr));  in qlnxr_post_send()
4535 			if (rc) {  in qlnxr_post_send()
4536 				QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc);  in qlnxr_post_send()
4541 			qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;  in qlnxr_post_send()
4542 			qp->prev_wqe_size = fwqe1->wqe_size;  in qlnxr_post_send()
4548 			QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode);  in qlnxr_post_send()
4550 			rc = -EINVAL;  in qlnxr_post_send()
4559 			ecore_chain_set_prod(&qp->sq.pbl,  in qlnxr_post_send()
4560 			     le16_to_cpu(qp->sq.db_data.data.value),  in qlnxr_post_send()
4563 			qp->prev_wqe_size = wqe->prev_wqe_size;  in qlnxr_post_send()
4564 			status = rc;  in qlnxr_post_send()
4570 		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;  in qlnxr_post_send()
4572 		qlnxr_inc_sw_prod(&qp->sq);  in qlnxr_post_send()
4574 		qp->sq.db_data.data.value++;  in qlnxr_post_send()
4576 		wr = wr->next;  in qlnxr_post_send()
4586 	//writel(qp->sq.db_data.raw, qp->sq.db);  in qlnxr_post_send()
4588 	reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells);  in qlnxr_post_send()
4589         bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw);  in qlnxr_post_send()
4590         bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);  in qlnxr_post_send()
4594 	spin_unlock_irqrestore(&qp->q_lock, flags);  in qlnxr_post_send()
4611 	used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;  in qlnxr_srq_elem_left()
4613 	return hw_srq->max_wr - used;  in qlnxr_srq_elem_left()
4622 	struct qlnxr_dev	*dev = qp->dev;  in qlnxr_post_recv()
4628 	ha = dev->ha;  in qlnxr_post_recv()
4630 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_post_recv()
4631 		return -EINVAL;  in qlnxr_post_recv()
4635 	if (qp->qp_type == IB_QPT_GSI) {  in qlnxr_post_recv()
4636 		QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n");  in qlnxr_post_recv()
4640 	if (qp->srq) {  in qlnxr_post_recv()
4641 		QL_DPRINT11(ha, "qp->srq [%p]"  in qlnxr_post_recv()
4643 			qp->srq);  in qlnxr_post_recv()
4644 		return -EINVAL;  in qlnxr_post_recv()
4647 	spin_lock_irqsave(&qp->q_lock, flags);  in qlnxr_post_recv()
4649 	if (qp->state == ECORE_ROCE_QP_STATE_RESET) {  in qlnxr_post_recv()
4650 		spin_unlock_irqrestore(&qp->q_lock, flags);  in qlnxr_post_recv()
4653 		QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n");  in qlnxr_post_recv()
4655 		return -EINVAL;  in qlnxr_post_recv()
4661 		if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) <  in qlnxr_post_recv()
4663 			(wr->num_sge > qp->rq.max_sges)) {  in qlnxr_post_recv()
4664 			status = -ENOMEM;  in qlnxr_post_recv()
4668 		for (i = 0; i < wr->num_sge; i++) {  in qlnxr_post_recv()
4670 			struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);  in qlnxr_post_recv()
4674 				SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge);  in qlnxr_post_recv()
4676 			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey);  in qlnxr_post_recv()
4678 			RQ_SGE_SET(rqe, wr->sg_list[i].addr, \  in qlnxr_post_recv()
4679 				wr->sg_list[i].length, flags);  in qlnxr_post_recv()
4681 		/* Special case of no sges. FW requires between 1-4 sges...  in qlnxr_post_recv()
4684 		if (!wr->num_sge) {  in qlnxr_post_recv()
4686 			struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl);  in qlnxr_post_recv()
4693 			rqe->addr.hi = 0;  in qlnxr_post_recv()
4694 			rqe->addr.lo = 0;  in qlnxr_post_recv()
4696 			rqe->length = 0;  in qlnxr_post_recv()
4697 			rqe->flags = cpu_to_le32(flags);  in qlnxr_post_recv()
4702 		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;  in qlnxr_post_recv()
4703 		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;  in qlnxr_post_recv()
4705 		qlnxr_inc_sw_prod(&qp->rq);  in qlnxr_post_recv()
4709 		qp->rq.db_data.data.value++;  in qlnxr_post_recv()
4711 	//	writel(qp->rq.db_data.raw, qp->rq.db);  in qlnxr_post_recv()
4714 	//		writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);  in qlnxr_post_recv()
4718 		reg_addr = (uint32_t)((uint8_t *)qp->rq.db -  in qlnxr_post_recv()
4719 				(uint8_t *)ha->cdev.doorbells);  in qlnxr_post_recv()
4721 		bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw);  in qlnxr_post_recv()
4722 		bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);  in qlnxr_post_recv()
4725 			reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 -  in qlnxr_post_recv()
4726 						(uint8_t *)ha->cdev.doorbells);  in qlnxr_post_recv()
4727 			bus_write_4(ha->pci_dbells, reg_addr, \  in qlnxr_post_recv()
4728 				qp->rq.iwarp_db2_data.raw);  in qlnxr_post_recv()
4729 			bus_barrier(ha->pci_dbells,  0, 0, \  in qlnxr_post_recv()
4733 		wr = wr->next;  in qlnxr_post_recv()
4736 	spin_unlock_irqrestore(&qp->q_lock, flags);  in qlnxr_post_recv()
4738 	QL_DPRINT12(ha, "exit status = 0x%x\n", status);  in qlnxr_post_recv()
4754 	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)  in qlnxr_chk_if_fmr()
4755 		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;  in qlnxr_chk_if_fmr()
4769 	qlnx_host_t	*ha = dev->ha;  in process_req()
4773 	while (num_entries && qp->sq.wqe_cons != hw_cons) {  in process_req()
4774 		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {  in process_req()
4781 		wc->status = status;  in process_req()
4782 		wc->vendor_err = 0;  in process_req()
4783 		wc->wc_flags = 0;  in process_req()
4784 		wc->src_qp = qp->id;  in process_req()
4785 		wc->qp = &qp->ibqp;  in process_req()
4788 		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;  in process_req()
4789 		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;  in process_req()
4791 		switch (wc->opcode) {  in process_req()
4794 			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;  in process_req()
4798 				qp->wqe_wr_id[qp->sq.cons].bytes_len);  in process_req()
4803 			wc->byte_len = 8;  in process_req()
4807 			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;  in process_req()
4813 			QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode);  in process_req()
4819 		num_entries--;  in process_req()
4823 		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)  in process_req()
4824 			ecore_chain_consume(&qp->sq.pbl);  in process_req()
4825 		qlnxr_inc_sw_cons(&qp->sq);  in process_req()
4828 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);  in process_req()
4841 	qlnx_host_t	*ha = dev->ha;  in qlnxr_poll_cq_req()
4843 	QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status);  in qlnxr_poll_cq_req()
4845 	switch (req->status) {  in qlnxr_poll_cq_req()
4848 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,  in qlnxr_poll_cq_req()
4854 		if (qp->state != ECORE_ROCE_QP_STATE_ERR)  in qlnxr_poll_cq_req()
4855 		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,  in qlnxr_poll_cq_req()
4862 		qp->state = ECORE_ROCE_QP_STATE_ERR;  in qlnxr_poll_cq_req()
4864 				req->sq_cons - 1, IB_WC_SUCCESS, 0);  in qlnxr_poll_cq_req()
4871 			switch (req->status) {  in qlnxr_poll_cq_req()
4906 			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,  in qlnxr_poll_cq_req()
4924 	qlnx_host_t		*ha = dev->ha;  in __process_resp_one()
4926 	QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n",  in __process_resp_one()
4927 		qp, resp->status);  in __process_resp_one()
4929 	wc->opcode = IB_WC_RECV;  in __process_resp_one()
4930 	wc->wc_flags = 0;  in __process_resp_one()
4932 	switch (resp->status) {  in __process_resp_one()
4958 		if (resp->flags & QLNXR_RESP_IMM) {  in __process_resp_one()
4959 			wc->ex.imm_data =  in __process_resp_one()
4960 				le32_to_cpu(resp->imm_data_or_inv_r_Key);  in __process_resp_one()
4961 			wc->wc_flags |= IB_WC_WITH_IMM;  in __process_resp_one()
4963 			if (resp->flags & QLNXR_RESP_RDMA)  in __process_resp_one()
4964 				wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;  in __process_resp_one()
4966 			if (resp->flags & QLNXR_RESP_INV) {  in __process_resp_one()
4968 					"Invalid flags QLNXR_RESP_INV [0x%x]"  in __process_resp_one()
4969 					"qp = %p qp->id = 0x%x cq = %p"  in __process_resp_one()
4970 					" cq->icid = 0x%x\n",  in __process_resp_one()
4971 					resp->flags, qp, qp->id, cq, cq->icid );  in __process_resp_one()
4973 		} else if (resp->flags & QLNXR_RESP_INV) {  in __process_resp_one()
4974 			wc->ex.imm_data =  in __process_resp_one()
4975 				le32_to_cpu(resp->imm_data_or_inv_r_Key);  in __process_resp_one()
4976 			wc->wc_flags |= IB_WC_WITH_INVALIDATE;  in __process_resp_one()
4978 			if (resp->flags & QLNXR_RESP_RDMA) {  in __process_resp_one()
4980 					"Invalid flags QLNXR_RESP_RDMA [0x%x]"  in __process_resp_one()
4981 					"qp = %p qp->id = 0x%x cq = %p"  in __process_resp_one()
4982 					" cq->icid = 0x%x\n",  in __process_resp_one()
4983 					resp->flags, qp, qp->id, cq, cq->icid );  in __process_resp_one()
4985 		} else if (resp->flags & QLNXR_RESP_RDMA) {  in __process_resp_one()
4986 			QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]"  in __process_resp_one()
4987 				"qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n",  in __process_resp_one()
4988 				resp->flags, qp, qp->id, cq, cq->icid );  in __process_resp_one()
4997 	wc->status = wc_status;  in __process_resp_one()
4998 	wc->vendor_err = 0;  in __process_resp_one()
4999 	wc->src_qp = qp->id;  in __process_resp_one()
5000 	wc->qp = &qp->ibqp;  in __process_resp_one()
5001 	wc->wr_id = wr_id;  in __process_resp_one()
5003 	QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status);  in __process_resp_one()
5015 	struct qlnxr_srq	*srq = qp->srq;  in process_resp_one_srq()
5017 	qlnx_host_t		*ha = dev->ha;  in process_resp_one_srq()
5021 	wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo);  in process_resp_one_srq()
5023 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {  in process_resp_one_srq()
5024 		wc->status = IB_WC_WR_FLUSH_ERR;  in process_resp_one_srq()
5025 		wc->vendor_err = 0;  in process_resp_one_srq()
5026 		wc->wr_id = wr_id;  in process_resp_one_srq()
5027 		wc->byte_len = 0;  in process_resp_one_srq()
5028 		wc->src_qp = qp->id;  in process_resp_one_srq()
5029 		wc->qp = &qp->ibqp;  in process_resp_one_srq()
5030 		wc->wr_id = wr_id;  in process_resp_one_srq()
5038 	srq->hw_srq.wr_cons_cnt++;  in process_resp_one_srq()
5051 	qlnx_host_t	*ha = dev->ha;  in process_resp_one()
5052 	u64		wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;  in process_resp_one()
5058 	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)  in process_resp_one()
5059 		ecore_chain_consume(&qp->rq.pbl);  in process_resp_one()
5060 	qlnxr_inc_sw_cons(&qp->rq);  in process_resp_one()
5073 	qlnx_host_t	*ha = qp->dev->ha;  in process_resp_flush()
5077 	while (num_entries && qp->rq.wqe_cons != hw_cons) {  in process_resp_flush()
5079 		wc->status = IB_WC_WR_FLUSH_ERR;  in process_resp_flush()
5080 		wc->vendor_err = 0;  in process_resp_flush()
5081 		wc->wc_flags = 0;  in process_resp_flush()
5082 		wc->src_qp = qp->id;  in process_resp_flush()
5083 		wc->byte_len = 0;  in process_resp_flush()
5084 		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;  in process_resp_flush()
5085 		wc->qp = &qp->ibqp;  in process_resp_flush()
5086 		num_entries--;  in process_resp_flush()
5089 		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)  in process_resp_flush()
5090 			ecore_chain_consume(&qp->rq.pbl);  in process_resp_flush()
5091 		qlnxr_inc_sw_cons(&qp->rq);  in process_resp_flush()
5094 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);  in process_resp_flush()
5104 	if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {  in try_consume_resp_cqe()
5120 	qlnx_host_t	*ha = dev->ha;  in qlnxr_poll_cq_resp_srq()
5128 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);  in qlnxr_poll_cq_resp_srq()
5142 	qlnx_host_t	*ha = dev->ha;  in qlnxr_poll_cq_resp()
5146 	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {  in qlnxr_poll_cq_resp()
5148 				resp->rq_cons);  in qlnxr_poll_cq_resp()
5156 	QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt);  in qlnxr_poll_cq_resp()
5164 	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {  in try_consume_req_cqe()
5174 	qlnx_host_t	*ha = dev->ha;  in doorbell_cq()
5179 	cq->db.data.agg_flags = flags;  in doorbell_cq()
5180 	cq->db.data.value = cpu_to_le32(cons);  in doorbell_cq()
5182 	reg_addr = (uint64_t)((uint8_t *)cq->db_addr -  in doorbell_cq()
5183 				(uint8_t *)(ha->cdev.doorbells));  in doorbell_cq()
5185 	bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw);  in doorbell_cq()
5186 	bus_barrier(ha->pci_dbells,  0, 0, BUS_SPACE_BARRIER_READ);  in doorbell_cq()
5192 //	writeq(cq->db.raw, cq->db_addr);  in doorbell_cq()
5198 //	writel(cq->db.raw & 0xffffffff, cq->db_addr);  in doorbell_cq()
5206 	struct rdma_cqe_requester *resp_cqe = &cqe->req;  in is_valid_cqe()
5207 	return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) ==  in is_valid_cqe()
5208 			cq->pbl_toggle;  in is_valid_cqe()
5215 	struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));  in qlnxr_poll_cq()
5217 	union rdma_cqe	*cqe = cq->latest_cqe;  in qlnxr_poll_cq()
5221 	qlnx_host_t	*ha = dev->ha;  in qlnxr_poll_cq()
5225 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_poll_cq()
5226 		return -EINVAL;  in qlnxr_poll_cq()
5228 	if (cq->destroyed) {  in qlnxr_poll_cq()
5230 			cq, cq->icid);  in qlnxr_poll_cq()
5234 	if (cq->cq_type == QLNXR_CQ_TYPE_GSI)  in qlnxr_poll_cq()
5237 	spin_lock_irqsave(&cq->cq_lock, flags);  in qlnxr_poll_cq()
5239 	old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);  in qlnxr_poll_cq()
5250 		resp_cqe = &cqe->req;  in qlnxr_poll_cq()
5251 		qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi,  in qlnxr_poll_cq()
5252 						resp_cqe->qp_handle.lo);  in qlnxr_poll_cq()
5259 		wc->qp = &qp->ibqp;  in qlnxr_poll_cq()
5261 		cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);  in qlnxr_poll_cq()
5266 					wc, &cqe->req);  in qlnxr_poll_cq()
5267 			try_consume_req_cqe(cq, qp, &cqe->req, &update);  in qlnxr_poll_cq()
5271 					wc, &cqe->resp, &update);  in qlnxr_poll_cq()
5275 					wc, &cqe->resp, &update);  in qlnxr_poll_cq()
5279 			QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type);  in qlnxr_poll_cq()
5282 		num_entries -= cnt;  in qlnxr_poll_cq()
5286 		cqe = cq->latest_cqe;  in qlnxr_poll_cq()
5288 	new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl);  in qlnxr_poll_cq()
5290 	cq->cq_cons += new_cons - old_cons;  in qlnxr_poll_cq()
5296 		doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags);  in qlnxr_poll_cq()
5297 		QL_DPRINT12(ha, "cq = %p cons = 0x%x "  in qlnxr_poll_cq()
5298 			"arm_flags = 0x%x db.icid = 0x%x\n", cq,  in qlnxr_poll_cq()
5299 			(cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid);  in qlnxr_poll_cq()
5302 	spin_unlock_irqrestore(&cq->cq_lock, flags);  in qlnxr_poll_cq()
5317 	dev = get_qlnxr_dev((ibcq->device));  in qlnxr_arm_cq()
5318 	ha = dev->ha;  in qlnxr_arm_cq()
5320 	QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x "  in qlnxr_arm_cq()
5321 		"cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq,  in qlnxr_arm_cq()
5322 		flags, cq, cq->cq_cons, cq->cq_type);  in qlnxr_arm_cq()
5324 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_arm_cq()
5325 		return -EINVAL;  in qlnxr_arm_cq()
5327 	if (cq->destroyed) {  in qlnxr_arm_cq()
5329 			cq, cq->icid);  in qlnxr_arm_cq()
5330 		return -EINVAL;  in qlnxr_arm_cq()
5333         if (cq->cq_type == QLNXR_CQ_TYPE_GSI) {  in qlnxr_arm_cq()
5337         spin_lock_irqsave(&cq->cq_lock, sflags);  in qlnxr_arm_cq()
5339         cq->arm_flags = 0;  in qlnxr_arm_cq()
5342                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;  in qlnxr_arm_cq()
5345                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;  in qlnxr_arm_cq()
5348         doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags);  in qlnxr_arm_cq()
5350         spin_unlock_irqrestore(&cq->cq_lock, sflags);  in qlnxr_arm_cq()
5352 	QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags);  in qlnxr_arm_cq()
5360 	struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device));  in __qlnxr_alloc_mr()
5362 	int		rc = -ENOMEM;  in __qlnxr_alloc_mr()  local
5365 	ha = dev->ha;  in __qlnxr_alloc_mr()
5369 		ibpd, pd, pd->pd_id, max_page_list_len);  in __qlnxr_alloc_mr()
5374 		return ERR_PTR(rc);  in __qlnxr_alloc_mr()
5377 	mr->dev = dev;  in __qlnxr_alloc_mr()
5378 	mr->type = QLNXR_MR_FRMR;  in __qlnxr_alloc_mr()
5380 	rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len,  in __qlnxr_alloc_mr()
5382 	if (rc) {  in __qlnxr_alloc_mr()
5387 	rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);  in __qlnxr_alloc_mr()
5388 	if (rc) {  in __qlnxr_alloc_mr()
5394 	mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR;  in __qlnxr_alloc_mr()
5395 	mr->hw_mr.key = 0;  in __qlnxr_alloc_mr()
5396 	mr->hw_mr.pd = pd->pd_id;  in __qlnxr_alloc_mr()
5397 	mr->hw_mr.local_read = 1;  in __qlnxr_alloc_mr()
5398 	mr->hw_mr.local_write = 0;  in __qlnxr_alloc_mr()
5399 	mr->hw_mr.remote_read = 0;  in __qlnxr_alloc_mr()
5400 	mr->hw_mr.remote_write = 0;  in __qlnxr_alloc_mr()
5401 	mr->hw_mr.remote_atomic = 0;  in __qlnxr_alloc_mr()
5402 	mr->hw_mr.mw_bind = false; /* TBD MW BIND */  in __qlnxr_alloc_mr()
5403 	mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */  in __qlnxr_alloc_mr()
5404 	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;  in __qlnxr_alloc_mr()
5405 	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);  in __qlnxr_alloc_mr()
5406 	mr->hw_mr.fbo = 0;  in __qlnxr_alloc_mr()
5407 	mr->hw_mr.length = 0;  in __qlnxr_alloc_mr()
5408 	mr->hw_mr.vaddr = 0;  in __qlnxr_alloc_mr()
5409 	mr->hw_mr.zbva = false; /* TBD figure when this should be true */  in __qlnxr_alloc_mr()
5410 	mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */  in __qlnxr_alloc_mr()
5411 	mr->hw_mr.dma_mr = false;  in __qlnxr_alloc_mr()
5413 	rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);  in __qlnxr_alloc_mr()
5414 	if (rc) {  in __qlnxr_alloc_mr()
5419 	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;  in __qlnxr_alloc_mr()
5420 	mr->ibmr.rkey = mr->ibmr.lkey;  in __qlnxr_alloc_mr()
5422 	QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n",  in __qlnxr_alloc_mr()
5423 		mr, mr->ibmr.lkey);  in __qlnxr_alloc_mr()
5428 	ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);  in __qlnxr_alloc_mr()
5434 	return ERR_PTR(rc);  in __qlnxr_alloc_mr()
5445 	dev = get_qlnxr_dev(ibpd->device);  in qlnxr_alloc_mr()
5446 	ha = dev->ha;  in qlnxr_alloc_mr()
5451 		return ERR_PTR(-EINVAL);  in qlnxr_alloc_mr()
5456 		return ERR_PTR(-EINVAL);  in qlnxr_alloc_mr()
5458 	QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr);  in qlnxr_alloc_mr()
5460 	return &mr->ibmr;  in qlnxr_alloc_mr()
5473 	dev = mr->dev;  in qlnxr_set_page()
5474 	ha = dev->ha;  in qlnxr_set_page()
5476 	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {  in qlnxr_set_page()
5477 		QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages);  in qlnxr_set_page()
5478 		return -ENOMEM;  in qlnxr_set_page()
5481 	QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages,  in qlnxr_set_page()
5484 	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);  in qlnxr_set_page()
5485 	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);  in qlnxr_set_page()
5486 	pbe = (struct regpair *)pbl_table->va;  in qlnxr_set_page()
5487 	pbe +=  mr->npages % pbes_in_page;  in qlnxr_set_page()
5488 	pbe->lo = cpu_to_le32((u32)addr);  in qlnxr_set_page()
5489 	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));  in qlnxr_set_page()
5491 	mr->npages++;  in qlnxr_set_page()
5493 	QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages,  in qlnxr_set_page()
5507 		return (-1);  in qlnxr_map_mr_sg()
5509 	if (mr->dev == NULL)  in qlnxr_map_mr_sg()
5510 		return (-1);  in qlnxr_map_mr_sg()
5512 	ha = mr->dev->ha;  in qlnxr_map_mr_sg()
5516 	mr->npages = 0;  in qlnxr_map_mr_sg()
5517 	qlnx_handle_completed_mrs(mr->dev, &mr->info);  in qlnxr_map_mr_sg()
5535 	dev = get_qlnxr_dev(ibah->device);  in qlnxr_create_ah()
5536 	ha = dev->ha;  in qlnxr_create_ah()
5540 	ah->attr = *attr;	  in qlnxr_create_ah()
5556 	dev = get_qlnxr_dev((ibah->device));  in qlnxr_query_ah()
5557 	ha = dev->ha;  in qlnxr_query_ah()
5559 	return -EINVAL;  in qlnxr_query_ah()
5568 	dev = get_qlnxr_dev((ibah->device));  in qlnxr_modify_ah()
5569 	ha = dev->ha;  in qlnxr_modify_ah()
5571 	return -ENOSYS;  in qlnxr_modify_ah()
5590 	ha = dev->ha;  in qlnxr_process_mad()
5593 	return -ENOSYS;  in qlnxr_process_mad()
5594 //	QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n",  in qlnxr_process_mad()
5595 //               in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version,  in qlnxr_process_mad()
5596 //               in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific,  in qlnxr_process_mad()
5597 //               in_mad->mad_hdr.class_version, in_mad->mad_hdr.method,  in qlnxr_process_mad()
5598 //               in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status);  in qlnxr_process_mad()
5613 	ha = dev->ha;  in qlnxr_get_port_immutable()
5622 		immutable->pkey_tbl_len = 1;  in qlnxr_get_port_immutable()
5623 		immutable->gid_tbl_len = 1;  in qlnxr_get_port_immutable()
5624 		immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;  in qlnxr_get_port_immutable()
5625 		immutable->max_mad_size = 0;  in qlnxr_get_port_immutable()
5627 		immutable->pkey_tbl_len = attr.pkey_tbl_len;  in qlnxr_get_port_immutable()
5628 		immutable->gid_tbl_len = attr.gid_tbl_len;  in qlnxr_get_port_immutable()
5629 		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;  in qlnxr_get_port_immutable()
5630 		immutable->max_mad_size = IB_MGMT_MAD_SIZE;  in qlnxr_get_port_immutable()
5644 	struct qlnxr_dev *dev = listener->dev;  in qlnxr_iw_mpa_request()
5651 	ha = dev->ha;  in qlnxr_iw_mpa_request()
5655 	if (params->cm_info->ip_version != ECORE_TCP_IPV4) {  in qlnxr_iw_mpa_request()
5656 		QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n",  in qlnxr_iw_mpa_request()
5657 			params->cm_info->ip_version);  in qlnxr_iw_mpa_request()
5668 	ep->dev = dev;  in qlnxr_iw_mpa_request()
5669 	ep->ecore_context = params->ep_context;  in qlnxr_iw_mpa_request()
5674 	event.status = params->status;  in qlnxr_iw_mpa_request()
5679 	laddr->sin_family = AF_INET;  in qlnxr_iw_mpa_request()
5680 	raddr->sin_family = AF_INET;  in qlnxr_iw_mpa_request()
5682 	laddr->sin_port = htons(params->cm_info->local_port);  in qlnxr_iw_mpa_request()
5683 	raddr->sin_port = htons(params->cm_info->remote_port);  in qlnxr_iw_mpa_request()
5685 	laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]);  in qlnxr_iw_mpa_request()
5686 	raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]);  in qlnxr_iw_mpa_request()
5689 	event.private_data = (void *)params->cm_info->private_data;  in qlnxr_iw_mpa_request()
5690 	event.private_data_len = (u8)params->cm_info->private_data_len;  in qlnxr_iw_mpa_request()
5692 	event.ord = params->cm_info->ord;  in qlnxr_iw_mpa_request()
5693 	event.ird = params->cm_info->ird;  in qlnxr_iw_mpa_request()
5695 	listener->cm_id->event_handler(listener->cm_id, &event);  in qlnxr_iw_mpa_request()
5709 	struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_issue_event()
5713 	ha = dev->ha;  in qlnxr_iw_issue_event()
5718 	event.status = params->status;  in qlnxr_iw_issue_event()
5721 	if (params->cm_info != NULL) {  in qlnxr_iw_issue_event()
5722 		event.ird = params->cm_info->ird;  in qlnxr_iw_issue_event()
5723 		event.ord = params->cm_info->ord;  in qlnxr_iw_issue_event()
5727 		event.private_data_len = params->cm_info->private_data_len;  in qlnxr_iw_issue_event()
5728 		event.private_data = (void *)params->cm_info->private_data;  in qlnxr_iw_issue_event()
5737 		if (ep->cm_id)  in qlnxr_iw_issue_event()
5738 			ep->cm_id->event_handler(ep->cm_id, &event);  in qlnxr_iw_issue_event()
5740 			QL_DPRINT11(ha, "ep->cm_id == NULL \n");  in qlnxr_iw_issue_event()
5755 	struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_close_event()
5758 	ha = dev->ha;  in qlnxr_iw_close_event()
5762 	if (ep->cm_id) {  in qlnxr_iw_close_event()
5767 		ep->cm_id->rem_ref(ep->cm_id);  in qlnxr_iw_close_event()
5768 		ep->cm_id = NULL;  in qlnxr_iw_close_event()
5781         struct qlnxr_dev        *dev = ep->dev;  in qlnxr_iw_passive_complete()
5784         ha = dev->ha;  in qlnxr_iw_passive_complete()
5789         if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) {  in qlnxr_iw_passive_complete()
5804         if (params->status < 0) {  in qlnxr_iw_passive_complete()
5824         struct qlnxr_iw_ep *ep = dwork->ep;  in qlnxr_iw_disconnect_worker()
5825         struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_disconnect_worker()
5826         struct qlnxr_qp *qp = ep->qp;  in qlnxr_iw_disconnect_worker()
5829         if (qp->destroyed) {  in qlnxr_iw_disconnect_worker()
5831                 qlnxr_iw_qp_rem_ref(&qp->ibqp);  in qlnxr_iw_disconnect_worker()
5836         event.status = dwork->status;  in qlnxr_iw_disconnect_worker()
5842         if (dwork->status)  in qlnxr_iw_disconnect_worker()
5849         if (ep->cm_id)  in qlnxr_iw_disconnect_worker()
5850                 ep->cm_id->event_handler(ep->cm_id, &event);  in qlnxr_iw_disconnect_worker()
5855         ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params);  in qlnxr_iw_disconnect_worker()
5857         qlnxr_iw_qp_rem_ref(&qp->ibqp);  in qlnxr_iw_disconnect_worker()
5868         struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_disconnect_event()
5869         struct qlnxr_qp *qp = ep->qp;  in qlnxr_iw_disconnect_event()
5875         qlnxr_iw_qp_add_ref(&qp->ibqp);  in qlnxr_iw_disconnect_event()
5876         work->ep = ep;  in qlnxr_iw_disconnect_event()
5877         work->event = params->event;  in qlnxr_iw_disconnect_event()
5878         work->status = params->status;  in qlnxr_iw_disconnect_event()
5880         INIT_WORK(&work->work, qlnxr_iw_disconnect_worker);  in qlnxr_iw_disconnect_event()
5881         queue_work(dev->iwarp_wq, &work->work);  in qlnxr_iw_disconnect_event()
5891         struct qlnxr_dev	*dev = ep->dev;  in qlnxr_iw_mpa_reply()
5893         int			rc;  in qlnxr_iw_mpa_reply()  local
5896 	ha = dev->ha;  in qlnxr_iw_mpa_reply()
5900 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_iw_mpa_reply()
5901 		return -EINVAL;  in qlnxr_iw_mpa_reply()
5904         rtr_in.ep_context = params->ep_context;  in qlnxr_iw_mpa_reply()
5906         rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in);  in qlnxr_iw_mpa_reply()
5908 	QL_DPRINT12(ha, "exit rc = %d\n", rc);  in qlnxr_iw_mpa_reply()
5909         return rc;  in qlnxr_iw_mpa_reply()
5919         struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_qp_event()
5920         struct ib_qp *ibqp = &(ep->qp->ibqp);  in qlnxr_iw_qp_event()
5924 	ha = dev->ha;  in qlnxr_iw_qp_event()
5927 		"[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n",  in qlnxr_iw_qp_event()
5928 		context, params->event, str, ibqp->event_handler);  in qlnxr_iw_qp_event()
5930         if (ibqp->event_handler) {  in qlnxr_iw_qp_event()
5932                 event.device = ibqp->device;  in qlnxr_iw_qp_event()
5934                 ibqp->event_handler(&event, ibqp->qp_context);  in qlnxr_iw_qp_event()
5945 	struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_event_handler()
5948 	ha = dev->ha;  in qlnxr_iw_event_handler()
5950 	QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "  in qlnxr_iw_event_handler()
5951 		"enter\n", context, params->event);  in qlnxr_iw_event_handler()
5953 	switch (params->event) {  in qlnxr_iw_event_handler()
5965 		ep->during_connect = 0;  in qlnxr_iw_event_handler()
5971 		ep->during_connect = 0;  in qlnxr_iw_event_handler()
5976 		if (params->status < 0) {  in qlnxr_iw_event_handler()
5979 			ep->cm_id->rem_ref(ep->cm_id);  in qlnxr_iw_event_handler()
5980 			ep->cm_id = NULL;  in qlnxr_iw_event_handler()
5989 		ep->during_connect = 0;  in qlnxr_iw_event_handler()
6040 			"Unknown event [0x%x] received \n", params->event);  in qlnxr_iw_event_handler()
6044 	QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] "  in qlnxr_iw_event_handler()
6045 		"exit\n", context, params->event);  in qlnxr_iw_event_handler()
6055 	int rc;  in qlnxr_addr4_resolve()  local
6057 	rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in,  in qlnxr_addr4_resolve()
6060 	QL_DPRINT12(dev->ha, "rc = %d "  in qlnxr_addr4_resolve()
6061 		"sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d "  in qlnxr_addr4_resolve()
6062 		"Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc,  in qlnxr_addr4_resolve()
6063 		dst_in->sin_len, dst_in->sin_family,  in qlnxr_addr4_resolve()
6064 		NIPQUAD((dst_in->sin_addr.s_addr)),  in qlnxr_addr4_resolve()
6068 	return rc;  in qlnxr_addr4_resolve()
6081 	int rc = 0;  in qlnxr_iw_connect()  local
6084 	dev = get_qlnxr_dev((cm_id->device));  in qlnxr_iw_connect()
6085 	ha = dev->ha;  in qlnxr_iw_connect()
6090 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_iw_connect()
6091 		return -EINVAL;  in qlnxr_iw_connect()
6093 	qp = idr_find(&dev->qpidr, conn_param->qpn);  in qlnxr_iw_connect()
6095 	laddr = (struct sockaddr_in *)&cm_id->local_addr;  in qlnxr_iw_connect()
6096 	raddr = (struct sockaddr_in *)&cm_id->remote_addr;  in qlnxr_iw_connect()
6100 		NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port,  in qlnxr_iw_connect()
6101 		NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port);  in qlnxr_iw_connect()
6107 		return -ENOMEM;  in qlnxr_iw_connect()
6110 	ep->dev = dev;  in qlnxr_iw_connect()
6111 	ep->qp = qp;  in qlnxr_iw_connect()
6112 	cm_id->add_ref(cm_id);  in qlnxr_iw_connect()
6113 	ep->cm_id = cm_id;  in qlnxr_iw_connect()
6123 	in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr);  in qlnxr_iw_connect()
6124 	in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr);  in qlnxr_iw_connect()
6125 	in_params.cm_info.remote_port = ntohs(raddr->sin_port);  in qlnxr_iw_connect()
6126 	in_params.cm_info.local_port = ntohs(laddr->sin_port);  in qlnxr_iw_connect()
6128 	in_params.mss = if_getmtu(dev->ha->ifp) - 40;  in qlnxr_iw_connect()
6138 	rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr);  in qlnxr_iw_connect()
6140 	if (rc) {  in qlnxr_iw_connect()
6147 		conn_param->ord, conn_param->ird, conn_param->private_data,  in qlnxr_iw_connect()
6148 		conn_param->private_data_len, qp->rq_psn);  in qlnxr_iw_connect()
6150 	in_params.cm_info.ord = conn_param->ord;  in qlnxr_iw_connect()
6151 	in_params.cm_info.ird = conn_param->ird;  in qlnxr_iw_connect()
6152 	in_params.cm_info.private_data = conn_param->private_data;  in qlnxr_iw_connect()
6153 	in_params.cm_info.private_data_len = conn_param->private_data_len;  in qlnxr_iw_connect()
6154 	in_params.qp = qp->ecore_qp;  in qlnxr_iw_connect()
6156 	memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN);  in qlnxr_iw_connect()
6158 	rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params);  in qlnxr_iw_connect()
6160 	if (rc) {  in qlnxr_iw_connect()
6167 	return rc;  in qlnxr_iw_connect()
6170 	cm_id->rem_ref(cm_id);  in qlnxr_iw_connect()
6173 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_iw_connect()
6174 	return rc;  in qlnxr_iw_connect()
6186 	int rc;  in qlnxr_iw_create_listen()  local
6188 	dev = get_qlnxr_dev((cm_id->device));  in qlnxr_iw_create_listen()
6189 	ha = dev->ha;  in qlnxr_iw_create_listen()
6193 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_iw_create_listen()
6194 		return -EINVAL;  in qlnxr_iw_create_listen()
6196 	laddr = (struct sockaddr_in *)&cm_id->local_addr;  in qlnxr_iw_create_listen()
6202 		return -ENOMEM;  in qlnxr_iw_create_listen()
6205 	listener->dev = dev;  in qlnxr_iw_create_listen()
6206 	cm_id->add_ref(cm_id);  in qlnxr_iw_create_listen()
6207 	listener->cm_id = cm_id;  in qlnxr_iw_create_listen()
6208 	listener->backlog = backlog;  in qlnxr_iw_create_listen()
6219 	iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr);  in qlnxr_iw_create_listen()
6220 	iparams.port = ntohs(laddr->sin_port);  in qlnxr_iw_create_listen()
6224 		NIPQUAD((laddr->sin_addr.s_addr)),  in qlnxr_iw_create_listen()
6225 		laddr->sin_port, iparams.port);  in qlnxr_iw_create_listen()
6227 	rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams);  in qlnxr_iw_create_listen()
6228 	if (rc) {  in qlnxr_iw_create_listen()
6230 			"ecore_iwarp_create_listen failed rc = %d\n", rc);  in qlnxr_iw_create_listen()
6234 	listener->ecore_handle = oparams.handle;  in qlnxr_iw_create_listen()
6235 	cm_id->provider_data = listener;  in qlnxr_iw_create_listen()
6238 	return rc;  in qlnxr_iw_create_listen()
6241 	cm_id->rem_ref(cm_id);  in qlnxr_iw_create_listen()
6244 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_iw_create_listen()
6245 	return rc;  in qlnxr_iw_create_listen()
6251 	struct qlnxr_iw_listener *listener = cm_id->provider_data;  in qlnxr_iw_destroy_listen()
6252 	struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device));  in qlnxr_iw_destroy_listen()
6253 	int rc = 0;  in qlnxr_iw_destroy_listen()  local
6256 	ha = dev->ha;  in qlnxr_iw_destroy_listen()
6260 	if (listener->ecore_handle)  in qlnxr_iw_destroy_listen()
6261 		rc = ecore_iwarp_destroy_listen(dev->rdma_ctx,  in qlnxr_iw_destroy_listen()
6262 				listener->ecore_handle);  in qlnxr_iw_destroy_listen()
6264 	cm_id->rem_ref(cm_id);  in qlnxr_iw_destroy_listen()
6266 	QL_DPRINT12(ha, "exit [%d]\n", rc);  in qlnxr_iw_destroy_listen()
6274 	struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;  in qlnxr_iw_accept()
6275 	struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_accept()
6278 	int rc;  in qlnxr_iw_accept()  local
6281 	ha = dev->ha;  in qlnxr_iw_accept()
6283 	QL_DPRINT12(ha, "enter  qpid=%d\n", conn_param->qpn);  in qlnxr_iw_accept()
6285 	if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING))  in qlnxr_iw_accept()
6286 		return -EINVAL;  in qlnxr_iw_accept()
6288 	qp = idr_find(&dev->qpidr, conn_param->qpn);  in qlnxr_iw_accept()
6291 			conn_param->qpn);  in qlnxr_iw_accept()
6292 		return -EINVAL;  in qlnxr_iw_accept()
6294 	ep->qp = qp;  in qlnxr_iw_accept()
6295 	qp->ep = ep;  in qlnxr_iw_accept()
6296 	cm_id->add_ref(cm_id);  in qlnxr_iw_accept()
6297 	ep->cm_id = cm_id;  in qlnxr_iw_accept()
6299 	params.ep_context = ep->ecore_context;  in qlnxr_iw_accept()
6301 	params.qp = ep->qp->ecore_qp;  in qlnxr_iw_accept()
6302 	params.private_data = conn_param->private_data;  in qlnxr_iw_accept()
6303 	params.private_data_len = conn_param->private_data_len;  in qlnxr_iw_accept()
6304 	params.ird = conn_param->ird;  in qlnxr_iw_accept()
6305 	params.ord = conn_param->ord;  in qlnxr_iw_accept()
6307 	rc = ecore_iwarp_accept(dev->rdma_ctx, &params);  in qlnxr_iw_accept()
6308 	if (rc) {  in qlnxr_iw_accept()
6309 		QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc);  in qlnxr_iw_accept()
6316 	cm_id->rem_ref(cm_id);  in qlnxr_iw_accept()
6317 	QL_DPRINT12(ha, "exit rc = %d\n", rc);  in qlnxr_iw_accept()
6318 	return rc;  in qlnxr_iw_accept()
6324         struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data;  in qlnxr_iw_reject()
6325         struct qlnxr_dev *dev = ep->dev;  in qlnxr_iw_reject()
6327         int rc;  in qlnxr_iw_reject()  local
6329         params.ep_context = ep->ecore_context;  in qlnxr_iw_reject()
6333         ep->qp = NULL;  in qlnxr_iw_reject()
6335         rc = ecore_iwarp_reject(dev->rdma_ctx, &params);  in qlnxr_iw_reject()
6337         return rc;  in qlnxr_iw_reject()
6346 	ha = qp->dev->ha;  in qlnxr_iw_qp_add_ref()
6350 	atomic_inc(&qp->refcnt);  in qlnxr_iw_qp_add_ref()
6362 	ha = qp->dev->ha;  in qlnxr_iw_qp_rem_ref()
6366 	if (atomic_dec_and_test(&qp->refcnt)) {  in qlnxr_iw_qp_rem_ref()
6367 		qlnxr_idr_remove(qp->dev, qp->qp_id);  in qlnxr_iw_qp_rem_ref()
6381 	ha = dev->ha;  in qlnxr_iw_get_qp()
6385 	qp = idr_find(&dev->qpidr, qpn);  in qlnxr_iw_get_qp()