49 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_query_device()  local
50 	struct pci_dev *pcidev = iwdev->rf->pcidev;  in irdma_query_device()
51 	struct irdma_hw_attrs *hw_attrs = &rf->sc_dev.hw_attrs;  in irdma_query_device()
59 	props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 |  in irdma_query_device()
60 	    irdma_fw_minor_ver(&rf->sc_dev);  in irdma_query_device()
69 	props->max_qp = rf->max_qp - rf->used_qps;  in irdma_query_device()
71 	set_max_sge(props, rf);  in irdma_query_device()
72 	props->max_cq = rf->max_cq - rf->used_cqs;  in irdma_query_device()
73 	props->max_cqe = rf->max_cqe - 1;  in irdma_query_device()
74 	props->max_mr = rf->max_mr - rf->used_mrs;  in irdma_query_device()
75 	props->max_pd = rf->max_pd - rf->used_pds;  in irdma_query_device()
82 		props->max_ah = rf->max_ah;  in irdma_query_device()
84 			props->max_mcast_grp = rf->max_mcg;  in irdma_query_device()
86 			props->max_total_mcast_qp_attach = rf->max_qp * IRDMA_MAX_MGS_PER_CTX;  in irdma_query_device()
106 	pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] +  in irdma_mmap_legacy()
107 	       pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;  in irdma_mmap_legacy()
167 		irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_mmap()
174 	irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_mmap()
179 	       pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;  in irdma_mmap()
197 		irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_mmap()
218 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_alloc_push_page()
230 	cqp_info->in.u.manage_push_page.cqp = &iwdev->rf->cqp.sc_cqp;  in irdma_alloc_push_page()
233 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_alloc_push_page()
235 	    iwdev->rf->sc_dev.hw_attrs.max_hw_device_pages) {  in irdma_alloc_push_page()
240 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_alloc_push_page()
283 	u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];  in irdma_compute_push_wqe_offset()
285 	if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) {  in irdma_compute_push_wqe_offset()
320 	WARN_ON_ONCE(iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev < IRDMA_GEN_2);  in irdma_setup_push_mmap_entries()
393 		irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_setup_umode_qp()
409 			irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_setup_umode_qp()
444 	    iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)  in irdma_setup_umode_qp()
509 	mem->va = irdma_allocate_dma_mem(&iwdev->rf->hw, mem, mem->size,  in irdma_setup_kmode_qp()
540 	struct irdma_pci_f *rf = iwqp->iwdev->rf;  in irdma_cqp_create_qp_cmd()  local
546 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_cqp_create_qp_cmd()
561 	status = irdma_handle_cqp_op(rf, cqp_request);  in irdma_cqp_create_qp_cmd()
562 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_cqp_create_qp_cmd()
572 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_roce_fill_and_set_qpctx_info()
612 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_iw_fill_and_set_qpctx_info()
641 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_validate_qp_attrs()
796 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_modify_qp_roce()
928 		    irdma_add_arp(iwdev->rf, local_ip,  in irdma_modify_qp_roce()
968 	irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_modify_qp_roce()
1021 				iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp);  in irdma_modify_qp_roce()
1124 				irdma_debug(&iwdev->rf->sc_dev,  in irdma_modify_qp_roce()
1154 	struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;  in irdma_modify_qp()
1179 	irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_modify_qp()
1372 			irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_modify_qp()
1387  * @rf: RDMA PCI function
1391 irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq)  in irdma_cq_free_rsrc()  argument
1396 		irdma_free_dma_mem(rf->sc_dev.hw, &iwcq->kmem);  in irdma_cq_free_rsrc()
1397 		irdma_free_dma_mem(rf->sc_dev.hw, &iwcq->kmem_shadow);  in irdma_cq_free_rsrc()
1400 	irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id);  in irdma_cq_free_rsrc()
1465 	struct irdma_pci_f *rf;  in irdma_resize_cq()  local
1471 	rf = iwdev->rf;  in irdma_resize_cq()
1473 	if (!(rf->sc_dev.hw_attrs.uk_attrs.feature_flags &  in irdma_resize_cq()
1480 	if (entries > rf->max_cqe)  in irdma_resize_cq()
1485 		if (rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)  in irdma_resize_cq()
1543 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);  in irdma_resize_cq()
1560 	ret = irdma_handle_cqp_op(rf, cqp_request);  in irdma_resize_cq()
1561 	irdma_put_cqp_request(&rf->cqp, cqp_request);  in irdma_resize_cq()
1620 	stag_idx = (stag & iwdev->rf->mr_stagmask) >> IRDMA_CQPSQ_STAG_IDX_S;  in irdma_free_stag()
1621 	irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_mrs, stag_idx);  in irdma_free_stag()
1642 	driver_key = random & ~iwdev->rf->mr_stagmask;  in irdma_create_stag()
1643 	next_stag_index = (random & iwdev->rf->mr_stagmask) >> 8;  in irdma_create_stag()
1644 	next_stag_index %= iwdev->rf->max_mr;  in irdma_create_stag()
1646 	ret = irdma_alloc_rsrc(iwdev->rf, iwdev->rf->allocated_mrs,  in irdma_create_stag()
1647 			       iwdev->rf->max_mr, &stag_index,  in irdma_create_stag()
1717  * @rf: RDMA PCI function
1722 irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,  in irdma_setup_pbles()  argument
1733 		status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,  in irdma_setup_pbles()
1778 	err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);  in irdma_handle_q_mem()
1825 		irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "MR type error\n");  in irdma_handle_q_mem()
1830 		irdma_free_pble(iwdev->rf->pble_rsrc, palloc);  in irdma_handle_q_mem()
1853 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_hw_alloc_stag()
1868 	cqp_info->in.u.alloc_stag.dev = &iwdev->rf->sc_dev;  in irdma_hw_alloc_stag()
1870 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_hw_alloc_stag()
1871 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_hw_alloc_stag()
1945 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_hwreg_mr()
1958 						       iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev);  in irdma_hwreg_mr()
1980 	cqp_info->in.u.mr_reg_non_shared.dev = &iwdev->rf->sc_dev;  in irdma_hwreg_mr()
1982 	ret = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_hwreg_mr()
1983 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_hwreg_mr()
2049 	err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);  in irdma_reg_user_mr_type_mem()
2057 			irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);  in irdma_reg_user_mr_type_mem()
2086 		irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);  in irdma_reg_user_mr_type_mem()
2143 …((iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) ? 0 : IRDMA_SHADOW_…  in irdma_reg_user_mr_type_cq()
2182 	if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)  in irdma_reg_user_mr()
2191 		irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_reg_user_mr()
2259 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_hwdereg_mr()
2274 	cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;  in irdma_hwdereg_mr()
2276 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_hwdereg_mr()
2277 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_hwdereg_mr()
2305 		irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_rereg_mr_trans()
2456 	dev = &iwqp->iwdev->rf->sc_dev;  in irdma_post_send()
2585 			irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_post_send()
2640 			irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_post_recv()
2875 	irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in __irdma_poll_cq()
2946  * @rf: RDMA PCI function
2950 mcast_list_add(struct irdma_pci_f *rf,  in mcast_list_add()  argument
2953 	list_add(&new_elem->list, &rf->mc_qht_list.list);  in mcast_list_add()
2969  * @rf: RDMA PCI function
2973 mcast_list_lookup_ip(struct irdma_pci_f *rf,  in mcast_list_lookup_ip()  argument
2979 	list_for_each_safe(pos, q, &rf->mc_qht_list.list) {  in mcast_list_lookup_ip()
3005 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);  in irdma_mcast_cqp_op()
3014 	cqp_info->in.u.mc_create.cqp = &iwdev->rf->cqp.sc_cqp;  in irdma_mcast_cqp_op()
3015 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_mcast_cqp_op()
3016 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_mcast_cqp_op()
3034 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_attach_mcast()  local
3054 		irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_attach_mcast()
3063 		irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_attach_mcast()
3069 	spin_lock_irqsave(&rf->qh_list_lock, flags);  in irdma_attach_mcast()
3070 	mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr);  in irdma_attach_mcast()
3074 		spin_unlock_irqrestore(&rf->qh_list_lock, flags);  in irdma_attach_mcast()
3082 		ret = irdma_alloc_rsrc(rf, rf->allocated_mcgs, rf->max_mcg,  in irdma_attach_mcast()
3083 				       &mgn, &rf->next_mcg);  in irdma_attach_mcast()
3092 		dma_mem_mc->va = irdma_allocate_dma_mem(&rf->hw, dma_mem_mc,  in irdma_attach_mcast()
3096 			irdma_free_rsrc(rf, rf->allocated_mcgs, mgn);  in irdma_attach_mcast()
3108 		mc_qht_elem->mc_grp_ctx.hmc_fcn_id = iwdev->rf->sc_dev.hmc_fn_id;  in irdma_attach_mcast()
3113 		spin_lock_irqsave(&rf->qh_list_lock, flags);  in irdma_attach_mcast()
3114 		mcast_list_add(rf, mc_qht_elem);  in irdma_attach_mcast()
3118 			spin_unlock_irqrestore(&rf->qh_list_lock, flags);  in irdma_attach_mcast()
3126 	spin_unlock_irqrestore(&rf->qh_list_lock, flags);  in irdma_attach_mcast()
3148 		irdma_free_dma_mem(&rf->hw,  in irdma_attach_mcast()
3150 		irdma_free_rsrc(rf, rf->allocated_mcgs,  in irdma_attach_mcast()
3171 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_detach_mcast()  local
3186 	spin_lock_irqsave(&rf->qh_list_lock, flags);  in irdma_detach_mcast()
3187 	mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr);  in irdma_detach_mcast()
3189 		spin_unlock_irqrestore(&rf->qh_list_lock, flags);  in irdma_detach_mcast()
3190 		irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_detach_mcast()
3199 		spin_unlock_irqrestore(&rf->qh_list_lock, flags);  in irdma_detach_mcast()
3203 			irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_detach_mcast()
3205 			spin_lock_irqsave(&rf->qh_list_lock, flags);  in irdma_detach_mcast()
3206 			mcast_list_add(rf, mc_qht_elem);  in irdma_detach_mcast()
3207 			spin_unlock_irqrestore(&rf->qh_list_lock, flags);  in irdma_detach_mcast()
3211 		irdma_free_dma_mem(&rf->hw,  in irdma_detach_mcast()
3213 		irdma_free_rsrc(rf, rf->allocated_mcgs,  in irdma_detach_mcast()
3217 		spin_unlock_irqrestore(&rf->qh_list_lock, flags);  in irdma_detach_mcast()
3221 			irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,  in irdma_detach_mcast()
3371 	if (iwdev->rf->rdma_ver == IRDMA_GEN_2)  in irdma_init_roce_device()
3428 	iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count;  in irdma_init_rdma_device()
3429 	iwdev->ibdev.dev.parent = iwdev->rf->dev_ctx.dev;  in irdma_init_rdma_device()
3430 	set_ibdev_dma_device(iwdev->ibdev, &iwdev->rf->pcidev->dev);  in irdma_init_rdma_device()
3432 	if (iwdev->rf->rdma_ver == IRDMA_GEN_1)  in irdma_init_rdma_device()
3497 	irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "Register RDMA device fail\n");  in irdma_ib_register_device()