1722 irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr,  in irdma_setup_pbles()  argument
1725 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_setup_pbles()
1733 		status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt,  in irdma_setup_pbles()
1744 		pbl = iwmr->pgaddrmem;  in irdma_setup_pbles()
1747 	irdma_copy_user_pgaddrs(iwmr, pbl, level);  in irdma_setup_pbles()
1750 		iwmr->pgaddrmem[0] = *pbl;  in irdma_setup_pbles()
1768 	struct irdma_mr *iwmr = iwpbl->iwmr;  in irdma_handle_q_mem()  local
1772 	u64 *arr = iwmr->pgaddrmem;  in irdma_handle_q_mem()
1777 	pg_size = iwmr->page_size;  in irdma_handle_q_mem()
1778 	err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);  in irdma_handle_q_mem()
1785 	switch (iwmr->type) {  in irdma_handle_q_mem()
1844 		    struct irdma_mr *iwmr)  in irdma_hw_alloc_stag()  argument
1847 	struct ib_pd *pd = iwmr->ibmr.pd;  in irdma_hw_alloc_stag()
1861 	info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;  in irdma_hw_alloc_stag()
1863 	info->total_len = iwmr->len;  in irdma_hw_alloc_stag()
1873 		iwmr->is_hwreg = 1;  in irdma_hw_alloc_stag()
1886 	struct irdma_mr *iwmr = to_iwmr(ibmr);  in irdma_set_page()  local
1887 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_set_page()
1891 	if (unlikely(iwmr->npages == iwmr->page_cnt))  in irdma_set_page()
1896 		palloc->level2.leaf + (iwmr->npages >> PBLE_512_SHIFT);  in irdma_set_page()
1898 		palloc_info->addr[iwmr->npages & (PBLE_PER_PAGE - 1)] = addr;  in irdma_set_page()
1901 		pbl[iwmr->npages] = addr;  in irdma_set_page()
1904 	iwmr->npages++;  in irdma_set_page()
1919 	struct irdma_mr *iwmr = to_iwmr(ibmr);  in irdma_map_mr_sg()  local
1921 	iwmr->npages = 0;  in irdma_map_mr_sg()
1933 irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,  in irdma_hwreg_mr()  argument
1936 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_hwreg_mr()
1938 	struct ib_pd *pd = iwmr->ibmr.pd;  in irdma_hwreg_mr()
1953 	stag_info->stag_idx = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;  in irdma_hwreg_mr()
1954 	stag_info->stag_key = (u8)iwmr->stag;  in irdma_hwreg_mr()
1955 	stag_info->total_len = iwmr->len;  in irdma_hwreg_mr()
1964 	stag_info->page_size = iwmr->page_size;  in irdma_hwreg_mr()
1975 		stag_info->reg_addr_pa = iwmr->pgaddrmem[0];  in irdma_hwreg_mr()
1986 		iwmr->is_hwreg = 1;  in irdma_hwreg_mr()
2001 	struct irdma_mr *iwmr;  in irdma_alloc_iwmr()  local
2003 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in irdma_alloc_iwmr()
2004 	if (!iwmr)  in irdma_alloc_iwmr()
2007 	iwpbl = &iwmr->iwpbl;  in irdma_alloc_iwmr()
2008 	iwpbl->iwmr = iwmr;  in irdma_alloc_iwmr()
2009 	iwmr->region = region;  in irdma_alloc_iwmr()
2010 	iwmr->ibmr.pd = pd;  in irdma_alloc_iwmr()
2011 	iwmr->ibmr.device = pd->device;  in irdma_alloc_iwmr()
2012 	iwmr->ibmr.iova = virt;  in irdma_alloc_iwmr()
2013 	iwmr->type = reg_type;  in irdma_alloc_iwmr()
2016 	iwmr->page_msk = ~(IRDMA_HW_PAGE_SIZE - 1);  in irdma_alloc_iwmr()
2017 	iwmr->page_size = IRDMA_HW_PAGE_SIZE;  in irdma_alloc_iwmr()
2018 	iwmr->len = region->length;  in irdma_alloc_iwmr()
2020 	iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt);  in irdma_alloc_iwmr()
2022 	return iwmr;  in irdma_alloc_iwmr()
2026 irdma_free_iwmr(struct irdma_mr *iwmr)  in irdma_free_iwmr()  argument
2028 	kfree(iwmr);  in irdma_free_iwmr()
2038 irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,  in irdma_reg_user_mr_type_mem()  argument
2041 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_reg_user_mr_type_mem()
2042 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_reg_user_mr_type_mem()
2047 	lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;  in irdma_reg_user_mr_type_mem()
2049 	err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);  in irdma_reg_user_mr_type_mem()
2055 						iwmr->page_size);  in irdma_reg_user_mr_type_mem()
2069 		iwmr->stag = stag;  in irdma_reg_user_mr_type_mem()
2070 		iwmr->ibmr.rkey = stag;  in irdma_reg_user_mr_type_mem()
2071 		iwmr->ibmr.lkey = stag;  in irdma_reg_user_mr_type_mem()
2073 	iwmr->access = access;  in irdma_reg_user_mr_type_mem()
2074 	err = irdma_hwreg_mr(iwdev, iwmr, access);  in irdma_reg_user_mr_type_mem()
2097 			  struct irdma_mr *iwmr)  in irdma_reg_user_mr_type_qp()  argument
2099 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_reg_user_mr_type_qp()
2100 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_reg_user_mr_type_qp()
2108 	if (total > iwmr->page_cnt)  in irdma_reg_user_mr_type_qp()
2132 			  struct irdma_mr *iwmr)  in irdma_reg_user_mr_type_cq()  argument
2134 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_reg_user_mr_type_cq()
2135 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_reg_user_mr_type_cq()
2144 	if (total > iwmr->page_cnt)  in irdma_reg_user_mr_type_cq()
2179 	struct irdma_mr *iwmr;  in irdma_reg_user_mr()  local
2201 	iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type);  in irdma_reg_user_mr()
2202 	if (IS_ERR(iwmr)) {  in irdma_reg_user_mr()
2204 		return (struct ib_mr *)iwmr;  in irdma_reg_user_mr()
2209 		err = irdma_reg_user_mr_type_qp(req, udata, iwmr);  in irdma_reg_user_mr()
2215 		err = irdma_reg_user_mr_type_cq(req, udata, iwmr);  in irdma_reg_user_mr()
2221 		err = irdma_reg_user_mr_type_mem(iwmr, access, true);  in irdma_reg_user_mr()
2231 	return &iwmr->ibmr;  in irdma_reg_user_mr()
2235 	irdma_free_iwmr(iwmr);  in irdma_reg_user_mr()
2244 	struct irdma_mr *iwmr = to_iwmr(ib_mr);  in irdma_hwdereg_mr()  local
2247 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_hwdereg_mr()
2256 	if (!iwmr->is_hwreg)  in irdma_hwdereg_mr()
2280 		iwmr->is_hwreg = 0;  in irdma_hwdereg_mr()
2293 irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,  in irdma_rereg_mr_trans()  argument
2296 	struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);  in irdma_rereg_mr_trans()
2297 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_rereg_mr_trans()
2298 	struct ib_pd *pd = iwmr->ibmr.pd;  in irdma_rereg_mr_trans()
2302 	region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0);  in irdma_rereg_mr_trans()
2310 	iwmr->region = region;  in irdma_rereg_mr_trans()
2311 	iwmr->ibmr.iova = virt;  in irdma_rereg_mr_trans()
2312 	iwmr->ibmr.pd = pd;  in irdma_rereg_mr_trans()
2313 	iwmr->page_size = PAGE_SIZE;  in irdma_rereg_mr_trans()
2315 	iwmr->len = region->length;  in irdma_rereg_mr_trans()
2317 	iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size,  in irdma_rereg_mr_trans()
2320 	err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false);  in irdma_rereg_mr_trans()
2324 	return &iwmr->ibmr;  in irdma_rereg_mr_trans()
2345 	struct irdma_mr *iwmr;  in irdma_reg_phys_mr()  local
2349 	iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);  in irdma_reg_phys_mr()
2350 	if (!iwmr)  in irdma_reg_phys_mr()
2353 	iwmr->ibmr.pd = pd;  in irdma_reg_phys_mr()
2354 	iwmr->ibmr.device = pd->device;  in irdma_reg_phys_mr()
2355 	iwpbl = &iwmr->iwpbl;  in irdma_reg_phys_mr()
2356 	iwpbl->iwmr = iwmr;  in irdma_reg_phys_mr()
2357 	iwmr->type = IRDMA_MEMREG_TYPE_MEM;  in irdma_reg_phys_mr()
2365 	iwmr->stag = stag;  in irdma_reg_phys_mr()
2366 	iwmr->ibmr.iova = *iova_start;  in irdma_reg_phys_mr()
2367 	iwmr->ibmr.rkey = stag;  in irdma_reg_phys_mr()
2368 	iwmr->ibmr.lkey = stag;  in irdma_reg_phys_mr()
2369 	iwmr->page_cnt = 1;  in irdma_reg_phys_mr()
2370 	iwmr->pgaddrmem[0] = addr;  in irdma_reg_phys_mr()
2371 	iwmr->len = size;  in irdma_reg_phys_mr()
2372 	iwmr->page_size = SZ_4K;  in irdma_reg_phys_mr()
2373 	ret = irdma_hwreg_mr(iwdev, iwmr, access);  in irdma_reg_phys_mr()
2379 	return &iwmr->ibmr;  in irdma_reg_phys_mr()
2382 	kfree(iwmr);  in irdma_reg_phys_mr()
2406 irdma_del_memlist(struct irdma_mr *iwmr,  in irdma_del_memlist()  argument
2409 	struct irdma_pbl *iwpbl = &iwmr->iwpbl;  in irdma_del_memlist()
2412 	switch (iwmr->type) {  in irdma_del_memlist()
2555 				struct irdma_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr);  in irdma_post_send()  local
2556 				struct irdma_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc;  in irdma_post_send()
2569 				stag_info.va = (void *)(uintptr_t)iwmr->ibmr.iova;  in irdma_post_send()
2570 				stag_info.total_len = iwmr->ibmr.length;  in irdma_post_send()