83 	if (mr->umem->odp_data) {  in update_odp_mr()
92 		mr->umem->odp_data->private = mr;  in update_odp_mr()
518 	mr->umem = NULL;  in mlx5_ib_get_dma_mr()
546 static int dma_map_mr_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,  in dma_map_mr_pas()  argument
564 	mlx5_ib_populate_pas(dev, umem, page_shift, pas, MLX5_IB_MTT_PRESENT);  in dma_map_mr_pas()
629 	struct ib_umem *umem = ib_umem_get(pd->uobject->context, start, length,  in mr_umem_get()  local
631 	if (IS_ERR(umem)) {  in mr_umem_get()
632 		mlx5_ib_err(dev, "umem get failed (%ld)\n", PTR_ERR(umem));  in mr_umem_get()
633 		return (void *)umem;  in mr_umem_get()
636 	mlx5_ib_cont_pages(umem, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages, page_shift, ncont, order);  in mr_umem_get()
639 		ib_umem_release(umem);  in mr_umem_get()
646 	return umem;  in mr_umem_get()
665 static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,  in reg_umr()  argument
698 	err = dma_map_mr_pas(dev, umem, npages, page_shift, &size, &mr_pas,  in reg_umr()
751 	struct ib_umem *umem = mr->umem;  in mlx5_ib_update_mtt()  local
807 			       ib_umem_num_pages(umem) - start_page_index);  in mlx5_ib_update_mtt()
810 			__mlx5_ib_populate_pas(dev, umem, PAGE_SHIFT,  in mlx5_ib_update_mtt()
872 				     struct ib_umem *umem, int npages,  in reg_create()  argument
896 	mlx5_ib_populate_pas(dev, umem, page_shift, pas,  in reg_create()
927 	mr->umem = umem;  in reg_create()
963 	struct ib_umem *umem;  in mlx5_ib_reg_user_mr()  local
972 	umem = mr_umem_get(pd, start, length, access_flags, &npages,  in mlx5_ib_reg_user_mr()
975 	if (IS_ERR(umem))  in mlx5_ib_reg_user_mr()
976 		return (void *)umem;  in mlx5_ib_reg_user_mr()
979 		mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift,  in mlx5_ib_reg_user_mr()
993 		mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,  in mlx5_ib_reg_user_mr()
1005 	mr->umem = umem;  in mlx5_ib_reg_user_mr()
1015 	ib_umem_release(umem);  in mlx5_ib_reg_user_mr()
1079 		err = dma_map_mr_pas(dev, mr->umem, npages, page_shift, &size,  in rereg_umr()
1135 	u64 addr = (flags & IB_MR_REREG_TRANS) ? virt_addr : mr->umem->address;  in mlx5_ib_rereg_user_mr()
1136 	u64 len = (flags & IB_MR_REREG_TRANS) ? length : mr->umem->length;  in mlx5_ib_rereg_user_mr()
1152 		ib_umem_release(mr->umem);  in mlx5_ib_rereg_user_mr()
1153 		mr->umem = mr_umem_get(pd, addr, len, access_flags, &npages,  in mlx5_ib_rereg_user_mr()
1155 		if (IS_ERR(mr->umem)) {  in mlx5_ib_rereg_user_mr()
1156 			err = PTR_ERR(mr->umem);  in mlx5_ib_rereg_user_mr()
1157 			mr->umem = NULL;  in mlx5_ib_rereg_user_mr()
1178 		mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,  in mlx5_ib_rereg_user_mr()
1312 	struct ib_umem *umem = mr->umem;  in mlx5_ib_dereg_mr()  local
1315 	if (umem && umem->odp_data) {  in mlx5_ib_dereg_mr()
1321 		mlx5_ib_invalidate_range(umem, ib_umem_start(umem),  in mlx5_ib_dereg_mr()
1322 					 ib_umem_end(umem));  in mlx5_ib_dereg_mr()
1328 		ib_umem_release(umem);  in mlx5_ib_dereg_mr()
1332 		umem = NULL;  in mlx5_ib_dereg_mr()
1338 	if (umem) {  in mlx5_ib_dereg_mr()
1339 		ib_umem_release(umem);  in mlx5_ib_dereg_mr()
1433 	mr->umem = NULL;  in mlx5_ib_alloc_mr()