133 	WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)));  in destroy_mkey()
135 	return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key);  in destroy_mkey()
756 		err = create_cache_mkey(ent, &mr->mmkey.key);  in _mlx5_mr_cache_alloc()
765 		mr->mmkey.key = pop_mkey_locked(ent);  in _mlx5_mr_cache_alloc()
769 	mr->mmkey.cache_ent = ent;  in _mlx5_mr_cache_alloc()
770 	mr->mmkey.type = MLX5_MKEY_MR;  in _mlx5_mr_cache_alloc()
771 	mr->mmkey.rb_key = ent->rb_key;  in _mlx5_mr_cache_alloc()
772 	mr->mmkey.cacheable = true;  in _mlx5_mr_cache_alloc()
773 	init_waitqueue_head(&mr->mmkey.wait);  in _mlx5_mr_cache_alloc()
1080 	err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);  in mlx5_ib_get_dma_mr()
1085 	mr->mmkey.type = MLX5_MKEY_MR;  in mlx5_ib_get_dma_mr()
1086 	mr->ibmr.lkey = mr->mmkey.key;  in mlx5_ib_get_dma_mr()
1087 	mr->ibmr.rkey = mr->mmkey.key;  in mlx5_ib_get_dma_mr()
1122 	mr->ibmr.lkey = mr->mmkey.key;  in set_mr_fields()
1123 	mr->ibmr.rkey = mr->mmkey.key;  in set_mr_fields()
1178 		mr->mmkey.rb_key = rb_key;  in alloc_cacheable_mr()
1179 		mr->mmkey.cacheable = true;  in alloc_cacheable_mr()
1234 	err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);  in reg_create_crossing_vhca_mr()
1238 	mr->mmkey.type = MLX5_MKEY_MR;  in reg_create_crossing_vhca_mr()
1242 	mlx5_ib_dbg(dev, "crossing mkey = 0x%x\n", mr->mmkey.key);  in reg_create_crossing_vhca_mr()
1341 	err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);  in reg_create()
1346 	mr->mmkey.type = MLX5_MKEY_MR;  in reg_create()
1347 	mr->mmkey.ndescs = get_octo_len(iova, umem->length, mr->page_shift);  in reg_create()
1352 	mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);  in reg_create()
1390 	err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);  in mlx5_ib_get_dm_mr()
1498 	mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);  in create_real_mr()
1564 	err = mlx5r_store_odp_mkey(dev, &mr->mmkey);  in create_user_odp_mr()
1676 	mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);  in reg_user_mr_dmabuf()
1681 		err = mlx5r_store_odp_mkey(dev, &mr->mmkey);  in reg_user_mr_dmabuf()
1818 	if (!mr->mmkey.cache_ent)  in can_use_umr_rereg_pas()
1824 		dev, new_umem, iova, mr->mmkey.cache_ent->rb_key.access_mode);  in can_use_umr_rereg_pas()
1827 	return (mr->mmkey.cache_ent->rb_key.ndescs) >=  in can_use_umr_rereg_pas()
1888 	    mr->mmkey.rb_key.ph != MLX5_IB_NO_PH)  in mlx5_ib_rereg_user_mr()
2036 	if (mr->mmkey.cache_ent) {  in cache_ent_find_and_store()
2037 		spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);  in cache_ent_find_and_store()
2042 	ent = mkey_cache_ent_from_rb_key(dev, mr->mmkey.rb_key);  in cache_ent_find_and_store()
2044 		if (ent->rb_key.ndescs == mr->mmkey.rb_key.ndescs) {  in cache_ent_find_and_store()
2049 			mr->mmkey.cache_ent = ent;  in cache_ent_find_and_store()
2050 			spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);  in cache_ent_find_and_store()
2056 	ent = mlx5r_cache_create_ent_locked(dev, mr->mmkey.rb_key, false);  in cache_ent_find_and_store()
2061 	mr->mmkey.cache_ent = ent;  in cache_ent_find_and_store()
2062 	spin_lock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);  in cache_ent_find_and_store()
2065 	ret = push_mkey_locked(mr->mmkey.cache_ent, mr->mmkey.key);  in cache_ent_find_and_store()
2066 	spin_unlock_irq(&mr->mmkey.cache_ent->mkeys_queue.lock);  in cache_ent_find_and_store()
2135 	struct mlx5_cache_ent *ent = mr->mmkey.cache_ent;  in mlx5r_handle_mkey_cleanup()
2140 	if (mr->mmkey.cacheable && !mlx5_umr_revoke_mr_with_lock(mr) &&  in mlx5r_handle_mkey_cleanup()
2142 		ent = mr->mmkey.cache_ent;  in mlx5r_handle_mkey_cleanup()
2159 		mr->mmkey.cache_ent = NULL;  in mlx5r_handle_mkey_cleanup()
2197 	    refcount_read(&mr->mmkey.usecount) != 0 &&  in __mlx5_ib_dereg_mr()
2198 	    xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key)))  in __mlx5_ib_dereg_mr()
2199 		mlx5r_deref_wait_odp_mkey(&mr->mmkey);  in __mlx5_ib_dereg_mr()
2202 		xa_cmpxchg(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key),  in __mlx5_ib_dereg_mr()
2245 	if (!mr->mmkey.cache_ent)  in __mlx5_ib_dereg_mr()
2320 	err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);  in _mlx5_alloc_mkey_descs()
2324 	mr->mmkey.type = MLX5_MKEY_MR;  in _mlx5_alloc_mkey_descs()
2325 	mr->ibmr.lkey = mr->mmkey.key;  in _mlx5_alloc_mkey_descs()
2326 	mr->ibmr.rkey = mr->mmkey.key;  in _mlx5_alloc_mkey_descs()
2444 	err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key),  in mlx5_alloc_integrity_descs()
2583 	err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen);  in mlx5_ib_alloc_mw()
2587 	mw->mmkey.type = MLX5_MKEY_MW;  in mlx5_ib_alloc_mw()
2588 	ibmw->rkey = mw->mmkey.key;  in mlx5_ib_alloc_mw()
2589 	mw->mmkey.ndescs = ndescs;  in mlx5_ib_alloc_mw()
2600 		err = mlx5r_store_odp_mkey(dev, &mw->mmkey);  in mlx5_ib_alloc_mw()
2609 	mlx5_core_destroy_mkey(dev->mdev, mw->mmkey.key);  in mlx5_ib_alloc_mw()
2621 	    xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key)))  in mlx5_ib_dealloc_mw()
2626 		mlx5r_deref_wait_odp_mkey(&mmw->mmkey);  in mlx5_ib_dealloc_mw()
2628 	return mlx5_core_destroy_mkey(dev->mdev, mmw->mmkey.key);  in mlx5_ib_dealloc_mw()
2685 		mr->mmkey.ndescs = 1;  in mlx5_ib_map_pa_mr_sg_pi()
2738 	mr->mmkey.ndescs = i;  in mlx5_ib_sg_to_klms()
2771 	if (unlikely(mr->mmkey.ndescs == mr->max_descs))  in mlx5_set_page()
2775 	descs[mr->mmkey.ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR);  in mlx5_set_page()
2785 	if (unlikely(mr->mmkey.ndescs + mr->meta_ndescs == mr->max_descs))  in mlx5_set_page_pi()
2789 	descs[mr->mmkey.ndescs + mr->meta_ndescs++] =  in mlx5_set_page_pi()
2805 	pi_mr->mmkey.ndescs = 0;  in mlx5_ib_map_mtt_mr_sg_pi()
2839 				 pi_mr->mmkey.ndescs * ibmr->page_size +  in mlx5_ib_map_mtt_mr_sg_pi()
2870 	pi_mr->mmkey.ndescs = 0;  in mlx5_ib_map_klm_mr_sg_pi()
2905 	mr->mmkey.ndescs = 0;  in mlx5_ib_map_mr_sg_pi()
2961 	mr->mmkey.ndescs = 0;  in mlx5_ib_map_mr_sg()