1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES 4 */ 5 #include <linux/iommu.h> 6 #include <uapi/linux/iommufd.h> 7 8 #include "iommufd_private.h" 9 10 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj) 11 { 12 struct iommufd_hw_pagetable *hwpt = 13 container_of(obj, struct iommufd_hw_pagetable, obj); 14 15 if (!list_empty(&hwpt->hwpt_item)) { 16 mutex_lock(&hwpt->ioas->mutex); 17 list_del(&hwpt->hwpt_item); 18 mutex_unlock(&hwpt->ioas->mutex); 19 20 iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain); 21 } 22 23 if (hwpt->domain) 24 iommu_domain_free(hwpt->domain); 25 26 refcount_dec(&hwpt->ioas->obj.users); 27 } 28 29 void iommufd_hw_pagetable_abort(struct iommufd_object *obj) 30 { 31 struct iommufd_hw_pagetable *hwpt = 32 container_of(obj, struct iommufd_hw_pagetable, obj); 33 34 /* The ioas->mutex must be held until finalize is called. */ 35 lockdep_assert_held(&hwpt->ioas->mutex); 36 37 if (!list_empty(&hwpt->hwpt_item)) { 38 list_del_init(&hwpt->hwpt_item); 39 iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain); 40 } 41 iommufd_hw_pagetable_destroy(obj); 42 } 43 44 int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt) 45 { 46 if (hwpt->enforce_cache_coherency) 47 return 0; 48 49 if (hwpt->domain->ops->enforce_cache_coherency) 50 hwpt->enforce_cache_coherency = 51 hwpt->domain->ops->enforce_cache_coherency( 52 hwpt->domain); 53 if (!hwpt->enforce_cache_coherency) 54 return -EINVAL; 55 return 0; 56 } 57 58 /** 59 * iommufd_hw_pagetable_alloc() - Get an iommu_domain for a device 60 * @ictx: iommufd context 61 * @ioas: IOAS to associate the domain with 62 * @idev: Device to get an iommu_domain for 63 * @immediate_attach: True if idev should be attached to the hwpt 64 * 65 * Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT 66 * will be linked to the given ioas and upon return the underlying iommu_domain 67 * is fully popoulated. 68 * 69 * The caller must hold the ioas->mutex until after 70 * iommufd_object_abort_and_destroy() or iommufd_object_finalize() is called on 71 * the returned hwpt. 72 */ 73 struct iommufd_hw_pagetable * 74 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, 75 struct iommufd_device *idev, bool immediate_attach) 76 { 77 struct iommufd_hw_pagetable *hwpt; 78 int rc; 79 80 lockdep_assert_held(&ioas->mutex); 81 82 hwpt = iommufd_object_alloc(ictx, hwpt, IOMMUFD_OBJ_HW_PAGETABLE); 83 if (IS_ERR(hwpt)) 84 return hwpt; 85 86 INIT_LIST_HEAD(&hwpt->hwpt_item); 87 /* Pairs with iommufd_hw_pagetable_destroy() */ 88 refcount_inc(&ioas->obj.users); 89 hwpt->ioas = ioas; 90 91 hwpt->domain = iommu_domain_alloc(idev->dev->bus); 92 if (!hwpt->domain) { 93 rc = -ENOMEM; 94 goto out_abort; 95 } 96 97 /* 98 * Set the coherency mode before we do iopt_table_add_domain() as some 99 * iommus have a per-PTE bit that controls it and need to decide before 100 * doing any maps. It is an iommu driver bug to report 101 * IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail enforce_cache_coherency on 102 * a new domain. 103 */ 104 if (idev->enforce_cache_coherency) { 105 rc = iommufd_hw_pagetable_enforce_cc(hwpt); 106 if (WARN_ON(rc)) 107 goto out_abort; 108 } 109 110 /* 111 * immediate_attach exists only to accommodate iommu drivers that cannot 112 * directly allocate a domain. These drivers do not finish creating the 113 * domain until attach is completed. Thus we must have this call 114 * sequence. Once those drivers are fixed this should be removed. 115 */ 116 if (immediate_attach) { 117 rc = iommufd_hw_pagetable_attach(hwpt, idev); 118 if (rc) 119 goto out_abort; 120 } 121 122 rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain); 123 if (rc) 124 goto out_detach; 125 list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list); 126 return hwpt; 127 128 out_detach: 129 if (immediate_attach) 130 iommufd_hw_pagetable_detach(idev); 131 out_abort: 132 iommufd_object_abort_and_destroy(ictx, &hwpt->obj); 133 return ERR_PTR(rc); 134 } 135 136 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd) 137 { 138 struct iommu_hwpt_alloc *cmd = ucmd->cmd; 139 struct iommufd_hw_pagetable *hwpt; 140 struct iommufd_device *idev; 141 struct iommufd_ioas *ioas; 142 int rc; 143 144 if (cmd->flags || cmd->__reserved) 145 return -EOPNOTSUPP; 146 147 idev = iommufd_get_device(ucmd, cmd->dev_id); 148 if (IS_ERR(idev)) 149 return PTR_ERR(idev); 150 151 ioas = iommufd_get_ioas(ucmd->ictx, cmd->pt_id); 152 if (IS_ERR(ioas)) { 153 rc = PTR_ERR(ioas); 154 goto out_put_idev; 155 } 156 157 mutex_lock(&ioas->mutex); 158 hwpt = iommufd_hw_pagetable_alloc(ucmd->ictx, ioas, idev, false); 159 if (IS_ERR(hwpt)) { 160 rc = PTR_ERR(hwpt); 161 goto out_unlock; 162 } 163 164 cmd->out_hwpt_id = hwpt->obj.id; 165 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 166 if (rc) 167 goto out_hwpt; 168 iommufd_object_finalize(ucmd->ictx, &hwpt->obj); 169 goto out_unlock; 170 171 out_hwpt: 172 iommufd_object_abort_and_destroy(ucmd->ictx, &hwpt->obj); 173 out_unlock: 174 mutex_unlock(&ioas->mutex); 175 iommufd_put_object(&ioas->obj); 176 out_put_idev: 177 iommufd_put_object(&idev->obj); 178 return rc; 179 } 180