1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES 3 */ 4 #include "iommufd_private.h" 5 6 struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, 7 size_t size, 8 enum iommufd_object_type type) 9 { 10 struct iommufd_object *obj; 11 int rc; 12 13 obj = kzalloc(size, GFP_KERNEL_ACCOUNT); 14 if (!obj) 15 return ERR_PTR(-ENOMEM); 16 obj->type = type; 17 /* Starts out bias'd by 1 until it is removed from the xarray */ 18 refcount_set(&obj->shortterm_users, 1); 19 refcount_set(&obj->users, 1); 20 21 /* 22 * Reserve an ID in the xarray but do not publish the pointer yet since 23 * the caller hasn't initialized it yet. Once the pointer is published 24 * in the xarray and visible to other threads we can't reliably destroy 25 * it anymore, so the caller must complete all errorable operations 26 * before calling iommufd_object_finalize(). 27 */ 28 rc = xa_alloc(&ictx->objects, &obj->id, XA_ZERO_ENTRY, xa_limit_31b, 29 GFP_KERNEL_ACCOUNT); 30 if (rc) 31 goto out_free; 32 return obj; 33 out_free: 34 kfree(obj); 35 return ERR_PTR(rc); 36 } 37 EXPORT_SYMBOL_NS_GPL(_iommufd_object_alloc, "IOMMUFD"); 38 39 /* Caller should xa_lock(&viommu->vdevs) to protect the return value */ 40 struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu, 41 unsigned long vdev_id) 42 { 43 struct iommufd_vdevice *vdev; 44 45 lockdep_assert_held(&viommu->vdevs.xa_lock); 46 47 vdev = xa_load(&viommu->vdevs, vdev_id); 48 return vdev ? vdev->dev : NULL; 49 } 50 EXPORT_SYMBOL_NS_GPL(iommufd_viommu_find_dev, "IOMMUFD"); 51 52 /* Return -ENOENT if device is not associated to the vIOMMU */ 53 int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu, 54 struct device *dev, unsigned long *vdev_id) 55 { 56 struct iommufd_vdevice *vdev; 57 unsigned long index; 58 int rc = -ENOENT; 59 60 if (WARN_ON_ONCE(!vdev_id)) 61 return -EINVAL; 62 63 xa_lock(&viommu->vdevs); 64 xa_for_each(&viommu->vdevs, index, vdev) { 65 if (vdev->dev == dev) { 66 *vdev_id = vdev->id; 67 rc = 0; 68 break; 69 } 70 } 71 xa_unlock(&viommu->vdevs); 72 return rc; 73 } 74 EXPORT_SYMBOL_NS_GPL(iommufd_viommu_get_vdev_id, "IOMMUFD"); 75 76 /* 77 * Typically called in driver's threaded IRQ handler. 78 * The @type and @event_data must be defined in include/uapi/linux/iommufd.h 79 */ 80 int iommufd_viommu_report_event(struct iommufd_viommu *viommu, 81 enum iommu_veventq_type type, void *event_data, 82 size_t data_len) 83 { 84 struct iommufd_veventq *veventq; 85 struct iommufd_vevent *vevent; 86 int rc = 0; 87 88 if (WARN_ON_ONCE(!data_len || !event_data)) 89 return -EINVAL; 90 91 down_read(&viommu->veventqs_rwsem); 92 93 veventq = iommufd_viommu_find_veventq(viommu, type); 94 if (!veventq) { 95 rc = -EOPNOTSUPP; 96 goto out_unlock_veventqs; 97 } 98 99 spin_lock(&veventq->common.lock); 100 if (veventq->num_events == veventq->depth) { 101 vevent = &veventq->lost_events_header; 102 goto out_set_header; 103 } 104 105 vevent = kzalloc(struct_size(vevent, event_data, data_len), GFP_ATOMIC); 106 if (!vevent) { 107 rc = -ENOMEM; 108 vevent = &veventq->lost_events_header; 109 goto out_set_header; 110 } 111 memcpy(vevent->event_data, event_data, data_len); 112 vevent->data_len = data_len; 113 veventq->num_events++; 114 115 out_set_header: 116 iommufd_vevent_handler(veventq, vevent); 117 spin_unlock(&veventq->common.lock); 118 out_unlock_veventqs: 119 up_read(&viommu->veventqs_rwsem); 120 return rc; 121 } 122 EXPORT_SYMBOL_NS_GPL(iommufd_viommu_report_event, "IOMMUFD"); 123 124 #ifdef CONFIG_IRQ_MSI_IOMMU 125 /* 126 * Get a iommufd_sw_msi_map for the msi physical address requested by the irq 127 * layer. The mapping to IOVA is global to the iommufd file descriptor, every 128 * domain that is attached to a device using the same MSI parameters will use 129 * the same IOVA. 130 */ 131 static struct iommufd_sw_msi_map * 132 iommufd_sw_msi_get_map(struct iommufd_ctx *ictx, phys_addr_t msi_addr, 133 phys_addr_t sw_msi_start) 134 { 135 struct iommufd_sw_msi_map *cur; 136 unsigned int max_pgoff = 0; 137 138 lockdep_assert_held(&ictx->sw_msi_lock); 139 140 list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) { 141 if (cur->sw_msi_start != sw_msi_start) 142 continue; 143 max_pgoff = max(max_pgoff, cur->pgoff + 1); 144 if (cur->msi_addr == msi_addr) 145 return cur; 146 } 147 148 if (ictx->sw_msi_id >= 149 BITS_PER_BYTE * sizeof_field(struct iommufd_sw_msi_maps, bitmap)) 150 return ERR_PTR(-EOVERFLOW); 151 152 cur = kzalloc(sizeof(*cur), GFP_KERNEL); 153 if (!cur) 154 return ERR_PTR(-ENOMEM); 155 156 cur->sw_msi_start = sw_msi_start; 157 cur->msi_addr = msi_addr; 158 cur->pgoff = max_pgoff; 159 cur->id = ictx->sw_msi_id++; 160 list_add_tail(&cur->sw_msi_item, &ictx->sw_msi_list); 161 return cur; 162 } 163 164 int iommufd_sw_msi_install(struct iommufd_ctx *ictx, 165 struct iommufd_hwpt_paging *hwpt_paging, 166 struct iommufd_sw_msi_map *msi_map) 167 { 168 unsigned long iova; 169 170 lockdep_assert_held(&ictx->sw_msi_lock); 171 172 iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE; 173 if (!test_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap)) { 174 int rc; 175 176 rc = iommu_map(hwpt_paging->common.domain, iova, 177 msi_map->msi_addr, PAGE_SIZE, 178 IOMMU_WRITE | IOMMU_READ | IOMMU_MMIO, 179 GFP_KERNEL_ACCOUNT); 180 if (rc) 181 return rc; 182 __set_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap); 183 } 184 return 0; 185 } 186 EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi_install, "IOMMUFD_INTERNAL"); 187 188 /* 189 * Called by the irq code if the platform translates the MSI address through the 190 * IOMMU. msi_addr is the physical address of the MSI page. iommufd will 191 * allocate a fd global iova for the physical page that is the same on all 192 * domains and devices. 193 */ 194 int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc, 195 phys_addr_t msi_addr) 196 { 197 struct device *dev = msi_desc_to_dev(desc); 198 struct iommufd_hwpt_paging *hwpt_paging; 199 struct iommu_attach_handle *raw_handle; 200 struct iommufd_attach_handle *handle; 201 struct iommufd_sw_msi_map *msi_map; 202 struct iommufd_ctx *ictx; 203 unsigned long iova; 204 int rc; 205 206 /* 207 * It is safe to call iommu_attach_handle_get() here because the iommu 208 * core code invokes this under the group mutex which also prevents any 209 * change of the attach handle for the duration of this function. 210 */ 211 iommu_group_mutex_assert(dev); 212 213 raw_handle = 214 iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0); 215 if (IS_ERR(raw_handle)) 216 return 0; 217 hwpt_paging = find_hwpt_paging(domain->iommufd_hwpt); 218 219 handle = to_iommufd_handle(raw_handle); 220 /* No IOMMU_RESV_SW_MSI means no change to the msi_msg */ 221 if (handle->idev->igroup->sw_msi_start == PHYS_ADDR_MAX) 222 return 0; 223 224 ictx = handle->idev->ictx; 225 guard(mutex)(&ictx->sw_msi_lock); 226 /* 227 * The input msi_addr is the exact byte offset of the MSI doorbell, we 228 * assume the caller has checked that it is contained with a MMIO region 229 * that is secure to map at PAGE_SIZE. 230 */ 231 msi_map = iommufd_sw_msi_get_map(handle->idev->ictx, 232 msi_addr & PAGE_MASK, 233 handle->idev->igroup->sw_msi_start); 234 if (IS_ERR(msi_map)) 235 return PTR_ERR(msi_map); 236 237 rc = iommufd_sw_msi_install(ictx, hwpt_paging, msi_map); 238 if (rc) 239 return rc; 240 __set_bit(msi_map->id, handle->idev->igroup->required_sw_msi.bitmap); 241 242 iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE; 243 msi_desc_set_iommu_msi_iova(desc, iova, PAGE_SHIFT); 244 return 0; 245 } 246 EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi, "IOMMUFD"); 247 #endif 248 249 MODULE_DESCRIPTION("iommufd code shared with builtin modules"); 250 MODULE_IMPORT_NS("IOMMUFD_INTERNAL"); 251 MODULE_LICENSE("GPL"); 252