1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES 3 */ 4 #include "iommufd_private.h" 5 6 /* Driver should use a per-structure helper in include/linux/iommufd.h */ 7 int _iommufd_object_depend(struct iommufd_object *obj_dependent, 8 struct iommufd_object *obj_depended) 9 { 10 /* Reject self dependency that dead locks */ 11 if (obj_dependent == obj_depended) 12 return -EINVAL; 13 /* Only support dependency between two objects of the same type */ 14 if (obj_dependent->type != obj_depended->type) 15 return -EINVAL; 16 17 refcount_inc(&obj_depended->users); 18 return 0; 19 } 20 EXPORT_SYMBOL_NS_GPL(_iommufd_object_depend, "IOMMUFD"); 21 22 /* Driver should use a per-structure helper in include/linux/iommufd.h */ 23 void _iommufd_object_undepend(struct iommufd_object *obj_dependent, 24 struct iommufd_object *obj_depended) 25 { 26 if (WARN_ON_ONCE(obj_dependent == obj_depended || 27 obj_dependent->type != obj_depended->type)) 28 return; 29 30 refcount_dec(&obj_depended->users); 31 } 32 EXPORT_SYMBOL_NS_GPL(_iommufd_object_undepend, "IOMMUFD"); 33 34 /* 35 * Allocate an @offset to return to user space to use for an mmap() syscall 36 * 37 * Driver should use a per-structure helper in include/linux/iommufd.h 38 */ 39 int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, 40 phys_addr_t mmio_addr, size_t length, 41 unsigned long *offset) 42 { 43 struct iommufd_mmap *immap; 44 unsigned long startp; 45 int rc; 46 47 if (!PAGE_ALIGNED(mmio_addr)) 48 return -EINVAL; 49 if (!length || !PAGE_ALIGNED(length)) 50 return -EINVAL; 51 52 immap = kzalloc(sizeof(*immap), GFP_KERNEL); 53 if (!immap) 54 return -ENOMEM; 55 immap->owner = owner; 56 immap->length = length; 57 immap->mmio_addr = mmio_addr; 58 59 /* Skip the first page to ease caller identifying the returned offset */ 60 rc = mtree_alloc_range(&ictx->mt_mmap, &startp, immap, immap->length, 61 PAGE_SIZE, ULONG_MAX, GFP_KERNEL); 62 if (rc < 0) { 63 kfree(immap); 64 return rc; 65 } 66 67 /* mmap() syscall will right-shift the offset in vma->vm_pgoff too */ 68 immap->vm_pgoff = startp >> PAGE_SHIFT; 69 *offset = startp; 70 return 0; 71 } 72 EXPORT_SYMBOL_NS_GPL(_iommufd_alloc_mmap, "IOMMUFD"); 73 74 /* Driver should use a per-structure helper in include/linux/iommufd.h */ 75 void _iommufd_destroy_mmap(struct iommufd_ctx *ictx, 76 struct iommufd_object *owner, unsigned long offset) 77 { 78 struct iommufd_mmap *immap; 79 80 immap = mtree_erase(&ictx->mt_mmap, offset); 81 WARN_ON_ONCE(!immap || immap->owner != owner); 82 kfree(immap); 83 } 84 EXPORT_SYMBOL_NS_GPL(_iommufd_destroy_mmap, "IOMMUFD"); 85 86 struct device *iommufd_vdevice_to_device(struct iommufd_vdevice *vdev) 87 { 88 return vdev->idev->dev; 89 } 90 EXPORT_SYMBOL_NS_GPL(iommufd_vdevice_to_device, "IOMMUFD"); 91 92 /* Caller should xa_lock(&viommu->vdevs) to protect the return value */ 93 struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu, 94 unsigned long vdev_id) 95 { 96 struct iommufd_vdevice *vdev; 97 98 lockdep_assert_held(&viommu->vdevs.xa_lock); 99 100 vdev = xa_load(&viommu->vdevs, vdev_id); 101 return vdev ? iommufd_vdevice_to_device(vdev) : NULL; 102 } 103 EXPORT_SYMBOL_NS_GPL(iommufd_viommu_find_dev, "IOMMUFD"); 104 105 /* Return -ENOENT if device is not associated to the vIOMMU */ 106 int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu, 107 struct device *dev, unsigned long *vdev_id) 108 { 109 struct iommufd_vdevice *vdev; 110 unsigned long index; 111 int rc = -ENOENT; 112 113 if (WARN_ON_ONCE(!vdev_id)) 114 return -EINVAL; 115 116 xa_lock(&viommu->vdevs); 117 xa_for_each(&viommu->vdevs, index, vdev) { 118 if (iommufd_vdevice_to_device(vdev) == dev) { 119 *vdev_id = vdev->virt_id; 120 rc = 0; 121 break; 122 } 123 } 124 xa_unlock(&viommu->vdevs); 125 return rc; 126 } 127 EXPORT_SYMBOL_NS_GPL(iommufd_viommu_get_vdev_id, "IOMMUFD"); 128 129 /* 130 * Typically called in driver's threaded IRQ handler. 131 * The @type and @event_data must be defined in include/uapi/linux/iommufd.h 132 */ 133 int iommufd_viommu_report_event(struct iommufd_viommu *viommu, 134 enum iommu_veventq_type type, void *event_data, 135 size_t data_len) 136 { 137 struct iommufd_veventq *veventq; 138 struct iommufd_vevent *vevent; 139 int rc = 0; 140 141 if (WARN_ON_ONCE(!data_len || !event_data)) 142 return -EINVAL; 143 144 down_read(&viommu->veventqs_rwsem); 145 146 veventq = iommufd_viommu_find_veventq(viommu, type); 147 if (!veventq) { 148 rc = -EOPNOTSUPP; 149 goto out_unlock_veventqs; 150 } 151 152 /* Pre-allocate to avoid GFP_ATOMIC; use GFP_NOWAIT to avoid sleeping */ 153 vevent = kzalloc_flex(*vevent, event_data, data_len, GFP_NOWAIT); 154 if (!vevent) { 155 spin_lock(&veventq->common.lock); 156 vevent = &veventq->lost_events_header; 157 rc = -ENOMEM; 158 goto out_set_header; 159 } 160 161 spin_lock(&veventq->common.lock); 162 if (veventq->num_events == veventq->depth) { 163 kfree(vevent); 164 vevent = &veventq->lost_events_header; 165 goto out_set_header; 166 } 167 vevent->data_len = data_len; 168 memcpy(vevent->event_data, event_data, data_len); 169 veventq->num_events++; 170 171 out_set_header: 172 iommufd_vevent_handler(veventq, vevent); 173 spin_unlock(&veventq->common.lock); 174 out_unlock_veventqs: 175 up_read(&viommu->veventqs_rwsem); 176 return rc; 177 } 178 EXPORT_SYMBOL_NS_GPL(iommufd_viommu_report_event, "IOMMUFD"); 179 180 #ifdef CONFIG_IRQ_MSI_IOMMU 181 /* 182 * Get a iommufd_sw_msi_map for the msi physical address requested by the irq 183 * layer. The mapping to IOVA is global to the iommufd file descriptor, every 184 * domain that is attached to a device using the same MSI parameters will use 185 * the same IOVA. 186 */ 187 static struct iommufd_sw_msi_map * 188 iommufd_sw_msi_get_map(struct iommufd_ctx *ictx, phys_addr_t msi_addr, 189 phys_addr_t sw_msi_start) 190 { 191 struct iommufd_sw_msi_map *cur; 192 unsigned int max_pgoff = 0; 193 194 lockdep_assert_held(&ictx->sw_msi_lock); 195 196 list_for_each_entry(cur, &ictx->sw_msi_list, sw_msi_item) { 197 if (cur->sw_msi_start != sw_msi_start) 198 continue; 199 max_pgoff = max(max_pgoff, cur->pgoff + 1); 200 if (cur->msi_addr == msi_addr) 201 return cur; 202 } 203 204 if (ictx->sw_msi_id >= 205 BITS_PER_BYTE * sizeof_field(struct iommufd_sw_msi_maps, bitmap)) 206 return ERR_PTR(-EOVERFLOW); 207 208 cur = kzalloc_obj(*cur); 209 if (!cur) 210 return ERR_PTR(-ENOMEM); 211 212 cur->sw_msi_start = sw_msi_start; 213 cur->msi_addr = msi_addr; 214 cur->pgoff = max_pgoff; 215 cur->id = ictx->sw_msi_id++; 216 list_add_tail(&cur->sw_msi_item, &ictx->sw_msi_list); 217 return cur; 218 } 219 220 int iommufd_sw_msi_install(struct iommufd_ctx *ictx, 221 struct iommufd_hwpt_paging *hwpt_paging, 222 struct iommufd_sw_msi_map *msi_map) 223 { 224 unsigned long iova; 225 226 lockdep_assert_held(&ictx->sw_msi_lock); 227 228 iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE; 229 if (!test_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap)) { 230 int rc; 231 232 rc = iommu_map(hwpt_paging->common.domain, iova, 233 msi_map->msi_addr, PAGE_SIZE, 234 IOMMU_WRITE | IOMMU_READ | IOMMU_MMIO, 235 GFP_KERNEL_ACCOUNT); 236 if (rc) 237 return rc; 238 __set_bit(msi_map->id, hwpt_paging->present_sw_msi.bitmap); 239 } 240 return 0; 241 } 242 EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi_install, "IOMMUFD_INTERNAL"); 243 244 /* 245 * Called by the irq code if the platform translates the MSI address through the 246 * IOMMU. msi_addr is the physical address of the MSI page. iommufd will 247 * allocate a fd global iova for the physical page that is the same on all 248 * domains and devices. 249 */ 250 int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc, 251 phys_addr_t msi_addr) 252 { 253 struct device *dev = msi_desc_to_dev(desc); 254 struct iommufd_hwpt_paging *hwpt_paging; 255 struct iommu_attach_handle *raw_handle; 256 struct iommufd_attach_handle *handle; 257 struct iommufd_sw_msi_map *msi_map; 258 struct iommufd_ctx *ictx; 259 unsigned long iova; 260 int rc; 261 262 /* 263 * It is safe to call iommu_attach_handle_get() here because the iommu 264 * core code invokes this under the group mutex which also prevents any 265 * change of the attach handle for the duration of this function. 266 */ 267 iommu_group_mutex_assert(dev); 268 269 raw_handle = 270 iommu_attach_handle_get(dev->iommu_group, IOMMU_NO_PASID, 0); 271 if (IS_ERR(raw_handle)) 272 return 0; 273 hwpt_paging = find_hwpt_paging(domain->iommufd_hwpt); 274 275 handle = to_iommufd_handle(raw_handle); 276 /* No IOMMU_RESV_SW_MSI means no change to the msi_msg */ 277 if (handle->idev->igroup->sw_msi_start == PHYS_ADDR_MAX) 278 return 0; 279 280 ictx = handle->idev->ictx; 281 guard(mutex)(&ictx->sw_msi_lock); 282 /* 283 * The input msi_addr is the exact byte offset of the MSI doorbell, we 284 * assume the caller has checked that it is contained with a MMIO region 285 * that is secure to map at PAGE_SIZE. 286 */ 287 msi_map = iommufd_sw_msi_get_map(handle->idev->ictx, 288 msi_addr & PAGE_MASK, 289 handle->idev->igroup->sw_msi_start); 290 if (IS_ERR(msi_map)) 291 return PTR_ERR(msi_map); 292 293 rc = iommufd_sw_msi_install(ictx, hwpt_paging, msi_map); 294 if (rc) 295 return rc; 296 __set_bit(msi_map->id, handle->idev->igroup->required_sw_msi.bitmap); 297 298 iova = msi_map->sw_msi_start + msi_map->pgoff * PAGE_SIZE; 299 msi_desc_set_iommu_msi_iova(desc, iova, PAGE_SHIFT); 300 return 0; 301 } 302 EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi, "IOMMUFD"); 303 #endif 304 305 MODULE_DESCRIPTION("iommufd code shared with builtin modules"); 306 MODULE_IMPORT_NS("IOMMUFD_INTERNAL"); 307 MODULE_LICENSE("GPL"); 308