iommu.c (6f47c7ae8c7afaf9ad291d39f0d3974f191a7946) iommu.c (14678219cf4093e897ab353fd78eab7994d1be7d)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 */
6
7#define pr_fmt(fmt) "iommu: " fmt
8

--- 567 unchanged lines hidden (view full) ---

576 /*
577 * With a group_list argument we defer the default_domain setup
578 * to the caller by providing a de-duplicated list of groups
579 * that need further setup.
580 */
581 if (list_empty(&group->entry))
582 list_add_tail(&group->entry, group_list);
583 }
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2007-2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <jroedel@suse.de>
5 */
6
7#define pr_fmt(fmt) "iommu: " fmt
8

--- 567 unchanged lines hidden (view full) ---

576 /*
577 * With a group_list argument we defer the default_domain setup
578 * to the caller by providing a de-duplicated list of groups
579 * that need further setup.
580 */
581 if (list_empty(&group->entry))
582 list_add_tail(&group->entry, group_list);
583 }
584 mutex_unlock(&group->mutex);
585
584
586 if (dev_is_pci(dev))
587 iommu_dma_set_pci_32bit_workaround(dev);
585 if (group->default_domain)
586 iommu_setup_dma_ops(dev);
588
587
588 mutex_unlock(&group->mutex);
589
589 return 0;
590
591err_remove_gdev:
592 list_del(&gdev->list);
593 __iommu_group_free_device(group, gdev);
594err_put_group:
595 iommu_deinit_device(dev);
596 mutex_unlock(&group->mutex);

--- 1226 unchanged lines hidden (view full) ---

1823 * that the cross-group default domain type and the setup of the
1824 * IOMMU_RESV_DIRECT will work correctly in non-hotpug scenarios.
1825 */
1826 ret = iommu_setup_default_domain(group, 0);
1827 if (ret) {
1828 mutex_unlock(&group->mutex);
1829 return ret;
1830 }
590 return 0;
591
592err_remove_gdev:
593 list_del(&gdev->list);
594 __iommu_group_free_device(group, gdev);
595err_put_group:
596 iommu_deinit_device(dev);
597 mutex_unlock(&group->mutex);

--- 1226 unchanged lines hidden (view full) ---

1824 * that the cross-group default domain type and the setup of the
1825 * IOMMU_RESV_DIRECT will work correctly in non-hotpug scenarios.
1826 */
1827 ret = iommu_setup_default_domain(group, 0);
1828 if (ret) {
1829 mutex_unlock(&group->mutex);
1830 return ret;
1831 }
1832 for_each_group_device(group, gdev)
1833 iommu_setup_dma_ops(gdev->dev);
1831 mutex_unlock(&group->mutex);
1832
1833 /*
1834 * FIXME: Mis-locked because the ops->probe_finalize() call-back
1835 * of some IOMMU drivers calls arm_iommu_attach_device() which
1836 * in-turn might call back into IOMMU core code, where it tries
1837 * to take group->mutex, resulting in a deadlock.
1838 */

--- 1222 unchanged lines hidden (view full) ---

3061 ret = -EPERM;
3062 goto out_unlock;
3063 }
3064
3065 ret = iommu_setup_default_domain(group, req_type);
3066 if (ret)
3067 goto out_unlock;
3068
1834 mutex_unlock(&group->mutex);
1835
1836 /*
1837 * FIXME: Mis-locked because the ops->probe_finalize() call-back
1838 * of some IOMMU drivers calls arm_iommu_attach_device() which
1839 * in-turn might call back into IOMMU core code, where it tries
1840 * to take group->mutex, resulting in a deadlock.
1841 */

--- 1222 unchanged lines hidden (view full) ---

3064 ret = -EPERM;
3065 goto out_unlock;
3066 }
3067
3068 ret = iommu_setup_default_domain(group, req_type);
3069 if (ret)
3070 goto out_unlock;
3071
3069 /*
3070 * Release the mutex here because ops->probe_finalize() call-back of
3071 * some vendor IOMMU drivers calls arm_iommu_attach_device() which
3072 * in-turn might call back into IOMMU core code, where it tries to take
3073 * group->mutex, resulting in a deadlock.
3074 */
3075 mutex_unlock(&group->mutex);
3076
3077 /* Make sure dma_ops is appropriatley set */
3078 for_each_group_device(group, gdev)
3072 /* Make sure dma_ops is appropriatley set */
3073 for_each_group_device(group, gdev)
3079 iommu_group_do_probe_finalize(gdev->dev);
3080 return count;
3074 iommu_setup_dma_ops(gdev->dev);
3081
3082out_unlock:
3083 mutex_unlock(&group->mutex);
3084 return ret ?: count;
3085}
3086
3087/**
3088 * iommu_device_use_default_domain() - Device driver wants to handle device

--- 223 unchanged lines hidden (view full) ---

3312
3313 return user;
3314}
3315EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
3316
3317static int __iommu_set_group_pasid(struct iommu_domain *domain,
3318 struct iommu_group *group, ioasid_t pasid)
3319{
3075
3076out_unlock:
3077 mutex_unlock(&group->mutex);
3078 return ret ?: count;
3079}
3080
3081/**
3082 * iommu_device_use_default_domain() - Device driver wants to handle device

--- 223 unchanged lines hidden (view full) ---

3306
3307 return user;
3308}
3309EXPORT_SYMBOL_GPL(iommu_group_dma_owner_claimed);
3310
3311static int __iommu_set_group_pasid(struct iommu_domain *domain,
3312 struct iommu_group *group, ioasid_t pasid)
3313{
3320 struct group_device *device;
3321 int ret = 0;
3314 struct group_device *device, *last_gdev;
3315 int ret;
3322
3323 for_each_group_device(group, device) {
3324 ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
3325 if (ret)
3316
3317 for_each_group_device(group, device) {
3318 ret = domain->ops->set_dev_pasid(domain, device->dev, pasid);
3319 if (ret)
3326 break;
3320 goto err_revert;
3327 }
3328
3321 }
3322
3323 return 0;
3324
3325err_revert:
3326 last_gdev = device;
3327 for_each_group_device(group, device) {
3328 const struct iommu_ops *ops = dev_iommu_ops(device->dev);
3329
3330 if (device == last_gdev)
3331 break;
3332 ops->remove_dev_pasid(device->dev, pasid, domain);
3333 }
3329 return ret;
3330}
3331
3332static void __iommu_remove_group_pasid(struct iommu_group *group,
3334 return ret;
3335}
3336
3337static void __iommu_remove_group_pasid(struct iommu_group *group,
3333 ioasid_t pasid)
3338 ioasid_t pasid,
3339 struct iommu_domain *domain)
3334{
3335 struct group_device *device;
3336 const struct iommu_ops *ops;
3337
3338 for_each_group_device(group, device) {
3339 ops = dev_iommu_ops(device->dev);
3340{
3341 struct group_device *device;
3342 const struct iommu_ops *ops;
3343
3344 for_each_group_device(group, device) {
3345 ops = dev_iommu_ops(device->dev);
3340 ops->remove_dev_pasid(device->dev, pasid);
3346 ops->remove_dev_pasid(device->dev, pasid, domain);
3341 }
3342}
3343
3344/*
3345 * iommu_attach_device_pasid() - Attach a domain to pasid of device
3346 * @domain: the iommu domain.
3347 * @dev: the attached device.
3348 * @pasid: the pasid of the device.
3347 }
3348}
3349
3350/*
3351 * iommu_attach_device_pasid() - Attach a domain to pasid of device
3352 * @domain: the iommu domain.
3353 * @dev: the attached device.
3354 * @pasid: the pasid of the device.
3355 * @handle: the attach handle.
3349 *
3350 * Return: 0 on success, or an error.
3351 */
3352int iommu_attach_device_pasid(struct iommu_domain *domain,
3356 *
3357 * Return: 0 on success, or an error.
3358 */
3359int iommu_attach_device_pasid(struct iommu_domain *domain,
3353 struct device *dev, ioasid_t pasid)
3360 struct device *dev, ioasid_t pasid,
3361 struct iommu_attach_handle *handle)
3354{
3355 /* Caller must be a probed driver on dev */
3356 struct iommu_group *group = dev->iommu_group;
3357 struct group_device *device;
3362{
3363 /* Caller must be a probed driver on dev */
3364 struct iommu_group *group = dev->iommu_group;
3365 struct group_device *device;
3358 void *curr;
3359 int ret;
3360
3361 if (!domain->ops->set_dev_pasid)
3362 return -EOPNOTSUPP;
3363
3364 if (!group)
3365 return -ENODEV;
3366

--- 4 unchanged lines hidden (view full) ---

3371 mutex_lock(&group->mutex);
3372 for_each_group_device(group, device) {
3373 if (pasid >= device->dev->iommu->max_pasids) {
3374 ret = -EINVAL;
3375 goto out_unlock;
3376 }
3377 }
3378
3366 int ret;
3367
3368 if (!domain->ops->set_dev_pasid)
3369 return -EOPNOTSUPP;
3370
3371 if (!group)
3372 return -ENODEV;
3373

--- 4 unchanged lines hidden (view full) ---

3378 mutex_lock(&group->mutex);
3379 for_each_group_device(group, device) {
3380 if (pasid >= device->dev->iommu->max_pasids) {
3381 ret = -EINVAL;
3382 goto out_unlock;
3383 }
3384 }
3385
3379 curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);
3380 if (curr) {
3381 ret = xa_err(curr) ? : -EBUSY;
3386 if (handle)
3387 handle->domain = domain;
3388
3389 ret = xa_insert(&group->pasid_array, pasid, handle, GFP_KERNEL);
3390 if (ret)
3382 goto out_unlock;
3391 goto out_unlock;
3383 }
3384
3385 ret = __iommu_set_group_pasid(domain, group, pasid);
3392
3393 ret = __iommu_set_group_pasid(domain, group, pasid);
3386 if (ret) {
3387 __iommu_remove_group_pasid(group, pasid);
3394 if (ret)
3388 xa_erase(&group->pasid_array, pasid);
3395 xa_erase(&group->pasid_array, pasid);
3389 }
3390out_unlock:
3391 mutex_unlock(&group->mutex);
3392 return ret;
3393}
3394EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
3395
3396/*
3397 * iommu_detach_device_pasid() - Detach the domain from pasid of device

--- 6 unchanged lines hidden (view full) ---

3404 */
3405void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
3406 ioasid_t pasid)
3407{
3408 /* Caller must be a probed driver on dev */
3409 struct iommu_group *group = dev->iommu_group;
3410
3411 mutex_lock(&group->mutex);
3396out_unlock:
3397 mutex_unlock(&group->mutex);
3398 return ret;
3399}
3400EXPORT_SYMBOL_GPL(iommu_attach_device_pasid);
3401
3402/*
3403 * iommu_detach_device_pasid() - Detach the domain from pasid of device

--- 6 unchanged lines hidden (view full) ---

3410 */
3411void iommu_detach_device_pasid(struct iommu_domain *domain, struct device *dev,
3412 ioasid_t pasid)
3413{
3414 /* Caller must be a probed driver on dev */
3415 struct iommu_group *group = dev->iommu_group;
3416
3417 mutex_lock(&group->mutex);
3412 __iommu_remove_group_pasid(group, pasid);
3413 WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);
3418 __iommu_remove_group_pasid(group, pasid, domain);
3419 xa_erase(&group->pasid_array, pasid);
3414 mutex_unlock(&group->mutex);
3415}
3416EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3417
3418/*
3419 * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
3420 * @dev: the queried device
3421 * @pasid: the pasid of the device

--- 8 unchanged lines hidden (view full) ---

3430 * Return: attached domain on success, NULL otherwise.
3431 */
3432struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
3433 ioasid_t pasid,
3434 unsigned int type)
3435{
3436 /* Caller must be a probed driver on dev */
3437 struct iommu_group *group = dev->iommu_group;
3420 mutex_unlock(&group->mutex);
3421}
3422EXPORT_SYMBOL_GPL(iommu_detach_device_pasid);
3423
3424/*
3425 * iommu_get_domain_for_dev_pasid() - Retrieve domain for @pasid of @dev
3426 * @dev: the queried device
3427 * @pasid: the pasid of the device

--- 8 unchanged lines hidden (view full) ---

3436 * Return: attached domain on success, NULL otherwise.
3437 */
3438struct iommu_domain *iommu_get_domain_for_dev_pasid(struct device *dev,
3439 ioasid_t pasid,
3440 unsigned int type)
3441{
3442 /* Caller must be a probed driver on dev */
3443 struct iommu_group *group = dev->iommu_group;
3438 struct iommu_domain *domain;
3444 struct iommu_attach_handle *handle;
3445 struct iommu_domain *domain = NULL;
3439
3440 if (!group)
3441 return NULL;
3442
3443 xa_lock(&group->pasid_array);
3446
3447 if (!group)
3448 return NULL;
3449
3450 xa_lock(&group->pasid_array);
3444 domain = xa_load(&group->pasid_array, pasid);
3451 handle = xa_load(&group->pasid_array, pasid);
3452 if (handle)
3453 domain = handle->domain;
3454
3445 if (type && domain && domain->type != type)
3455 if (type && domain && domain->type != type)
3446 domain = ERR_PTR(-EBUSY);
3456 domain = NULL;
3447 xa_unlock(&group->pasid_array);
3448
3449 return domain;
3450}
3451EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
3452
3453ioasid_t iommu_alloc_global_pasid(struct device *dev)
3454{

--- 24 unchanged lines hidden --- 		3457 xa_unlock(&group->pasid_array);
3458
3459 return domain;
3460}
3461EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev_pasid);
3462
3463ioasid_t iommu_alloc_global_pasid(struct device *dev)
3464{

--- 24 unchanged lines hidden ---