diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/amd/iommu_v2.c | 10 | ||||
| -rw-r--r-- | drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 29 | ||||
| -rw-r--r-- | drivers/iommu/intel/svm.c | 8 | ||||
| -rw-r--r-- | drivers/iommu/iommu-sva.c | 3 | ||||
| -rw-r--r-- | drivers/iommu/iommu.c | 27 | ||||
| -rw-r--r-- | drivers/iommu/iommufd/device.c | 12 | ||||
| -rw-r--r-- | drivers/iommu/iommufd/iommufd_private.h | 15 | ||||
| -rw-r--r-- | drivers/iommu/iommufd/main.c | 78 | ||||
| -rw-r--r-- | drivers/iommu/iommufd/pages.c | 2 |
9 files changed, 121 insertions, 63 deletions
diff --git a/drivers/iommu/amd/iommu_v2.c b/drivers/iommu/amd/iommu_v2.c index 261352a23271..2596466cd5a6 100644 --- a/drivers/iommu/amd/iommu_v2.c +++ b/drivers/iommu/amd/iommu_v2.c @@ -355,9 +355,9 @@ static struct pasid_state *mn_to_state(struct mmu_notifier *mn) return container_of(mn, struct pasid_state, mn); } -static void mn_invalidate_range(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, unsigned long end) +static void mn_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) { struct pasid_state *pasid_state; struct device_state *dev_state; @@ -391,8 +391,8 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) } static const struct mmu_notifier_ops iommu_mn = { - .release = mn_release, - .invalidate_range = mn_invalidate_range, + .release = mn_release, + .arch_invalidate_secondary_tlbs = mn_arch_invalidate_secondary_tlbs, }; static void set_pri_tag_status(struct pasid_state *pasid_state, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index a5a63b1c947e..dbc812a0e57e 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -186,9 +186,10 @@ static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) } } -static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, unsigned long end) +static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, + unsigned long end) { struct arm_smmu_mmu_notifier *smmu_mn = mn_to_smmu(mn); struct arm_smmu_domain *smmu_domain = smmu_mn->domain; @@ -200,10 +201,20 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn, * range. So do a simple translation here by calculating size correctly. */ size = end - start; + if (size == ULONG_MAX) + size = 0; + + if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) { + if (!size) + arm_smmu_tlb_inv_asid(smmu_domain->smmu, + smmu_mn->cd->asid); + else + arm_smmu_tlb_inv_range_asid(start, size, + smmu_mn->cd->asid, + PAGE_SIZE, false, + smmu_domain); + } - if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM)) - arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid, - PAGE_SIZE, false, smmu_domain); arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size); } @@ -237,9 +248,9 @@ static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) } static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { - .invalidate_range = arm_smmu_mm_invalidate_range, - .release = arm_smmu_mm_release, - .free_notifier = arm_smmu_mmu_notifier_free, + .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs, + .release = arm_smmu_mm_release, + .free_notifier = arm_smmu_mmu_notifier_free, }; /* Allocate or get existing MMU notifier for this {domain, mm} pair */ diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e95b339e9cdc..8f6d68006ab6 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -219,9 +219,9 @@ static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, } /* Pages have been freed at this point */ -static void intel_invalidate_range(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, unsigned long end) +static void intel_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) { struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); @@ -256,7 +256,7 @@ static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) static const struct mmu_notifier_ops intel_mmuops = { .release = intel_mm_release, - .invalidate_range = intel_invalidate_range, + .arch_invalidate_secondary_tlbs = intel_arch_invalidate_secondary_tlbs, }; static DEFINE_MUTEX(pasid_mutex); diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index 3ebd4b6586b3..05c0fb2acbc4 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -34,8 +34,9 @@ static int iommu_sva_alloc_pasid(struct mm_struct *mm, ioasid_t min, ioasid_t ma } ret = ida_alloc_range(&iommu_global_pasid_ida, min, max, GFP_KERNEL); - if (ret < min) + if (ret < 0) goto out; + mm->pasid = ret; ret = 0; out: diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index da340f11c5f5..caaf563d38ae 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2891,14 +2891,11 @@ static int iommu_setup_default_domain(struct iommu_group *group, ret = __iommu_group_set_domain_internal( group, dom, IOMMU_SET_DOMAIN_MUST_SUCCEED); if (WARN_ON(ret)) - goto out_free; + goto out_free_old; } else { ret = __iommu_group_set_domain(group, dom); - if (ret) { - iommu_domain_free(dom); - group->default_domain = old_dom; - return ret; - } + if (ret) + goto err_restore_def_domain; } /* @@ -2911,20 +2908,24 @@ static int iommu_setup_default_domain(struct iommu_group *group, for_each_group_device(group, gdev) { ret = iommu_create_device_direct_mappings(dom, gdev->dev); if (ret) - goto err_restore; + goto err_restore_domain; } } -err_restore: - if (old_dom) { +out_free_old: + if (old_dom) + iommu_domain_free(old_dom); + return ret; + +err_restore_domain: + if (old_dom) __iommu_group_set_domain_internal( group, old_dom, IOMMU_SET_DOMAIN_MUST_SUCCEED); +err_restore_def_domain: + if (old_dom) { iommu_domain_free(dom); - old_dom = NULL; + group->default_domain = old_dom; } -out_free: - if (old_dom) - iommu_domain_free(old_dom); return ret; } diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 29d05663d4d1..ed2937a4e196 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -109,10 +109,7 @@ EXPORT_SYMBOL_NS_GPL(iommufd_device_bind, IOMMUFD); */ void iommufd_device_unbind(struct iommufd_device *idev) { - bool was_destroyed; - - was_destroyed = iommufd_object_destroy_user(idev->ictx, &idev->obj); - WARN_ON(!was_destroyed); + iommufd_object_destroy_user(idev->ictx, &idev->obj); } EXPORT_SYMBOL_NS_GPL(iommufd_device_unbind, IOMMUFD); @@ -382,7 +379,7 @@ void iommufd_device_detach(struct iommufd_device *idev) mutex_unlock(&hwpt->devices_lock); if (hwpt->auto_domain) - iommufd_object_destroy_user(idev->ictx, &hwpt->obj); + iommufd_object_deref_user(idev->ictx, &hwpt->obj); else refcount_dec(&hwpt->obj.users); @@ -456,10 +453,7 @@ EXPORT_SYMBOL_NS_GPL(iommufd_access_create, IOMMUFD); */ void iommufd_access_destroy(struct iommufd_access *access) { - bool was_destroyed; - - was_destroyed = iommufd_object_destroy_user(access->ictx, &access->obj); - WARN_ON(!was_destroyed); + iommufd_object_destroy_user(access->ictx, &access->obj); } EXPORT_SYMBOL_NS_GPL(iommufd_access_destroy, IOMMUFD); diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index b38e67d1988b..f9790983699c 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -176,8 +176,19 @@ void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx, struct iommufd_object *obj); void iommufd_object_finalize(struct iommufd_ctx *ictx, struct iommufd_object *obj); -bool iommufd_object_destroy_user(struct iommufd_ctx *ictx, - struct iommufd_object *obj); +void __iommufd_object_destroy_user(struct iommufd_ctx *ictx, + struct iommufd_object *obj, bool allow_fail); +static inline void iommufd_object_destroy_user(struct iommufd_ctx *ictx, + struct iommufd_object *obj) +{ + __iommufd_object_destroy_user(ictx, obj, false); +} +static inline void iommufd_object_deref_user(struct iommufd_ctx *ictx, + struct iommufd_object *obj) +{ + __iommufd_object_destroy_user(ictx, obj, true); +} + struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, size_t size, enum iommufd_object_type type); diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c index 3fbe636c3d8a..4cf5f73f2708 100644 --- a/drivers/iommu/iommufd/main.c +++ b/drivers/iommu/iommufd/main.c @@ -117,13 +117,55 @@ struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, } /* + * Remove the given object id from the xarray if the only reference to the + * object is held by the xarray. The caller must call ops destroy(). + */ +static struct iommufd_object *iommufd_object_remove(struct iommufd_ctx *ictx, + u32 id, bool extra_put) +{ + struct iommufd_object *obj; + XA_STATE(xas, &ictx->objects, id); + + xa_lock(&ictx->objects); + obj = xas_load(&xas); + if (xa_is_zero(obj) || !obj) { + obj = ERR_PTR(-ENOENT); + goto out_xa; + } + + /* + * If the caller is holding a ref on obj we put it here under the + * spinlock. + */ + if (extra_put) + refcount_dec(&obj->users); + + if (!refcount_dec_if_one(&obj->users)) { + obj = ERR_PTR(-EBUSY); + goto out_xa; + } + + xas_store(&xas, NULL); + if (ictx->vfio_ioas == container_of(obj, struct iommufd_ioas, obj)) + ictx->vfio_ioas = NULL; + +out_xa: + xa_unlock(&ictx->objects); + + /* The returned object reference count is zero */ + return obj; +} + +/* * The caller holds a users refcount and wants to destroy the object. Returns * true if the object was destroyed. In all cases the caller no longer has a * reference on obj. */ -bool iommufd_object_destroy_user(struct iommufd_ctx *ictx, - struct iommufd_object *obj) +void __iommufd_object_destroy_user(struct iommufd_ctx *ictx, + struct iommufd_object *obj, bool allow_fail) { + struct iommufd_object *ret; + /* * The purpose of the destroy_rwsem is to ensure deterministic * destruction of objects used by external drivers and destroyed by this @@ -131,22 +173,22 @@ bool iommufd_object_destroy_user(struct iommufd_ctx *ictx, * side of this, such as during ioctl execution. */ down_write(&obj->destroy_rwsem); - xa_lock(&ictx->objects); - refcount_dec(&obj->users); - if (!refcount_dec_if_one(&obj->users)) { - xa_unlock(&ictx->objects); - up_write(&obj->destroy_rwsem); - return false; - } - __xa_erase(&ictx->objects, obj->id); - if (ictx->vfio_ioas && &ictx->vfio_ioas->obj == obj) - ictx->vfio_ioas = NULL; - xa_unlock(&ictx->objects); + ret = iommufd_object_remove(ictx, obj->id, true); up_write(&obj->destroy_rwsem); + if (allow_fail && IS_ERR(ret)) + return; + + /* + * If there is a bug and we couldn't destroy the object then we did put + * back the caller's refcount and will eventually try to free it again + * during close. + */ + if (WARN_ON(IS_ERR(ret))) + return; + iommufd_object_ops[obj->type].destroy(obj); kfree(obj); - return true; } static int iommufd_destroy(struct iommufd_ucmd *ucmd) @@ -154,13 +196,11 @@ static int iommufd_destroy(struct iommufd_ucmd *ucmd) struct iommu_destroy *cmd = ucmd->cmd; struct iommufd_object *obj; - obj = iommufd_get_object(ucmd->ictx, cmd->id, IOMMUFD_OBJ_ANY); + obj = iommufd_object_remove(ucmd->ictx, cmd->id, false); if (IS_ERR(obj)) return PTR_ERR(obj); - iommufd_ref_to_users(obj); - /* See iommufd_ref_to_users() */ - if (!iommufd_object_destroy_user(ucmd->ictx, obj)) - return -EBUSY; + iommufd_object_ops[obj->type].destroy(obj); + kfree(obj); return 0; } diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 412ca96be128..8d9aa297c117 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -297,7 +297,7 @@ static void batch_clear_carry(struct pfn_batch *batch, unsigned int keep_pfns) batch->pfns[0] = batch->pfns[batch->end - 1] + (batch->npfns[batch->end - 1] - keep_pfns); batch->npfns[0] = keep_pfns; - batch->end = 0; + batch->end = 1; } static void batch_skip_carry(struct pfn_batch *batch, unsigned int skip_pfns) |