diff options
| author | Ankit Soni <Ankit.Soni@amd.com> | 2026-01-22 18:30:38 +0300 |
|---|---|---|
| committer | Joerg Roedel <joerg.roedel@amd.com> | 2026-02-03 16:27:05 +0300 |
| commit | 9e249c48412828e807afddc21527eb734dc9bd3d (patch) | |
| tree | 8db2544375a42daf8a53a5cc1f3109aca5c18016 | |
| parent | 5b0530bb16ec5af3cc96fc25891d6a860fd37a8c (diff) | |
| download | linux-9e249c48412828e807afddc21527eb734dc9bd3d.tar.xz | |
iommu/amd: serialize sequence allocation under concurrent TLB invalidations
With concurrent TLB invalidations, completion wait randomly gets timed out
because cmd_sem_val was incremented outside the IOMMU spinlock, allowing
CMD_COMPL_WAIT commands to be queued out of sequence and breaking the
ordering assumption in wait_on_sem().
Move the cmd_sem_val increment under iommu->lock so completion sequence
allocation is serialized with command queuing.
And remove the unnecessary return.
Fixes: d2a0cac10597 ("iommu/amd: move wait_on_sem() out of spinlock")
Tested-by: Srikanth Aithal <sraithal@amd.com>
Reported-by: Srikanth Aithal <sraithal@amd.com>
Signed-off-by: Ankit Soni <Ankit.Soni@amd.com>
Reviewed-by: Vasant Hegde <vasant.hegde@amd.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>
| -rw-r--r-- | drivers/iommu/amd/amd_iommu_types.h | 2 | ||||
| -rw-r--r-- | drivers/iommu/amd/init.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/amd/iommu.c | 18 |
3 files changed, 14 insertions, 8 deletions
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index cfcbad6c28ff..c685d3771436 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -752,7 +752,7 @@ struct amd_iommu { u32 flags; volatile u64 *cmd_sem; - atomic64_t cmd_sem_val; + u64 cmd_sem_val; /* * Track physical address to directly use it in build_completion_wait() * and avoid adding any special checks and handling for kdump. diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index b1c344ed7dbd..02c0c64c5f6b 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1885,7 +1885,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, iommu->pci_seg = pci_seg; raw_spin_lock_init(&iommu->lock); - atomic64_set(&iommu->cmd_sem_val, 0); + iommu->cmd_sem_val = 0; /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 285ae635c324..58be841d624e 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1439,6 +1439,12 @@ static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) return iommu_queue_command_sync(iommu, cmd, true); } +static u64 get_cmdsem_val(struct amd_iommu *iommu) +{ + lockdep_assert_held(&iommu->lock); + return ++iommu->cmd_sem_val; +} + /* * This function queues a completion wait command into the command * buffer of an IOMMU @@ -1453,11 +1459,11 @@ static int iommu_completion_wait(struct amd_iommu *iommu) if (!iommu->need_sync) return 0; - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd, iommu, data); - raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, false); raw_spin_unlock_irqrestore(&iommu->lock, flags); @@ -3177,10 +3183,11 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) return; build_inv_irt(&cmd, devid); - data = atomic64_inc_return(&iommu->cmd_sem_val); - build_completion_wait(&cmd2, iommu, data); raw_spin_lock_irqsave(&iommu->lock, flags); + data = get_cmdsem_val(iommu); + build_completion_wait(&cmd2, iommu, data); + ret = __iommu_queue_command_sync(iommu, &cmd, true); if (ret) goto out_err; @@ -3194,7 +3201,6 @@ static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) out_err: raw_spin_unlock_irqrestore(&iommu->lock, flags); - return; } static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data) |