From 4a20ce0ff68eb6fc4b1e8f25139c93b312f21229 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 4 May 2023 22:10:55 +0100 Subject: iommu: Add a capability for flush queue support Passing a special type to domain_alloc to indirectly query whether flush queues are a worthwhile optimisation with the given driver is a bit clunky, and looking increasingly anachronistic. Let's put that into an explicit capability instead. Signed-off-by: Robin Murphy Reviewed-by: Lu Baolu Tested-by: Jerry Snitselaar # amd, intel, smmu-v3 Reviewed-by: Jerry Snitselaar Reviewed-by: Jason Gunthorpe Link: https://lore.kernel.org/r/f0086a93dbccb92622e1ace775846d81c1c4b174.1683233867.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/iommu/amd/iommu.c') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4a314647d1f7..9b7bd6bed664 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -2293,6 +2293,8 @@ static bool amd_iommu_capable(struct device *dev, enum iommu_cap cap) return amdr_ivrs_remap_support; case IOMMU_CAP_ENFORCE_CACHE_COHERENCY: return true; + case IOMMU_CAP_DEFERRED_FLUSH: + return true; default: break; } -- cgit v1.2.3 From a42f0c7a4118ffa395866cc7f5522d71a86fc4dd Mon Sep 17 00:00:00 2001 From: Joao Martins Date: Tue, 30 May 2023 10:11:33 -0400 Subject: iommu/amd: Switch amd_iommu_update_ga() to use modify_irte_ga() The modify_irte_ga() uses cmpxchg_double() to update the IRTE in one shot, which is necessary when adding IRTE cache disabling support since the driver no longer need to flush the IRT for hardware to take effect. Please note that there is a functional change where the IsRun and Destination bits of IRTE are now cached in the struct amd_ir_data.entry. Reviewed-by: Jerry Snitselaar Signed-off-by: Joao Martins Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20230530141137.14376-2-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) (limited to 'drivers/iommu/amd/iommu.c') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 4a314647d1f7..12d6844128f9 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3702,44 +3702,26 @@ int amd_iommu_create_irq_domain(struct amd_iommu *iommu) int amd_iommu_update_ga(int cpu, bool is_run, void *data) { - unsigned long flags; - struct amd_iommu *iommu; - struct irq_remap_table *table; struct amd_ir_data *ir_data = (struct amd_ir_data *)data; - int devid = ir_data->irq_2_irte.devid; struct irte_ga *entry = (struct irte_ga *) ir_data->entry; - struct irte_ga *ref = (struct irte_ga *) ir_data->ref; if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) || - !ref || !entry || !entry->lo.fields_vapic.guest_mode) + !entry || !entry->lo.fields_vapic.guest_mode) return 0; - iommu = ir_data->iommu; - if (!iommu) - return -ENODEV; - - table = get_irq_table(iommu, devid); - if (!table) + if (!ir_data->iommu) return -ENODEV; - raw_spin_lock_irqsave(&table->lock, flags); - - if (ref->lo.fields_vapic.guest_mode) { - if (cpu >= 0) { - ref->lo.fields_vapic.destination = - APICID_TO_IRTE_DEST_LO(cpu); - ref->hi.fields.destination = - APICID_TO_IRTE_DEST_HI(cpu); - } - ref->lo.fields_vapic.is_run = is_run; - barrier(); + if (cpu >= 0) { + entry->lo.fields_vapic.destination = + APICID_TO_IRTE_DEST_LO(cpu); + entry->hi.fields.destination = + APICID_TO_IRTE_DEST_HI(cpu); } + entry->lo.fields_vapic.is_run = is_run; - raw_spin_unlock_irqrestore(&table->lock, flags); - - iommu_flush_irt(iommu, devid); - iommu_completion_wait(iommu); - return 0; + return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, + ir_data->irq_2_irte.index, entry, ir_data); } EXPORT_SYMBOL(amd_iommu_update_ga); #endif -- cgit v1.2.3 From 74a37817bd1567330fb372eb01223e31b45b1cc0 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 30 May 2023 10:11:34 -0400 Subject: iommu/amd: Remove the unused struct amd_ir_data.ref Since the amd_iommu_update_ga() has been switched to use the modify_irte_ga() helper function to update the IRTE, the parameter is no longer needed. Reviewed-by: Jerry Snitselaar Suggested-by: Vasant Hegde Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20230530141137.14376-3-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 1 - drivers/iommu/amd/iommu.c | 17 +++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) (limited to 'drivers/iommu/amd/iommu.c') diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 2ddbda3a4374..8c072be68875 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -1001,7 +1001,6 @@ struct amd_ir_data { struct irq_2_irte irq_2_irte; struct msi_msg msi_entry; void *entry; /* Pointer to union irte or struct irte_ga */ - void *ref; /* Pointer to the actual irte */ /** * Store information for activate/de-activate diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 12d6844128f9..e6daf0f39fd8 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -3001,7 +3001,7 @@ out: } static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, - struct irte_ga *irte, struct amd_ir_data *data) + struct irte_ga *irte) { bool ret; struct irq_remap_table *table; @@ -3028,9 +3028,6 @@ static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, */ WARN_ON(!ret); - if (data) - data->ref = entry; - raw_spin_unlock_irqrestore(&table->lock, flags); iommu_flush_irt(iommu, devid); @@ -3119,7 +3116,7 @@ static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u1 struct irte_ga *irte = (struct irte_ga *) entry; irte->lo.fields_remap.valid = 1; - modify_irte_ga(iommu, devid, index, irte, NULL); + modify_irte_ga(iommu, devid, index, irte); } static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index) @@ -3135,7 +3132,7 @@ static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, struct irte_ga *irte = (struct irte_ga *) entry; irte->lo.fields_remap.valid = 0; - modify_irte_ga(iommu, devid, index, irte, NULL); + modify_irte_ga(iommu, devid, index, irte); } static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index, @@ -3159,7 +3156,7 @@ static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid APICID_TO_IRTE_DEST_LO(dest_apicid); irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid); - modify_irte_ga(iommu, devid, index, irte, NULL); + modify_irte_ga(iommu, devid, index, irte); } } @@ -3510,7 +3507,7 @@ int amd_iommu_activate_guest_mode(void *data) entry->lo.fields_vapic.ga_tag = ir_data->ga_tag; return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, - ir_data->irq_2_irte.index, entry, ir_data); + ir_data->irq_2_irte.index, entry); } EXPORT_SYMBOL(amd_iommu_activate_guest_mode); @@ -3540,7 +3537,7 @@ int amd_iommu_deactivate_guest_mode(void *data) APICID_TO_IRTE_DEST_HI(cfg->dest_apicid); return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, - ir_data->irq_2_irte.index, entry, ir_data); + ir_data->irq_2_irte.index, entry); } EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode); @@ -3721,7 +3718,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) entry->lo.fields_vapic.is_run = is_run; return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, - ir_data->irq_2_irte.index, entry, ir_data); + ir_data->irq_2_irte.index, entry); } EXPORT_SYMBOL(amd_iommu_update_ga); #endif -- cgit v1.2.3 From 98aeb4ea5599c5f7fbb1645bdd2050d0be96dfa3 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 30 May 2023 10:11:36 -0400 Subject: iommu/amd: Do not Invalidate IRT when IRTE caching is disabled With the Interrupt Remapping Table cache disabled, there is no need to issue invalidate IRT and wait for its completion. Therefore, add logic to bypass the operation. Reviewed-by: Jerry Snitselaar Suggested-by: Joao Martins Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20230530141137.14376-5-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'drivers/iommu/amd/iommu.c') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e6daf0f39fd8..8fdd6ebf8711 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1266,12 +1266,24 @@ static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) u32 devid; u16 last_bdf = iommu->pci_seg->last_bdf; + if (iommu->irtcachedis_enabled) + return; + for (devid = 0; devid <= last_bdf; devid++) iommu_flush_irt(iommu, devid); iommu_completion_wait(iommu); } +static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) +{ + if (iommu->irtcachedis_enabled) + return; + + iommu_flush_irt(iommu, devid); + iommu_completion_wait(iommu); +} + void iommu_flush_all_caches(struct amd_iommu *iommu) { if (iommu_feature(iommu, FEATURE_IA)) { @@ -3030,8 +3042,7 @@ static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index, raw_spin_unlock_irqrestore(&table->lock, flags); - iommu_flush_irt(iommu, devid); - iommu_completion_wait(iommu); + iommu_flush_irt_and_complete(iommu, devid); return 0; } @@ -3050,8 +3061,7 @@ static int modify_irte(struct amd_iommu *iommu, table->table[index] = irte->val; raw_spin_unlock_irqrestore(&table->lock, flags); - iommu_flush_irt(iommu, devid); - iommu_completion_wait(iommu); + iommu_flush_irt_and_complete(iommu, devid); return 0; } @@ -3069,8 +3079,7 @@ static void free_irte(struct amd_iommu *iommu, u16 devid, int index) iommu->irte_ops->clear_allocated(table, index); raw_spin_unlock_irqrestore(&table->lock, flags); - iommu_flush_irt(iommu, devid); - iommu_completion_wait(iommu); + iommu_flush_irt_and_complete(iommu, devid); } static void irte_prepare(void *entry, -- cgit v1.2.3 From bccc37a8a2fb002a302a526656c56a793a708670 Mon Sep 17 00:00:00 2001 From: Suravee Suthikulpanit Date: Tue, 30 May 2023 10:11:37 -0400 Subject: iommu/amd: Improving Interrupt Remapping Table Invalidation Invalidating Interrupt Remapping Table (IRT) requires, the AMD IOMMU driver to issue INVALIDATE_INTERRUPT_TABLE and COMPLETION_WAIT commands. Currently, the driver issues the two commands separately, which requires calling raw_spin_lock_irqsave() twice. In addition, the COMPLETION_WAIT could potentially be interleaved with other commands causing delay of the COMPLETION_WAIT command. Therefore, combine issuing of the two commands in one spin-lock, and changing struct amd_iommu.cmd_sem_val to use atomic64 to minimize locking. Reviewed-by: Jerry Snitselaar Signed-off-by: Suravee Suthikulpanit Link: https://lore.kernel.org/r/20230530141137.14376-6-suravee.suthikulpanit@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/init.c | 2 +- drivers/iommu/amd/iommu.c | 27 ++++++++++++++++++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) (limited to 'drivers/iommu/amd/iommu.c') diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 8eeea1f25e69..5a4e04404cfd 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -752,7 +752,7 @@ struct amd_iommu { u32 flags; volatile u64 *cmd_sem; - u64 cmd_sem_val; + atomic64_t cmd_sem_val; #ifdef CONFIG_AMD_IOMMU_DEBUGFS /* DebugFS Info */ diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 418da641ee3d..4c68dc382286 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -1733,7 +1733,7 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, iommu->pci_seg = pci_seg; raw_spin_lock_init(&iommu->lock); - iommu->cmd_sem_val = 0; + atomic64_set(&iommu->cmd_sem_val, 0); /* Add IOMMU to internal data structures */ list_add_tail(&iommu->list, &amd_iommu_list); diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 8fdd6ebf8711..e47c8c520708 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1175,11 +1175,11 @@ static int iommu_completion_wait(struct amd_iommu *iommu) if (!iommu->need_sync) return 0; - raw_spin_lock_irqsave(&iommu->lock, flags); - - data = ++iommu->cmd_sem_val; + data = atomic64_add_return(1, &iommu->cmd_sem_val); build_completion_wait(&cmd, iommu, data); + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __iommu_queue_command_sync(iommu, &cmd, false); if (ret) goto out_unlock; @@ -1277,11 +1277,28 @@ static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) { + int ret; + u64 data; + unsigned long flags; + struct iommu_cmd cmd, cmd2; + if (iommu->irtcachedis_enabled) return; - iommu_flush_irt(iommu, devid); - iommu_completion_wait(iommu); + build_inv_irt(&cmd, devid); + data = atomic64_add_return(1, &iommu->cmd_sem_val); + build_completion_wait(&cmd2, iommu, data); + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __iommu_queue_command_sync(iommu, &cmd, true); + if (ret) + goto out; + ret = __iommu_queue_command_sync(iommu, &cmd2, false); + if (ret) + goto out; + wait_on_sem(iommu, data); +out: + raw_spin_unlock_irqrestore(&iommu->lock, flags); } void iommu_flush_all_caches(struct amd_iommu *iommu) -- cgit v1.2.3 From 1ce018df87640adb139c8418785ad3b6e4376bd3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 9 Jun 2023 15:15:08 +0200 Subject: iommu/amd: Fix compile error for unused function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recent changes introduced a compile error: drivers/iommu/amd/iommu.c:1285:13: error: ‘iommu_flush_irt_and_complete’ defined but not used [-Werror=unused-function] 1285 | static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~ This happens with defconfig-x86_64 because AMD IOMMU is enabled but CONFIG_IRQ_REMAP is disabled. Move the function under #ifdef CONFIG_IRQ_REMAP to fix the error. Signed-off-by: Joerg Roedel --- drivers/iommu/amd/iommu.c | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) (limited to 'drivers/iommu/amd/iommu.c') diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index e47c8c520708..13097619fc4c 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1275,32 +1275,6 @@ static void amd_iommu_flush_irt_all(struct amd_iommu *iommu) iommu_completion_wait(iommu); } -static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) -{ - int ret; - u64 data; - unsigned long flags; - struct iommu_cmd cmd, cmd2; - - if (iommu->irtcachedis_enabled) - return; - - build_inv_irt(&cmd, devid); - data = atomic64_add_return(1, &iommu->cmd_sem_val); - build_completion_wait(&cmd2, iommu, data); - - raw_spin_lock_irqsave(&iommu->lock, flags); - ret = __iommu_queue_command_sync(iommu, &cmd, true); - if (ret) - goto out; - ret = __iommu_queue_command_sync(iommu, &cmd2, false); - if (ret) - goto out; - wait_on_sem(iommu, data); -out: - raw_spin_unlock_irqrestore(&iommu->lock, flags); -} - void iommu_flush_all_caches(struct amd_iommu *iommu) { if (iommu_feature(iommu, FEATURE_IA)) { @@ -2831,6 +2805,32 @@ EXPORT_SYMBOL(amd_iommu_device_info); static struct irq_chip amd_ir_chip; static DEFINE_SPINLOCK(iommu_table_lock); +static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid) +{ + int ret; + u64 data; + unsigned long flags; + struct iommu_cmd cmd, cmd2; + + if (iommu->irtcachedis_enabled) + return; + + build_inv_irt(&cmd, devid); + data = atomic64_add_return(1, &iommu->cmd_sem_val); + build_completion_wait(&cmd2, iommu, data); + + raw_spin_lock_irqsave(&iommu->lock, flags); + ret = __iommu_queue_command_sync(iommu, &cmd, true); + if (ret) + goto out; + ret = __iommu_queue_command_sync(iommu, &cmd2, false); + if (ret) + goto out; + wait_on_sem(iommu, data); +out: + raw_spin_unlock_irqrestore(&iommu->lock, flags); +} + static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid, struct irq_remap_table *table) { -- cgit v1.2.3