From 5b438f4ba315db4f8c1489d175656798d58c014f Mon Sep 17 00:00:00 2001 From: Jacob Pan Date: Fri, 11 Jan 2019 13:04:57 +0800 Subject: iommu/vt-d: Support page request in scalable mode VT-d Rev3.0 has made a few changes to the page request interface, 1. widened PRQ descriptor from 128 bits to 256 bits; 2. removed streaming response type; 3. introduced private data that requires page response even the request is not last request in group (LPIG). This is a supplement to commit 1c4f88b7f1f92 ("iommu/vt-d: Shared virtual address in scalable mode") and makes the svm code compliant with VT-d Rev3.0. Cc: Ashok Raj Cc: Liu Yi L Cc: Kevin Tian Signed-off-by: Jacob Pan Fixes: 1c4f88b7f1f92 ("iommu/vt-d: Shared virtual address in scalable mode") Signed-off-by: Lu Baolu Signed-off-by: Joerg Roedel --- include/linux/intel-iommu.h | 21 +++++++++------------ include/linux/intel-svm.h | 2 +- 2 files changed, 10 insertions(+), 13 deletions(-) (limited to 'include/linux') diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 0605f3bf6e79..fa364de9db18 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -374,20 +374,17 @@ enum { #define QI_DEV_EIOTLB_PFSID(pfsid) (((u64)(pfsid & 0xf) << 12) | ((u64)(pfsid & 0xfff) << 52)) #define QI_DEV_EIOTLB_MAX_INVS 32 -#define QI_PGRP_IDX(idx) (((u64)(idx)) << 55) -#define QI_PGRP_PRIV(priv) (((u64)(priv)) << 32) -#define QI_PGRP_RESP_CODE(res) ((u64)(res)) -#define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) -#define QI_PGRP_DID(did) (((u64)(did)) << 16) +/* Page group response descriptor QW0 */ #define QI_PGRP_PASID_P(p) (((u64)(p)) << 4) +#define QI_PGRP_PDP(p) (((u64)(p)) << 5) +#define QI_PGRP_RESP_CODE(res) (((u64)(res)) << 12) +#define QI_PGRP_DID(rid) (((u64)(rid)) << 16) +#define QI_PGRP_PASID(pasid) (((u64)(pasid)) << 32) + +/* Page group response descriptor QW1 */ +#define QI_PGRP_LPIG(x) (((u64)(x)) << 2) +#define QI_PGRP_IDX(idx) (((u64)(idx)) << 3) -#define QI_PSTRM_ADDR(addr) (((u64)(addr)) & VTD_PAGE_MASK) -#define QI_PSTRM_DEVFN(devfn) (((u64)(devfn)) << 4) -#define QI_PSTRM_RESP_CODE(res) ((u64)(res)) -#define QI_PSTRM_IDX(idx) (((u64)(idx)) << 55) -#define QI_PSTRM_PRIV(priv) (((u64)(priv)) << 32) -#define QI_PSTRM_BUS(bus) (((u64)(bus)) << 24) -#define QI_PSTRM_PASID(pasid) (((u64)(pasid)) << 4) #define QI_RESP_SUCCESS 0x0 #define QI_RESP_INVALID 0x1 diff --git a/include/linux/intel-svm.h b/include/linux/intel-svm.h index 99bc5b3ae26e..e3f76315ca4d 100644 --- a/include/linux/intel-svm.h +++ b/include/linux/intel-svm.h @@ -20,7 +20,7 @@ struct device; struct svm_dev_ops { void (*fault_cb)(struct device *dev, int pasid, u64 address, - u32 private, int rwxp, int response); + void *private, int rwxp, int response); }; /* Values for rxwp in fault_cb callback */ -- cgit v1.2.3 From 1d7ae53b152dbc5ba0a4f6a83ecc42ac66f52d11 Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Wed, 12 Dec 2018 23:38:47 +0300 Subject: iommu: Introduce iotlb_sync_map callback Introduce iotlb_sync_map() callback that is invoked in the end of iommu_map(). This new callback allows IOMMU drivers to avoid syncing after mapping of each contiguous chunk and sync only when the whole mapping is completed, optimizing performance of the mapping operation. Signed-off-by: Dmitry Osipenko Reviewed-by: Robin Murphy Reviewed-by: Thierry Reding Signed-off-by: Joerg Roedel --- drivers/iommu/iommu.c | 8 ++++++-- include/linux/iommu.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3ed4db334341..ed0e63f2cd9b 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1585,13 +1585,14 @@ static size_t iommu_pgsize(struct iommu_domain *domain, int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { + const struct iommu_ops *ops = domain->ops; unsigned long orig_iova = iova; unsigned int min_pagesz; size_t orig_size = size; phys_addr_t orig_paddr = paddr; int ret = 0; - if (unlikely(domain->ops->map == NULL || + if (unlikely(ops->map == NULL || domain->pgsize_bitmap == 0UL)) return -ENODEV; @@ -1620,7 +1621,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n", iova, &paddr, pgsize); - ret = domain->ops->map(domain, iova, paddr, pgsize, prot); + ret = ops->map(domain, iova, paddr, pgsize, prot); if (ret) break; @@ -1629,6 +1630,9 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, size -= pgsize; } + if (ops->iotlb_sync_map) + ops->iotlb_sync_map(domain); + /* unroll mapping in case something went wrong */ if (ret) iommu_unmap(domain, orig_iova, orig_size - size); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e90da6b6f3d1..477ef47c357c 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -201,6 +201,7 @@ struct iommu_ops { void (*flush_iotlb_all)(struct iommu_domain *domain); void (*iotlb_range_add)(struct iommu_domain *domain, unsigned long iova, size_t size); + void (*iotlb_sync_map)(struct iommu_domain *domain); void (*iotlb_sync)(struct iommu_domain *domain); phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova); int (*add_device)(struct device *dev); -- cgit v1.2.3 From b77cf11f094136a9d7d0ee6a56cf49db1f412871 Mon Sep 17 00:00:00 2001 From: Rob Herring Date: Tue, 5 Feb 2019 10:37:31 -0600 Subject: iommu: Allow io-pgtable to be used outside of drivers/iommu/ Move io-pgtable.h to include/linux/ and export alloc_io_pgtable_ops and free_io_pgtable_ops. This enables drivers outside drivers/iommu/ to use the page table library. Specifically, some ARM Mali GPUs use the ARM page table formats. Cc: Will Deacon Cc: Robin Murphy Cc: Joerg Roedel Cc: Matthias Brugger Cc: Rob Clark Cc: linux-arm-kernel@lists.infradead.org Cc: iommu@lists.linux-foundation.org Cc: linux-mediatek@lists.infradead.org Cc: linux-arm-msm@vger.kernel.org Signed-off-by: Rob Herring Signed-off-by: Joerg Roedel --- drivers/iommu/arm-smmu-v3.c | 3 +- drivers/iommu/arm-smmu.c | 2 +- drivers/iommu/io-pgtable-arm-v7s.c | 3 +- drivers/iommu/io-pgtable-arm.c | 3 +- drivers/iommu/io-pgtable.c | 5 +- drivers/iommu/io-pgtable.h | 213 ------------------------------------- drivers/iommu/ipmmu-vmsa.c | 3 +- drivers/iommu/msm_iommu.c | 2 +- drivers/iommu/mtk_iommu.h | 3 +- drivers/iommu/qcom_iommu.c | 2 +- include/linux/io-pgtable.h | 213 +++++++++++++++++++++++++++++++++++++ 11 files changed, 224 insertions(+), 228 deletions(-) delete mode 100644 drivers/iommu/io-pgtable.h create mode 100644 include/linux/io-pgtable.h (limited to 'include/linux') diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 0d284029dc73..d3880010c6cf 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -32,8 +33,6 @@ #include -#include "io-pgtable.h" - /* MMIO registers */ #define ARM_SMMU_IDR0 0x0 #define IDR0_ST_LVL GENMASK(28, 27) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index af18a7e7f917..045d93884164 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -56,7 +57,6 @@ #include #include -#include "io-pgtable.h" #include "arm-smmu-regs.h" #define ARM_MMU500_ACTLR_CPRE (1 << 1) diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index cec29bf45c9b..75a8273d1ae9 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -45,8 +46,6 @@ #include -#include "io-pgtable.h" - /* Struct accessors */ #define io_pgtable_to_data(x) \ container_of((x), struct arm_v7s_io_pgtable, iop) diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 237cacd4a62b..d3700ec15cbd 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -31,8 +32,6 @@ #include -#include "io-pgtable.h" - #define ARM_LPAE_MAX_ADDR_BITS 52 #define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 #define ARM_LPAE_MAX_LEVELS 4 diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 127558d83667..93f2880be6c6 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -19,11 +19,10 @@ */ #include +#include #include #include -#include "io-pgtable.h" - static const struct io_pgtable_init_fns * io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { #ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE @@ -61,6 +60,7 @@ struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, return &iop->ops; } +EXPORT_SYMBOL_GPL(alloc_io_pgtable_ops); /* * It is the IOMMU driver's responsibility to ensure that the page table @@ -77,3 +77,4 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) io_pgtable_tlb_flush_all(iop); io_pgtable_init_table[iop->fmt]->free(iop); } +EXPORT_SYMBOL_GPL(free_io_pgtable_ops); diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h deleted file mode 100644 index 47d5ae559329..000000000000 --- a/drivers/iommu/io-pgtable.h +++ /dev/null @@ -1,213 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __IO_PGTABLE_H -#define __IO_PGTABLE_H -#include - -/* - * Public API for use by IOMMU drivers - */ -enum io_pgtable_fmt { - ARM_32_LPAE_S1, - ARM_32_LPAE_S2, - ARM_64_LPAE_S1, - ARM_64_LPAE_S2, - ARM_V7S, - IO_PGTABLE_NUM_FMTS, -}; - -/** - * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management. - * - * @tlb_flush_all: Synchronously invalidate the entire TLB context. - * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. - * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and - * any corresponding page table updates are visible to the - * IOMMU. - * - * Note that these can all be called in atomic context and must therefore - * not block. - */ -struct iommu_gather_ops { - void (*tlb_flush_all)(void *cookie); - void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule, - bool leaf, void *cookie); - void (*tlb_sync)(void *cookie); -}; - -/** - * struct io_pgtable_cfg - Configuration data for a set of page tables. - * - * @quirks: A bitmap of hardware quirks that require some special - * action by the low-level page table allocator. - * @pgsize_bitmap: A bitmap of page sizes supported by this set of page - * tables. - * @ias: Input address (iova) size, in bits. - * @oas: Output address (paddr) size, in bits. - * @tlb: TLB management callbacks for this set of tables. - * @iommu_dev: The device representing the DMA configuration for the - * page table walker. - */ -struct io_pgtable_cfg { - /* - * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in - * stage 1 PTEs, for hardware which insists on validating them - * even in non-secure state where they should normally be ignored. - * - * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and - * IOMMU_NOEXEC flags and map everything with full access, for - * hardware which does not implement the permissions of a given - * format, and/or requires some format-specific default value. - * - * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid - * (unmapped) entries but the hardware might do so anyway, perform - * TLB maintenance when mapping as well as when unmapping. - * - * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all - * PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit - * when the SoC is in "4GB mode" and they can only access the high - * remap of DRAM (0x1_00000000 to 0x1_ffffffff). - * - * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever - * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a - * software-emulated IOMMU), such that pagetable updates need not - * be treated as explicit DMA data. - * - * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs - * on unmap, for DMA domains using the flush queue mechanism for - * delayed invalidation. - */ - #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) - #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) - #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) - #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) - #define IO_PGTABLE_QUIRK_NO_DMA BIT(4) - #define IO_PGTABLE_QUIRK_NON_STRICT BIT(5) - unsigned long quirks; - unsigned long pgsize_bitmap; - unsigned int ias; - unsigned int oas; - const struct iommu_gather_ops *tlb; - struct device *iommu_dev; - - /* Low-level data specific to the table format */ - union { - struct { - u64 ttbr[2]; - u64 tcr; - u64 mair[2]; - } arm_lpae_s1_cfg; - - struct { - u64 vttbr; - u64 vtcr; - } arm_lpae_s2_cfg; - - struct { - u32 ttbr[2]; - u32 tcr; - u32 nmrr; - u32 prrr; - } arm_v7s_cfg; - }; -}; - -/** - * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. - * - * @map: Map a physically contiguous memory region. - * @unmap: Unmap a physically contiguous memory region. - * @iova_to_phys: Translate iova to physical address. - * - * These functions map directly onto the iommu_ops member functions with - * the same names. - */ -struct io_pgtable_ops { - int (*map)(struct io_pgtable_ops *ops, unsigned long iova, - phys_addr_t paddr, size_t size, int prot); - size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, - size_t size); - phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, - unsigned long iova); -}; - -/** - * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. - * - * @fmt: The page table format. - * @cfg: The page table configuration. This will be modified to represent - * the configuration actually provided by the allocator (e.g. the - * pgsize_bitmap may be restricted). - * @cookie: An opaque token provided by the IOMMU driver and passed back to - * the callback routines in cfg->tlb. - */ -struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, - struct io_pgtable_cfg *cfg, - void *cookie); - -/** - * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller - * *must* ensure that the page table is no longer - * live, but the TLB can be dirty. - * - * @ops: The ops returned from alloc_io_pgtable_ops. - */ -void free_io_pgtable_ops(struct io_pgtable_ops *ops); - - -/* - * Internal structures for page table allocator implementations. - */ - -/** - * struct io_pgtable - Internal structure describing a set of page tables. - * - * @fmt: The page table format. - * @cookie: An opaque token provided by the IOMMU driver and passed back to - * any callback routines. - * @cfg: A copy of the page table configuration. - * @ops: The page table operations in use for this set of page tables. - */ -struct io_pgtable { - enum io_pgtable_fmt fmt; - void *cookie; - struct io_pgtable_cfg cfg; - struct io_pgtable_ops ops; -}; - -#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) - -static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) -{ - iop->cfg.tlb->tlb_flush_all(iop->cookie); -} - -static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, - unsigned long iova, size_t size, size_t granule, bool leaf) -{ - iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); -} - -static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) -{ - iop->cfg.tlb->tlb_sync(iop->cookie); -} - -/** - * struct io_pgtable_init_fns - Alloc/free a set of page tables for a - * particular format. - * - * @alloc: Allocate a set of page tables described by cfg. - * @free: Free the page tables associated with iop. - */ -struct io_pgtable_init_fns { - struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); - void (*free)(struct io_pgtable *iop); -}; - -extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; - -#endif /* __IO_PGTABLE_H */ diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 7a4529c61c19..9a380c10655e 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -35,8 +36,6 @@ #define arm_iommu_detach_device(...) do {} while (0) #endif -#include "io-pgtable.h" - #define IPMMU_CTX_MAX 8 struct ipmmu_features { diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index fc4270733f11..ef7d1f995d6b 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -37,7 +38,6 @@ #include "msm_iommu_hw-8xxx.h" #include "msm_iommu.h" -#include "io-pgtable.h" #define MRC(reg, processor, op1, crn, crm, op2) \ __asm__ __volatile__ ( \ diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h index 778498b8633f..62c2c3e8c5df 100644 --- a/drivers/iommu/mtk_iommu.h +++ b/drivers/iommu/mtk_iommu.h @@ -19,13 +19,12 @@ #include #include #include +#include #include #include #include #include -#include "io-pgtable.h" - struct mtk_iommu_suspend_reg { u32 standard_axi_mode; u32 dcm_dis; diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c index d8595f0a987d..8cdd3f059513 100644 --- a/drivers/iommu/qcom_iommu.c +++ b/drivers/iommu/qcom_iommu.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,6 @@ #include #include -#include "io-pgtable.h" #include "arm-smmu-regs.h" #define SMMU_INTR_SEL_NS 0x2000 diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h new file mode 100644 index 000000000000..47d5ae559329 --- /dev/null +++ b/include/linux/io-pgtable.h @@ -0,0 +1,213 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __IO_PGTABLE_H +#define __IO_PGTABLE_H +#include + +/* + * Public API for use by IOMMU drivers + */ +enum io_pgtable_fmt { + ARM_32_LPAE_S1, + ARM_32_LPAE_S2, + ARM_64_LPAE_S1, + ARM_64_LPAE_S2, + ARM_V7S, + IO_PGTABLE_NUM_FMTS, +}; + +/** + * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management. + * + * @tlb_flush_all: Synchronously invalidate the entire TLB context. + * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. + * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and + * any corresponding page table updates are visible to the + * IOMMU. + * + * Note that these can all be called in atomic context and must therefore + * not block. + */ +struct iommu_gather_ops { + void (*tlb_flush_all)(void *cookie); + void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule, + bool leaf, void *cookie); + void (*tlb_sync)(void *cookie); +}; + +/** + * struct io_pgtable_cfg - Configuration data for a set of page tables. + * + * @quirks: A bitmap of hardware quirks that require some special + * action by the low-level page table allocator. + * @pgsize_bitmap: A bitmap of page sizes supported by this set of page + * tables. + * @ias: Input address (iova) size, in bits. + * @oas: Output address (paddr) size, in bits. + * @tlb: TLB management callbacks for this set of tables. + * @iommu_dev: The device representing the DMA configuration for the + * page table walker. + */ +struct io_pgtable_cfg { + /* + * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in + * stage 1 PTEs, for hardware which insists on validating them + * even in non-secure state where they should normally be ignored. + * + * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and + * IOMMU_NOEXEC flags and map everything with full access, for + * hardware which does not implement the permissions of a given + * format, and/or requires some format-specific default value. + * + * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid + * (unmapped) entries but the hardware might do so anyway, perform + * TLB maintenance when mapping as well as when unmapping. + * + * IO_PGTABLE_QUIRK_ARM_MTK_4GB: (ARM v7s format) Set bit 9 in all + * PTEs, for Mediatek IOMMUs which treat it as a 33rd address bit + * when the SoC is in "4GB mode" and they can only access the high + * remap of DRAM (0x1_00000000 to 0x1_ffffffff). + * + * IO_PGTABLE_QUIRK_NO_DMA: Guarantees that the tables will only ever + * be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a + * software-emulated IOMMU), such that pagetable updates need not + * be treated as explicit DMA data. + * + * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs + * on unmap, for DMA domains using the flush queue mechanism for + * delayed invalidation. + */ + #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) + #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) + #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) + #define IO_PGTABLE_QUIRK_ARM_MTK_4GB BIT(3) + #define IO_PGTABLE_QUIRK_NO_DMA BIT(4) + #define IO_PGTABLE_QUIRK_NON_STRICT BIT(5) + unsigned long quirks; + unsigned long pgsize_bitmap; + unsigned int ias; + unsigned int oas; + const struct iommu_gather_ops *tlb; + struct device *iommu_dev; + + /* Low-level data specific to the table format */ + union { + struct { + u64 ttbr[2]; + u64 tcr; + u64 mair[2]; + } arm_lpae_s1_cfg; + + struct { + u64 vttbr; + u64 vtcr; + } arm_lpae_s2_cfg; + + struct { + u32 ttbr[2]; + u32 tcr; + u32 nmrr; + u32 prrr; + } arm_v7s_cfg; + }; +}; + +/** + * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. + * + * @map: Map a physically contiguous memory region. + * @unmap: Unmap a physically contiguous memory region. + * @iova_to_phys: Translate iova to physical address. + * + * These functions map directly onto the iommu_ops member functions with + * the same names. + */ +struct io_pgtable_ops { + int (*map)(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot); + size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, + size_t size); + phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, + unsigned long iova); +}; + +/** + * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. + * + * @fmt: The page table format. + * @cfg: The page table configuration. This will be modified to represent + * the configuration actually provided by the allocator (e.g. the + * pgsize_bitmap may be restricted). + * @cookie: An opaque token provided by the IOMMU driver and passed back to + * the callback routines in cfg->tlb. + */ +struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + struct io_pgtable_cfg *cfg, + void *cookie); + +/** + * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller + * *must* ensure that the page table is no longer + * live, but the TLB can be dirty. + * + * @ops: The ops returned from alloc_io_pgtable_ops. + */ +void free_io_pgtable_ops(struct io_pgtable_ops *ops); + + +/* + * Internal structures for page table allocator implementations. + */ + +/** + * struct io_pgtable - Internal structure describing a set of page tables. + * + * @fmt: The page table format. + * @cookie: An opaque token provided by the IOMMU driver and passed back to + * any callback routines. + * @cfg: A copy of the page table configuration. + * @ops: The page table operations in use for this set of page tables. + */ +struct io_pgtable { + enum io_pgtable_fmt fmt; + void *cookie; + struct io_pgtable_cfg cfg; + struct io_pgtable_ops ops; +}; + +#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) + +static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) +{ + iop->cfg.tlb->tlb_flush_all(iop->cookie); +} + +static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, + unsigned long iova, size_t size, size_t granule, bool leaf) +{ + iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); +} + +static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) +{ + iop->cfg.tlb->tlb_sync(iop->cookie); +} + +/** + * struct io_pgtable_init_fns - Alloc/free a set of page tables for a + * particular format. + * + * @alloc: Allocate a set of page tables described by cfg. + * @free: Free the page tables associated with iop. + */ +struct io_pgtable_init_fns { + struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); + void (*free)(struct io_pgtable *iop); +}; + +extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; + +#endif /* __IO_PGTABLE_H */ -- cgit v1.2.3 From db04d4a3d72f0c5ee34609559f535d11ab47303c Mon Sep 17 00:00:00 2001 From: Tom Murphy Date: Mon, 11 Feb 2019 15:50:33 +0000 Subject: iommu: Fix flush_tlb_all typo Fix typo, flush_tlb_all should be flush_iotlb_all. Signed-off-by: Tom Murphy Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index e90da6b6f3d1..2b402dcbcf81 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -167,7 +167,7 @@ struct iommu_resv_region { * @detach_dev: detach device from an iommu domain * @map: map a physically contiguous memory region to an iommu domain * @unmap: unmap a physically contiguous memory region from an iommu domain - * @flush_tlb_all: Synchronously flush all hardware TLBs for this domain + * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain * @iotlb_range_add: Add a given iova range to the flush queue for this domain * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush * queue -- cgit v1.2.3 From e5567f5f67621877726f99be040af9fbedda37dc Mon Sep 17 00:00:00 2001 From: Kuppuswamy Sathyanarayanan Date: Tue, 19 Feb 2019 11:04:51 -0800 Subject: PCI/ATS: Add pci_prg_resp_pasid_required() interface. Return the PRG Response PASID Required bit in the Page Request Status Register. As per PCIe spec r4.0, sec 10.5.2.3, if this bit is Set, the device expects a PASID TLP Prefix on PRG Response Messages when the corresponding Page Requests had a PASID TLP Prefix. If Clear, the device does not expect PASID TLP Prefixes on any PRG Response Message, and the device behavior is undefined if the device receives a PRG Response Message with a PASID TLP Prefix. Also the device behavior is undefined if this bit is Set and the device receives a PRG Response Message with no PASID TLP Prefix when the corresponding Page Requests had a PASID TLP Prefix. This function will be used by drivers like IOMMU, if it is required to check the status of the PRG Response PASID Required bit before enabling the PASID support of the device. Cc: Ashok Raj Cc: Jacob Pan Cc: Keith Busch Suggested-by: Ashok Raj Signed-off-by: Kuppuswamy Sathyanarayanan Acked-by: Bjorn Helgaas Signed-off-by: Joerg Roedel --- drivers/pci/ats.c | 30 ++++++++++++++++++++++++++++++ include/linux/pci-ats.h | 5 +++++ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 36 insertions(+) (limited to 'include/linux') diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index 5b78f3b1b918..420cd0a578d0 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -368,6 +368,36 @@ int pci_pasid_features(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(pci_pasid_features); +/** + * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit + * status. + * @pdev: PCI device structure + * + * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. + * + * Even though the PRG response PASID status is read from PRI Status + * Register, since this API will mainly be used by PASID users, this + * function is defined within #ifdef CONFIG_PCI_PASID instead of + * CONFIG_PCI_PRI. + */ +int pci_prg_resp_pasid_required(struct pci_dev *pdev) +{ + u16 status; + int pos; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); + if (!pos) + return 0; + + pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status); + + if (status & PCI_PRI_STATUS_PASID) + return 1; + + return 0; +} +EXPORT_SYMBOL_GPL(pci_prg_resp_pasid_required); + #define PASID_NUMBER_SHIFT 8 #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT) /** diff --git a/include/linux/pci-ats.h b/include/linux/pci-ats.h index 7c4b8e27268c..facfd6a18fe1 100644 --- a/include/linux/pci-ats.h +++ b/include/linux/pci-ats.h @@ -40,6 +40,7 @@ void pci_disable_pasid(struct pci_dev *pdev); void pci_restore_pasid_state(struct pci_dev *pdev); int pci_pasid_features(struct pci_dev *pdev); int pci_max_pasids(struct pci_dev *pdev); +int pci_prg_resp_pasid_required(struct pci_dev *pdev); #else /* CONFIG_PCI_PASID */ @@ -66,6 +67,10 @@ static inline int pci_max_pasids(struct pci_dev *pdev) return -EINVAL; } +static int pci_prg_resp_pasid_required(struct pci_dev *pdev) +{ + return 0; +} #endif /* CONFIG_PCI_PASID */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index e1e9888c85e6..898be572b010 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -880,6 +880,7 @@ #define PCI_PRI_STATUS_RF 0x001 /* Response Failure */ #define PCI_PRI_STATUS_UPRGI 0x002 /* Unexpected PRG index */ #define PCI_PRI_STATUS_STOPPED 0x100 /* PRI Stopped */ +#define PCI_PRI_STATUS_PASID 0x8000 /* PRG Response PASID Required */ #define PCI_PRI_MAX_REQ 0x08 /* PRI max reqs supported */ #define PCI_PRI_ALLOC_REQ 0x0c /* PRI max reqs allowed */ #define PCI_EXT_CAP_PRI_SIZEOF 16 -- cgit v1.2.3 From 8c938ddc6df3bbe72809db1be6c9f3af83f5d7a9 Mon Sep 17 00:00:00 2001 From: Kuppuswamy Sathyanarayanan Date: Tue, 19 Feb 2019 11:06:09 -0800 Subject: PCI/ATS: Add pci_ats_page_aligned() interface Return the Page Aligned Request bit in the ATS Capability Register. As per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit is set, it indicates the Untranslated Addresses generated by the device are always aligned to a 4096 byte boundary. An IOMMU that can only translate page-aligned addresses can only be used with devices that always produce aligned Untranslated Addresses. This interface will be used by drivers for such IOMMUs to determine whether devices can use the ATS service. Cc: Ashok Raj Cc: Jacob Pan Cc: Keith Busch Suggested-by: Ashok Raj Signed-off-by: Kuppuswamy Sathyanarayanan Acked-by: Bjorn Helgaas Signed-off-by: Joerg Roedel --- drivers/pci/ats.c | 27 +++++++++++++++++++++++++++ include/linux/pci.h | 2 ++ include/uapi/linux/pci_regs.h | 1 + 3 files changed, 30 insertions(+) (limited to 'include/linux') diff --git a/drivers/pci/ats.c b/drivers/pci/ats.c index 420cd0a578d0..97c08146534a 100644 --- a/drivers/pci/ats.c +++ b/drivers/pci/ats.c @@ -142,6 +142,33 @@ int pci_ats_queue_depth(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pci_ats_queue_depth); +/** + * pci_ats_page_aligned - Return Page Aligned Request bit status. + * @pdev: the PCI device + * + * Returns 1, if the Untranslated Addresses generated by the device + * are always aligned or 0 otherwise. + * + * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit + * is set, it indicates the Untranslated Addresses generated by the + * device are always aligned to a 4096 byte boundary. + */ +int pci_ats_page_aligned(struct pci_dev *pdev) +{ + u16 cap; + + if (!pdev->ats_cap) + return 0; + + pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); + + if (cap & PCI_ATS_CAP_PAGE_ALIGNED) + return 1; + + return 0; +} +EXPORT_SYMBOL_GPL(pci_ats_page_aligned); + #ifdef CONFIG_PCI_PRI /** * pci_enable_pri - Enable PRI capability diff --git a/include/linux/pci.h b/include/linux/pci.h index 65f1d8c2f082..9724a8c0496b 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1524,11 +1524,13 @@ void pci_ats_init(struct pci_dev *dev); int pci_enable_ats(struct pci_dev *dev, int ps); void pci_disable_ats(struct pci_dev *dev); int pci_ats_queue_depth(struct pci_dev *dev); +int pci_ats_page_aligned(struct pci_dev *dev); #else static inline void pci_ats_init(struct pci_dev *d) { } static inline int pci_enable_ats(struct pci_dev *d, int ps) { return -ENODEV; } static inline void pci_disable_ats(struct pci_dev *d) { } static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; } +static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; } #endif #ifdef CONFIG_PCIE_PTM diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 898be572b010..5c98133f2c94 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -866,6 +866,7 @@ #define PCI_ATS_CAP 0x04 /* ATS Capability Register */ #define PCI_ATS_CAP_QDEP(x) ((x) & 0x1f) /* Invalidate Queue Depth */ #define PCI_ATS_MAX_QDEP 32 /* Max Invalidate Queue Depth */ +#define PCI_ATS_CAP_PAGE_ALIGNED 0x0020 /* Page Aligned Request */ #define PCI_ATS_CTRL 0x06 /* ATS Control Register */ #define PCI_ATS_CTRL_ENABLE 0x8000 /* ATS Enable */ #define PCI_ATS_CTRL_STU(x) ((x) & 0x1f) /* Smallest Translation Unit */ -- cgit v1.2.3 From 2405bc162583e1d7c40b13bf078e87428d2dfe4e Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 20 Feb 2019 14:00:52 +0100 Subject: iommu: Document iommu_ops.iotlb_sync_map() Add missing kerneldoc for iommu_ops.iotlb_sync_map(). Fixes: 1d7ae53b152dbc5b ("iommu: Introduce iotlb_sync_map callback") Signed-off-by: Geert Uytterhoeven Reviewed-by: Dmitry Osipenko Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 2b402dcbcf81..28ad97801032 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -169,6 +169,7 @@ struct iommu_resv_region { * @unmap: unmap a physically contiguous memory region from an iommu domain * @flush_iotlb_all: Synchronously flush all hardware TLBs for this domain * @iotlb_range_add: Add a given iova range to the flush queue for this domain + * @iotlb_sync_map: Sync mappings created recently using @map to the hardware * @iotlb_sync: Flush all queued ranges from the hardware TLBs and empty flush * queue * @iova_to_phys: translate iova to physical address -- cgit v1.2.3 From a7055d572c51338bed8673331ead6759cae6b70b Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Wed, 20 Feb 2019 14:00:53 +0100 Subject: iommu: Document iommu_ops.is_attach_deferred() Add missing kerneldoc for iommu_ops.is_attach_deferred(). Fixes: e01d1913b0d08171 ("iommu: Add is_attach_deferred call-back to iommu-ops") Signed-off-by: Geert Uytterhoeven Signed-off-by: Joerg Roedel --- include/linux/iommu.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/linux') diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 28ad97801032..41fa7958592d 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -184,6 +184,8 @@ struct iommu_resv_region { * @domain_window_enable: Configure and enable a particular window for a domain * @domain_window_disable: Disable a particular window for a domain * @of_xlate: add OF master IDs to iommu grouping + * @is_attach_deferred: Check if domain attach should be deferred from iommu + * driver init to device driver init (default no) * @pgsize_bitmap: bitmap of all possible supported page sizes */ struct iommu_ops { -- cgit v1.2.3 From fff42928ade591969836ff49888d063b829ac888 Mon Sep 17 00:00:00 2001 From: Kuppuswamy Sathyanarayanan Date: Wed, 27 Feb 2019 11:26:46 -0800 Subject: PCI/ATS: Add inline to pci_prg_resp_pasid_required() Fix unused function warning when compiled with CONFIG_PCI_PASID disabled. Fixes: e5567f5f6762 ("PCI/ATS: Add pci_prg_resp_pasid_required() interface.") Signed-off-by: Kuppuswamy Sathyanarayanan Signed-off-by: Joerg Roedel --- include/linux/pci-ats.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux') diff --git a/include/linux/pci-ats.h b/include/linux/pci-ats.h index facfd6a18fe1..1ebb88e7c184 100644 --- a/include/linux/pci-ats.h +++ b/include/linux/pci-ats.h @@ -67,7 +67,7 @@ static inline int pci_max_pasids(struct pci_dev *pdev) return -EINVAL; } -static int pci_prg_resp_pasid_required(struct pci_dev *pdev) +static inline int pci_prg_resp_pasid_required(struct pci_dev *pdev) { return 0; } -- cgit v1.2.3