diff options
| -rw-r--r-- | Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt | 5 | ||||
| -rw-r--r-- | drivers/iommu/Kconfig | 2 | ||||
| -rw-r--r-- | drivers/iommu/exynos-iommu.c | 187 |
3 files changed, 143 insertions, 51 deletions
diff --git a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt index f61ca25ca136..85f068805dd8 100644 --- a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt +++ b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt @@ -35,9 +35,10 @@ Required properties: - interrupts: An interrupt specifier for interrupt signal of System MMU, according to the format defined by a particular interrupt controller. -- clock-names: Should be "sysmmu" if the System MMU is needed to gate its clock. +- clock-names: Should be "sysmmu" or a pair of "aclk" and "pclk" to gate + SYSMMU core clocks. Optional "master" if the clock to the System MMU is gated by - another gate clock other than "sysmmu" (usually main gate clock + another gate clock other core (usually main gate clock of peripheral device this SYSMMU belongs to). - clocks: Phandles for respective clocks described by clock-names. - power-domains: Required if the System MMU is needed to gate its power. diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a1e75cba18e0..1674de1cfed0 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -243,7 +243,7 @@ config TEGRA_IOMMU_SMMU config EXYNOS_IOMMU bool "Exynos IOMMU Support" - depends on ARCH_EXYNOS && ARM && MMU + depends on ARCH_EXYNOS && MMU select IOMMU_API select ARM_DMA_USE_IOMMU help diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 8e289e2a05fb..9c8ce951158d 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1,6 +1,5 @@ -/* linux/drivers/iommu/exynos_iommu.c - * - * Copyright (c) 2011 Samsung Electronics Co., Ltd. +/* + * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. * http://www.samsung.com * * This program is free software; you can redistribute it and/or modify @@ -55,17 +54,25 @@ typedef u32 sysmmu_pte_t; #define lv2ent_small(pent) ((*(pent) & 2) == 2) #define lv2ent_large(pent) ((*(pent) & 3) == 1) -static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size) -{ - return iova & (size - 1); -} - -#define section_phys(sent) (*(sent) & SECT_MASK) -#define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE) -#define lpage_phys(pent) (*(pent) & LPAGE_MASK) -#define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE) -#define spage_phys(pent) (*(pent) & SPAGE_MASK) -#define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE) +/* + * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces + * v5.0 introduced support for 36bit physical address space by shifting + * all page entry values by 4 bits. + * All SYSMMU controllers in the system support the address spaces of the same + * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper + * value (0 or 4). + */ +static short PG_ENT_SHIFT = -1; +#define SYSMMU_PG_ENT_SHIFT 0 +#define SYSMMU_V5_PG_ENT_SHIFT 4 + +#define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT) +#define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK) +#define section_offs(iova) (iova & (SECT_SIZE - 1)) +#define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK) +#define lpage_offs(iova) (iova & (LPAGE_SIZE - 1)) +#define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK) +#define spage_offs(iova) (iova & (SPAGE_SIZE - 1)) #define NUM_LV1ENTRIES 4096 #define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE) @@ -84,13 +91,12 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) +#define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0)) -#define lv2table_base(sent) (*(sent) & 0xFFFFFC00) - -#define mk_lv1ent_sect(pa) ((pa) | 2) -#define mk_lv1ent_page(pa) ((pa) | 1) -#define mk_lv2ent_lpage(pa) ((pa) | 1) -#define mk_lv2ent_spage(pa) ((pa) | 2) +#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2) +#define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1) +#define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1) +#define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2) #define CTRL_ENABLE 0x5 #define CTRL_BLOCK 0x7 @@ -98,14 +104,23 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define CFG_LRU 0x1 #define CFG_QOS(n) ((n & 0xF) << 7) -#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */ #define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ #define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */ +/* common registers */ #define REG_MMU_CTRL 0x000 #define REG_MMU_CFG 0x004 #define REG_MMU_STATUS 0x008 +#define REG_MMU_VERSION 0x034 + +#define MMU_MAJ_VER(val) ((val) >> 7) +#define MMU_MIN_VER(val) ((val) & 0x7F) +#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ + +#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) + +/* v1.x - v3.x registers */ #define REG_MMU_FLUSH 0x00C #define REG_MMU_FLUSH_ENTRY 0x010 #define REG_PT_BASE_ADDR 0x014 @@ -117,18 +132,14 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define REG_AR_FAULT_ADDR 0x02C #define REG_DEFAULT_SLAVE_ADDR 0x030 -#define REG_MMU_VERSION 0x034 - -#define MMU_MAJ_VER(val) ((val) >> 7) -#define MMU_MIN_VER(val) ((val) & 0x7F) -#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ - -#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) - -#define REG_PB0_SADDR 0x04C -#define REG_PB0_EADDR 0x050 -#define REG_PB1_SADDR 0x054 -#define REG_PB1_EADDR 0x058 +/* v5.x registers */ +#define REG_V5_PT_BASE_PFN 0x00C +#define REG_V5_MMU_FLUSH_ALL 0x010 +#define REG_V5_MMU_FLUSH_ENTRY 0x014 +#define REG_V5_INT_STATUS 0x060 +#define REG_V5_INT_CLEAR 0x064 +#define REG_V5_FAULT_AR_VA 0x070 +#define REG_V5_FAULT_AW_VA 0x080 #define has_sysmmu(dev) (dev->archdata.iommu != NULL) @@ -169,6 +180,19 @@ static const struct sysmmu_fault_info sysmmu_faults[] = { { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, }; +static const struct sysmmu_fault_info sysmmu_v5_faults[] = { + { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, + { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, + { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, + { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, + { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, + { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, + { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, + { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, + { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, + { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, +}; + /* * This structure is attached to dev.archdata.iommu of the master device * on device add, contains a list of SYSMMU controllers defined by device tree, @@ -205,6 +229,8 @@ struct sysmmu_drvdata { struct device *master; /* master device (owner) */ void __iomem *sfrbase; /* our registers */ struct clk *clk; /* SYSMMU's clock */ + struct clk *aclk; /* SYSMMU's aclk clock */ + struct clk *pclk; /* SYSMMU's pclk clock */ struct clk *clk_master; /* master's device clock */ int activations; /* number of calls to sysmmu_enable */ spinlock_t lock; /* lock for modyfying state */ @@ -262,7 +288,10 @@ static bool sysmmu_block(struct sysmmu_drvdata *data) static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) { - __raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH); + if (MMU_MAJ_VER(data->version) < 5) + __raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH); + else + __raw_writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL); } static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, @@ -271,15 +300,23 @@ static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, unsigned int i; for (i = 0; i < num_inv; i++) { - __raw_writel((iova & SPAGE_MASK) | 1, - data->sfrbase + REG_MMU_FLUSH_ENTRY); + if (MMU_MAJ_VER(data->version) < 5) + __raw_writel((iova & SPAGE_MASK) | 1, + data->sfrbase + REG_MMU_FLUSH_ENTRY); + else + __raw_writel((iova & SPAGE_MASK) | 1, + data->sfrbase + REG_V5_MMU_FLUSH_ENTRY); iova += SPAGE_SIZE; } } static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) { - __raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); + if (MMU_MAJ_VER(data->version) < 5) + __raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); + else + __raw_writel(pgd >> PAGE_SHIFT, + data->sfrbase + REG_V5_PT_BASE_PFN); __sysmmu_tlb_invalidate(data); } @@ -290,6 +327,8 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) clk_enable(data->clk_master); clk_enable(data->clk); + clk_enable(data->pclk); + clk_enable(data->aclk); ver = __raw_readl(data->sfrbase + REG_MMU_VERSION); @@ -302,6 +341,8 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) dev_dbg(data->sysmmu, "hardware version: %d.%d\n", MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); + clk_disable(data->aclk); + clk_disable(data->pclk); clk_disable(data->clk); clk_disable(data->clk_master); } @@ -326,19 +367,31 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { /* SYSMMU is in blocked state when interrupt occurred. */ struct sysmmu_drvdata *data = dev_id; - const struct sysmmu_fault_info *finfo = sysmmu_faults; - int i, n = ARRAY_SIZE(sysmmu_faults); - unsigned int itype; + const struct sysmmu_fault_info *finfo; + unsigned int i, n, itype; sysmmu_iova_t fault_addr = -1; + unsigned short reg_status, reg_clear; int ret = -ENOSYS; WARN_ON(!is_sysmmu_active(data)); + if (MMU_MAJ_VER(data->version) < 5) { + reg_status = REG_INT_STATUS; + reg_clear = REG_INT_CLEAR; + finfo = sysmmu_faults; + n = ARRAY_SIZE(sysmmu_faults); + } else { + reg_status = REG_V5_INT_STATUS; + reg_clear = REG_V5_INT_CLEAR; + finfo = sysmmu_v5_faults; + n = ARRAY_SIZE(sysmmu_v5_faults); + } + spin_lock(&data->lock); clk_enable(data->clk_master); - itype = __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); + itype = __ffs(__raw_readl(data->sfrbase + reg_status)); for (i = 0; i < n; i++, finfo++) if (finfo->bit == itype) break; @@ -355,7 +408,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) /* fault is not recovered by fault handler */ BUG_ON(ret != 0); - __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); + __raw_writel(1 << itype, data->sfrbase + reg_clear); sysmmu_unblock(data); @@ -373,6 +426,8 @@ static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); __raw_writel(0, data->sfrbase + REG_MMU_CFG); + clk_disable(data->aclk); + clk_disable(data->pclk); clk_disable(data->clk); clk_disable(data->clk_master); } @@ -421,6 +476,8 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) { clk_enable(data->clk_master); clk_enable(data->clk); + clk_enable(data->pclk); + clk_enable(data->aclk); __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); @@ -544,22 +601,47 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) } data->clk = devm_clk_get(dev, "sysmmu"); - if (IS_ERR(data->clk)) { - dev_err(dev, "Failed to get clock!\n"); - return PTR_ERR(data->clk); - } else { + if (!IS_ERR(data->clk)) { ret = clk_prepare(data->clk); if (ret) { dev_err(dev, "Failed to prepare clk\n"); return ret; } + } else { + data->clk = NULL; + } + + data->aclk = devm_clk_get(dev, "aclk"); + if (!IS_ERR(data->aclk)) { + ret = clk_prepare(data->aclk); + if (ret) { + dev_err(dev, "Failed to prepare aclk\n"); + return ret; + } + } else { + data->aclk = NULL; + } + + data->pclk = devm_clk_get(dev, "pclk"); + if (!IS_ERR(data->pclk)) { + ret = clk_prepare(data->pclk); + if (ret) { + dev_err(dev, "Failed to prepare pclk\n"); + return ret; + } + } else { + data->pclk = NULL; + } + + if (!data->clk && (!data->aclk || !data->pclk)) { + dev_err(dev, "Failed to get device clock(s)!\n"); + return -ENOSYS; } data->clk_master = devm_clk_get(dev, "master"); if (!IS_ERR(data->clk_master)) { ret = clk_prepare(data->clk_master); if (ret) { - clk_unprepare(data->clk); dev_err(dev, "Failed to prepare master's clk\n"); return ret; } @@ -573,6 +655,13 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); __sysmmu_get_version(data); + if (PG_ENT_SHIFT < 0) { + if (MMU_MAJ_VER(data->version) < 5) + PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT; + else + PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT; + } + pm_runtime_enable(dev); return 0; @@ -637,6 +726,8 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) dma_addr_t handle; int i; + /* Check if correct PTE offsets are initialized */ + BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev); domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) @@ -816,7 +907,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, bool need_flush_flpd_cache = lv1ent_zero(sent); pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); - BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1)); + BUG_ON((phys_addr_t)pent & (LV2TABLE_SIZE - 1)); if (!pent) return ERR_PTR(-ENOMEM); |