diff options
Diffstat (limited to 'arch/arm/mm')
32 files changed, 525 insertions, 286 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 60cdfdc151aa..fd9077a74fce 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 comment "Processor Type" # Select CPU types depending on the architecture selected. This selects diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index 950d19babb5f..01bcc33f59e3 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # # Makefile for the linux arm-specific parts of the memory manager. # @@ -9,6 +10,7 @@ obj-$(CONFIG_MMU) += fault-armv.o flush.o idmap.o ioremap.o \ ifneq ($(CONFIG_MMU),y) obj-y += nommu.o +obj-$(CONFIG_ARM_MPU) += pmsa-v7.o endif obj-$(CONFIG_ARM_PTDUMP) += dump.o diff --git a/arch/arm/mm/abort-ev4.S b/arch/arm/mm/abort-ev4.S index b3b31e30cadd..a10bcb89594d 100644 --- a/arch/arm/mm/abort-ev4.S +++ b/arch/arm/mm/abort-ev4.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> /* diff --git a/arch/arm/mm/abort-ev4t.S b/arch/arm/mm/abort-ev4t.S index 9da704e7b86e..14743a2f6997 100644 --- a/arch/arm/mm/abort-ev4t.S +++ b/arch/arm/mm/abort-ev4t.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> #include "abort-macro.S" diff --git a/arch/arm/mm/abort-ev5t.S b/arch/arm/mm/abort-ev5t.S index a6a381a6caa5..98c523118820 100644 --- a/arch/arm/mm/abort-ev5t.S +++ b/arch/arm/mm/abort-ev5t.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> #include "abort-macro.S" diff --git a/arch/arm/mm/abort-ev5tj.S b/arch/arm/mm/abort-ev5tj.S index 00ab011bef58..fec72f4fbaf5 100644 --- a/arch/arm/mm/abort-ev5tj.S +++ b/arch/arm/mm/abort-ev5tj.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> #include "abort-macro.S" diff --git a/arch/arm/mm/abort-ev6.S b/arch/arm/mm/abort-ev6.S index 8801a15aa105..c58bf8b43fea 100644 --- a/arch/arm/mm/abort-ev6.S +++ b/arch/arm/mm/abort-ev6.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> #include "abort-macro.S" diff --git a/arch/arm/mm/abort-ev7.S b/arch/arm/mm/abort-ev7.S index e8d0e08c227f..f7cc5d68444b 100644 --- a/arch/arm/mm/abort-ev7.S +++ b/arch/arm/mm/abort-ev7.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> /* diff --git a/arch/arm/mm/abort-lv4t.S b/arch/arm/mm/abort-lv4t.S index 4cdfab31a0b6..fbd60a120f66 100644 --- a/arch/arm/mm/abort-lv4t.S +++ b/arch/arm/mm/abort-lv4t.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> /* diff --git a/arch/arm/mm/abort-macro.S b/arch/arm/mm/abort-macro.S index 4509bee4e081..bacf53fd0b70 100644 --- a/arch/arm/mm/abort-macro.S +++ b/arch/arm/mm/abort-macro.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * The ARM LDRD and Thumb LDRSB instructions use bit 20/11 (ARM/Thumb) * differently than every other instruction, so it is set to 0 (write) diff --git a/arch/arm/mm/abort-nommu.S b/arch/arm/mm/abort-nommu.S index 119cb479c2ab..6e2366a26321 100644 --- a/arch/arm/mm/abort-nommu.S +++ b/arch/arm/mm/abort-nommu.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> /* diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index fcf1473d6fed..ada8eb206a90 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -382,9 +382,9 @@ static void __dma_free_remap(void *cpu_addr, size_t size) } #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K -static struct gen_pool *atomic_pool; +static struct gen_pool *atomic_pool __ro_after_init; -static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE; +static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; static int __init early_coherent_pool(char *p) { @@ -393,21 +393,6 @@ static int __init early_coherent_pool(char *p) } early_param("coherent_pool", early_coherent_pool); -void __init init_dma_coherent_pool_size(unsigned long size) -{ - /* - * Catch any attempt to set the pool size too late. - */ - BUG_ON(atomic_pool); - - /* - * Set architecture specific coherent pool size only if - * it has not been changed by kernel command line parameter. - */ - if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE) - atomic_pool_size = size; -} - /* * Initialise the coherent pool for atomic allocations. */ @@ -443,7 +428,7 @@ static int __init atomic_pool_init(void) gen_pool_set_algo(atomic_pool, gen_pool_first_fit_order_align, - (void *)PAGE_SHIFT); + NULL); pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n", atomic_pool_size / 1024); return 0; diff --git a/arch/arm/mm/dma.h b/arch/arm/mm/dma.h index 70ea6852f94e..aaef64b7f177 100644 --- a/arch/arm/mm/dma.h +++ b/arch/arm/mm/dma.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef DMA_H #define DMA_H diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c index 35ff45470dbf..fc3b44028cfb 100644 --- a/arch/arm/mm/dump.c +++ b/arch/arm/mm/dump.c @@ -129,8 +129,8 @@ static const struct prot_bits section_bits[] = { .val = PMD_SECT_USER, .set = "USR", }, { - .mask = L_PMD_SECT_RDONLY, - .val = L_PMD_SECT_RDONLY, + .mask = L_PMD_SECT_RDONLY | PMD_SECT_AP2, + .val = L_PMD_SECT_RDONLY | PMD_SECT_AP2, .set = "ro", .clear = "RW", #elif __LINUX_ARM_ARCH__ >= 6 diff --git a/arch/arm/mm/extable.c b/arch/arm/mm/extable.c index f436f7439e46..fc33564597b8 100644 --- a/arch/arm/mm/extable.c +++ b/arch/arm/mm/extable.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/arch/arm/mm/extable.c */ diff --git a/arch/arm/mm/fault.h b/arch/arm/mm/fault.h index afc1f84e763b..c063708fa503 100644 --- a/arch/arm/mm/fault.h +++ b/arch/arm/mm/fault.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __ARCH_ARM_FAULT_H #define __ARCH_ARM_FAULT_H diff --git a/arch/arm/mm/fsr-2level.c b/arch/arm/mm/fsr-2level.c index 18ca74c0f341..f2be95197265 100644 --- a/arch/arm/mm/fsr-2level.c +++ b/arch/arm/mm/fsr-2level.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 static struct fsr_info fsr_info[] = { /* * The following are the standard ARMv3 and ARMv4 aborts. ARMv5 diff --git a/arch/arm/mm/fsr-3level.c b/arch/arm/mm/fsr-3level.c index ab4409a2307e..d0ae2963656a 100644 --- a/arch/arm/mm/fsr-3level.c +++ b/arch/arm/mm/fsr-3level.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 static struct fsr_info fsr_info[] = { { do_bad, SIGBUS, 0, "unknown 0" }, { do_bad, SIGBUS, 0, "unknown 1" }, diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index 3e511bec69b8..10bfba85eb96 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/module.h> #include <linux/kernel.h> #include <linux/slab.h> diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index ad80548325fe..a1f11a7ee81b 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -580,16 +580,6 @@ void __init mem_init(void) BUILD_BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); BUG_ON(PKMAP_BASE + LAST_PKMAP * PAGE_SIZE > PAGE_OFFSET); #endif - - if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { - extern int sysctl_overcommit_memory; - /* - * On a machine this small we won't get - * anywhere without overcommit, so turn - * it on by default. - */ - sysctl_overcommit_memory = OVERCOMMIT_ALWAYS; - } } #ifdef CONFIG_STRICT_KERNEL_RWX @@ -639,8 +629,8 @@ static struct section_perm ro_perms[] = { .start = (unsigned long)_stext, .end = (unsigned long)__init_begin, #ifdef CONFIG_ARM_LPAE - .mask = ~L_PMD_SECT_RDONLY, - .prot = L_PMD_SECT_RDONLY, + .mask = ~(L_PMD_SECT_RDONLY | PMD_SECT_AP2), + .prot = L_PMD_SECT_RDONLY | PMD_SECT_AP2, #else .mask = ~(PMD_SECT_APX | PMD_SECT_AP_WRITE), .prot = PMD_SECT_APX | PMD_SECT_AP_WRITE, diff --git a/arch/arm/mm/iomap.c b/arch/arm/mm/iomap.c index 4614208369f1..091ddc56827e 100644 --- a/arch/arm/mm/iomap.c +++ b/arch/arm/mm/iomap.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/arch/arm/mm/iomap.c * diff --git a/arch/arm/mm/l2c-l2x0-resume.S b/arch/arm/mm/l2c-l2x0-resume.S index fda415e4ca8f..fc01f1b18523 100644 --- a/arch/arm/mm/l2c-l2x0-resume.S +++ b/arch/arm/mm/l2c-l2x0-resume.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * L2C-310 early resume code. This can be used by platforms to restore * the settings of their L2 cache controller before restoring the diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index ce727d47275c..6b045c6653ea 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifdef CONFIG_MMU #include <linux/list.h> #include <linux/vmalloc.h> diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index f0701d8d24df..eb1de66517d5 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * linux/arch/arm/mm/mmap.c */ diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 3b8e728cc944..e4370810f4f1 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -27,259 +27,7 @@ unsigned long vectors_base; #ifdef CONFIG_ARM_MPU struct mpu_rgn_info mpu_rgn_info; - -/* Region number */ -static void rgnr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c2, 0" : : "r" (v)); -} - -/* Data-side / unified region attributes */ - -/* Region access control register */ -static void dracr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 4" : : "r" (v)); -} - -/* Region size register */ -static void drsr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 2" : : "r" (v)); -} - -/* Region base address register */ -static void drbar_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 0" : : "r" (v)); -} - -static u32 drbar_read(void) -{ - u32 v; - asm("mrc p15, 0, %0, c6, c1, 0" : "=r" (v)); - return v; -} -/* Optional instruction-side region attributes */ - -/* I-side Region access control register */ -static void iracr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 5" : : "r" (v)); -} - -/* I-side Region size register */ -static void irsr_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 3" : : "r" (v)); -} - -/* I-side Region base address register */ -static void irbar_write(u32 v) -{ - asm("mcr p15, 0, %0, c6, c1, 1" : : "r" (v)); -} - -static unsigned long irbar_read(void) -{ - unsigned long v; - asm("mrc p15, 0, %0, c6, c1, 1" : "=r" (v)); - return v; -} - -/* MPU initialisation functions */ -void __init adjust_lowmem_bounds_mpu(void) -{ - phys_addr_t phys_offset = PHYS_OFFSET; - phys_addr_t aligned_region_size, specified_mem_size, rounded_mem_size; - struct memblock_region *reg; - bool first = true; - phys_addr_t mem_start; - phys_addr_t mem_end; - - for_each_memblock(memory, reg) { - if (first) { - /* - * Initially only use memory continuous from - * PHYS_OFFSET */ - if (reg->base != phys_offset) - panic("First memory bank must be contiguous from PHYS_OFFSET"); - - mem_start = reg->base; - mem_end = reg->base + reg->size; - specified_mem_size = reg->size; - first = false; - } else { - /* - * memblock auto merges contiguous blocks, remove - * all blocks afterwards in one go (we can't remove - * blocks separately while iterating) - */ - pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n", - &mem_end, ®->base); - memblock_remove(reg->base, 0 - reg->base); - break; - } - } - - /* - * MPU has curious alignment requirements: Size must be power of 2, and - * region start must be aligned to the region size - */ - if (phys_offset != 0) - pr_info("PHYS_OFFSET != 0 => MPU Region size constrained by alignment requirements\n"); - - /* - * Maximum aligned region might overflow phys_addr_t if phys_offset is - * 0. Hence we keep everything below 4G until we take the smaller of - * the aligned_region_size and rounded_mem_size, one of which is - * guaranteed to be smaller than the maximum physical address. - */ - aligned_region_size = (phys_offset - 1) ^ (phys_offset); - /* Find the max power-of-two sized region that fits inside our bank */ - rounded_mem_size = (1 << __fls(specified_mem_size)) - 1; - - /* The actual region size is the smaller of the two */ - aligned_region_size = aligned_region_size < rounded_mem_size - ? aligned_region_size + 1 - : rounded_mem_size + 1; - - if (aligned_region_size != specified_mem_size) { - pr_warn("Truncating memory from %pa to %pa (MPU region constraints)", - &specified_mem_size, &aligned_region_size); - memblock_remove(mem_start + aligned_region_size, - specified_mem_size - aligned_region_size); - - mem_end = mem_start + aligned_region_size; - } - - pr_debug("MPU Region from %pa size %pa (end %pa))\n", - &phys_offset, &aligned_region_size, &mem_end); - -} - -static int mpu_present(void) -{ - return ((read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA) == MMFR0_PMSAv7); -} - -static int mpu_max_regions(void) -{ - /* - * We don't support a different number of I/D side regions so if we - * have separate instruction and data memory maps then return - * whichever side has a smaller number of supported regions. - */ - u32 dregions, iregions, mpuir; - mpuir = read_cpuid(CPUID_MPUIR); - - dregions = iregions = (mpuir & MPUIR_DREGION_SZMASK) >> MPUIR_DREGION; - - /* Check for separate d-side and i-side memory maps */ - if (mpuir & MPUIR_nU) - iregions = (mpuir & MPUIR_IREGION_SZMASK) >> MPUIR_IREGION; - - /* Use the smallest of the two maxima */ - return min(dregions, iregions); -} - -static int mpu_iside_independent(void) -{ - /* MPUIR.nU specifies whether there is *not* a unified memory map */ - return read_cpuid(CPUID_MPUIR) & MPUIR_nU; -} - -static int mpu_min_region_order(void) -{ - u32 drbar_result, irbar_result; - /* We've kept a region free for this probing */ - rgnr_write(MPU_PROBE_REGION); - isb(); - /* - * As per ARM ARM, write 0xFFFFFFFC to DRBAR to find the minimum - * region order - */ - drbar_write(0xFFFFFFFC); - drbar_result = irbar_result = drbar_read(); - drbar_write(0x0); - /* If the MPU is non-unified, we use the larger of the two minima*/ - if (mpu_iside_independent()) { - irbar_write(0xFFFFFFFC); - irbar_result = irbar_read(); - irbar_write(0x0); - } - isb(); /* Ensure that MPU region operations have completed */ - /* Return whichever result is larger */ - return __ffs(max(drbar_result, irbar_result)); -} - -static int mpu_setup_region(unsigned int number, phys_addr_t start, - unsigned int size_order, unsigned int properties) -{ - u32 size_data; - - /* We kept a region free for probing resolution of MPU regions*/ - if (number > mpu_max_regions() || number == MPU_PROBE_REGION) - return -ENOENT; - - if (size_order > 32) - return -ENOMEM; - - if (size_order < mpu_min_region_order()) - return -ENOMEM; - - /* Writing N to bits 5:1 (RSR_SZ) specifies region size 2^N+1 */ - size_data = ((size_order - 1) << MPU_RSR_SZ) | 1 << MPU_RSR_EN; - - dsb(); /* Ensure all previous data accesses occur with old mappings */ - rgnr_write(number); - isb(); - drbar_write(start); - dracr_write(properties); - isb(); /* Propagate properties before enabling region */ - drsr_write(size_data); - - /* Check for independent I-side registers */ - if (mpu_iside_independent()) { - irbar_write(start); - iracr_write(properties); - isb(); - irsr_write(size_data); - } - isb(); - - /* Store region info (we treat i/d side the same, so only store d) */ - mpu_rgn_info.rgns[number].dracr = properties; - mpu_rgn_info.rgns[number].drbar = start; - mpu_rgn_info.rgns[number].drsr = size_data; - return 0; -} - -/* -* Set up default MPU regions, doing nothing if there is no MPU -*/ -void __init mpu_setup(void) -{ - int region_err; - if (!mpu_present()) - return; - - region_err = mpu_setup_region(MPU_RAM_REGION, PHYS_OFFSET, - ilog2(memblock.memory.regions[0].size), - MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL); - if (region_err) { - panic("MPU region initialization failure! %d", region_err); - } else { - pr_info("Using ARMv7 PMSA Compliant MPU. " - "Region independence: %s, Max regions: %d\n", - mpu_iside_independent() ? "Yes" : "No", - mpu_max_regions()); - } -} -#else -static void adjust_lowmem_bounds_mpu(void) {} -static void __init mpu_setup(void) {} -#endif /* CONFIG_ARM_MPU */ +#endif #ifdef CONFIG_CPU_CP15 #ifdef CONFIG_CPU_HIGH_VECTOR @@ -344,6 +92,11 @@ void __init arm_mm_memblock_reserve(void) * reserved here. */ #endif + /* + * In any case, always ensure address 0 is never used as many things + * get very confused if 0 is returned as a legitimate address. + */ + memblock_reserve(0, 1); } void __init adjust_lowmem_bounds(void) diff --git a/arch/arm/mm/pabort-legacy.S b/arch/arm/mm/pabort-legacy.S index 8bbff025269a..b2ffce420106 100644 --- a/arch/arm/mm/pabort-legacy.S +++ b/arch/arm/mm/pabort-legacy.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> diff --git a/arch/arm/mm/pabort-v6.S b/arch/arm/mm/pabort-v6.S index 9627646ce783..8686265dc941 100644 --- a/arch/arm/mm/pabort-v6.S +++ b/arch/arm/mm/pabort-v6.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> diff --git a/arch/arm/mm/pabort-v7.S b/arch/arm/mm/pabort-v7.S index 875761f44f3b..9c70b1a21dc9 100644 --- a/arch/arm/mm/pabort-v7.S +++ b/arch/arm/mm/pabort-v7.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #include <linux/linkage.h> #include <asm/assembler.h> diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index c1c1a5c67da1..61e281cb29fb 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -141,7 +141,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) pte = pmd_pgtable(*pmd); pmd_clear(pmd); pte_free(mm, pte); - atomic_long_dec(&mm->nr_ptes); + mm_dec_nr_ptes(mm); no_pmd: pud_clear(pud); pmd_free(mm, pmd); diff --git a/arch/arm/mm/physaddr.c b/arch/arm/mm/physaddr.c index 02e60f495608..cf75819e4c13 100644 --- a/arch/arm/mm/physaddr.c +++ b/arch/arm/mm/physaddr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/bug.h> #include <linux/export.h> #include <linux/types.h> diff --git a/arch/arm/mm/pmsa-v7.c b/arch/arm/mm/pmsa-v7.c new file mode 100644 index 000000000000..976df60ac426 --- /dev/null +++ b/arch/arm/mm/pmsa-v7.c @@ -0,0 +1,484 @@ +/* + * Based on linux/arch/arm/mm/nommu.c + * + * ARM PMSAv7 supporting functions. + */ + +#include <linux/bitops.h> +#include <linux/memblock.h> + +#include <asm/cacheflush.h> +#include <asm/cp15.h> +#include <asm/cputype.h> +#include <asm/mpu.h> +#include <asm/sections.h> + +#include "mm.h" + +struct region { + phys_addr_t base; + phys_addr_t size; + unsigned long subreg; +}; + +static struct region __initdata mem[MPU_MAX_REGIONS]; +#ifdef CONFIG_XIP_KERNEL +static struct region __initdata xip[MPU_MAX_REGIONS]; +#endif + +static unsigned int __initdata mpu_min_region_order; +static unsigned int __initdata mpu_max_regions; + +static int __init __mpu_min_region_order(void); +static int __init __mpu_max_regions(void); + +#ifndef CONFIG_CPU_V7M + +#define DRBAR __ACCESS_CP15(c6, 0, c1, 0) +#define IRBAR __ACCESS_CP15(c6, 0, c1, 1) +#define DRSR __ACCESS_CP15(c6, 0, c1, 2) +#define IRSR __ACCESS_CP15(c6, 0, c1, 3) +#define DRACR __ACCESS_CP15(c6, 0, c1, 4) +#define IRACR __ACCESS_CP15(c6, 0, c1, 5) +#define RNGNR __ACCESS_CP15(c6, 0, c2, 0) + +/* Region number */ +static inline void rgnr_write(u32 v) +{ + write_sysreg(v, RNGNR); +} + +/* Data-side / unified region attributes */ + +/* Region access control register */ +static inline void dracr_write(u32 v) +{ + write_sysreg(v, DRACR); +} + +/* Region size register */ +static inline void drsr_write(u32 v) +{ + write_sysreg(v, DRSR); +} + +/* Region base address register */ +static inline void drbar_write(u32 v) +{ + write_sysreg(v, DRBAR); +} + +static inline u32 drbar_read(void) +{ + return read_sysreg(DRBAR); +} +/* Optional instruction-side region attributes */ + +/* I-side Region access control register */ +static inline void iracr_write(u32 v) +{ + write_sysreg(v, IRACR); +} + +/* I-side Region size register */ +static inline void irsr_write(u32 v) +{ + write_sysreg(v, IRSR); +} + +/* I-side Region base address register */ +static inline void irbar_write(u32 v) +{ + write_sysreg(v, IRBAR); +} + +static inline u32 irbar_read(void) +{ + return read_sysreg(IRBAR); +} + +#else + +static inline void rgnr_write(u32 v) +{ + writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RNR); +} + +/* Data-side / unified region attributes */ + +/* Region access control register */ +static inline void dracr_write(u32 v) +{ + u32 rsr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(15, 0); + + writel_relaxed((v << 16) | rsr, BASEADDR_V7M_SCB + MPU_RASR); +} + +/* Region size register */ +static inline void drsr_write(u32 v) +{ + u32 racr = readl_relaxed(BASEADDR_V7M_SCB + MPU_RASR) & GENMASK(31, 16); + + writel_relaxed(v | racr, BASEADDR_V7M_SCB + MPU_RASR); +} + +/* Region base address register */ +static inline void drbar_write(u32 v) +{ + writel_relaxed(v, BASEADDR_V7M_SCB + MPU_RBAR); +} + +static inline u32 drbar_read(void) +{ + return readl_relaxed(BASEADDR_V7M_SCB + MPU_RBAR); +} + +/* ARMv7-M only supports a unified MPU, so I-side operations are nop */ + +static inline void iracr_write(u32 v) {} +static inline void irsr_write(u32 v) {} +static inline void irbar_write(u32 v) {} +static inline unsigned long irbar_read(void) {return 0;} + +#endif + +static int __init mpu_present(void) +{ + return ((read_cpuid_ext(CPUID_EXT_MMFR0) & MMFR0_PMSA) == MMFR0_PMSAv7); +} + +static bool __init try_split_region(phys_addr_t base, phys_addr_t size, struct region *region) +{ + unsigned long subreg, bslots, sslots; + phys_addr_t abase = base & ~(size - 1); + phys_addr_t asize = base + size - abase; + phys_addr_t p2size = 1 << __fls(asize); + phys_addr_t bdiff, sdiff; + + if (p2size != asize) + p2size *= 2; + + bdiff = base - abase; + sdiff = p2size - asize; + subreg = p2size / MPU_NR_SUBREGS; + + if ((bdiff % subreg) || (sdiff % subreg)) + return false; + + bslots = bdiff / subreg; + sslots = sdiff / subreg; + + if (bslots || sslots) { + int i; + + if (subreg < MPU_MIN_SUBREG_SIZE) + return false; + + if (bslots + sslots > MPU_NR_SUBREGS) + return false; + + for (i = 0; i < bslots; i++) + _set_bit(i, ®ion->subreg); + + for (i = 1; i <= sslots; i++) + _set_bit(MPU_NR_SUBREGS - i, ®ion->subreg); + } + + region->base = abase; + region->size = p2size; + + return true; +} + +static int __init allocate_region(phys_addr_t base, phys_addr_t size, + unsigned int limit, struct region *regions) +{ + int count = 0; + phys_addr_t diff = size; + int attempts = MPU_MAX_REGIONS; + + while (diff) { + /* Try cover region as is (maybe with help of subregions) */ + if (try_split_region(base, size, ®ions[count])) { + count++; + base += size; + diff -= size; + size = diff; + } else { + /* + * Maximum aligned region might overflow phys_addr_t + * if "base" is 0. Hence we keep everything below 4G + * until we take the smaller of the aligned region + * size ("asize") and rounded region size ("p2size"), + * one of which is guaranteed to be smaller than the + * maximum physical address. + */ + phys_addr_t asize = (base - 1) ^ base; + phys_addr_t p2size = (1 << __fls(diff)) - 1; + + size = asize < p2size ? asize + 1 : p2size + 1; + } + + if (count > limit) + break; + + if (!attempts) + break; + + attempts--; + } + + return count; +} + +/* MPU initialisation functions */ +void __init adjust_lowmem_bounds_mpu(void) +{ + phys_addr_t specified_mem_size = 0, total_mem_size = 0; + struct memblock_region *reg; + bool first = true; + phys_addr_t mem_start; + phys_addr_t mem_end; + unsigned int mem_max_regions; + int num, i; + + if (!mpu_present()) + return; + + /* Free-up MPU_PROBE_REGION */ + mpu_min_region_order = __mpu_min_region_order(); + + /* How many regions are supported */ + mpu_max_regions = __mpu_max_regions(); + + mem_max_regions = min((unsigned int)MPU_MAX_REGIONS, mpu_max_regions); + + /* We need to keep one slot for background region */ + mem_max_regions--; + +#ifndef CONFIG_CPU_V7M + /* ... and one for vectors */ + mem_max_regions--; +#endif + +#ifdef CONFIG_XIP_KERNEL + /* plus some regions to cover XIP ROM */ + num = allocate_region(CONFIG_XIP_PHYS_ADDR, __pa(_exiprom) - CONFIG_XIP_PHYS_ADDR, + mem_max_regions, xip); + + mem_max_regions -= num; +#endif + + for_each_memblock(memory, reg) { + if (first) { + phys_addr_t phys_offset = PHYS_OFFSET; + + /* + * Initially only use memory continuous from + * PHYS_OFFSET */ + if (reg->base != phys_offset) + panic("First memory bank must be contiguous from PHYS_OFFSET"); + + mem_start = reg->base; + mem_end = reg->base + reg->size; + specified_mem_size = reg->size; + first = false; + } else { + /* + * memblock auto merges contiguous blocks, remove + * all blocks afterwards in one go (we can't remove + * blocks separately while iterating) + */ + pr_notice("Ignoring RAM after %pa, memory at %pa ignored\n", + &mem_end, ®->base); + memblock_remove(reg->base, 0 - reg->base); + break; + } + } + + num = allocate_region(mem_start, specified_mem_size, mem_max_regions, mem); + + for (i = 0; i < num; i++) { + unsigned long subreg = mem[i].size / MPU_NR_SUBREGS; + + total_mem_size += mem[i].size - subreg * hweight_long(mem[i].subreg); + + pr_debug("MPU: base %pa size %pa disable subregions: %*pbl\n", + &mem[i].base, &mem[i].size, MPU_NR_SUBREGS, &mem[i].subreg); + } + + if (total_mem_size != specified_mem_size) { + pr_warn("Truncating memory from %pa to %pa (MPU region constraints)", + &specified_mem_size, &total_mem_size); + memblock_remove(mem_start + total_mem_size, + specified_mem_size - total_mem_size); + } +} + +static int __init __mpu_max_regions(void) +{ + /* + * We don't support a different number of I/D side regions so if we + * have separate instruction and data memory maps then return + * whichever side has a smaller number of supported regions. + */ + u32 dregions, iregions, mpuir; + + mpuir = read_cpuid_mputype(); + + dregions = iregions = (mpuir & MPUIR_DREGION_SZMASK) >> MPUIR_DREGION; + + /* Check for separate d-side and i-side memory maps */ + if (mpuir & MPUIR_nU) + iregions = (mpuir & MPUIR_IREGION_SZMASK) >> MPUIR_IREGION; + + /* Use the smallest of the two maxima */ + return min(dregions, iregions); +} + +static int __init mpu_iside_independent(void) +{ + /* MPUIR.nU specifies whether there is *not* a unified memory map */ + return read_cpuid_mputype() & MPUIR_nU; +} + +static int __init __mpu_min_region_order(void) +{ + u32 drbar_result, irbar_result; + + /* We've kept a region free for this probing */ + rgnr_write(MPU_PROBE_REGION); + isb(); + /* + * As per ARM ARM, write 0xFFFFFFFC to DRBAR to find the minimum + * region order + */ + drbar_write(0xFFFFFFFC); + drbar_result = irbar_result = drbar_read(); + drbar_write(0x0); + /* If the MPU is non-unified, we use the larger of the two minima*/ + if (mpu_iside_independent()) { + irbar_write(0xFFFFFFFC); + irbar_result = irbar_read(); + irbar_write(0x0); + } + isb(); /* Ensure that MPU region operations have completed */ + /* Return whichever result is larger */ + + return __ffs(max(drbar_result, irbar_result)); +} + +static int __init mpu_setup_region(unsigned int number, phys_addr_t start, + unsigned int size_order, unsigned int properties, + unsigned int subregions, bool need_flush) +{ + u32 size_data; + + /* We kept a region free for probing resolution of MPU regions*/ + if (number > mpu_max_regions + || number >= MPU_MAX_REGIONS) + return -ENOENT; + + if (size_order > 32) + return -ENOMEM; + + if (size_order < mpu_min_region_order) + return -ENOMEM; + + /* Writing N to bits 5:1 (RSR_SZ) specifies region size 2^N+1 */ + size_data = ((size_order - 1) << MPU_RSR_SZ) | 1 << MPU_RSR_EN; + size_data |= subregions << MPU_RSR_SD; + + if (need_flush) + flush_cache_all(); + + dsb(); /* Ensure all previous data accesses occur with old mappings */ + rgnr_write(number); + isb(); + drbar_write(start); + dracr_write(properties); + isb(); /* Propagate properties before enabling region */ + drsr_write(size_data); + + /* Check for independent I-side registers */ + if (mpu_iside_independent()) { + irbar_write(start); + iracr_write(properties); + isb(); + irsr_write(size_data); + } + isb(); + + /* Store region info (we treat i/d side the same, so only store d) */ + mpu_rgn_info.rgns[number].dracr = properties; + mpu_rgn_info.rgns[number].drbar = start; + mpu_rgn_info.rgns[number].drsr = size_data; + + mpu_rgn_info.used++; + + return 0; +} + +/* +* Set up default MPU regions, doing nothing if there is no MPU +*/ +void __init mpu_setup(void) +{ + int i, region = 0, err = 0; + + if (!mpu_present()) + return; + + /* Setup MPU (order is important) */ + + /* Background */ + err |= mpu_setup_region(region++, 0, 32, + MPU_ACR_XN | MPU_RGN_STRONGLY_ORDERED | MPU_AP_PL1RW_PL0NA, + 0, false); + +#ifdef CONFIG_XIP_KERNEL + /* ROM */ + for (i = 0; i < ARRAY_SIZE(xip); i++) { + /* + * In case we overwrite RAM region we set earlier in + * head-nommu.S (which is cachable) all subsequent + * data access till we setup RAM bellow would be done + * with BG region (which is uncachable), thus we need + * to clean and invalidate cache. + */ + bool need_flush = region == MPU_RAM_REGION; + + if (!xip[i].size) + continue; + + err |= mpu_setup_region(region++, xip[i].base, ilog2(xip[i].size), + MPU_AP_PL1RO_PL0NA | MPU_RGN_NORMAL, + xip[i].subreg, need_flush); + } +#endif + + /* RAM */ + for (i = 0; i < ARRAY_SIZE(mem); i++) { + if (!mem[i].size) + continue; + + err |= mpu_setup_region(region++, mem[i].base, ilog2(mem[i].size), + MPU_AP_PL1RW_PL0RW | MPU_RGN_NORMAL, + mem[i].subreg, false); + } + + /* Vectors */ +#ifndef CONFIG_CPU_V7M + err |= mpu_setup_region(region++, vectors_base, ilog2(2 * PAGE_SIZE), + MPU_AP_PL1RW_PL0NA | MPU_RGN_NORMAL, + 0, false); +#endif + if (err) { + panic("MPU region initialization failure! %d", err); + } else { + pr_info("Using ARMv7 PMSA Compliant MPU. " + "Region independence: %s, Used %d of %d regions\n", + mpu_iside_independent() ? "Yes" : "No", + mpu_rgn_info.used, mpu_max_regions); + } +} diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index f944836da8a2..f10e31d0730a 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * We need constants.h for: * VMA_VM_MM |