diff options
Diffstat (limited to 'arch/arm64')
213 files changed, 10691 insertions, 2665 deletions
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 9532f8d5857e..7796af4b1d6f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1,7 +1,10 @@ config ARM64 def_bool y - select ARCH_BINFMT_ELF_RANDOMIZE_PIE + select ACPI_GENERIC_GSI if ACPI + select ACPI_REDUCED_HARDWARE_ONLY if ACPI select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE + select ARCH_HAS_ELF_RANDOMIZE + select ARCH_HAS_GCOV_PROFILE_ALL select ARCH_HAS_SG_CHAIN select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_USE_CMPXCHG_LOCKREF @@ -13,7 +16,9 @@ config ARM64 select ARM_ARCH_TIMER select ARM_GIC select AUDIT_ARCH_COMPAT_GENERIC + select ARM_GIC_V2M if PCI_MSI select ARM_GIC_V3 + select ARM_GIC_V3_ITS if PCI_MSI select BUILDTIME_EXTABLE_SORT select CLONE_BACKWARDS select COMMON_CLK @@ -24,9 +29,10 @@ config ARM64 select GENERIC_CLOCKEVENTS_BROADCAST if SMP select GENERIC_CPU_AUTOPROBE select GENERIC_EARLY_IOREMAP - select GENERIC_IOMAP select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW + select GENERIC_IRQ_SHOW_LEVEL + select GENERIC_PCI_IOMAP select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD select GENERIC_STRNCPY_FROM_USER @@ -34,13 +40,17 @@ config ARM64 select GENERIC_TIME_VSYSCALL select HANDLE_DOMAIN_IRQ select HARDIRQS_SW_RESEND + select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_BITREVERSE select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_KGDB + select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_BPF_JIT select HAVE_C_RECORDMCOUNT select HAVE_CC_STACKPROTECTOR + select HAVE_CMPXCHG_DOUBLE select HAVE_DEBUG_BUGVERBOSE select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG @@ -136,12 +146,90 @@ config KERNEL_MODE_NEON config FIX_EARLYCON_MEM def_bool y +config PGTABLE_LEVELS + int + default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 + default 3 if ARM64_64K_PAGES && ARM64_VA_BITS_48 + default 3 if ARM64_4K_PAGES && ARM64_VA_BITS_39 + default 4 if ARM64_4K_PAGES && ARM64_VA_BITS_48 + source "init/Kconfig" source "kernel/Kconfig.freezer" menu "Platform selection" +config ARCH_EXYNOS + bool + help + This enables support for Samsung Exynos SoC family + +config ARCH_EXYNOS7 + bool "ARMv8 based Samsung Exynos7" + select ARCH_EXYNOS + select COMMON_CLK_SAMSUNG + select HAVE_S3C2410_WATCHDOG if WATCHDOG + select HAVE_S3C_RTC if RTC_CLASS + select PINCTRL + select PINCTRL_EXYNOS + + help + This enables support for Samsung Exynos7 SoC family + +config ARCH_FSL_LS2085A + bool "Freescale LS2085A SOC" + help + This enables support for Freescale LS2085A SOC. + +config ARCH_MEDIATEK + bool "Mediatek MT65xx & MT81xx ARMv8 SoC" + select ARM_GIC + select PINCTRL + help + Support for Mediatek MT65xx & MT81xx ARMv8 SoCs + +config ARCH_QCOM + bool "Qualcomm Platforms" + select PINCTRL + help + This enables support for the ARMv8 based Qualcomm chipsets. + +config ARCH_SEATTLE + bool "AMD Seattle SoC Family" + help + This enables support for AMD Seattle SOC Family + +config ARCH_TEGRA + bool "NVIDIA Tegra SoC Family" + select ARCH_HAS_RESET_CONTROLLER + select ARCH_REQUIRE_GPIOLIB + select CLKDEV_LOOKUP + select CLKSRC_MMIO + select CLKSRC_OF + select GENERIC_CLOCKEVENTS + select HAVE_CLK + select PINCTRL + select RESET_CONTROLLER + help + This enables support for the NVIDIA Tegra SoC family. + +config ARCH_TEGRA_132_SOC + bool "NVIDIA Tegra132 SoC" + depends on ARCH_TEGRA + select PINCTRL_TEGRA124 + select USB_ULPI if USB_PHY + select USB_ULPI_VIEWPORT if USB_PHY + help + Enable support for NVIDIA Tegra132 SoC, based on the Denver + ARMv8 CPU. The Tegra132 SoC is similar to the Tegra124 SoC, + but contains an NVIDIA Denver CPU complex in place of + Tegra124's "4+1" Cortex-A15 CPU complex. + +config ARCH_SPRD + bool "Spreadtrum SoC platform" + help + Support for Spreadtrum ARM based SoCs + config ARCH_THUNDER bool "Cavium Inc. Thunder SoC Family" help @@ -162,13 +250,15 @@ config ARCH_XGENE help This enables support for AppliedMicro X-Gene SOC Family +config ARCH_ZYNQMP + bool "Xilinx ZynqMP Family" + help + This enables support for Xilinx ZynqMP Family + endmenu menu "Bus support" -config ARM_AMBA - bool - config PCI bool "PCI support" help @@ -193,6 +283,135 @@ endmenu menu "Kernel Features" +menu "ARM errata workarounds via the alternatives framework" + +config ARM64_ERRATUM_826319 + bool "Cortex-A53: 826319: System might deadlock if a write cannot complete until read data is accepted" + default y + help + This option adds an alternative code sequence to work around ARM + erratum 826319 on Cortex-A53 parts up to r0p2 with an AMBA 4 ACE or + AXI master interface and an L2 cache. + + If a Cortex-A53 uses an AMBA AXI4 ACE interface to other processors + and is unable to accept a certain write via this interface, it will + not progress on read data presented on the read data channel and the + system can deadlock. + + The workaround promotes data cache clean instructions to + data cache clean-and-invalidate. + Please note that this does not necessarily enable the workaround, + as it depends on the alternative framework, which will only patch + the kernel if an affected CPU is detected. + + If unsure, say Y. + +config ARM64_ERRATUM_827319 + bool "Cortex-A53: 827319: Data cache clean instructions might cause overlapping transactions to the interconnect" + default y + help + This option adds an alternative code sequence to work around ARM + erratum 827319 on Cortex-A53 parts up to r0p2 with an AMBA 5 CHI + master interface and an L2 cache. + + Under certain conditions this erratum can cause a clean line eviction + to occur at the same time as another transaction to the same address + on the AMBA 5 CHI interface, which can cause data corruption if the + interconnect reorders the two transactions. + + The workaround promotes data cache clean instructions to + data cache clean-and-invalidate. + Please note that this does not necessarily enable the workaround, + as it depends on the alternative framework, which will only patch + the kernel if an affected CPU is detected. + + If unsure, say Y. + +config ARM64_ERRATUM_824069 + bool "Cortex-A53: 824069: Cache line might not be marked as clean after a CleanShared snoop" + default y + help + This option adds an alternative code sequence to work around ARM + erratum 824069 on Cortex-A53 parts up to r0p2 when it is connected + to a coherent interconnect. + + If a Cortex-A53 processor is executing a store or prefetch for + write instruction at the same time as a processor in another + cluster is executing a cache maintenance operation to the same + address, then this erratum might cause a clean cache line to be + incorrectly marked as dirty. + + The workaround promotes data cache clean instructions to + data cache clean-and-invalidate. + Please note that this option does not necessarily enable the + workaround, as it depends on the alternative framework, which will + only patch the kernel if an affected CPU is detected. + + If unsure, say Y. + +config ARM64_ERRATUM_819472 + bool "Cortex-A53: 819472: Store exclusive instructions might cause data corruption" + default y + help + This option adds an alternative code sequence to work around ARM + erratum 819472 on Cortex-A53 parts up to r0p1 with an L2 cache + present when it is connected to a coherent interconnect. + + If the processor is executing a load and store exclusive sequence at + the same time as a processor in another cluster is executing a cache + maintenance operation to the same address, then this erratum might + cause data corruption. + + The workaround promotes data cache clean instructions to + data cache clean-and-invalidate. + Please note that this does not necessarily enable the workaround, + as it depends on the alternative framework, which will only patch + the kernel if an affected CPU is detected. + + If unsure, say Y. + +config ARM64_ERRATUM_832075 + bool "Cortex-A57: 832075: possible deadlock on mixing exclusive memory accesses with device loads" + default y + help + This option adds an alternative code sequence to work around ARM + erratum 832075 on Cortex-A57 parts up to r1p2. + + Affected Cortex-A57 parts might deadlock when exclusive load/store + instructions to Write-Back memory are mixed with Device loads. + + The workaround is to promote device loads to use Load-Acquire + semantics. + Please note that this does not necessarily enable the workaround, + as it depends on the alternative framework, which will only patch + the kernel if an affected CPU is detected. + + If unsure, say Y. + +config ARM64_ERRATUM_845719 + bool "Cortex-A53: 845719: a load might read incorrect data" + depends on COMPAT + default y + help + This option adds an alternative code sequence to work around ARM + erratum 845719 on Cortex-A53 parts up to r0p4. + + When running a compat (AArch32) userspace on an affected Cortex-A53 + part, a load at EL0 from a virtual address that matches the bottom 32 + bits of the virtual address used by a recent load at (AArch64) EL1 + might return incorrect data. + + The workaround is to write the contextidr_el1 register on exception + return to a 32-bit task. + Please note that this does not necessarily enable the workaround, + as it depends on the alternative framework, which will only patch + the kernel if an affected CPU is detected. + + If unsure, say Y. + +endmenu + + choice prompt "Page size" default ARM64_4K_PAGES @@ -233,7 +452,6 @@ config ARM64_VA_BITS_42 config ARM64_VA_BITS_48 bool "48-bit" - depends on !ARM_SMMU endchoice @@ -243,13 +461,6 @@ config ARM64_VA_BITS default 42 if ARM64_VA_BITS_42 default 48 if ARM64_VA_BITS_48 -config ARM64_PGTABLE_LEVELS - int - default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42 - default 3 if ARM64_64K_PAGES && ARM64_VA_BITS_48 - default 3 if ARM64_4K_PAGES && ARM64_VA_BITS_39 - default 4 if ARM64_4K_PAGES && ARM64_VA_BITS_48 - config CPU_BIG_ENDIAN bool "Build big-endian kernel" help @@ -285,8 +496,8 @@ config SCHED_SMT places. If unsure say N here. config NR_CPUS - int "Maximum number of CPUs (2-64)" - range 2 64 + int "Maximum number of CPUs (2-4096)" + range 2 4096 depends on SMP # These have to remain sorted largest to smallest default "64" @@ -300,6 +511,10 @@ config HOTPLUG_CPU source kernel/Kconfig.preempt +config UP_LATE_INIT + def_bool y + depends on !SMP + config HZ int default 100 @@ -345,6 +560,19 @@ config ARCH_HAS_CACHE_LINE_SIZE source "mm/Kconfig" +config SECCOMP + bool "Enable seccomp to safely compute untrusted bytecode" + ---help--- + This kernel feature is useful for number crunching applications + that may need to compute untrusted bytecode during their + execution. By using pipes or other transports made available to + the process as file descriptors supporting the read/write + syscalls, it's possible to isolate those applications in + their own address space using seccomp. Once seccomp is + enabled via prctl(PR_SET_SECCOMP), it cannot be disabled + and the task is only allowed to execute a few safe syscalls + defined by each seccomp mode. + config XEN_DOM0 def_bool y depends on XEN @@ -361,6 +589,73 @@ config FORCE_MAX_ZONEORDER default "14" if (ARM64_64K_PAGES && TRANSPARENT_HUGEPAGE) default "11" +menuconfig ARMV8_DEPRECATED + bool "Emulate deprecated/obsolete ARMv8 instructions" + depends on COMPAT + help + Legacy software support may require certain instructions + that have been deprecated or obsoleted in the architecture. + + Enable this config to enable selective emulation of these + features. + + If unsure, say Y + +if ARMV8_DEPRECATED + +config SWP_EMULATION + bool "Emulate SWP/SWPB instructions" + help + ARMv8 obsoletes the use of A32 SWP/SWPB instructions such that + they are always undefined. Say Y here to enable software + emulation of these instructions for userspace using LDXR/STXR. + + In some older versions of glibc [<=2.8] SWP is used during futex + trylock() operations with the assumption that the code will not + be preempted. This invalid assumption may be more likely to fail + with SWP emulation enabled, leading to deadlock of the user + application. + + NOTE: when accessing uncached shared regions, LDXR/STXR rely + on an external transaction monitoring block called a global + monitor to maintain update atomicity. If your system does not + implement a global monitor, this option can cause programs that + perform SWP operations to uncached memory to deadlock. + + If unsure, say Y + +config CP15_BARRIER_EMULATION + bool "Emulate CP15 Barrier instructions" + help + The CP15 barrier instructions - CP15ISB, CP15DSB, and + CP15DMB - are deprecated in ARMv8 (and ARMv7). It is + strongly recommended to use the ISB, DSB, and DMB + instructions instead. + + Say Y here to enable software emulation of these + instructions for AArch32 userspace code. When this option is + enabled, CP15 barrier usage is traced which can help + identify software that needs updating. + + If unsure, say Y + +config SETEND_EMULATION + bool "Emulate SETEND instruction" + help + The SETEND instruction alters the data-endianness of the + AArch32 EL0, and is deprecated in ARMv8. + + Say Y here to enable software emulation of the instruction + for AArch32 userspace code. + + Note: All the cpus on the system must have mixed endian support at EL0 + for this feature to be enabled. If a new CPU - which doesn't support mixed + endian - is hotplugged in after this feature has been enabled, there could + be unexpected results in the applications. + + If unsure, say Y +endif + endmenu menu "Boot options" @@ -401,6 +696,17 @@ config EFI allow the kernel to be booted as an EFI application. This is only useful on systems that have UEFI firmware. +config DMI + bool "Enable support for SMBIOS (DMI) tables" + depends on EFI + default y + help + This enables SMBIOS/DMI feature for systems. + + This option is only useful on systems that have UEFI firmware. + However, even with this option, the resultant kernel should + continue to boot on existing non-UEFI platforms. + endmenu menu "Userspace binary formats" @@ -409,7 +715,7 @@ source "fs/Kconfig.binfmt" config COMPAT bool "Kernel support for 32-bit EL0" - depends on !ARM64_64K_PAGES + depends on !ARM64_64K_PAGES || EXPERT select COMPAT_BINFMT_ELF select HAVE_UID16 select OLD_SIGSUSPEND3 @@ -420,6 +726,10 @@ config COMPAT the user helper functions, VFP support and the ptrace interface are handled appropriately by the kernel. + If you also enabled CONFIG_ARM64_64K_PAGES, please be aware that you + will only be able to execute AArch32 binaries that were compiled with + 64k aligned segments. + If you want to execute 32-bit userspace applications, say Y. config SYSVIPC_COMPAT @@ -435,9 +745,6 @@ source "kernel/power/Kconfig" config ARCH_SUSPEND_POSSIBLE def_bool y -config ARM64_CPU_SUSPEND - def_bool PM_SLEEP - endmenu menu "CPU Power Management" @@ -454,6 +761,8 @@ source "drivers/Kconfig" source "drivers/firmware/Kconfig" +source "drivers/acpi/Kconfig" + source "fs/Kconfig" source "arch/arm64/kvm/Kconfig" diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 0a12933e50ed..d6285ef9b5f9 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -6,6 +6,18 @@ config FRAME_POINTER bool default y +config ARM64_PTDUMP + bool "Export kernel pagetable layout to userspace via debugfs" + depends on DEBUG_KERNEL + select DEBUG_FS + help + Say Y here if you want to show the kernel pagetable layout in a + debugfs file. This information is only useful for kernel developers + who are working in architecture specific areas of the kernel. + It is probably not a good idea to enable this feature in a production + kernel. + If in doubt, say "N" + config STRICT_DEVMEM bool "Filter access to /dev/mem" depends on MMU @@ -54,4 +66,29 @@ config DEBUG_SET_MODULE_RONX against certain classes of kernel exploits. If in doubt, say "N". +config DEBUG_RODATA + bool "Make kernel text and rodata read-only" + help + If this is set, kernel text and rodata will be made read-only. This + is to help catch accidental or malicious attempts to change the + kernel's executable code. Additionally splits rodata from kernel + text so it can be made explicitly non-executable. + + If in doubt, say Y + +config DEBUG_ALIGN_RODATA + depends on DEBUG_RODATA && !ARM64_64K_PAGES + bool "Align linker sections up to SECTION_SIZE" + help + If this option is enabled, sections that may potentially be marked as + read only or non-executable will be aligned up to the section size of + the kernel. This prevents sections from being split into pages and + avoids a potential TLB penalty. The downside is an increase in + alignment and potentially wasted space. Turn on this option if + performance is more important than memory pressure. + + If in doubt, say N + +source "drivers/hwtracing/coresight/Kconfig" + endmenu diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 20901ffed182..4d2a925998f9 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -15,8 +15,6 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET) OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S GZFLAGS :=-9 -LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name) - KBUILD_DEFCONFIG := defconfig KBUILD_CFLAGS += -mgeneral-regs-only @@ -50,8 +48,7 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/ core-$(CONFIG_XEN) += arch/arm64/xen/ core-$(CONFIG_CRYPTO) += arch/arm64/crypto/ libs-y := arch/arm64/lib/ $(libs-y) -libs-y += $(LIBGCC) -libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/ +core-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a # Default target when executing plain make KBUILD_IMAGE := Image.gz @@ -70,8 +67,13 @@ zinstall install: vmlinux %.dtb: scripts $(Q)$(MAKE) $(build)=$(boot)/dts $(boot)/dts/$@ -dtbs: scripts - $(Q)$(MAKE) $(build)=$(boot)/dts dtbs +PHONY += dtbs dtbs_install + +dtbs: prepare scripts + $(Q)$(MAKE) $(build)=$(boot)/dts + +dtbs_install: + $(Q)$(MAKE) $(dtbinst)=$(boot)/dts PHONY += vdso_install vdso_install: @@ -80,11 +82,13 @@ vdso_install: # We use MRPROPER_FILES and CLEAN_FILES now archclean: $(Q)$(MAKE) $(clean)=$(boot) + $(Q)$(MAKE) $(clean)=$(boot)/dts define archhelp echo '* Image.gz - Compressed kernel image (arch/$(ARCH)/boot/Image.gz)' echo ' Image - Uncompressed kernel image (arch/$(ARCH)/boot/Image)' echo '* dtbs - Build device tree blobs for enabled boards' + echo ' dtbs_install - Install dtbs to $(INSTALL_DTBS_PATH)' echo ' install - Install uncompressed kernel' echo ' zinstall - Install compressed kernel' echo ' Install using (your) ~/bin/installkernel or' diff --git a/arch/arm64/boot/dts/Makefile b/arch/arm64/boot/dts/Makefile index f8001a62029c..ad26a752b976 100644 --- a/arch/arm64/boot/dts/Makefile +++ b/arch/arm64/boot/dts/Makefile @@ -1,10 +1,12 @@ -dtb-$(CONFIG_ARCH_THUNDER) += thunder-88xx.dtb -dtb-$(CONFIG_ARCH_VEXPRESS) += rtsm_ve-aemv8a.dtb foundation-v8.dtb -dtb-$(CONFIG_ARCH_XGENE) += apm-mustang.dtb +dts-dirs += amd +dts-dirs += apm +dts-dirs += arm +dts-dirs += cavium +dts-dirs += exynos +dts-dirs += freescale +dts-dirs += mediatek +dts-dirs += qcom +dts-dirs += sprd +dts-dirs += xilinx -targets += dtbs -targets += $(dtb-y) - -dtbs: $(addprefix $(obj)/, $(dtb-y)) - -clean-files := *.dtb +subdir-y := $(dts-dirs) diff --git a/arch/arm64/boot/dts/amd/Makefile b/arch/arm64/boot/dts/amd/Makefile new file mode 100644 index 000000000000..cfdf701e05df --- /dev/null +++ b/arch/arm64/boot/dts/amd/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_SEATTLE) += amd-overdrive.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/amd/amd-overdrive.dts b/arch/arm64/boot/dts/amd/amd-overdrive.dts new file mode 100644 index 000000000000..564a3f7df71d --- /dev/null +++ b/arch/arm64/boot/dts/amd/amd-overdrive.dts @@ -0,0 +1,66 @@ +/* + * DTS file for AMD Seattle Overdrive Development Board + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + */ + +/dts-v1/; + +/include/ "amd-seattle-soc.dtsi" + +/ { + model = "AMD Seattle Development Board (Overdrive)"; + compatible = "amd,seattle-overdrive", "amd,seattle"; + + chosen { + stdout-path = &serial0; + linux,pci-probe-only; + }; +}; + +&ccp0 { + status = "ok"; +}; + +&gpio0 { + status = "ok"; +}; + +&gpio1 { + status = "ok"; +}; + +&i2c0 { + status = "ok"; +}; + +&pcie0 { + status = "ok"; +}; + +&spi0 { + status = "ok"; +}; + +&spi1 { + status = "ok"; + sdcard0: sdcard@0 { + compatible = "mmc-spi-slot"; + reg = <0>; + spi-max-frequency = <20000000>; + voltage-ranges = <3200 3400>; + gpios = <&gpio0 7 0>; + interrupt-parent = <&gpio0>; + interrupts = <7 3>; + pl022,hierarchy = <0>; + pl022,interface = <0>; + pl022,com-mode = <0x0>; + pl022,rx-level-trig = <0>; + pl022,tx-level-trig = <0>; + }; +}; + +&v2m0 { + arm,msi-base-spi = <64>; + arm,msi-num-spis = <256>; +}; diff --git a/arch/arm64/boot/dts/amd/amd-seattle-clks.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-clks.dtsi new file mode 100644 index 000000000000..f623c46525a6 --- /dev/null +++ b/arch/arm64/boot/dts/amd/amd-seattle-clks.dtsi @@ -0,0 +1,54 @@ +/* + * DTS file for AMD Seattle Clocks + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + */ + + adl3clk_100mhz: clk100mhz_0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + clock-output-names = "adl3clk_100mhz"; + }; + + ccpclk_375mhz: clk375mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <375000000>; + clock-output-names = "ccpclk_375mhz"; + }; + + sataclk_333mhz: clk333mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <333000000>; + clock-output-names = "sataclk_333mhz"; + }; + + pcieclk_500mhz: clk500mhz_0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <500000000>; + clock-output-names = "pcieclk_500mhz"; + }; + + dmaclk_500mhz: clk500mhz_1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <500000000>; + clock-output-names = "dmaclk_500mhz"; + }; + + miscclk_250mhz: clk250mhz_4 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <250000000>; + clock-output-names = "miscclk_250mhz"; + }; + + uartspiclk_100mhz: clk100mhz_1 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + clock-output-names = "uartspiclk_100mhz"; + }; diff --git a/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi new file mode 100644 index 000000000000..2874d92881fd --- /dev/null +++ b/arch/arm64/boot/dts/amd/amd-seattle-soc.dtsi @@ -0,0 +1,172 @@ +/* + * DTS file for AMD Seattle SoC + * + * Copyright (C) 2014 Advanced Micro Devices, Inc. + */ + +/ { + compatible = "amd,seattle"; + interrupt-parent = <&gic0>; + #address-cells = <2>; + #size-cells = <2>; + + gic0: interrupt-controller@e1101000 { + compatible = "arm,gic-400", "arm,cortex-a15-gic"; + interrupt-controller; + #interrupt-cells = <3>; + #address-cells = <2>; + #size-cells = <2>; + reg = <0x0 0xe1110000 0 0x1000>, + <0x0 0xe112f000 0 0x2000>, + <0x0 0xe1140000 0 0x10000>, + <0x0 0xe1160000 0 0x10000>; + interrupts = <1 9 0xf04>; + ranges = <0 0 0 0xe1100000 0 0x100000>; + v2m0: v2m@e0080000 { + compatible = "arm,gic-v2m-frame"; + msi-controller; + reg = <0x0 0x00080000 0 0x1000>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = <1 13 0xff04>, + <1 14 0xff04>, + <1 11 0xff04>, + <1 10 0xff04>; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = <0 7 4>, + <0 8 4>, + <0 9 4>, + <0 10 4>, + <0 11 4>, + <0 12 4>, + <0 13 4>, + <0 14 4>; + }; + + smb0: smb { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + /* DDR range is 40-bit addressing */ + dma-ranges = <0x80 0x0 0x80 0x0 0x7f 0xffffffff>; + + /include/ "amd-seattle-clks.dtsi" + + sata0: sata@e0300000 { + compatible = "snps,dwc-ahci"; + reg = <0 0xe0300000 0 0x800>; + interrupts = <0 355 4>; + clocks = <&sataclk_333mhz>; + dma-coherent; + }; + + i2c0: i2c@e1000000 { + status = "disabled"; + compatible = "snps,designware-i2c"; + reg = <0 0xe1000000 0 0x1000>; + interrupts = <0 357 4>; + clocks = <&uartspiclk_100mhz>; + }; + + serial0: serial@e1010000 { + compatible = "arm,pl011", "arm,primecell"; + reg = <0 0xe1010000 0 0x1000>; + interrupts = <0 328 4>; + clocks = <&uartspiclk_100mhz>, <&uartspiclk_100mhz>; + clock-names = "uartclk", "apb_pclk"; + }; + + spi0: ssp@e1020000 { + status = "disabled"; + compatible = "arm,pl022", "arm,primecell"; + #gpio-cells = <2>; + reg = <0 0xe1020000 0 0x1000>; + spi-controller; + interrupts = <0 330 4>; + clocks = <&uartspiclk_100mhz>; + clock-names = "apb_pclk"; + }; + + spi1: ssp@e1030000 { + status = "disabled"; + compatible = "arm,pl022", "arm,primecell"; + #gpio-cells = <2>; + reg = <0 0xe1030000 0 0x1000>; + spi-controller; + interrupts = <0 329 4>; + clocks = <&uartspiclk_100mhz>; + clock-names = "apb_pclk"; + num-cs = <1>; + #address-cells = <1>; + #size-cells = <0>; + }; + + gpio0: gpio@e1040000 { + status = "disabled"; + compatible = "arm,pl061", "arm,primecell"; + #gpio-cells = <2>; + reg = <0 0xe1040000 0 0x1000>; + gpio-controller; + interrupts = <0 359 4>; + interrupt-controller; + #interrupt-cells = <2>; + clocks = <&uartspiclk_100mhz>; + clock-names = "apb_pclk"; + }; + + gpio1: gpio@e1050000 { + status = "disabled"; + compatible = "arm,pl061", "arm,primecell"; + #gpio-cells = <2>; + reg = <0 0xe1050000 0 0x1000>; + gpio-controller; + interrupts = <0 358 4>; + clocks = <&uartspiclk_100mhz>; + clock-names = "apb_pclk"; + }; + + ccp0: ccp@e0100000 { + status = "disabled"; + compatible = "amd,ccp-seattle-v1a"; + reg = <0 0xe0100000 0 0x10000>; + interrupts = <0 3 4>; + dma-coherent; + }; + + pcie0: pcie@f0000000 { + compatible = "pci-host-ecam-generic"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + device_type = "pci"; + bus-range = <0 0x7f>; + msi-parent = <&v2m0>; + reg = <0 0xf0000000 0 0x10000000>; + + interrupt-map-mask = <0xf800 0x0 0x0 0x7>; + interrupt-map = + <0x1000 0x0 0x0 0x1 &gic0 0x0 0x0 0x0 0x120 0x1>, + <0x1000 0x0 0x0 0x2 &gic0 0x0 0x0 0x0 0x121 0x1>, + <0x1000 0x0 0x0 0x3 &gic0 0x0 0x0 0x0 0x122 0x1>, + <0x1000 0x0 0x0 0x4 &gic0 0x0 0x0 0x0 0x123 0x1>; + + dma-coherent; + dma-ranges = <0x43000000 0x80 0x0 0x80 0x0 0x7f 0xffffffff>; + ranges = + /* I/O Memory (size=64K) */ + <0x01000000 0x00 0x00000000 0x00 0xefff0000 0x00 0x00010000>, + /* 32-bit MMIO (size=2G) */ + <0x02000000 0x00 0x40000000 0x00 0x40000000 0x00 0x80000000>, + /* 64-bit MMIO (size= 124G) */ + <0x03000000 0x01 0x00000000 0x01 0x00000000 0x7f 0x00000000>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apm/Makefile b/arch/arm64/boot/dts/apm/Makefile new file mode 100644 index 000000000000..a2afabbc1717 --- /dev/null +++ b/arch/arm64/boot/dts/apm/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_XGENE) += apm-mustang.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/apm-mustang.dts b/arch/arm64/boot/dts/apm/apm-mustang.dts index 2e25de0800b9..83578e766b94 100644 --- a/arch/arm64/boot/dts/apm-mustang.dts +++ b/arch/arm64/boot/dts/apm/apm-mustang.dts @@ -45,6 +45,10 @@ status = "ok"; }; +&sgenet1 { + status = "ok"; +}; + &xgenet { status = "ok"; }; diff --git a/arch/arm64/boot/dts/apm-storm.dtsi b/arch/arm64/boot/dts/apm/apm-storm.dtsi index f1ad9c2ab2e9..d8f3a1c65ecd 100644 --- a/arch/arm64/boot/dts/apm-storm.dtsi +++ b/arch/arm64/boot/dts/apm/apm-storm.dtsi @@ -102,6 +102,7 @@ #address-cells = <2>; #size-cells = <2>; ranges; + dma-ranges = <0x0 0x0 0x0 0x0 0x400 0x0>; clocks { #address-cells = <2>; @@ -186,6 +187,16 @@ clock-output-names = "sge0clk"; }; + sge1clk: sge1clk@1f21c000 { + compatible = "apm,xgene-device-clock"; + #clock-cells = <1>; + clocks = <&socplldiv2 0>; + reg = <0x0 0x1f21c000 0x0 0x1000>; + reg-names = "csr-reg"; + csr-mask = <0xc>; + clock-output-names = "sge1clk"; + }; + xge0clk: xge0clk@1f61c000 { compatible = "apm,xgene-device-clock"; #clock-cells = <1>; @@ -352,6 +363,37 @@ reg-names = "csr-reg"; clock-output-names = "pcie4clk"; }; + + dmaclk: dmaclk@1f27c000 { + compatible = "apm,xgene-device-clock"; + #clock-cells = <1>; + clocks = <&socplldiv2 0>; + reg = <0x0 0x1f27c000 0x0 0x1000>; + reg-names = "csr-reg"; + clock-output-names = "dmaclk"; + }; + }; + + msi: msi@79000000 { + compatible = "apm,xgene1-msi"; + msi-controller; + reg = <0x00 0x79000000 0x0 0x900000>; + interrupts = < 0x0 0x10 0x4 + 0x0 0x11 0x4 + 0x0 0x12 0x4 + 0x0 0x13 0x4 + 0x0 0x14 0x4 + 0x0 0x15 0x4 + 0x0 0x16 0x4 + 0x0 0x17 0x4 + 0x0 0x18 0x4 + 0x0 0x19 0x4 + 0x0 0x1a 0x4 + 0x0 0x1b 0x4 + 0x0 0x1c 0x4 + 0x0 0x1d 0x4 + 0x0 0x1e 0x4 + 0x0 0x1f 0x4>; }; pcie0: pcie@1f2b0000 { @@ -375,6 +417,7 @@ 0x0 0x0 0x0 0x4 &gic 0x0 0xc5 0x1>; dma-coherent; clocks = <&pcie0clk 0>; + msi-parent = <&msi>; }; pcie1: pcie@1f2c0000 { @@ -398,6 +441,7 @@ 0x0 0x0 0x0 0x4 &gic 0x0 0xcb 0x1>; dma-coherent; clocks = <&pcie1clk 0>; + msi-parent = <&msi>; }; pcie2: pcie@1f2d0000 { @@ -421,6 +465,7 @@ 0x0 0x0 0x0 0x4 &gic 0x0 0xd1 0x1>; dma-coherent; clocks = <&pcie2clk 0>; + msi-parent = <&msi>; }; pcie3: pcie@1f500000 { @@ -444,6 +489,7 @@ 0x0 0x0 0x0 0x4 &gic 0x0 0xd7 0x1>; dma-coherent; clocks = <&pcie3clk 0>; + msi-parent = <&msi>; }; pcie4: pcie@1f510000 { @@ -467,6 +513,7 @@ 0x0 0x0 0x0 0x4 &gic 0x0 0xdd 0x1>; dma-coherent; clocks = <&pcie4clk 0>; + msi-parent = <&msi>; }; serial0: serial@1c020000 { @@ -622,27 +669,45 @@ }; sgenet0: ethernet@1f210000 { - compatible = "apm,xgene-enet"; + compatible = "apm,xgene1-sgenet"; status = "disabled"; reg = <0x0 0x1f210000 0x0 0xd100>, <0x0 0x1f200000 0x0 0Xc300>, <0x0 0x1B000000 0x0 0X200>; reg-names = "enet_csr", "ring_csr", "ring_cmd"; - interrupts = <0x0 0xA0 0x4>; + interrupts = <0x0 0xA0 0x4>, + <0x0 0xA1 0x4>; dma-coherent; clocks = <&sge0clk 0>; local-mac-address = [00 00 00 00 00 00]; phy-connection-type = "sgmii"; }; + sgenet1: ethernet@1f210030 { + compatible = "apm,xgene1-sgenet"; + status = "disabled"; + reg = <0x0 0x1f210030 0x0 0xd100>, + <0x0 0x1f200000 0x0 0Xc300>, + <0x0 0x1B000000 0x0 0X8000>; + reg-names = "enet_csr", "ring_csr", "ring_cmd"; + interrupts = <0x0 0xAC 0x4>, + <0x0 0xAD 0x4>; + port-id = <1>; + dma-coherent; + clocks = <&sge1clk 0>; + local-mac-address = [00 00 00 00 00 00]; + phy-connection-type = "sgmii"; + }; + xgenet: ethernet@1f610000 { - compatible = "apm,xgene-enet"; + compatible = "apm,xgene1-xgenet"; status = "disabled"; reg = <0x0 0x1f610000 0x0 0xd100>, <0x0 0x1f600000 0x0 0Xc300>, <0x0 0x18000000 0x0 0X200>; reg-names = "enet_csr", "ring_csr", "ring_cmd"; - interrupts = <0x0 0x60 0x4>; + interrupts = <0x0 0x60 0x4>, + <0x0 0x61 0x4>; dma-coherent; clocks = <&xge0clk 0>; /* mac address will be overwritten by the bootloader */ @@ -656,5 +721,21 @@ interrupts = <0x0 0x41 0x4>; clocks = <&rngpkaclk 0>; }; + + dma: dma@1f270000 { + compatible = "apm,xgene-storm-dma"; + device_type = "dma"; + reg = <0x0 0x1f270000 0x0 0x10000>, + <0x0 0x1f200000 0x0 0x10000>, + <0x0 0x1b008000 0x0 0x2000>, + <0x0 0x1054a000 0x0 0x100>; + interrupts = <0x0 0x82 0x4>, + <0x0 0xb8 0x4>, + <0x0 0xb9 0x4>, + <0x0 0xba 0x4>, + <0x0 0xbb 0x4>; + dma-coherent; + clocks = <&dmaclk 0>; + }; }; }; diff --git a/arch/arm64/boot/dts/arm/Makefile b/arch/arm64/boot/dts/arm/Makefile new file mode 100644 index 000000000000..301a0dada1fe --- /dev/null +++ b/arch/arm64/boot/dts/arm/Makefile @@ -0,0 +1,7 @@ +dtb-$(CONFIG_ARCH_VEXPRESS) += foundation-v8.dtb +dtb-$(CONFIG_ARCH_VEXPRESS) += juno.dtb +dtb-$(CONFIG_ARCH_VEXPRESS) += rtsm_ve-aemv8a.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/foundation-v8.dts b/arch/arm64/boot/dts/arm/foundation-v8.dts index 4a060906809d..4eac8dcea423 100644 --- a/arch/arm64/boot/dts/foundation-v8.dts +++ b/arch/arm64/boot/dts/arm/foundation-v8.dts @@ -34,6 +34,7 @@ reg = <0x0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@1 { device_type = "cpu"; @@ -41,6 +42,7 @@ reg = <0x0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@2 { device_type = "cpu"; @@ -48,6 +50,7 @@ reg = <0x0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@3 { device_type = "cpu"; @@ -55,6 +58,11 @@ reg = <0x0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; + }; + + L2_0: l2-cache0 { + compatible = "cache"; }; }; @@ -78,10 +86,10 @@ timer { compatible = "arm,armv8-timer"; - interrupts = <1 13 0xff01>, - <1 14 0xff01>, - <1 11 0xff01>, - <1 10 0xff01>; + interrupts = <1 13 0xf08>, + <1 14 0xf08>, + <1 11 0xf08>, + <1 10 0xf08>; clock-frequency = <100000000>; }; diff --git a/arch/arm64/boot/dts/arm/juno-clocks.dtsi b/arch/arm64/boot/dts/arm/juno-clocks.dtsi new file mode 100644 index 000000000000..c9b89efe0f56 --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno-clocks.dtsi @@ -0,0 +1,44 @@ +/* + * ARM Juno Platform clocks + * + * Copyright (c) 2013-2014 ARM Ltd + * + * This file is licensed under a dual GPLv2 or BSD license. + * + */ + + /* SoC fixed clocks */ + soc_uartclk: refclk7273800hz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <7273800>; + clock-output-names = "juno:uartclk"; + }; + + soc_usb48mhz: clk48mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <48000000>; + clock-output-names = "clk48mhz"; + }; + + soc_smc50mhz: clk50mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <50000000>; + clock-output-names = "smc_clk"; + }; + + soc_refclk100mhz: refclk100mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <100000000>; + clock-output-names = "apb_pclk"; + }; + + soc_faxiclk: refclk533mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <533000000>; + clock-output-names = "faxi_clk"; + }; diff --git a/arch/arm64/boot/dts/arm/juno-motherboard.dtsi b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi new file mode 100644 index 000000000000..351c95bda89e --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno-motherboard.dtsi @@ -0,0 +1,152 @@ +/* + * ARM Juno Platform motherboard peripherals + * + * Copyright (c) 2013-2014 ARM Ltd + * + * This file is licensed under a dual GPLv2 or BSD license. + * + */ + + mb_clk24mhz: clk24mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "juno_mb:clk24mhz"; + }; + + mb_clk25mhz: clk25mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <25000000>; + clock-output-names = "juno_mb:clk25mhz"; + }; + + v2m_refclk1mhz: refclk1mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <1000000>; + clock-output-names = "juno_mb:refclk1mhz"; + }; + + v2m_refclk32khz: refclk32khz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <32768>; + clock-output-names = "juno_mb:refclk32khz"; + }; + + motherboard { + compatible = "arm,vexpress,v2p-p1", "simple-bus"; + #address-cells = <2>; /* SMB chipselect number and offset */ + #size-cells = <1>; + #interrupt-cells = <1>; + ranges; + model = "V2M-Juno"; + arm,hbi = <0x252>; + arm,vexpress,site = <0>; + arm,v2m-memory-map = "rs1"; + + mb_fixed_3v3: fixedregulator@0 { + compatible = "regulator-fixed"; + regulator-name = "MCC_SB_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + regulator-always-on; + }; + + ethernet@2,00000000 { + compatible = "smsc,lan9118", "smsc,lan9115"; + reg = <2 0x00000000 0x10000>; + interrupts = <3>; + phy-mode = "mii"; + reg-io-width = <4>; + smsc,irq-active-high; + smsc,irq-push-pull; + clocks = <&mb_clk25mhz>; + vdd33a-supply = <&mb_fixed_3v3>; + vddvario-supply = <&mb_fixed_3v3>; + }; + + usb@5,00000000 { + compatible = "nxp,usb-isp1763"; + reg = <5 0x00000000 0x20000>; + bus-width = <16>; + interrupts = <4>; + }; + + iofpga@3,00000000 { + compatible = "arm,amba-bus", "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 3 0 0x200000>; + + v2m_sysctl: sysctl@020000 { + compatible = "arm,sp810", "arm,primecell"; + reg = <0x020000 0x1000>; + clocks = <&v2m_refclk32khz>, <&v2m_refclk1mhz>, <&mb_clk24mhz>; + clock-names = "refclk", "timclk", "apb_pclk"; + #clock-cells = <1>; + clock-output-names = "timerclken0", "timerclken1", "timerclken2", "timerclken3"; + }; + + mmci@050000 { + compatible = "arm,pl180", "arm,primecell"; + reg = <0x050000 0x1000>; + interrupts = <5>; + /* cd-gpios = <&v2m_mmc_gpios 0 0>; + wp-gpios = <&v2m_mmc_gpios 1 0>; */ + max-frequency = <12000000>; + vmmc-supply = <&mb_fixed_3v3>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "mclk", "apb_pclk"; + }; + + kmi@060000 { + compatible = "arm,pl050", "arm,primecell"; + reg = <0x060000 0x1000>; + interrupts = <8>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "KMIREFCLK", "apb_pclk"; + }; + + kmi@070000 { + compatible = "arm,pl050", "arm,primecell"; + reg = <0x070000 0x1000>; + interrupts = <8>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "KMIREFCLK", "apb_pclk"; + }; + + wdt@0f0000 { + compatible = "arm,sp805", "arm,primecell"; + reg = <0x0f0000 0x10000>; + interrupts = <7>; + clocks = <&mb_clk24mhz>, <&soc_smc50mhz>; + clock-names = "wdogclk", "apb_pclk"; + }; + + v2m_timer01: timer@110000 { + compatible = "arm,sp804", "arm,primecell"; + reg = <0x110000 0x10000>; + interrupts = <9>; + clocks = <&v2m_sysctl 0>, <&v2m_sysctl 1>, <&mb_clk24mhz>; + clock-names = "timclken1", "timclken2", "apb_pclk"; + }; + + v2m_timer23: timer@120000 { + compatible = "arm,sp804", "arm,primecell"; + reg = <0x120000 0x10000>; + interrupts = <9>; + clocks = <&v2m_sysctl 2>, <&v2m_sysctl 3>, <&mb_clk24mhz>; + clock-names = "timclken1", "timclken2", "apb_pclk"; + }; + + rtc@170000 { + compatible = "arm,pl031", "arm,primecell"; + reg = <0x170000 0x10000>; + interrupts = <0>; + clocks = <&soc_smc50mhz>; + clock-names = "apb_pclk"; + }; + }; + }; diff --git a/arch/arm64/boot/dts/arm/juno.dts b/arch/arm64/boot/dts/arm/juno.dts new file mode 100644 index 000000000000..5e9110a3353d --- /dev/null +++ b/arch/arm64/boot/dts/arm/juno.dts @@ -0,0 +1,238 @@ +/* + * ARM Ltd. Juno Platform + * + * Copyright (c) 2013-2014 ARM Ltd. + * + * This file is licensed under a dual GPLv2 or BSD license. + */ + +/dts-v1/; + +#include <dt-bindings/interrupt-controller/arm-gic.h> + +/ { + model = "ARM Juno development board (r0)"; + compatible = "arm,juno", "arm,vexpress"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + serial0 = &soc_uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + A57_0: cpu@0 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x0>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + }; + + A57_1: cpu@1 { + compatible = "arm,cortex-a57","arm,armv8"; + reg = <0x0 0x1>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A57_L2>; + }; + + A53_0: cpu@100 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x100>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_1: cpu@101 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x101>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_2: cpu@102 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x102>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A53_3: cpu@103 { + compatible = "arm,cortex-a53","arm,armv8"; + reg = <0x0 0x103>; + device_type = "cpu"; + enable-method = "psci"; + next-level-cache = <&A53_L2>; + }; + + A57_L2: l2-cache0 { + compatible = "cache"; + }; + + A53_L2: l2-cache1 { + compatible = "cache"; + }; + }; + + memory@80000000 { + device_type = "memory"; + /* last 16MB of the first memory area is reserved for secure world use by firmware */ + reg = <0x00000000 0x80000000 0x0 0x7f000000>, + <0x00000008 0x80000000 0x1 0x80000000>; + }; + + gic: interrupt-controller@2c001000 { + compatible = "arm,gic-400", "arm,cortex-a15-gic"; + reg = <0x0 0x2c010000 0 0x1000>, + <0x0 0x2c02f000 0 0x2000>, + <0x0 0x2c04f000 0 0x2000>, + <0x0 0x2c06f000 0 0x2000>; + #address-cells = <0>; + #interrupt-cells = <3>; + interrupt-controller; + interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_HIGH)>; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(6) | IRQ_TYPE_LEVEL_LOW)>; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = <GIC_SPI 02 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 06 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 18 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>; + interrupt-affinity = <&A57_0>, + <&A57_1>, + <&A53_0>, + <&A53_1>, + <&A53_2>, + <&A53_3>; + }; + + /include/ "juno-clocks.dtsi" + + dma@7ff00000 { + compatible = "arm,pl330", "arm,primecell"; + reg = <0x0 0x7ff00000 0 0x1000>; + #dma-cells = <1>; + #dma-channels = <8>; + #dma-requests = <32>; + interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_faxiclk>; + clock-names = "apb_pclk"; + }; + + soc_uart0: uart@7ff80000 { + compatible = "arm,pl011", "arm,primecell"; + reg = <0x0 0x7ff80000 0x0 0x1000>; + interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_uartclk>, <&soc_refclk100mhz>; + clock-names = "uartclk", "apb_pclk"; + }; + + i2c@7ffa0000 { + compatible = "snps,designware-i2c"; + reg = <0x0 0x7ffa0000 0x0 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>; + clock-frequency = <400000>; + i2c-sda-hold-time-ns = <500>; + clocks = <&soc_smc50mhz>; + + dvi0: dvi-transmitter@70 { + compatible = "nxp,tda998x"; + reg = <0x70>; + }; + + dvi1: dvi-transmitter@71 { + compatible = "nxp,tda998x"; + reg = <0x71>; + }; + }; + + ohci@7ffb0000 { + compatible = "generic-ohci"; + reg = <0x0 0x7ffb0000 0x0 0x10000>; + interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_usb48mhz>; + }; + + ehci@7ffc0000 { + compatible = "generic-ehci"; + reg = <0x0 0x7ffc0000 0x0 0x10000>; + interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_usb48mhz>; + }; + + memory-controller@7ffd0000 { + compatible = "arm,pl354", "arm,primecell"; + reg = <0 0x7ffd0000 0 0x1000>; + interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&soc_smc50mhz>; + clock-names = "apb_pclk"; + }; + + smb { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <1>; + ranges = <0 0 0 0x08000000 0x04000000>, + <1 0 0 0x14000000 0x04000000>, + <2 0 0 0x18000000 0x04000000>, + <3 0 0 0x1c000000 0x04000000>, + <4 0 0 0x0c000000 0x04000000>, + <5 0 0 0x10000000 0x04000000>; + + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 15>; + interrupt-map = <0 0 0 &gic 0 68 IRQ_TYPE_LEVEL_HIGH>, + <0 0 1 &gic 0 69 IRQ_TYPE_LEVEL_HIGH>, + <0 0 2 &gic 0 70 IRQ_TYPE_LEVEL_HIGH>, + <0 0 3 &gic 0 160 IRQ_TYPE_LEVEL_HIGH>, + <0 0 4 &gic 0 161 IRQ_TYPE_LEVEL_HIGH>, + <0 0 5 &gic 0 162 IRQ_TYPE_LEVEL_HIGH>, + <0 0 6 &gic 0 163 IRQ_TYPE_LEVEL_HIGH>, + <0 0 7 &gic 0 164 IRQ_TYPE_LEVEL_HIGH>, + <0 0 8 &gic 0 165 IRQ_TYPE_LEVEL_HIGH>, + <0 0 9 &gic 0 166 IRQ_TYPE_LEVEL_HIGH>, + <0 0 10 &gic 0 167 IRQ_TYPE_LEVEL_HIGH>, + <0 0 11 &gic 0 168 IRQ_TYPE_LEVEL_HIGH>, + <0 0 12 &gic 0 169 IRQ_TYPE_LEVEL_HIGH>; + + /include/ "juno-motherboard.dtsi" + }; +}; diff --git a/arch/arm64/boot/dts/rtsm_ve-aemv8a.dts b/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts index 572005ea2217..20addabbd127 100644 --- a/arch/arm64/boot/dts/rtsm_ve-aemv8a.dts +++ b/arch/arm64/boot/dts/arm/rtsm_ve-aemv8a.dts @@ -37,6 +37,7 @@ reg = <0x0 0x0>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@1 { device_type = "cpu"; @@ -44,6 +45,7 @@ reg = <0x0 0x1>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@2 { device_type = "cpu"; @@ -51,6 +53,7 @@ reg = <0x0 0x2>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; }; cpu@3 { device_type = "cpu"; @@ -58,6 +61,11 @@ reg = <0x0 0x3>; enable-method = "spin-table"; cpu-release-addr = <0x0 0x8000fff8>; + next-level-cache = <&L2_0>; + }; + + L2_0: l2-cache0 { + compatible = "cache"; }; }; @@ -81,10 +89,10 @@ timer { compatible = "arm,armv8-timer"; - interrupts = <1 13 0xff01>, - <1 14 0xff01>, - <1 11 0xff01>, - <1 10 0xff01>; + interrupts = <1 13 0xf08>, + <1 14 0xf08>, + <1 11 0xf08>, + <1 10 0xf08>; clock-frequency = <100000000>; }; diff --git a/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi index c46cbb29f3c6..c46cbb29f3c6 100644 --- a/arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi +++ b/arch/arm64/boot/dts/arm/rtsm_ve-motherboard.dtsi diff --git a/arch/arm64/boot/dts/cavium/Makefile b/arch/arm64/boot/dts/cavium/Makefile new file mode 100644 index 000000000000..e34f89ddabb2 --- /dev/null +++ b/arch/arm64/boot/dts/cavium/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_THUNDER) += thunder-88xx.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/thunder-88xx.dts b/arch/arm64/boot/dts/cavium/thunder-88xx.dts index 800ba65991f7..800ba65991f7 100644 --- a/arch/arm64/boot/dts/thunder-88xx.dts +++ b/arch/arm64/boot/dts/cavium/thunder-88xx.dts diff --git a/arch/arm64/boot/dts/thunder-88xx.dtsi b/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi index d8c0bdc51882..d8c0bdc51882 100644 --- a/arch/arm64/boot/dts/thunder-88xx.dtsi +++ b/arch/arm64/boot/dts/cavium/thunder-88xx.dtsi diff --git a/arch/arm64/boot/dts/exynos/Makefile b/arch/arm64/boot/dts/exynos/Makefile new file mode 100644 index 000000000000..20310e5b6d6f --- /dev/null +++ b/arch/arm64/boot/dts/exynos/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_EXYNOS7) += exynos7-espresso.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/exynos/exynos7-espresso.dts b/arch/arm64/boot/dts/exynos/exynos7-espresso.dts new file mode 100644 index 000000000000..5424cc450f72 --- /dev/null +++ b/arch/arm64/boot/dts/exynos/exynos7-espresso.dts @@ -0,0 +1,84 @@ +/* + * SAMSUNG Exynos7 Espresso board device tree source + * + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. +*/ + +/dts-v1/; +#include "exynos7.dtsi" + +/ { + model = "Samsung Exynos7 Espresso board based on EXYNOS7"; + compatible = "samsung,exynos7-espresso", "samsung,exynos7"; + + aliases { + serial0 = &serial_2; + mshc0 = &mmc_0; + mshc2 = &mmc_2; + }; + + chosen { + linux,stdout-path = &serial_2; + }; + + memory@40000000 { + device_type = "memory"; + reg = <0x0 0x40000000 0x0 0xC0000000>; + }; +}; + +&fin_pll { + clock-frequency = <24000000>; +}; + +&serial_2 { + status = "okay"; +}; + +&rtc { + status = "okay"; +}; + +&watchdog { + status = "okay"; +}; + +&adc { + status = "okay"; +}; + +&mmc_0 { + status = "okay"; + num-slots = <1>; + broken-cd; + cap-mmc-highspeed; + non-removable; + card-detect-delay = <200>; + clock-frequency = <800000000>; + samsung,dw-mshc-ciu-div = <3>; + samsung,dw-mshc-sdr-timing = <0 4>; + samsung,dw-mshc-ddr-timing = <0 2>; + pinctrl-names = "default"; + pinctrl-0 = <&sd0_clk &sd0_cmd &sd0_qrdy &sd0_bus1 &sd0_bus4 &sd0_bus8>; + bus-width = <8>; +}; + +&mmc_2 { + status = "okay"; + num-slots = <1>; + cap-sd-highspeed; + card-detect-delay = <200>; + clock-frequency = <400000000>; + samsung,dw-mshc-ciu-div = <3>; + samsung,dw-mshc-sdr-timing = <2 3>; + samsung,dw-mshc-ddr-timing = <1 2>; + pinctrl-names = "default"; + pinctrl-0 = <&sd2_clk &sd2_cmd &sd2_cd &sd2_bus1 &sd2_bus4>; + bus-width = <4>; + disable-wp; +}; diff --git a/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi new file mode 100644 index 000000000000..2eef4a279131 --- /dev/null +++ b/arch/arm64/boot/dts/exynos/exynos7-pinctrl.dtsi @@ -0,0 +1,588 @@ +/* + * Samsung's Exynos7 SoC pin-mux and pin-config device tree source + * + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * Samsung's Exynos7 SoC pin-mux and pin-config options are listed as + * device tree nodes in this file. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. +*/ + +&pinctrl_alive { + gpa0: gpa0 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + interrupt-parent = <&gic>; + #interrupt-cells = <2>; + interrupts = <0 0 0>, <0 1 0>, <0 2 0>, <0 3 0>, + <0 4 0>, <0 5 0>, <0 6 0>, <0 7 0>; + }; + + gpa1: gpa1 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + interrupt-parent = <&gic>; + #interrupt-cells = <2>; + interrupts = <0 8 0>, <0 9 0>, <0 10 0>, <0 11 0>, + <0 12 0>, <0 13 0>, <0 14 0>, <0 15 0>; + }; + + gpa2: gpa2 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpa3: gpa3 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; +}; + +&pinctrl_bus0 { + gpb0: gpb0 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpc0: gpc0 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpc1: gpc1 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpc2: gpc2 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpc3: gpc3 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd0: gpd0 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd1: gpd1 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd2: gpd2 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd4: gpd4 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd5: gpd5 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd6: gpd6 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd7: gpd7 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpd8: gpd8 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpg0: gpg0 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpg3: gpg3 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + hs_i2c10_bus: hs-i2c10-bus { + samsung,pins = "gpb0-1", "gpb0-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + hs_i2c11_bus: hs-i2c11-bus { + samsung,pins = "gpb0-3", "gpb0-2"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + hs_i2c2_bus: hs-i2c2-bus { + samsung,pins = "gpd0-3", "gpd0-2"; + samsung,pin-function = <3>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + uart0_data: uart0-data { + samsung,pins = "gpd0-0", "gpd0-1"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + uart0_fctl: uart0-fctl { + samsung,pins = "gpd0-2", "gpd0-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + uart2_data: uart2-data { + samsung,pins = "gpd1-4", "gpd1-5"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + hs_i2c3_bus: hs-i2c3-bus { + samsung,pins = "gpd1-3", "gpd1-2"; + samsung,pin-function = <3>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + uart1_data: uart1-data { + samsung,pins = "gpd1-0", "gpd1-1"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + uart1_fctl: uart1-fctl { + samsung,pins = "gpd1-2", "gpd1-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + hs_i2c0_bus: hs-i2c0-bus { + samsung,pins = "gpd2-1", "gpd2-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + hs_i2c1_bus: hs-i2c1-bus { + samsung,pins = "gpd2-3", "gpd2-2"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + hs_i2c9_bus: hs-i2c9-bus { + samsung,pins = "gpd2-7", "gpd2-6"; + samsung,pin-function = <3>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + pwm0_out: pwm0-out { + samsung,pins = "gpd2-4"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + pwm1_out: pwm1-out { + samsung,pins = "gpd2-5"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + pwm2_out: pwm2-out { + samsung,pins = "gpd2-6"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + pwm3_out: pwm3-out { + samsung,pins = "gpd2-7"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + hs_i2c8_bus: hs-i2c8-bus { + samsung,pins = "gpd5-3", "gpd5-2"; + samsung,pin-function = <3>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + uart3_data: uart3-data { + samsung,pins = "gpd5-0", "gpd5-1"; + samsung,pin-function = <3>; + samsung,pin-pud = <0>; + samsung,pin-drv = <0>; + }; + + spi2_bus: spi2-bus { + samsung,pins = "gpd5-0", "gpd5-1", "gpd5-2", "gpd5-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + spi1_bus: spi1-bus { + samsung,pins = "gpd6-2", "gpd6-3", "gpd6-4", "gpd6-5"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + spi0_bus: spi0-bus { + samsung,pins = "gpd8-0", "gpd8-1", "gpd6-0", "gpd6-1"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + hs_i2c4_bus: hs-i2c4-bus { + samsung,pins = "gpg3-1", "gpg3-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; + + hs_i2c5_bus: hs-i2c5-bus { + samsung,pins = "gpg3-3", "gpg3-2"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; +}; + +&pinctrl_nfc { + gpj0: gpj0 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + hs_i2c6_bus: hs-i2c6-bus { + samsung,pins = "gpj0-1", "gpj0-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; +}; + +&pinctrl_touch { + gpj1: gpj1 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + hs_i2c7_bus: hs-i2c7-bus { + samsung,pins = "gpj1-1", "gpj1-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; +}; + +&pinctrl_ff { + gpg4: gpg4 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + spi3_bus: spi3-bus { + samsung,pins = "gpg4-0", "gpg4-1", "gpg4-2", "gpg4-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; +}; + +&pinctrl_ese { + gpv7: gpv7 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + spi4_bus: spi4-bus { + samsung,pins = "gpv7-0", "gpv7-1", "gpv7-2", "gpv7-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <0>; + }; +}; + +&pinctrl_fsys0 { + gpr4: gpr4 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + sd2_clk: sd2-clk { + samsung,pins = "gpr4-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <3>; + }; + + sd2_cmd: sd2-cmd { + samsung,pins = "gpr4-1"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <3>; + }; + + sd2_cd: sd2-cd { + samsung,pins = "gpr4-2"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; + }; + + sd2_bus1: sd2-bus-width1 { + samsung,pins = "gpr4-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; + }; + + sd2_bus4: sd2-bus-width4 { + samsung,pins = "gpr4-4", "gpr4-5", "gpr4-6"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; + }; +}; + +&pinctrl_fsys1 { + gpr0: gpr0 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpr1: gpr1 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpr2: gpr2 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + gpr3: gpr3 { + gpio-controller; + #gpio-cells = <2>; + + interrupt-controller; + #interrupt-cells = <2>; + }; + + sd0_clk: sd0-clk { + samsung,pins = "gpr0-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <3>; + }; + + sd0_cmd: sd0-cmd { + samsung,pins = "gpr0-1"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; + }; + + sd0_ds: sd0-ds { + samsung,pins = "gpr0-2"; + samsung,pin-function = <2>; + samsung,pin-pud = <1>; + samsung,pin-drv = <3>; + }; + + sd0_qrdy: sd0-qrdy { + samsung,pins = "gpr0-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <1>; + samsung,pin-drv = <3>; + }; + + sd0_bus1: sd0-bus-width1 { + samsung,pins = "gpr1-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; + }; + + sd0_bus4: sd0-bus-width4 { + samsung,pins = "gpr1-1", "gpr1-2", "gpr1-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; + }; + + sd0_bus8: sd0-bus-width8 { + samsung,pins = "gpr1-4", "gpr1-5", "gpr1-6", "gpr1-7"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <3>; + }; + + sd1_clk: sd1-clk { + samsung,pins = "gpr2-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <2>; + }; + + sd1_cmd: sd1-cmd { + samsung,pins = "gpr2-1"; + samsung,pin-function = <2>; + samsung,pin-pud = <0>; + samsung,pin-drv = <2>; + }; + + sd1_ds: sd1-ds { + samsung,pins = "gpr2-2"; + samsung,pin-function = <2>; + samsung,pin-pud = <1>; + samsung,pin-drv = <6>; + }; + + sd1_qrdy: sd1-qrdy { + samsung,pins = "gpr2-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <1>; + samsung,pin-drv = <6>; + }; + + sd1_int: sd1-int { + samsung,pins = "gpr2-4"; + samsung,pin-function = <2>; + samsung,pin-pud = <1>; + samsung,pin-drv = <6>; + }; + + sd1_bus1: sd1-bus-width1 { + samsung,pins = "gpr3-0"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <2>; + }; + + sd1_bus4: sd1-bus-width4 { + samsung,pins = "gpr3-1", "gpr3-2", "gpr3-3"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <2>; + }; + + sd1_bus8: sd1-bus-width8 { + samsung,pins = "gpr3-4", "gpr3-5", "gpr3-6", "gpr3-7"; + samsung,pin-function = <2>; + samsung,pin-pud = <3>; + samsung,pin-drv = <2>; + }; +}; diff --git a/arch/arm64/boot/dts/exynos/exynos7.dtsi b/arch/arm64/boot/dts/exynos/exynos7.dtsi new file mode 100644 index 000000000000..d7a37c3a6b52 --- /dev/null +++ b/arch/arm64/boot/dts/exynos/exynos7.dtsi @@ -0,0 +1,530 @@ +/* + * SAMSUNG EXYNOS7 SoC device tree source + * + * Copyright (c) 2014 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <dt-bindings/clock/exynos7-clk.h> + +/ { + compatible = "samsung,exynos7"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + aliases { + pinctrl0 = &pinctrl_alive; + pinctrl1 = &pinctrl_bus0; + pinctrl2 = &pinctrl_nfc; + pinctrl3 = &pinctrl_touch; + pinctrl4 = &pinctrl_ff; + pinctrl5 = &pinctrl_ese; + pinctrl6 = &pinctrl_fsys0; + pinctrl7 = &pinctrl_fsys1; + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0x0>; + enable-method = "psci"; + }; + + cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0x1>; + enable-method = "psci"; + }; + + cpu@2 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0x2>; + enable-method = "psci"; + }; + + cpu@3 { + device_type = "cpu"; + compatible = "arm,cortex-a57", "arm,armv8"; + reg = <0x3>; + enable-method = "psci"; + }; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + soc: soc { + compatible = "simple-bus"; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0 0 0x18000000>; + + chipid@10000000 { + compatible = "samsung,exynos4210-chipid"; + reg = <0x10000000 0x100>; + }; + + fin_pll: xxti { + compatible = "fixed-clock"; + clock-output-names = "fin_pll"; + #clock-cells = <0>; + }; + + gic: interrupt-controller@11001000 { + compatible = "arm,gic-400"; + #interrupt-cells = <3>; + #address-cells = <0>; + interrupt-controller; + reg = <0x11001000 0x1000>, + <0x11002000 0x1000>, + <0x11004000 0x2000>, + <0x11006000 0x2000>; + }; + + clock_topc: clock-controller@10570000 { + compatible = "samsung,exynos7-clock-topc"; + reg = <0x10570000 0x10000>; + #clock-cells = <1>; + }; + + clock_top0: clock-controller@105d0000 { + compatible = "samsung,exynos7-clock-top0"; + reg = <0x105d0000 0xb000>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_topc DOUT_SCLK_BUS0_PLL>, + <&clock_topc DOUT_SCLK_BUS1_PLL>, + <&clock_topc DOUT_SCLK_CC_PLL>, + <&clock_topc DOUT_SCLK_MFC_PLL>; + clock-names = "fin_pll", "dout_sclk_bus0_pll", + "dout_sclk_bus1_pll", "dout_sclk_cc_pll", + "dout_sclk_mfc_pll"; + }; + + clock_top1: clock-controller@105e0000 { + compatible = "samsung,exynos7-clock-top1"; + reg = <0x105e0000 0xb000>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_topc DOUT_SCLK_BUS0_PLL>, + <&clock_topc DOUT_SCLK_BUS1_PLL>, + <&clock_topc DOUT_SCLK_CC_PLL>, + <&clock_topc DOUT_SCLK_MFC_PLL>; + clock-names = "fin_pll", "dout_sclk_bus0_pll", + "dout_sclk_bus1_pll", "dout_sclk_cc_pll", + "dout_sclk_mfc_pll"; + }; + + clock_ccore: clock-controller@105b0000 { + compatible = "samsung,exynos7-clock-ccore"; + reg = <0x105b0000 0xd00>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_topc DOUT_ACLK_CCORE_133>; + clock-names = "fin_pll", "dout_aclk_ccore_133"; + }; + + clock_peric0: clock-controller@13610000 { + compatible = "samsung,exynos7-clock-peric0"; + reg = <0x13610000 0xd00>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_top0 DOUT_ACLK_PERIC0>, + <&clock_top0 CLK_SCLK_UART0>; + clock-names = "fin_pll", "dout_aclk_peric0_66", + "sclk_uart0"; + }; + + clock_peric1: clock-controller@14c80000 { + compatible = "samsung,exynos7-clock-peric1"; + reg = <0x14c80000 0xd00>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_top0 DOUT_ACLK_PERIC1>, + <&clock_top0 CLK_SCLK_UART1>, + <&clock_top0 CLK_SCLK_UART2>, + <&clock_top0 CLK_SCLK_UART3>; + clock-names = "fin_pll", "dout_aclk_peric1_66", + "sclk_uart1", "sclk_uart2", "sclk_uart3"; + }; + + clock_peris: clock-controller@10040000 { + compatible = "samsung,exynos7-clock-peris"; + reg = <0x10040000 0xd00>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_topc DOUT_ACLK_PERIS>; + clock-names = "fin_pll", "dout_aclk_peris_66"; + }; + + clock_fsys0: clock-controller@10e90000 { + compatible = "samsung,exynos7-clock-fsys0"; + reg = <0x10e90000 0xd00>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_top1 DOUT_ACLK_FSYS0_200>, + <&clock_top1 DOUT_SCLK_MMC2>; + clock-names = "fin_pll", "dout_aclk_fsys0_200", + "dout_sclk_mmc2"; + }; + + clock_fsys1: clock-controller@156e0000 { + compatible = "samsung,exynos7-clock-fsys1"; + reg = <0x156e0000 0xd00>; + #clock-cells = <1>; + clocks = <&fin_pll>, <&clock_top1 DOUT_ACLK_FSYS1_200>, + <&clock_top1 DOUT_SCLK_MMC0>, + <&clock_top1 DOUT_SCLK_MMC1>; + clock-names = "fin_pll", "dout_aclk_fsys1_200", + "dout_sclk_mmc0", "dout_sclk_mmc1"; + }; + + serial_0: serial@13630000 { + compatible = "samsung,exynos4210-uart"; + reg = <0x13630000 0x100>; + interrupts = <0 440 0>; + clocks = <&clock_peric0 PCLK_UART0>, + <&clock_peric0 SCLK_UART0>; + clock-names = "uart", "clk_uart_baud0"; + status = "disabled"; + }; + + serial_1: serial@14c20000 { + compatible = "samsung,exynos4210-uart"; + reg = <0x14c20000 0x100>; + interrupts = <0 456 0>; + clocks = <&clock_peric1 PCLK_UART1>, + <&clock_peric1 SCLK_UART1>; + clock-names = "uart", "clk_uart_baud0"; + status = "disabled"; + }; + + serial_2: serial@14c30000 { + compatible = "samsung,exynos4210-uart"; + reg = <0x14c30000 0x100>; + interrupts = <0 457 0>; + clocks = <&clock_peric1 PCLK_UART2>, + <&clock_peric1 SCLK_UART2>; + clock-names = "uart", "clk_uart_baud0"; + status = "disabled"; + }; + + serial_3: serial@14c40000 { + compatible = "samsung,exynos4210-uart"; + reg = <0x14c40000 0x100>; + interrupts = <0 458 0>; + clocks = <&clock_peric1 PCLK_UART3>, + <&clock_peric1 SCLK_UART3>; + clock-names = "uart", "clk_uart_baud0"; + status = "disabled"; + }; + + pinctrl_alive: pinctrl@10580000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x10580000 0x1000>; + + wakeup-interrupt-controller { + compatible = "samsung,exynos7-wakeup-eint"; + interrupt-parent = <&gic>; + interrupts = <0 16 0>; + }; + }; + + pinctrl_bus0: pinctrl@13470000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x13470000 0x1000>; + interrupts = <0 383 0>; + }; + + pinctrl_nfc: pinctrl@14cd0000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x14cd0000 0x1000>; + interrupts = <0 473 0>; + }; + + pinctrl_touch: pinctrl@14ce0000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x14ce0000 0x1000>; + interrupts = <0 474 0>; + }; + + pinctrl_ff: pinctrl@14c90000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x14c90000 0x1000>; + interrupts = <0 475 0>; + }; + + pinctrl_ese: pinctrl@14ca0000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x14ca0000 0x1000>; + interrupts = <0 476 0>; + }; + + pinctrl_fsys0: pinctrl@10e60000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x10e60000 0x1000>; + interrupts = <0 221 0>; + }; + + pinctrl_fsys1: pinctrl@15690000 { + compatible = "samsung,exynos7-pinctrl"; + reg = <0x15690000 0x1000>; + interrupts = <0 203 0>; + }; + + hsi2c_0: hsi2c@13640000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x13640000 0x1000>; + interrupts = <0 441 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c0_bus>; + clocks = <&clock_peric0 PCLK_HSI2C0>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_1: hsi2c@13650000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x13650000 0x1000>; + interrupts = <0 442 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c1_bus>; + clocks = <&clock_peric0 PCLK_HSI2C1>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_2: hsi2c@14e60000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x14e60000 0x1000>; + interrupts = <0 459 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c2_bus>; + clocks = <&clock_peric1 PCLK_HSI2C2>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_3: hsi2c@14e70000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x14e70000 0x1000>; + interrupts = <0 460 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c3_bus>; + clocks = <&clock_peric1 PCLK_HSI2C3>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_4: hsi2c@13660000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x13660000 0x1000>; + interrupts = <0 443 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c4_bus>; + clocks = <&clock_peric0 PCLK_HSI2C4>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_5: hsi2c@13670000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x13670000 0x1000>; + interrupts = <0 444 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c5_bus>; + clocks = <&clock_peric0 PCLK_HSI2C5>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_6: hsi2c@14e00000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x14e00000 0x1000>; + interrupts = <0 461 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c6_bus>; + clocks = <&clock_peric1 PCLK_HSI2C6>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_7: hsi2c@13e10000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x13e10000 0x1000>; + interrupts = <0 462 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c7_bus>; + clocks = <&clock_peric1 PCLK_HSI2C7>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_8: hsi2c@14e20000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x14e20000 0x1000>; + interrupts = <0 463 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c8_bus>; + clocks = <&clock_peric1 PCLK_HSI2C8>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_9: hsi2c@13680000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x13680000 0x1000>; + interrupts = <0 445 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c9_bus>; + clocks = <&clock_peric0 PCLK_HSI2C9>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_10: hsi2c@13690000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x13690000 0x1000>; + interrupts = <0 446 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c10_bus>; + clocks = <&clock_peric0 PCLK_HSI2C10>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + hsi2c_11: hsi2c@136a0000 { + compatible = "samsung,exynos7-hsi2c"; + reg = <0x136a0000 0x1000>; + interrupts = <0 447 0>; + #address-cells = <1>; + #size-cells = <0>; + pinctrl-names = "default"; + pinctrl-0 = <&hs_i2c11_bus>; + clocks = <&clock_peric0 PCLK_HSI2C11>; + clock-names = "hsi2c"; + status = "disabled"; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = <1 13 0xff01>, + <1 14 0xff01>, + <1 11 0xff01>, + <1 10 0xff01>; + }; + + pmu_system_controller: system-controller@105c0000 { + compatible = "samsung,exynos7-pmu", "syscon"; + reg = <0x105c0000 0x5000>; + }; + + rtc: rtc@10590000 { + compatible = "samsung,s3c6410-rtc"; + reg = <0x10590000 0x100>; + interrupts = <0 355 0>, <0 356 0>; + clocks = <&clock_ccore PCLK_RTC>; + clock-names = "rtc"; + status = "disabled"; + }; + + watchdog: watchdog@101d0000 { + compatible = "samsung,exynos7-wdt"; + reg = <0x101d0000 0x100>; + interrupts = <0 110 0>; + clocks = <&clock_peris PCLK_WDT>; + clock-names = "watchdog"; + samsung,syscon-phandle = <&pmu_system_controller>; + status = "disabled"; + }; + + mmc_0: mmc@15740000 { + compatible = "samsung,exynos7-dw-mshc-smu"; + interrupts = <0 201 0>; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x15740000 0x2000>; + clocks = <&clock_fsys1 ACLK_MMC0>, + <&clock_top1 CLK_SCLK_MMC0>; + clock-names = "biu", "ciu"; + fifo-depth = <0x40>; + status = "disabled"; + }; + + mmc_1: mmc@15750000 { + compatible = "samsung,exynos7-dw-mshc"; + interrupts = <0 202 0>; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x15750000 0x2000>; + clocks = <&clock_fsys1 ACLK_MMC1>, + <&clock_top1 CLK_SCLK_MMC1>; + clock-names = "biu", "ciu"; + fifo-depth = <0x40>; + status = "disabled"; + }; + + mmc_2: mmc@15560000 { + compatible = "samsung,exynos7-dw-mshc-smu"; + interrupts = <0 216 0>; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x15560000 0x2000>; + clocks = <&clock_fsys0 ACLK_MMC2>, + <&clock_top1 CLK_SCLK_MMC2>; + clock-names = "biu", "ciu"; + fifo-depth = <0x40>; + status = "disabled"; + }; + + adc: adc@13620000 { + compatible = "samsung,exynos7-adc"; + reg = <0x13620000 0x100>; + interrupts = <0 448 0>; + clocks = <&clock_peric0 PCLK_ADCIF>; + clock-names = "adc"; + #io-channel-cells = <1>; + io-channel-ranges; + status = "disabled"; + }; + + pwm: pwm@136c0000 { + compatible = "samsung,exynos4210-pwm"; + reg = <0x136c0000 0x100>; + samsung,pwm-outputs = <0>, <1>, <2>, <3>; + #pwm-cells = <3>; + clocks = <&clock_peric0 PCLK_PWM>; + clock-names = "timers"; + }; + }; +}; + +#include "exynos7-pinctrl.dtsi" diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile new file mode 100644 index 000000000000..4f2de3e789ee --- /dev/null +++ b/arch/arm64/boot/dts/freescale/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_FSL_LS2085A) += fsl-ls2085a-simu.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts b/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts new file mode 100644 index 000000000000..82e2a6fccc64 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/fsl-ls2085a-simu.dts @@ -0,0 +1,65 @@ +/* + * Device Tree file for Freescale LS2085a software Simulator model + * + * Copyright (C) 2014, Freescale Semiconductor + * + * Bhupesh Sharma <bhupesh.sharma@freescale.com> + * + * This file is dual-licensed: you can use it either under the terms + * of the GPL or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/dts-v1/; + +/include/ "fsl-ls2085a.dtsi" + +/ { + model = "Freescale Layerscape 2085a software Simulator model"; + compatible = "fsl,ls2085a-simu", "fsl,ls2085a"; + + ethernet@2210000 { + compatible = "smsc,lan91c111"; + reg = <0x0 0x2210000 0x0 0x100>; + interrupts = <0 58 0x1>; + }; +}; diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi new file mode 100644 index 000000000000..e281ceb338c3 --- /dev/null +++ b/arch/arm64/boot/dts/freescale/fsl-ls2085a.dtsi @@ -0,0 +1,163 @@ +/* + * Device Tree Include file for Freescale Layerscape-2085A family SoC. + * + * Copyright (C) 2014, Freescale Semiconductor + * + * Bhupesh Sharma <bhupesh.sharma@freescale.com> + * + * This file is dual-licensed: you can use it either under the terms + * of the GPLv2 or the X11 license, at your option. Note that this dual + * licensing only applies to this file, and not this project as a + * whole. + * + * a) This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this library; if not, write to the Free + * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, + * MA 02110-1301 USA + * + * Or, alternatively, + * + * b) Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/ { + compatible = "fsl,ls2085a"; + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + /* + * We expect the enable-method for cpu's to be "psci", but this + * is dependent on the SoC FW, which will fill this in. + * + * Currently supported enable-method is psci v0.2 + */ + + /* We have 4 clusters having 2 Cortex-A57 cores each */ + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x0>; + }; + + cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x1>; + }; + + cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x100>; + }; + + cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x101>; + }; + + cpu@200 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x200>; + }; + + cpu@201 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x201>; + }; + + cpu@300 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x300>; + }; + + cpu@301 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x0 0x301>; + }; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0x00000000 0x80000000 0 0x80000000>; + /* DRAM space - 1, size : 2 GB DRAM */ + }; + + gic: interrupt-controller@6000000 { + compatible = "arm,gic-v3"; + reg = <0x0 0x06000000 0 0x10000>, /* GIC Dist */ + <0x0 0x06100000 0 0x100000>; /* GICR (RD_base + SGI_base) */ + #interrupt-cells = <3>; + interrupt-controller; + interrupts = <1 9 0x4>; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = <1 13 0x8>, /* Physical Secure PPI, active-low */ + <1 14 0x8>, /* Physical Non-Secure PPI, active-low */ + <1 11 0x8>, /* Virtual PPI, active-low */ + <1 10 0x8>; /* Hypervisor PPI, active-low */ + }; + + serial0: serial@21c0500 { + device_type = "serial"; + compatible = "fsl,ns16550", "ns16550a"; + reg = <0x0 0x21c0500 0x0 0x100>; + clock-frequency = <0>; /* Updated by bootloader */ + interrupts = <0 32 0x1>; /* edge triggered */ + }; + + serial1: serial@21c0600 { + device_type = "serial"; + compatible = "fsl,ns16550", "ns16550a"; + reg = <0x0 0x21c0600 0x0 0x100>; + clock-frequency = <0>; /* Updated by bootloader */ + interrupts = <0 32 0x1>; /* edge triggered */ + }; + + fsl_mc: fsl-mc@80c000000 { + compatible = "fsl,qoriq-mc"; + reg = <0x00000008 0x0c000000 0 0x40>, /* MC portal base */ + <0x00000000 0x08340000 0 0x40000>; /* MC control reg */ + }; +}; diff --git a/arch/arm64/boot/dts/include/dt-bindings b/arch/arm64/boot/dts/include/dt-bindings new file mode 120000 index 000000000000..08c00e4972fa --- /dev/null +++ b/arch/arm64/boot/dts/include/dt-bindings @@ -0,0 +1 @@ +../../../../../include/dt-bindings
\ No newline at end of file diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile new file mode 100644 index 000000000000..3ce24622b231 --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_MEDIATEK) += mt8173-evb.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/mediatek/mt8173-evb.dts b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts new file mode 100644 index 000000000000..d0ab012fa379 --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt8173-evb.dts @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2014 MediaTek Inc. + * Author: Eddie Huang <eddie.huang@mediatek.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/dts-v1/; +#include "mt8173.dtsi" + +/ { + model = "MediaTek MT8173 evaluation board"; + compatible = "mediatek,mt8173-evb", "mediatek,mt8173"; + + aliases { + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + }; + + memory@40000000 { + device_type = "memory"; + reg = <0 0x40000000 0 0x80000000>; + }; + + chosen { }; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/mediatek/mt8173-pinfunc.h b/arch/arm64/boot/dts/mediatek/mt8173-pinfunc.h new file mode 100644 index 000000000000..d2f3809af70e --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt8173-pinfunc.h @@ -0,0 +1,682 @@ +/* + * Copyright (c) 2014 MediaTek Inc. + * Author: Hongzhou.Yang <hongzhou.yang@mediatek.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __DTS_MT8173_PINFUNC_H +#define __DTS_MT8173_PINFUNC_H + +#include <dt-bindings/pinctrl/mt65xx.h> + +#define MT8173_PIN_0_EINT0__FUNC_GPIO0 (MTK_PIN_NO(0) | 0) +#define MT8173_PIN_0_EINT0__FUNC_IRDA_PDN (MTK_PIN_NO(0) | 1) +#define MT8173_PIN_0_EINT0__FUNC_I2S1_WS (MTK_PIN_NO(0) | 2) +#define MT8173_PIN_0_EINT0__FUNC_AUD_SPDIF (MTK_PIN_NO(0) | 3) +#define MT8173_PIN_0_EINT0__FUNC_UTXD0 (MTK_PIN_NO(0) | 4) +#define MT8173_PIN_0_EINT0__FUNC_DBG_MON_A_20_ (MTK_PIN_NO(0) | 7) + +#define MT8173_PIN_1_EINT1__FUNC_GPIO1 (MTK_PIN_NO(1) | 0) +#define MT8173_PIN_1_EINT1__FUNC_IRDA_RXD (MTK_PIN_NO(1) | 1) +#define MT8173_PIN_1_EINT1__FUNC_I2S1_BCK (MTK_PIN_NO(1) | 2) +#define MT8173_PIN_1_EINT1__FUNC_SDA5 (MTK_PIN_NO(1) | 3) +#define MT8173_PIN_1_EINT1__FUNC_URXD0 (MTK_PIN_NO(1) | 4) +#define MT8173_PIN_1_EINT1__FUNC_DBG_MON_A_21_ (MTK_PIN_NO(1) | 7) + +#define MT8173_PIN_2_EINT2__FUNC_GPIO2 (MTK_PIN_NO(2) | 0) +#define MT8173_PIN_2_EINT2__FUNC_IRDA_TXD (MTK_PIN_NO(2) | 1) +#define MT8173_PIN_2_EINT2__FUNC_I2S1_MCK (MTK_PIN_NO(2) | 2) +#define MT8173_PIN_2_EINT2__FUNC_SCL5 (MTK_PIN_NO(2) | 3) +#define MT8173_PIN_2_EINT2__FUNC_UTXD3 (MTK_PIN_NO(2) | 4) +#define MT8173_PIN_2_EINT2__FUNC_DBG_MON_A_22_ (MTK_PIN_NO(2) | 7) + +#define MT8173_PIN_3_EINT3__FUNC_GPIO3 (MTK_PIN_NO(3) | 0) +#define MT8173_PIN_3_EINT3__FUNC_DSI1_TE (MTK_PIN_NO(3) | 1) +#define MT8173_PIN_3_EINT3__FUNC_I2S1_DO_1 (MTK_PIN_NO(3) | 2) +#define MT8173_PIN_3_EINT3__FUNC_SDA3 (MTK_PIN_NO(3) | 3) +#define MT8173_PIN_3_EINT3__FUNC_URXD3 (MTK_PIN_NO(3) | 4) +#define MT8173_PIN_3_EINT3__FUNC_DBG_MON_A_23_ (MTK_PIN_NO(3) | 7) + +#define MT8173_PIN_4_EINT4__FUNC_GPIO4 (MTK_PIN_NO(4) | 0) +#define MT8173_PIN_4_EINT4__FUNC_DISP_PWM1 (MTK_PIN_NO(4) | 1) +#define MT8173_PIN_4_EINT4__FUNC_I2S1_DO_2 (MTK_PIN_NO(4) | 2) +#define MT8173_PIN_4_EINT4__FUNC_SCL3 (MTK_PIN_NO(4) | 3) +#define MT8173_PIN_4_EINT4__FUNC_UCTS3 (MTK_PIN_NO(4) | 4) +#define MT8173_PIN_4_EINT4__FUNC_SFWP_B (MTK_PIN_NO(4) | 6) + +#define MT8173_PIN_5_EINT5__FUNC_GPIO5 (MTK_PIN_NO(5) | 0) +#define MT8173_PIN_5_EINT5__FUNC_PCM1_CLK (MTK_PIN_NO(5) | 1) +#define MT8173_PIN_5_EINT5__FUNC_I2S2_WS (MTK_PIN_NO(5) | 2) +#define MT8173_PIN_5_EINT5__FUNC_SPI_CK_3_ (MTK_PIN_NO(5) | 3) +#define MT8173_PIN_5_EINT5__FUNC_URTS3 (MTK_PIN_NO(5) | 4) +#define MT8173_PIN_5_EINT5__FUNC_AP_MD32_JTAG_TMS (MTK_PIN_NO(5) | 5) +#define MT8173_PIN_5_EINT5__FUNC_SFOUT (MTK_PIN_NO(5) | 6) + +#define MT8173_PIN_6_EINT6__FUNC_GPIO6 (MTK_PIN_NO(6) | 0) +#define MT8173_PIN_6_EINT6__FUNC_PCM1_SYNC (MTK_PIN_NO(6) | 1) +#define MT8173_PIN_6_EINT6__FUNC_I2S2_BCK (MTK_PIN_NO(6) | 2) +#define MT8173_PIN_6_EINT6__FUNC_SPI_MI_3_ (MTK_PIN_NO(6) | 3) +#define MT8173_PIN_6_EINT6__FUNC_AP_MD32_JTAG_TCK (MTK_PIN_NO(6) | 5) +#define MT8173_PIN_6_EINT6__FUNC_SFCS0 (MTK_PIN_NO(6) | 6) + +#define MT8173_PIN_7_EINT7__FUNC_GPIO7 (MTK_PIN_NO(7) | 0) +#define MT8173_PIN_7_EINT7__FUNC_PCM1_DI (MTK_PIN_NO(7) | 1) +#define MT8173_PIN_7_EINT7__FUNC_I2S2_DI_1 (MTK_PIN_NO(7) | 2) +#define MT8173_PIN_7_EINT7__FUNC_SPI_MO_3_ (MTK_PIN_NO(7) | 3) +#define MT8173_PIN_7_EINT7__FUNC_AP_MD32_JTAG_TDI (MTK_PIN_NO(7) | 5) +#define MT8173_PIN_7_EINT7__FUNC_SFHOLD (MTK_PIN_NO(7) | 6) + +#define MT8173_PIN_8_EINT8__FUNC_GPIO8 (MTK_PIN_NO(8) | 0) +#define MT8173_PIN_8_EINT8__FUNC_PCM1_DO (MTK_PIN_NO(8) | 1) +#define MT8173_PIN_8_EINT8__FUNC_I2S2_DI_2 (MTK_PIN_NO(8) | 2) +#define MT8173_PIN_8_EINT8__FUNC_SPI_CS_3_ (MTK_PIN_NO(8) | 3) +#define MT8173_PIN_8_EINT8__FUNC_AUD_SPDIF (MTK_PIN_NO(8) | 4) +#define MT8173_PIN_8_EINT8__FUNC_AP_MD32_JTAG_TDO (MTK_PIN_NO(8) | 5) +#define MT8173_PIN_8_EINT8__FUNC_SFIN (MTK_PIN_NO(8) | 6) + +#define MT8173_PIN_9_EINT9__FUNC_GPIO9 (MTK_PIN_NO(9) | 0) +#define MT8173_PIN_9_EINT9__FUNC_USB_DRVVBUS_P0 (MTK_PIN_NO(9) | 1) +#define MT8173_PIN_9_EINT9__FUNC_I2S2_MCK (MTK_PIN_NO(9) | 2) +#define MT8173_PIN_9_EINT9__FUNC_USB_DRVVBUS_P1 (MTK_PIN_NO(9) | 4) +#define MT8173_PIN_9_EINT9__FUNC_AP_MD32_JTAG_TRST (MTK_PIN_NO(9) | 5) +#define MT8173_PIN_9_EINT9__FUNC_SFCK (MTK_PIN_NO(9) | 6) + +#define MT8173_PIN_10_EINT10__FUNC_GPIO10 (MTK_PIN_NO(10) | 0) +#define MT8173_PIN_10_EINT10__FUNC_CLKM0 (MTK_PIN_NO(10) | 1) +#define MT8173_PIN_10_EINT10__FUNC_DSI1_TE (MTK_PIN_NO(10) | 2) +#define MT8173_PIN_10_EINT10__FUNC_DISP_PWM1 (MTK_PIN_NO(10) | 3) +#define MT8173_PIN_10_EINT10__FUNC_PWM4 (MTK_PIN_NO(10) | 4) +#define MT8173_PIN_10_EINT10__FUNC_IRDA_RXD (MTK_PIN_NO(10) | 5) + +#define MT8173_PIN_11_EINT11__FUNC_GPIO11 (MTK_PIN_NO(11) | 0) +#define MT8173_PIN_11_EINT11__FUNC_CLKM1 (MTK_PIN_NO(11) | 1) +#define MT8173_PIN_11_EINT11__FUNC_I2S3_WS (MTK_PIN_NO(11) | 2) +#define MT8173_PIN_11_EINT11__FUNC_USB_DRVVBUS_P0 (MTK_PIN_NO(11) | 3) +#define MT8173_PIN_11_EINT11__FUNC_PWM5 (MTK_PIN_NO(11) | 4) +#define MT8173_PIN_11_EINT11__FUNC_IRDA_TXD (MTK_PIN_NO(11) | 5) +#define MT8173_PIN_11_EINT11__FUNC_USB_DRVVBUS_P1 (MTK_PIN_NO(11) | 6) +#define MT8173_PIN_11_EINT11__FUNC_DBG_MON_B_30_ (MTK_PIN_NO(11) | 7) + +#define MT8173_PIN_12_EINT12__FUNC_GPIO12 (MTK_PIN_NO(12) | 0) +#define MT8173_PIN_12_EINT12__FUNC_CLKM2 (MTK_PIN_NO(12) | 1) +#define MT8173_PIN_12_EINT12__FUNC_I2S3_BCK (MTK_PIN_NO(12) | 2) +#define MT8173_PIN_12_EINT12__FUNC_SRCLKENA0 (MTK_PIN_NO(12) | 3) +#define MT8173_PIN_12_EINT12__FUNC_I2S2_WS (MTK_PIN_NO(12) | 5) +#define MT8173_PIN_12_EINT12__FUNC_DBG_MON_B_32_ (MTK_PIN_NO(12) | 7) + +#define MT8173_PIN_13_EINT13__FUNC_GPIO13 (MTK_PIN_NO(13) | 0) +#define MT8173_PIN_13_EINT13__FUNC_CLKM3 (MTK_PIN_NO(13) | 1) +#define MT8173_PIN_13_EINT13__FUNC_I2S3_MCK (MTK_PIN_NO(13) | 2) +#define MT8173_PIN_13_EINT13__FUNC_SRCLKENA0 (MTK_PIN_NO(13) | 3) +#define MT8173_PIN_13_EINT13__FUNC_I2S2_BCK (MTK_PIN_NO(13) | 5) +#define MT8173_PIN_13_EINT13__FUNC_DBG_MON_A_32_ (MTK_PIN_NO(13) | 7) + +#define MT8173_PIN_14_EINT14__FUNC_GPIO14 (MTK_PIN_NO(14) | 0) +#define MT8173_PIN_14_EINT14__FUNC_CMDAT0 (MTK_PIN_NO(14) | 1) +#define MT8173_PIN_14_EINT14__FUNC_CMCSD0 (MTK_PIN_NO(14) | 2) +#define MT8173_PIN_14_EINT14__FUNC_CLKM2 (MTK_PIN_NO(14) | 4) +#define MT8173_PIN_14_EINT14__FUNC_DBG_MON_B_6_ (MTK_PIN_NO(14) | 7) + +#define MT8173_PIN_15_EINT15__FUNC_GPIO15 (MTK_PIN_NO(15) | 0) +#define MT8173_PIN_15_EINT15__FUNC_CMDAT1 (MTK_PIN_NO(15) | 1) +#define MT8173_PIN_15_EINT15__FUNC_CMCSD1 (MTK_PIN_NO(15) | 2) +#define MT8173_PIN_15_EINT15__FUNC_CMFLASH (MTK_PIN_NO(15) | 3) +#define MT8173_PIN_15_EINT15__FUNC_CLKM3 (MTK_PIN_NO(15) | 4) +#define MT8173_PIN_15_EINT15__FUNC_DBG_MON_B_29_ (MTK_PIN_NO(15) | 7) + +#define MT8173_PIN_16_IDDIG__FUNC_GPIO16 (MTK_PIN_NO(16) | 0) +#define MT8173_PIN_16_IDDIG__FUNC_IDDIG (MTK_PIN_NO(16) | 1) +#define MT8173_PIN_16_IDDIG__FUNC_CMFLASH (MTK_PIN_NO(16) | 2) +#define MT8173_PIN_16_IDDIG__FUNC_PWM5 (MTK_PIN_NO(16) | 4) + +#define MT8173_PIN_17_WATCHDOG__FUNC_GPIO17 (MTK_PIN_NO(17) | 0) +#define MT8173_PIN_17_WATCHDOG__FUNC_WATCHDOG_AO (MTK_PIN_NO(17) | 1) + +#define MT8173_PIN_18_CEC__FUNC_GPIO18 (MTK_PIN_NO(18) | 0) +#define MT8173_PIN_18_CEC__FUNC_CEC (MTK_PIN_NO(18) | 1) + +#define MT8173_PIN_19_HDMISCK__FUNC_GPIO19 (MTK_PIN_NO(19) | 0) +#define MT8173_PIN_19_HDMISCK__FUNC_HDMISCK (MTK_PIN_NO(19) | 1) +#define MT8173_PIN_19_HDMISCK__FUNC_HDCP_SCL (MTK_PIN_NO(19) | 2) + +#define MT8173_PIN_20_HDMISD__FUNC_GPIO20 (MTK_PIN_NO(20) | 0) +#define MT8173_PIN_20_HDMISD__FUNC_HDMISD (MTK_PIN_NO(20) | 1) +#define MT8173_PIN_20_HDMISD__FUNC_HDCP_SDA (MTK_PIN_NO(20) | 2) + +#define MT8173_PIN_21_HTPLG__FUNC_GPIO21 (MTK_PIN_NO(21) | 0) +#define MT8173_PIN_21_HTPLG__FUNC_HTPLG (MTK_PIN_NO(21) | 1) + +#define MT8173_PIN_22_MSDC3_DAT0__FUNC_GPIO22 (MTK_PIN_NO(22) | 0) +#define MT8173_PIN_22_MSDC3_DAT0__FUNC_MSDC3_DAT0 (MTK_PIN_NO(22) | 1) + +#define MT8173_PIN_23_MSDC3_DAT1__FUNC_GPIO23 (MTK_PIN_NO(23) | 0) +#define MT8173_PIN_23_MSDC3_DAT1__FUNC_MSDC3_DAT1 (MTK_PIN_NO(23) | 1) + +#define MT8173_PIN_24_MSDC3_DAT2__FUNC_GPIO24 (MTK_PIN_NO(24) | 0) +#define MT8173_PIN_24_MSDC3_DAT2__FUNC_MSDC3_DAT2 (MTK_PIN_NO(24) | 1) + +#define MT8173_PIN_25_MSDC3_DAT3__FUNC_GPIO25 (MTK_PIN_NO(25) | 0) +#define MT8173_PIN_25_MSDC3_DAT3__FUNC_MSDC3_DAT3 (MTK_PIN_NO(25) | 1) + +#define MT8173_PIN_26_MSDC3_CLK__FUNC_GPIO26 (MTK_PIN_NO(26) | 0) +#define MT8173_PIN_26_MSDC3_CLK__FUNC_MSDC3_CLK (MTK_PIN_NO(26) | 1) + +#define MT8173_PIN_27_MSDC3_CMD__FUNC_GPIO27 (MTK_PIN_NO(27) | 0) +#define MT8173_PIN_27_MSDC3_CMD__FUNC_MSDC3_CMD (MTK_PIN_NO(27) | 1) + +#define MT8173_PIN_28_MSDC3_DSL__FUNC_GPIO28 (MTK_PIN_NO(28) | 0) +#define MT8173_PIN_28_MSDC3_DSL__FUNC_MSDC3_DSL (MTK_PIN_NO(28) | 1) + +#define MT8173_PIN_29_UCTS2__FUNC_GPIO29 (MTK_PIN_NO(29) | 0) +#define MT8173_PIN_29_UCTS2__FUNC_UCTS2 (MTK_PIN_NO(29) | 1) + +#define MT8173_PIN_30_URTS2__FUNC_GPIO30 (MTK_PIN_NO(30) | 0) +#define MT8173_PIN_30_URTS2__FUNC_URTS2 (MTK_PIN_NO(30) | 1) + +#define MT8173_PIN_31_URXD2__FUNC_GPIO31 (MTK_PIN_NO(31) | 0) +#define MT8173_PIN_31_URXD2__FUNC_URXD2 (MTK_PIN_NO(31) | 1) +#define MT8173_PIN_31_URXD2__FUNC_UTXD2 (MTK_PIN_NO(31) | 2) + +#define MT8173_PIN_32_UTXD2__FUNC_GPIO32 (MTK_PIN_NO(32) | 0) +#define MT8173_PIN_32_UTXD2__FUNC_UTXD2 (MTK_PIN_NO(32) | 1) +#define MT8173_PIN_32_UTXD2__FUNC_URXD2 (MTK_PIN_NO(32) | 2) + +#define MT8173_PIN_33_DAICLK__FUNC_GPIO33 (MTK_PIN_NO(33) | 0) +#define MT8173_PIN_33_DAICLK__FUNC_MRG_CLK (MTK_PIN_NO(33) | 1) +#define MT8173_PIN_33_DAICLK__FUNC_PCM0_CLK (MTK_PIN_NO(33) | 2) + +#define MT8173_PIN_34_DAIPCMIN__FUNC_GPIO34 (MTK_PIN_NO(34) | 0) +#define MT8173_PIN_34_DAIPCMIN__FUNC_MRG_DI (MTK_PIN_NO(34) | 1) +#define MT8173_PIN_34_DAIPCMIN__FUNC_PCM0_DI (MTK_PIN_NO(34) | 2) + +#define MT8173_PIN_35_DAIPCMOUT__FUNC_GPIO35 (MTK_PIN_NO(35) | 0) +#define MT8173_PIN_35_DAIPCMOUT__FUNC_MRG_DO (MTK_PIN_NO(35) | 1) +#define MT8173_PIN_35_DAIPCMOUT__FUNC_PCM0_DO (MTK_PIN_NO(35) | 2) + +#define MT8173_PIN_36_DAISYNC__FUNC_GPIO36 (MTK_PIN_NO(36) | 0) +#define MT8173_PIN_36_DAISYNC__FUNC_MRG_SYNC (MTK_PIN_NO(36) | 1) +#define MT8173_PIN_36_DAISYNC__FUNC_PCM0_SYNC (MTK_PIN_NO(36) | 2) + +#define MT8173_PIN_37_EINT16__FUNC_GPIO37 (MTK_PIN_NO(37) | 0) +#define MT8173_PIN_37_EINT16__FUNC_USB_DRVVBUS_P0 (MTK_PIN_NO(37) | 1) +#define MT8173_PIN_37_EINT16__FUNC_USB_DRVVBUS_P1 (MTK_PIN_NO(37) | 2) +#define MT8173_PIN_37_EINT16__FUNC_PWM0 (MTK_PIN_NO(37) | 3) +#define MT8173_PIN_37_EINT16__FUNC_PWM1 (MTK_PIN_NO(37) | 4) +#define MT8173_PIN_37_EINT16__FUNC_PWM2 (MTK_PIN_NO(37) | 5) +#define MT8173_PIN_37_EINT16__FUNC_CLKM0 (MTK_PIN_NO(37) | 6) + +#define MT8173_PIN_38_CONN_RST__FUNC_GPIO38 (MTK_PIN_NO(38) | 0) +#define MT8173_PIN_38_CONN_RST__FUNC_USB_DRVVBUS_P0 (MTK_PIN_NO(38) | 1) +#define MT8173_PIN_38_CONN_RST__FUNC_USB_DRVVBUS_P1 (MTK_PIN_NO(38) | 2) +#define MT8173_PIN_38_CONN_RST__FUNC_CLKM1 (MTK_PIN_NO(38) | 6) + +#define MT8173_PIN_39_CM2MCLK__FUNC_GPIO39 (MTK_PIN_NO(39) | 0) +#define MT8173_PIN_39_CM2MCLK__FUNC_CM2MCLK (MTK_PIN_NO(39) | 1) +#define MT8173_PIN_39_CM2MCLK__FUNC_CMCSD0 (MTK_PIN_NO(39) | 2) +#define MT8173_PIN_39_CM2MCLK__FUNC_DBG_MON_A_17_ (MTK_PIN_NO(39) | 7) + +#define MT8173_PIN_40_CMPCLK__FUNC_GPIO40 (MTK_PIN_NO(40) | 0) +#define MT8173_PIN_40_CMPCLK__FUNC_CMPCLK (MTK_PIN_NO(40) | 1) +#define MT8173_PIN_40_CMPCLK__FUNC_CMCSK (MTK_PIN_NO(40) | 2) +#define MT8173_PIN_40_CMPCLK__FUNC_CMCSD2 (MTK_PIN_NO(40) | 3) +#define MT8173_PIN_40_CMPCLK__FUNC_DBG_MON_A_18_ (MTK_PIN_NO(40) | 7) + +#define MT8173_PIN_41_CMMCLK__FUNC_GPIO41 (MTK_PIN_NO(41) | 0) +#define MT8173_PIN_41_CMMCLK__FUNC_CMMCLK (MTK_PIN_NO(41) | 1) +#define MT8173_PIN_41_CMMCLK__FUNC_DBG_MON_A_19_ (MTK_PIN_NO(41) | 7) + +#define MT8173_PIN_42_DSI_TE__FUNC_GPIO42 (MTK_PIN_NO(42) | 0) +#define MT8173_PIN_42_DSI_TE__FUNC_DSI_TE (MTK_PIN_NO(42) | 1) + +#define MT8173_PIN_43_SDA2__FUNC_GPIO43 (MTK_PIN_NO(43) | 0) +#define MT8173_PIN_43_SDA2__FUNC_SDA2 (MTK_PIN_NO(43) | 1) + +#define MT8173_PIN_44_SCL2__FUNC_GPIO44 (MTK_PIN_NO(44) | 0) +#define MT8173_PIN_44_SCL2__FUNC_SCL2 (MTK_PIN_NO(44) | 1) + +#define MT8173_PIN_45_SDA0__FUNC_GPIO45 (MTK_PIN_NO(45) | 0) +#define MT8173_PIN_45_SDA0__FUNC_SDA0 (MTK_PIN_NO(45) | 1) + +#define MT8173_PIN_46_SCL0__FUNC_GPIO46 (MTK_PIN_NO(46) | 0) +#define MT8173_PIN_46_SCL0__FUNC_SCL0 (MTK_PIN_NO(46) | 1) + +#define MT8173_PIN_47_RDN0_A__FUNC_GPIO47 (MTK_PIN_NO(47) | 0) +#define MT8173_PIN_47_RDN0_A__FUNC_CMDAT2 (MTK_PIN_NO(47) | 1) + +#define MT8173_PIN_48_RDP0_A__FUNC_GPIO48 (MTK_PIN_NO(48) | 0) +#define MT8173_PIN_48_RDP0_A__FUNC_CMDAT3 (MTK_PIN_NO(48) | 1) + +#define MT8173_PIN_49_RDN1_A__FUNC_GPIO49 (MTK_PIN_NO(49) | 0) +#define MT8173_PIN_49_RDN1_A__FUNC_CMDAT4 (MTK_PIN_NO(49) | 1) + +#define MT8173_PIN_50_RDP1_A__FUNC_GPIO50 (MTK_PIN_NO(50) | 0) +#define MT8173_PIN_50_RDP1_A__FUNC_CMDAT5 (MTK_PIN_NO(50) | 1) + +#define MT8173_PIN_51_RCN_A__FUNC_GPIO51 (MTK_PIN_NO(51) | 0) +#define MT8173_PIN_51_RCN_A__FUNC_CMDAT6 (MTK_PIN_NO(51) | 1) + +#define MT8173_PIN_52_RCP_A__FUNC_GPIO52 (MTK_PIN_NO(52) | 0) +#define MT8173_PIN_52_RCP_A__FUNC_CMDAT7 (MTK_PIN_NO(52) | 1) + +#define MT8173_PIN_53_RDN2_A__FUNC_GPIO53 (MTK_PIN_NO(53) | 0) +#define MT8173_PIN_53_RDN2_A__FUNC_CMDAT8 (MTK_PIN_NO(53) | 1) +#define MT8173_PIN_53_RDN2_A__FUNC_CMCSD3 (MTK_PIN_NO(53) | 2) + +#define MT8173_PIN_54_RDP2_A__FUNC_GPIO54 (MTK_PIN_NO(54) | 0) +#define MT8173_PIN_54_RDP2_A__FUNC_CMDAT9 (MTK_PIN_NO(54) | 1) +#define MT8173_PIN_54_RDP2_A__FUNC_CMCSD2 (MTK_PIN_NO(54) | 2) + +#define MT8173_PIN_55_RDN3_A__FUNC_GPIO55 (MTK_PIN_NO(55) | 0) +#define MT8173_PIN_55_RDN3_A__FUNC_CMHSYNC (MTK_PIN_NO(55) | 1) +#define MT8173_PIN_55_RDN3_A__FUNC_CMCSD1 (MTK_PIN_NO(55) | 2) + +#define MT8173_PIN_56_RDP3_A__FUNC_GPIO56 (MTK_PIN_NO(56) | 0) +#define MT8173_PIN_56_RDP3_A__FUNC_CMVSYNC (MTK_PIN_NO(56) | 1) +#define MT8173_PIN_56_RDP3_A__FUNC_CMCSD0 (MTK_PIN_NO(56) | 2) + +#define MT8173_PIN_57_MSDC0_DAT0__FUNC_GPIO57 (MTK_PIN_NO(57) | 0) +#define MT8173_PIN_57_MSDC0_DAT0__FUNC_MSDC0_DAT0 (MTK_PIN_NO(57) | 1) +#define MT8173_PIN_57_MSDC0_DAT0__FUNC_I2S1_WS (MTK_PIN_NO(57) | 2) +#define MT8173_PIN_57_MSDC0_DAT0__FUNC_DBG_MON_B_7_ (MTK_PIN_NO(57) | 7) + +#define MT8173_PIN_58_MSDC0_DAT1__FUNC_GPIO58 (MTK_PIN_NO(58) | 0) +#define MT8173_PIN_58_MSDC0_DAT1__FUNC_MSDC0_DAT1 (MTK_PIN_NO(58) | 1) +#define MT8173_PIN_58_MSDC0_DAT1__FUNC_I2S1_BCK (MTK_PIN_NO(58) | 2) +#define MT8173_PIN_58_MSDC0_DAT1__FUNC_DBG_MON_B_8_ (MTK_PIN_NO(58) | 7) + +#define MT8173_PIN_59_MSDC0_DAT2__FUNC_GPIO59 (MTK_PIN_NO(59) | 0) +#define MT8173_PIN_59_MSDC0_DAT2__FUNC_MSDC0_DAT2 (MTK_PIN_NO(59) | 1) +#define MT8173_PIN_59_MSDC0_DAT2__FUNC_I2S1_MCK (MTK_PIN_NO(59) | 2) +#define MT8173_PIN_59_MSDC0_DAT2__FUNC_DBG_MON_B_9_ (MTK_PIN_NO(59) | 7) + +#define MT8173_PIN_60_MSDC0_DAT3__FUNC_GPIO60 (MTK_PIN_NO(60) | 0) +#define MT8173_PIN_60_MSDC0_DAT3__FUNC_MSDC0_DAT3 (MTK_PIN_NO(60) | 1) +#define MT8173_PIN_60_MSDC0_DAT3__FUNC_I2S1_DO_1 (MTK_PIN_NO(60) | 2) +#define MT8173_PIN_60_MSDC0_DAT3__FUNC_DBG_MON_B_10_ (MTK_PIN_NO(60) | 7) + +#define MT8173_PIN_61_MSDC0_DAT4__FUNC_GPIO61 (MTK_PIN_NO(61) | 0) +#define MT8173_PIN_61_MSDC0_DAT4__FUNC_MSDC0_DAT4 (MTK_PIN_NO(61) | 1) +#define MT8173_PIN_61_MSDC0_DAT4__FUNC_I2S1_DO_2 (MTK_PIN_NO(61) | 2) +#define MT8173_PIN_61_MSDC0_DAT4__FUNC_DBG_MON_B_11_ (MTK_PIN_NO(61) | 7) + +#define MT8173_PIN_62_MSDC0_DAT5__FUNC_GPIO62 (MTK_PIN_NO(62) | 0) +#define MT8173_PIN_62_MSDC0_DAT5__FUNC_MSDC0_DAT5 (MTK_PIN_NO(62) | 1) +#define MT8173_PIN_62_MSDC0_DAT5__FUNC_I2S2_WS (MTK_PIN_NO(62) | 2) +#define MT8173_PIN_62_MSDC0_DAT5__FUNC_DBG_MON_B_12_ (MTK_PIN_NO(62) | 7) + +#define MT8173_PIN_63_MSDC0_DAT6__FUNC_GPIO63 (MTK_PIN_NO(63) | 0) +#define MT8173_PIN_63_MSDC0_DAT6__FUNC_MSDC0_DAT6 (MTK_PIN_NO(63) | 1) +#define MT8173_PIN_63_MSDC0_DAT6__FUNC_I2S2_BCK (MTK_PIN_NO(63) | 2) +#define MT8173_PIN_63_MSDC0_DAT6__FUNC_DBG_MON_B_13_ (MTK_PIN_NO(63) | 7) + +#define MT8173_PIN_64_MSDC0_DAT7__FUNC_GPIO64 (MTK_PIN_NO(64) | 0) +#define MT8173_PIN_64_MSDC0_DAT7__FUNC_MSDC0_DAT7 (MTK_PIN_NO(64) | 1) +#define MT8173_PIN_64_MSDC0_DAT7__FUNC_I2S2_DI_1 (MTK_PIN_NO(64) | 2) +#define MT8173_PIN_64_MSDC0_DAT7__FUNC_DBG_MON_B_14_ (MTK_PIN_NO(64) | 7) + +#define MT8173_PIN_65_MSDC0_CLK__FUNC_GPIO65 (MTK_PIN_NO(65) | 0) +#define MT8173_PIN_65_MSDC0_CLK__FUNC_MSDC0_CLK (MTK_PIN_NO(65) | 1) +#define MT8173_PIN_65_MSDC0_CLK__FUNC_DBG_MON_B_16_ (MTK_PIN_NO(65) | 7) + +#define MT8173_PIN_66_MSDC0_CMD__FUNC_GPIO66 (MTK_PIN_NO(66) | 0) +#define MT8173_PIN_66_MSDC0_CMD__FUNC_MSDC0_CMD (MTK_PIN_NO(66) | 1) +#define MT8173_PIN_66_MSDC0_CMD__FUNC_I2S2_DI_2 (MTK_PIN_NO(66) | 2) +#define MT8173_PIN_66_MSDC0_CMD__FUNC_DBG_MON_B_15_ (MTK_PIN_NO(66) | 7) + +#define MT8173_PIN_67_MSDC0_DSL__FUNC_GPIO67 (MTK_PIN_NO(67) | 0) +#define MT8173_PIN_67_MSDC0_DSL__FUNC_MSDC0_DSL (MTK_PIN_NO(67) | 1) +#define MT8173_PIN_67_MSDC0_DSL__FUNC_DBG_MON_B_17_ (MTK_PIN_NO(67) | 7) + +#define MT8173_PIN_68_MSDC0_RST___FUNC_GPIO68 (MTK_PIN_NO(68) | 0) +#define MT8173_PIN_68_MSDC0_RST___FUNC_MSDC0_RSTB (MTK_PIN_NO(68) | 1) +#define MT8173_PIN_68_MSDC0_RST___FUNC_I2S2_MCK (MTK_PIN_NO(68) | 2) +#define MT8173_PIN_68_MSDC0_RST___FUNC_DBG_MON_B_18_ (MTK_PIN_NO(68) | 7) + +#define MT8173_PIN_69_SPI_CK__FUNC_GPIO69 (MTK_PIN_NO(69) | 0) +#define MT8173_PIN_69_SPI_CK__FUNC_SPI_CK_0_ (MTK_PIN_NO(69) | 1) +#define MT8173_PIN_69_SPI_CK__FUNC_I2S3_DO_1 (MTK_PIN_NO(69) | 2) +#define MT8173_PIN_69_SPI_CK__FUNC_PWM0 (MTK_PIN_NO(69) | 3) +#define MT8173_PIN_69_SPI_CK__FUNC_PWM5 (MTK_PIN_NO(69) | 4) +#define MT8173_PIN_69_SPI_CK__FUNC_I2S2_MCK (MTK_PIN_NO(69) | 5) +#define MT8173_PIN_69_SPI_CK__FUNC_DBG_MON_B_19_ (MTK_PIN_NO(69) | 7) + +#define MT8173_PIN_70_SPI_MI__FUNC_GPIO70 (MTK_PIN_NO(70) | 0) +#define MT8173_PIN_70_SPI_MI__FUNC_SPI_MI_0_ (MTK_PIN_NO(70) | 1) +#define MT8173_PIN_70_SPI_MI__FUNC_I2S3_DO_2 (MTK_PIN_NO(70) | 2) +#define MT8173_PIN_70_SPI_MI__FUNC_PWM1 (MTK_PIN_NO(70) | 3) +#define MT8173_PIN_70_SPI_MI__FUNC_SPI_MO_0_ (MTK_PIN_NO(70) | 4) +#define MT8173_PIN_70_SPI_MI__FUNC_I2S2_DI_1 (MTK_PIN_NO(70) | 5) +#define MT8173_PIN_70_SPI_MI__FUNC_DSI1_TE (MTK_PIN_NO(70) | 6) +#define MT8173_PIN_70_SPI_MI__FUNC_DBG_MON_B_20_ (MTK_PIN_NO(70) | 7) + +#define MT8173_PIN_71_SPI_MO__FUNC_GPIO71 (MTK_PIN_NO(71) | 0) +#define MT8173_PIN_71_SPI_MO__FUNC_SPI_MO_0_ (MTK_PIN_NO(71) | 1) +#define MT8173_PIN_71_SPI_MO__FUNC_I2S3_DO_3 (MTK_PIN_NO(71) | 2) +#define MT8173_PIN_71_SPI_MO__FUNC_PWM2 (MTK_PIN_NO(71) | 3) +#define MT8173_PIN_71_SPI_MO__FUNC_SPI_MI_0_ (MTK_PIN_NO(71) | 4) +#define MT8173_PIN_71_SPI_MO__FUNC_I2S2_DI_2 (MTK_PIN_NO(71) | 5) +#define MT8173_PIN_71_SPI_MO__FUNC_DBG_MON_B_21_ (MTK_PIN_NO(71) | 7) + +#define MT8173_PIN_72_SPI_CS__FUNC_GPIO72 (MTK_PIN_NO(72) | 0) +#define MT8173_PIN_72_SPI_CS__FUNC_SPI_CS_0_ (MTK_PIN_NO(72) | 1) +#define MT8173_PIN_72_SPI_CS__FUNC_I2S3_DO_4 (MTK_PIN_NO(72) | 2) +#define MT8173_PIN_72_SPI_CS__FUNC_PWM3 (MTK_PIN_NO(72) | 3) +#define MT8173_PIN_72_SPI_CS__FUNC_PWM6 (MTK_PIN_NO(72) | 4) +#define MT8173_PIN_72_SPI_CS__FUNC_DISP_PWM1 (MTK_PIN_NO(72) | 5) +#define MT8173_PIN_72_SPI_CS__FUNC_DBG_MON_B_22_ (MTK_PIN_NO(72) | 7) + +#define MT8173_PIN_73_MSDC1_DAT0__FUNC_GPIO73 (MTK_PIN_NO(73) | 0) +#define MT8173_PIN_73_MSDC1_DAT0__FUNC_MSDC1_DAT0 (MTK_PIN_NO(73) | 1) +#define MT8173_PIN_73_MSDC1_DAT0__FUNC_DBG_MON_B_24_ (MTK_PIN_NO(73) | 7) + +#define MT8173_PIN_74_MSDC1_DAT1__FUNC_GPIO74 (MTK_PIN_NO(74) | 0) +#define MT8173_PIN_74_MSDC1_DAT1__FUNC_MSDC1_DAT1 (MTK_PIN_NO(74) | 1) +#define MT8173_PIN_74_MSDC1_DAT1__FUNC_DBG_MON_B_25_ (MTK_PIN_NO(74) | 7) + +#define MT8173_PIN_75_MSDC1_DAT2__FUNC_GPIO75 (MTK_PIN_NO(75) | 0) +#define MT8173_PIN_75_MSDC1_DAT2__FUNC_MSDC1_DAT2 (MTK_PIN_NO(75) | 1) +#define MT8173_PIN_75_MSDC1_DAT2__FUNC_DBG_MON_B_26_ (MTK_PIN_NO(75) | 7) + +#define MT8173_PIN_76_MSDC1_DAT3__FUNC_GPIO76 (MTK_PIN_NO(76) | 0) +#define MT8173_PIN_76_MSDC1_DAT3__FUNC_MSDC1_DAT3 (MTK_PIN_NO(76) | 1) +#define MT8173_PIN_76_MSDC1_DAT3__FUNC_DBG_MON_B_27_ (MTK_PIN_NO(76) | 7) + +#define MT8173_PIN_77_MSDC1_CLK__FUNC_GPIO77 (MTK_PIN_NO(77) | 0) +#define MT8173_PIN_77_MSDC1_CLK__FUNC_MSDC1_CLK (MTK_PIN_NO(77) | 1) +#define MT8173_PIN_77_MSDC1_CLK__FUNC_DBG_MON_B_28_ (MTK_PIN_NO(77) | 7) + +#define MT8173_PIN_78_MSDC1_CMD__FUNC_GPIO78 (MTK_PIN_NO(78) | 0) +#define MT8173_PIN_78_MSDC1_CMD__FUNC_MSDC1_CMD (MTK_PIN_NO(78) | 1) +#define MT8173_PIN_78_MSDC1_CMD__FUNC_DBG_MON_B_23_ (MTK_PIN_NO(78) | 7) + +#define MT8173_PIN_79_PWRAP_SPI0_MI__FUNC_GPIO79 (MTK_PIN_NO(79) | 0) +#define MT8173_PIN_79_PWRAP_SPI0_MI__FUNC_PWRAP_SPIMI (MTK_PIN_NO(79) | 1) +#define MT8173_PIN_79_PWRAP_SPI0_MI__FUNC_PWRAP_SPIMO (MTK_PIN_NO(79) | 2) + +#define MT8173_PIN_80_PWRAP_SPI0_MO__FUNC_GPIO80 (MTK_PIN_NO(80) | 0) +#define MT8173_PIN_80_PWRAP_SPI0_MO__FUNC_PWRAP_SPIMO (MTK_PIN_NO(80) | 1) +#define MT8173_PIN_80_PWRAP_SPI0_MO__FUNC_PWRAP_SPIMI (MTK_PIN_NO(80) | 2) + +#define MT8173_PIN_81_PWRAP_SPI0_CK__FUNC_GPIO81 (MTK_PIN_NO(81) | 0) +#define MT8173_PIN_81_PWRAP_SPI0_CK__FUNC_PWRAP_SPICK (MTK_PIN_NO(81) | 1) + +#define MT8173_PIN_82_PWRAP_SPI0_CSN__FUNC_GPIO82 (MTK_PIN_NO(82) | 0) +#define MT8173_PIN_82_PWRAP_SPI0_CSN__FUNC_PWRAP_SPICS (MTK_PIN_NO(82) | 1) + +#define MT8173_PIN_83_AUD_CLK_MOSI__FUNC_GPIO83 (MTK_PIN_NO(83) | 0) +#define MT8173_PIN_83_AUD_CLK_MOSI__FUNC_AUD_CLK_MOSI (MTK_PIN_NO(83) | 1) + +#define MT8173_PIN_84_AUD_DAT_MISO__FUNC_GPIO84 (MTK_PIN_NO(84) | 0) +#define MT8173_PIN_84_AUD_DAT_MISO__FUNC_AUD_DAT_MISO (MTK_PIN_NO(84) | 1) +#define MT8173_PIN_84_AUD_DAT_MISO__FUNC_AUD_DAT_MOSI (MTK_PIN_NO(84) | 2) + +#define MT8173_PIN_85_AUD_DAT_MOSI__FUNC_GPIO85 (MTK_PIN_NO(85) | 0) +#define MT8173_PIN_85_AUD_DAT_MOSI__FUNC_AUD_DAT_MOSI (MTK_PIN_NO(85) | 1) +#define MT8173_PIN_85_AUD_DAT_MOSI__FUNC_AUD_DAT_MISO (MTK_PIN_NO(85) | 2) + +#define MT8173_PIN_86_RTC32K_CK__FUNC_GPIO86 (MTK_PIN_NO(86) | 0) +#define MT8173_PIN_86_RTC32K_CK__FUNC_RTC32K_CK (MTK_PIN_NO(86) | 1) + +#define MT8173_PIN_87_DISP_PWM0__FUNC_GPIO87 (MTK_PIN_NO(87) | 0) +#define MT8173_PIN_87_DISP_PWM0__FUNC_DISP_PWM0 (MTK_PIN_NO(87) | 1) +#define MT8173_PIN_87_DISP_PWM0__FUNC_DISP_PWM1 (MTK_PIN_NO(87) | 2) +#define MT8173_PIN_87_DISP_PWM0__FUNC_DBG_MON_B_31_ (MTK_PIN_NO(87) | 7) + +#define MT8173_PIN_88_SRCLKENAI__FUNC_GPIO88 (MTK_PIN_NO(88) | 0) +#define MT8173_PIN_88_SRCLKENAI__FUNC_SRCLKENAI (MTK_PIN_NO(88) | 1) + +#define MT8173_PIN_89_SRCLKENAI2__FUNC_GPIO89 (MTK_PIN_NO(89) | 0) +#define MT8173_PIN_89_SRCLKENAI2__FUNC_SRCLKENAI2 (MTK_PIN_NO(89) | 1) + +#define MT8173_PIN_90_SRCLKENA0__FUNC_GPIO90 (MTK_PIN_NO(90) | 0) +#define MT8173_PIN_90_SRCLKENA0__FUNC_SRCLKENA0 (MTK_PIN_NO(90) | 1) + +#define MT8173_PIN_91_SRCLKENA1__FUNC_GPIO91 (MTK_PIN_NO(91) | 0) +#define MT8173_PIN_91_SRCLKENA1__FUNC_SRCLKENA1 (MTK_PIN_NO(91) | 1) + +#define MT8173_PIN_92_PCM_CLK__FUNC_GPIO92 (MTK_PIN_NO(92) | 0) +#define MT8173_PIN_92_PCM_CLK__FUNC_PCM1_CLK (MTK_PIN_NO(92) | 1) +#define MT8173_PIN_92_PCM_CLK__FUNC_I2S0_BCK (MTK_PIN_NO(92) | 2) +#define MT8173_PIN_92_PCM_CLK__FUNC_DBG_MON_A_24_ (MTK_PIN_NO(92) | 7) + +#define MT8173_PIN_93_PCM_SYNC__FUNC_GPIO93 (MTK_PIN_NO(93) | 0) +#define MT8173_PIN_93_PCM_SYNC__FUNC_PCM1_SYNC (MTK_PIN_NO(93) | 1) +#define MT8173_PIN_93_PCM_SYNC__FUNC_I2S0_WS (MTK_PIN_NO(93) | 2) +#define MT8173_PIN_93_PCM_SYNC__FUNC_DBG_MON_A_25_ (MTK_PIN_NO(93) | 7) + +#define MT8173_PIN_94_PCM_RX__FUNC_GPIO94 (MTK_PIN_NO(94) | 0) +#define MT8173_PIN_94_PCM_RX__FUNC_PCM1_DI (MTK_PIN_NO(94) | 1) +#define MT8173_PIN_94_PCM_RX__FUNC_I2S0_DI (MTK_PIN_NO(94) | 2) +#define MT8173_PIN_94_PCM_RX__FUNC_DBG_MON_A_26_ (MTK_PIN_NO(94) | 7) + +#define MT8173_PIN_95_PCM_TX__FUNC_GPIO95 (MTK_PIN_NO(95) | 0) +#define MT8173_PIN_95_PCM_TX__FUNC_PCM1_DO (MTK_PIN_NO(95) | 1) +#define MT8173_PIN_95_PCM_TX__FUNC_I2S0_DO (MTK_PIN_NO(95) | 2) +#define MT8173_PIN_95_PCM_TX__FUNC_DBG_MON_A_27_ (MTK_PIN_NO(95) | 7) + +#define MT8173_PIN_96_URXD1__FUNC_GPIO96 (MTK_PIN_NO(96) | 0) +#define MT8173_PIN_96_URXD1__FUNC_URXD1 (MTK_PIN_NO(96) | 1) +#define MT8173_PIN_96_URXD1__FUNC_UTXD1 (MTK_PIN_NO(96) | 2) +#define MT8173_PIN_96_URXD1__FUNC_DBG_MON_A_28_ (MTK_PIN_NO(96) | 7) + +#define MT8173_PIN_97_UTXD1__FUNC_GPIO97 (MTK_PIN_NO(97) | 0) +#define MT8173_PIN_97_UTXD1__FUNC_UTXD1 (MTK_PIN_NO(97) | 1) +#define MT8173_PIN_97_UTXD1__FUNC_URXD1 (MTK_PIN_NO(97) | 2) +#define MT8173_PIN_97_UTXD1__FUNC_DBG_MON_A_29_ (MTK_PIN_NO(97) | 7) + +#define MT8173_PIN_98_URTS1__FUNC_GPIO98 (MTK_PIN_NO(98) | 0) +#define MT8173_PIN_98_URTS1__FUNC_URTS1 (MTK_PIN_NO(98) | 1) +#define MT8173_PIN_98_URTS1__FUNC_UCTS1 (MTK_PIN_NO(98) | 2) +#define MT8173_PIN_98_URTS1__FUNC_DBG_MON_A_30_ (MTK_PIN_NO(98) | 7) + +#define MT8173_PIN_99_UCTS1__FUNC_GPIO99 (MTK_PIN_NO(99) | 0) +#define MT8173_PIN_99_UCTS1__FUNC_UCTS1 (MTK_PIN_NO(99) | 1) +#define MT8173_PIN_99_UCTS1__FUNC_URTS1 (MTK_PIN_NO(99) | 2) +#define MT8173_PIN_99_UCTS1__FUNC_DBG_MON_A_31_ (MTK_PIN_NO(99) | 7) + +#define MT8173_PIN_100_MSDC2_DAT0__FUNC_GPIO100 (MTK_PIN_NO(100) | 0) +#define MT8173_PIN_100_MSDC2_DAT0__FUNC_MSDC2_DAT0 (MTK_PIN_NO(100) | 1) +#define MT8173_PIN_100_MSDC2_DAT0__FUNC_USB_DRVVBUS_P0 (MTK_PIN_NO(100) | 3) +#define MT8173_PIN_100_MSDC2_DAT0__FUNC_SDA5 (MTK_PIN_NO(100) | 4) +#define MT8173_PIN_100_MSDC2_DAT0__FUNC_USB_DRVVBUS_P1 (MTK_PIN_NO(100) | 5) +#define MT8173_PIN_100_MSDC2_DAT0__FUNC_DBG_MON_B_0_ (MTK_PIN_NO(100) | 7) + +#define MT8173_PIN_101_MSDC2_DAT1__FUNC_GPIO101 (MTK_PIN_NO(101) | 0) +#define MT8173_PIN_101_MSDC2_DAT1__FUNC_MSDC2_DAT1 (MTK_PIN_NO(101) | 1) +#define MT8173_PIN_101_MSDC2_DAT1__FUNC_AUD_SPDIF (MTK_PIN_NO(101) | 3) +#define MT8173_PIN_101_MSDC2_DAT1__FUNC_SCL5 (MTK_PIN_NO(101) | 4) +#define MT8173_PIN_101_MSDC2_DAT1__FUNC_DBG_MON_B_1_ (MTK_PIN_NO(101) | 7) + +#define MT8173_PIN_102_MSDC2_DAT2__FUNC_GPIO102 (MTK_PIN_NO(102) | 0) +#define MT8173_PIN_102_MSDC2_DAT2__FUNC_MSDC2_DAT2 (MTK_PIN_NO(102) | 1) +#define MT8173_PIN_102_MSDC2_DAT2__FUNC_UTXD0 (MTK_PIN_NO(102) | 3) +#define MT8173_PIN_102_MSDC2_DAT2__FUNC_PWM0 (MTK_PIN_NO(102) | 5) +#define MT8173_PIN_102_MSDC2_DAT2__FUNC_SPI_CK_1_ (MTK_PIN_NO(102) | 6) +#define MT8173_PIN_102_MSDC2_DAT2__FUNC_DBG_MON_B_2_ (MTK_PIN_NO(102) | 7) + +#define MT8173_PIN_103_MSDC2_DAT3__FUNC_GPIO103 (MTK_PIN_NO(103) | 0) +#define MT8173_PIN_103_MSDC2_DAT3__FUNC_MSDC2_DAT3 (MTK_PIN_NO(103) | 1) +#define MT8173_PIN_103_MSDC2_DAT3__FUNC_URXD0 (MTK_PIN_NO(103) | 3) +#define MT8173_PIN_103_MSDC2_DAT3__FUNC_PWM1 (MTK_PIN_NO(103) | 5) +#define MT8173_PIN_103_MSDC2_DAT3__FUNC_SPI_MI_1_ (MTK_PIN_NO(103) | 6) +#define MT8173_PIN_103_MSDC2_DAT3__FUNC_DBG_MON_B_3_ (MTK_PIN_NO(103) | 7) + +#define MT8173_PIN_104_MSDC2_CLK__FUNC_GPIO104 (MTK_PIN_NO(104) | 0) +#define MT8173_PIN_104_MSDC2_CLK__FUNC_MSDC2_CLK (MTK_PIN_NO(104) | 1) +#define MT8173_PIN_104_MSDC2_CLK__FUNC_UTXD3 (MTK_PIN_NO(104) | 3) +#define MT8173_PIN_104_MSDC2_CLK__FUNC_SDA3 (MTK_PIN_NO(104) | 4) +#define MT8173_PIN_104_MSDC2_CLK__FUNC_PWM2 (MTK_PIN_NO(104) | 5) +#define MT8173_PIN_104_MSDC2_CLK__FUNC_SPI_MO_1_ (MTK_PIN_NO(104) | 6) +#define MT8173_PIN_104_MSDC2_CLK__FUNC_DBG_MON_B_4_ (MTK_PIN_NO(104) | 7) + +#define MT8173_PIN_105_MSDC2_CMD__FUNC_GPIO105 (MTK_PIN_NO(105) | 0) +#define MT8173_PIN_105_MSDC2_CMD__FUNC_MSDC2_CMD (MTK_PIN_NO(105) | 1) +#define MT8173_PIN_105_MSDC2_CMD__FUNC_URXD3 (MTK_PIN_NO(105) | 3) +#define MT8173_PIN_105_MSDC2_CMD__FUNC_SCL3 (MTK_PIN_NO(105) | 4) +#define MT8173_PIN_105_MSDC2_CMD__FUNC_PWM3 (MTK_PIN_NO(105) | 5) +#define MT8173_PIN_105_MSDC2_CMD__FUNC_SPI_CS_1_ (MTK_PIN_NO(105) | 6) +#define MT8173_PIN_105_MSDC2_CMD__FUNC_DBG_MON_B_5_ (MTK_PIN_NO(105) | 7) + +#define MT8173_PIN_106_SDA3__FUNC_GPIO106 (MTK_PIN_NO(106) | 0) +#define MT8173_PIN_106_SDA3__FUNC_SDA3 (MTK_PIN_NO(106) | 1) + +#define MT8173_PIN_107_SCL3__FUNC_GPIO107 (MTK_PIN_NO(107) | 0) +#define MT8173_PIN_107_SCL3__FUNC_SCL3 (MTK_PIN_NO(107) | 1) + +#define MT8173_PIN_108_JTMS__FUNC_GPIO108 (MTK_PIN_NO(108) | 0) +#define MT8173_PIN_108_JTMS__FUNC_JTMS (MTK_PIN_NO(108) | 1) +#define MT8173_PIN_108_JTMS__FUNC_MFG_JTAG_TMS (MTK_PIN_NO(108) | 2) +#define MT8173_PIN_108_JTMS__FUNC_AP_MD32_JTAG_TMS (MTK_PIN_NO(108) | 5) +#define MT8173_PIN_108_JTMS__FUNC_DFD_TMS (MTK_PIN_NO(108) | 6) + +#define MT8173_PIN_109_JTCK__FUNC_GPIO109 (MTK_PIN_NO(109) | 0) +#define MT8173_PIN_109_JTCK__FUNC_JTCK (MTK_PIN_NO(109) | 1) +#define MT8173_PIN_109_JTCK__FUNC_MFG_JTAG_TCK (MTK_PIN_NO(109) | 2) +#define MT8173_PIN_109_JTCK__FUNC_AP_MD32_JTAG_TCK (MTK_PIN_NO(109) | 5) +#define MT8173_PIN_109_JTCK__FUNC_DFD_TCK (MTK_PIN_NO(109) | 6) + +#define MT8173_PIN_110_JTDI__FUNC_GPIO110 (MTK_PIN_NO(110) | 0) +#define MT8173_PIN_110_JTDI__FUNC_JTDI (MTK_PIN_NO(110) | 1) +#define MT8173_PIN_110_JTDI__FUNC_MFG_JTAG_TDI (MTK_PIN_NO(110) | 2) +#define MT8173_PIN_110_JTDI__FUNC_AP_MD32_JTAG_TDI (MTK_PIN_NO(110) | 5) +#define MT8173_PIN_110_JTDI__FUNC_DFD_TDI (MTK_PIN_NO(110) | 6) + +#define MT8173_PIN_111_JTDO__FUNC_GPIO111 (MTK_PIN_NO(111) | 0) +#define MT8173_PIN_111_JTDO__FUNC_JTDO (MTK_PIN_NO(111) | 1) +#define MT8173_PIN_111_JTDO__FUNC_MFG_JTAG_TDO (MTK_PIN_NO(111) | 2) +#define MT8173_PIN_111_JTDO__FUNC_AP_MD32_JTAG_TDO (MTK_PIN_NO(111) | 5) +#define MT8173_PIN_111_JTDO__FUNC_DFD_TDO (MTK_PIN_NO(111) | 6) + +#define MT8173_PIN_112_JTRST_B__FUNC_GPIO112 (MTK_PIN_NO(112) | 0) +#define MT8173_PIN_112_JTRST_B__FUNC_JTRST_B (MTK_PIN_NO(112) | 1) +#define MT8173_PIN_112_JTRST_B__FUNC_MFG_JTAG_TRSTN (MTK_PIN_NO(112) | 2) +#define MT8173_PIN_112_JTRST_B__FUNC_AP_MD32_JTAG_TRST (MTK_PIN_NO(112) | 5) +#define MT8173_PIN_112_JTRST_B__FUNC_DFD_NTRST (MTK_PIN_NO(112) | 6) + +#define MT8173_PIN_113_URXD0__FUNC_GPIO113 (MTK_PIN_NO(113) | 0) +#define MT8173_PIN_113_URXD0__FUNC_URXD0 (MTK_PIN_NO(113) | 1) +#define MT8173_PIN_113_URXD0__FUNC_UTXD0 (MTK_PIN_NO(113) | 2) +#define MT8173_PIN_113_URXD0__FUNC_I2S2_WS (MTK_PIN_NO(113) | 6) +#define MT8173_PIN_113_URXD0__FUNC_DBG_MON_A_0_ (MTK_PIN_NO(113) | 7) + +#define MT8173_PIN_114_UTXD0__FUNC_GPIO114 (MTK_PIN_NO(114) | 0) +#define MT8173_PIN_114_UTXD0__FUNC_UTXD0 (MTK_PIN_NO(114) | 1) +#define MT8173_PIN_114_UTXD0__FUNC_URXD0 (MTK_PIN_NO(114) | 2) +#define MT8173_PIN_114_UTXD0__FUNC_I2S2_BCK (MTK_PIN_NO(114) | 6) +#define MT8173_PIN_114_UTXD0__FUNC_DBG_MON_A_1_ (MTK_PIN_NO(114) | 7) + +#define MT8173_PIN_115_URTS0__FUNC_GPIO115 (MTK_PIN_NO(115) | 0) +#define MT8173_PIN_115_URTS0__FUNC_URTS0 (MTK_PIN_NO(115) | 1) +#define MT8173_PIN_115_URTS0__FUNC_UCTS0 (MTK_PIN_NO(115) | 2) +#define MT8173_PIN_115_URTS0__FUNC_I2S2_MCK (MTK_PIN_NO(115) | 6) +#define MT8173_PIN_115_URTS0__FUNC_DBG_MON_A_2_ (MTK_PIN_NO(115) | 7) + +#define MT8173_PIN_116_UCTS0__FUNC_GPIO116 (MTK_PIN_NO(116) | 0) +#define MT8173_PIN_116_UCTS0__FUNC_UCTS0 (MTK_PIN_NO(116) | 1) +#define MT8173_PIN_116_UCTS0__FUNC_URTS0 (MTK_PIN_NO(116) | 2) +#define MT8173_PIN_116_UCTS0__FUNC_I2S2_DI_1 (MTK_PIN_NO(116) | 6) +#define MT8173_PIN_116_UCTS0__FUNC_DBG_MON_A_3_ (MTK_PIN_NO(116) | 7) + +#define MT8173_PIN_117_URXD3__FUNC_GPIO117 (MTK_PIN_NO(117) | 0) +#define MT8173_PIN_117_URXD3__FUNC_URXD3 (MTK_PIN_NO(117) | 1) +#define MT8173_PIN_117_URXD3__FUNC_UTXD3 (MTK_PIN_NO(117) | 2) +#define MT8173_PIN_117_URXD3__FUNC_DBG_MON_A_9_ (MTK_PIN_NO(117) | 7) + +#define MT8173_PIN_118_UTXD3__FUNC_GPIO118 (MTK_PIN_NO(118) | 0) +#define MT8173_PIN_118_UTXD3__FUNC_UTXD3 (MTK_PIN_NO(118) | 1) +#define MT8173_PIN_118_UTXD3__FUNC_URXD3 (MTK_PIN_NO(118) | 2) +#define MT8173_PIN_118_UTXD3__FUNC_DBG_MON_A_10_ (MTK_PIN_NO(118) | 7) + +#define MT8173_PIN_119_KPROW0__FUNC_GPIO119 (MTK_PIN_NO(119) | 0) +#define MT8173_PIN_119_KPROW0__FUNC_KROW0 (MTK_PIN_NO(119) | 1) +#define MT8173_PIN_119_KPROW0__FUNC_DBG_MON_A_11_ (MTK_PIN_NO(119) | 7) + +#define MT8173_PIN_120_KPROW1__FUNC_GPIO120 (MTK_PIN_NO(120) | 0) +#define MT8173_PIN_120_KPROW1__FUNC_KROW1 (MTK_PIN_NO(120) | 1) +#define MT8173_PIN_120_KPROW1__FUNC_PWM6 (MTK_PIN_NO(120) | 3) +#define MT8173_PIN_120_KPROW1__FUNC_DBG_MON_A_12_ (MTK_PIN_NO(120) | 7) + +#define MT8173_PIN_121_KPROW2__FUNC_GPIO121 (MTK_PIN_NO(121) | 0) +#define MT8173_PIN_121_KPROW2__FUNC_KROW2 (MTK_PIN_NO(121) | 1) +#define MT8173_PIN_121_KPROW2__FUNC_IRDA_PDN (MTK_PIN_NO(121) | 2) +#define MT8173_PIN_121_KPROW2__FUNC_USB_DRVVBUS_P0 (MTK_PIN_NO(121) | 3) +#define MT8173_PIN_121_KPROW2__FUNC_PWM4 (MTK_PIN_NO(121) | 4) +#define MT8173_PIN_121_KPROW2__FUNC_USB_DRVVBUS_P1 (MTK_PIN_NO(121) | 5) +#define MT8173_PIN_121_KPROW2__FUNC_DBG_MON_A_13_ (MTK_PIN_NO(121) | 7) + +#define MT8173_PIN_122_KPCOL0__FUNC_GPIO122 (MTK_PIN_NO(122) | 0) +#define MT8173_PIN_122_KPCOL0__FUNC_KCOL0 (MTK_PIN_NO(122) | 1) +#define MT8173_PIN_122_KPCOL0__FUNC_DBG_MON_A_14_ (MTK_PIN_NO(122) | 7) + +#define MT8173_PIN_123_KPCOL1__FUNC_GPIO123 (MTK_PIN_NO(123) | 0) +#define MT8173_PIN_123_KPCOL1__FUNC_KCOL1 (MTK_PIN_NO(123) | 1) +#define MT8173_PIN_123_KPCOL1__FUNC_IRDA_RXD (MTK_PIN_NO(123) | 2) +#define MT8173_PIN_123_KPCOL1__FUNC_PWM5 (MTK_PIN_NO(123) | 3) +#define MT8173_PIN_123_KPCOL1__FUNC_DBG_MON_A_15_ (MTK_PIN_NO(123) | 7) + +#define MT8173_PIN_124_KPCOL2__FUNC_GPIO124 (MTK_PIN_NO(124) | 0) +#define MT8173_PIN_124_KPCOL2__FUNC_KCOL2 (MTK_PIN_NO(124) | 1) +#define MT8173_PIN_124_KPCOL2__FUNC_IRDA_TXD (MTK_PIN_NO(124) | 2) +#define MT8173_PIN_124_KPCOL2__FUNC_USB_DRVVBUS_P0 (MTK_PIN_NO(124) | 3) +#define MT8173_PIN_124_KPCOL2__FUNC_PWM3 (MTK_PIN_NO(124) | 4) +#define MT8173_PIN_124_KPCOL2__FUNC_USB_DRVVBUS_P1 (MTK_PIN_NO(124) | 5) +#define MT8173_PIN_124_KPCOL2__FUNC_DBG_MON_A_16_ (MTK_PIN_NO(124) | 7) + +#define MT8173_PIN_125_SDA1__FUNC_GPIO125 (MTK_PIN_NO(125) | 0) +#define MT8173_PIN_125_SDA1__FUNC_SDA1 (MTK_PIN_NO(125) | 1) + +#define MT8173_PIN_126_SCL1__FUNC_GPIO126 (MTK_PIN_NO(126) | 0) +#define MT8173_PIN_126_SCL1__FUNC_SCL1 (MTK_PIN_NO(126) | 1) + +#define MT8173_PIN_127_LCM_RST__FUNC_GPIO127 (MTK_PIN_NO(127) | 0) +#define MT8173_PIN_127_LCM_RST__FUNC_LCM_RST (MTK_PIN_NO(127) | 1) + +#define MT8173_PIN_128_I2S0_LRCK__FUNC_GPIO128 (MTK_PIN_NO(128) | 0) +#define MT8173_PIN_128_I2S0_LRCK__FUNC_I2S0_WS (MTK_PIN_NO(128) | 1) +#define MT8173_PIN_128_I2S0_LRCK__FUNC_I2S1_WS (MTK_PIN_NO(128) | 2) +#define MT8173_PIN_128_I2S0_LRCK__FUNC_I2S2_WS (MTK_PIN_NO(128) | 3) +#define MT8173_PIN_128_I2S0_LRCK__FUNC_SPI_CK_2_ (MTK_PIN_NO(128) | 5) +#define MT8173_PIN_128_I2S0_LRCK__FUNC_DBG_MON_A_4_ (MTK_PIN_NO(128) | 7) + +#define MT8173_PIN_129_I2S0_BCK__FUNC_GPIO129 (MTK_PIN_NO(129) | 0) +#define MT8173_PIN_129_I2S0_BCK__FUNC_I2S0_BCK (MTK_PIN_NO(129) | 1) +#define MT8173_PIN_129_I2S0_BCK__FUNC_I2S1_BCK (MTK_PIN_NO(129) | 2) +#define MT8173_PIN_129_I2S0_BCK__FUNC_I2S2_BCK (MTK_PIN_NO(129) | 3) +#define MT8173_PIN_129_I2S0_BCK__FUNC_SPI_MI_2_ (MTK_PIN_NO(129) | 5) +#define MT8173_PIN_129_I2S0_BCK__FUNC_DBG_MON_A_5_ (MTK_PIN_NO(129) | 7) + +#define MT8173_PIN_130_I2S0_MCK__FUNC_GPIO130 (MTK_PIN_NO(130) | 0) +#define MT8173_PIN_130_I2S0_MCK__FUNC_I2S0_MCK (MTK_PIN_NO(130) | 1) +#define MT8173_PIN_130_I2S0_MCK__FUNC_I2S1_MCK (MTK_PIN_NO(130) | 2) +#define MT8173_PIN_130_I2S0_MCK__FUNC_I2S2_MCK (MTK_PIN_NO(130) | 3) +#define MT8173_PIN_130_I2S0_MCK__FUNC_SPI_MO_2_ (MTK_PIN_NO(130) | 5) +#define MT8173_PIN_130_I2S0_MCK__FUNC_DBG_MON_A_6_ (MTK_PIN_NO(130) | 7) + +#define MT8173_PIN_131_I2S0_DATA0__FUNC_GPIO131 (MTK_PIN_NO(131) | 0) +#define MT8173_PIN_131_I2S0_DATA0__FUNC_I2S0_DO (MTK_PIN_NO(131) | 1) +#define MT8173_PIN_131_I2S0_DATA0__FUNC_I2S1_DO_1 (MTK_PIN_NO(131) | 2) +#define MT8173_PIN_131_I2S0_DATA0__FUNC_I2S2_DI_1 (MTK_PIN_NO(131) | 3) +#define MT8173_PIN_131_I2S0_DATA0__FUNC_SPI_CS_2_ (MTK_PIN_NO(131) | 5) +#define MT8173_PIN_131_I2S0_DATA0__FUNC_DBG_MON_A_7_ (MTK_PIN_NO(131) | 7) + +#define MT8173_PIN_132_I2S0_DATA1__FUNC_GPIO132 (MTK_PIN_NO(132) | 0) +#define MT8173_PIN_132_I2S0_DATA1__FUNC_I2S0_DI (MTK_PIN_NO(132) | 1) +#define MT8173_PIN_132_I2S0_DATA1__FUNC_I2S1_DO_2 (MTK_PIN_NO(132) | 2) +#define MT8173_PIN_132_I2S0_DATA1__FUNC_I2S2_DI_2 (MTK_PIN_NO(132) | 3) +#define MT8173_PIN_132_I2S0_DATA1__FUNC_DBG_MON_A_8_ (MTK_PIN_NO(132) | 7) + +#define MT8173_PIN_133_SDA4__FUNC_GPIO133 (MTK_PIN_NO(133) | 0) +#define MT8173_PIN_133_SDA4__FUNC_SDA4 (MTK_PIN_NO(133) | 1) + +#define MT8173_PIN_134_SCL4__FUNC_GPIO134 (MTK_PIN_NO(134) | 0) +#define MT8173_PIN_134_SCL4__FUNC_SCL4 (MTK_PIN_NO(134) | 1) + +#endif /* __DTS_MT8173_PINFUNC_H */ diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi new file mode 100644 index 000000000000..924fdb6673ff --- /dev/null +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2014 MediaTek Inc. + * Author: Eddie Huang <eddie.huang@mediatek.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <dt-bindings/interrupt-controller/irq.h> +#include <dt-bindings/interrupt-controller/arm-gic.h> +#include "mt8173-pinfunc.h" + +/ { + compatible = "mediatek,mt8173"; + interrupt-parent = <&sysirq>; + #address-cells = <2>; + #size-cells = <2>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { + cpu = <&cpu0>; + }; + core1 { + cpu = <&cpu1>; + }; + }; + + cluster1 { + core0 { + cpu = <&cpu2>; + }; + core1 { + cpu = <&cpu3>; + }; + }; + }; + + cpu0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + reg = <0x000>; + }; + + cpu1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a53"; + reg = <0x001>; + enable-method = "psci"; + }; + + cpu2: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x100>; + enable-method = "psci"; + }; + + cpu3: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a57"; + reg = <0x101>; + enable-method = "psci"; + }; + }; + + psci { + compatible = "arm,psci"; + method = "smc"; + cpu_suspend = <0x84000001>; + cpu_off = <0x84000002>; + cpu_on = <0x84000003>; + }; + + uart_clk: dummy26m { + compatible = "fixed-clock"; + clock-frequency = <26000000>; + #clock-cells = <0>; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&gic>; + interrupts = <GIC_PPI 13 + (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 14 + (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 11 + (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 10 + (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>; + }; + + soc { + #address-cells = <2>; + #size-cells = <2>; + compatible = "simple-bus"; + ranges; + + syscfg_pctl_a: syscfg_pctl_a@10005000 { + compatible = "mediatek,mt8173-pctl-a-syscfg", "syscon"; + reg = <0 0x10005000 0 0x1000>; + }; + + pio: pinctrl@0x10005000 { + compatible = "mediatek,mt8173-pinctrl"; + reg = <0 0x1000B000 0 0x1000>; + mediatek,pctl-regmap = <&syscfg_pctl_a>; + pins-are-numbered; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>; + }; + + sysirq: intpol-controller@10200620 { + compatible = "mediatek,mt8173-sysirq", + "mediatek,mt6577-sysirq"; + interrupt-controller; + #interrupt-cells = <3>; + interrupt-parent = <&gic>; + reg = <0 0x10200620 0 0x20>; + }; + + gic: interrupt-controller@10220000 { + compatible = "arm,gic-400"; + #interrupt-cells = <3>; + interrupt-parent = <&gic>; + interrupt-controller; + reg = <0 0x10221000 0 0x1000>, + <0 0x10222000 0 0x2000>, + <0 0x10224000 0 0x2000>, + <0 0x10226000 0 0x2000>; + interrupts = <GIC_PPI 9 + (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>; + }; + + uart0: serial@11002000 { + compatible = "mediatek,mt8173-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11002000 0 0x400>; + interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_LOW>; + clocks = <&uart_clk>; + status = "disabled"; + }; + + uart1: serial@11003000 { + compatible = "mediatek,mt8173-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11003000 0 0x400>; + interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_LOW>; + clocks = <&uart_clk>; + status = "disabled"; + }; + + uart2: serial@11004000 { + compatible = "mediatek,mt8173-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11004000 0 0x400>; + interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_LOW>; + clocks = <&uart_clk>; + status = "disabled"; + }; + + uart3: serial@11005000 { + compatible = "mediatek,mt8173-uart", + "mediatek,mt6577-uart"; + reg = <0 0x11005000 0 0x400>; + interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_LOW>; + clocks = <&uart_clk>; + status = "disabled"; + }; + }; + +}; + diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile new file mode 100644 index 000000000000..8e94af64ee94 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_QCOM) += apq8016-sbc.dtb msm8916-mtp.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc.dts b/arch/arm64/boot/dts/qcom/apq8016-sbc.dts new file mode 100644 index 000000000000..825f489a2af7 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/apq8016-sbc.dts @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/dts-v1/; + +#include "apq8016-sbc.dtsi" + +/ { + model = "Qualcomm Technologies, Inc. APQ 8016 SBC"; + compatible = "qcom,apq8016-sbc", "qcom,apq8016", "qcom,sbc"; +}; diff --git a/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi new file mode 100644 index 000000000000..703a4f16e711 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/apq8016-sbc.dtsi @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "msm8916.dtsi" + +/ { + aliases { + serial0 = &blsp1_uart2; + }; + + chosen { + stdout-path = "serial0"; + }; + + soc { + serial@78b0000 { + status = "okay"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp1_uart2_default>; + pinctrl-1 = <&blsp1_uart2_sleep>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916-mtp.dts b/arch/arm64/boot/dts/qcom/msm8916-mtp.dts new file mode 100644 index 000000000000..fced77f0fd3a --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916-mtp.dts @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/dts-v1/; + +#include "msm8916-mtp.dtsi" + +/ { + model = "Qualcomm Technologies, Inc. MSM 8916 MTP"; + compatible = "qcom,msm8916-mtp", "qcom,msm8916-mtp-smb1360", + "qcom,msm8916", "qcom,mtp"; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916-mtp.dtsi b/arch/arm64/boot/dts/qcom/msm8916-mtp.dtsi new file mode 100644 index 000000000000..bea871b0df13 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916-mtp.dtsi @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include "msm8916.dtsi" + +/ { + aliases { + serial0 = &blsp1_uart2; + }; + + chosen { + stdout-path = "serial0"; + }; + + soc { + serial@78b0000 { + status = "okay"; + pinctrl-names = "default", "sleep"; + pinctrl-0 = <&blsp1_uart2_default>; + pinctrl-1 = <&blsp1_uart2_sleep>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi new file mode 100644 index 000000000000..f212b8303d04 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <dt-bindings/interrupt-controller/arm-gic.h> +#include <dt-bindings/clock/qcom,gcc-msm8916.h> +#include <dt-bindings/reset/qcom,gcc-msm8916.h> + +/ { + model = "Qualcomm Technologies, Inc. MSM8916"; + compatible = "qcom,msm8916"; + + interrupt-parent = <&intc>; + + #address-cells = <2>; + #size-cells = <2>; + + aliases { }; + + chosen { }; + + memory { + device_type = "memory"; + /* We expect the bootloader to fill in the reg */ + reg = <0 0 0 0>; + }; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + CPU0: cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0>; + }; + + CPU1: cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x1>; + }; + + CPU2: cpu@2 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x2>; + }; + + CPU3: cpu@3 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x3>; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = <GIC_PPI 2 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 3 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 4 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 1 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>; + }; + + soc: soc { + #address-cells = <1>; + #size-cells = <1>; + ranges = <0 0 0 0xffffffff>; + compatible = "simple-bus"; + + pinctrl@1000000 { + compatible = "qcom,msm8916-pinctrl"; + reg = <0x1000000 0x300000>; + interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>; + gpio-controller; + #gpio-cells = <2>; + interrupt-controller; + #interrupt-cells = <2>; + + blsp1_uart2_default: blsp1_uart2_default { + pinmux { + function = "blsp_uart2"; + pins = "gpio4", "gpio5"; + }; + pinconf { + pins = "gpio4", "gpio5"; + drive-strength = <16>; + bias-disable; + }; + }; + + blsp1_uart2_sleep: blsp1_uart2_sleep { + pinmux { + function = "blsp_uart2"; + pins = "gpio4", "gpio5"; + }; + pinconf { + pins = "gpio4", "gpio5"; + drive-strength = <2>; + bias-pull-down; + }; + }; + }; + + gcc: qcom,gcc@1800000 { + compatible = "qcom,gcc-msm8916"; + #clock-cells = <1>; + #reset-cells = <1>; + reg = <0x1800000 0x80000>; + }; + + blsp1_uart2: serial@78b0000 { + compatible = "qcom,msm-uartdm-v1.4", "qcom,msm-uartdm"; + reg = <0x78b0000 0x200>; + interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&gcc GCC_BLSP1_UART2_APPS_CLK>, <&gcc GCC_BLSP1_AHB_CLK>; + clock-names = "core", "iface"; + status = "disabled"; + }; + + intc: interrupt-controller@b000000 { + compatible = "qcom,msm-qgic2"; + interrupt-controller; + #interrupt-cells = <3>; + reg = <0x0b000000 0x1000>, <0x0b002000 0x1000>; + }; + + timer@b020000 { + #address-cells = <1>; + #size-cells = <1>; + ranges; + compatible = "arm,armv7-timer-mem"; + reg = <0xb020000 0x1000>; + clock-frequency = <19200000>; + + frame@b021000 { + frame-number = <0>; + interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>; + reg = <0xb021000 0x1000>, + <0xb022000 0x1000>; + }; + + frame@b023000 { + frame-number = <1>; + interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>; + reg = <0xb023000 0x1000>; + status = "disabled"; + }; + + frame@b024000 { + frame-number = <2>; + interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>; + reg = <0xb024000 0x1000>; + status = "disabled"; + }; + + frame@b025000 { + frame-number = <3>; + interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>; + reg = <0xb025000 0x1000>; + status = "disabled"; + }; + + frame@b026000 { + frame-number = <4>; + interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>; + reg = <0xb026000 0x1000>; + status = "disabled"; + }; + + frame@b027000 { + frame-number = <5>; + interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>; + reg = <0xb027000 0x1000>; + status = "disabled"; + }; + + frame@b028000 { + frame-number = <6>; + interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>; + reg = <0xb028000 0x1000>; + status = "disabled"; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/sprd/Makefile b/arch/arm64/boot/dts/sprd/Makefile new file mode 100644 index 000000000000..b658c5e09b15 --- /dev/null +++ b/arch/arm64/boot/dts/sprd/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_SPRD) += sc9836-openphone.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/sprd/sc9836-openphone.dts b/arch/arm64/boot/dts/sprd/sc9836-openphone.dts new file mode 100644 index 000000000000..e5657c35cd10 --- /dev/null +++ b/arch/arm64/boot/dts/sprd/sc9836-openphone.dts @@ -0,0 +1,49 @@ +/* + * Spreadtrum SC9836 openphone board DTS file + * + * Copyright (C) 2014, Spreadtrum Communications Inc. + * + * This file is licensed under a dual GPLv2 or X11 license. + */ + +/dts-v1/; + +#include "sc9836.dtsi" + +/ { + model = "Spreadtrum SC9836 Openphone Board"; + + compatible = "sprd,sc9836-openphone", "sprd,sc9836"; + + aliases { + serial0 = &uart0; + serial1 = &uart1; + serial2 = &uart2; + serial3 = &uart3; + }; + + memory@80000000 { + device_type = "memory"; + reg = <0 0x80000000 0 0x20000000>; + }; + + chosen { + stdout-path = "serial1:115200n8"; + }; +}; + +&uart0 { + status = "okay"; +}; + +&uart1 { + status = "okay"; +}; + +&uart2 { + status = "okay"; +}; + +&uart3 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/sprd/sc9836.dtsi b/arch/arm64/boot/dts/sprd/sc9836.dtsi new file mode 100644 index 000000000000..ee34e1a36e03 --- /dev/null +++ b/arch/arm64/boot/dts/sprd/sc9836.dtsi @@ -0,0 +1,129 @@ +/* + * Spreadtrum SC9836 SoC DTS file + * + * Copyright (C) 2014, Spreadtrum Communications Inc. + * + * This file is licensed under a dual GPLv2 or X11 license. + */ + +#include "sharkl64.dtsi" +#include <dt-bindings/interrupt-controller/arm-gic.h> + +/ { + compatible = "sprd,sc9836"; + + cpus { + #address-cells = <2>; + #size-cells = <0>; + + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x0>; + enable-method = "psci"; + }; + + cpu@1 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x1>; + enable-method = "psci"; + }; + + cpu@2 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x2>; + enable-method = "psci"; + }; + + cpu@3 { + device_type = "cpu"; + compatible = "arm,cortex-a53", "arm,armv8"; + reg = <0x0 0x3>; + enable-method = "psci"; + }; + }; + + etf@10003000 { + compatible = "arm,coresight-tmc", "arm,primecell"; + reg = <0 0x10003000 0 0x1000>; + clocks = <&clk26mhz>; + clock-names = "apb_pclk"; + port { + etf_in: endpoint { + slave-mode; + remote-endpoint = <&funnel_out_port0>; + }; + }; + }; + + funnel@10001000 { + compatible = "arm,coresight-funnel", "arm,primecell"; + reg = <0 0x10001000 0 0x1000>; + clocks = <&clk26mhz>; + clock-names = "apb_pclk"; + ports { + #address-cells = <1>; + #size-cells = <0>; + + /* funnel output port */ + port@0 { + reg = <0>; + funnel_out_port0: endpoint { + remote-endpoint = <&etf_in>; + }; + }; + + /* funnel input port 0~3 is reserved for ETMs */ + port@1 { + reg = <4>; + funnel_in_port4: endpoint { + slave-mode; + remote-endpoint = <&stm_out>; + }; + }; + }; + }; + + stm@10006000 { + compatible = "arm,coresight-stm", "arm,primecell"; + reg = <0 0x10006000 0 0x1000>, + <0 0x01000000 0 0x180000>; + reg-names = "stm-base", "stm-stimulus-base"; + clocks = <&clk26mhz>; + clock-names = "apb_pclk"; + port { + stm_out: endpoint { + remote-endpoint = <&funnel_in_port4>; + }; + }; + }; + + gic: interrupt-controller@12001000 { + compatible = "arm,gic-400"; + reg = <0 0x12001000 0 0x1000>, + <0 0x12002000 0 0x2000>, + <0 0x12004000 0 0x2000>, + <0 0x12006000 0 0x2000>; + #interrupt-cells = <3>; + interrupt-controller; + interrupts = <GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>; + }; + + psci { + compatible = "arm,psci"; + method = "smc"; + cpu_on = <0xc4000003>; + cpu_off = <0x84000002>; + cpu_suspend = <0xc4000001>; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>, + <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_LOW)>; + }; +}; diff --git a/arch/arm64/boot/dts/sprd/sharkl64.dtsi b/arch/arm64/boot/dts/sprd/sharkl64.dtsi new file mode 100644 index 000000000000..69f64e7fce7c --- /dev/null +++ b/arch/arm64/boot/dts/sprd/sharkl64.dtsi @@ -0,0 +1,65 @@ +/* + * Spreadtrum Sharkl64 platform DTS file + * + * Copyright (C) 2014, Spreadtrum Communications Inc. + * + * This file is licensed under a dual GPLv2 or X11 license. + */ + +/ { + interrupt-parent = <&gic>; + #address-cells = <2>; + #size-cells = <2>; + + soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + ap-apb { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + ranges; + + uart0: serial@70000000 { + compatible = "sprd,sc9836-uart"; + reg = <0 0x70000000 0 0x100>; + interrupts = <0 2 0xf04>; + clocks = <&clk26mhz>; + status = "disabled"; + }; + + uart1: serial@70100000 { + compatible = "sprd,sc9836-uart"; + reg = <0 0x70100000 0 0x100>; + interrupts = <0 3 0xf04>; + clocks = <&clk26mhz>; + status = "disabled"; + }; + + uart2: serial@70200000 { + compatible = "sprd,sc9836-uart"; + reg = <0 0x70200000 0 0x100>; + interrupts = <0 4 0xf04>; + clocks = <&clk26mhz>; + status = "disabled"; + }; + + uart3: serial@70300000 { + compatible = "sprd,sc9836-uart"; + reg = <0 0x70300000 0 0x100>; + interrupts = <0 5 0xf04>; + clocks = <&clk26mhz>; + status = "disabled"; + }; + }; + }; + + clk26mhz: clk26mhz { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <26000000>; + }; +}; diff --git a/arch/arm64/boot/dts/xilinx/Makefile b/arch/arm64/boot/dts/xilinx/Makefile new file mode 100644 index 000000000000..ae16427f6a4a --- /dev/null +++ b/arch/arm64/boot/dts/xilinx/Makefile @@ -0,0 +1,5 @@ +dtb-$(CONFIG_ARCH_ZYNQMP) += zynqmp-ep108.dtb + +always := $(dtb-y) +subdir-y := $(dts-dirs) +clean-files := *.dtb diff --git a/arch/arm64/boot/dts/xilinx/zynqmp-ep108.dts b/arch/arm64/boot/dts/xilinx/zynqmp-ep108.dts new file mode 100644 index 000000000000..0a3f40ecd06d --- /dev/null +++ b/arch/arm64/boot/dts/xilinx/zynqmp-ep108.dts @@ -0,0 +1,47 @@ +/* + * dts file for Xilinx ZynqMP ep108 development board + * + * (C) Copyright 2014 - 2015, Xilinx, Inc. + * + * Michal Simek <michal.simek@xilinx.com> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +/dts-v1/; + +/include/ "zynqmp.dtsi" + +/ { + model = "ZynqMP EP108"; + + aliases { + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0:115200n8"; + }; + + memory { + device_type = "memory"; + reg = <0x0 0x0 0x40000000>; + }; +}; + +&gem0 { + status = "okay"; + phy-handle = <&phy0>; + phy-mode = "rgmii-id"; + phy0: phy@0{ + reg = <0>; + max-speed = <100>; + }; +}; + +&uart0 { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/xilinx/zynqmp.dtsi b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi new file mode 100644 index 000000000000..11e0b00045cf --- /dev/null +++ b/arch/arm64/boot/dts/xilinx/zynqmp.dtsi @@ -0,0 +1,305 @@ +/* + * dts file for Xilinx ZynqMP + * + * (C) Copyright 2014 - 2015, Xilinx, Inc. + * + * Michal Simek <michal.simek@xilinx.com> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + */ + +/ { + compatible = "xlnx,zynqmp"; + #address-cells = <2>; + #size-cells = <1>; + + cpus { + #address-cells = <1>; + #size-cells = <0>; + + cpu@0 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + enable-method = "psci"; + reg = <0x0>; + }; + + cpu@1 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + enable-method = "psci"; + reg = <0x1>; + }; + + cpu@2 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + enable-method = "psci"; + reg = <0x2>; + }; + + cpu@3 { + compatible = "arm,cortex-a53", "arm,armv8"; + device_type = "cpu"; + enable-method = "psci"; + reg = <0x3>; + }; + }; + + pmu { + compatible = "arm,armv8-pmuv3"; + interrupts = <0 143 4>, + <0 144 4>, + <0 145 4>, + <0 146 4>; + }; + + psci { + compatible = "arm,psci-0.2"; + method = "smc"; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&gic>; + interrupts = <1 13 0xf01>, + <1 14 0xf01>, + <1 11 0xf01>, + <1 10 0xf01>; + }; + + amba_apu { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <1>; + ranges; + + gic: interrupt-controller@f9010000 { + compatible = "arm,gic-400", "arm,cortex-a15-gic"; + #interrupt-cells = <3>; + reg = <0x0 0xf9010000 0x10000>, + <0x0 0xf902f000 0x2000>, + <0x0 0xf9040000 0x20000>, + <0x0 0xf906f000 0x2000>; + interrupt-controller; + interrupt-parent = <&gic>; + interrupts = <1 9 0xf04>; + }; + }; + + amba { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <1>; + ranges; + + misc_clk: misc_clk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <25000000>; + }; + + ttc0: timer@ff110000 { + compatible = "cdns,ttc"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 36 4>, <0 37 4>, <0 38 4>; + reg = <0x0 0xff110000 0x1000>; + clocks = <&misc_clk>; + timer-width = <32>; + }; + + ttc1: timer@ff120000 { + compatible = "cdns,ttc"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 39 4>, <0 40 4>, <0 41 4>; + reg = <0x0 0xff120000 0x1000>; + clocks = <&misc_clk>; + timer-width = <32>; + }; + + ttc2: timer@ff130000 { + compatible = "cdns,ttc"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 42 4>, <0 43 4>, <0 44 4>; + reg = <0x0 0xff130000 0x1000>; + clocks = <&misc_clk>; + timer-width = <32>; + }; + + ttc3: timer@ff140000 { + compatible = "cdns,ttc"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 45 4>, <0 46 4>, <0 47 4>; + reg = <0x0 0xff140000 0x1000>; + clocks = <&misc_clk>; + timer-width = <32>; + }; + + uart0: serial@ff000000 { + compatible = "cdns,uart-r1p8"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 21 4>; + reg = <0x0 0xff000000 0x1000>; + clock-names = "uart_clk", "pclk"; + clocks = <&misc_clk &misc_clk>; + }; + + uart1: serial@ff010000 { + compatible = "cdns,uart-r1p8"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 22 4>; + reg = <0x0 0xff010000 0x1000>; + clock-names = "uart_clk", "pclk"; + clocks = <&misc_clk &misc_clk>; + }; + + gpio: gpio@ff0a0000 { + compatible = "xlnx,zynq-gpio-1.0"; + status = "disabled"; + #gpio-cells = <0x2>; + clocks = <&misc_clk>; + interrupt-parent = <&gic>; + interrupts = <0 16 4>; + reg = <0x0 0xff0a0000 0x1000>; + }; + + gem0: ethernet@ff0b0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 57 4>, <0 57 4>; + reg = <0x0 0xff0b0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + gem1: ethernet@ff0c0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 59 4>, <0 59 4>; + reg = <0x0 0xff0c0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + gem2: ethernet@ff0d0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 61 4>, <0 61 4>; + reg = <0x0 0xff0d0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + gem3: ethernet@ff0e0000 { + compatible = "cdns,gem"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 63 4>, <0 63 4>; + reg = <0x0 0xff0e0000 0x1000>; + clock-names = "pclk", "hclk", "tx_clk"; + clocks = <&misc_clk>, <&misc_clk>, <&misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + spi0: spi@ff040000 { + compatible = "cdns,spi-r1p6"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 19 4>; + reg = <0x0 0xff040000 0x1000>; + clock-names = "ref_clk", "pclk"; + clocks = <&misc_clk &misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + spi1: spi@ff050000 { + compatible = "cdns,spi-r1p6"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 20 4>; + reg = <0x0 0xff050000 0x1000>; + clock-names = "ref_clk", "pclk"; + clocks = <&misc_clk &misc_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c_clk: i2c_clk { + compatible = "fixed-clock"; + #clock-cells = <0x0>; + clock-frequency = <111111111>; + }; + + i2c0: i2c@ff020000 { + compatible = "cdns,i2c-r1p10"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 17 4>; + reg = <0x0 0xff020000 0x1000>; + clocks = <&i2c_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c1: i2c@ff030000 { + compatible = "cdns,i2c-r1p10"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 18 4>; + reg = <0x0 0xff030000 0x1000>; + clocks = <&i2c_clk>; + #address-cells = <1>; + #size-cells = <0>; + }; + + sdhci0: sdhci@ff160000 { + compatible = "arasan,sdhci-8.9a"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 48 4>; + reg = <0x0 0xff160000 0x1000>; + clock-names = "clk_xin", "clk_ahb"; + clocks = <&misc_clk>, <&misc_clk>; + }; + + sdhci1: sdhci@ff170000 { + compatible = "arasan,sdhci-8.9a"; + status = "disabled"; + interrupt-parent = <&gic>; + interrupts = <0 49 4>; + reg = <0x0 0xff170000 0x1000>; + clock-names = "clk_xin", "clk_ahb"; + clocks = <&misc_clk>, <&misc_clk>; + }; + + watchdog0: watchdog@fd4d0000 { + compatible = "cdns,wdt-r1p2"; + status = "disabled"; + clocks= <&misc_clk>; + interrupt-parent = <&gic>; + interrupts = <0 52 1>; + reg = <0x0 0xfd4d0000 0x1000>; + timeout-sec = <10>; + }; + }; +}; diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index dd301be89ecc..2ed7449d9273 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1,6 +1,7 @@ # CONFIG_LOCALVERSION_AUTO is not set CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y +CONFIG_FHANDLE=y CONFIG_AUDIT=y CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y @@ -13,14 +14,12 @@ CONFIG_TASK_IO_ACCOUNTING=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 -CONFIG_RESOURCE_COUNTERS=y CONFIG_MEMCG=y CONFIG_MEMCG_SWAP=y CONFIG_MEMCG_KMEM=y CONFIG_CGROUP_HUGETLB=y # CONFIG_UTS_NS is not set # CONFIG_IPC_NS is not set -# CONFIG_PID_NS is not set # CONFIG_NET_NS is not set CONFIG_SCHED_AUTOGROUP=y CONFIG_BLK_DEV_INITRD=y @@ -32,9 +31,18 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set # CONFIG_IOSCHED_DEADLINE is not set +CONFIG_ARCH_EXYNOS7=y +CONFIG_ARCH_FSL_LS2085A=y +CONFIG_ARCH_MEDIATEK=y +CONFIG_ARCH_SEATTLE=y +CONFIG_ARCH_TEGRA=y +CONFIG_ARCH_TEGRA_132_SOC=y +CONFIG_ARCH_QCOM=y +CONFIG_ARCH_SPRD=y CONFIG_ARCH_THUNDER=y CONFIG_ARCH_VEXPRESS=y CONFIG_ARCH_XGENE=y +CONFIG_ARCH_ZYNQMP=y CONFIG_PCI=y CONFIG_PCI_MSI=y CONFIG_PCI_XGENE=y @@ -46,6 +54,8 @@ CONFIG_CMA=y CONFIG_CMDLINE="console=ttyAMA0" # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set CONFIG_COMPAT=y +CONFIG_CPU_IDLE=y +CONFIG_ARM_CPUIDLE=y CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y @@ -59,6 +69,7 @@ CONFIG_BPF_JIT=y # CONFIG_WIRELESS is not set CONFIG_NET_9P=y CONFIG_NET_9P_VIRTIO=y +# CONFIG_TEGRA_AHB is not set CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y @@ -78,6 +89,7 @@ CONFIG_NETDEVICES=y CONFIG_TUN=y CONFIG_VIRTIO_NET=y CONFIG_NET_XGENE=y +CONFIG_SKY2=y CONFIG_SMC91X=y CONFIG_SMSC911X=y # CONFIG_WLAN is not set @@ -87,16 +99,23 @@ CONFIG_SERIO_AMBAKMI=y CONFIG_LEGACY_PTY_COUNT=16 CONFIG_SERIAL_8250=y CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_8250_MT6577=y CONFIG_SERIAL_AMBA_PL011=y CONFIG_SERIAL_AMBA_PL011_CONSOLE=y +CONFIG_SERIAL_MSM=y +CONFIG_SERIAL_MSM_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_SERIAL_XILINX_PS_UART=y +CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y CONFIG_VIRTIO_CONSOLE=y # CONFIG_HW_RANDOM is not set -# CONFIG_HMC_DRV is not set CONFIG_SPI=y CONFIG_SPI_PL022=y +CONFIG_PINCTRL_MSM8916=y CONFIG_GPIO_PL061=y CONFIG_GPIO_XGENE=y +CONFIG_POWER_RESET_XGENE=y +CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_REGULATOR=y CONFIG_REGULATOR_FIXED_VOLTAGE=y @@ -109,10 +128,10 @@ CONFIG_LOGO=y CONFIG_USB=y CONFIG_USB_EHCI_HCD=y CONFIG_USB_EHCI_HCD_PLATFORM=y -CONFIG_USB_ISP1760_HCD=y CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_STORAGE=y +CONFIG_USB_ISP1760=y CONFIG_USB_ULPI=y CONFIG_MMC=y CONFIG_MMC_ARMMMCI=y @@ -122,8 +141,11 @@ CONFIG_MMC_SPI=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_EFI=y CONFIG_RTC_DRV_XGENE=y +CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_BALLOON=y CONFIG_VIRTIO_MMIO=y +CONFIG_COMMON_CLK_QCOM=y +CONFIG_MSM_GCC_8916=y # CONFIG_IOMMU_SUPPORT is not set CONFIG_PHY_XGENE=y CONFIG_EXT2_FS=y @@ -133,13 +155,17 @@ CONFIG_EXT3_FS=y CONFIG_EXT4_FS=y CONFIG_FANOTIFY=y CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y +CONFIG_QUOTA=y +CONFIG_AUTOFS4_FS=y CONFIG_FUSE_FS=y CONFIG_CUSE=y CONFIG_VFAT_FS=y CONFIG_TMPFS=y CONFIG_HUGETLBFS=y +CONFIG_EFIVAR_FS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y +CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_9P_FS=y CONFIG_NLS_CODEPAGE_437=y @@ -152,6 +178,7 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_LOCKUP_DETECTOR=y # CONFIG_SCHED_DEBUG is not set +# CONFIG_DEBUG_PREEMPT is not set # CONFIG_FTRACE is not set CONFIG_SECURITY=y CONFIG_CRYPTO_ANSI_CPRNG=y @@ -159,7 +186,6 @@ CONFIG_ARM64_CRYPTO=y CONFIG_CRYPTO_SHA1_ARM64_CE=y CONFIG_CRYPTO_SHA2_ARM64_CE=y CONFIG_CRYPTO_GHASH_ARM64_CE=y -CONFIG_CRYPTO_AES_ARM64_CE=y CONFIG_CRYPTO_AES_ARM64_CE_CCM=y CONFIG_CRYPTO_AES_ARM64_CE_BLK=y CONFIG_CRYPTO_AES_ARM64_NEON_BLK=y diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig index 5562652c5316..2cf32e9887e1 100644 --- a/arch/arm64/crypto/Kconfig +++ b/arch/arm64/crypto/Kconfig @@ -27,20 +27,19 @@ config CRYPTO_AES_ARM64_CE tristate "AES core cipher using ARMv8 Crypto Extensions" depends on ARM64 && KERNEL_MODE_NEON select CRYPTO_ALGAPI - select CRYPTO_AES config CRYPTO_AES_ARM64_CE_CCM tristate "AES in CCM mode using ARMv8 Crypto Extensions" depends on ARM64 && KERNEL_MODE_NEON select CRYPTO_ALGAPI - select CRYPTO_AES + select CRYPTO_AES_ARM64_CE select CRYPTO_AEAD config CRYPTO_AES_ARM64_CE_BLK tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions" depends on ARM64 && KERNEL_MODE_NEON select CRYPTO_BLKCIPHER - select CRYPTO_AES + select CRYPTO_AES_ARM64_CE select CRYPTO_ABLK_HELPER config CRYPTO_AES_ARM64_NEON_BLK @@ -50,4 +49,8 @@ config CRYPTO_AES_ARM64_NEON_BLK select CRYPTO_AES select CRYPTO_ABLK_HELPER +config CRYPTO_CRC32_ARM64 + tristate "CRC32 and CRC32C using optional ARMv8 instructions" + depends on ARM64 + select CRYPTO_HASH endif diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index a3f935fde975..abb79b3cfcfe 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -29,10 +29,14 @@ aes-ce-blk-y := aes-glue-ce.o aes-ce.o obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o aes-neon-blk-y := aes-glue-neon.o aes-neon.o -AFLAGS_aes-ce.o := -DINTERLEAVE=2 -DINTERLEAVE_INLINE +AFLAGS_aes-ce.o := -DINTERLEAVE=4 AFLAGS_aes-neon.o := -DINTERLEAVE=4 CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS +obj-$(CONFIG_CRYPTO_CRC32_ARM64) += crc32-arm64.o + +CFLAGS_crc32-arm64.o := -mcpu=generic+crc + $(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE $(call if_changed_rule,cc_o_c) diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index 432e4841cd81..a2a7fbcacc14 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -101,19 +101,19 @@ ENTRY(ce_aes_ccm_final) 0: mov v4.16b, v3.16b 1: ld1 {v5.2d}, [x2], #16 /* load next round key */ aese v0.16b, v4.16b - aese v1.16b, v4.16b aesmc v0.16b, v0.16b + aese v1.16b, v4.16b aesmc v1.16b, v1.16b 2: ld1 {v3.2d}, [x2], #16 /* load next round key */ aese v0.16b, v5.16b - aese v1.16b, v5.16b aesmc v0.16b, v0.16b + aese v1.16b, v5.16b aesmc v1.16b, v1.16b 3: ld1 {v4.2d}, [x2], #16 /* load next round key */ subs w3, w3, #3 aese v0.16b, v3.16b - aese v1.16b, v3.16b aesmc v0.16b, v0.16b + aese v1.16b, v3.16b aesmc v1.16b, v1.16b bpl 1b aese v0.16b, v4.16b @@ -146,19 +146,19 @@ ENDPROC(ce_aes_ccm_final) ld1 {v5.2d}, [x10], #16 /* load 2nd round key */ 2: /* inner loop: 3 rounds, 2x interleaved */ aese v0.16b, v4.16b - aese v1.16b, v4.16b aesmc v0.16b, v0.16b + aese v1.16b, v4.16b aesmc v1.16b, v1.16b 3: ld1 {v3.2d}, [x10], #16 /* load next round key */ aese v0.16b, v5.16b - aese v1.16b, v5.16b aesmc v0.16b, v0.16b + aese v1.16b, v5.16b aesmc v1.16b, v1.16b 4: ld1 {v4.2d}, [x10], #16 /* load next round key */ subs w7, w7, #3 aese v0.16b, v3.16b - aese v1.16b, v3.16b aesmc v0.16b, v0.16b + aese v1.16b, v3.16b aesmc v1.16b, v1.16b ld1 {v5.2d}, [x10], #16 /* load next round key */ bpl 2b diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 9e6cdde9b43d..3303e8a7b837 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -13,9 +13,11 @@ #include <crypto/aes.h> #include <crypto/algapi.h> #include <crypto/scatterwalk.h> -#include <linux/crypto.h> +#include <crypto/internal/aead.h> #include <linux/module.h> +#include "aes-ce-setkey.h" + static int num_rounds(struct crypto_aes_ctx *ctx) { /* @@ -48,7 +50,7 @@ static int ccm_setkey(struct crypto_aead *tfm, const u8 *in_key, struct crypto_aes_ctx *ctx = crypto_aead_ctx(tfm); int ret; - ret = crypto_aes_expand_key(ctx, in_key, key_len); + ret = ce_aes_expandkey(ctx, in_key, key_len); if (!ret) return 0; @@ -294,4 +296,4 @@ module_exit(aes_mod_exit); MODULE_DESCRIPTION("Synchronous AES in CCM mode using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("ccm(aes)"); +MODULE_ALIAS_CRYPTO("ccm(aes)"); diff --git a/arch/arm64/crypto/aes-ce-cipher.c b/arch/arm64/crypto/aes-ce-cipher.c index 2075e1acae6b..ce47792a983d 100644 --- a/arch/arm64/crypto/aes-ce-cipher.c +++ b/arch/arm64/crypto/aes-ce-cipher.c @@ -14,6 +14,8 @@ #include <linux/crypto.h> #include <linux/module.h> +#include "aes-ce-setkey.h" + MODULE_DESCRIPTION("Synchronous AES cipher using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); @@ -124,6 +126,114 @@ static void aes_cipher_decrypt(struct crypto_tfm *tfm, u8 dst[], u8 const src[]) kernel_neon_end(); } +/* + * aes_sub() - use the aese instruction to perform the AES sbox substitution + * on each byte in 'input' + */ +static u32 aes_sub(u32 input) +{ + u32 ret; + + __asm__("dup v1.4s, %w[in] ;" + "movi v0.16b, #0 ;" + "aese v0.16b, v1.16b ;" + "umov %w[out], v0.4s[0] ;" + + : [out] "=r"(ret) + : [in] "r"(input) + : "v0","v1"); + + return ret; +} + +int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + /* + * The AES key schedule round constants + */ + static u8 const rcon[] = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + }; + + u32 kwords = key_len / sizeof(u32); + struct aes_block *key_enc, *key_dec; + int i, j; + + if (key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key_enc, in_key, key_len); + ctx->key_length = key_len; + + kernel_neon_begin_partial(2); + for (i = 0; i < sizeof(rcon); i++) { + u32 *rki = ctx->key_enc + (i * kwords); + u32 *rko = rki + kwords; + + rko[0] = ror32(aes_sub(rki[kwords - 1]), 8) ^ rcon[i] ^ rki[0]; + rko[1] = rko[0] ^ rki[1]; + rko[2] = rko[1] ^ rki[2]; + rko[3] = rko[2] ^ rki[3]; + + if (key_len == AES_KEYSIZE_192) { + if (i >= 7) + break; + rko[4] = rko[3] ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + } else if (key_len == AES_KEYSIZE_256) { + if (i >= 6) + break; + rko[4] = aes_sub(rko[3]) ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + rko[6] = rko[5] ^ rki[6]; + rko[7] = rko[6] ^ rki[7]; + } + } + + /* + * Generate the decryption keys for the Equivalent Inverse Cipher. + * This involves reversing the order of the round keys, and applying + * the Inverse Mix Columns transformation on all but the first and + * the last one. + */ + key_enc = (struct aes_block *)ctx->key_enc; + key_dec = (struct aes_block *)ctx->key_dec; + j = num_rounds(ctx); + + key_dec[0] = key_enc[j]; + for (i = 1, j--; j > 0; i++, j--) + __asm__("ld1 {v0.16b}, %[in] ;" + "aesimc v1.16b, v0.16b ;" + "st1 {v1.16b}, %[out] ;" + + : [out] "=Q"(key_dec[i]) + : [in] "Q"(key_enc[j]) + : "v0","v1"); + key_dec[i] = key_enc[0]; + + kernel_neon_end(); + return 0; +} +EXPORT_SYMBOL(ce_aes_expandkey); + +int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = ce_aes_expandkey(ctx, in_key, key_len); + if (!ret) + return 0; + + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; +} +EXPORT_SYMBOL(ce_aes_setkey); + static struct crypto_alg aes_alg = { .cra_name = "aes", .cra_driver_name = "aes-ce", @@ -135,7 +245,7 @@ static struct crypto_alg aes_alg = { .cra_cipher = { .cia_min_keysize = AES_MIN_KEY_SIZE, .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = crypto_aes_set_key, + .cia_setkey = ce_aes_setkey, .cia_encrypt = aes_cipher_encrypt, .cia_decrypt = aes_cipher_decrypt } diff --git a/arch/arm64/crypto/aes-ce-setkey.h b/arch/arm64/crypto/aes-ce-setkey.h new file mode 100644 index 000000000000..f08a6471d034 --- /dev/null +++ b/arch/arm64/crypto/aes-ce-setkey.h @@ -0,0 +1,5 @@ + +int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len); +int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, + unsigned int key_len); diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S index 685a18f731eb..78f3cfe92c08 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/arch/arm64/crypto/aes-ce.S @@ -45,18 +45,14 @@ .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3 aes\de \i0\().16b, \k\().16b - .ifnb \i1 - aes\de \i1\().16b, \k\().16b - .ifnb \i3 - aes\de \i2\().16b, \k\().16b - aes\de \i3\().16b, \k\().16b - .endif - .endif aes\mc \i0\().16b, \i0\().16b .ifnb \i1 + aes\de \i1\().16b, \k\().16b aes\mc \i1\().16b, \i1\().16b .ifnb \i3 + aes\de \i2\().16b, \k\().16b aes\mc \i2\().16b, \i2\().16b + aes\de \i3\().16b, \k\().16b aes\mc \i3\().16b, \i3\().16b .endif .endif diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index 79cd911ef88c..05d9e16c0dfd 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -16,9 +16,13 @@ #include <linux/module.h> #include <linux/cpufeature.h> +#include "aes-ce-setkey.h" + #ifdef USE_V8_CRYPTO_EXTENSIONS #define MODE "ce" #define PRIO 300 +#define aes_setkey ce_aes_setkey +#define aes_expandkey ce_aes_expandkey #define aes_ecb_encrypt ce_aes_ecb_encrypt #define aes_ecb_decrypt ce_aes_ecb_decrypt #define aes_cbc_encrypt ce_aes_cbc_encrypt @@ -30,6 +34,8 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #else #define MODE "neon" #define PRIO 200 +#define aes_setkey crypto_aes_set_key +#define aes_expandkey crypto_aes_expand_key #define aes_ecb_encrypt neon_aes_ecb_encrypt #define aes_ecb_decrypt neon_aes_ecb_decrypt #define aes_cbc_encrypt neon_aes_cbc_encrypt @@ -38,10 +44,10 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); #define aes_xts_encrypt neon_aes_xts_encrypt #define aes_xts_decrypt neon_aes_xts_decrypt MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON"); -MODULE_ALIAS("ecb(aes)"); -MODULE_ALIAS("cbc(aes)"); -MODULE_ALIAS("ctr(aes)"); -MODULE_ALIAS("xts(aes)"); +MODULE_ALIAS_CRYPTO("ecb(aes)"); +MODULE_ALIAS_CRYPTO("cbc(aes)"); +MODULE_ALIAS_CRYPTO("ctr(aes)"); +MODULE_ALIAS_CRYPTO("xts(aes)"); #endif MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); @@ -79,10 +85,10 @@ static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key, struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); int ret; - ret = crypto_aes_expand_key(&ctx->key1, in_key, key_len / 2); + ret = aes_expandkey(&ctx->key1, in_key, key_len / 2); if (!ret) - ret = crypto_aes_expand_key(&ctx->key2, &in_key[key_len / 2], - key_len / 2); + ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2], + key_len / 2); if (!ret) return 0; @@ -278,7 +284,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__ecb-aes-" MODE, .cra_driver_name = "__driver-ecb-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, @@ -288,7 +295,7 @@ static struct crypto_alg aes_algs[] = { { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, - .setkey = crypto_aes_set_key, + .setkey = aes_setkey, .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, @@ -296,7 +303,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__cbc-aes-" MODE, .cra_driver_name = "__driver-cbc-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, @@ -306,7 +314,7 @@ static struct crypto_alg aes_algs[] = { { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, - .setkey = crypto_aes_set_key, + .setkey = aes_setkey, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, @@ -314,7 +322,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__ctr-aes-" MODE, .cra_driver_name = "__driver-ctr-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, @@ -324,7 +333,7 @@ static struct crypto_alg aes_algs[] = { { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, - .setkey = crypto_aes_set_key, + .setkey = aes_setkey, .encrypt = ctr_encrypt, .decrypt = ctr_encrypt, }, @@ -332,7 +341,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__xts-aes-" MODE, .cra_driver_name = "__driver-xts-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), .cra_alignmask = 7, diff --git a/arch/arm64/crypto/crc32-arm64.c b/arch/arm64/crypto/crc32-arm64.c new file mode 100644 index 000000000000..6a37c3c6b11d --- /dev/null +++ b/arch/arm64/crypto/crc32-arm64.c @@ -0,0 +1,290 @@ +/* + * crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions + * + * Module based on crypto/crc32c_generic.c + * + * CRC32 loop taken from Ed Nevill's Hadoop CRC patch + * http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E + * + * Using inline assembly instead of intrinsics in order to be backwards + * compatible with older compilers. + * + * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/unaligned/access_ok.h> +#include <linux/cpufeature.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> + +#include <crypto/internal/hash.h> + +MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>"); +MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions"); +MODULE_LICENSE("GPL v2"); + +#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) +#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value)) + +static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len) +{ + s64 length = len; + + while ((length -= sizeof(u64)) >= 0) { + CRC32X(crc, get_unaligned_le64(p)); + p += sizeof(u64); + } + + /* The following is more efficient than the straight loop */ + if (length & sizeof(u32)) { + CRC32W(crc, get_unaligned_le32(p)); + p += sizeof(u32); + } + if (length & sizeof(u16)) { + CRC32H(crc, get_unaligned_le16(p)); + p += sizeof(u16); + } + if (length & sizeof(u8)) + CRC32B(crc, *p); + + return crc; +} + +static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len) +{ + s64 length = len; + + while ((length -= sizeof(u64)) >= 0) { + CRC32CX(crc, get_unaligned_le64(p)); + p += sizeof(u64); + } + + /* The following is more efficient than the straight loop */ + if (length & sizeof(u32)) { + CRC32CW(crc, get_unaligned_le32(p)); + p += sizeof(u32); + } + if (length & sizeof(u16)) { + CRC32CH(crc, get_unaligned_le16(p)); + p += sizeof(u16); + } + if (length & sizeof(u8)) + CRC32CB(crc, *p); + + return crc; +} + +#define CHKSUM_BLOCK_SIZE 1 +#define CHKSUM_DIGEST_SIZE 4 + +struct chksum_ctx { + u32 key; +}; + +struct chksum_desc_ctx { + u32 crc; +}; + +static int chksum_init(struct shash_desc *desc) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + ctx->crc = mctx->key; + + return 0; +} + +/* + * Setting the seed allows arbitrary accumulators and flexible XOR policy + * If your algorithm starts with ~0, then XOR with ~0 before you set + * the seed. + */ +static int chksum_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(tfm); + + if (keylen != sizeof(mctx->key)) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + mctx->key = get_unaligned_le32(key); + return 0; +} + +static int chksum_update(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length); + return 0; +} + +static int chksumc_update(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length); + return 0; +} + +static int chksum_final(struct shash_desc *desc, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + put_unaligned_le32(ctx->crc, out); + return 0; +} + +static int chksumc_final(struct shash_desc *desc, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + put_unaligned_le32(~ctx->crc, out); + return 0; +} + +static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) +{ + put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out); + return 0; +} + +static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) +{ + put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out); + return 0; +} + +static int chksum_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + return __chksum_finup(ctx->crc, data, len, out); +} + +static int chksumc_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + return __chksumc_finup(ctx->crc, data, len, out); +} + +static int chksum_digest(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); + + return __chksum_finup(mctx->key, data, length, out); +} + +static int chksumc_digest(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); + + return __chksumc_finup(mctx->key, data, length, out); +} + +static int crc32_cra_init(struct crypto_tfm *tfm) +{ + struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = 0; + return 0; +} + +static int crc32c_cra_init(struct crypto_tfm *tfm) +{ + struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); + + mctx->key = ~0; + return 0; +} + +static struct shash_alg crc32_alg = { + .digestsize = CHKSUM_DIGEST_SIZE, + .setkey = chksum_setkey, + .init = chksum_init, + .update = chksum_update, + .final = chksum_final, + .finup = chksum_finup, + .digest = chksum_digest, + .descsize = sizeof(struct chksum_desc_ctx), + .base = { + .cra_name = "crc32", + .cra_driver_name = "crc32-arm64-hw", + .cra_priority = 300, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_alignmask = 0, + .cra_ctxsize = sizeof(struct chksum_ctx), + .cra_module = THIS_MODULE, + .cra_init = crc32_cra_init, + } +}; + +static struct shash_alg crc32c_alg = { + .digestsize = CHKSUM_DIGEST_SIZE, + .setkey = chksum_setkey, + .init = chksum_init, + .update = chksumc_update, + .final = chksumc_final, + .finup = chksumc_finup, + .digest = chksumc_digest, + .descsize = sizeof(struct chksum_desc_ctx), + .base = { + .cra_name = "crc32c", + .cra_driver_name = "crc32c-arm64-hw", + .cra_priority = 300, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_alignmask = 0, + .cra_ctxsize = sizeof(struct chksum_ctx), + .cra_module = THIS_MODULE, + .cra_init = crc32c_cra_init, + } +}; + +static int __init crc32_mod_init(void) +{ + int err; + + err = crypto_register_shash(&crc32_alg); + + if (err) + return err; + + err = crypto_register_shash(&crc32c_alg); + + if (err) { + crypto_unregister_shash(&crc32_alg); + return err; + } + + return 0; +} + +static void __exit crc32_mod_exit(void) +{ + crypto_unregister_shash(&crc32_alg); + crypto_unregister_shash(&crc32c_alg); +} + +module_cpu_feature_match(CRC32, crc32_mod_init); +module_exit(crc32_mod_exit); diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S index 09d57d98609c..033aae6d732a 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/arch/arm64/crypto/sha1-ce-core.S @@ -66,8 +66,8 @@ .word 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6 /* - * void sha1_ce_transform(int blocks, u8 const *src, u32 *state, - * u8 *head, long bytes) + * void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + * int blocks) */ ENTRY(sha1_ce_transform) /* load round constants */ @@ -78,25 +78,22 @@ ENTRY(sha1_ce_transform) ld1r {k3.4s}, [x6] /* load state */ - ldr dga, [x2] - ldr dgb, [x2, #16] + ldr dga, [x0] + ldr dgb, [x0, #16] - /* load partial state (if supplied) */ - cbz x3, 0f - ld1 {v8.4s-v11.4s}, [x3] - b 1f + /* load sha1_ce_state::finalize */ + ldr w4, [x0, #:lo12:sha1_ce_offsetof_finalize] /* load input */ 0: ld1 {v8.4s-v11.4s}, [x1], #64 - sub w0, w0, #1 + sub w2, w2, #1 -1: CPU_LE( rev32 v8.16b, v8.16b ) CPU_LE( rev32 v9.16b, v9.16b ) CPU_LE( rev32 v10.16b, v10.16b ) CPU_LE( rev32 v11.16b, v11.16b ) -2: add t0.4s, v8.4s, k0.4s +1: add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b add_update c, ev, k0, 8, 9, 10, 11, dgb @@ -127,15 +124,15 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - cbnz w0, 0b + cbnz w2, 0b /* * Final block: add padding and total bit count. - * Skip if we have no total byte count in x4. In that case, the input - * size was not a round multiple of the block size, and the padding is - * handled by the C code. + * Skip if the input size was not a round multiple of the block size, + * the padding is handled by the C code in that case. */ cbz x4, 3f + ldr x4, [x0, #:lo12:sha1_ce_offsetof_count] movi v9.2d, #0 mov x8, #0x80000000 movi v10.2d, #0 @@ -144,10 +141,10 @@ CPU_LE( rev32 v11.16b, v11.16b ) mov x4, #0 mov v11.d[0], xzr mov v11.d[1], x7 - b 2b + b 1b /* store new state */ -3: str dga, [x2] - str dgb, [x2, #16] +3: str dga, [x0] + str dgb, [x0, #16] ret ENDPROC(sha1_ce_transform) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index 6fe83f37a750..aefda9868627 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -12,144 +12,84 @@ #include <asm/unaligned.h> #include <crypto/internal/hash.h> #include <crypto/sha.h> +#include <crypto/sha1_base.h> #include <linux/cpufeature.h> #include <linux/crypto.h> #include <linux/module.h> +#define ASM_EXPORT(sym, val) \ + asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); + MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -asmlinkage void sha1_ce_transform(int blocks, u8 const *src, u32 *state, - u8 *head, long bytes); +struct sha1_ce_state { + struct sha1_state sst; + u32 finalize; +}; -static int sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); +asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + int blocks); - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - return 0; -} - -static int sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha1_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->count += len; - - if ((partial + len) >= SHA1_BLOCK_SIZE) { - int blocks; - - if (partial) { - int p = SHA1_BLOCK_SIZE - partial; + struct sha1_ce_state *sctx = shash_desc_ctx(desc); - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - } - - blocks = len / SHA1_BLOCK_SIZE; - len %= SHA1_BLOCK_SIZE; - - kernel_neon_begin_partial(16); - sha1_ce_transform(blocks, data, sctx->state, - partial ? sctx->buffer : NULL, 0); - kernel_neon_end(); + sctx->finalize = 0; + kernel_neon_begin_partial(16); + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_ce_transform); + kernel_neon_end(); - data += blocks * SHA1_BLOCK_SIZE; - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); return 0; } -static int sha1_final(struct shash_desc *desc, u8 *out) +static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; - - struct sha1_state *sctx = shash_desc_ctx(desc); - __be64 bits = cpu_to_be64(sctx->count << 3); - __be32 *dst = (__be32 *)out; - int i; - - u32 padlen = SHA1_BLOCK_SIZE - - ((sctx->count + sizeof(bits)) % SHA1_BLOCK_SIZE); - - sha1_update(desc, padding, padlen); - sha1_update(desc, (const u8 *)&bits, sizeof(bits)); + struct sha1_ce_state *sctx = shash_desc_ctx(desc); + bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha1_state){}; - return 0; -} - -static int sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int blocks; - int i; - - if (sctx->count || !len || (len % SHA1_BLOCK_SIZE)) { - sha1_update(desc, data, len); - return sha1_final(desc, out); - } + ASM_EXPORT(sha1_ce_offsetof_count, + offsetof(struct sha1_ce_state, sst.count)); + ASM_EXPORT(sha1_ce_offsetof_finalize, + offsetof(struct sha1_ce_state, finalize)); /* - * Use a fast path if the input is a multiple of 64 bytes. In - * this case, there is no need to copy data around, and we can - * perform the entire digest calculation in a single invocation - * of sha1_ce_transform() + * Allow the asm code to perform the finalization if there is no + * partial data and the input is a round multiple of the block size. */ - blocks = len / SHA1_BLOCK_SIZE; + sctx->finalize = finalize; kernel_neon_begin_partial(16); - sha1_ce_transform(blocks, data, sctx->state, NULL, len); + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_ce_transform); + if (!finalize) + sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); kernel_neon_end(); - - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha1_state){}; - return 0; + return sha1_base_finish(desc, out); } -static int sha1_export(struct shash_desc *desc, void *out) +static int sha1_ce_final(struct shash_desc *desc, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - struct sha1_state *dst = out; + struct sha1_ce_state *sctx = shash_desc_ctx(desc); - *dst = *sctx; - return 0; -} - -static int sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - struct sha1_state const *src = in; - - *sctx = *src; - return 0; + sctx->finalize = 0; + kernel_neon_begin_partial(16); + sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); + kernel_neon_end(); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { - .init = sha1_init, - .update = sha1_update, - .final = sha1_final, - .finup = sha1_finup, - .export = sha1_export, - .import = sha1_import, - .descsize = sizeof(struct sha1_state), + .init = sha1_base_init, + .update = sha1_ce_update, + .final = sha1_ce_final, + .finup = sha1_ce_finup, + .descsize = sizeof(struct sha1_ce_state), .digestsize = SHA1_DIGEST_SIZE, - .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ce", diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S index 7f29fc031ea8..5df9d9d470ad 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/crypto/sha2-ce-core.S @@ -73,8 +73,8 @@ .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 /* - * void sha2_ce_transform(int blocks, u8 const *src, u32 *state, - * u8 *head, long bytes) + * void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, + * int blocks) */ ENTRY(sha2_ce_transform) /* load round constants */ @@ -85,24 +85,21 @@ ENTRY(sha2_ce_transform) ld1 {v12.4s-v15.4s}, [x8] /* load state */ - ldp dga, dgb, [x2] + ldp dga, dgb, [x0] - /* load partial input (if supplied) */ - cbz x3, 0f - ld1 {v16.4s-v19.4s}, [x3] - b 1f + /* load sha256_ce_state::finalize */ + ldr w4, [x0, #:lo12:sha256_ce_offsetof_finalize] /* load input */ 0: ld1 {v16.4s-v19.4s}, [x1], #64 - sub w0, w0, #1 + sub w2, w2, #1 -1: CPU_LE( rev32 v16.16b, v16.16b ) CPU_LE( rev32 v17.16b, v17.16b ) CPU_LE( rev32 v18.16b, v18.16b ) CPU_LE( rev32 v19.16b, v19.16b ) -2: add t0.4s, v16.4s, v0.4s +1: add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b mov dg1v.16b, dgbv.16b @@ -131,15 +128,15 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgbv.4s, dgbv.4s, dg1v.4s /* handled all input blocks? */ - cbnz w0, 0b + cbnz w2, 0b /* * Final block: add padding and total bit count. - * Skip if we have no total byte count in x4. In that case, the input - * size was not a round multiple of the block size, and the padding is - * handled by the C code. + * Skip if the input size was not a round multiple of the block size, + * the padding is handled by the C code in that case. */ cbz x4, 3f + ldr x4, [x0, #:lo12:sha256_ce_offsetof_count] movi v17.2d, #0 mov x8, #0x80000000 movi v18.2d, #0 @@ -148,9 +145,9 @@ CPU_LE( rev32 v19.16b, v19.16b ) mov x4, #0 mov v19.d[0], xzr mov v19.d[1], x7 - b 2b + b 1b /* store new state */ -3: stp dga, dgb, [x2] +3: stp dga, dgb, [x0] ret ENDPROC(sha2_ce_transform) diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index ae67e88c28b9..7cd587564a41 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -12,206 +12,85 @@ #include <asm/unaligned.h> #include <crypto/internal/hash.h> #include <crypto/sha.h> +#include <crypto/sha256_base.h> #include <linux/cpufeature.h> #include <linux/crypto.h> #include <linux/module.h> +#define ASM_EXPORT(sym, val) \ + asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); + MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -asmlinkage int sha2_ce_transform(int blocks, u8 const *src, u32 *state, - u8 *head, long bytes); - -static int sha224_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha256_state){ - .state = { - SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, - SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, - } - }; - return 0; -} - -static int sha256_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha256_state){ - .state = { - SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, - } - }; - return 0; -} - -static int sha2_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->count += len; - - if ((partial + len) >= SHA256_BLOCK_SIZE) { - int blocks; - - if (partial) { - int p = SHA256_BLOCK_SIZE - partial; - - memcpy(sctx->buf + partial, data, p); - data += p; - len -= p; - } - - blocks = len / SHA256_BLOCK_SIZE; - len %= SHA256_BLOCK_SIZE; - - kernel_neon_begin_partial(28); - sha2_ce_transform(blocks, data, sctx->state, - partial ? sctx->buf : NULL, 0); - kernel_neon_end(); - - data += blocks * SHA256_BLOCK_SIZE; - partial = 0; - } - if (len) - memcpy(sctx->buf + partial, data, len); - return 0; -} - -static void sha2_final(struct shash_desc *desc) -{ - static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; - - struct sha256_state *sctx = shash_desc_ctx(desc); - __be64 bits = cpu_to_be64(sctx->count << 3); - u32 padlen = SHA256_BLOCK_SIZE - - ((sctx->count + sizeof(bits)) % SHA256_BLOCK_SIZE); - - sha2_update(desc, padding, padlen); - sha2_update(desc, (const u8 *)&bits, sizeof(bits)); -} - -static int sha224_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_final(desc); +struct sha256_ce_state { + struct sha256_state sst; + u32 finalize; +}; - for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); +asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, + int blocks); - *sctx = (struct sha256_state){}; - return 0; -} - -static int sha256_final(struct shash_desc *desc, u8 *out) +static int sha256_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_final(desc); + struct sha256_ce_state *sctx = shash_desc_ctx(desc); - for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); + sctx->finalize = 0; + kernel_neon_begin_partial(28); + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha2_ce_transform); + kernel_neon_end(); - *sctx = (struct sha256_state){}; return 0; } -static void sha2_finup(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - int blocks; + struct sha256_ce_state *sctx = shash_desc_ctx(desc); + bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); - if (sctx->count || !len || (len % SHA256_BLOCK_SIZE)) { - sha2_update(desc, data, len); - sha2_final(desc); - return; - } + ASM_EXPORT(sha256_ce_offsetof_count, + offsetof(struct sha256_ce_state, sst.count)); + ASM_EXPORT(sha256_ce_offsetof_finalize, + offsetof(struct sha256_ce_state, finalize)); /* - * Use a fast path if the input is a multiple of 64 bytes. In - * this case, there is no need to copy data around, and we can - * perform the entire digest calculation in a single invocation - * of sha2_ce_transform() + * Allow the asm code to perform the finalization if there is no + * partial data and the input is a round multiple of the block size. */ - blocks = len / SHA256_BLOCK_SIZE; + sctx->finalize = finalize; kernel_neon_begin_partial(28); - sha2_ce_transform(blocks, data, sctx->state, NULL, len); + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha2_ce_transform); + if (!finalize) + sha256_base_do_finalize(desc, + (sha256_block_fn *)sha2_ce_transform); kernel_neon_end(); + return sha256_base_finish(desc, out); } -static int sha224_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int sha256_ce_final(struct shash_desc *desc, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_finup(desc, data, len); - - for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha256_state){}; - return 0; -} - -static int sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_finup(desc, data, len); + struct sha256_ce_state *sctx = shash_desc_ctx(desc); - for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha256_state){}; - return 0; -} - -static int sha2_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - struct sha256_state *dst = out; - - *dst = *sctx; - return 0; -} - -static int sha2_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - struct sha256_state const *src = in; - - *sctx = *src; - return 0; + sctx->finalize = 0; + kernel_neon_begin_partial(28); + sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); + kernel_neon_end(); + return sha256_base_finish(desc, out); } static struct shash_alg algs[] = { { - .init = sha224_init, - .update = sha2_update, - .final = sha224_final, - .finup = sha224_finup, - .export = sha2_export, - .import = sha2_import, - .descsize = sizeof(struct sha256_state), + .init = sha224_base_init, + .update = sha256_ce_update, + .final = sha256_ce_final, + .finup = sha256_ce_finup, + .descsize = sizeof(struct sha256_ce_state), .digestsize = SHA224_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ce", @@ -221,15 +100,12 @@ static struct shash_alg algs[] = { { .cra_module = THIS_MODULE, } }, { - .init = sha256_init, - .update = sha2_update, - .final = sha256_final, - .finup = sha256_finup, - .export = sha2_export, - .import = sha2_import, - .descsize = sizeof(struct sha256_state), + .init = sha256_base_init, + .update = sha256_ce_update, + .final = sha256_ce_final, + .finup = sha256_ce_finup, + .descsize = sizeof(struct sha256_ce_state), .digestsize = SHA256_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ce", diff --git a/arch/arm64/include/asm/Kbuild b/arch/arm64/include/asm/Kbuild index dc770bd4f5a5..55103e50c51b 100644 --- a/arch/arm64/include/asm/Kbuild +++ b/arch/arm64/include/asm/Kbuild @@ -14,7 +14,6 @@ generic-y += early_ioremap.h generic-y += emergency-restart.h generic-y += errno.h generic-y += ftrace.h -generic-y += hash.h generic-y += hw_irq.h generic-y += ioctl.h generic-y += ioctls.h @@ -28,6 +27,7 @@ generic-y += local64.h generic-y += mcs_spinlock.h generic-y += mman.h generic-y += msgbuf.h +generic-y += msi.h generic-y += mutex.h generic-y += pci.h generic-y += pci-bridge.h diff --git a/arch/arm64/include/asm/acenv.h b/arch/arm64/include/asm/acenv.h new file mode 100644 index 000000000000..b49166fde7ea --- /dev/null +++ b/arch/arm64/include/asm/acenv.h @@ -0,0 +1,18 @@ +/* + * ARM64 specific ACPICA environments and implementation + * + * Copyright (C) 2014, Linaro Ltd. + * Author: Hanjun Guo <hanjun.guo@linaro.org> + * Author: Graeme Gregory <graeme.gregory@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef _ASM_ACENV_H +#define _ASM_ACENV_H + +/* It is required unconditionally by ACPI core, update it when needed. */ + +#endif /* _ASM_ACENV_H */ diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h new file mode 100644 index 000000000000..59c05d8ea4a0 --- /dev/null +++ b/arch/arm64/include/asm/acpi.h @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2013-2014, Linaro Ltd. + * Author: Al Stone <al.stone@linaro.org> + * Author: Graeme Gregory <graeme.gregory@linaro.org> + * Author: Hanjun Guo <hanjun.guo@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation; + */ + +#ifndef _ASM_ACPI_H +#define _ASM_ACPI_H + +#include <linux/mm.h> +#include <linux/irqchip/arm-gic-acpi.h> + +#include <asm/cputype.h> +#include <asm/smp_plat.h> + +/* Basic configuration for ACPI */ +#ifdef CONFIG_ACPI +/* ACPI table mapping after acpi_gbl_permanent_mmap is set */ +static inline void __iomem *acpi_os_ioremap(acpi_physical_address phys, + acpi_size size) +{ + if (!page_is_ram(phys >> PAGE_SHIFT)) + return ioremap(phys, size); + + return ioremap_cache(phys, size); +} +#define acpi_os_ioremap acpi_os_ioremap + +typedef u64 phys_cpuid_t; +#define PHYS_CPUID_INVALID INVALID_HWID + +#define acpi_strict 1 /* No out-of-spec workarounds on ARM64 */ +extern int acpi_disabled; +extern int acpi_noirq; +extern int acpi_pci_disabled; + +/* 1 to indicate PSCI 0.2+ is implemented */ +static inline bool acpi_psci_present(void) +{ + return acpi_gbl_FADT.arm_boot_flags & ACPI_FADT_PSCI_COMPLIANT; +} + +/* 1 to indicate HVC must be used instead of SMC as the PSCI conduit */ +static inline bool acpi_psci_use_hvc(void) +{ + return acpi_gbl_FADT.arm_boot_flags & ACPI_FADT_PSCI_USE_HVC; +} + +static inline void disable_acpi(void) +{ + acpi_disabled = 1; + acpi_pci_disabled = 1; + acpi_noirq = 1; +} + +static inline void enable_acpi(void) +{ + acpi_disabled = 0; + acpi_pci_disabled = 0; + acpi_noirq = 0; +} + +/* + * The ACPI processor driver for ACPI core code needs this macro + * to find out this cpu was already mapped (mapping from CPU hardware + * ID to CPU logical ID) or not. + */ +#define cpu_physical_id(cpu) cpu_logical_map(cpu) + +/* + * It's used from ACPI core in kdump to boot UP system with SMP kernel, + * with this check the ACPI core will not override the CPU index + * obtained from GICC with 0 and not print some error message as well. + * Since MADT must provide at least one GICC structure for GIC + * initialization, CPU will be always available in MADT on ARM64. + */ +static inline bool acpi_has_cpu_in_madt(void) +{ + return true; +} + +static inline void arch_fix_phys_package_id(int num, u32 slot) { } +void __init acpi_init_cpus(void); + +#else +static inline bool acpi_psci_present(void) { return false; } +static inline bool acpi_psci_use_hvc(void) { return false; } +static inline void acpi_init_cpus(void) { } +#endif /* CONFIG_ACPI */ + +#endif /*_ASM_ACPI_H*/ diff --git a/arch/arm64/include/asm/alternative-asm.h b/arch/arm64/include/asm/alternative-asm.h new file mode 100644 index 000000000000..919a67855b63 --- /dev/null +++ b/arch/arm64/include/asm/alternative-asm.h @@ -0,0 +1,29 @@ +#ifndef __ASM_ALTERNATIVE_ASM_H +#define __ASM_ALTERNATIVE_ASM_H + +#ifdef __ASSEMBLY__ + +.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len + .word \orig_offset - . + .word \alt_offset - . + .hword \feature + .byte \orig_len + .byte \alt_len +.endm + +.macro alternative_insn insn1 insn2 cap +661: \insn1 +662: .pushsection .altinstructions, "a" + altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f + .popsection + .pushsection .altinstr_replacement, "ax" +663: \insn2 +664: .popsection + .if ((664b-663b) != (662b-661b)) + .error "Alternatives instruction length mismatch" + .endif +.endm + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_ALTERNATIVE_ASM_H */ diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h new file mode 100644 index 000000000000..d261f01e2bae --- /dev/null +++ b/arch/arm64/include/asm/alternative.h @@ -0,0 +1,44 @@ +#ifndef __ASM_ALTERNATIVE_H +#define __ASM_ALTERNATIVE_H + +#include <linux/types.h> +#include <linux/stddef.h> +#include <linux/stringify.h> + +struct alt_instr { + s32 orig_offset; /* offset to original instruction */ + s32 alt_offset; /* offset to replacement instruction */ + u16 cpufeature; /* cpufeature bit set for replacement */ + u8 orig_len; /* size of original instruction(s) */ + u8 alt_len; /* size of new instruction(s), <= orig_len */ +}; + +void apply_alternatives_all(void); +void apply_alternatives(void *start, size_t length); +void free_alternatives_memory(void); + +#define ALTINSTR_ENTRY(feature) \ + " .word 661b - .\n" /* label */ \ + " .word 663f - .\n" /* new instruction */ \ + " .hword " __stringify(feature) "\n" /* feature bit */ \ + " .byte 662b-661b\n" /* source len */ \ + " .byte 664f-663f\n" /* replacement len */ + +/* alternative assembly primitive: */ +#define ALTERNATIVE(oldinstr, newinstr, feature) \ + "661:\n\t" \ + oldinstr "\n" \ + "662:\n" \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY(feature) \ + ".popsection\n" \ + ".pushsection .altinstr_replacement, \"a\"\n" \ + "663:\n\t" \ + newinstr "\n" \ + "664:\n\t" \ + ".popsection\n\t" \ + ".if ((664b-663b) != (662b-661b))\n\t" \ + " .error \"Alternatives instruction length mismatch\"\n\t"\ + ".endif\n" + +#endif /* __ASM_ALTERNATIVE_H */ diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index f19097134b02..fbe0ca31a99c 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -21,6 +21,7 @@ #include <asm/barrier.h> +#include <linux/bug.h> #include <linux/init.h> #include <linux/types.h> @@ -104,6 +105,15 @@ static inline void arch_timer_set_cntkctl(u32 cntkctl) asm volatile("msr cntkctl_el1, %0" : : "r" (cntkctl)); } +static inline u64 arch_counter_get_cntpct(void) +{ + /* + * AArch64 kernel and user space mandate the use of CNTVCT. + */ + BUG(); + return 0; +} + static inline u64 arch_counter_get_cntvct(void) { u64 cval; diff --git a/arch/arm64/include/asm/syscalls.h b/arch/arm64/include/asm/arm-cci.h index 48fe7c600e98..f0b63712e10e 100644 --- a/arch/arm64/include/asm/syscalls.h +++ b/arch/arm64/include/asm/arm-cci.h @@ -1,5 +1,7 @@ /* - * Copyright (C) 2012 ARM Ltd. + * arch/arm64/include/asm/arm-cci.h + * + * Copyright (C) 2015 ARM Ltd. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -13,18 +15,13 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef __ASM_SYSCALLS_H -#define __ASM_SYSCALLS_H - -#include <linux/linkage.h> -#include <linux/compiler.h> -#include <linux/signal.h> -/* - * System call wrappers implemented in kernel/entry.S. - */ -asmlinkage long sys_rt_sigreturn_wrapper(void); +#ifndef __ASM_ARM_CCI_H +#define __ASM_ARM_CCI_H -#include <asm-generic/syscalls.h> +static inline bool platform_has_secure_cci_access(void) +{ + return false; +} -#endif /* __ASM_SYSCALLS_H */ +#endif diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 5901480bfdca..144b64ad96c3 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -20,6 +20,9 @@ #error "Only include this from assembly code" #endif +#ifndef __ASM_ASSEMBLER_H +#define __ASM_ASSEMBLER_H + #include <asm/ptrace.h> #include <asm/thread_info.h> @@ -155,3 +158,53 @@ lr .req x30 // link register #endif orr \rd, \lbits, \hbits, lsl #32 .endm + +/* + * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where + * <symbol> is within the range +/- 4 GB of the PC. + */ + /* + * @dst: destination register (64 bit wide) + * @sym: name of the symbol + * @tmp: optional scratch register to be used if <dst> == sp, which + * is not allowed in an adrp instruction + */ + .macro adr_l, dst, sym, tmp= + .ifb \tmp + adrp \dst, \sym + add \dst, \dst, :lo12:\sym + .else + adrp \tmp, \sym + add \dst, \tmp, :lo12:\sym + .endif + .endm + + /* + * @dst: destination register (32 or 64 bit wide) + * @sym: name of the symbol + * @tmp: optional 64-bit scratch register to be used if <dst> is a + * 32-bit wide register, in which case it cannot be used to hold + * the address + */ + .macro ldr_l, dst, sym, tmp= + .ifb \tmp + adrp \dst, \sym + ldr \dst, [\dst, :lo12:\sym] + .else + adrp \tmp, \sym + ldr \dst, [\tmp, :lo12:\sym] + .endif + .endm + + /* + * @src: source register (32 or 64 bit wide) + * @sym: name of the symbol + * @tmp: mandatory 64-bit scratch register to calculate the address + * while <src> needs to be preserved. + */ + .macro str_l, src, sym, tmp + adrp \tmp, \sym + str \src, [\tmp, :lo12:\sym] + .endm + +#endif /* __ASM_ASSEMBLER_H */ diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 6389d60574d9..0fa47c4275cb 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -32,6 +32,9 @@ #define rmb() dsb(ld) #define wmb() dsb(st) +#define dma_rmb() dmb(oshld) +#define dma_wmb() dmb(oshst) + #ifndef CONFIG_SMP #define smp_mb() barrier() #define smp_rmb() barrier() @@ -62,6 +65,14 @@ do { \ do { \ compiletime_assert_atomic_type(*p); \ switch (sizeof(*p)) { \ + case 1: \ + asm volatile ("stlrb %w1, %0" \ + : "=Q" (*p) : "r" (v) : "memory"); \ + break; \ + case 2: \ + asm volatile ("stlrh %w1, %0" \ + : "=Q" (*p) : "r" (v) : "memory"); \ + break; \ case 4: \ asm volatile ("stlr %w1, %0" \ : "=Q" (*p) : "r" (v) : "memory"); \ @@ -78,6 +89,14 @@ do { \ typeof(*p) ___p1; \ compiletime_assert_atomic_type(*p); \ switch (sizeof(*p)) { \ + case 1: \ + asm volatile ("ldarb %w0, %1" \ + : "=r" (___p1) : "Q" (*p) : "memory"); \ + break; \ + case 2: \ + asm volatile ("ldarh %w0, %1" \ + : "=r" (___p1) : "Q" (*p) : "memory"); \ + break; \ case 4: \ asm volatile ("ldar %w0, %1" \ : "=r" (___p1) : "Q" (*p) : "memory"); \ @@ -95,7 +114,7 @@ do { \ #define read_barrier_depends() do { } while(0) #define smp_read_barrier_depends() do { } while(0) -#define set_mb(var, value) do { var = value; smp_mb(); } while (0) +#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) #define nop() asm volatile("nop"); #define smp_mb__before_atomic() smp_mb() diff --git a/arch/arm64/include/asm/bitrev.h b/arch/arm64/include/asm/bitrev.h new file mode 100644 index 000000000000..a5a0c3660137 --- /dev/null +++ b/arch/arm64/include/asm/bitrev.h @@ -0,0 +1,19 @@ +#ifndef __ASM_BITREV_H +#define __ASM_BITREV_H +static __always_inline __attribute_const__ u32 __arch_bitrev32(u32 x) +{ + __asm__ ("rbit %w0, %w1" : "=r" (x) : "r" (x)); + return x; +} + +static __always_inline __attribute_const__ u16 __arch_bitrev16(u16 x) +{ + return __arch_bitrev32((u32)x) >> 16; +} + +static __always_inline __attribute_const__ u8 __arch_bitrev8(u8 x) +{ + return __arch_bitrev32((u32)x) >> 24; +} + +#endif diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index 88cc05b5f3ac..bde449936e2f 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -32,6 +32,8 @@ #ifndef __ASSEMBLY__ +#define __read_mostly __attribute__((__section__(".data..read_mostly"))) + static inline int cache_line_size(void) { u32 cwg = cache_type_cwg(); diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 689b6379188c..67d309cc3b6b 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -73,7 +73,7 @@ extern void flush_cache_all(void); extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void flush_icache_range(unsigned long start, unsigned long end); extern void __flush_dcache_area(void *addr, size_t len); -extern void __flush_cache_user_range(unsigned long start, unsigned long end); +extern long __flush_cache_user_range(unsigned long start, unsigned long end); static inline void flush_cache_mm(struct mm_struct *mm) { @@ -152,4 +152,9 @@ int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); int set_memory_x(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages); + +#ifdef CONFIG_DEBUG_RODATA +void mark_rodata_ro(void); +#endif + #endif diff --git a/arch/arm64/include/asm/cachetype.h b/arch/arm64/include/asm/cachetype.h index 4c631a0a3609..da2fc9e3cedd 100644 --- a/arch/arm64/include/asm/cachetype.h +++ b/arch/arm64/include/asm/cachetype.h @@ -39,24 +39,41 @@ extern unsigned long __icache_flags; +/* + * NumSets, bits[27:13] - (Number of sets in cache) - 1 + * Associativity, bits[12:3] - (Associativity of cache) - 1 + * LineSize, bits[2:0] - (Log2(Number of words in cache line)) - 2 + */ +#define CCSIDR_EL1_WRITE_THROUGH BIT(31) +#define CCSIDR_EL1_WRITE_BACK BIT(30) +#define CCSIDR_EL1_READ_ALLOCATE BIT(29) +#define CCSIDR_EL1_WRITE_ALLOCATE BIT(28) #define CCSIDR_EL1_LINESIZE_MASK 0x7 #define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK) - +#define CCSIDR_EL1_ASSOCIATIVITY_SHIFT 3 +#define CCSIDR_EL1_ASSOCIATIVITY_MASK 0x3ff +#define CCSIDR_EL1_ASSOCIATIVITY(x) \ + (((x) >> CCSIDR_EL1_ASSOCIATIVITY_SHIFT) & CCSIDR_EL1_ASSOCIATIVITY_MASK) #define CCSIDR_EL1_NUMSETS_SHIFT 13 -#define CCSIDR_EL1_NUMSETS_MASK (0x7fff << CCSIDR_EL1_NUMSETS_SHIFT) +#define CCSIDR_EL1_NUMSETS_MASK 0x7fff #define CCSIDR_EL1_NUMSETS(x) \ - (((x) & CCSIDR_EL1_NUMSETS_MASK) >> CCSIDR_EL1_NUMSETS_SHIFT) + (((x) >> CCSIDR_EL1_NUMSETS_SHIFT) & CCSIDR_EL1_NUMSETS_MASK) + +#define CACHE_LINESIZE(x) (16 << CCSIDR_EL1_LINESIZE(x)) +#define CACHE_NUMSETS(x) (CCSIDR_EL1_NUMSETS(x) + 1) +#define CACHE_ASSOCIATIVITY(x) (CCSIDR_EL1_ASSOCIATIVITY(x) + 1) -extern u64 __attribute_const__ icache_get_ccsidr(void); +extern u64 __attribute_const__ cache_get_ccsidr(u64 csselr); +/* Helpers for Level 1 Instruction cache csselr = 1L */ static inline int icache_get_linesize(void) { - return 16 << CCSIDR_EL1_LINESIZE(icache_get_ccsidr()); + return CACHE_LINESIZE(cache_get_ccsidr(1L)); } static inline int icache_get_numsets(void) { - return 1 + CCSIDR_EL1_NUMSETS(icache_get_ccsidr()); + return CACHE_NUMSETS(cache_get_ccsidr(1L)); } /* diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index ddb9d7830558..d8c25b7b18fb 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -19,6 +19,7 @@ #define __ASM_CMPXCHG_H #include <linux/bug.h> +#include <linux/mmdebug.h> #include <asm/barrier.h> @@ -152,6 +153,51 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, return oldval; } +#define system_has_cmpxchg_double() 1 + +static inline int __cmpxchg_double(volatile void *ptr1, volatile void *ptr2, + unsigned long old1, unsigned long old2, + unsigned long new1, unsigned long new2, int size) +{ + unsigned long loop, lost; + + switch (size) { + case 8: + VM_BUG_ON((unsigned long *)ptr2 - (unsigned long *)ptr1 != 1); + do { + asm volatile("// __cmpxchg_double8\n" + " ldxp %0, %1, %2\n" + " eor %0, %0, %3\n" + " eor %1, %1, %4\n" + " orr %1, %0, %1\n" + " mov %w0, #0\n" + " cbnz %1, 1f\n" + " stxp %w0, %5, %6, %2\n" + "1:\n" + : "=&r"(loop), "=&r"(lost), "+Q" (*(u64 *)ptr1) + : "r" (old1), "r"(old2), "r"(new1), "r"(new2)); + } while (loop); + break; + default: + BUILD_BUG(); + } + + return !lost; +} + +static inline int __cmpxchg_double_mb(volatile void *ptr1, volatile void *ptr2, + unsigned long old1, unsigned long old2, + unsigned long new1, unsigned long new2, int size) +{ + int ret; + + smp_mb(); + ret = __cmpxchg_double(ptr1, ptr2, old1, old2, new1, new2, size); + smp_mb(); + + return ret; +} + static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, unsigned long new, int size) { @@ -182,6 +228,49 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, __ret; \ }) +#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \ +({\ + int __ret;\ + __ret = __cmpxchg_double_mb((ptr1), (ptr2), (unsigned long)(o1), \ + (unsigned long)(o2), (unsigned long)(n1), \ + (unsigned long)(n2), sizeof(*(ptr1)));\ + __ret; \ +}) + +#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \ +({\ + int __ret;\ + __ret = __cmpxchg_double((ptr1), (ptr2), (unsigned long)(o1), \ + (unsigned long)(o2), (unsigned long)(n1), \ + (unsigned long)(n2), sizeof(*(ptr1)));\ + __ret; \ +}) + +#define _protect_cmpxchg_local(pcp, o, n) \ +({ \ + typeof(*raw_cpu_ptr(&(pcp))) __ret; \ + preempt_disable(); \ + __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \ + preempt_enable(); \ + __ret; \ +}) + +#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) +#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) +#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) +#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n) + +#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ +({ \ + int __ret; \ + preempt_disable(); \ + __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \ + raw_cpu_ptr(&(ptr2)), \ + o1, o2, n1, n2); \ + preempt_enable(); \ + __ret; \ +}) + #define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) #define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h index 56de5aadede2..7fbed6919b54 100644 --- a/arch/arm64/include/asm/compat.h +++ b/arch/arm64/include/asm/compat.h @@ -161,7 +161,6 @@ typedef struct compat_siginfo { int si_code; union { - /* The padding is the same size as AArch64. */ int _pad[128/sizeof(int) - 3]; /* kill() */ @@ -205,6 +204,13 @@ typedef struct compat_siginfo { compat_long_t _band; /* POLL_IN, POLL_OUT, POLL_MSG */ int _fd; } _sigpoll; + + /* SIGSYS */ + struct { + compat_uptr_t _call_addr; /* calling user insn */ + int _syscall; /* triggering system call number */ + compat_uint_t _arch; /* AUDIT_ARCH_* of syscall */ + } _sigsys; } _sifields; } compat_siginfo_t; diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index 056443086019..8e797b2fcc01 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -30,6 +30,8 @@ struct cpuinfo_arm64 { u32 reg_dczid; u32 reg_midr; + u64 reg_id_aa64dfr0; + u64 reg_id_aa64dfr1; u64 reg_id_aa64isar0; u64 reg_id_aa64isar1; u64 reg_id_aa64mmfr0; @@ -37,6 +39,7 @@ struct cpuinfo_arm64 { u64 reg_id_aa64pfr0; u64 reg_id_aa64pfr1; + u32 reg_id_dfr0; u32 reg_id_isar0; u32 reg_id_isar1; u32 reg_id_isar2; @@ -49,6 +52,10 @@ struct cpuinfo_arm64 { u32 reg_id_mmfr3; u32 reg_id_pfr0; u32 reg_id_pfr1; + + u32 reg_mvfr0; + u32 reg_mvfr1; + u32 reg_mvfr2; }; DECLARE_PER_CPU(struct cpuinfo_arm64, cpu_data); diff --git a/arch/arm64/include/asm/cpu_ops.h b/arch/arm64/include/asm/cpu_ops.h index 6f8e2ef9094a..5a31d6716914 100644 --- a/arch/arm64/include/asm/cpu_ops.h +++ b/arch/arm64/include/asm/cpu_ops.h @@ -28,8 +28,6 @@ struct device_node; * enable-method property. * @cpu_init: Reads any data necessary for a specific enable-method from the * devicetree, for a given cpu node and proposed logical id. - * @cpu_init_idle: Reads any data necessary to initialize CPU idle states from - * devicetree, for a given cpu node and proposed logical id. * @cpu_prepare: Early one-time preparation step for a cpu. If there is a * mechanism for doing so, tests whether it is possible to boot * the given CPU. @@ -42,6 +40,8 @@ struct device_node; * @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the * cpu being killed. * @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu. + * @cpu_init_idle: Reads any data necessary to initialize CPU idle states from + * devicetree, for a given cpu node and proposed logical id. * @cpu_suspend: Suspends a cpu and saves the required context. May fail owing * to wrong parameters or error conditions. Called from the * CPU being suspended. Must be called with IRQs disabled. @@ -49,7 +49,6 @@ struct device_node; struct cpu_operations { const char *name; int (*cpu_init)(struct device_node *, unsigned int); - int (*cpu_init_idle)(struct device_node *, unsigned int); int (*cpu_prepare)(unsigned int); int (*cpu_boot)(unsigned int); void (*cpu_postboot)(void); @@ -58,7 +57,8 @@ struct cpu_operations { void (*cpu_die)(unsigned int cpu); int (*cpu_kill)(unsigned int cpu); #endif -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_IDLE + int (*cpu_init_idle)(struct device_node *, unsigned int); int (*cpu_suspend)(unsigned long); #endif }; @@ -66,5 +66,6 @@ struct cpu_operations { extern const struct cpu_operations *cpu_ops[NR_CPUS]; int __init cpu_read_ops(struct device_node *dn, int cpu); void __init cpu_read_bootcpu_ops(void); +const struct cpu_operations *cpu_get_ops(const char *name); #endif /* ifndef __ASM_CPU_OPS_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index cd4ac0516488..82cb9f98ba1a 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -21,9 +21,56 @@ #define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap)) #define cpu_feature(x) ilog2(HWCAP_ ## x) +#define ARM64_WORKAROUND_CLEAN_CACHE 0 +#define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 +#define ARM64_WORKAROUND_845719 2 + +#define ARM64_NCAPS 3 + +#ifndef __ASSEMBLY__ + +struct arm64_cpu_capabilities { + const char *desc; + u16 capability; + bool (*matches)(const struct arm64_cpu_capabilities *); + union { + struct { /* To be used for erratum handling only */ + u32 midr_model; + u32 midr_range_min, midr_range_max; + }; + }; +}; + +extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); + static inline bool cpu_have_feature(unsigned int num) { return elf_hwcap & (1UL << num); } +static inline bool cpus_have_cap(unsigned int num) +{ + if (num >= ARM64_NCAPS) + return false; + return test_bit(num, cpu_hwcaps); +} + +static inline void cpus_set_cap(unsigned int num) +{ + if (num >= ARM64_NCAPS) + pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", + num, ARM64_NCAPS); + else + __set_bit(num, cpu_hwcaps); +} + +void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, + const char *info); +void check_local_cpu_errata(void); +void check_local_cpu_features(void); +bool cpu_supports_mixed_endian_el0(void); +bool system_supports_mixed_endian_el0(void); + +#endif /* __ASSEMBLY__ */ + #endif diff --git a/arch/arm64/include/asm/cpuidle.h b/arch/arm64/include/asm/cpuidle.h index b52a9932e2b1..141b2fcabaa6 100644 --- a/arch/arm64/include/asm/cpuidle.h +++ b/arch/arm64/include/asm/cpuidle.h @@ -1,13 +1,24 @@ #ifndef __ASM_CPUIDLE_H #define __ASM_CPUIDLE_H +#include <asm/proc-fns.h> + #ifdef CONFIG_CPU_IDLE -extern int cpu_init_idle(unsigned int cpu); +extern int arm_cpuidle_init(unsigned int cpu); +extern int cpu_suspend(unsigned long arg); #else -static inline int cpu_init_idle(unsigned int cpu) +static inline int arm_cpuidle_init(unsigned int cpu) { return -EOPNOTSUPP; } -#endif +static inline int cpu_suspend(unsigned long arg) +{ + return -EOPNOTSUPP; +} +#endif +static inline int arm_cpuidle_suspend(int index) +{ + return cpu_suspend(index); +} #endif diff --git a/arch/arm64/include/asm/cputable.h b/arch/arm64/include/asm/cputable.h deleted file mode 100644 index e3bd983d3661..000000000000 --- a/arch/arm64/include/asm/cputable.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * arch/arm64/include/asm/cputable.h - * - * Copyright (C) 2012 ARM Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ -#ifndef __ASM_CPUTABLE_H -#define __ASM_CPUTABLE_H - -struct cpu_info { - unsigned int cpu_id_val; - unsigned int cpu_id_mask; - const char *cpu_name; - unsigned long (*cpu_setup)(void); -}; - -extern struct cpu_info *lookup_processor_type(unsigned int); - -#endif diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 379d0b874328..a84ec605bed8 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -57,6 +57,11 @@ #define MIDR_IMPLEMENTOR(midr) \ (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) +#define MIDR_CPU_PART(imp, partnum) \ + (((imp) << MIDR_IMPLEMENTOR_SHIFT) | \ + (0xf << MIDR_ARCHITECTURE_SHIFT) | \ + ((partnum) << MIDR_PARTNUM_SHIFT)) + #define ARM_CPU_IMP_ARM 0x41 #define ARM_CPU_IMP_APM 0x50 @@ -67,6 +72,18 @@ #define APM_CPU_PART_POTENZA 0x000 +#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16 +#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT) +#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \ + (((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT) +#define ID_AA64MMFR0_BIGEND_SHIFT 8 +#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT) +#define ID_AA64MMFR0_BIGEND(mmfr0) \ + (((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT) + +#define SCTLR_EL1_CP15BEN (0x1 << 5) +#define SCTLR_EL1_SED (0x1 << 8) + #ifndef __ASSEMBLY__ /* @@ -99,6 +116,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void) return read_cpuid(CTR_EL0); } +static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0) +{ + return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) || + (ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1); +} #endif /* __ASSEMBLY__ */ #endif diff --git a/arch/arm64/include/asm/device.h b/arch/arm64/include/asm/device.h index cf98b362094b..243ef256b8c9 100644 --- a/arch/arm64/include/asm/device.h +++ b/arch/arm64/include/asm/device.h @@ -21,6 +21,7 @@ struct dev_archdata { #ifdef CONFIG_IOMMU_API void *iommu; /* private IOMMU data */ #endif + bool dma_coherent; }; struct pdev_archdata { diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h index adeae3f6f0fc..9437e3dc5833 100644 --- a/arch/arm64/include/asm/dma-mapping.h +++ b/arch/arm64/include/asm/dma-mapping.h @@ -28,8 +28,6 @@ #define DMA_ERROR_CODE (~(dma_addr_t)0) extern struct dma_map_ops *dma_ops; -extern struct dma_map_ops coherent_swiotlb_dma_ops; -extern struct dma_map_ops noncoherent_swiotlb_dma_ops; static inline struct dma_map_ops *__generic_dma_ops(struct device *dev) { @@ -47,17 +45,20 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) return __generic_dma_ops(dev); } -static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops) +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, + struct iommu_ops *iommu, bool coherent) { - dev->archdata.dma_ops = ops; + dev->archdata.dma_coherent = coherent; } +#define arch_setup_dma_ops arch_setup_dma_ops -static inline int set_arch_dma_coherent_ops(struct device *dev) +/* do not use this function in a driver */ +static inline bool is_device_dma_coherent(struct device *dev) { - set_dma_ops(dev, &coherent_swiotlb_dma_ops); - return 0; + if (!dev) + return false; + return dev->archdata.dma_coherent; } -#define set_arch_dma_coherent_ops set_arch_dma_coherent_ops #include <asm-generic/dma-mapping-common.h> @@ -96,7 +97,7 @@ static inline int dma_set_mask(struct device *dev, u64 mask) static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { if (!dev->dma_mask) - return 0; + return false; return addr + size - 1 <= *dev->dma_mask; } diff --git a/arch/arm64/include/asm/dmi.h b/arch/arm64/include/asm/dmi.h new file mode 100644 index 000000000000..69d37d87b159 --- /dev/null +++ b/arch/arm64/include/asm/dmi.h @@ -0,0 +1,31 @@ +/* + * arch/arm64/include/asm/dmi.h + * + * Copyright (C) 2013 Linaro Limited. + * Written by: Yi Li (yi.li@linaro.org) + * + * based on arch/ia64/include/asm/dmi.h + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#ifndef __ASM_DMI_H +#define __ASM_DMI_H + +#include <linux/io.h> +#include <linux/slab.h> + +/* + * According to section 2.3.6 of the UEFI spec, the firmware should not + * request a virtual mapping for configuration tables such as SMBIOS. + * This means we have to map them before use. + */ +#define dmi_early_remap(x, l) ioremap_cache(x, l) +#define dmi_early_unmap(x, l) iounmap(x) +#define dmi_remap(x, l) ioremap_cache(x, l) +#define dmi_unmap(x) iounmap(x) +#define dmi_alloc(l) kzalloc(l, GFP_KERNEL) + +#endif diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index a34fd3b12e2b..ef572206f1c3 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -6,29 +6,33 @@ #ifdef CONFIG_EFI extern void efi_init(void); -extern void efi_idmap_init(void); #else #define efi_init() -#define efi_idmap_init() #endif #define efi_call_virt(f, ...) \ ({ \ - efi_##f##_t *__f = efi.systab->runtime->f; \ + efi_##f##_t *__f; \ efi_status_t __s; \ \ kernel_neon_begin(); \ + efi_virtmap_load(); \ + __f = efi.systab->runtime->f; \ __s = __f(__VA_ARGS__); \ + efi_virtmap_unload(); \ kernel_neon_end(); \ __s; \ }) #define __efi_call_virt(f, ...) \ ({ \ - efi_##f##_t *__f = efi.systab->runtime->f; \ + efi_##f##_t *__f; \ \ kernel_neon_begin(); \ + efi_virtmap_load(); \ + __f = efi.systab->runtime->f; \ __f(__VA_ARGS__); \ + efi_virtmap_unload(); \ kernel_neon_end(); \ }) @@ -44,4 +48,22 @@ extern void efi_idmap_init(void); #define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__) +#define EFI_ALLOC_ALIGN SZ_64K + +/* + * On ARM systems, virtually remapped UEFI runtime services are set up in two + * distinct stages: + * - The stub retrieves the final version of the memory map from UEFI, populates + * the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime + * service to communicate the new mapping to the firmware (Note that the new + * mapping is not live at this time) + * - During an early initcall(), the EFI system table is permanently remapped + * and the virtual remapping of the UEFI Runtime Services regions is loaded + * into a private set of page tables. If this all succeeds, the Runtime + * Services are enabled and the EFI_RUNTIME_SERVICES bit set. + */ + +void efi_virtmap_load(void); +void efi_virtmap_unload(void); + #endif /* _ASM_EFI_H */ diff --git a/arch/arm64/include/asm/elf.h b/arch/arm64/include/asm/elf.h index 1f65be393139..faad6df49e5b 100644 --- a/arch/arm64/include/asm/elf.h +++ b/arch/arm64/include/asm/elf.h @@ -125,7 +125,6 @@ typedef struct user_fpsimd_state elf_fpregset_t; * the loader. We need to make sure that it is out of the way of the program * that it will "exec", and that there is sufficient room for the brk. */ -extern unsigned long randomize_et_dyn(unsigned long base); #define ELF_ET_DYN_BASE (2 * TASK_SIZE_64 / 3) /* @@ -157,10 +156,6 @@ extern int arch_setup_additional_pages(struct linux_binprm *bprm, #define STACK_RND_MASK (0x3ffff >> (PAGE_SHIFT - 12)) #endif -struct mm_struct; -extern unsigned long arch_randomize_brk(struct mm_struct *mm); -#define arch_randomize_brk arch_randomize_brk - #ifdef CONFIG_COMPAT #ifdef __AARCH64EB__ diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index 72674f4c3871..70522450ca23 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -18,40 +18,91 @@ #ifndef __ASM_ESR_H #define __ASM_ESR_H -#define ESR_EL1_WRITE (1 << 6) -#define ESR_EL1_CM (1 << 8) -#define ESR_EL1_IL (1 << 25) +#define ESR_ELx_EC_UNKNOWN (0x00) +#define ESR_ELx_EC_WFx (0x01) +/* Unallocated EC: 0x02 */ +#define ESR_ELx_EC_CP15_32 (0x03) +#define ESR_ELx_EC_CP15_64 (0x04) +#define ESR_ELx_EC_CP14_MR (0x05) +#define ESR_ELx_EC_CP14_LS (0x06) +#define ESR_ELx_EC_FP_ASIMD (0x07) +#define ESR_ELx_EC_CP10_ID (0x08) +/* Unallocated EC: 0x09 - 0x0B */ +#define ESR_ELx_EC_CP14_64 (0x0C) +/* Unallocated EC: 0x0d */ +#define ESR_ELx_EC_ILL (0x0E) +/* Unallocated EC: 0x0F - 0x10 */ +#define ESR_ELx_EC_SVC32 (0x11) +#define ESR_ELx_EC_HVC32 (0x12) +#define ESR_ELx_EC_SMC32 (0x13) +/* Unallocated EC: 0x14 */ +#define ESR_ELx_EC_SVC64 (0x15) +#define ESR_ELx_EC_HVC64 (0x16) +#define ESR_ELx_EC_SMC64 (0x17) +#define ESR_ELx_EC_SYS64 (0x18) +/* Unallocated EC: 0x19 - 0x1E */ +#define ESR_ELx_EC_IMP_DEF (0x1f) +#define ESR_ELx_EC_IABT_LOW (0x20) +#define ESR_ELx_EC_IABT_CUR (0x21) +#define ESR_ELx_EC_PC_ALIGN (0x22) +/* Unallocated EC: 0x23 */ +#define ESR_ELx_EC_DABT_LOW (0x24) +#define ESR_ELx_EC_DABT_CUR (0x25) +#define ESR_ELx_EC_SP_ALIGN (0x26) +/* Unallocated EC: 0x27 */ +#define ESR_ELx_EC_FP_EXC32 (0x28) +/* Unallocated EC: 0x29 - 0x2B */ +#define ESR_ELx_EC_FP_EXC64 (0x2C) +/* Unallocated EC: 0x2D - 0x2E */ +#define ESR_ELx_EC_SERROR (0x2F) +#define ESR_ELx_EC_BREAKPT_LOW (0x30) +#define ESR_ELx_EC_BREAKPT_CUR (0x31) +#define ESR_ELx_EC_SOFTSTP_LOW (0x32) +#define ESR_ELx_EC_SOFTSTP_CUR (0x33) +#define ESR_ELx_EC_WATCHPT_LOW (0x34) +#define ESR_ELx_EC_WATCHPT_CUR (0x35) +/* Unallocated EC: 0x36 - 0x37 */ +#define ESR_ELx_EC_BKPT32 (0x38) +/* Unallocated EC: 0x39 */ +#define ESR_ELx_EC_VECTOR32 (0x3A) +/* Unallocted EC: 0x3B */ +#define ESR_ELx_EC_BRK64 (0x3C) +/* Unallocated EC: 0x3D - 0x3F */ +#define ESR_ELx_EC_MAX (0x3F) -#define ESR_EL1_EC_SHIFT (26) -#define ESR_EL1_EC_UNKNOWN (0x00) -#define ESR_EL1_EC_WFI (0x01) -#define ESR_EL1_EC_CP15_32 (0x03) -#define ESR_EL1_EC_CP15_64 (0x04) -#define ESR_EL1_EC_CP14_MR (0x05) -#define ESR_EL1_EC_CP14_LS (0x06) -#define ESR_EL1_EC_FP_ASIMD (0x07) -#define ESR_EL1_EC_CP10_ID (0x08) -#define ESR_EL1_EC_CP14_64 (0x0C) -#define ESR_EL1_EC_ILL_ISS (0x0E) -#define ESR_EL1_EC_SVC32 (0x11) -#define ESR_EL1_EC_SVC64 (0x15) -#define ESR_EL1_EC_SYS64 (0x18) -#define ESR_EL1_EC_IABT_EL0 (0x20) -#define ESR_EL1_EC_IABT_EL1 (0x21) -#define ESR_EL1_EC_PC_ALIGN (0x22) -#define ESR_EL1_EC_DABT_EL0 (0x24) -#define ESR_EL1_EC_DABT_EL1 (0x25) -#define ESR_EL1_EC_SP_ALIGN (0x26) -#define ESR_EL1_EC_FP_EXC32 (0x28) -#define ESR_EL1_EC_FP_EXC64 (0x2C) -#define ESR_EL1_EC_SERROR (0x2F) -#define ESR_EL1_EC_BREAKPT_EL0 (0x30) -#define ESR_EL1_EC_BREAKPT_EL1 (0x31) -#define ESR_EL1_EC_SOFTSTP_EL0 (0x32) -#define ESR_EL1_EC_SOFTSTP_EL1 (0x33) -#define ESR_EL1_EC_WATCHPT_EL0 (0x34) -#define ESR_EL1_EC_WATCHPT_EL1 (0x35) -#define ESR_EL1_EC_BKPT32 (0x38) -#define ESR_EL1_EC_BRK64 (0x3C) +#define ESR_ELx_EC_SHIFT (26) +#define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT) + +#define ESR_ELx_IL (UL(1) << 25) +#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1) +#define ESR_ELx_ISV (UL(1) << 24) +#define ESR_ELx_SAS_SHIFT (22) +#define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT) +#define ESR_ELx_SSE (UL(1) << 21) +#define ESR_ELx_SRT_SHIFT (16) +#define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT) +#define ESR_ELx_SF (UL(1) << 15) +#define ESR_ELx_AR (UL(1) << 14) +#define ESR_ELx_EA (UL(1) << 9) +#define ESR_ELx_CM (UL(1) << 8) +#define ESR_ELx_S1PTW (UL(1) << 7) +#define ESR_ELx_WNR (UL(1) << 6) +#define ESR_ELx_FSC (0x3F) +#define ESR_ELx_FSC_TYPE (0x3C) +#define ESR_ELx_FSC_EXTABT (0x10) +#define ESR_ELx_FSC_ACCESS (0x08) +#define ESR_ELx_FSC_FAULT (0x04) +#define ESR_ELx_FSC_PERM (0x0C) +#define ESR_ELx_CV (UL(1) << 24) +#define ESR_ELx_COND_SHIFT (20) +#define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT) +#define ESR_ELx_WFx_ISS_WFE (UL(1) << 0) +#define ESR_ELx_xVC_IMM_MASK ((1UL << 16) - 1) + +#ifndef __ASSEMBLY__ +#include <asm/types.h> + +const char *esr_get_class_string(u32 esr); +#endif /* __ASSEMBLY */ #endif /* __ASM_ESR_H */ diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 5f7bfe6df723..95e6b6dcbe37 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -31,7 +31,9 @@ * */ enum fixed_addresses { + FIX_HOLE, FIX_EARLYCON_MEM_BASE, + FIX_TEXT_POKE0, __end_of_permanent_fixed_addresses, /* @@ -56,10 +58,14 @@ enum fixed_addresses { #define FIXMAP_PAGE_IO __pgprot(PROT_DEVICE_nGnRE) -extern void __early_set_fixmap(enum fixed_addresses idx, - phys_addr_t phys, pgprot_t flags); +void __init early_fixmap_init(void); -#define __set_fixmap __early_set_fixmap +#define __early_set_fixmap __set_fixmap + +#define __late_set_fixmap __set_fixmap +#define __late_clear_fixmap(idx) __set_fixmap((idx), 0, FIXMAP_PAGE_CLEAR) + +extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); #include <asm-generic/fixmap.h> diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index 007618b8188c..a2daf1293028 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -76,7 +76,6 @@ fpsimd_restore_fpcr x\tmpnr, \state .endm -.altmacro .macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2 mrs x\tmpnr1, fpsr str w\numnr, [\state, #8] @@ -86,11 +85,22 @@ add \state, \state, x\numnr, lsl #4 sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1 br x\tmpnr1 - .irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 - .irp qb, %(qa + 1) - stp q\qa, q\qb, [\state, # -16 * \qa - 16] - .endr - .endr + stp q30, q31, [\state, #-16 * 30 - 16] + stp q28, q29, [\state, #-16 * 28 - 16] + stp q26, q27, [\state, #-16 * 26 - 16] + stp q24, q25, [\state, #-16 * 24 - 16] + stp q22, q23, [\state, #-16 * 22 - 16] + stp q20, q21, [\state, #-16 * 20 - 16] + stp q18, q19, [\state, #-16 * 18 - 16] + stp q16, q17, [\state, #-16 * 16 - 16] + stp q14, q15, [\state, #-16 * 14 - 16] + stp q12, q13, [\state, #-16 * 12 - 16] + stp q10, q11, [\state, #-16 * 10 - 16] + stp q8, q9, [\state, #-16 * 8 - 16] + stp q6, q7, [\state, #-16 * 6 - 16] + stp q4, q5, [\state, #-16 * 4 - 16] + stp q2, q3, [\state, #-16 * 2 - 16] + stp q0, q1, [\state, #-16 * 0 - 16] 0: .endm @@ -103,10 +113,21 @@ add \state, \state, x\tmpnr2, lsl #4 sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1 br x\tmpnr1 - .irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0 - .irp qb, %(qa + 1) - ldp q\qa, q\qb, [\state, # -16 * \qa - 16] - .endr - .endr + ldp q30, q31, [\state, #-16 * 30 - 16] + ldp q28, q29, [\state, #-16 * 28 - 16] + ldp q26, q27, [\state, #-16 * 26 - 16] + ldp q24, q25, [\state, #-16 * 24 - 16] + ldp q22, q23, [\state, #-16 * 22 - 16] + ldp q20, q21, [\state, #-16 * 20 - 16] + ldp q18, q19, [\state, #-16 * 18 - 16] + ldp q16, q17, [\state, #-16 * 16 - 16] + ldp q14, q15, [\state, #-16 * 14 - 16] + ldp q12, q13, [\state, #-16 * 12 - 16] + ldp q10, q11, [\state, #-16 * 10 - 16] + ldp q8, q9, [\state, #-16 * 8 - 16] + ldp q6, q7, [\state, #-16 * 6 - 16] + ldp q4, q5, [\state, #-16 * 4 - 16] + ldp q2, q3, [\state, #-16 * 2 - 16] + ldp q0, q1, [\state, #-16 * 0 - 16] 0: .endm diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index 5f750dc96e0f..74069b3bd919 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -58,7 +58,7 @@ futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; - pagefault_disable(); /* implies preempt_disable() */ + pagefault_disable(); switch (op) { case FUTEX_OP_SET: @@ -85,7 +85,7 @@ futex_atomic_op_inuser (int encoded_op, u32 __user *uaddr) ret = -ENOSYS; } - pagefault_enable(); /* subsumes preempt_enable() */ + pagefault_enable(); if (!ret) { switch (cmp) { diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h index e8a3268a891c..6aae421f4d73 100644 --- a/arch/arm64/include/asm/hardirq.h +++ b/arch/arm64/include/asm/hardirq.h @@ -20,7 +20,7 @@ #include <linux/threads.h> #include <asm/irq.h> -#define NR_IPI 6 +#define NR_IPI 5 typedef struct { unsigned int __softirq_pending; diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 024c46183c3c..0ad735166d9f 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -30,6 +30,7 @@ #define COMPAT_HWCAP_IDIVA (1 << 17) #define COMPAT_HWCAP_IDIVT (1 << 18) #define COMPAT_HWCAP_IDIV (COMPAT_HWCAP_IDIVA|COMPAT_HWCAP_IDIVT) +#define COMPAT_HWCAP_LPAE (1 << 20) #define COMPAT_HWCAP_EVTSTRM (1 << 21) #define COMPAT_HWCAP2_AES (1 << 0) diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 56a9e63b6c33..f81b328d9cf4 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -264,8 +264,10 @@ __AARCH64_INSN_FUNCS(ands, 0x7F200000, 0x6A000000) __AARCH64_INSN_FUNCS(bics, 0x7F200000, 0x6A200000) __AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000) __AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000) -__AARCH64_INSN_FUNCS(cbz, 0xFE000000, 0x34000000) -__AARCH64_INSN_FUNCS(cbnz, 0xFE000000, 0x35000000) +__AARCH64_INSN_FUNCS(cbz, 0x7F000000, 0x34000000) +__AARCH64_INSN_FUNCS(cbnz, 0x7F000000, 0x35000000) +__AARCH64_INSN_FUNCS(tbz, 0x7F000000, 0x36000000) +__AARCH64_INSN_FUNCS(tbnz, 0x7F000000, 0x37000000) __AARCH64_INSN_FUNCS(bcond, 0xFF000010, 0x54000000) __AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001) __AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002) @@ -283,6 +285,7 @@ bool aarch64_insn_is_nop(u32 insn); int aarch64_insn_read(void *addr, u32 *insnp); int aarch64_insn_write(void *addr, u32 insn); enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); +u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn); u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, u32 insn, u64 imm); u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, @@ -354,6 +357,16 @@ bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); int aarch64_insn_patch_text_nosync(void *addr, u32 insn); int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); + +bool aarch32_insn_is_wide(u32 insn); + +#define A32_RN_OFFSET 16 +#define A32_RT_OFFSET 12 +#define A32_RT2_OFFSET 0 + +u32 aarch32_insn_extract_reg_num(u32 insn, int offset); +u32 aarch32_insn_mcr_extract_opc2(u32 insn); +u32 aarch32_insn_mcr_extract_crm(u32 insn); #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 79f1d519221f..7116d3973058 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -26,59 +26,83 @@ #include <asm/byteorder.h> #include <asm/barrier.h> +#include <asm/memory.h> #include <asm/pgtable.h> #include <asm/early_ioremap.h> +#include <asm/alternative.h> +#include <asm/cpufeature.h> #include <xen/xen.h> /* * Generic IO read/write. These perform native-endian accesses. */ +#define __raw_writeb __raw_writeb static inline void __raw_writeb(u8 val, volatile void __iomem *addr) { asm volatile("strb %w0, [%1]" : : "r" (val), "r" (addr)); } +#define __raw_writew __raw_writew static inline void __raw_writew(u16 val, volatile void __iomem *addr) { asm volatile("strh %w0, [%1]" : : "r" (val), "r" (addr)); } +#define __raw_writel __raw_writel static inline void __raw_writel(u32 val, volatile void __iomem *addr) { asm volatile("str %w0, [%1]" : : "r" (val), "r" (addr)); } +#define __raw_writeq __raw_writeq static inline void __raw_writeq(u64 val, volatile void __iomem *addr) { asm volatile("str %0, [%1]" : : "r" (val), "r" (addr)); } +#define __raw_readb __raw_readb static inline u8 __raw_readb(const volatile void __iomem *addr) { u8 val; - asm volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr)); + asm volatile(ALTERNATIVE("ldrb %w0, [%1]", + "ldarb %w0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } +#define __raw_readw __raw_readw static inline u16 __raw_readw(const volatile void __iomem *addr) { u16 val; - asm volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr)); + + asm volatile(ALTERNATIVE("ldrh %w0, [%1]", + "ldarh %w0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } +#define __raw_readl __raw_readl static inline u32 __raw_readl(const volatile void __iomem *addr) { u32 val; - asm volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr)); + asm volatile(ALTERNATIVE("ldr %w0, [%1]", + "ldar %w0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } +#define __raw_readq __raw_readq static inline u64 __raw_readq(const volatile void __iomem *addr) { u64 val; - asm volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr)); + asm volatile(ALTERNATIVE("ldr %0, [%1]", + "ldar %0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } @@ -122,96 +146,8 @@ static inline u64 __raw_readq(const volatile void __iomem *addr) * I/O port access primitives. */ #define arch_has_dev_port() (1) -#define IO_SPACE_LIMIT (SZ_32M - 1) -#define PCI_IOBASE ((void __iomem *)(MODULES_VADDR - SZ_32M)) - -static inline u8 inb(unsigned long addr) -{ - return readb(addr + PCI_IOBASE); -} - -static inline u16 inw(unsigned long addr) -{ - return readw(addr + PCI_IOBASE); -} - -static inline u32 inl(unsigned long addr) -{ - return readl(addr + PCI_IOBASE); -} - -static inline void outb(u8 b, unsigned long addr) -{ - writeb(b, addr + PCI_IOBASE); -} - -static inline void outw(u16 b, unsigned long addr) -{ - writew(b, addr + PCI_IOBASE); -} - -static inline void outl(u32 b, unsigned long addr) -{ - writel(b, addr + PCI_IOBASE); -} - -#define inb_p(addr) inb(addr) -#define inw_p(addr) inw(addr) -#define inl_p(addr) inl(addr) - -#define outb_p(x, addr) outb((x), (addr)) -#define outw_p(x, addr) outw((x), (addr)) -#define outl_p(x, addr) outl((x), (addr)) - -static inline void insb(unsigned long addr, void *buffer, int count) -{ - u8 *buf = buffer; - while (count--) - *buf++ = __raw_readb(addr + PCI_IOBASE); -} - -static inline void insw(unsigned long addr, void *buffer, int count) -{ - u16 *buf = buffer; - while (count--) - *buf++ = __raw_readw(addr + PCI_IOBASE); -} - -static inline void insl(unsigned long addr, void *buffer, int count) -{ - u32 *buf = buffer; - while (count--) - *buf++ = __raw_readl(addr + PCI_IOBASE); -} - -static inline void outsb(unsigned long addr, const void *buffer, int count) -{ - const u8 *buf = buffer; - while (count--) - __raw_writeb(*buf++, addr + PCI_IOBASE); -} - -static inline void outsw(unsigned long addr, const void *buffer, int count) -{ - const u16 *buf = buffer; - while (count--) - __raw_writew(*buf++, addr + PCI_IOBASE); -} - -static inline void outsl(unsigned long addr, const void *buffer, int count) -{ - const u32 *buf = buffer; - while (count--) - __raw_writel(*buf++, addr + PCI_IOBASE); -} - -#define insb_p(port,to,len) insb(port,to,len) -#define insw_p(port,to,len) insw(port,to,len) -#define insl_p(port,to,len) insl(port,to,len) - -#define outsb_p(port,from,len) outsb(port,from,len) -#define outsw_p(port,from,len) outsw(port,from,len) -#define outsl_p(port,from,len) outsl(port,from,len) +#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1) +#define PCI_IOBASE ((void __iomem *)PCI_IO_START) /* * String version of I/O memory access operations. @@ -234,20 +170,17 @@ extern void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size); #define ioremap(addr, size) __ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) #define ioremap_nocache(addr, size) __ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) #define ioremap_wc(addr, size) __ioremap((addr), (size), __pgprot(PROT_NORMAL_NC)) +#define ioremap_wt(addr, size) __ioremap((addr), (size), __pgprot(PROT_DEVICE_nGnRE)) #define iounmap __iounmap -#define ARCH_HAS_IOREMAP_WC -#include <asm-generic/iomap.h> - /* - * More restrictive address range checking than the default implementation - * (PHYS_OFFSET and PHYS_MASK taken into account). + * io{read,write}{16,32}be() macros */ -#define ARCH_HAS_VALID_PHYS_ADDR_RANGE -extern int valid_phys_addr_range(phys_addr_t addr, size_t size); -extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size); +#define ioread16be(p) ({ __u16 __v = be16_to_cpu((__force __be16)__raw_readw(p)); __iormb(); __v; }) +#define ioread32be(p) ({ __u32 __v = be32_to_cpu((__force __be32)__raw_readl(p)); __iormb(); __v; }) -extern int devmem_is_allowed(unsigned long pfn); +#define iowrite16be(v,p) ({ __iowmb(); __raw_writew((__force __u16)cpu_to_be16(v), p); }) +#define iowrite32be(v,p) ({ __iowmb(); __raw_writel((__force __u32)cpu_to_be32(v), p); }) /* * Convert a physical pointer to a virtual kernel pointer for /dev/mem @@ -260,6 +193,18 @@ extern int devmem_is_allowed(unsigned long pfn); */ #define xlate_dev_kmem_ptr(p) p +#include <asm-generic/io.h> + +/* + * More restrictive address range checking than the default implementation + * (PHYS_OFFSET and PHYS_MASK taken into account). + */ +#define ARCH_HAS_VALID_PHYS_ADDR_RANGE +extern int valid_phys_addr_range(phys_addr_t addr, size_t size); +extern int valid_mmap_phys_addr_range(unsigned long pfn, size_t size); + +extern int devmem_is_allowed(unsigned long pfn); + struct bio_vec; extern bool xen_biovec_phys_mergeable(const struct bio_vec *vec1, const struct bio_vec *vec2); diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h index e1f7ecdde11f..bbb251b14746 100644 --- a/arch/arm64/include/asm/irq.h +++ b/arch/arm64/include/asm/irq.h @@ -1,10 +1,24 @@ #ifndef __ASM_IRQ_H #define __ASM_IRQ_H +#include <linux/irqchip/arm-gic-acpi.h> + #include <asm-generic/irq.h> -extern void (*handle_arch_irq)(struct pt_regs *); +struct pt_regs; + extern void migrate_irqs(void); extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); +static inline void acpi_irq_init(void) +{ + /* + * Hardcode ACPI IRQ chip initialization to GICv2 for now. + * Proper irqchip infrastructure will be implemented along with + * incoming GICv2m|GICv3|ITS bits. + */ + acpi_gic_init(); +} +#define acpi_irq_init acpi_irq_init + #endif diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h index 076a1c714049..c0e5165c2f76 100644 --- a/arch/arm64/include/asm/jump_label.h +++ b/arch/arm64/include/asm/jump_label.h @@ -18,11 +18,12 @@ */ #ifndef __ASM_JUMP_LABEL_H #define __ASM_JUMP_LABEL_H + +#ifndef __ASSEMBLY__ + #include <linux/types.h> #include <asm/insn.h> -#ifdef __KERNEL__ - #define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE static __always_inline bool arch_static_branch(struct static_key *key) @@ -39,8 +40,6 @@ l_yes: return true; } -#endif /* __KERNEL__ */ - typedef u64 jump_label_t; struct jump_entry { @@ -49,4 +48,5 @@ struct jump_entry { jump_label_t key; }; +#endif /* __ASSEMBLY__ */ #endif /* __ASM_JUMP_LABEL_H */ diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 7fd3e27e3ccc..ac6fafb95fe7 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -18,6 +18,8 @@ #ifndef __ARM64_KVM_ARM_H__ #define __ARM64_KVM_ARM_H__ +#include <asm/esr.h> +#include <asm/memory.h> #include <asm/types.h> /* Hyp Configuration Register (HCR) bits */ @@ -127,6 +129,9 @@ * 40 bits wide (T0SZ = 24). Systems with a PARange smaller than 40 bits are * not known to exist and will break with this configuration. * + * VTCR_EL2.PS is extracted from ID_AA64MMFR0_EL1.PARange at boot time + * (see hyp-init.S). + * * Note that when using 4K pages, we concatenate two first level page tables * together. * @@ -136,7 +141,6 @@ #ifdef CONFIG_ARM64_64K_PAGES /* * Stage2 translation configuration: - * 40bits output (PS = 2) * 40bits input (T0SZ = 24) * 64kB pages (TG0 = 1) * 2 level page tables (SL = 1) @@ -148,7 +152,6 @@ #else /* * Stage2 translation configuration: - * 40bits output (PS = 2) * 40bits input (T0SZ = 24) * 4kB pages (TG0 = 0) * 3 level page tables (SL = 1) @@ -160,9 +163,9 @@ #endif #define VTTBR_BADDR_SHIFT (VTTBR_X - 1) -#define VTTBR_BADDR_MASK (((1LLU << (PHYS_MASK_SHIFT - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT) -#define VTTBR_VMID_SHIFT (48LLU) -#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT) +#define VTTBR_BADDR_MASK (((UL(1) << (PHYS_MASK_SHIFT - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT) +#define VTTBR_VMID_SHIFT (UL(48)) +#define VTTBR_VMID_MASK (UL(0xFF) << VTTBR_VMID_SHIFT) /* Hyp System Trap Register */ #define HSTR_EL2_TTEE (1 << 16) @@ -183,77 +186,12 @@ #define MDCR_EL2_TPMCR (1 << 5) #define MDCR_EL2_HPMN_MASK (0x1F) -/* Exception Syndrome Register (ESR) bits */ -#define ESR_EL2_EC_SHIFT (26) -#define ESR_EL2_EC (0x3fU << ESR_EL2_EC_SHIFT) -#define ESR_EL2_IL (1U << 25) -#define ESR_EL2_ISS (ESR_EL2_IL - 1) -#define ESR_EL2_ISV_SHIFT (24) -#define ESR_EL2_ISV (1U << ESR_EL2_ISV_SHIFT) -#define ESR_EL2_SAS_SHIFT (22) -#define ESR_EL2_SAS (3U << ESR_EL2_SAS_SHIFT) -#define ESR_EL2_SSE (1 << 21) -#define ESR_EL2_SRT_SHIFT (16) -#define ESR_EL2_SRT_MASK (0x1f << ESR_EL2_SRT_SHIFT) -#define ESR_EL2_SF (1 << 15) -#define ESR_EL2_AR (1 << 14) -#define ESR_EL2_EA (1 << 9) -#define ESR_EL2_CM (1 << 8) -#define ESR_EL2_S1PTW (1 << 7) -#define ESR_EL2_WNR (1 << 6) -#define ESR_EL2_FSC (0x3f) -#define ESR_EL2_FSC_TYPE (0x3c) - -#define ESR_EL2_CV_SHIFT (24) -#define ESR_EL2_CV (1U << ESR_EL2_CV_SHIFT) -#define ESR_EL2_COND_SHIFT (20) -#define ESR_EL2_COND (0xfU << ESR_EL2_COND_SHIFT) - - -#define FSC_FAULT (0x04) -#define FSC_PERM (0x0c) +/* For compatibility with fault code shared with 32-bit */ +#define FSC_FAULT ESR_ELx_FSC_FAULT +#define FSC_ACCESS ESR_ELx_FSC_ACCESS +#define FSC_PERM ESR_ELx_FSC_PERM /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ -#define HPFAR_MASK (~0xFUL) - -#define ESR_EL2_EC_UNKNOWN (0x00) -#define ESR_EL2_EC_WFI (0x01) -#define ESR_EL2_EC_CP15_32 (0x03) -#define ESR_EL2_EC_CP15_64 (0x04) -#define ESR_EL2_EC_CP14_MR (0x05) -#define ESR_EL2_EC_CP14_LS (0x06) -#define ESR_EL2_EC_FP_ASIMD (0x07) -#define ESR_EL2_EC_CP10_ID (0x08) -#define ESR_EL2_EC_CP14_64 (0x0C) -#define ESR_EL2_EC_ILL_ISS (0x0E) -#define ESR_EL2_EC_SVC32 (0x11) -#define ESR_EL2_EC_HVC32 (0x12) -#define ESR_EL2_EC_SMC32 (0x13) -#define ESR_EL2_EC_SVC64 (0x15) -#define ESR_EL2_EC_HVC64 (0x16) -#define ESR_EL2_EC_SMC64 (0x17) -#define ESR_EL2_EC_SYS64 (0x18) -#define ESR_EL2_EC_IABT (0x20) -#define ESR_EL2_EC_IABT_HYP (0x21) -#define ESR_EL2_EC_PC_ALIGN (0x22) -#define ESR_EL2_EC_DABT (0x24) -#define ESR_EL2_EC_DABT_HYP (0x25) -#define ESR_EL2_EC_SP_ALIGN (0x26) -#define ESR_EL2_EC_FP_EXC32 (0x28) -#define ESR_EL2_EC_FP_EXC64 (0x2C) -#define ESR_EL2_EC_SERROR (0x2F) -#define ESR_EL2_EC_BREAKPT (0x30) -#define ESR_EL2_EC_BREAKPT_HYP (0x31) -#define ESR_EL2_EC_SOFTSTP (0x32) -#define ESR_EL2_EC_SOFTSTP_HYP (0x33) -#define ESR_EL2_EC_WATCHPT (0x34) -#define ESR_EL2_EC_WATCHPT_HYP (0x35) -#define ESR_EL2_EC_BKPT32 (0x38) -#define ESR_EL2_EC_VECTOR32 (0x3A) -#define ESR_EL2_EC_BRK64 (0x3C) - -#define ESR_EL2_EC_xABT_xFSR_EXTABT 0x10 - -#define ESR_EL2_EC_WFI_ISS_WFE (1 << 0) +#define HPFAR_MASK (~UL(0xf)) #endif /* __ARM64_KVM_ARM_H__ */ diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 7e04f6ab3d5e..c4c11d20ca17 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -126,6 +126,7 @@ extern char __kvm_hyp_vector[]; extern void __kvm_flush_vm_context(void); extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa); +extern void __kvm_tlb_flush_vmid(struct kvm *kvm); extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 5674a55b5518..17e92f05b1fe 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -23,10 +23,13 @@ #define __ARM64_KVM_EMULATE_H__ #include <linux/kvm_host.h> -#include <asm/kvm_asm.h> + +#include <asm/esr.h> #include <asm/kvm_arm.h> +#include <asm/kvm_asm.h> #include <asm/kvm_mmio.h> #include <asm/ptrace.h> +#include <asm/cputype.h> unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num); unsigned long *vcpu_spsr32(const struct kvm_vcpu *vcpu); @@ -38,6 +41,23 @@ void kvm_inject_undefined(struct kvm_vcpu *vcpu); void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr); void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr); +static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) +{ + vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; + if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features)) + vcpu->arch.hcr_el2 &= ~HCR_RW; +} + +static inline unsigned long vcpu_get_hcr(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.hcr_el2; +} + +static inline void vcpu_set_hcr(struct kvm_vcpu *vcpu, unsigned long hcr) +{ + vcpu->arch.hcr_el2 = hcr; +} + static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu) { return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc; @@ -121,70 +141,75 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu) return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8; } +static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu) +{ + return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_xVC_IMM_MASK; +} + static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_ISV); + return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV); } static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_WNR); + return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WNR); } static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SSE); + return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SSE); } static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu) { - return (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SRT_MASK) >> ESR_EL2_SRT_SHIFT; + return (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT; } static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EA); + return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_EA); } static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_S1PTW); + return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_S1PTW); } static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu) { - return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SAS) >> ESR_EL2_SAS_SHIFT); + return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT); } /* This one is not specific to Data Abort */ static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu) { - return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_IL); + return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_IL); } static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_hsr(vcpu) >> ESR_EL2_EC_SHIFT; + return kvm_vcpu_get_hsr(vcpu) >> ESR_ELx_EC_SHIFT; } static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_trap_get_class(vcpu) == ESR_EL2_EC_IABT; + return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW; } static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC; + return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC; } static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu) { - return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC_TYPE; + return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE; } -static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu) +static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) { - return vcpu_sys_reg(vcpu, MPIDR_EL1); + return vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; } static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 2012c4ba8d67..f0f58c9beec0 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -28,6 +28,8 @@ #include <asm/kvm_asm.h> #include <asm/kvm_mmio.h> +#define __KVM_HAVE_ARCH_INTC_INITIALIZED + #if defined(CONFIG_KVM_ARM_MAX_VCPUS) #define KVM_MAX_VCPUS CONFIG_KVM_ARM_MAX_VCPUS #else @@ -59,6 +61,9 @@ struct kvm_arch { /* VTTBR value associated with above pgd and vmid */ u64 vttbr; + /* The maximum number of vCPUs depends on the used GIC model */ + int max_vcpus; + /* Interrupt controller */ struct vgic_dist vgic; @@ -116,9 +121,6 @@ struct kvm_vcpu_arch { * Anything that is not used directly from assembly code goes * here. */ - /* dcache set/way operation pending */ - int last_pcpu; - cpumask_t require_dcache_flush; /* Don't run the guest */ bool pause; @@ -162,11 +164,10 @@ struct kvm_vm_stat { }; struct kvm_vcpu_stat { + u32 halt_successful_poll; u32 halt_wakeup; }; -int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, - const struct kvm_vcpu_init *init); int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init); unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu); int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices); @@ -178,19 +179,10 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva); int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end); void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); +int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end); +int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); /* We do not have shadow page tables, hence the empty hooks */ -static inline int kvm_age_hva(struct kvm *kvm, unsigned long start, - unsigned long end) -{ - return 0; -} - -static inline int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) -{ - return 0; -} - static inline void kvm_arch_mmu_notifier_invalidate_page(struct kvm *kvm, unsigned long address) { @@ -200,6 +192,8 @@ struct kvm_vcpu *kvm_arm_get_running_vcpu(void); struct kvm_vcpu * __percpu *kvm_get_running_vcpus(void); u64 kvm_call_hyp(void *hypfn, ...); +void force_vm_exit(const cpumask_t *mask); +void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot); int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int exception_index); @@ -207,6 +201,8 @@ int handle_exit(struct kvm_vcpu *vcpu, struct kvm_run *run, int kvm_perf_init(void); int kvm_perf_teardown(void); +struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr); + static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr, phys_addr_t pgd_ptr, unsigned long hyp_stack_ptr, diff --git a/arch/arm64/include/asm/kvm_mmio.h b/arch/arm64/include/asm/kvm_mmio.h index fc2f689c0694..889c908ee631 100644 --- a/arch/arm64/include/asm/kvm_mmio.h +++ b/arch/arm64/include/asm/kvm_mmio.h @@ -31,27 +31,6 @@ struct kvm_decode { bool sign_extend; }; -/* - * The in-kernel MMIO emulation code wants to use a copy of run->mmio, - * which is an anonymous type. Use our own type instead. - */ -struct kvm_exit_mmio { - phys_addr_t phys_addr; - u8 data[8]; - u32 len; - bool is_write; -}; - -static inline void kvm_prepare_mmio(struct kvm_run *run, - struct kvm_exit_mmio *mmio) -{ - run->mmio.phys_addr = mmio->phys_addr; - run->mmio.len = mmio->len; - run->mmio.is_write = mmio->is_write; - memcpy(run->mmio.data, mmio->data, mmio->len); - run->exit_reason = KVM_EXIT_MMIO; -} - int kvm_handle_mmio_return(struct kvm_vcpu *vcpu, struct kvm_run *run); int io_mem_abort(struct kvm_vcpu *vcpu, struct kvm_run *run, phys_addr_t fault_ipa); diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 0caf7a59f6a1..61505676d085 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -68,6 +68,8 @@ #include <asm/pgalloc.h> #include <asm/cachetype.h> #include <asm/cacheflush.h> +#include <asm/mmu_context.h> +#include <asm/pgtable.h> #define KERN_TO_HYP(kva) ((unsigned long)kva - PAGE_OFFSET + HYP_PAGE_OFFSET) @@ -83,6 +85,7 @@ int create_hyp_io_mappings(void *from, void *to, phys_addr_t); void free_boot_hyp_pgd(void); void free_hyp_pgds(void); +void stage2_unmap_vm(struct kvm *kvm); int kvm_alloc_stage2_pgd(struct kvm *kvm); void kvm_free_stage2_pgd(struct kvm *kvm); int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa, @@ -117,6 +120,27 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd) pmd_val(*pmd) |= PMD_S2_RDWR; } +static inline void kvm_set_s2pte_readonly(pte_t *pte) +{ + pte_val(*pte) = (pte_val(*pte) & ~PTE_S2_RDWR) | PTE_S2_RDONLY; +} + +static inline bool kvm_s2pte_readonly(pte_t *pte) +{ + return (pte_val(*pte) & PTE_S2_RDWR) == PTE_S2_RDONLY; +} + +static inline void kvm_set_s2pmd_readonly(pmd_t *pmd) +{ + pmd_val(*pmd) = (pmd_val(*pmd) & ~PMD_S2_RDWR) | PMD_S2_RDONLY; +} + +static inline bool kvm_s2pmd_readonly(pmd_t *pmd) +{ + return (pmd_val(*pmd) & PMD_S2_RDWR) == PMD_S2_RDONLY; +} + + #define kvm_pgd_addr_end(addr, end) pgd_addr_end(addr, end) #define kvm_pud_addr_end(addr, end) pud_addr_end(addr, end) #define kvm_pmd_addr_end(addr, end) pmd_addr_end(addr, end) @@ -136,56 +160,21 @@ static inline void kvm_set_s2pmd_writable(pmd_t *pmd) #define PTRS_PER_S2_PGD (1 << PTRS_PER_S2_PGD_SHIFT) #define S2_PGD_ORDER get_order(PTRS_PER_S2_PGD * sizeof(pgd_t)) +#define kvm_pgd_index(addr) (((addr) >> PGDIR_SHIFT) & (PTRS_PER_S2_PGD - 1)) + /* * If we are concatenating first level stage-2 page tables, we would have less * than or equal to 16 pointers in the fake PGD, because that's what the - * architecture allows. In this case, (4 - CONFIG_ARM64_PGTABLE_LEVELS) + * architecture allows. In this case, (4 - CONFIG_PGTABLE_LEVELS) * represents the first level for the host, and we add 1 to go to the next * level (which uses contatenation) for the stage-2 tables. */ #if PTRS_PER_S2_PGD <= 16 -#define KVM_PREALLOC_LEVEL (4 - CONFIG_ARM64_PGTABLE_LEVELS + 1) +#define KVM_PREALLOC_LEVEL (4 - CONFIG_PGTABLE_LEVELS + 1) #else #define KVM_PREALLOC_LEVEL (0) #endif -/** - * kvm_prealloc_hwpgd - allocate inital table for VTTBR - * @kvm: The KVM struct pointer for the VM. - * @pgd: The kernel pseudo pgd - * - * When the kernel uses more levels of page tables than the guest, we allocate - * a fake PGD and pre-populate it to point to the next-level page table, which - * will be the real initial page table pointed to by the VTTBR. - * - * When KVM_PREALLOC_LEVEL==2, we allocate a single page for the PMD and - * the kernel will use folded pud. When KVM_PREALLOC_LEVEL==1, we - * allocate 2 consecutive PUD pages. - */ -static inline int kvm_prealloc_hwpgd(struct kvm *kvm, pgd_t *pgd) -{ - unsigned int i; - unsigned long hwpgd; - - if (KVM_PREALLOC_LEVEL == 0) - return 0; - - hwpgd = __get_free_pages(GFP_KERNEL | __GFP_ZERO, PTRS_PER_S2_PGD_SHIFT); - if (!hwpgd) - return -ENOMEM; - - for (i = 0; i < PTRS_PER_S2_PGD; i++) { - if (KVM_PREALLOC_LEVEL == 1) - pgd_populate(NULL, pgd + i, - (pud_t *)hwpgd + i * PTRS_PER_PUD); - else if (KVM_PREALLOC_LEVEL == 2) - pud_populate(NULL, pud_offset(pgd, 0) + i, - (pmd_t *)hwpgd + i * PTRS_PER_PMD); - } - - return 0; -} - static inline void *kvm_get_hwpgd(struct kvm *kvm) { pgd_t *pgd = kvm->arch.pgd; @@ -202,12 +191,11 @@ static inline void *kvm_get_hwpgd(struct kvm *kvm) return pmd_offset(pud, 0); } -static inline void kvm_free_hwpgd(struct kvm *kvm) +static inline unsigned int kvm_get_hwpgd_size(void) { - if (KVM_PREALLOC_LEVEL > 0) { - unsigned long hwpgd = (unsigned long)kvm_get_hwpgd(kvm); - free_pages(hwpgd, PTRS_PER_S2_PGD_SHIFT); - } + if (KVM_PREALLOC_LEVEL > 0) + return PTRS_PER_S2_PGD * PAGE_SIZE; + return PTRS_PER_S2_PGD * sizeof(pgd_t); } static inline bool kvm_page_empty(void *ptr) @@ -242,23 +230,77 @@ static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu) return (vcpu_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101; } -static inline void coherent_cache_guest_page(struct kvm_vcpu *vcpu, hva_t hva, - unsigned long size) +static inline void __coherent_cache_guest_page(struct kvm_vcpu *vcpu, pfn_t pfn, + unsigned long size, + bool ipa_uncached) { - if (!vcpu_has_cache_enabled(vcpu)) - kvm_flush_dcache_to_poc((void *)hva, size); + void *va = page_address(pfn_to_page(pfn)); + + if (!vcpu_has_cache_enabled(vcpu) || ipa_uncached) + kvm_flush_dcache_to_poc(va, size); if (!icache_is_aliasing()) { /* PIPT */ - flush_icache_range(hva, hva + size); + flush_icache_range((unsigned long)va, + (unsigned long)va + size); } else if (!icache_is_aivivt()) { /* non ASID-tagged VIVT */ /* any kind of VIPT cache */ __flush_icache_all(); } } +static inline void __kvm_flush_dcache_pte(pte_t pte) +{ + struct page *page = pte_page(pte); + kvm_flush_dcache_to_poc(page_address(page), PAGE_SIZE); +} + +static inline void __kvm_flush_dcache_pmd(pmd_t pmd) +{ + struct page *page = pmd_page(pmd); + kvm_flush_dcache_to_poc(page_address(page), PMD_SIZE); +} + +static inline void __kvm_flush_dcache_pud(pud_t pud) +{ + struct page *page = pud_page(pud); + kvm_flush_dcache_to_poc(page_address(page), PUD_SIZE); +} + #define kvm_virt_to_phys(x) __virt_to_phys((unsigned long)(x)) -void stage2_flush_vm(struct kvm *kvm); +void kvm_set_way_flush(struct kvm_vcpu *vcpu); +void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); + +static inline bool __kvm_cpu_uses_extended_idmap(void) +{ + return __cpu_uses_extended_idmap(); +} + +static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, + pgd_t *hyp_pgd, + pgd_t *merged_hyp_pgd, + unsigned long hyp_idmap_start) +{ + int idmap_idx; + + /* + * Use the first entry to access the HYP mappings. It is + * guaranteed to be free, otherwise we wouldn't use an + * extended idmap. + */ + VM_BUG_ON(pgd_val(merged_hyp_pgd[0])); + merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE); + + /* + * Create another extended level entry that points to the boot HYP map, + * which contains an ID mapping of the HYP init code. We essentially + * merge the boot and runtime HYP maps by doing so, but they don't + * overlap anyway, so this is fine. + */ + idmap_idx = hyp_idmap_start >> VA_BITS; + VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx])); + merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE); +} #endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index a62cd077457b..f800d45ea226 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -33,6 +33,12 @@ #define UL(x) _AC(x, UL) /* + * Size of the PCI I/O space. This must remain a power of two so that + * IO_SPACE_LIMIT acts as a mask for the low bits of I/O addresses. + */ +#define PCI_IO_SIZE SZ_16M + +/* * PAGE_OFFSET - the virtual address of the start of the kernel image (top * (VA_BITS - 1)) * VA_BITS - the maximum number of bits for virtual addresses. @@ -45,7 +51,9 @@ #define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1)) #define MODULES_END (PAGE_OFFSET) #define MODULES_VADDR (MODULES_END - SZ_64M) -#define FIXADDR_TOP (MODULES_VADDR - SZ_2M - PAGE_SIZE) +#define PCI_IO_END (MODULES_VADDR - SZ_2M) +#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE) +#define FIXADDR_TOP (PCI_IO_START - SZ_2M) #define TASK_SIZE_64 (UL(1) << VA_BITS) #ifdef CONFIG_COMPAT @@ -120,11 +128,13 @@ extern phys_addr_t memstart_addr; * translation for translating DMA addresses. Use the driver * DMA support - see dma-mapping.h. */ +#define virt_to_phys virt_to_phys static inline phys_addr_t virt_to_phys(const volatile void *x) { return __virt_to_phys((unsigned long)(x)); } +#define phys_to_virt phys_to_virt static inline void *phys_to_virt(phys_addr_t x) { return (void *)(__phys_to_virt(x)); diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h index c2f006c48bdb..3d311761e3c2 100644 --- a/arch/arm64/include/asm/mmu.h +++ b/arch/arm64/include/asm/mmu.h @@ -31,7 +31,8 @@ extern void paging_init(void); extern void setup_mm_for_reboot(void); extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt); extern void init_mem_pgprot(void); -/* create an identity mapping for memory (or io if map_io is true) */ -extern void create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io); +extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot); #endif diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index a9eee33dfa62..8ec41e5f56f0 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -64,6 +64,49 @@ static inline void cpu_set_reserved_ttbr0(void) : "r" (ttbr)); } +/* + * TCR.T0SZ value to use when the ID map is active. Usually equals + * TCR_T0SZ(VA_BITS), unless system RAM is positioned very high in + * physical memory, in which case it will be smaller. + */ +extern u64 idmap_t0sz; + +static inline bool __cpu_uses_extended_idmap(void) +{ + return (!IS_ENABLED(CONFIG_ARM64_VA_BITS_48) && + unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS))); +} + +static inline void __cpu_set_tcr_t0sz(u64 t0sz) +{ + unsigned long tcr; + + if (__cpu_uses_extended_idmap()) + asm volatile ( + " mrs %0, tcr_el1 ;" + " bfi %0, %1, %2, %3 ;" + " msr tcr_el1, %0 ;" + " isb" + : "=&r" (tcr) + : "r"(t0sz), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH)); +} + +/* + * Set TCR.T0SZ to the value appropriate for activating the identity map. + */ +static inline void cpu_set_idmap_tcr_t0sz(void) +{ + __cpu_set_tcr_t0sz(idmap_t0sz); +} + +/* + * Set TCR.T0SZ to its default value (based on VA_BITS) + */ +static inline void cpu_set_default_tcr_t0sz(void) +{ + __cpu_set_tcr_t0sz(TCR_T0SZ(VA_BITS)); +} + static inline void switch_new_context(struct mm_struct *mm) { unsigned long flags; @@ -151,6 +194,15 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, { unsigned int cpu = smp_processor_id(); + /* + * init_mm.pgd does not contain any user mappings and it is always + * active for kernel addresses in TTBR1. Just set the reserved TTBR0. + */ + if (next == &init_mm) { + cpu_set_reserved_ttbr0(); + return; + } + if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next)) || prev != next) check_and_switch_context(next, tsk); } diff --git a/arch/arm64/include/asm/opcodes.h b/arch/arm64/include/asm/opcodes.h new file mode 100644 index 000000000000..4e603ea36ad3 --- /dev/null +++ b/arch/arm64/include/asm/opcodes.h @@ -0,0 +1 @@ +#include <../../arm/include/asm/opcodes.h> diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 22b16232bd60..7d9c7e4a424b 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -33,16 +33,18 @@ * image. Both require pgd, pud (4 levels only) and pmd tables to (section) * map the kernel. With the 64K page configuration, swapper and idmap need to * map to pte level. The swapper also maps the FDT (see __create_page_tables - * for more information). + * for more information). Note that the number of ID map translation levels + * could be increased on the fly if system RAM is out of reach for the default + * VA range, so 3 pages are reserved in all cases. */ #ifdef CONFIG_ARM64_64K_PAGES -#define SWAPPER_PGTABLE_LEVELS (CONFIG_ARM64_PGTABLE_LEVELS) +#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS) #else -#define SWAPPER_PGTABLE_LEVELS (CONFIG_ARM64_PGTABLE_LEVELS - 1) +#define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1) #endif #define SWAPPER_DIR_SIZE (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE) -#define IDMAP_DIR_SIZE (SWAPPER_DIR_SIZE) +#define IDMAP_DIR_SIZE (3 * PAGE_SIZE) #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/pci.h b/arch/arm64/include/asm/pci.h index 872ba939fcb2..b008a72f8bc0 100644 --- a/arch/arm64/include/asm/pci.h +++ b/arch/arm64/include/asm/pci.h @@ -27,6 +27,12 @@ extern int isa_dma_bridge_buggy; #ifdef CONFIG_PCI +static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) +{ + /* no legacy IRQ on arm64 */ + return -ENODEV; +} + static inline int pci_proc_domain(struct pci_bus *bus) { return 1; diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 5279e5733386..4fde8c1df97f 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -44,6 +44,243 @@ static inline unsigned long __my_cpu_offset(void) #endif /* CONFIG_SMP */ +#define PERCPU_OP(op, asm_op) \ +static inline unsigned long __percpu_##op(void *ptr, \ + unsigned long val, int size) \ +{ \ + unsigned long loop, ret; \ + \ + switch (size) { \ + case 1: \ + do { \ + asm ("//__per_cpu_" #op "_1\n" \ + "ldxrb %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxrb %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u8 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 2: \ + do { \ + asm ("//__per_cpu_" #op "_2\n" \ + "ldxrh %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxrh %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u16 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 4: \ + do { \ + asm ("//__per_cpu_" #op "_4\n" \ + "ldxr %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxr %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u32 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 8: \ + do { \ + asm ("//__per_cpu_" #op "_8\n" \ + "ldxr %[ret], %[ptr]\n" \ + #asm_op " %[ret], %[ret], %[val]\n" \ + "stxr %w[loop], %[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u64 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + \ + return ret; \ +} + +PERCPU_OP(add, add) +PERCPU_OP(and, and) +PERCPU_OP(or, orr) +#undef PERCPU_OP + +static inline unsigned long __percpu_read(void *ptr, int size) +{ + unsigned long ret; + + switch (size) { + case 1: + ret = ACCESS_ONCE(*(u8 *)ptr); + break; + case 2: + ret = ACCESS_ONCE(*(u16 *)ptr); + break; + case 4: + ret = ACCESS_ONCE(*(u32 *)ptr); + break; + case 8: + ret = ACCESS_ONCE(*(u64 *)ptr); + break; + default: + BUILD_BUG(); + } + + return ret; +} + +static inline void __percpu_write(void *ptr, unsigned long val, int size) +{ + switch (size) { + case 1: + ACCESS_ONCE(*(u8 *)ptr) = (u8)val; + break; + case 2: + ACCESS_ONCE(*(u16 *)ptr) = (u16)val; + break; + case 4: + ACCESS_ONCE(*(u32 *)ptr) = (u32)val; + break; + case 8: + ACCESS_ONCE(*(u64 *)ptr) = (u64)val; + break; + default: + BUILD_BUG(); + } +} + +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, + int size) +{ + unsigned long ret, loop; + + switch (size) { + case 1: + do { + asm ("//__percpu_xchg_1\n" + "ldxrb %w[ret], %[ptr]\n" + "stxrb %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u8 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 2: + do { + asm ("//__percpu_xchg_2\n" + "ldxrh %w[ret], %[ptr]\n" + "stxrh %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u16 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 4: + do { + asm ("//__percpu_xchg_4\n" + "ldxr %w[ret], %[ptr]\n" + "stxr %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u32 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 8: + do { + asm ("//__percpu_xchg_8\n" + "ldxr %[ret], %[ptr]\n" + "stxr %w[loop], %[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u64 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + default: + BUILD_BUG(); + } + + return ret; +} + +#define _percpu_read(pcp) \ +({ \ + typeof(pcp) __retval; \ + preempt_disable(); \ + __retval = (typeof(pcp))__percpu_read(raw_cpu_ptr(&(pcp)), \ + sizeof(pcp)); \ + preempt_enable(); \ + __retval; \ +}) + +#define _percpu_write(pcp, val) \ +do { \ + preempt_disable(); \ + __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), \ + sizeof(pcp)); \ + preempt_enable(); \ +} while(0) \ + +#define _pcp_protect(operation, pcp, val) \ +({ \ + typeof(pcp) __retval; \ + preempt_disable(); \ + __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \ + (val), sizeof(pcp)); \ + preempt_enable(); \ + __retval; \ +}) + +#define _percpu_add(pcp, val) \ + _pcp_protect(__percpu_add, pcp, val) + +#define _percpu_add_return(pcp, val) _percpu_add(pcp, val) + +#define _percpu_and(pcp, val) \ + _pcp_protect(__percpu_and, pcp, val) + +#define _percpu_or(pcp, val) \ + _pcp_protect(__percpu_or, pcp, val) + +#define _percpu_xchg(pcp, val) (typeof(pcp)) \ + _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)) + +#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) + +#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) + +#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) + +#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) + +#define this_cpu_read_1(pcp) _percpu_read(pcp) +#define this_cpu_read_2(pcp) _percpu_read(pcp) +#define this_cpu_read_4(pcp) _percpu_read(pcp) +#define this_cpu_read_8(pcp) _percpu_read(pcp) + +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) + +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) + #include <asm-generic/percpu.h> #endif /* __ASM_PERCPU_H */ diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index d5bed02073d6..76420568d66a 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -26,11 +26,13 @@ #define check_pgt_cache() do { } while (0) -#if CONFIG_ARM64_PGTABLE_LEVELS > 2 +#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) + +#if CONFIG_PGTABLE_LEVELS > 2 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + return (pmd_t *)__get_free_page(PGALLOC_GFP); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) @@ -44,13 +46,13 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE)); } -#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 2 */ +#endif /* CONFIG_PGTABLE_LEVELS > 2 */ -#if CONFIG_ARM64_PGTABLE_LEVELS > 3 +#if CONFIG_PGTABLE_LEVELS > 3 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + return (pud_t *)__get_free_page(PGALLOC_GFP); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -64,13 +66,11 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) set_pgd(pgd, __pgd(__pa(pud) | PUD_TYPE_TABLE)); } -#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 3 */ +#endif /* CONFIG_PGTABLE_LEVELS > 3 */ extern pgd_t *pgd_alloc(struct mm_struct *mm); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); -#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) - static inline pte_t * pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) { diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 88174e0bfafe..59bfae75dc98 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -21,7 +21,7 @@ /* * PMD_SHIFT determines the size a level 2 page table entry can map. */ -#if CONFIG_ARM64_PGTABLE_LEVELS > 2 +#if CONFIG_PGTABLE_LEVELS > 2 #define PMD_SHIFT ((PAGE_SHIFT - 3) * 2 + 3) #define PMD_SIZE (_AC(1, UL) << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) @@ -31,7 +31,7 @@ /* * PUD_SHIFT determines the size a level 1 page table entry can map. */ -#if CONFIG_ARM64_PGTABLE_LEVELS > 3 +#if CONFIG_PGTABLE_LEVELS > 3 #define PUD_SHIFT ((PAGE_SHIFT - 3) * 3 + 3) #define PUD_SIZE (_AC(1, UL) << PUD_SHIFT) #define PUD_MASK (~(PUD_SIZE-1)) @@ -42,7 +42,7 @@ * PGDIR_SHIFT determines the size a top-level page table entry can map * (depending on the configuration, this level can be 0, 1 or 2). */ -#define PGDIR_SHIFT ((PAGE_SHIFT - 3) * CONFIG_ARM64_PGTABLE_LEVELS + 3) +#define PGDIR_SHIFT ((PAGE_SHIFT - 3) * CONFIG_PGTABLE_LEVELS + 3) #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) @@ -119,6 +119,7 @@ #define PTE_S2_RDONLY (_AT(pteval_t, 1) << 6) /* HAP[2:1] */ #define PTE_S2_RDWR (_AT(pteval_t, 3) << 6) /* HAP[2:1] */ +#define PMD_S2_RDONLY (_AT(pmdval_t, 1) << 6) /* HAP[2:1] */ #define PMD_S2_RDWR (_AT(pmdval_t, 3) << 6) /* HAP[2:1] */ /* @@ -142,7 +143,12 @@ /* * TCR flags. */ -#define TCR_TxSZ(x) (((UL(64) - (x)) << 16) | ((UL(64) - (x)) << 0)) +#define TCR_T0SZ_OFFSET 0 +#define TCR_T1SZ_OFFSET 16 +#define TCR_T0SZ(x) ((UL(64) - (x)) << TCR_T0SZ_OFFSET) +#define TCR_T1SZ(x) ((UL(64) - (x)) << TCR_T1SZ_OFFSET) +#define TCR_TxSZ(x) (TCR_T0SZ(x) | TCR_T1SZ(x)) +#define TCR_TxSZ_WIDTH 6 #define TCR_IRGN_NC ((UL(0) << 8) | (UL(0) << 24)) #define TCR_IRGN_WBWA ((UL(1) << 8) | (UL(1) << 24)) #define TCR_IRGN_WT ((UL(2) << 8) | (UL(2) << 24)) diff --git a/arch/arm64/include/asm/pgtable-types.h b/arch/arm64/include/asm/pgtable-types.h index ca9df80af896..2b1bd7e52c3b 100644 --- a/arch/arm64/include/asm/pgtable-types.h +++ b/arch/arm64/include/asm/pgtable-types.h @@ -38,13 +38,13 @@ typedef struct { pteval_t pte; } pte_t; #define pte_val(x) ((x).pte) #define __pte(x) ((pte_t) { (x) } ) -#if CONFIG_ARM64_PGTABLE_LEVELS > 2 +#if CONFIG_PGTABLE_LEVELS > 2 typedef struct { pmdval_t pmd; } pmd_t; #define pmd_val(x) ((x).pmd) #define __pmd(x) ((pmd_t) { (x) } ) #endif -#if CONFIG_ARM64_PGTABLE_LEVELS > 3 +#if CONFIG_PGTABLE_LEVELS > 3 typedef struct { pudval_t pud; } pud_t; #define pud_val(x) ((x).pud) #define __pud(x) ((pud_t) { (x) } ) @@ -64,13 +64,13 @@ typedef pteval_t pte_t; #define pte_val(x) (x) #define __pte(x) (x) -#if CONFIG_ARM64_PGTABLE_LEVELS > 2 +#if CONFIG_PGTABLE_LEVELS > 2 typedef pmdval_t pmd_t; #define pmd_val(x) (x) #define __pmd(x) (x) #endif -#if CONFIG_ARM64_PGTABLE_LEVELS > 3 +#if CONFIG_PGTABLE_LEVELS > 3 typedef pudval_t pud_t; #define pud_val(x) (x) #define __pud(x) (x) @@ -86,9 +86,9 @@ typedef pteval_t pgprot_t; #endif /* STRICT_MM_TYPECHECKS */ -#if CONFIG_ARM64_PGTABLE_LEVELS == 2 +#if CONFIG_PGTABLE_LEVELS == 2 #include <asm-generic/pgtable-nopmd.h> -#elif CONFIG_ARM64_PGTABLE_LEVELS == 3 +#elif CONFIG_PGTABLE_LEVELS == 3 #include <asm-generic/pgtable-nopud.h> #endif diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 41a43bf26492..56283f8a675c 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -25,7 +25,6 @@ * Software defined PTE bits definition. */ #define PTE_VALID (_AT(pteval_t, 1) << 0) -#define PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !pte_present() */ #define PTE_DIRTY (_AT(pteval_t, 1) << 55) #define PTE_SPECIAL (_AT(pteval_t, 1) << 56) #define PTE_WRITE (_AT(pteval_t, 1) << 57) @@ -46,7 +45,7 @@ #define vmemmap ((struct page *)(VMALLOC_END + SZ_64K)) -#define FIRST_USER_ADDRESS 0 +#define FIRST_USER_ADDRESS 0UL #ifndef __ASSEMBLY__ extern void __pte_error(const char *file, int line, unsigned long val); @@ -264,6 +263,11 @@ static inline pmd_t pte_pmd(pte_t pte) return __pmd(pte_val(pte)); } +static inline pgprot_t mk_sect_prot(pgprot_t prot) +{ + return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT); +} + /* * THP definitions. */ @@ -279,6 +283,7 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd)) #define pmd_young(pmd) pte_young(pmd_pte(pmd)) #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd))) #define pmd_mksplitting(pmd) pte_pmd(pte_mkspecial(pmd_pte(pmd))) @@ -297,7 +302,6 @@ void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address, #define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))) #define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot) -#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) #define pud_write(pud) pte_write(pud_pte(pud)) #define pud_pfn(pud) (((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT) @@ -337,9 +341,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, #ifdef CONFIG_ARM64_64K_PAGES #define pud_sect(pud) (0) +#define pud_table(pud) (1) #else #define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ PUD_TYPE_SECT) +#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \ + PUD_TYPE_TABLE) #endif static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) @@ -367,7 +374,7 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) */ #define mk_pte(page,prot) pfn_pte(page_to_pfn(page),prot) -#if CONFIG_ARM64_PGTABLE_LEVELS > 2 +#if CONFIG_PGTABLE_LEVELS > 2 #define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd)) @@ -400,11 +407,11 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr); } -#define pud_page(pud) pmd_page(pud_pmd(pud)) +#define pud_page(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK)) -#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 2 */ +#endif /* CONFIG_PGTABLE_LEVELS > 2 */ -#if CONFIG_ARM64_PGTABLE_LEVELS > 3 +#if CONFIG_PGTABLE_LEVELS > 3 #define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud)) @@ -436,7 +443,9 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr) return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(addr); } -#endif /* CONFIG_ARM64_PGTABLE_LEVELS > 3 */ +#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK)) + +#endif /* CONFIG_PGTABLE_LEVELS > 3 */ #define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd_val(pgd)) @@ -451,7 +460,7 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long addr) static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY | - PTE_PROT_NONE | PTE_VALID | PTE_WRITE; + PTE_PROT_NONE | PTE_WRITE | PTE_TYPE_MASK; pte_val(pte) = (pte_val(pte) & ~mask) | (pgprot_val(newprot) & mask); return pte; } @@ -467,13 +476,12 @@ extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; /* * Encode and decode a swap entry: * bits 0-1: present (must be zero) - * bit 2: PTE_FILE - * bits 3-8: swap type - * bits 9-57: swap offset + * bits 2-7: swap type + * bits 8-57: swap offset */ -#define __SWP_TYPE_SHIFT 3 +#define __SWP_TYPE_SHIFT 2 #define __SWP_TYPE_BITS 6 -#define __SWP_OFFSET_BITS 49 +#define __SWP_OFFSET_BITS 50 #define __SWP_TYPE_MASK ((1 << __SWP_TYPE_BITS) - 1) #define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT) #define __SWP_OFFSET_MASK ((1UL << __SWP_OFFSET_BITS) - 1) @@ -491,18 +499,6 @@ extern pgd_t idmap_pg_dir[PTRS_PER_PGD]; */ #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > __SWP_TYPE_BITS) -/* - * Encode and decode a file entry: - * bits 0-1: present (must be zero) - * bit 2: PTE_FILE - * bits 3-57: file offset / PAGE_SIZE - */ -#define pte_file(pte) (pte_val(pte) & PTE_FILE) -#define pte_to_pgoff(x) (pte_val(x) >> 3) -#define pgoff_to_pte(x) __pte(((x) << 3) | PTE_FILE) - -#define PTE_FILE_MAX_BITS 55 - extern int kern_addr_valid(unsigned long addr); #include <asm-generic/pgtable.h> diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h index e6f087806aaf..b7710a59672c 100644 --- a/arch/arm64/include/asm/pmu.h +++ b/arch/arm64/include/asm/pmu.h @@ -44,6 +44,7 @@ struct pmu_hw_events { struct arm_pmu { struct pmu pmu; cpumask_t active_irqs; + int *irq_affinity; const char *name; irqreturn_t (*handle_irq)(int irq_num, void *dev); void (*enable)(struct hw_perf_event *evt, int idx); diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h index 9a8fd84f8fb2..220633b791b8 100644 --- a/arch/arm64/include/asm/proc-fns.h +++ b/arch/arm64/include/asm/proc-fns.h @@ -39,16 +39,11 @@ extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr); #include <asm/memory.h> -#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm) - -#define cpu_get_pgd() \ -({ \ - unsigned long pg; \ - asm("mrs %0, ttbr0_el1\n" \ - : "=r" (pg)); \ - pg &= ~0xffff000000003ffful; \ - (pgd_t *)phys_to_virt(pg); \ -}) +#define cpu_switch_mm(pgd,mm) \ +do { \ + BUG_ON(pgd == swapper_pg_dir); \ + cpu_do_switch_mm(virt_to_phys(pgd),mm); \ +} while (0) #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 286b1bec547c..d2c37a1df0eb 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -31,6 +31,7 @@ #include <asm/fpsimd.h> #include <asm/hw_breakpoint.h> +#include <asm/pgtable-hwdef.h> #include <asm/ptrace.h> #include <asm/types.h> @@ -44,7 +45,8 @@ #define STACK_TOP STACK_TOP_MAX #endif /* CONFIG_COMPAT */ -#define ARCH_LOW_ADDRESS_LIMIT PHYS_MASK +extern phys_addr_t arm64_dma_phys_limit; +#define ARCH_LOW_ADDRESS_LIMIT (arm64_dma_phys_limit - 1) #endif /* __KERNEL__ */ struct debug_info { @@ -123,12 +125,13 @@ struct task_struct; /* Free all resources held by a thread. */ extern void release_thread(struct task_struct *); -/* Prepare to copy thread state - unlazy all lazy status */ -#define prepare_to_copy(tsk) do { } while (0) - unsigned long get_wchan(struct task_struct *p); -#define cpu_relax() barrier() +static inline void cpu_relax(void) +{ + asm volatile("yield" ::: "memory"); +} + #define cpu_relax_lowlatency() cpu_relax() /* Thread switching */ diff --git a/arch/arm64/include/asm/psci.h b/arch/arm64/include/asm/psci.h index e5312ea0ec1a..2454bc59c916 100644 --- a/arch/arm64/include/asm/psci.h +++ b/arch/arm64/include/asm/psci.h @@ -14,6 +14,7 @@ #ifndef __ASM_PSCI_H #define __ASM_PSCI_H -int psci_init(void); +int psci_dt_init(void); +int psci_acpi_init(void); #endif /* __ASM_PSCI_H */ diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 41ed9e13795e..d6dd9fdbc3be 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -58,6 +58,13 @@ #define COMPAT_PSR_Z_BIT 0x40000000 #define COMPAT_PSR_N_BIT 0x80000000 #define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */ + +#ifdef CONFIG_CPU_BIG_ENDIAN +#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT +#else +#define COMPAT_PSR_ENDSTATE 0 +#endif + /* * These are 'magic' values for PTRACE_PEEKUSR that return info about where a * process is located in memory. diff --git a/arch/arm64/include/asm/seccomp.h b/arch/arm64/include/asm/seccomp.h new file mode 100644 index 000000000000..c76fac979629 --- /dev/null +++ b/arch/arm64/include/asm/seccomp.h @@ -0,0 +1,25 @@ +/* + * arch/arm64/include/asm/seccomp.h + * + * Copyright (C) 2014 Linaro Limited + * Author: AKASHI Takahiro <takahiro.akashi@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H + +#include <asm/unistd.h> + +#ifdef CONFIG_COMPAT +#define __NR_seccomp_read_32 __NR_compat_read +#define __NR_seccomp_write_32 __NR_compat_write +#define __NR_seccomp_exit_32 __NR_compat_exit +#define __NR_seccomp_sigreturn_32 __NR_compat_rt_sigreturn +#endif /* CONFIG_COMPAT */ + +#include <asm-generic/seccomp.h> + +#endif /* _ASM_SECCOMP_H */ diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 780f82c827b6..bf22650b1a78 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -39,9 +39,10 @@ extern void show_ipi_list(struct seq_file *p, int prec); extern void handle_IPI(int ipinr, struct pt_regs *regs); /* - * Setup the set of possible CPUs (via set_cpu_possible) + * Discover the set of possible CPUs and determine their + * SMP operations. */ -extern void smp_init_cpus(void); +extern void of_smp_init_cpus(void); /* * Provide a function to raise an IPI cross call on CPUs in callmap. diff --git a/arch/arm64/include/asm/smp_plat.h b/arch/arm64/include/asm/smp_plat.h index 59e282311b58..8dcd61e32176 100644 --- a/arch/arm64/include/asm/smp_plat.h +++ b/arch/arm64/include/asm/smp_plat.h @@ -40,4 +40,6 @@ static inline u32 mpidr_hash_size(void) extern u64 __cpu_logical_map[NR_CPUS]; #define cpu_logical_map(cpu) __cpu_logical_map[cpu] +void __init do_post_cpus_up_work(void); + #endif /* __ASM_SMP_PLAT_H */ diff --git a/arch/arm64/include/asm/spinlock.h b/arch/arm64/include/asm/spinlock.h index c45b7b1b7197..cee128732435 100644 --- a/arch/arm64/include/asm/spinlock.h +++ b/arch/arm64/include/asm/spinlock.h @@ -99,12 +99,12 @@ static inline int arch_spin_value_unlocked(arch_spinlock_t lock) static inline int arch_spin_is_locked(arch_spinlock_t *lock) { - return !arch_spin_value_unlocked(ACCESS_ONCE(*lock)); + return !arch_spin_value_unlocked(READ_ONCE(*lock)); } static inline int arch_spin_is_contended(arch_spinlock_t *lock) { - arch_spinlock_t lockval = ACCESS_ONCE(*lock); + arch_spinlock_t lockval = READ_ONCE(*lock); return (lockval.next - lockval.owner) > 1; } #define arch_spin_is_contended arch_spin_is_contended diff --git a/arch/arm64/include/asm/suspend.h b/arch/arm64/include/asm/suspend.h index 456d67c1f0fa..003802f58963 100644 --- a/arch/arm64/include/asm/suspend.h +++ b/arch/arm64/include/asm/suspend.h @@ -23,6 +23,4 @@ struct sleep_save_sp { extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)); extern void cpu_resume(void); -extern int cpu_suspend(unsigned long); - #endif diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 459bf8e53208..dcd06d18a42a 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -33,7 +33,6 @@ #ifndef __ASSEMBLY__ struct task_struct; -struct exec_domain; #include <asm/types.h> @@ -47,8 +46,6 @@ struct thread_info { unsigned long flags; /* low level flags */ mm_segment_t addr_limit; /* address limit */ struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ - struct restart_block restart_block; int preempt_count; /* 0 => preemptable, <0 => bug */ int cpu; /* cpu */ }; @@ -56,13 +53,9 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ - .restart_block = { \ - .fn = do_no_restart_syscall, \ - }, \ } #define init_thread_info (init_thread_union.thread_info) diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index a82c0c5c8b52..3a0242c7eb8d 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -19,10 +19,6 @@ #ifndef __ASM_TLB_H #define __ASM_TLB_H -#define __tlb_remove_pmd_tlb_entry __tlb_remove_pmd_tlb_entry - -#include <asm-generic/tlb.h> - #include <linux/pagemap.h> #include <linux/swap.h> @@ -37,96 +33,42 @@ static inline void __tlb_remove_table(void *_table) #define tlb_remove_entry(tlb, entry) tlb_remove_page(tlb, entry) #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ -/* - * There's three ways the TLB shootdown code is used: - * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). - * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. - * 2. Unmapping all vmas. See exit_mmap(). - * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. - * Page tables will be freed. - * 3. Unmapping argument pages. See shift_arg_pages(). - * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. - */ +#include <asm-generic/tlb.h> + static inline void tlb_flush(struct mmu_gather *tlb) { if (tlb->fullmm) { flush_tlb_mm(tlb->mm); - } else if (tlb->end > 0) { + } else { struct vm_area_struct vma = { .vm_mm = tlb->mm, }; flush_tlb_range(&vma, tlb->start, tlb->end); - tlb->start = TASK_SIZE; - tlb->end = 0; - } -} - -static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) -{ - if (!tlb->fullmm) { - tlb->start = min(tlb->start, addr); - tlb->end = max(tlb->end, addr + PAGE_SIZE); - } -} - -/* - * Memorize the range for the TLB flush. - */ -static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, - unsigned long addr) -{ - tlb_add_flush(tlb, addr); -} - -/* - * In the case of tlb vma handling, we can optimise these away in the - * case where we're doing a full MM flush. When we're doing a munmap, - * the vmas are adjusted to only cover the region to be torn down. - */ -static inline void tlb_start_vma(struct mmu_gather *tlb, - struct vm_area_struct *vma) -{ - if (!tlb->fullmm) { - tlb->start = TASK_SIZE; - tlb->end = 0; } } -static inline void tlb_end_vma(struct mmu_gather *tlb, - struct vm_area_struct *vma) -{ - if (!tlb->fullmm) - tlb_flush(tlb); -} - static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { + __flush_tlb_pgtable(tlb->mm, addr); pgtable_page_dtor(pte); - tlb_add_flush(tlb, addr); tlb_remove_entry(tlb, pte); } -#if CONFIG_ARM64_PGTABLE_LEVELS > 2 +#if CONFIG_PGTABLE_LEVELS > 2 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) { - tlb_add_flush(tlb, addr); + __flush_tlb_pgtable(tlb->mm, addr); tlb_remove_entry(tlb, virt_to_page(pmdp)); } #endif -#if CONFIG_ARM64_PGTABLE_LEVELS > 3 +#if CONFIG_PGTABLE_LEVELS > 3 static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, unsigned long addr) { - tlb_add_flush(tlb, addr); + __flush_tlb_pgtable(tlb->mm, addr); tlb_remove_entry(tlb, virt_to_page(pudp)); } #endif -static inline void __tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, - unsigned long address) -{ - tlb_add_flush(tlb, address); -} - #endif diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index 73f0ce570fb3..c3bb05b98616 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -24,11 +24,6 @@ #include <linux/sched.h> #include <asm/cputype.h> -extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *); -extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long); - -extern struct cpu_tlb_fns cpu_tlb; - /* * TLB Management * ============== @@ -149,6 +144,19 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end } /* + * Used to invalidate the TLB (walk caches) corresponding to intermediate page + * table levels (pgd/pud/pmd). + */ +static inline void __flush_tlb_pgtable(struct mm_struct *mm, + unsigned long uaddr) +{ + unsigned long addr = uaddr >> 12 | ((unsigned long)ASID(mm) << 48); + + dsb(ishst); + asm("tlbi vae1is, %0" : : "r" (addr)); + dsb(ish); +} +/* * On AArch64, the cache coherency is handled via the set_pte_at() function. */ static inline void update_mmu_cache(struct vm_area_struct *vma, diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h index 7ebcd31ce51c..225ec3524fbf 100644 --- a/arch/arm64/include/asm/topology.h +++ b/arch/arm64/include/asm/topology.h @@ -18,7 +18,7 @@ extern struct cpu_topology cpu_topology[NR_CPUS]; #define topology_physical_package_id(cpu) (cpu_topology[cpu].cluster_id) #define topology_core_id(cpu) (cpu_topology[cpu].core_id) #define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling) -#define topology_thread_cpumask(cpu) (&cpu_topology[cpu].thread_sibling) +#define topology_sibling_cpumask(cpu) (&cpu_topology[cpu].thread_sibling) void init_cpu_topology(void); void store_cpu_topology(unsigned int cpuid); diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h index 10ca8ff93cc2..232e4ba5d314 100644 --- a/arch/arm64/include/asm/traps.h +++ b/arch/arm64/include/asm/traps.h @@ -18,6 +18,22 @@ #ifndef __ASM_TRAP_H #define __ASM_TRAP_H +#include <linux/list.h> + +struct pt_regs; + +struct undef_hook { + struct list_head node; + u32 instr_mask; + u32 instr_val; + u64 pstate_mask; + u64 pstate_val; + int (*fn)(struct pt_regs *regs, u32 instr); +}; + +void register_undef_hook(struct undef_hook *hook); +void unregister_undef_hook(struct undef_hook *hook); + static inline int in_exception_text(unsigned long ptr) { extern char __exception_text_start[]; diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 3bf8f4e99a51..07e1ba449bf1 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -63,7 +63,7 @@ static inline void set_fs(mm_segment_t fs) current_thread_info()->addr_limit = fs; } -#define segment_eq(a,b) ((a) == (b)) +#define segment_eq(a, b) ((a) == (b)) /* * Return 1 if addr < current->addr_limit, 0 otherwise. @@ -147,7 +147,7 @@ do { \ default: \ BUILD_BUG(); \ } \ - (x) = (__typeof__(*(ptr)))__gu_val; \ + (x) = (__force __typeof__(*(ptr)))__gu_val; \ } while (0) #define __get_user(x, ptr) \ diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 6d2bf419431d..3bc498c250dc 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -31,6 +31,9 @@ * Compat syscall numbers used by the AArch64 kernel. */ #define __NR_compat_restart_syscall 0 +#define __NR_compat_exit 1 +#define __NR_compat_read 3 +#define __NR_compat_write 4 #define __NR_compat_sigreturn 119 #define __NR_compat_rt_sigreturn 173 @@ -41,10 +44,13 @@ #define __ARM_NR_compat_cacheflush (__ARM_NR_COMPAT_BASE+2) #define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE+5) -#define __NR_compat_syscalls 386 +#define __NR_compat_syscalls 388 #endif #define __ARCH_WANT_SYS_CLONE + +#ifndef __COMPAT_SYSCALL_NR #include <uapi/asm/unistd.h> +#endif #define NR_syscalls (__NR_syscalls) diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index 9dfdac4a74a1..cef934a90f17 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -406,7 +406,7 @@ __SYSCALL(__NR_vfork, sys_vfork) #define __NR_ugetrlimit 191 /* SuS compliant getrlimit */ __SYSCALL(__NR_ugetrlimit, compat_sys_getrlimit) /* SuS compliant getrlimit */ #define __NR_mmap2 192 -__SYSCALL(__NR_mmap2, sys_mmap_pgoff) +__SYSCALL(__NR_mmap2, compat_sys_mmap2_wrapper) #define __NR_truncate64 193 __SYSCALL(__NR_truncate64, compat_sys_truncate64_wrapper) #define __NR_ftruncate64 194 @@ -787,10 +787,13 @@ __SYSCALL(__NR_sched_setattr, sys_sched_setattr) __SYSCALL(__NR_sched_getattr, sys_sched_getattr) #define __NR_renameat2 382 __SYSCALL(__NR_renameat2, sys_renameat2) - /* 383 for seccomp */ +#define __NR_seccomp 383 +__SYSCALL(__NR_seccomp, sys_seccomp) #define __NR_getrandom 384 __SYSCALL(__NR_getrandom, sys_getrandom) #define __NR_memfd_create 385 __SYSCALL(__NR_memfd_create, sys_memfd_create) #define __NR_bpf 386 __SYSCALL(__NR_bpf, sys_bpf) +#define __NR_execveat 387 +__SYSCALL(__NR_execveat, compat_sys_execveat) diff --git a/arch/arm64/include/asm/xen/page-coherent.h b/arch/arm64/include/asm/xen/page-coherent.h index dde3fc9c49f0..2052102b4e02 100644 --- a/arch/arm64/include/asm/xen/page-coherent.h +++ b/arch/arm64/include/asm/xen/page-coherent.h @@ -1,43 +1 @@ -#ifndef _ASM_ARM64_XEN_PAGE_COHERENT_H -#define _ASM_ARM64_XEN_PAGE_COHERENT_H - -#include <asm/page.h> -#include <linux/dma-attrs.h> -#include <linux/dma-mapping.h> - -static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size, - dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) -{ - return __generic_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs); -} - -static inline void xen_free_coherent_pages(struct device *hwdev, size_t size, - void *cpu_addr, dma_addr_t dma_handle, - struct dma_attrs *attrs) -{ - __generic_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs); -} - -static inline void xen_dma_map_page(struct device *hwdev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) -{ -} - -static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) -{ -} - -static inline void xen_dma_sync_single_for_cpu(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ -} - -static inline void xen_dma_sync_single_for_device(struct device *hwdev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ -} -#endif /* _ASM_ARM64_XEN_PAGE_COHERENT_H */ +#include <../../arm/include/asm/xen/page-coherent.h> diff --git a/arch/arm64/include/uapi/asm/Kbuild b/arch/arm64/include/uapi/asm/Kbuild index 942376d37d22..825b0fe51c2b 100644 --- a/arch/arm64/include/uapi/asm/Kbuild +++ b/arch/arm64/include/uapi/asm/Kbuild @@ -18,4 +18,5 @@ header-y += siginfo.h header-y += signal.h header-y += stat.h header-y += statfs.h +header-y += ucontext.h header-y += unistd.h diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 8e38878c87c6..d26832022127 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -78,6 +78,13 @@ struct kvm_regs { #define KVM_VGIC_V2_DIST_SIZE 0x1000 #define KVM_VGIC_V2_CPU_SIZE 0x2000 +/* Supported VGICv3 address types */ +#define KVM_VGIC_V3_ADDR_TYPE_DIST 2 +#define KVM_VGIC_V3_ADDR_TYPE_REDIST 3 + +#define KVM_VGIC_V3_DIST_SIZE SZ_64K +#define KVM_VGIC_V3_REDIST_SIZE (2 * SZ_64K) + #define KVM_ARM_VCPU_POWER_OFF 0 /* CPU is started in OFF state */ #define KVM_ARM_VCPU_EL1_32BIT 1 /* CPU running a 32bit VM */ #define KVM_ARM_VCPU_PSCI_0_2 2 /* CPU uses PSCI v0.2 */ @@ -161,6 +168,8 @@ struct kvm_arch_memory_slot { #define KVM_DEV_ARM_VGIC_OFFSET_SHIFT 0 #define KVM_DEV_ARM_VGIC_OFFSET_MASK (0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) #define KVM_DEV_ARM_VGIC_GRP_NR_IRQS 3 +#define KVM_DEV_ARM_VGIC_GRP_CTRL 4 +#define KVM_DEV_ARM_VGIC_CTRL_INIT 0 /* KVM_IRQ_LINE irq field index values */ #define KVM_ARM_IRQ_TYPE_SHIFT 24 @@ -179,8 +188,17 @@ struct kvm_arch_memory_slot { #define KVM_ARM_IRQ_CPU_IRQ 0 #define KVM_ARM_IRQ_CPU_FIQ 1 -/* Highest supported SPI, from VGIC_NR_IRQS */ +/* + * This used to hold the highest supported SPI, but it is now obsolete + * and only here to provide source code level compatibility with older + * userland. The highest SPI number can be set via KVM_DEV_ARM_VGIC_GRP_NR_IRQS. + */ +#ifndef __KERNEL__ #define KVM_ARM_IRQ_GIC_MAX 127 +#endif + +/* One single KVM irqchip, ie. the VGIC */ +#define KVM_NR_IRQCHIPS 1 /* PSCI interface */ #define KVM_PSCI_FN_BASE 0x95c1ba5e diff --git a/arch/arm64/include/asm/ucontext.h b/arch/arm64/include/uapi/asm/ucontext.h index 42e04c877428..791de8e89e35 100644 --- a/arch/arm64/include/asm/ucontext.h +++ b/arch/arm64/include/uapi/asm/ucontext.h @@ -13,8 +13,10 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#ifndef __ASM_UCONTEXT_H -#define __ASM_UCONTEXT_H +#ifndef _UAPI__ASM_UCONTEXT_H +#define _UAPI__ASM_UCONTEXT_H + +#include <linux/types.h> struct ucontext { unsigned long uc_flags; @@ -27,4 +29,4 @@ struct ucontext { struct sigcontext uc_mcontext; }; -#endif /* __ASM_UCONTEXT_H */ +#endif /* _UAPI__ASM_UCONTEXT_H */ diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 5bd029b43644..426d0763c81b 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -5,32 +5,37 @@ CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) CFLAGS_efi-stub.o := -DTEXT_OFFSET=$(TEXT_OFFSET) +CFLAGS_armv8_deprecated.o := -I$(src) CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_insn.o = -pg CFLAGS_REMOVE_return_address.o = -pg # Object file lists. -arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ +arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ - hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \ - cpuinfo.o + hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \ + return_address.o cpuinfo.o cpu_errata.o \ + cpufeature.o alternative.o cacheinfo.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ - sys_compat.o + sys_compat.o entry32.o \ + ../../arm/kernel/opcodes.o arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o -arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o +arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o arm64-obj-$(CONFIG_KGDB) += kgdb.o arm64-obj-$(CONFIG_EFI) += efi.o efi-stub.o efi-entry.o arm64-obj-$(CONFIG_PCI) += pci.o +arm64-obj-$(CONFIG_ARMV8_DEPRECATED) += armv8_deprecated.o +arm64-obj-$(CONFIG_ACPI) += acpi.o obj-y += $(arm64-obj-y) vdso/ obj-m += $(arm64-obj-m) diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c new file mode 100644 index 000000000000..8b839558838e --- /dev/null +++ b/arch/arm64/kernel/acpi.c @@ -0,0 +1,345 @@ +/* + * ARM64 Specific Low-Level ACPI Boot Support + * + * Copyright (C) 2013-2014, Linaro Ltd. + * Author: Al Stone <al.stone@linaro.org> + * Author: Graeme Gregory <graeme.gregory@linaro.org> + * Author: Hanjun Guo <hanjun.guo@linaro.org> + * Author: Tomasz Nowicki <tomasz.nowicki@linaro.org> + * Author: Naresh Bhat <naresh.bhat@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#define pr_fmt(fmt) "ACPI: " fmt + +#include <linux/acpi.h> +#include <linux/bootmem.h> +#include <linux/cpumask.h> +#include <linux/init.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/memblock.h> +#include <linux/of_fdt.h> +#include <linux/smp.h> + +#include <asm/cputype.h> +#include <asm/cpu_ops.h> +#include <asm/smp_plat.h> + +int acpi_noirq = 1; /* skip ACPI IRQ initialization */ +int acpi_disabled = 1; +EXPORT_SYMBOL(acpi_disabled); + +int acpi_pci_disabled = 1; /* skip ACPI PCI scan and IRQ initialization */ +EXPORT_SYMBOL(acpi_pci_disabled); + +/* Processors with enabled flag and sane MPIDR */ +static int enabled_cpus; + +/* Boot CPU is valid or not in MADT */ +static bool bootcpu_valid __initdata; + +static bool param_acpi_off __initdata; +static bool param_acpi_force __initdata; + +static int __init parse_acpi(char *arg) +{ + if (!arg) + return -EINVAL; + + /* "acpi=off" disables both ACPI table parsing and interpreter */ + if (strcmp(arg, "off") == 0) + param_acpi_off = true; + else if (strcmp(arg, "force") == 0) /* force ACPI to be enabled */ + param_acpi_force = true; + else + return -EINVAL; /* Core will print when we return error */ + + return 0; +} +early_param("acpi", parse_acpi); + +static int __init dt_scan_depth1_nodes(unsigned long node, + const char *uname, int depth, + void *data) +{ + /* + * Return 1 as soon as we encounter a node at depth 1 that is + * not the /chosen node. + */ + if (depth == 1 && (strcmp(uname, "chosen") != 0)) + return 1; + return 0; +} + +/* + * __acpi_map_table() will be called before page_init(), so early_ioremap() + * or early_memremap() should be called here to for ACPI table mapping. + */ +char *__init __acpi_map_table(unsigned long phys, unsigned long size) +{ + if (!size) + return NULL; + + return early_memremap(phys, size); +} + +void __init __acpi_unmap_table(char *map, unsigned long size) +{ + if (!map || !size) + return; + + early_memunmap(map, size); +} + +/** + * acpi_map_gic_cpu_interface - generates a logical cpu number + * and map to MPIDR represented by GICC structure + */ +static void __init +acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) +{ + int i; + u64 mpidr = processor->arm_mpidr & MPIDR_HWID_BITMASK; + bool enabled = !!(processor->flags & ACPI_MADT_ENABLED); + + if (mpidr == INVALID_HWID) { + pr_info("Skip MADT cpu entry with invalid MPIDR\n"); + return; + } + + total_cpus++; + if (!enabled) + return; + + if (enabled_cpus >= NR_CPUS) { + pr_warn("NR_CPUS limit of %d reached, Processor %d/0x%llx ignored.\n", + NR_CPUS, total_cpus, mpidr); + return; + } + + /* Check if GICC structure of boot CPU is available in the MADT */ + if (cpu_logical_map(0) == mpidr) { + if (bootcpu_valid) { + pr_err("Firmware bug, duplicate CPU MPIDR: 0x%llx in MADT\n", + mpidr); + return; + } + + bootcpu_valid = true; + } + + /* + * Duplicate MPIDRs are a recipe for disaster. Scan + * all initialized entries and check for + * duplicates. If any is found just ignore the CPU. + */ + for (i = 1; i < enabled_cpus; i++) { + if (cpu_logical_map(i) == mpidr) { + pr_err("Firmware bug, duplicate CPU MPIDR: 0x%llx in MADT\n", + mpidr); + return; + } + } + + if (!acpi_psci_present()) + return; + + cpu_ops[enabled_cpus] = cpu_get_ops("psci"); + /* CPU 0 was already initialized */ + if (enabled_cpus) { + if (!cpu_ops[enabled_cpus]) + return; + + if (cpu_ops[enabled_cpus]->cpu_init(NULL, enabled_cpus)) + return; + + /* map the logical cpu id to cpu MPIDR */ + cpu_logical_map(enabled_cpus) = mpidr; + } + + enabled_cpus++; +} + +static int __init +acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header, + const unsigned long end) +{ + struct acpi_madt_generic_interrupt *processor; + + processor = (struct acpi_madt_generic_interrupt *)header; + + if (BAD_MADT_ENTRY(processor, end)) + return -EINVAL; + + acpi_table_print_madt_entry(header); + acpi_map_gic_cpu_interface(processor); + return 0; +} + +/* Parse GIC cpu interface entries in MADT for SMP init */ +void __init acpi_init_cpus(void) +{ + int count, i; + + /* + * do a partial walk of MADT to determine how many CPUs + * we have including disabled CPUs, and get information + * we need for SMP init + */ + count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, + acpi_parse_gic_cpu_interface, 0); + + if (!count) { + pr_err("No GIC CPU interface entries present\n"); + return; + } else if (count < 0) { + pr_err("Error parsing GIC CPU interface entry\n"); + return; + } + + if (!bootcpu_valid) { + pr_err("MADT missing boot CPU MPIDR, not enabling secondaries\n"); + return; + } + + for (i = 0; i < enabled_cpus; i++) + set_cpu_possible(i, true); + + /* Make boot-up look pretty */ + pr_info("%d CPUs enabled, %d CPUs total\n", enabled_cpus, total_cpus); +} + +/* + * acpi_fadt_sanity_check() - Check FADT presence and carry out sanity + * checks on it + * + * Return 0 on success, <0 on failure + */ +static int __init acpi_fadt_sanity_check(void) +{ + struct acpi_table_header *table; + struct acpi_table_fadt *fadt; + acpi_status status; + acpi_size tbl_size; + int ret = 0; + + /* + * FADT is required on arm64; retrieve it to check its presence + * and carry out revision and ACPI HW reduced compliancy tests + */ + status = acpi_get_table_with_size(ACPI_SIG_FADT, 0, &table, &tbl_size); + if (ACPI_FAILURE(status)) { + const char *msg = acpi_format_exception(status); + + pr_err("Failed to get FADT table, %s\n", msg); + return -ENODEV; + } + + fadt = (struct acpi_table_fadt *)table; + + /* + * Revision in table header is the FADT Major revision, and there + * is a minor revision of FADT which was introduced by ACPI 5.1, + * we only deal with ACPI 5.1 or newer revision to get GIC and SMP + * boot protocol configuration data. + */ + if (table->revision < 5 || + (table->revision == 5 && fadt->minor_revision < 1)) { + pr_err("Unsupported FADT revision %d.%d, should be 5.1+\n", + table->revision, fadt->minor_revision); + ret = -EINVAL; + goto out; + } + + if (!(fadt->flags & ACPI_FADT_HW_REDUCED)) { + pr_err("FADT not ACPI hardware reduced compliant\n"); + ret = -EINVAL; + } + +out: + /* + * acpi_get_table_with_size() creates FADT table mapping that + * should be released after parsing and before resuming boot + */ + early_acpi_os_unmap_memory(table, tbl_size); + return ret; +} + +/* + * acpi_boot_table_init() called from setup_arch(), always. + * 1. find RSDP and get its address, and then find XSDT + * 2. extract all tables and checksums them all + * 3. check ACPI FADT revision + * 4. check ACPI FADT HW reduced flag + * + * We can parse ACPI boot-time tables such as MADT after + * this function is called. + * + * On return ACPI is enabled if either: + * + * - ACPI tables are initialized and sanity checks passed + * - acpi=force was passed in the command line and ACPI was not disabled + * explicitly through acpi=off command line parameter + * + * ACPI is disabled on function return otherwise + */ +void __init acpi_boot_table_init(void) +{ + /* + * Enable ACPI instead of device tree unless + * - ACPI has been disabled explicitly (acpi=off), or + * - the device tree is not empty (it has more than just a /chosen node) + * and ACPI has not been force enabled (acpi=force) + */ + if (param_acpi_off || + (!param_acpi_force && of_scan_flat_dt(dt_scan_depth1_nodes, NULL))) + return; + + /* + * ACPI is disabled at this point. Enable it in order to parse + * the ACPI tables and carry out sanity checks + */ + enable_acpi(); + + /* + * If ACPI tables are initialized and FADT sanity checks passed, + * leave ACPI enabled and carry on booting; otherwise disable ACPI + * on initialization error. + * If acpi=force was passed on the command line it forces ACPI + * to be enabled even if its initialization failed. + */ + if (acpi_table_init() || acpi_fadt_sanity_check()) { + pr_err("Failed to init ACPI tables\n"); + if (!param_acpi_force) + disable_acpi(); + } +} + +void __init acpi_gic_init(void) +{ + struct acpi_table_header *table; + acpi_status status; + acpi_size tbl_size; + int err; + + if (acpi_disabled) + return; + + status = acpi_get_table_with_size(ACPI_SIG_MADT, 0, &table, &tbl_size); + if (ACPI_FAILURE(status)) { + const char *msg = acpi_format_exception(status); + + pr_err("Failed to get MADT table, %s\n", msg); + return; + } + + err = gic_v2_acpi_init(table); + if (err) + pr_err("Failed to initialize GIC IRQ controller"); + + early_acpi_os_unmap_memory((char *)table, tbl_size); +} diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c new file mode 100644 index 000000000000..28f8365edc4c --- /dev/null +++ b/arch/arm64/kernel/alternative.c @@ -0,0 +1,85 @@ +/* + * alternative runtime patching + * inspired by the x86 version + * + * Copyright (C) 2014 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) "alternatives: " fmt + +#include <linux/init.h> +#include <linux/cpu.h> +#include <asm/cacheflush.h> +#include <asm/alternative.h> +#include <asm/cpufeature.h> +#include <linux/stop_machine.h> + +extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; + +struct alt_region { + struct alt_instr *begin; + struct alt_instr *end; +}; + +static int __apply_alternatives(void *alt_region) +{ + struct alt_instr *alt; + struct alt_region *region = alt_region; + u8 *origptr, *replptr; + + for (alt = region->begin; alt < region->end; alt++) { + if (!cpus_have_cap(alt->cpufeature)) + continue; + + BUG_ON(alt->alt_len != alt->orig_len); + + pr_info_once("patching kernel code\n"); + + origptr = (u8 *)&alt->orig_offset + alt->orig_offset; + replptr = (u8 *)&alt->alt_offset + alt->alt_offset; + memcpy(origptr, replptr, alt->alt_len); + flush_icache_range((uintptr_t)origptr, + (uintptr_t)(origptr + alt->alt_len)); + } + + return 0; +} + +void apply_alternatives_all(void) +{ + struct alt_region region = { + .begin = __alt_instructions, + .end = __alt_instructions_end, + }; + + /* better not try code patching on a live SMP system */ + stop_machine(__apply_alternatives, ®ion, NULL); +} + +void apply_alternatives(void *start, size_t length) +{ + struct alt_region region = { + .begin = start, + .end = start + length, + }; + + __apply_alternatives(®ion); +} + +void free_alternatives_memory(void) +{ + free_reserved_area(__alt_instructions, __alt_instructions_end, + 0, "alternatives"); +} diff --git a/arch/arm64/kernel/armv8_deprecated.c b/arch/arm64/kernel/armv8_deprecated.c new file mode 100644 index 000000000000..7922c2e710ca --- /dev/null +++ b/arch/arm64/kernel/armv8_deprecated.c @@ -0,0 +1,662 @@ +/* + * Copyright (C) 2014 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/cpu.h> +#include <linux/init.h> +#include <linux/list.h> +#include <linux/perf_event.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/sysctl.h> + +#include <asm/insn.h> +#include <asm/opcodes.h> +#include <asm/system_misc.h> +#include <asm/traps.h> +#include <asm/uaccess.h> +#include <asm/cpufeature.h> + +#define CREATE_TRACE_POINTS +#include "trace-events-emulation.h" + +/* + * The runtime support for deprecated instruction support can be in one of + * following three states - + * + * 0 = undef + * 1 = emulate (software emulation) + * 2 = hw (supported in hardware) + */ +enum insn_emulation_mode { + INSN_UNDEF, + INSN_EMULATE, + INSN_HW, +}; + +enum legacy_insn_status { + INSN_DEPRECATED, + INSN_OBSOLETE, +}; + +struct insn_emulation_ops { + const char *name; + enum legacy_insn_status status; + struct undef_hook *hooks; + int (*set_hw_mode)(bool enable); +}; + +struct insn_emulation { + struct list_head node; + struct insn_emulation_ops *ops; + int current_mode; + int min; + int max; +}; + +static LIST_HEAD(insn_emulation); +static int nr_insn_emulated; +static DEFINE_RAW_SPINLOCK(insn_emulation_lock); + +static void register_emulation_hooks(struct insn_emulation_ops *ops) +{ + struct undef_hook *hook; + + BUG_ON(!ops->hooks); + + for (hook = ops->hooks; hook->instr_mask; hook++) + register_undef_hook(hook); + + pr_notice("Registered %s emulation handler\n", ops->name); +} + +static void remove_emulation_hooks(struct insn_emulation_ops *ops) +{ + struct undef_hook *hook; + + BUG_ON(!ops->hooks); + + for (hook = ops->hooks; hook->instr_mask; hook++) + unregister_undef_hook(hook); + + pr_notice("Removed %s emulation handler\n", ops->name); +} + +static void enable_insn_hw_mode(void *data) +{ + struct insn_emulation *insn = (struct insn_emulation *)data; + if (insn->ops->set_hw_mode) + insn->ops->set_hw_mode(true); +} + +static void disable_insn_hw_mode(void *data) +{ + struct insn_emulation *insn = (struct insn_emulation *)data; + if (insn->ops->set_hw_mode) + insn->ops->set_hw_mode(false); +} + +/* Run set_hw_mode(mode) on all active CPUs */ +static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable) +{ + if (!insn->ops->set_hw_mode) + return -EINVAL; + if (enable) + on_each_cpu(enable_insn_hw_mode, (void *)insn, true); + else + on_each_cpu(disable_insn_hw_mode, (void *)insn, true); + return 0; +} + +/* + * Run set_hw_mode for all insns on a starting CPU. + * Returns: + * 0 - If all the hooks ran successfully. + * -EINVAL - At least one hook is not supported by the CPU. + */ +static int run_all_insn_set_hw_mode(unsigned long cpu) +{ + int rc = 0; + unsigned long flags; + struct insn_emulation *insn; + + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_for_each_entry(insn, &insn_emulation, node) { + bool enable = (insn->current_mode == INSN_HW); + if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) { + pr_warn("CPU[%ld] cannot support the emulation of %s", + cpu, insn->ops->name); + rc = -EINVAL; + } + } + raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + return rc; +} + +static int update_insn_emulation_mode(struct insn_emulation *insn, + enum insn_emulation_mode prev) +{ + int ret = 0; + + switch (prev) { + case INSN_UNDEF: /* Nothing to be done */ + break; + case INSN_EMULATE: + remove_emulation_hooks(insn->ops); + break; + case INSN_HW: + if (!run_all_cpu_set_hw_mode(insn, false)) + pr_notice("Disabled %s support\n", insn->ops->name); + break; + } + + switch (insn->current_mode) { + case INSN_UNDEF: + break; + case INSN_EMULATE: + register_emulation_hooks(insn->ops); + break; + case INSN_HW: + ret = run_all_cpu_set_hw_mode(insn, true); + if (!ret) + pr_notice("Enabled %s support\n", insn->ops->name); + break; + } + + return ret; +} + +static void register_insn_emulation(struct insn_emulation_ops *ops) +{ + unsigned long flags; + struct insn_emulation *insn; + + insn = kzalloc(sizeof(*insn), GFP_KERNEL); + insn->ops = ops; + insn->min = INSN_UNDEF; + + switch (ops->status) { + case INSN_DEPRECATED: + insn->current_mode = INSN_EMULATE; + /* Disable the HW mode if it was turned on at early boot time */ + run_all_cpu_set_hw_mode(insn, false); + insn->max = INSN_HW; + break; + case INSN_OBSOLETE: + insn->current_mode = INSN_UNDEF; + insn->max = INSN_EMULATE; + break; + } + + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_add(&insn->node, &insn_emulation); + nr_insn_emulated++; + raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + + /* Register any handlers if required */ + update_insn_emulation_mode(insn, INSN_UNDEF); +} + +static int emulation_proc_handler(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, + loff_t *ppos) +{ + int ret = 0; + struct insn_emulation *insn = (struct insn_emulation *) table->data; + enum insn_emulation_mode prev_mode = insn->current_mode; + + table->data = &insn->current_mode; + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (ret || !write || prev_mode == insn->current_mode) + goto ret; + + ret = update_insn_emulation_mode(insn, prev_mode); + if (ret) { + /* Mode change failed, revert to previous mode. */ + insn->current_mode = prev_mode; + update_insn_emulation_mode(insn, INSN_UNDEF); + } +ret: + table->data = insn; + return ret; +} + +static struct ctl_table ctl_abi[] = { + { + .procname = "abi", + .mode = 0555, + }, + { } +}; + +static void register_insn_emulation_sysctl(struct ctl_table *table) +{ + unsigned long flags; + int i = 0; + struct insn_emulation *insn; + struct ctl_table *insns_sysctl, *sysctl; + + insns_sysctl = kzalloc(sizeof(*sysctl) * (nr_insn_emulated + 1), + GFP_KERNEL); + + raw_spin_lock_irqsave(&insn_emulation_lock, flags); + list_for_each_entry(insn, &insn_emulation, node) { + sysctl = &insns_sysctl[i]; + + sysctl->mode = 0644; + sysctl->maxlen = sizeof(int); + + sysctl->procname = insn->ops->name; + sysctl->data = insn; + sysctl->extra1 = &insn->min; + sysctl->extra2 = &insn->max; + sysctl->proc_handler = emulation_proc_handler; + i++; + } + raw_spin_unlock_irqrestore(&insn_emulation_lock, flags); + + table->child = insns_sysctl; + register_sysctl_table(table); +} + +/* + * Implement emulation of the SWP/SWPB instructions using load-exclusive and + * store-exclusive. + * + * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] + * Where: Rt = destination + * Rt2 = source + * Rn = address + */ + +/* + * Error-checking SWP macros implemented using ldxr{b}/stxr{b} + */ +#define __user_swpX_asm(data, addr, res, temp, B) \ + __asm__ __volatile__( \ + " mov %w2, %w1\n" \ + "0: ldxr"B" %w1, [%3]\n" \ + "1: stxr"B" %w0, %w2, [%3]\n" \ + " cbz %w0, 2f\n" \ + " mov %w0, %w4\n" \ + "2:\n" \ + " .pushsection .fixup,\"ax\"\n" \ + " .align 2\n" \ + "3: mov %w0, %w5\n" \ + " b 2b\n" \ + " .popsection" \ + " .pushsection __ex_table,\"a\"\n" \ + " .align 3\n" \ + " .quad 0b, 3b\n" \ + " .quad 1b, 3b\n" \ + " .popsection" \ + : "=&r" (res), "+r" (data), "=&r" (temp) \ + : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ + : "memory") + +#define __user_swp_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "") +#define __user_swpb_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "b") + +/* + * Bit 22 of the instruction encoding distinguishes between + * the SWP and SWPB variants (bit set means SWPB). + */ +#define TYPE_SWPB (1 << 22) + +/* + * Set up process info to signal segmentation fault - called on access error. + */ +static void set_segfault(struct pt_regs *regs, unsigned long addr) +{ + siginfo_t info; + + down_read(¤t->mm->mmap_sem); + if (find_vma(current->mm, addr) == NULL) + info.si_code = SEGV_MAPERR; + else + info.si_code = SEGV_ACCERR; + up_read(¤t->mm->mmap_sem); + + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void *) instruction_pointer(regs); + + pr_debug("SWP{B} emulation: access caused memory abort!\n"); + arm64_notify_die("Illegal memory access", regs, &info, 0); +} + +static int emulate_swpX(unsigned int address, unsigned int *data, + unsigned int type) +{ + unsigned int res = 0; + + if ((type != TYPE_SWPB) && (address & 0x3)) { + /* SWP to unaligned address not permitted */ + pr_debug("SWP instruction on unaligned pointer!\n"); + return -EFAULT; + } + + while (1) { + unsigned long temp; + + if (type == TYPE_SWPB) + __user_swpb_asm(*data, address, res, temp); + else + __user_swp_asm(*data, address, res, temp); + + if (likely(res != -EAGAIN) || signal_pending(current)) + break; + + cond_resched(); + } + + return res; +} + +/* + * swp_handler logs the id of calling process, dissects the instruction, sanity + * checks the memory location, calls emulate_swpX for the actual operation and + * deals with fixup/error handling before returning + */ +static int swp_handler(struct pt_regs *regs, u32 instr) +{ + u32 destreg, data, type, address = 0; + int rn, rt2, res = 0; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); + + type = instr & TYPE_SWPB; + + switch (arm_check_condition(instr, regs->pstate)) { + case ARM_OPCODE_CONDTEST_PASS: + break; + case ARM_OPCODE_CONDTEST_FAIL: + /* Condition failed - return to next instruction */ + goto ret; + case ARM_OPCODE_CONDTEST_UNCOND: + /* If unconditional encoding - not a SWP, undef */ + return -EFAULT; + default: + return -EINVAL; + } + + rn = aarch32_insn_extract_reg_num(instr, A32_RN_OFFSET); + rt2 = aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET); + + address = (u32)regs->user_regs.regs[rn]; + data = (u32)regs->user_regs.regs[rt2]; + destreg = aarch32_insn_extract_reg_num(instr, A32_RT_OFFSET); + + pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", + rn, address, destreg, + aarch32_insn_extract_reg_num(instr, A32_RT2_OFFSET), data); + + /* Check access in reasonable access range for both SWP and SWPB */ + if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { + pr_debug("SWP{B} emulation: access to 0x%08x not allowed!\n", + address); + goto fault; + } + + res = emulate_swpX(address, &data, type); + if (res == -EFAULT) + goto fault; + else if (res == 0) + regs->user_regs.regs[destreg] = data; + +ret: + if (type == TYPE_SWPB) + trace_instruction_emulation("swpb", regs->pc); + else + trace_instruction_emulation("swp", regs->pc); + + pr_warn_ratelimited("\"%s\" (%ld) uses obsolete SWP{B} instruction at 0x%llx\n", + current->comm, (unsigned long)current->pid, regs->pc); + + regs->pc += 4; + return 0; + +fault: + set_segfault(regs, address); + + return 0; +} + +/* + * Only emulate SWP/SWPB executed in ARM state/User mode. + * The kernel must be SWP free and SWP{B} does not exist in Thumb. + */ +static struct undef_hook swp_hooks[] = { + { + .instr_mask = 0x0fb00ff0, + .instr_val = 0x01000090, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = swp_handler + }, + { } +}; + +static struct insn_emulation_ops swp_ops = { + .name = "swp", + .status = INSN_OBSOLETE, + .hooks = swp_hooks, + .set_hw_mode = NULL, +}; + +static int cp15barrier_handler(struct pt_regs *regs, u32 instr) +{ + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); + + switch (arm_check_condition(instr, regs->pstate)) { + case ARM_OPCODE_CONDTEST_PASS: + break; + case ARM_OPCODE_CONDTEST_FAIL: + /* Condition failed - return to next instruction */ + goto ret; + case ARM_OPCODE_CONDTEST_UNCOND: + /* If unconditional encoding - not a barrier instruction */ + return -EFAULT; + default: + return -EINVAL; + } + + switch (aarch32_insn_mcr_extract_crm(instr)) { + case 10: + /* + * dmb - mcr p15, 0, Rt, c7, c10, 5 + * dsb - mcr p15, 0, Rt, c7, c10, 4 + */ + if (aarch32_insn_mcr_extract_opc2(instr) == 5) { + dmb(sy); + trace_instruction_emulation( + "mcr p15, 0, Rt, c7, c10, 5 ; dmb", regs->pc); + } else { + dsb(sy); + trace_instruction_emulation( + "mcr p15, 0, Rt, c7, c10, 4 ; dsb", regs->pc); + } + break; + case 5: + /* + * isb - mcr p15, 0, Rt, c7, c5, 4 + * + * Taking an exception or returning from one acts as an + * instruction barrier. So no explicit barrier needed here. + */ + trace_instruction_emulation( + "mcr p15, 0, Rt, c7, c5, 4 ; isb", regs->pc); + break; + } + +ret: + pr_warn_ratelimited("\"%s\" (%ld) uses deprecated CP15 Barrier instruction at 0x%llx\n", + current->comm, (unsigned long)current->pid, regs->pc); + + regs->pc += 4; + return 0; +} + +static inline void config_sctlr_el1(u32 clear, u32 set) +{ + u32 val; + + asm volatile("mrs %0, sctlr_el1" : "=r" (val)); + val &= ~clear; + val |= set; + asm volatile("msr sctlr_el1, %0" : : "r" (val)); +} + +static int cp15_barrier_set_hw_mode(bool enable) +{ + if (enable) + config_sctlr_el1(0, SCTLR_EL1_CP15BEN); + else + config_sctlr_el1(SCTLR_EL1_CP15BEN, 0); + return 0; +} + +static struct undef_hook cp15_barrier_hooks[] = { + { + .instr_mask = 0x0fff0fdf, + .instr_val = 0x0e070f9a, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = cp15barrier_handler, + }, + { + .instr_mask = 0x0fff0fff, + .instr_val = 0x0e070f95, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = cp15barrier_handler, + }, + { } +}; + +static struct insn_emulation_ops cp15_barrier_ops = { + .name = "cp15_barrier", + .status = INSN_DEPRECATED, + .hooks = cp15_barrier_hooks, + .set_hw_mode = cp15_barrier_set_hw_mode, +}; + +static int setend_set_hw_mode(bool enable) +{ + if (!cpu_supports_mixed_endian_el0()) + return -EINVAL; + + if (enable) + config_sctlr_el1(SCTLR_EL1_SED, 0); + else + config_sctlr_el1(0, SCTLR_EL1_SED); + return 0; +} + +static int compat_setend_handler(struct pt_regs *regs, u32 big_endian) +{ + char *insn; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc); + + if (big_endian) { + insn = "setend be"; + regs->pstate |= COMPAT_PSR_E_BIT; + } else { + insn = "setend le"; + regs->pstate &= ~COMPAT_PSR_E_BIT; + } + + trace_instruction_emulation(insn, regs->pc); + pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n", + current->comm, (unsigned long)current->pid, regs->pc); + + return 0; +} + +static int a32_setend_handler(struct pt_regs *regs, u32 instr) +{ + int rc = compat_setend_handler(regs, (instr >> 9) & 1); + regs->pc += 4; + return rc; +} + +static int t16_setend_handler(struct pt_regs *regs, u32 instr) +{ + int rc = compat_setend_handler(regs, (instr >> 3) & 1); + regs->pc += 2; + return rc; +} + +static struct undef_hook setend_hooks[] = { + { + .instr_mask = 0xfffffdff, + .instr_val = 0xf1010000, + .pstate_mask = COMPAT_PSR_MODE_MASK, + .pstate_val = COMPAT_PSR_MODE_USR, + .fn = a32_setend_handler, + }, + { + /* Thumb mode */ + .instr_mask = 0x0000fff7, + .instr_val = 0x0000b650, + .pstate_mask = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK), + .pstate_val = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR), + .fn = t16_setend_handler, + }, + {} +}; + +static struct insn_emulation_ops setend_ops = { + .name = "setend", + .status = INSN_DEPRECATED, + .hooks = setend_hooks, + .set_hw_mode = setend_set_hw_mode, +}; + +static int insn_cpu_hotplug_notify(struct notifier_block *b, + unsigned long action, void *hcpu) +{ + int rc = 0; + if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING) + rc = run_all_insn_set_hw_mode((unsigned long)hcpu); + + return notifier_from_errno(rc); +} + +static struct notifier_block insn_cpu_hotplug_notifier = { + .notifier_call = insn_cpu_hotplug_notify, +}; + +/* + * Invoked as late_initcall, since not needed before init spawned. + */ +static int __init armv8_deprecated_init(void) +{ + if (IS_ENABLED(CONFIG_SWP_EMULATION)) + register_insn_emulation(&swp_ops); + + if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION)) + register_insn_emulation(&cp15_barrier_ops); + + if (IS_ENABLED(CONFIG_SETEND_EMULATION)) { + if(system_supports_mixed_endian_el0()) + register_insn_emulation(&setend_ops); + else + pr_info("setend instruction emulation is not supported on the system"); + } + + register_cpu_notifier(&insn_cpu_hotplug_notifier); + register_insn_emulation_sysctl(ctl_abi); + + return 0; +} + +late_initcall(armv8_deprecated_init); diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 9a9fce090d58..da675cc5dfae 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -24,7 +24,6 @@ #include <linux/kvm_host.h> #include <asm/thread_info.h> #include <asm/memory.h> -#include <asm/cputable.h> #include <asm/smp_plat.h> #include <asm/suspend.h> #include <asm/vdso_datapage.h> @@ -38,7 +37,6 @@ int main(void) DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); @@ -71,9 +69,6 @@ int main(void) BLANK(); DEFINE(PAGE_SZ, PAGE_SIZE); BLANK(); - DEFINE(CPU_INFO_SZ, sizeof(struct cpu_info)); - DEFINE(CPU_INFO_SETUP, offsetof(struct cpu_info, cpu_setup)); - BLANK(); DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL); DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); @@ -140,6 +135,7 @@ int main(void) DEFINE(VGIC_V2_CPU_ELRSR, offsetof(struct vgic_cpu, vgic_v2.vgic_elrsr)); DEFINE(VGIC_V2_CPU_APR, offsetof(struct vgic_cpu, vgic_v2.vgic_apr)); DEFINE(VGIC_V2_CPU_LR, offsetof(struct vgic_cpu, vgic_v2.vgic_lr)); + DEFINE(VGIC_V3_CPU_SRE, offsetof(struct vgic_cpu, vgic_v3.vgic_sre)); DEFINE(VGIC_V3_CPU_HCR, offsetof(struct vgic_cpu, vgic_v3.vgic_hcr)); DEFINE(VGIC_V3_CPU_VMCR, offsetof(struct vgic_cpu, vgic_v3.vgic_vmcr)); DEFINE(VGIC_V3_CPU_MISR, offsetof(struct vgic_cpu, vgic_v3.vgic_misr)); @@ -152,7 +148,7 @@ int main(void) DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); #endif -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_PM DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx)); DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp)); DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask)); diff --git a/arch/arm64/kernel/cacheinfo.c b/arch/arm64/kernel/cacheinfo.c new file mode 100644 index 000000000000..b8629d52fba9 --- /dev/null +++ b/arch/arm64/kernel/cacheinfo.c @@ -0,0 +1,128 @@ +/* + * ARM64 cacheinfo support + * + * Copyright (C) 2015 ARM Ltd. + * All Rights Reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/bitops.h> +#include <linux/cacheinfo.h> +#include <linux/cpu.h> +#include <linux/compiler.h> +#include <linux/of.h> + +#include <asm/cachetype.h> +#include <asm/processor.h> + +#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */ +/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */ +#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1)) +#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level)) +#define CLIDR_CTYPE(clidr, level) \ + (((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level)) + +static inline enum cache_type get_cache_type(int level) +{ + u64 clidr; + + if (level > MAX_CACHE_LEVEL) + return CACHE_TYPE_NOCACHE; + asm volatile ("mrs %x0, clidr_el1" : "=r" (clidr)); + return CLIDR_CTYPE(clidr, level); +} + +/* + * Cache Size Selection Register(CSSELR) selects which Cache Size ID + * Register(CCSIDR) is accessible by specifying the required cache + * level and the cache type. We need to ensure that no one else changes + * CSSELR by calling this in non-preemtible context + */ +u64 __attribute_const__ cache_get_ccsidr(u64 csselr) +{ + u64 ccsidr; + + WARN_ON(preemptible()); + + /* Put value into CSSELR */ + asm volatile("msr csselr_el1, %x0" : : "r" (csselr)); + isb(); + /* Read result out of CCSIDR */ + asm volatile("mrs %x0, ccsidr_el1" : "=r" (ccsidr)); + + return ccsidr; +} + +static void ci_leaf_init(struct cacheinfo *this_leaf, + enum cache_type type, unsigned int level) +{ + bool is_icache = type & CACHE_TYPE_INST; + u64 tmp = cache_get_ccsidr((level - 1) << 1 | is_icache); + + this_leaf->level = level; + this_leaf->type = type; + this_leaf->coherency_line_size = CACHE_LINESIZE(tmp); + this_leaf->number_of_sets = CACHE_NUMSETS(tmp); + this_leaf->ways_of_associativity = CACHE_ASSOCIATIVITY(tmp); + this_leaf->size = this_leaf->number_of_sets * + this_leaf->coherency_line_size * this_leaf->ways_of_associativity; + this_leaf->attributes = + ((tmp & CCSIDR_EL1_WRITE_THROUGH) ? CACHE_WRITE_THROUGH : 0) | + ((tmp & CCSIDR_EL1_WRITE_BACK) ? CACHE_WRITE_BACK : 0) | + ((tmp & CCSIDR_EL1_READ_ALLOCATE) ? CACHE_READ_ALLOCATE : 0) | + ((tmp & CCSIDR_EL1_WRITE_ALLOCATE) ? CACHE_WRITE_ALLOCATE : 0); +} + +static int __init_cache_level(unsigned int cpu) +{ + unsigned int ctype, level, leaves; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + + for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) { + ctype = get_cache_type(level); + if (ctype == CACHE_TYPE_NOCACHE) { + level--; + break; + } + /* Separate instruction and data caches */ + leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1; + } + + this_cpu_ci->num_levels = level; + this_cpu_ci->num_leaves = leaves; + return 0; +} + +static int __populate_cache_leaves(unsigned int cpu) +{ + unsigned int level, idx; + enum cache_type type; + struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu); + struct cacheinfo *this_leaf = this_cpu_ci->info_list; + + for (idx = 0, level = 1; level <= this_cpu_ci->num_levels && + idx < this_cpu_ci->num_leaves; idx++, level++) { + type = get_cache_type(level); + if (type == CACHE_TYPE_SEPARATE) { + ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level); + ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level); + } else { + ci_leaf_init(this_leaf++, type, level); + } + } + return 0; +} + +DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level) +DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves) diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c new file mode 100644 index 000000000000..6ffd91438560 --- /dev/null +++ b/arch/arm64/kernel/cpu_errata.c @@ -0,0 +1,92 @@ +/* + * Contains CPU specific errata definitions + * + * Copyright (C) 2014 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/types.h> +#include <asm/cpu.h> +#include <asm/cputype.h> +#include <asm/cpufeature.h> + +#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) +#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) + +#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \ + MIDR_ARCHITECTURE_MASK) + +static bool __maybe_unused +is_affected_midr_range(const struct arm64_cpu_capabilities *entry) +{ + u32 midr = read_cpuid_id(); + + if ((midr & CPU_MODEL_MASK) != entry->midr_model) + return false; + + midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK; + + return (midr >= entry->midr_range_min && midr <= entry->midr_range_max); +} + +#define MIDR_RANGE(model, min, max) \ + .matches = is_affected_midr_range, \ + .midr_model = model, \ + .midr_range_min = min, \ + .midr_range_max = max + +const struct arm64_cpu_capabilities arm64_errata[] = { +#if defined(CONFIG_ARM64_ERRATUM_826319) || \ + defined(CONFIG_ARM64_ERRATUM_827319) || \ + defined(CONFIG_ARM64_ERRATUM_824069) + { + /* Cortex-A53 r0p[012] */ + .desc = "ARM errata 826319, 827319, 824069", + .capability = ARM64_WORKAROUND_CLEAN_CACHE, + MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x02), + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_819472 + { + /* Cortex-A53 r0p[01] */ + .desc = "ARM errata 819472", + .capability = ARM64_WORKAROUND_CLEAN_CACHE, + MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x01), + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_832075 + { + /* Cortex-A57 r0p0 - r1p2 */ + .desc = "ARM erratum 832075", + .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE, + MIDR_RANGE(MIDR_CORTEX_A57, 0x00, + (1 << MIDR_VARIANT_SHIFT) | 2), + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_845719 + { + /* Cortex-A53 r0p[01234] */ + .desc = "ARM erratum 845719", + .capability = ARM64_WORKAROUND_845719, + MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04), + }, +#endif + { + } +}; + +void check_local_cpu_errata(void) +{ + check_cpu_capabilities(arm64_errata, "enabling workaround for"); +} diff --git a/arch/arm64/kernel/cpu_ops.c b/arch/arm64/kernel/cpu_ops.c index cce952440c64..fb8ff9ba467a 100644 --- a/arch/arm64/kernel/cpu_ops.c +++ b/arch/arm64/kernel/cpu_ops.c @@ -35,7 +35,7 @@ static const struct cpu_operations *supported_cpu_ops[] __initconst = { NULL, }; -static const struct cpu_operations * __init cpu_get_ops(const char *name) +const struct cpu_operations * __init cpu_get_ops(const char *name) { const struct cpu_operations **ops = supported_cpu_ops; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c new file mode 100644 index 000000000000..3d9967e43d89 --- /dev/null +++ b/arch/arm64/kernel/cpufeature.c @@ -0,0 +1,47 @@ +/* + * Contains CPU feature definitions + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) "alternatives: " fmt + +#include <linux/types.h> +#include <asm/cpu.h> +#include <asm/cpufeature.h> + +static const struct arm64_cpu_capabilities arm64_features[] = { + {}, +}; + +void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, + const char *info) +{ + int i; + + for (i = 0; caps[i].desc; i++) { + if (!caps[i].matches(&caps[i])) + continue; + + if (!cpus_have_cap(caps[i].capability)) + pr_info("%s %s\n", info, caps[i].desc); + cpus_set_cap(caps[i].capability); + } +} + +void check_local_cpu_features(void) +{ + check_cpu_capabilities(arm64_features, "detected feature"); +} diff --git a/arch/arm64/kernel/cpuidle.c b/arch/arm64/kernel/cpuidle.c index 19d17f51db37..a78143a5c99f 100644 --- a/arch/arm64/kernel/cpuidle.c +++ b/arch/arm64/kernel/cpuidle.c @@ -15,7 +15,7 @@ #include <asm/cpuidle.h> #include <asm/cpu_ops.h> -int cpu_init_idle(unsigned int cpu) +int arm_cpuidle_init(unsigned int cpu) { int ret = -EOPNOTSUPP; struct device_node *cpu_node = of_cpu_device_node_get(cpu); @@ -29,3 +29,23 @@ int cpu_init_idle(unsigned int cpu) of_node_put(cpu_node); return ret; } + +/** + * cpu_suspend() - function to enter a low-power idle state + * @arg: argument to pass to CPU suspend operations + * + * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU + * operations back-end error code otherwise. + */ +int cpu_suspend(unsigned long arg) +{ + int cpu = smp_processor_id(); + + /* + * If cpu_ops have not been registered or suspend + * has not been initialized, cpu_suspend call fails early. + */ + if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend) + return -EOPNOTSUPP; + return cpu_ops[cpu]->cpu_suspend(arg); +} diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 504fdaa8367e..75d5a867e7fb 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -18,6 +18,7 @@ #include <asm/cachetype.h> #include <asm/cpu.h> #include <asm/cputype.h> +#include <asm/cpufeature.h> #include <linux/bitops.h> #include <linux/bug.h> @@ -34,6 +35,7 @@ */ DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data); static struct cpuinfo_arm64 boot_cpu_data; +static bool mixed_endian_el0 = true; static char *icache_policy_str[] = { [ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN", @@ -67,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); } +bool cpu_supports_mixed_endian_el0(void) +{ + return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); +} + +bool system_supports_mixed_endian_el0(void) +{ + return mixed_endian_el0; +} + +static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info) +{ + mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0); +} + +static void update_cpu_features(struct cpuinfo_arm64 *info) +{ + update_mixed_endian_el0_support(info); +} + static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu) { if ((boot & mask) == (cur & mask)) @@ -111,6 +133,15 @@ static void cpuinfo_sanity_check(struct cpuinfo_arm64 *cur) diff |= CHECK(cntfrq, boot, cur, cpu); /* + * The kernel uses self-hosted debug features and expects CPUs to + * support identical debug features. We presently need CTX_CMPs, WRPs, + * and BRPs to be identical. + * ID_AA64DFR1 is currently RES0. + */ + diff |= CHECK(id_aa64dfr0, boot, cur, cpu); + diff |= CHECK(id_aa64dfr1, boot, cur, cpu); + + /* * Even in big.LITTLE, processors should be identical instruction-set * wise. */ @@ -137,25 +168,35 @@ static void cpuinfo_sanity_check(struct cpuinfo_arm64 *cur) * If we have AArch32, we care about 32-bit features for compat. These * registers should be RES0 otherwise. */ + diff |= CHECK(id_dfr0, boot, cur, cpu); diff |= CHECK(id_isar0, boot, cur, cpu); diff |= CHECK(id_isar1, boot, cur, cpu); diff |= CHECK(id_isar2, boot, cur, cpu); diff |= CHECK(id_isar3, boot, cur, cpu); diff |= CHECK(id_isar4, boot, cur, cpu); diff |= CHECK(id_isar5, boot, cur, cpu); - diff |= CHECK(id_mmfr0, boot, cur, cpu); + /* + * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and + * ACTLR formats could differ across CPUs and therefore would have to + * be trapped for virtualization anyway. + */ + diff |= CHECK_MASK(id_mmfr0, 0xff0fffff, boot, cur, cpu); diff |= CHECK(id_mmfr1, boot, cur, cpu); diff |= CHECK(id_mmfr2, boot, cur, cpu); diff |= CHECK(id_mmfr3, boot, cur, cpu); diff |= CHECK(id_pfr0, boot, cur, cpu); diff |= CHECK(id_pfr1, boot, cur, cpu); + diff |= CHECK(mvfr0, boot, cur, cpu); + diff |= CHECK(mvfr1, boot, cur, cpu); + diff |= CHECK(mvfr2, boot, cur, cpu); + /* * Mismatched CPU features are a recipe for disaster. Don't even * pretend to support them. */ WARN_TAINT_ONCE(diff, TAINT_CPU_OUT_OF_SPEC, - "Unsupported CPU feature variation."); + "Unsupported CPU feature variation.\n"); } static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) @@ -165,6 +206,8 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_dczid = read_cpuid(DCZID_EL0); info->reg_midr = read_cpuid_id(); + info->reg_id_aa64dfr0 = read_cpuid(ID_AA64DFR0_EL1); + info->reg_id_aa64dfr1 = read_cpuid(ID_AA64DFR1_EL1); info->reg_id_aa64isar0 = read_cpuid(ID_AA64ISAR0_EL1); info->reg_id_aa64isar1 = read_cpuid(ID_AA64ISAR1_EL1); info->reg_id_aa64mmfr0 = read_cpuid(ID_AA64MMFR0_EL1); @@ -172,6 +215,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_id_aa64pfr0 = read_cpuid(ID_AA64PFR0_EL1); info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1); + info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1); info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1); info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1); info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1); @@ -185,7 +229,15 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1); info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1); + info->reg_mvfr0 = read_cpuid(MVFR0_EL1); + info->reg_mvfr1 = read_cpuid(MVFR1_EL1); + info->reg_mvfr2 = read_cpuid(MVFR2_EL1); + cpuinfo_detect_icache_policy(info); + + check_local_cpu_errata(); + check_local_cpu_features(); + update_cpu_features(info); } void cpuinfo_store_cpu(void) @@ -202,15 +254,3 @@ void __init cpuinfo_store_boot_cpu(void) boot_cpu_data = *info; } - -u64 __attribute_const__ icache_get_ccsidr(void) -{ - u64 ccsidr; - - WARN_ON(preemptible()); - - /* Select L1 I-cache and read its size ID register */ - asm("msr csselr_el1, %1; isb; mrs %0, ccsidr_el1" - : "=r"(ccsidr) : "r"(1L)); - return ccsidr; -} diff --git a/arch/arm64/kernel/cputable.c b/arch/arm64/kernel/cputable.c deleted file mode 100644 index fd3993cb060f..000000000000 --- a/arch/arm64/kernel/cputable.c +++ /dev/null @@ -1,33 +0,0 @@ -/* - * arch/arm64/kernel/cputable.c - * - * Copyright (C) 2012 ARM Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ - -#include <linux/init.h> - -#include <asm/cputable.h> - -extern unsigned long __cpu_setup(void); - -struct cpu_info cpu_table[] = { - { - .cpu_id_val = 0x000f0000, - .cpu_id_mask = 0x000f0000, - .cpu_name = "AArch64 Processor", - .cpu_setup = __cpu_setup, - }, - { /* Empty */ }, -}; diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S index d18a44940968..8ce9b0577442 100644 --- a/arch/arm64/kernel/efi-entry.S +++ b/arch/arm64/kernel/efi-entry.S @@ -61,7 +61,8 @@ ENTRY(efi_stub_entry) */ mov x20, x0 // DTB address ldr x0, [sp, #16] // relocated _text address - mov x21, x0 + ldr x21, =stext_offset + add x21, x0, x21 /* * Calculate size of the kernel Image (same for original and copy). diff --git a/arch/arm64/kernel/efi-stub.c b/arch/arm64/kernel/efi-stub.c index d27dd982ff26..f5374065ad53 100644 --- a/arch/arm64/kernel/efi-stub.c +++ b/arch/arm64/kernel/efi-stub.c @@ -13,13 +13,13 @@ #include <asm/efi.h> #include <asm/sections.h> -efi_status_t handle_kernel_image(efi_system_table_t *sys_table, - unsigned long *image_addr, - unsigned long *image_size, - unsigned long *reserve_addr, - unsigned long *reserve_size, - unsigned long dram_base, - efi_loaded_image_t *image) +efi_status_t __init handle_kernel_image(efi_system_table_t *sys_table, + unsigned long *image_addr, + unsigned long *image_size, + unsigned long *reserve_addr, + unsigned long *reserve_size, + unsigned long dram_base, + efi_loaded_image_t *image) { efi_status_t status; unsigned long kernel_size, kernel_memsize = 0; diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 95c49ebc660d..ab21e0d58278 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -11,26 +11,46 @@ * */ +#include <linux/atomic.h> +#include <linux/dmi.h> #include <linux/efi.h> #include <linux/export.h> #include <linux/memblock.h> +#include <linux/mm_types.h> #include <linux/bootmem.h> #include <linux/of.h> #include <linux/of_fdt.h> +#include <linux/preempt.h> +#include <linux/rbtree.h> +#include <linux/rwsem.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/spinlock.h> #include <asm/cacheflush.h> #include <asm/efi.h> #include <asm/tlbflush.h> #include <asm/mmu_context.h> +#include <asm/mmu.h> +#include <asm/pgtable.h> struct efi_memory_map memmap; -static efi_runtime_services_t *runtime; - static u64 efi_system_table; +static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss; + +static struct mm_struct efi_mm = { + .mm_rb = RB_ROOT, + .pgd = efi_pgd, + .mm_users = ATOMIC_INIT(2), + .mm_count = ATOMIC_INIT(1), + .mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem), + .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock), + .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), + INIT_MM_CONTEXT(efi_mm) +}; + static int uefi_debug __initdata; static int __init uefi_debug_setup(char *str) { @@ -47,30 +67,33 @@ static int __init is_normal_ram(efi_memory_desc_t *md) return 0; } -static void __init efi_setup_idmap(void) +/* + * Translate a EFI virtual address into a physical address: this is necessary, + * as some data members of the EFI system table are virtually remapped after + * SetVirtualAddressMap() has been called. + */ +static phys_addr_t efi_to_phys(unsigned long addr) { - struct memblock_region *r; efi_memory_desc_t *md; - u64 paddr, npages, size; - - for_each_memblock(memory, r) - create_id_mapping(r->base, r->size, 0); - /* map runtime io spaces */ for_each_efi_memory_desc(&memmap, md) { - if (!(md->attribute & EFI_MEMORY_RUNTIME) || is_normal_ram(md)) + if (!(md->attribute & EFI_MEMORY_RUNTIME)) continue; - paddr = md->phys_addr; - npages = md->num_pages; - memrange_efi_to_native(&paddr, &npages); - size = npages << PAGE_SHIFT; - create_id_mapping(paddr, size, 1); + if (md->virt_addr == 0) + /* no virtual mapping has been installed by the stub */ + break; + if (md->virt_addr <= addr && + (addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT)) + return md->phys_addr + addr - md->virt_addr; } + return addr; } static int __init uefi_init(void) { efi_char16_t *c16; + void *config_tables; + u64 table_size; char vendor[100] = "unknown"; int i, retval; @@ -98,7 +121,7 @@ static int __init uefi_init(void) efi.systab->hdr.revision & 0xffff); /* Show what we know for posterity */ - c16 = early_memremap(efi.systab->fw_vendor, + c16 = early_memremap(efi_to_phys(efi.systab->fw_vendor), sizeof(vendor)); if (c16) { for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i) @@ -111,10 +134,14 @@ static int __init uefi_init(void) efi.systab->hdr.revision >> 16, efi.systab->hdr.revision & 0xffff, vendor); - retval = efi_config_init(NULL); - if (retval == 0) - set_bit(EFI_CONFIG_TABLES, &efi.flags); + table_size = sizeof(efi_config_table_64_t) * efi.systab->nr_tables; + config_tables = early_memremap(efi_to_phys(efi.systab->tables), + table_size); + retval = efi_config_parse_tables(config_tables, efi.systab->nr_tables, + sizeof(efi_config_table_64_t), NULL); + + early_memunmap(config_tables, table_size); out: early_memunmap(efi.systab, sizeof(efi_system_table_t)); return retval; @@ -125,17 +152,17 @@ out: */ static __init int is_reserve_region(efi_memory_desc_t *md) { - if (!is_normal_ram(md)) + switch (md->type) { + case EFI_LOADER_CODE: + case EFI_LOADER_DATA: + case EFI_BOOT_SERVICES_CODE: + case EFI_BOOT_SERVICES_DATA: + case EFI_CONVENTIONAL_MEMORY: return 0; - - if (md->attribute & EFI_MEMORY_RUNTIME) - return 1; - - if (md->type == EFI_ACPI_RECLAIM_MEMORY || - md->type == EFI_RESERVED_TYPE) - return 1; - - return 0; + default: + break; + } + return is_normal_ram(md); } static __init void reserve_regions(void) @@ -164,9 +191,7 @@ static __init void reserve_regions(void) if (is_normal_ram(md)) early_init_dt_add_memory_arch(paddr, size); - if (is_reserve_region(md) || - md->type == EFI_BOOT_SERVICES_CODE || - md->type == EFI_BOOT_SERVICES_DATA) { + if (is_reserve_region(md)) { memblock_reserve(paddr, size); if (uefi_debug) pr_cont("*"); @@ -179,123 +204,6 @@ static __init void reserve_regions(void) set_bit(EFI_MEMMAP, &efi.flags); } - -static u64 __init free_one_region(u64 start, u64 end) -{ - u64 size = end - start; - - if (uefi_debug) - pr_info(" EFI freeing: 0x%012llx-0x%012llx\n", start, end - 1); - - free_bootmem_late(start, size); - return size; -} - -static u64 __init free_region(u64 start, u64 end) -{ - u64 map_start, map_end, total = 0; - - if (end <= start) - return total; - - map_start = (u64)memmap.phys_map; - map_end = PAGE_ALIGN(map_start + (memmap.map_end - memmap.map)); - map_start &= PAGE_MASK; - - if (start < map_end && end > map_start) { - /* region overlaps UEFI memmap */ - if (start < map_start) - total += free_one_region(start, map_start); - - if (map_end < end) - total += free_one_region(map_end, end); - } else - total += free_one_region(start, end); - - return total; -} - -static void __init free_boot_services(void) -{ - u64 total_freed = 0; - u64 keep_end, free_start, free_end; - efi_memory_desc_t *md; - - /* - * If kernel uses larger pages than UEFI, we have to be careful - * not to inadvertantly free memory we want to keep if there is - * overlap at the kernel page size alignment. We do not want to - * free is_reserve_region() memory nor the UEFI memmap itself. - * - * The memory map is sorted, so we keep track of the end of - * any previous region we want to keep, remember any region - * we want to free and defer freeing it until we encounter - * the next region we want to keep. This way, before freeing - * it, we can clip it as needed to avoid freeing memory we - * want to keep for UEFI. - */ - - keep_end = 0; - free_start = 0; - - for_each_efi_memory_desc(&memmap, md) { - u64 paddr, npages, size; - - if (is_reserve_region(md)) { - /* - * We don't want to free any memory from this region. - */ - if (free_start) { - /* adjust free_end then free region */ - if (free_end > md->phys_addr) - free_end -= PAGE_SIZE; - total_freed += free_region(free_start, free_end); - free_start = 0; - } - keep_end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT); - continue; - } - - if (md->type != EFI_BOOT_SERVICES_CODE && - md->type != EFI_BOOT_SERVICES_DATA) { - /* no need to free this region */ - continue; - } - - /* - * We want to free memory from this region. - */ - paddr = md->phys_addr; - npages = md->num_pages; - memrange_efi_to_native(&paddr, &npages); - size = npages << PAGE_SHIFT; - - if (free_start) { - if (paddr <= free_end) - free_end = paddr + size; - else { - total_freed += free_region(free_start, free_end); - free_start = paddr; - free_end = paddr + size; - } - } else { - free_start = paddr; - free_end = paddr + size; - } - if (free_start < keep_end) { - free_start += PAGE_SIZE; - if (free_start >= free_end) - free_start = 0; - } - } - if (free_start) - total_freed += free_region(free_start, free_end); - - if (total_freed) - pr_info("Freed 0x%llx bytes of EFI boot services memory", - total_freed); -} - void __init efi_init(void) { struct efi_fdt_params params; @@ -318,156 +226,144 @@ void __init efi_init(void) return; reserve_regions(); + early_memunmap(memmap.map, params.mmap_size); } -void __init efi_idmap_init(void) +static bool __init efi_virtmap_init(void) { - if (!efi_enabled(EFI_BOOT)) - return; - - /* boot time idmap_pg_dir is incomplete, so fill in missing parts */ - efi_setup_idmap(); -} - -static int __init remap_region(efi_memory_desc_t *md, void **new) -{ - u64 paddr, vaddr, npages, size; - - paddr = md->phys_addr; - npages = md->num_pages; - memrange_efi_to_native(&paddr, &npages); - size = npages << PAGE_SHIFT; + efi_memory_desc_t *md; - if (is_normal_ram(md)) - vaddr = (__force u64)ioremap_cache(paddr, size); - else - vaddr = (__force u64)ioremap(paddr, size); + for_each_efi_memory_desc(&memmap, md) { + u64 paddr, npages, size; + pgprot_t prot; - if (!vaddr) { - pr_err("Unable to remap 0x%llx pages @ %p\n", - npages, (void *)paddr); - return 0; - } + if (!(md->attribute & EFI_MEMORY_RUNTIME)) + continue; + if (md->virt_addr == 0) + return false; - /* adjust for any rounding when EFI and system pagesize differs */ - md->virt_addr = vaddr + (md->phys_addr - paddr); + paddr = md->phys_addr; + npages = md->num_pages; + memrange_efi_to_native(&paddr, &npages); + size = npages << PAGE_SHIFT; - if (uefi_debug) - pr_info(" EFI remap 0x%012llx => %p\n", + pr_info(" EFI remap 0x%016llx => %p\n", md->phys_addr, (void *)md->virt_addr); - memcpy(*new, md, memmap.desc_size); - *new += memmap.desc_size; - - return 1; + /* + * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be + * executable, everything else can be mapped with the XN bits + * set. + */ + if (!is_normal_ram(md)) + prot = __pgprot(PROT_DEVICE_nGnRE); + else if (md->type == EFI_RUNTIME_SERVICES_CODE) + prot = PAGE_KERNEL_EXEC; + else + prot = PAGE_KERNEL; + + create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot); + } + return true; } /* - * Switch UEFI from an identity map to a kernel virtual map + * Enable the UEFI Runtime Services if all prerequisites are in place, i.e., + * non-early mapping of the UEFI system table and virtual mappings for all + * EFI_MEMORY_RUNTIME regions. */ -static int __init arm64_enter_virtual_mode(void) +static int __init arm64_enable_runtime_services(void) { - efi_memory_desc_t *md; - phys_addr_t virtmap_phys; - void *virtmap, *virt_md; - efi_status_t status; u64 mapsize; - int count = 0; - unsigned long flags; if (!efi_enabled(EFI_BOOT)) { pr_info("EFI services will not be available.\n"); return -1; } - mapsize = memmap.map_end - memmap.map; - early_memunmap(memmap.map, mapsize); - if (efi_runtime_disabled()) { pr_info("EFI runtime services will be disabled.\n"); return -1; } pr_info("Remapping and enabling EFI services.\n"); - /* replace early memmap mapping with permanent mapping */ + + mapsize = memmap.map_end - memmap.map; memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map, mapsize); - memmap.map_end = memmap.map + mapsize; - - efi.memmap = &memmap; - - /* Map the runtime regions */ - virtmap = kmalloc(mapsize, GFP_KERNEL); - if (!virtmap) { - pr_err("Failed to allocate EFI virtual memmap\n"); + if (!memmap.map) { + pr_err("Failed to remap EFI memory map\n"); return -1; } - virtmap_phys = virt_to_phys(virtmap); - virt_md = virtmap; - - for_each_efi_memory_desc(&memmap, md) { - if (!(md->attribute & EFI_MEMORY_RUNTIME)) - continue; - if (!remap_region(md, &virt_md)) - goto err_unmap; - ++count; - } + memmap.map_end = memmap.map + mapsize; + efi.memmap = &memmap; - efi.systab = (__force void *)efi_lookup_mapped_addr(efi_system_table); + efi.systab = (__force void *)ioremap_cache(efi_system_table, + sizeof(efi_system_table_t)); if (!efi.systab) { - /* - * If we have no virtual mapping for the System Table at this - * point, the memory map doesn't cover the physical offset where - * it resides. This means the System Table will be inaccessible - * to Runtime Services themselves once the virtual mapping is - * installed. - */ - pr_err("Failed to remap EFI System Table -- buggy firmware?\n"); - goto err_unmap; + pr_err("Failed to remap EFI System Table\n"); + return -1; } set_bit(EFI_SYSTEM_TABLES, &efi.flags); - local_irq_save(flags); - cpu_switch_mm(idmap_pg_dir, &init_mm); - - /* Call SetVirtualAddressMap with the physical address of the map */ - runtime = efi.systab->runtime; - efi.set_virtual_address_map = runtime->set_virtual_address_map; - - status = efi.set_virtual_address_map(count * memmap.desc_size, - memmap.desc_size, - memmap.desc_version, - (efi_memory_desc_t *)virtmap_phys); - cpu_set_reserved_ttbr0(); - flush_tlb_all(); - local_irq_restore(flags); - - kfree(virtmap); - - free_boot_services(); - - if (status != EFI_SUCCESS) { - pr_err("Failed to set EFI virtual address map! [%lx]\n", - status); + if (!efi_virtmap_init()) { + pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n"); return -1; } /* Set up runtime services function pointers */ - runtime = efi.systab->runtime; efi_native_runtime_setup(); set_bit(EFI_RUNTIME_SERVICES, &efi.flags); efi.runtime_version = efi.systab->hdr.revision; return 0; +} +early_initcall(arm64_enable_runtime_services); -err_unmap: - /* unmap all mappings that succeeded: there are 'count' of those */ - for (virt_md = virtmap; count--; virt_md += memmap.desc_size) { - md = virt_md; - iounmap((__force void __iomem *)md->virt_addr); - } - kfree(virtmap); - return -1; +static int __init arm64_dmi_init(void) +{ + /* + * On arm64, DMI depends on UEFI, and dmi_scan_machine() needs to + * be called early because dmi_id_init(), which is an arch_initcall + * itself, depends on dmi_scan_machine() having been called already. + */ + dmi_scan_machine(); + if (dmi_available) + dmi_set_dump_stack_arch_desc(); + return 0; +} +core_initcall(arm64_dmi_init); + +static void efi_set_pgd(struct mm_struct *mm) +{ + if (mm == &init_mm) + cpu_set_reserved_ttbr0(); + else + cpu_switch_mm(mm->pgd, mm); + + flush_tlb_all(); + if (icache_is_aivivt()) + __flush_icache_all(); +} + +void efi_virtmap_load(void) +{ + preempt_disable(); + efi_set_pgd(&efi_mm); +} + +void efi_virtmap_unload(void) +{ + efi_set_pgd(current->active_mm); + preempt_enable(); +} + +/* + * UpdateCapsule() depends on the system being shutdown via + * ResetSystem(). + */ +bool efi_poweroff_required(void) +{ + return efi_enabled(EFI_RUNTIME_SERVICES); } -early_initcall(arm64_enter_virtual_mode); diff --git a/arch/arm64/kernel/entry-ftrace.S b/arch/arm64/kernel/entry-ftrace.S index 38e704e597f7..08cafc518b9a 100644 --- a/arch/arm64/kernel/entry-ftrace.S +++ b/arch/arm64/kernel/entry-ftrace.S @@ -98,8 +98,8 @@ ENTRY(_mcount) mcount_enter - ldr x0, =ftrace_trace_function - ldr x2, [x0] + adrp x0, ftrace_trace_function + ldr x2, [x0, #:lo12:ftrace_trace_function] adr x0, ftrace_stub cmp x0, x2 // if (ftrace_trace_function b.eq skip_ftrace_call // != ftrace_stub) { @@ -115,14 +115,15 @@ skip_ftrace_call: // return; mcount_exit // return; // } skip_ftrace_call: - ldr x1, =ftrace_graph_return - ldr x2, [x1] // if ((ftrace_graph_return - cmp x0, x2 // != ftrace_stub) - b.ne ftrace_graph_caller - - ldr x1, =ftrace_graph_entry // || (ftrace_graph_entry - ldr x2, [x1] // != ftrace_graph_entry_stub)) - ldr x0, =ftrace_graph_entry_stub + adrp x1, ftrace_graph_return + ldr x2, [x1, #:lo12:ftrace_graph_return] + cmp x0, x2 // if ((ftrace_graph_return + b.ne ftrace_graph_caller // != ftrace_stub) + + adrp x1, ftrace_graph_entry // || (ftrace_graph_entry + adrp x0, ftrace_graph_entry_stub // != ftrace_graph_entry_stub)) + ldr x2, [x1, #:lo12:ftrace_graph_entry] + add x0, x0, #:lo12:ftrace_graph_entry_stub cmp x0, x2 b.ne ftrace_graph_caller // ftrace_graph_caller(); diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 726b910fe6ec..959fe8733560 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -21,8 +21,10 @@ #include <linux/init.h> #include <linux/linkage.h> +#include <asm/alternative-asm.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> +#include <asm/cpufeature.h> #include <asm/errno.h> #include <asm/esr.h> #include <asm/thread_info.h> @@ -64,25 +66,26 @@ #define BAD_ERROR 3 .macro kernel_entry, el, regsize = 64 - sub sp, sp, #S_FRAME_SIZE - S_LR // room for LR, SP, SPSR, ELR + sub sp, sp, #S_FRAME_SIZE .if \regsize == 32 mov w0, w0 // zero upper 32 bits of x0 .endif - push x28, x29 - push x26, x27 - push x24, x25 - push x22, x23 - push x20, x21 - push x18, x19 - push x16, x17 - push x14, x15 - push x12, x13 - push x10, x11 - push x8, x9 - push x6, x7 - push x4, x5 - push x2, x3 - push x0, x1 + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + .if \el == 0 mrs x21, sp_el0 get_thread_info tsk // Ensure MDSCR_EL1.SS is clear, @@ -118,33 +121,49 @@ .if \el == 0 ct_user_enter ldr x23, [sp, #S_SP] // load return stack pointer + msr sp_el0, x23 + +#ifdef CONFIG_ARM64_ERRATUM_845719 + alternative_insn \ + "nop", \ + "tbz x22, #4, 1f", \ + ARM64_WORKAROUND_845719 +#ifdef CONFIG_PID_IN_CONTEXTIDR + alternative_insn \ + "nop; nop", \ + "mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:", \ + ARM64_WORKAROUND_845719 +#else + alternative_insn \ + "nop", \ + "msr contextidr_el1, xzr; 1:", \ + ARM64_WORKAROUND_845719 +#endif +#endif .endif + msr elr_el1, x21 // set up the return data + msr spsr_el1, x22 .if \ret ldr x1, [sp, #S_X1] // preserve x0 (syscall return) - add sp, sp, S_X2 .else - pop x0, x1 + ldp x0, x1, [sp, #16 * 0] .endif - pop x2, x3 // load the rest of the registers - pop x4, x5 - pop x6, x7 - pop x8, x9 - msr elr_el1, x21 // set up the return data - msr spsr_el1, x22 - .if \el == 0 - msr sp_el0, x23 - .endif - pop x10, x11 - pop x12, x13 - pop x14, x15 - pop x16, x17 - pop x18, x19 - pop x20, x21 - pop x22, x23 - pop x24, x25 - pop x26, x27 - pop x28, x29 - ldr lr, [sp], #S_FRAME_SIZE - S_LR // load LR and restore SP + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + ldr lr, [sp, #S_LR] + add sp, sp, #S_FRAME_SIZE // restore sp eret // return to kernel .endm @@ -168,7 +187,8 @@ tsk .req x28 // current thread_info * Interrupt handling. */ .macro irq_handler - ldr x1, handle_arch_irq + adrp x1, handle_arch_irq + ldr x1, [x1, #:lo12:handle_arch_irq] mov x0, sp blr x1 .endm @@ -269,18 +289,18 @@ ENDPROC(el1_error_invalid) el1_sync: kernel_entry 1 mrs x1, esr_el1 // read the syndrome register - lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class - cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1 + lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class + cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1 b.eq el1_da - cmp x24, #ESR_EL1_EC_SYS64 // configurable trap + cmp x24, #ESR_ELx_EC_SYS64 // configurable trap b.eq el1_undef - cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception + cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception b.eq el1_sp_pc - cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception + cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception b.eq el1_sp_pc - cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1 + cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1 b.eq el1_undef - cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1 + cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1 b.ge el1_dbg b el1_inv el1_da: @@ -318,7 +338,7 @@ el1_dbg: /* * Debug exception handling */ - cmp x24, #ESR_EL1_EC_BRK64 // if BRK64 + cmp x24, #ESR_ELx_EC_BRK64 // if BRK64 cinc x24, x24, eq // set bit '0' tbz x24, #0, el1_inv // EL1 only mrs x0, far_el1 @@ -375,26 +395,26 @@ el1_preempt: el0_sync: kernel_entry 0 mrs x25, esr_el1 // read the syndrome register - lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class - cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state + lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class + cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state b.eq el0_svc - cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 + cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0 b.eq el0_da - cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 + cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0 b.eq el0_ia - cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access + cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access b.eq el0_fpsimd_acc - cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception + cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception b.eq el0_fpsimd_exc - cmp x24, #ESR_EL1_EC_SYS64 // configurable trap + cmp x24, #ESR_ELx_EC_SYS64 // configurable trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception + cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception b.eq el0_sp_pc - cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception + cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception b.eq el0_sp_pc - cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 + cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 b.eq el0_undef - cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 + cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0 b.ge el0_dbg b el0_inv @@ -403,37 +423,37 @@ el0_sync: el0_sync_compat: kernel_entry 0, 32 mrs x25, esr_el1 // read the syndrome register - lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class - cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state + lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class + cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state b.eq el0_svc_compat - cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0 + cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0 b.eq el0_da - cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0 + cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0 b.eq el0_ia - cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access + cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access b.eq el0_fpsimd_acc - cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception + cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception b.eq el0_fpsimd_exc - cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0 + cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0 b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap + cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap + cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap + cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap + cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap + cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap b.eq el0_undef - cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0 + cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0 b.ge el0_dbg b el0_inv el0_svc_compat: /* * AArch32 syscall handling */ - adr stbl, compat_sys_call_table // load compat syscall table pointer + adrp stbl, compat_sys_call_table // load compat syscall table pointer uxtw scno, w7 // syscall number in w7 (r7) mov sc_nr, #__NR_compat_syscalls b el0_svc_naked @@ -455,8 +475,8 @@ el0_da: bic x0, x26, #(0xff << 56) mov x1, x25 mov x2, sp - adr lr, ret_to_user - b do_mem_abort + bl do_mem_abort + b ret_to_user el0_ia: /* * Instruction abort handling @@ -468,8 +488,8 @@ el0_ia: mov x0, x26 orr x1, x25, #1 << 24 // use reserved ISS bit for instruction aborts mov x2, sp - adr lr, ret_to_user - b do_mem_abort + bl do_mem_abort + b ret_to_user el0_fpsimd_acc: /* * Floating Point or Advanced SIMD access @@ -478,8 +498,8 @@ el0_fpsimd_acc: ct_user_exit mov x0, x25 mov x1, sp - adr lr, ret_to_user - b do_fpsimd_acc + bl do_fpsimd_acc + b ret_to_user el0_fpsimd_exc: /* * Floating Point or Advanced SIMD exception @@ -488,8 +508,8 @@ el0_fpsimd_exc: ct_user_exit mov x0, x25 mov x1, sp - adr lr, ret_to_user - b do_fpsimd_exc + bl do_fpsimd_exc + b ret_to_user el0_sp_pc: /* * Stack or PC alignment exception handling @@ -500,8 +520,8 @@ el0_sp_pc: mov x0, x26 mov x1, x25 mov x2, sp - adr lr, ret_to_user - b do_sp_pc_abort + bl do_sp_pc_abort + b ret_to_user el0_undef: /* * Undefined instruction @@ -510,8 +530,8 @@ el0_undef: enable_dbg_and_irq ct_user_exit mov x0, sp - adr lr, ret_to_user - b do_undefinstr + bl do_undefinstr + b ret_to_user el0_dbg: /* * Debug exception handling @@ -530,8 +550,8 @@ el0_inv: mov x0, sp mov x1, #BAD_SYNC mrs x2, esr_el1 - adr lr, ret_to_user - b bad_mode + bl bad_mode + b ret_to_user ENDPROC(el0_sync) .align 6 @@ -653,14 +673,15 @@ el0_svc_naked: // compat entry point ldr x16, [tsk, #TI_FLAGS] // check for syscall hooks tst x16, #_TIF_SYSCALL_WORK b.ne __sys_trace - adr lr, ret_fast_syscall // return address cmp scno, sc_nr // check upper syscall limit b.hs ni_sys ldr x16, [stbl, scno, lsl #3] // address in the syscall table - br x16 // call sys_* routine + blr x16 // call sys_* routine + b ret_fast_syscall ni_sys: mov x0, sp - b do_ni_syscall + bl do_ni_syscall + b ret_fast_syscall ENDPROC(el0_svc) /* @@ -668,26 +689,38 @@ ENDPROC(el0_svc) * switches, and waiting for our parent to respond. */ __sys_trace: - mov x0, sp + mov w0, #-1 // set default errno for + cmp scno, x0 // user-issued syscall(-1) + b.ne 1f + mov x0, #-ENOSYS + str x0, [sp, #S_X0] +1: mov x0, sp bl syscall_trace_enter - adr lr, __sys_trace_return // return address + cmp w0, #-1 // skip the syscall? + b.eq __sys_trace_return_skipped uxtw scno, w0 // syscall number (possibly new) mov x1, sp // pointer to regs cmp scno, sc_nr // check upper syscall limit - b.hs ni_sys + b.hs __ni_sys_trace ldp x0, x1, [sp] // restore the syscall args ldp x2, x3, [sp, #S_X2] ldp x4, x5, [sp, #S_X4] ldp x6, x7, [sp, #S_X6] ldr x16, [stbl, scno, lsl #3] // address in the syscall table - br x16 // call sys_* routine + blr x16 // call sys_* routine __sys_trace_return: - str x0, [sp] // save returned x0 + str x0, [sp, #S_X0] // save returned x0 +__sys_trace_return_skipped: mov x0, sp bl syscall_trace_exit b ret_to_user +__ni_sys_trace: + mov x0, sp + bl do_ni_syscall + b __sys_trace_return + /* * Special system call wrappers. */ @@ -695,6 +728,3 @@ ENTRY(sys_rt_sigreturn_wrapper) mov x0, sp b sys_rt_sigreturn ENDPROC(sys_rt_sigreturn_wrapper) - -ENTRY(handle_arch_irq) - .quad 0 diff --git a/arch/arm64/kernel/sys32.S b/arch/arm64/kernel/entry32.S index 423a5b3fc2be..bd9bfaa9269b 100644 --- a/arch/arm64/kernel/sys32.S +++ b/arch/arm64/kernel/entry32.S @@ -19,34 +19,37 @@ */ #include <linux/linkage.h> +#include <linux/const.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> +#include <asm/errno.h> +#include <asm/page.h> /* * System call wrappers for the AArch32 compatibility layer. */ -compat_sys_sigreturn_wrapper: +ENTRY(compat_sys_sigreturn_wrapper) mov x0, sp mov x27, #0 // prevent syscall restart handling (why) b compat_sys_sigreturn ENDPROC(compat_sys_sigreturn_wrapper) -compat_sys_rt_sigreturn_wrapper: +ENTRY(compat_sys_rt_sigreturn_wrapper) mov x0, sp mov x27, #0 // prevent syscall restart handling (why) b compat_sys_rt_sigreturn ENDPROC(compat_sys_rt_sigreturn_wrapper) -compat_sys_statfs64_wrapper: +ENTRY(compat_sys_statfs64_wrapper) mov w3, #84 cmp w1, #88 csel w1, w3, w1, eq b compat_sys_statfs64 ENDPROC(compat_sys_statfs64_wrapper) -compat_sys_fstatfs64_wrapper: +ENTRY(compat_sys_fstatfs64_wrapper) mov w3, #84 cmp w1, #88 csel w1, w3, w1, eq @@ -54,37 +57,52 @@ compat_sys_fstatfs64_wrapper: ENDPROC(compat_sys_fstatfs64_wrapper) /* + * Note: off_4k (w5) is always in units of 4K. If we can't do the + * requested offset because it is not page-aligned, we return -EINVAL. + */ +ENTRY(compat_sys_mmap2_wrapper) +#if PAGE_SHIFT > 12 + tst w5, #~PAGE_MASK >> 12 + b.ne 1f + lsr w5, w5, #PAGE_SHIFT - 12 +#endif + b sys_mmap_pgoff +1: mov x0, #-EINVAL + ret +ENDPROC(compat_sys_mmap2_wrapper) + +/* * Wrappers for AArch32 syscalls that either take 64-bit parameters * in registers or that take 32-bit parameters which require sign * extension. */ -compat_sys_pread64_wrapper: +ENTRY(compat_sys_pread64_wrapper) regs_to_64 x3, x4, x5 b sys_pread64 ENDPROC(compat_sys_pread64_wrapper) -compat_sys_pwrite64_wrapper: +ENTRY(compat_sys_pwrite64_wrapper) regs_to_64 x3, x4, x5 b sys_pwrite64 ENDPROC(compat_sys_pwrite64_wrapper) -compat_sys_truncate64_wrapper: +ENTRY(compat_sys_truncate64_wrapper) regs_to_64 x1, x2, x3 b sys_truncate ENDPROC(compat_sys_truncate64_wrapper) -compat_sys_ftruncate64_wrapper: +ENTRY(compat_sys_ftruncate64_wrapper) regs_to_64 x1, x2, x3 b sys_ftruncate ENDPROC(compat_sys_ftruncate64_wrapper) -compat_sys_readahead_wrapper: +ENTRY(compat_sys_readahead_wrapper) regs_to_64 x1, x2, x3 mov w2, w4 b sys_readahead ENDPROC(compat_sys_readahead_wrapper) -compat_sys_fadvise64_64_wrapper: +ENTRY(compat_sys_fadvise64_64_wrapper) mov w6, w1 regs_to_64 x1, x2, x3 regs_to_64 x2, x4, x5 @@ -92,24 +110,14 @@ compat_sys_fadvise64_64_wrapper: b sys_fadvise64_64 ENDPROC(compat_sys_fadvise64_64_wrapper) -compat_sys_sync_file_range2_wrapper: +ENTRY(compat_sys_sync_file_range2_wrapper) regs_to_64 x2, x2, x3 regs_to_64 x3, x4, x5 b sys_sync_file_range2 ENDPROC(compat_sys_sync_file_range2_wrapper) -compat_sys_fallocate_wrapper: +ENTRY(compat_sys_fallocate_wrapper) regs_to_64 x2, x2, x3 regs_to_64 x3, x4, x5 b sys_fallocate ENDPROC(compat_sys_fallocate_wrapper) - -#undef __SYSCALL -#define __SYSCALL(x, y) .quad y // x - -/* - * The system calls table must be 4KB aligned. - */ - .align 12 -ENTRY(compat_sys_call_table) -#include <asm/unistd32.h> diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index cf8556ae09d0..c851be795080 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -156,7 +156,7 @@ static int ftrace_modify_graph_caller(bool enable) branch = aarch64_insn_gen_branch_imm(pc, (unsigned long)ftrace_graph_caller, - AARCH64_INSN_BRANCH_LINK); + AARCH64_INSN_BRANCH_NOLINK); nop = aarch64_insn_gen_nop(); if (enable) diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 0a6e4f924df8..19f915e8f6e0 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -36,7 +36,7 @@ #include <asm/page.h> #include <asm/virt.h> -#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) +#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET) #if (TEXT_OFFSET & 0xfff) != 0 #error TEXT_OFFSET must be at least 4KB aligned @@ -46,13 +46,6 @@ #error TEXT_OFFSET must be less than 2MB #endif - .macro pgtbl, ttb0, ttb1, virt_to_phys - ldr \ttb1, =swapper_pg_dir - ldr \ttb0, =idmap_pg_dir - add \ttb1, \ttb1, \virt_to_phys - add \ttb0, \ttb0, \virt_to_phys - .endm - #ifdef CONFIG_ARM64_64K_PAGES #define BLOCK_SHIFT PAGE_SHIFT #define BLOCK_SIZE PAGE_SIZE @@ -63,7 +56,7 @@ #define TABLE_SHIFT PUD_SHIFT #endif -#define KERNEL_START KERNEL_RAM_VADDR +#define KERNEL_START _text #define KERNEL_END _end /* @@ -132,6 +125,8 @@ efi_head: #endif #ifdef CONFIG_EFI + .globl stext_offset + .set stext_offset, stext - efi_head .align 3 pe_header: .ascii "PE" @@ -155,12 +150,12 @@ optional_header: .long 0 // SizeOfInitializedData .long 0 // SizeOfUninitializedData .long efi_stub_entry - efi_head // AddressOfEntryPoint - .long stext - efi_head // BaseOfCode + .long stext_offset // BaseOfCode extra_header_fields: .quad 0 // ImageBase - .long 0x20 // SectionAlignment - .long 0x8 // FileAlignment + .long 0x1000 // SectionAlignment + .long PECOFF_FILE_ALIGNMENT // FileAlignment .short 0 // MajorOperatingSystemVersion .short 0 // MinorOperatingSystemVersion .short 0 // MajorImageVersion @@ -172,7 +167,7 @@ extra_header_fields: .long _end - efi_head // SizeOfImage // Everything before the kernel image is considered part of the header - .long stext - efi_head // SizeOfHeaders + .long stext_offset // SizeOfHeaders .long 0 // CheckSum .short 0xa // Subsystem (EFI application) .short 0 // DllCharacteristics @@ -217,280 +212,81 @@ section_table: .byte 0 .byte 0 // end of 0 padding of section name .long _end - stext // VirtualSize - .long stext - efi_head // VirtualAddress + .long stext_offset // VirtualAddress .long _edata - stext // SizeOfRawData - .long stext - efi_head // PointerToRawData + .long stext_offset // PointerToRawData .long 0 // PointerToRelocations (0 for executables) .long 0 // PointerToLineNumbers (0 for executables) .short 0 // NumberOfRelocations (0 for executables) .short 0 // NumberOfLineNumbers (0 for executables) .long 0xe0500020 // Characteristics (section flags) - .align 5 + + /* + * EFI will load stext onwards at the 4k section alignment + * described in the PE/COFF header. To ensure that instruction + * sequences using an adrp and a :lo12: immediate will function + * correctly at this alignment, we must ensure that stext is + * placed at a 4k boundary in the Image to begin with. + */ + .align 12 #endif ENTRY(stext) - mov x21, x0 // x21=FDT + bl preserve_boot_args bl el2_setup // Drop to EL1, w20=cpu_boot_mode - bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET + adrp x24, __PHYS_OFFSET bl set_cpu_boot_mode_flag - mrs x22, midr_el1 // x22=cpuid - mov x0, x22 - bl lookup_processor_type - mov x23, x0 // x23=current cpu_table - cbz x23, __error_p // invalid processor (x23=0)? + bl __vet_fdt bl __create_page_tables // x25=TTBR0, x26=TTBR1 /* - * The following calls CPU specific code in a position independent - * manner. See arch/arm64/mm/proc.S for details. x23 = base of - * cpu_info structure selected by lookup_processor_type above. + * The following calls CPU setup code, see arch/arm64/mm/proc.S for + * details. * On return, the CPU will be ready for the MMU to be turned on and * the TCR will have been set. */ - ldr x27, __switch_data // address to jump to after + ldr x27, =__mmap_switched // address to jump to after // MMU has been enabled - adr lr, __enable_mmu // return (PIC) address - ldr x12, [x23, #CPU_INFO_SETUP] - add x12, x12, x28 // __virt_to_phys - br x12 // initialise processor + adr_l lr, __enable_mmu // return (PIC) address + b __cpu_setup // initialise processor ENDPROC(stext) /* - * If we're fortunate enough to boot at EL2, ensure that the world is - * sane before dropping to EL1. - * - * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if - * booted in EL1 or EL2 respectively. + * Preserve the arguments passed by the bootloader in x0 .. x3 */ -ENTRY(el2_setup) - mrs x0, CurrentEL - cmp x0, #CurrentEL_EL2 - b.ne 1f - mrs x0, sctlr_el2 -CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2 -CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2 - msr sctlr_el2, x0 - b 2f -1: mrs x0, sctlr_el1 -CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1 -CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1 - msr sctlr_el1, x0 - mov w20, #BOOT_CPU_MODE_EL1 // This cpu booted in EL1 - isb - ret - - /* Hyp configuration. */ -2: mov x0, #(1 << 31) // 64-bit EL1 - msr hcr_el2, x0 - - /* Generic timers. */ - mrs x0, cnthctl_el2 - orr x0, x0, #3 // Enable EL1 physical timers - msr cnthctl_el2, x0 - msr cntvoff_el2, xzr // Clear virtual offset - -#ifdef CONFIG_ARM_GIC_V3 - /* GICv3 system register access */ - mrs x0, id_aa64pfr0_el1 - ubfx x0, x0, #24, #4 - cmp x0, #1 - b.ne 3f - - mrs_s x0, ICC_SRE_EL2 - orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1 - orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1 - msr_s ICC_SRE_EL2, x0 - isb // Make sure SRE is now set - msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults - -3: -#endif - - /* Populate ID registers. */ - mrs x0, midr_el1 - mrs x1, mpidr_el1 - msr vpidr_el2, x0 - msr vmpidr_el2, x1 - - /* sctlr_el1 */ - mov x0, #0x0800 // Set/clear RES{1,0} bits -CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems -CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems - msr sctlr_el1, x0 - - /* Coprocessor traps. */ - mov x0, #0x33ff - msr cptr_el2, x0 // Disable copro. traps to EL2 - -#ifdef CONFIG_COMPAT - msr hstr_el2, xzr // Disable CP15 traps to EL2 -#endif +preserve_boot_args: + mov x21, x0 // x21=FDT - /* Stage-2 translation */ - msr vttbr_el2, xzr + adr_l x0, boot_args // record the contents of + stp x21, x1, [x0] // x0 .. x3 at kernel entry + stp x2, x3, [x0, #16] - /* Hypervisor stub */ - adr x0, __hyp_stub_vectors - msr vbar_el2, x0 + dmb sy // needed before dc ivac with + // MMU off - /* spsr */ - mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ - PSR_MODE_EL1h) - msr spsr_el2, x0 - msr elr_el2, lr - mov w20, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2 - eret -ENDPROC(el2_setup) + add x1, x0, #0x20 // 4 x 8 bytes + b __inval_cache_range // tail call +ENDPROC(preserve_boot_args) /* - * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed - * in x20. See arch/arm64/include/asm/virt.h for more info. + * Determine validity of the x21 FDT pointer. + * The dtb must be 8-byte aligned and live in the first 512M of memory. */ -ENTRY(set_cpu_boot_mode_flag) - ldr x1, =__boot_cpu_mode // Compute __boot_cpu_mode - add x1, x1, x28 - cmp w20, #BOOT_CPU_MODE_EL2 +__vet_fdt: + tst x21, #0x7 b.ne 1f - add x1, x1, #4 -1: str w20, [x1] // This CPU has booted in EL1 - dmb sy - dc ivac, x1 // Invalidate potentially stale cache line + cmp x21, x24 + b.lt 1f + mov x0, #(1 << 29) + add x0, x0, x24 + cmp x21, x0 + b.ge 1f ret -ENDPROC(set_cpu_boot_mode_flag) - -/* - * We need to find out the CPU boot mode long after boot, so we need to - * store it in a writable variable. - * - * This is not in .bss, because we set it sufficiently early that the boot-time - * zeroing of .bss would clobber it. - */ - .pushsection .data..cacheline_aligned -ENTRY(__boot_cpu_mode) - .align L1_CACHE_SHIFT - .long BOOT_CPU_MODE_EL2 - .long 0 - .popsection - -#ifdef CONFIG_SMP - .align 3 -1: .quad . - .quad secondary_holding_pen_release - - /* - * This provides a "holding pen" for platforms to hold all secondary - * cores are held until we're ready for them to initialise. - */ -ENTRY(secondary_holding_pen) - bl el2_setup // Drop to EL1, w20=cpu_boot_mode - bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET - bl set_cpu_boot_mode_flag - mrs x0, mpidr_el1 - ldr x1, =MPIDR_HWID_BITMASK - and x0, x0, x1 - adr x1, 1b - ldp x2, x3, [x1] - sub x1, x1, x2 - add x3, x3, x1 -pen: ldr x4, [x3] - cmp x4, x0 - b.eq secondary_startup - wfe - b pen -ENDPROC(secondary_holding_pen) - - /* - * Secondary entry point that jumps straight into the kernel. Only to - * be used where CPUs are brought online dynamically by the kernel. - */ -ENTRY(secondary_entry) - bl el2_setup // Drop to EL1 - bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET - bl set_cpu_boot_mode_flag - b secondary_startup -ENDPROC(secondary_entry) - -ENTRY(secondary_startup) - /* - * Common entry point for secondary CPUs. - */ - mrs x22, midr_el1 // x22=cpuid - mov x0, x22 - bl lookup_processor_type - mov x23, x0 // x23=current cpu_table - cbz x23, __error_p // invalid processor (x23=0)? - - pgtbl x25, x26, x28 // x25=TTBR0, x26=TTBR1 - ldr x12, [x23, #CPU_INFO_SETUP] - add x12, x12, x28 // __virt_to_phys - blr x12 // initialise processor - - ldr x21, =secondary_data - ldr x27, =__secondary_switched // address to jump to after enabling the MMU - b __enable_mmu -ENDPROC(secondary_startup) - -ENTRY(__secondary_switched) - ldr x0, [x21] // get secondary_data.stack - mov sp, x0 - mov x29, #0 - b secondary_start_kernel -ENDPROC(__secondary_switched) -#endif /* CONFIG_SMP */ - -/* - * Setup common bits before finally enabling the MMU. Essentially this is just - * loading the page table pointer and vector base registers. - * - * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on - * the MMU. - */ -__enable_mmu: - ldr x5, =vectors - msr vbar_el1, x5 - msr ttbr0_el1, x25 // load TTBR0 - msr ttbr1_el1, x26 // load TTBR1 - isb - b __turn_mmu_on -ENDPROC(__enable_mmu) - -/* - * Enable the MMU. This completely changes the structure of the visible memory - * space. You will not be able to trace execution through this. - * - * x0 = system control register - * x27 = *virtual* address to jump to upon completion - * - * other registers depend on the function called upon completion - * - * We align the entire function to the smallest power of two larger than it to - * ensure it fits within a single block map entry. Otherwise were PHYS_OFFSET - * close to the end of a 512MB or 1GB block we might require an additional - * table to map the entire function. - */ - .align 4 -__turn_mmu_on: - msr sctlr_el1, x0 - isb - br x27 -ENDPROC(__turn_mmu_on) - -/* - * Calculate the start of physical memory. - */ -__calc_phys_offset: - adr x0, 1f - ldp x1, x2, [x0] - sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET - add x24, x2, x28 // x24 = PHYS_OFFSET +1: + mov x21, #0 ret -ENDPROC(__calc_phys_offset) - - .align 3 -1: .quad . - .quad PAGE_OFFSET - +ENDPROC(__vet_fdt) /* * Macro to create a table entry to the next page. * @@ -556,7 +352,8 @@ ENDPROC(__calc_phys_offset) * - pgd entry for fixed mappings (TTBR1) */ __create_page_tables: - pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses + adrp x25, idmap_pg_dir + adrp x26, swapper_pg_dir mov x27, lr /* @@ -585,12 +382,50 @@ __create_page_tables: * Create the identity mapping. */ mov x0, x25 // idmap_pg_dir - ldr x3, =KERNEL_START - add x3, x3, x28 // __pa(KERNEL_START) + adrp x3, KERNEL_START // __pa(KERNEL_START) + +#ifndef CONFIG_ARM64_VA_BITS_48 +#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3) +#define EXTRA_PTRS (1 << (48 - EXTRA_SHIFT)) + + /* + * If VA_BITS < 48, it may be too small to allow for an ID mapping to be + * created that covers system RAM if that is located sufficiently high + * in the physical address space. So for the ID map, use an extended + * virtual range in that case, by configuring an additional translation + * level. + * First, we have to verify our assumption that the current value of + * VA_BITS was chosen such that all translation levels are fully + * utilised, and that lowering T0SZ will always result in an additional + * translation level to be configured. + */ +#if VA_BITS != EXTRA_SHIFT +#error "Mismatch between VA_BITS and page size/number of translation levels" +#endif + + /* + * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the + * entire kernel image can be ID mapped. As T0SZ == (64 - #bits used), + * this number conveniently equals the number of leading zeroes in + * the physical address of KERNEL_END. + */ + adrp x5, KERNEL_END + clz x5, x5 + cmp x5, TCR_T0SZ(VA_BITS) // default T0SZ small enough? + b.ge 1f // .. then skip additional level + + adr_l x6, idmap_t0sz + str x5, [x6] + dmb sy + dc ivac, x6 // Invalidate potentially stale cache line + + create_table_entry x0, x3, EXTRA_SHIFT, EXTRA_PTRS, x5, x6 +1: +#endif + create_pgd_entry x0, x3, x5, x6 - ldr x6, =KERNEL_END mov x5, x3 // __pa(KERNEL_START) - add x6, x6, x28 // __pa(KERNEL_END) + adr_l x6, KERNEL_END // __pa(KERNEL_END) create_block_map x0, x7, x3, x5, x6 /* @@ -599,7 +434,7 @@ __create_page_tables: mov x0, x26 // swapper_pg_dir mov x5, #PAGE_OFFSET create_pgd_entry x0, x5, x3, x6 - ldr x6, =KERNEL_END + ldr x6, =KERNEL_END // __va(KERNEL_END) mov x3, x24 // phys offset create_block_map x0, x7, x3, x5, x6 @@ -626,6 +461,7 @@ __create_page_tables: */ mov x0, x25 add x1, x26, #SWAPPER_DIR_SIZE + dmb sy bl __inval_cache_range mov lr, x27 @@ -633,104 +469,213 @@ __create_page_tables: ENDPROC(__create_page_tables) .ltorg - .align 3 - .type __switch_data, %object -__switch_data: - .quad __mmap_switched - .quad __bss_start // x6 - .quad __bss_stop // x7 - .quad processor_id // x4 - .quad __fdt_pointer // x5 - .quad memstart_addr // x6 - .quad init_thread_union + THREAD_START_SP // sp - /* - * The following fragment of code is executed with the MMU on in MMU mode, and - * uses absolute addresses; this is not position independent. + * The following fragment of code is executed with the MMU enabled. */ + .set initial_sp, init_thread_union + THREAD_START_SP __mmap_switched: - adr x3, __switch_data + 8 + adr_l x6, __bss_start + adr_l x7, __bss_stop - ldp x6, x7, [x3], #16 1: cmp x6, x7 b.hs 2f str xzr, [x6], #8 // Clear BSS b 1b 2: - ldp x4, x5, [x3], #16 - ldr x6, [x3], #8 - ldr x16, [x3] - mov sp, x16 - str x22, [x4] // Save processor ID - str x21, [x5] // Save FDT pointer - str x24, [x6] // Save PHYS_OFFSET + adr_l sp, initial_sp, x4 + str_l x21, __fdt_pointer, x5 // Save FDT pointer + str_l x24, memstart_addr, x6 // Save PHYS_OFFSET mov x29, #0 b start_kernel ENDPROC(__mmap_switched) /* - * Exception handling. Something went wrong and we can't proceed. We ought to - * tell the user, but since we don't have any guarantee that we're even - * running on the right architecture, we do virtually nothing. + * end early head section, begin head code that is also used for + * hotplug and needs to have the same protections as the text region */ -__error_p: -ENDPROC(__error_p) - -__error: -1: nop - b 1b -ENDPROC(__error) - + .section ".text","ax" /* - * This function gets the processor ID in w0 and searches the cpu_table[] for - * a match. It returns a pointer to the struct cpu_info it found. The - * cpu_table[] must end with an empty (all zeros) structure. + * If we're fortunate enough to boot at EL2, ensure that the world is + * sane before dropping to EL1. * - * This routine can be called via C code and it needs to work with the MMU - * both disabled and enabled (the offset is calculated automatically). + * Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in x20 if + * booted in EL1 or EL2 respectively. */ -ENTRY(lookup_processor_type) - adr x1, __lookup_processor_type_data - ldp x2, x3, [x1] - sub x1, x1, x2 // get offset between VA and PA - add x3, x3, x1 // convert VA to PA -1: - ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask - cbz w5, 2f // end of list? - and w6, w6, w0 - cmp w5, w6 - b.eq 3f - add x3, x3, #CPU_INFO_SZ - b 1b -2: - mov x3, #0 // unknown processor -3: - mov x0, x3 +ENTRY(el2_setup) + mrs x0, CurrentEL + cmp x0, #CurrentEL_EL2 + b.ne 1f + mrs x0, sctlr_el2 +CPU_BE( orr x0, x0, #(1 << 25) ) // Set the EE bit for EL2 +CPU_LE( bic x0, x0, #(1 << 25) ) // Clear the EE bit for EL2 + msr sctlr_el2, x0 + b 2f +1: mrs x0, sctlr_el1 +CPU_BE( orr x0, x0, #(3 << 24) ) // Set the EE and E0E bits for EL1 +CPU_LE( bic x0, x0, #(3 << 24) ) // Clear the EE and E0E bits for EL1 + msr sctlr_el1, x0 + mov w20, #BOOT_CPU_MODE_EL1 // This cpu booted in EL1 + isb ret -ENDPROC(lookup_processor_type) - .align 3 - .type __lookup_processor_type_data, %object -__lookup_processor_type_data: - .quad . - .quad cpu_table - .size __lookup_processor_type_data, . - __lookup_processor_type_data + /* Hyp configuration. */ +2: mov x0, #(1 << 31) // 64-bit EL1 + msr hcr_el2, x0 + + /* Generic timers. */ + mrs x0, cnthctl_el2 + orr x0, x0, #3 // Enable EL1 physical timers + msr cnthctl_el2, x0 + msr cntvoff_el2, xzr // Clear virtual offset + +#ifdef CONFIG_ARM_GIC_V3 + /* GICv3 system register access */ + mrs x0, id_aa64pfr0_el1 + ubfx x0, x0, #24, #4 + cmp x0, #1 + b.ne 3f + + mrs_s x0, ICC_SRE_EL2 + orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1 + orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1 + msr_s ICC_SRE_EL2, x0 + isb // Make sure SRE is now set + msr_s ICH_HCR_EL2, xzr // Reset ICC_HCR_EL2 to defaults + +3: +#endif + + /* Populate ID registers. */ + mrs x0, midr_el1 + mrs x1, mpidr_el1 + msr vpidr_el2, x0 + msr vmpidr_el2, x1 + + /* sctlr_el1 */ + mov x0, #0x0800 // Set/clear RES{1,0} bits +CPU_BE( movk x0, #0x33d0, lsl #16 ) // Set EE and E0E on BE systems +CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems + msr sctlr_el1, x0 + + /* Coprocessor traps. */ + mov x0, #0x33ff + msr cptr_el2, x0 // Disable copro. traps to EL2 + +#ifdef CONFIG_COMPAT + msr hstr_el2, xzr // Disable CP15 traps to EL2 +#endif + + /* Stage-2 translation */ + msr vttbr_el2, xzr + + /* Hypervisor stub */ + adrp x0, __hyp_stub_vectors + add x0, x0, #:lo12:__hyp_stub_vectors + msr vbar_el2, x0 + + /* spsr */ + mov x0, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ + PSR_MODE_EL1h) + msr spsr_el2, x0 + msr elr_el2, lr + mov w20, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2 + eret +ENDPROC(el2_setup) /* - * Determine validity of the x21 FDT pointer. - * The dtb must be 8-byte aligned and live in the first 512M of memory. + * Sets the __boot_cpu_mode flag depending on the CPU boot mode passed + * in x20. See arch/arm64/include/asm/virt.h for more info. */ -__vet_fdt: - tst x21, #0x7 +ENTRY(set_cpu_boot_mode_flag) + adr_l x1, __boot_cpu_mode + cmp w20, #BOOT_CPU_MODE_EL2 b.ne 1f - cmp x21, x24 - b.lt 1f - mov x0, #(1 << 29) - add x0, x0, x24 - cmp x21, x0 - b.ge 1f - ret -1: - mov x21, #0 + add x1, x1, #4 +1: str w20, [x1] // This CPU has booted in EL1 + dmb sy + dc ivac, x1 // Invalidate potentially stale cache line ret -ENDPROC(__vet_fdt) +ENDPROC(set_cpu_boot_mode_flag) + +/* + * We need to find out the CPU boot mode long after boot, so we need to + * store it in a writable variable. + * + * This is not in .bss, because we set it sufficiently early that the boot-time + * zeroing of .bss would clobber it. + */ + .pushsection .data..cacheline_aligned + .align L1_CACHE_SHIFT +ENTRY(__boot_cpu_mode) + .long BOOT_CPU_MODE_EL2 + .long BOOT_CPU_MODE_EL1 + .popsection + +#ifdef CONFIG_SMP + /* + * This provides a "holding pen" for platforms to hold all secondary + * cores are held until we're ready for them to initialise. + */ +ENTRY(secondary_holding_pen) + bl el2_setup // Drop to EL1, w20=cpu_boot_mode + bl set_cpu_boot_mode_flag + mrs x0, mpidr_el1 + ldr x1, =MPIDR_HWID_BITMASK + and x0, x0, x1 + adr_l x3, secondary_holding_pen_release +pen: ldr x4, [x3] + cmp x4, x0 + b.eq secondary_startup + wfe + b pen +ENDPROC(secondary_holding_pen) + + /* + * Secondary entry point that jumps straight into the kernel. Only to + * be used where CPUs are brought online dynamically by the kernel. + */ +ENTRY(secondary_entry) + bl el2_setup // Drop to EL1 + bl set_cpu_boot_mode_flag + b secondary_startup +ENDPROC(secondary_entry) + +ENTRY(secondary_startup) + /* + * Common entry point for secondary CPUs. + */ + adrp x25, idmap_pg_dir + adrp x26, swapper_pg_dir + bl __cpu_setup // initialise processor + + ldr x21, =secondary_data + ldr x27, =__secondary_switched // address to jump to after enabling the MMU + b __enable_mmu +ENDPROC(secondary_startup) + +ENTRY(__secondary_switched) + ldr x0, [x21] // get secondary_data.stack + mov sp, x0 + mov x29, #0 + b secondary_start_kernel +ENDPROC(__secondary_switched) +#endif /* CONFIG_SMP */ + +/* + * Enable the MMU. + * + * x0 = SCTLR_EL1 value for turning on the MMU. + * x27 = *virtual* address to jump to upon completion + * + * other registers depend on the function called upon completion + */ +__enable_mmu: + ldr x5, =vectors + msr vbar_el1, x5 + msr ttbr0_el1, x25 // load TTBR0 + msr ttbr1_el1, x26 // load TTBR1 + isb + msr sctlr_el1, x0 + isb + br x27 +ENDPROC(__enable_mmu) diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c index df1cf15377b4..e7d934d3afe0 100644 --- a/arch/arm64/kernel/hw_breakpoint.c +++ b/arch/arm64/kernel/hw_breakpoint.c @@ -527,7 +527,7 @@ int arch_validate_hwbkpt_settings(struct perf_event *bp) * Disallow per-task kernel breakpoints since these would * complicate the stepping code. */ - if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.bp_target) + if (info->ctrl.privilege == AARCH64_BREAKPOINT_EL1 && bp->hw.target) return -EINVAL; return 0; @@ -894,7 +894,7 @@ static struct notifier_block hw_breakpoint_reset_nb = { .notifier_call = hw_breakpoint_reset_notify, }; -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_PM extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)); #else static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index 8cd27fedc8b6..924902083e47 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -17,14 +17,19 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ #include <linux/bitops.h> +#include <linux/bug.h> #include <linux/compiler.h> #include <linux/kernel.h> +#include <linux/mm.h> #include <linux/smp.h> +#include <linux/spinlock.h> #include <linux/stop_machine.h> +#include <linux/types.h> #include <linux/uaccess.h> #include <asm/cacheflush.h> #include <asm/debug-monitors.h> +#include <asm/fixmap.h> #include <asm/insn.h> #define AARCH64_INSN_SF_BIT BIT(31) @@ -72,6 +77,31 @@ bool __kprobes aarch64_insn_is_nop(u32 insn) } } +static DEFINE_SPINLOCK(patch_lock); + +static void __kprobes *patch_map(void *addr, int fixmap) +{ + unsigned long uintaddr = (uintptr_t) addr; + bool module = !core_kernel_text(uintaddr); + struct page *page; + + if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX)) + page = vmalloc_to_page(addr); + else if (!module && IS_ENABLED(CONFIG_DEBUG_RODATA)) + page = virt_to_page(addr); + else + return addr; + + BUG_ON(!page); + set_fixmap(fixmap, page_to_phys(page)); + + return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK)); +} + +static void __kprobes patch_unmap(int fixmap) +{ + clear_fixmap(fixmap); +} /* * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always * little-endian. @@ -88,10 +118,27 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp) return ret; } +static int __kprobes __aarch64_insn_write(void *addr, u32 insn) +{ + void *waddr = addr; + unsigned long flags = 0; + int ret; + + spin_lock_irqsave(&patch_lock, flags); + waddr = patch_map(addr, FIX_TEXT_POKE0); + + ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE); + + patch_unmap(FIX_TEXT_POKE0); + spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} + int __kprobes aarch64_insn_write(void *addr, u32 insn) { insn = cpu_to_le32(insn); - return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE); + return __aarch64_insn_write(addr, insn); } static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn) @@ -218,23 +265,13 @@ int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) return aarch64_insn_patch_text_sync(addrs, insns, cnt); } -u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, - u32 insn, u64 imm) +static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, + u32 *maskp, int *shiftp) { - u32 immlo, immhi, lomask, himask, mask; + u32 mask; int shift; switch (type) { - case AARCH64_INSN_IMM_ADR: - lomask = 0x3; - himask = 0x7ffff; - immlo = imm & lomask; - imm >>= 2; - immhi = imm & himask; - imm = (immlo << 24) | (immhi); - mask = (lomask << 24) | (himask); - shift = 5; - break; case AARCH64_INSN_IMM_26: mask = BIT(26) - 1; shift = 0; @@ -273,9 +310,68 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, shift = 16; break; default: - pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", - type); - return 0; + return -EINVAL; + } + + *maskp = mask; + *shiftp = shift; + + return 0; +} + +#define ADR_IMM_HILOSPLIT 2 +#define ADR_IMM_SIZE SZ_2M +#define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1) +#define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1) +#define ADR_IMM_LOSHIFT 29 +#define ADR_IMM_HISHIFT 5 + +u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn) +{ + u32 immlo, immhi, mask; + int shift; + + switch (type) { + case AARCH64_INSN_IMM_ADR: + shift = 0; + immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK; + immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK; + insn = (immhi << ADR_IMM_HILOSPLIT) | immlo; + mask = ADR_IMM_SIZE - 1; + break; + default: + if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { + pr_err("aarch64_insn_decode_immediate: unknown immediate encoding %d\n", + type); + return 0; + } + } + + return (insn >> shift) & mask; +} + +u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, + u32 insn, u64 imm) +{ + u32 immlo, immhi, mask; + int shift; + + switch (type) { + case AARCH64_INSN_IMM_ADR: + shift = 0; + immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT; + imm >>= ADR_IMM_HILOSPLIT; + immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT; + imm = immlo | immhi; + mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) | + (ADR_IMM_HIMASK << ADR_IMM_HISHIFT)); + break; + default: + if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { + pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", + type); + return 0; + } } /* Update the immediate field. */ @@ -960,3 +1056,29 @@ u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst, return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift); } + +bool aarch32_insn_is_wide(u32 insn) +{ + return insn >= 0xe800; +} + +/* + * Macros/defines for extracting register numbers from instruction. + */ +u32 aarch32_insn_extract_reg_num(u32 insn, int offset) +{ + return (insn & (0xf << offset)) >> offset; +} + +#define OPC2_MASK 0x7 +#define OPC2_OFFSET 5 +u32 aarch32_insn_mcr_extract_opc2(u32 insn) +{ + return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET; +} + +#define CRM_MASK 0xf +u32 aarch32_insn_mcr_extract_crm(u32 insn) +{ + return insn & CRM_MASK; +} diff --git a/arch/arm64/kernel/io.c b/arch/arm64/kernel/io.c index 7d37ead4d199..354be2a872ae 100644 --- a/arch/arm64/kernel/io.c +++ b/arch/arm64/kernel/io.c @@ -25,12 +25,26 @@ */ void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count) { - unsigned char *t = to; - while (count) { + while (count && (!IS_ALIGNED((unsigned long)from, 8) || + !IS_ALIGNED((unsigned long)to, 8))) { + *(u8 *)to = __raw_readb(from); + from++; + to++; count--; - *t = readb(from); - t++; + } + + while (count >= 8) { + *(u64 *)to = __raw_readq(from); + from += 8; + to += 8; + count -= 8; + } + + while (count) { + *(u8 *)to = __raw_readb(from); from++; + to++; + count--; } } EXPORT_SYMBOL(__memcpy_fromio); @@ -40,12 +54,26 @@ EXPORT_SYMBOL(__memcpy_fromio); */ void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count) { - const unsigned char *f = from; - while (count) { + while (count && (!IS_ALIGNED((unsigned long)to, 8) || + !IS_ALIGNED((unsigned long)from, 8))) { + __raw_writeb(*(volatile u8 *)from, to); + from++; + to++; count--; - writeb(*f, to); - f++; + } + + while (count >= 8) { + __raw_writeq(*(volatile u64 *)from, to); + from += 8; + to += 8; + count -= 8; + } + + while (count) { + __raw_writeb(*(volatile u8 *)from, to); + from++; to++; + count--; } } EXPORT_SYMBOL(__memcpy_toio); @@ -55,10 +83,28 @@ EXPORT_SYMBOL(__memcpy_toio); */ void __memset_io(volatile void __iomem *dst, int c, size_t count) { - while (count) { + u64 qc = (u8)c; + + qc |= qc << 8; + qc |= qc << 16; + qc |= qc << 32; + + while (count && !IS_ALIGNED((unsigned long)dst, 8)) { + __raw_writeb(c, dst); + dst++; count--; - writeb(c, dst); + } + + while (count >= 8) { + __raw_writeq(qc, dst); + dst += 8; + count -= 8; + } + + while (count) { + __raw_writeb(c, dst); dst++; + count--; } } EXPORT_SYMBOL(__memset_io); diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index 071a6ec13bd8..240b75c0e94f 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -40,6 +40,8 @@ int arch_show_interrupts(struct seq_file *p, int prec) return 0; } +void (*handle_arch_irq)(struct pt_regs *) = NULL; + void __init set_handle_irq(void (*handle_irq)(struct pt_regs *)) { if (handle_arch_irq) diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c index 263a166291fb..4f1fec7a46db 100644 --- a/arch/arm64/kernel/jump_label.c +++ b/arch/arm64/kernel/jump_label.c @@ -22,9 +22,8 @@ #ifdef HAVE_JUMP_LABEL -static void __arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type, - bool is_static) +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) { void *addr = (void *)entry->code; u32 insn; @@ -37,22 +36,18 @@ static void __arch_jump_label_transform(struct jump_entry *entry, insn = aarch64_insn_gen_nop(); } - if (is_static) - aarch64_insn_patch_text_nosync(addr, insn); - else - aarch64_insn_patch_text(&addr, &insn, 1); -} - -void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type) -{ - __arch_jump_label_transform(entry, type, false); + aarch64_insn_patch_text(&addr, &insn, 1); } void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type) { - __arch_jump_label_transform(entry, type, true); + /* + * We use the architected A64 NOP in arch_static_branch, so there's no + * need to patch an identical A64 NOP over the top of it here. The core + * will call arch_jump_label_transform from a module notifier if the + * NOP needs to be replaced by a branch. + */ } #endif /* HAVE_JUMP_LABEL */ diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 1eb1cc955139..67bf4107f6ef 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -25,7 +25,9 @@ #include <linux/mm.h> #include <linux/moduleloader.h> #include <linux/vmalloc.h> +#include <asm/alternative.h> #include <asm/insn.h> +#include <asm/sections.h> #define AARCH64_INSN_IMM_MOVNZ AARCH64_INSN_IMM_MAX #define AARCH64_INSN_IMM_MOVK AARCH64_INSN_IMM_16 @@ -33,8 +35,8 @@ void *module_alloc(unsigned long size) { return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL_EXEC, NUMA_NO_NODE, - __builtin_return_address(0)); + GFP_KERNEL, PAGE_KERNEL_EXEC, 0, + NUMA_NO_NODE, __builtin_return_address(0)); } enum aarch64_reloc_op { @@ -394,3 +396,20 @@ overflow: me->name, (int)ELF64_R_TYPE(rel[i].r_info), val); return -ENOEXEC; } + +int module_finalize(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, + struct module *me) +{ + const Elf_Shdr *s, *se; + const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + + for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) { + if (strcmp(".altinstructions", secstrs + s->sh_name) == 0) { + apply_alternatives((void *)s->sh_addr, s->sh_size); + return 0; + } + } + + return 0; +} diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c index ce5836c14ec1..4095379dc069 100644 --- a/arch/arm64/kernel/pci.c +++ b/arch/arm64/kernel/pci.c @@ -10,6 +10,7 @@ * */ +#include <linux/acpi.h> #include <linux/init.h> #include <linux/io.h> #include <linux/kernel.h> @@ -47,24 +48,26 @@ int pcibios_add_device(struct pci_dev *dev) return 0; } - -#ifdef CONFIG_PCI_DOMAINS_GENERIC -static bool dt_domain_found = false; - -void pci_bus_assign_domain_nr(struct pci_bus *bus, struct device *parent) +/* + * raw_pci_read/write - Platform-specific PCI config space access. + */ +int raw_pci_read(unsigned int domain, unsigned int bus, + unsigned int devfn, int reg, int len, u32 *val) { - int domain = of_get_pci_domain_nr(parent->of_node); + return -ENXIO; +} - if (domain >= 0) { - dt_domain_found = true; - } else if (dt_domain_found == true) { - dev_err(parent, "Node %s is missing \"linux,pci-domain\" property in DT\n", - parent->of_node->full_name); - return; - } else { - domain = pci_get_new_domain_nr(); - } +int raw_pci_write(unsigned int domain, unsigned int bus, + unsigned int devfn, int reg, int len, u32 val) +{ + return -ENXIO; +} - bus->domain_nr = domain; +#ifdef CONFIG_ACPI +/* Root bridge scanning */ +struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root) +{ + /* TODO: Should be revisited when implementing PCI on ACPI */ + return NULL; } #endif diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index aa29ecb4f800..cce18c85d2e8 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -25,8 +25,10 @@ #include <linux/irq.h> #include <linux/kernel.h> #include <linux/export.h> +#include <linux/of.h> #include <linux/perf_event.h> #include <linux/platform_device.h> +#include <linux/slab.h> #include <linux/spinlock.h> #include <linux/uaccess.h> @@ -169,8 +171,14 @@ armpmu_event_set_period(struct perf_event *event, ret = 1; } - if (left > (s64)armpmu->max_period) - left = armpmu->max_period; + /* + * Limit the maximum period to prevent the counter value + * from overtaking the one we are about to program. In + * effect we are reducing max_period to account for + * interrupt latency (and we are being very conservative). + */ + if (left > (armpmu->max_period >> 1)) + left = armpmu->max_period >> 1; local64_set(&hwc->prev_count, (u64)-left); @@ -316,22 +324,31 @@ out: } static int -validate_event(struct pmu_hw_events *hw_events, - struct perf_event *event) +validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, + struct perf_event *event) { - struct arm_pmu *armpmu = to_arm_pmu(event->pmu); + struct arm_pmu *armpmu; struct hw_perf_event fake_event = event->hw; struct pmu *leader_pmu = event->group_leader->pmu; if (is_software_event(event)) return 1; + /* + * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The + * core perf code won't check that the pmu->ctx == leader->ctx + * until after pmu->event_init(event). + */ + if (event->pmu != pmu) + return 0; + if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF) return 1; if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) return 1; + armpmu = to_arm_pmu(event->pmu); return armpmu->get_event_idx(hw_events, &fake_event) >= 0; } @@ -349,15 +366,15 @@ validate_group(struct perf_event *event) memset(fake_used_mask, 0, sizeof(fake_used_mask)); fake_pmu.used_mask = fake_used_mask; - if (!validate_event(&fake_pmu, leader)) + if (!validate_event(event->pmu, &fake_pmu, leader)) return -EINVAL; list_for_each_entry(sibling, &leader->sibling_list, group_entry) { - if (!validate_event(&fake_pmu, sibling)) + if (!validate_event(event->pmu, &fake_pmu, sibling)) return -EINVAL; } - if (!validate_event(&fake_pmu, event)) + if (!validate_event(event->pmu, &fake_pmu, event)) return -EINVAL; return 0; @@ -390,7 +407,12 @@ armpmu_release_hardware(struct arm_pmu *armpmu) free_percpu_irq(irq, &cpu_hw_events); } else { for (i = 0; i < irqs; ++i) { - if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs)) + int cpu = i; + + if (armpmu->irq_affinity) + cpu = armpmu->irq_affinity[i]; + + if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs)) continue; irq = platform_get_irq(pmu_device, i); if (irq > 0) @@ -444,19 +466,24 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) on_each_cpu(armpmu_enable_percpu_irq, &irq, 1); } else { for (i = 0; i < irqs; ++i) { + int cpu = i; + err = 0; irq = platform_get_irq(pmu_device, i); if (irq <= 0) continue; + if (armpmu->irq_affinity) + cpu = armpmu->irq_affinity[i]; + /* * If we have a single PMU interrupt that we can't shift, * assume that we're running on a uniprocessor machine and * continue. Otherwise, continue without this interrupt. */ - if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { + if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) { pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", - irq, i); + irq, cpu); continue; } @@ -470,7 +497,7 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) return err; } - cpumask_set_cpu(i, &armpmu->active_irqs); + cpumask_set_cpu(cpu, &armpmu->active_irqs); } } @@ -1283,9 +1310,51 @@ static const struct of_device_id armpmu_of_device_ids[] = { static int armpmu_device_probe(struct platform_device *pdev) { + int i, irq, *irqs; + if (!cpu_pmu) return -ENODEV; + /* Don't bother with PPIs; they're already affine */ + irq = platform_get_irq(pdev, 0); + if (irq >= 0 && irq_is_percpu(irq)) + return 0; + + irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL); + if (!irqs) + return -ENOMEM; + + for (i = 0; i < pdev->num_resources; ++i) { + struct device_node *dn; + int cpu; + + dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity", + i); + if (!dn) { + pr_warn("Failed to parse %s/interrupt-affinity[%d]\n", + of_node_full_name(pdev->dev.of_node), i); + break; + } + + for_each_possible_cpu(cpu) + if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL)) + break; + + of_node_put(dn); + if (cpu >= nr_cpu_ids) { + pr_warn("Failed to find logical CPU for %s\n", + dn->name); + break; + } + + irqs[i] = cpu; + } + + if (i == pdev->num_resources) + cpu_pmu->irq_affinity = irqs; + else + kfree(irqs); + cpu_pmu->plat_device = pdev; return 0; } diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c index 6762ad705587..3f62b35fb6f1 100644 --- a/arch/arm64/kernel/perf_regs.c +++ b/arch/arm64/kernel/perf_regs.c @@ -50,3 +50,11 @@ u64 perf_reg_abi(struct task_struct *task) else return PERF_SAMPLE_REGS_ABI_64; } + +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs, + struct pt_regs *regs_user_copy) +{ + regs_user->regs = task_pt_regs(current); + regs_user->abi = perf_reg_abi(current); +} diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index fde9923af859..c6b1f3b96f45 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -21,6 +21,7 @@ #include <stdarg.h> #include <linux/compat.h> +#include <linux/efi.h> #include <linux/export.h> #include <linux/sched.h> #include <linux/kernel.h> @@ -150,6 +151,13 @@ void machine_restart(char *cmd) local_irq_disable(); smp_send_stop(); + /* + * UpdateCapsule() depends on the system being reset via + * ResetSystem(). + */ + if (efi_enabled(EFI_RUNTIME_SERVICES)) + efi_reboot(reboot_mode, NULL); + /* Now call the architecture specific reboot code. */ if (arm_pm_restart) arm_pm_restart(reboot_mode, cmd); diff --git a/arch/arm64/kernel/psci-call.S b/arch/arm64/kernel/psci-call.S new file mode 100644 index 000000000000..cf83e61cd3b5 --- /dev/null +++ b/arch/arm64/kernel/psci-call.S @@ -0,0 +1,28 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Copyright (C) 2015 ARM Limited + * + * Author: Will Deacon <will.deacon@arm.com> + */ + +#include <linux/linkage.h> + +/* int __invoke_psci_fn_hvc(u64 function_id, u64 arg0, u64 arg1, u64 arg2) */ +ENTRY(__invoke_psci_fn_hvc) + hvc #0 + ret +ENDPROC(__invoke_psci_fn_hvc) + +/* int __invoke_psci_fn_smc(u64 function_id, u64 arg0, u64 arg1, u64 arg2) */ +ENTRY(__invoke_psci_fn_smc) + smc #0 + ret +ENDPROC(__invoke_psci_fn_smc) diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c index 663da771580a..ea18cb53921e 100644 --- a/arch/arm64/kernel/psci.c +++ b/arch/arm64/kernel/psci.c @@ -15,6 +15,7 @@ #define pr_fmt(fmt) "psci: " fmt +#include <linux/acpi.h> #include <linux/init.h> #include <linux/of.h> #include <linux/smp.h> @@ -24,6 +25,7 @@ #include <linux/slab.h> #include <uapi/linux/psci.h> +#include <asm/acpi.h> #include <asm/compiler.h> #include <asm/cpu_ops.h> #include <asm/errno.h> @@ -57,6 +59,9 @@ static struct psci_operations psci_ops; static int (*invoke_psci_fn)(u64, u64, u64, u64); typedef int (*psci_initcall_t)(const struct device_node *); +asmlinkage int __invoke_psci_fn_hvc(u64, u64, u64, u64); +asmlinkage int __invoke_psci_fn_smc(u64, u64, u64, u64); + enum psci_function { PSCI_FN_CPU_SUSPEND, PSCI_FN_CPU_ON, @@ -109,40 +114,6 @@ static void psci_power_state_unpack(u32 power_state, PSCI_0_2_POWER_STATE_AFFL_SHIFT; } -/* - * The following two functions are invoked via the invoke_psci_fn pointer - * and will not be inlined, allowing us to piggyback on the AAPCS. - */ -static noinline int __invoke_psci_fn_hvc(u64 function_id, u64 arg0, u64 arg1, - u64 arg2) -{ - asm volatile( - __asmeq("%0", "x0") - __asmeq("%1", "x1") - __asmeq("%2", "x2") - __asmeq("%3", "x3") - "hvc #0\n" - : "+r" (function_id) - : "r" (arg0), "r" (arg1), "r" (arg2)); - - return function_id; -} - -static noinline int __invoke_psci_fn_smc(u64 function_id, u64 arg0, u64 arg1, - u64 arg2) -{ - asm volatile( - __asmeq("%0", "x0") - __asmeq("%1", "x1") - __asmeq("%2", "x2") - __asmeq("%3", "x3") - "smc #0\n" - : "+r" (function_id) - : "r" (arg0), "r" (arg1), "r" (arg2)); - - return function_id; -} - static int psci_get_version(void) { int err; @@ -304,39 +275,8 @@ static void psci_sys_poweroff(void) invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0); } -/* - * PSCI Function IDs for v0.2+ are well defined so use - * standard values. - */ -static int __init psci_0_2_init(struct device_node *np) +static void __init psci_0_2_set_functions(void) { - int err, ver; - - err = get_set_conduit_method(np); - - if (err) - goto out_put_node; - - ver = psci_get_version(); - - if (ver == PSCI_RET_NOT_SUPPORTED) { - /* PSCI v0.2 mandates implementation of PSCI_ID_VERSION. */ - pr_err("PSCI firmware does not comply with the v0.2 spec.\n"); - err = -EOPNOTSUPP; - goto out_put_node; - } else { - pr_info("PSCIv%d.%d detected in firmware.\n", - PSCI_VERSION_MAJOR(ver), - PSCI_VERSION_MINOR(ver)); - - if (PSCI_VERSION_MAJOR(ver) == 0 && - PSCI_VERSION_MINOR(ver) < 2) { - err = -EINVAL; - pr_err("Conflicting PSCI version detected.\n"); - goto out_put_node; - } - } - pr_info("Using standard PSCI v0.2 function IDs\n"); psci_function_id[PSCI_FN_CPU_SUSPEND] = PSCI_0_2_FN64_CPU_SUSPEND; psci_ops.cpu_suspend = psci_cpu_suspend; @@ -360,6 +300,60 @@ static int __init psci_0_2_init(struct device_node *np) arm_pm_restart = psci_sys_reset; pm_power_off = psci_sys_poweroff; +} + +/* + * Probe function for PSCI firmware versions >= 0.2 + */ +static int __init psci_probe(void) +{ + int ver = psci_get_version(); + + if (ver == PSCI_RET_NOT_SUPPORTED) { + /* + * PSCI versions >=0.2 mandates implementation of + * PSCI_VERSION. + */ + pr_err("PSCI firmware does not comply with the v0.2 spec.\n"); + return -EOPNOTSUPP; + } else { + pr_info("PSCIv%d.%d detected in firmware.\n", + PSCI_VERSION_MAJOR(ver), + PSCI_VERSION_MINOR(ver)); + + if (PSCI_VERSION_MAJOR(ver) == 0 && + PSCI_VERSION_MINOR(ver) < 2) { + pr_err("Conflicting PSCI version detected.\n"); + return -EINVAL; + } + } + + psci_0_2_set_functions(); + + return 0; +} + +/* + * PSCI init function for PSCI versions >=0.2 + * + * Probe based on PSCI PSCI_VERSION function + */ +static int __init psci_0_2_init(struct device_node *np) +{ + int err; + + err = get_set_conduit_method(np); + + if (err) + goto out_put_node; + /* + * Starting with v0.2, the PSCI specification introduced a call + * (PSCI_VERSION) that allows probing the firmware version, so + * that PSCI function IDs and version specific initialization + * can be carried out according to the specific version reported + * by firmware + */ + err = psci_probe(); out_put_node: of_node_put(np); @@ -412,7 +406,7 @@ static const struct of_device_id psci_of_match[] __initconst = { {}, }; -int __init psci_init(void) +int __init psci_dt_init(void) { struct device_node *np; const struct of_device_id *matched_np; @@ -427,6 +421,27 @@ int __init psci_init(void) return init_fn(np); } +/* + * We use PSCI 0.2+ when ACPI is deployed on ARM64 and it's + * explicitly clarified in SBBR + */ +int __init psci_acpi_init(void) +{ + if (!acpi_psci_present()) { + pr_info("is not implemented in ACPI.\n"); + return -EOPNOTSUPP; + } + + pr_info("probing for conduit method from ACPI.\n"); + + if (acpi_psci_use_hvc()) + invoke_psci_fn = __invoke_psci_fn_hvc; + else + invoke_psci_fn = __invoke_psci_fn_smc; + + return psci_probe(); +} + #ifdef CONFIG_SMP static int __init cpu_psci_cpu_init(struct device_node *dn, unsigned int cpu) @@ -511,7 +526,7 @@ static int cpu_psci_cpu_kill(unsigned int cpu) static int psci_suspend_finisher(unsigned long index) { - struct psci_power_state *state = __get_cpu_var(psci_power_state); + struct psci_power_state *state = __this_cpu_read(psci_power_state); return psci_ops.cpu_suspend(state[index - 1], virt_to_phys(cpu_resume)); @@ -520,7 +535,7 @@ static int psci_suspend_finisher(unsigned long index) static int __maybe_unused cpu_psci_cpu_suspend(unsigned long index) { int ret; - struct psci_power_state *state = __get_cpu_var(psci_power_state); + struct psci_power_state *state = __this_cpu_read(psci_power_state); /* * idle state index 0 corresponds to wfi, should never be called * from the cpu_suspend operations diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 8a4ae8e73213..d882b833dbdb 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -27,6 +27,7 @@ #include <linux/smp.h> #include <linux/ptrace.h> #include <linux/user.h> +#include <linux/seccomp.h> #include <linux/security.h> #include <linux/init.h> #include <linux/signal.h> @@ -551,6 +552,32 @@ static int tls_set(struct task_struct *target, const struct user_regset *regset, return ret; } +static int system_call_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + int syscallno = task_pt_regs(target)->syscallno; + + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, + &syscallno, 0, -1); +} + +static int system_call_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int syscallno, ret; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1); + if (ret) + return ret; + + task_pt_regs(target)->syscallno = syscallno; + return ret; +} + enum aarch64_regset { REGSET_GPR, REGSET_FPR, @@ -559,6 +586,7 @@ enum aarch64_regset { REGSET_HW_BREAK, REGSET_HW_WATCH, #endif + REGSET_SYSTEM_CALL, }; static const struct user_regset aarch64_regsets[] = { @@ -608,6 +636,14 @@ static const struct user_regset aarch64_regsets[] = { .set = hw_break_set, }, #endif + [REGSET_SYSTEM_CALL] = { + .core_note_type = NT_ARM_SYSTEM_CALL, + .n = 1, + .size = sizeof(int), + .align = sizeof(int), + .get = system_call_get, + .set = system_call_set, + }, }; static const struct user_regset_view user_aarch64_view = { @@ -1114,6 +1150,10 @@ static void tracehook_report_syscall(struct pt_regs *regs, asmlinkage int syscall_trace_enter(struct pt_regs *regs) { + /* Do the secure computing check first; failures should be fast. */ + if (secure_computing() == -1) + return -1; + if (test_thread_flag(TIF_SYSCALL_TRACE)) tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER); diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 2437196cc5d4..74753132c3ac 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -17,6 +17,7 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <linux/acpi.h> #include <linux/export.h> #include <linux/kernel.h> #include <linux/stddef.h> @@ -40,15 +41,18 @@ #include <linux/fs.h> #include <linux/proc_fs.h> #include <linux/memblock.h> +#include <linux/of_iommu.h> #include <linux/of_fdt.h> #include <linux/of_platform.h> #include <linux/efi.h> +#include <linux/personality.h> +#include <asm/acpi.h> #include <asm/fixmap.h> #include <asm/cpu.h> #include <asm/cputype.h> #include <asm/elf.h> -#include <asm/cputable.h> +#include <asm/cpufeature.h> #include <asm/cpu_ops.h> #include <asm/sections.h> #include <asm/setup.h> @@ -59,9 +63,7 @@ #include <asm/memblock.h> #include <asm/psci.h> #include <asm/efi.h> - -unsigned int processor_id; -EXPORT_SYMBOL(processor_id); +#include <asm/virt.h> unsigned long elf_hwcap __read_mostly; EXPORT_SYMBOL_GPL(elf_hwcap); @@ -72,13 +74,14 @@ EXPORT_SYMBOL_GPL(elf_hwcap); COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\ COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\ COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\ - COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV) + COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\ + COMPAT_HWCAP_LPAE) unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT; unsigned int compat_elf_hwcap2 __read_mostly; #endif -static const char *cpu_name; -static const char *machine_name; +DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); + phys_addr_t __fdt_pointer __initdata; /* @@ -114,14 +117,23 @@ void __init early_print(const char *str, ...) printk("%s", buf); } +/* + * The recorded values of x0 .. x3 upon kernel entry. + */ +u64 __cacheline_aligned boot_args[4]; + void __init smp_setup_processor_id(void) { + u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; + cpu_logical_map(0) = mpidr; + /* * clear __my_cpu_offset on boot CPU to avoid hang caused by * using percpu variable early, for example, lockdep will * access percpu variable inside lock_release */ set_my_cpu_offset(0); + pr_info("Booting Linux on physical CPU 0x%lx\n", (unsigned long)mpidr); } bool arch_match_cpu_phys_id(int cpu, u64 phys_id) @@ -198,24 +210,38 @@ static void __init smp_build_mpidr_hash(void) } #endif +static void __init hyp_mode_check(void) +{ + if (is_hyp_mode_available()) + pr_info("CPU: All CPU(s) started at EL2\n"); + else if (is_hyp_mode_mismatched()) + WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC, + "CPU: CPUs started in inconsistent modes"); + else + pr_info("CPU: All CPU(s) started at EL1\n"); +} + +void __init do_post_cpus_up_work(void) +{ + hyp_mode_check(); + apply_alternatives_all(); +} + +#ifdef CONFIG_UP_LATE_INIT +void __init up_late_init(void) +{ + do_post_cpus_up_work(); +} +#endif /* CONFIG_UP_LATE_INIT */ + static void __init setup_processor(void) { - struct cpu_info *cpu_info; u64 features, block; u32 cwg; int cls; - cpu_info = lookup_processor_type(read_cpuid_id()); - if (!cpu_info) { - printk("CPU configuration botched (ID %08x), unable to continue.\n", - read_cpuid_id()); - while (1); - } - - cpu_name = cpu_info->cpu_name; - - printk("CPU: %s [%08x] revision %d\n", - cpu_name, read_cpuid_id(), read_cpuid_id() & 15); + printk("CPU: AArch64 Processor [%08x] revision %d\n", + read_cpuid_id(), read_cpuid_id() & 15); sprintf(init_utsname()->machine, ELF_PLATFORM); elf_hwcap = 0; @@ -311,27 +337,8 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys) cpu_relax(); } - machine_name = of_flat_dt_get_machine_name(); -} - -/* - * Limit the memory size that was specified via FDT. - */ -static int __init early_mem(char *p) -{ - phys_addr_t limit; - - if (!p) - return 1; - - limit = memparse(p, &p) & PAGE_MASK; - pr_notice("Memory limited to %lldMB\n", limit >> 20); - - memblock_enforce_memory_limit(limit); - - return 0; + dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name()); } -early_param("mem", early_mem); static void __init request_standard_resources(void) { @@ -376,6 +383,7 @@ void __init setup_arch(char **cmdline_p) *cmdline_p = boot_command_line; + early_fixmap_init(); early_ioremap_init(); parse_early_param(); @@ -389,19 +397,27 @@ void __init setup_arch(char **cmdline_p) efi_init(); arm64_memblock_init(); + /* Parse the ACPI tables for possible boot-time configuration */ + acpi_boot_table_init(); + paging_init(); request_standard_resources(); - efi_idmap_init(); + early_ioremap_reset(); - unflatten_device_tree(); - - psci_init(); + if (acpi_disabled) { + unflatten_device_tree(); + psci_dt_init(); + cpu_read_bootcpu_ops(); +#ifdef CONFIG_SMP + of_smp_init_cpus(); +#endif + } else { + psci_acpi_init(); + acpi_init_cpus(); + } - cpu_logical_map(0) = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; - cpu_read_bootcpu_ops(); #ifdef CONFIG_SMP - smp_init_cpus(); smp_build_mpidr_hash(); #endif @@ -412,10 +428,17 @@ void __init setup_arch(char **cmdline_p) conswitchp = &dummy_con; #endif #endif + if (boot_args[1] || boot_args[2] || boot_args[3]) { + pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" + "\tx1: %016llx\n\tx2: %016llx\n\tx3: %016llx\n" + "This indicates a broken bootloader or old kernel\n", + boot_args[1], boot_args[2], boot_args[3]); + } } static int __init arm64_device_init(void) { + of_iommu_init(); of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); return 0; } @@ -447,14 +470,50 @@ static const char *hwcap_str[] = { NULL }; +#ifdef CONFIG_COMPAT +static const char *compat_hwcap_str[] = { + "swp", + "half", + "thumb", + "26bit", + "fastmult", + "fpa", + "vfp", + "edsp", + "java", + "iwmmxt", + "crunch", + "thumbee", + "neon", + "vfpv3", + "vfpv3d16", + "tls", + "vfpv4", + "idiva", + "idivt", + "vfpd32", + "lpae", + "evtstrm" +}; + +static const char *compat_hwcap2_str[] = { + "aes", + "pmull", + "sha1", + "sha2", + "crc32", + NULL +}; +#endif /* CONFIG_COMPAT */ + static int c_show(struct seq_file *m, void *v) { - int i; - - seq_printf(m, "Processor\t: %s rev %d (%s)\n", - cpu_name, read_cpuid_id() & 15, ELF_PLATFORM); + int i, j; for_each_online_cpu(i) { + struct cpuinfo_arm64 *cpuinfo = &per_cpu(cpu_data, i); + u32 midr = cpuinfo->reg_midr; + /* * glibc reads /proc/cpuinfo to determine the number of * online processors, looking for lines beginning with @@ -463,24 +522,38 @@ static int c_show(struct seq_file *m, void *v) #ifdef CONFIG_SMP seq_printf(m, "processor\t: %d\n", i); #endif - } - - /* dump out the processor features */ - seq_puts(m, "Features\t: "); - - for (i = 0; hwcap_str[i]; i++) - if (elf_hwcap & (1 << i)) - seq_printf(m, "%s ", hwcap_str[i]); - seq_printf(m, "\nCPU implementer\t: 0x%02x\n", read_cpuid_id() >> 24); - seq_printf(m, "CPU architecture: AArch64\n"); - seq_printf(m, "CPU variant\t: 0x%x\n", (read_cpuid_id() >> 20) & 15); - seq_printf(m, "CPU part\t: 0x%03x\n", (read_cpuid_id() >> 4) & 0xfff); - seq_printf(m, "CPU revision\t: %d\n", read_cpuid_id() & 15); - - seq_puts(m, "\n"); - - seq_printf(m, "Hardware\t: %s\n", machine_name); + /* + * Dump out the common processor features in a single line. + * Userspace should read the hwcaps with getauxval(AT_HWCAP) + * rather than attempting to parse this, but there's a body of + * software which does already (at least for 32-bit). + */ + seq_puts(m, "Features\t:"); + if (personality(current->personality) == PER_LINUX32) { +#ifdef CONFIG_COMPAT + for (j = 0; compat_hwcap_str[j]; j++) + if (compat_elf_hwcap & (1 << j)) + seq_printf(m, " %s", compat_hwcap_str[j]); + + for (j = 0; compat_hwcap2_str[j]; j++) + if (compat_elf_hwcap2 & (1 << j)) + seq_printf(m, " %s", compat_hwcap2_str[j]); +#endif /* CONFIG_COMPAT */ + } else { + for (j = 0; hwcap_str[j]; j++) + if (elf_hwcap & (1 << j)) + seq_printf(m, " %s", hwcap_str[j]); + } + seq_puts(m, "\n"); + + seq_printf(m, "CPU implementer\t: 0x%02x\n", + MIDR_IMPLEMENTOR(midr)); + seq_printf(m, "CPU architecture: 8\n"); + seq_printf(m, "CPU variant\t: 0x%x\n", MIDR_VARIANT(midr)); + seq_printf(m, "CPU part\t: 0x%03x\n", MIDR_PARTNUM(midr)); + seq_printf(m, "CPU revision\t: %d\n\n", MIDR_REVISION(midr)); + } return 0; } diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 6fa792137eda..e18c48cb6db1 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -131,7 +131,7 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs *regs) struct rt_sigframe __user *frame; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; /* * Since we stacked the signal on a 128-bit boundary, then 'sp' should @@ -287,19 +287,12 @@ static void setup_restart_syscall(struct pt_regs *regs) */ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) { - struct thread_info *thread = current_thread_info(); struct task_struct *tsk = current; sigset_t *oldset = sigmask_to_save(); int usig = ksig->sig; int ret; /* - * translate the signal - */ - if (usig < 32 && thread->exec_domain && thread->exec_domain->signal_invmap) - usig = thread->exec_domain->signal_invmap[usig]; - - /* * Set up the stack frame */ if (is_compat_task()) { diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c index 1b9ad02837cf..d26fcd4cd6e6 100644 --- a/arch/arm64/kernel/signal32.c +++ b/arch/arm64/kernel/signal32.c @@ -154,8 +154,7 @@ int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from) case __SI_TIMER: err |= __put_user(from->si_tid, &to->si_tid); err |= __put_user(from->si_overrun, &to->si_overrun); - err |= __put_user((compat_uptr_t)(unsigned long)from->si_ptr, - &to->si_ptr); + err |= __put_user(from->si_int, &to->si_int); break; case __SI_POLL: err |= __put_user(from->si_band, &to->si_band); @@ -184,7 +183,13 @@ int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from) case __SI_MESGQ: /* But this is */ err |= __put_user(from->si_pid, &to->si_pid); err |= __put_user(from->si_uid, &to->si_uid); - err |= __put_user((compat_uptr_t)(unsigned long)from->si_ptr, &to->si_ptr); + err |= __put_user(from->si_int, &to->si_int); + break; + case __SI_SYS: + err |= __put_user((compat_uptr_t)(unsigned long) + from->si_call_addr, &to->si_call_addr); + err |= __put_user(from->si_syscall, &to->si_syscall); + err |= __put_user(from->si_arch, &to->si_arch); break; default: /* this is just in case for now ... */ err |= __put_user(from->si_pid, &to->si_pid); @@ -341,7 +346,7 @@ asmlinkage int compat_sys_sigreturn(struct pt_regs *regs) struct compat_sigframe __user *frame; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; /* * Since we stacked the signal on a 64-bit boundary, @@ -375,7 +380,7 @@ asmlinkage int compat_sys_rt_sigreturn(struct pt_regs *regs) struct compat_rt_sigframe __user *frame; /* Always make any pending restarted system calls return -EINTR */ - current_thread_info()->restart_block.fn = do_no_restart_syscall; + current->restart_block.fn = do_no_restart_syscall; /* * Since we stacked the signal on a 64-bit boundary, @@ -434,7 +439,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, { compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler); compat_ulong_t retcode; - compat_ulong_t spsr = regs->pstate & ~PSR_f; + compat_ulong_t spsr = regs->pstate & ~(PSR_f | COMPAT_PSR_E_BIT); int thumb; /* Check if the handler is written for ARM or Thumb */ @@ -448,6 +453,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka, /* The IT state must be cleared for both ARM and Thumb-2 */ spsr &= ~COMPAT_PSR_IT_MASK; + /* Restore the original endianness */ + spsr |= COMPAT_PSR_ENDSTATE; + if (ka->sa.sa_flags & SA_RESTORER) { retcode = ptr_to_compat(ka->sa.sa_restorer); } else { @@ -495,7 +503,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf, __put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err); /* set the compat FSR WnR */ - __put_user_error(!!(current->thread.fault_code & ESR_EL1_WRITE) << + __put_user_error(!!(current->thread.fault_code & ESR_ELx_WNR) << FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err); __put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err); __put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err); diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S index a564b440416a..ede186cdd452 100644 --- a/arch/arm64/kernel/sleep.S +++ b/arch/arm64/kernel/sleep.S @@ -147,14 +147,12 @@ cpu_resume_after_mmu: ret ENDPROC(cpu_resume_after_mmu) - .data ENTRY(cpu_resume) bl el2_setup // if in EL2 drop to EL1 cleanly #ifdef CONFIG_SMP mrs x1, mpidr_el1 - adr x4, mpidr_hash_ptr - ldr x5, [x4] - add x8, x4, x5 // x8 = struct mpidr_hash phys address + adrp x8, mpidr_hash + add x8, x8, #:lo12:mpidr_hash // x8 = struct mpidr_hash phys address /* retrieve mpidr_hash members to compute the hash */ ldr x2, [x8, #MPIDR_HASH_MASK] ldp w3, w4, [x8, #MPIDR_HASH_SHIFTS] @@ -164,14 +162,15 @@ ENTRY(cpu_resume) #else mov x7, xzr #endif - adr x0, sleep_save_sp + adrp x0, sleep_save_sp + add x0, x0, #:lo12:sleep_save_sp ldr x0, [x0, #SLEEP_SAVE_SP_PHYS] ldr x0, [x0, x7, lsl #3] /* load sp from context */ ldr x2, [x0, #CPU_CTX_SP] - adr x1, sleep_idmap_phys + adrp x1, sleep_idmap_phys /* load physical address of identity map page table in x1 */ - ldr x1, [x1] + ldr x1, [x1, #:lo12:sleep_idmap_phys] mov sp, x2 /* * cpu_do_resume expects x0 to contain context physical address @@ -180,26 +179,3 @@ ENTRY(cpu_resume) bl cpu_do_resume // PC relative jump, MMU off b cpu_resume_mmu // Resume MMU, never returns ENDPROC(cpu_resume) - - .align 3 -mpidr_hash_ptr: - /* - * offset of mpidr_hash symbol from current location - * used to obtain run-time mpidr_hash address with MMU off - */ - .quad mpidr_hash - . -/* - * physical address of identity mapped page tables - */ - .type sleep_idmap_phys, #object -ENTRY(sleep_idmap_phys) - .quad 0 -/* - * struct sleep_save_sp { - * phys_addr_t *save_ptr_stash; - * phys_addr_t save_ptr_stash_phys; - * }; - */ - .type sleep_save_sp, #object -ENTRY(sleep_save_sp) - .space SLEEP_SAVE_SP_SZ // struct sleep_save_sp diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index b06d1d90ee8c..2cb008177252 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -37,6 +37,7 @@ #include <linux/of.h> #include <linux/irq_work.h> +#include <asm/alternative.h> #include <asm/atomic.h> #include <asm/cacheflush.h> #include <asm/cpu.h> @@ -64,7 +65,6 @@ struct secondary_data secondary_data; enum ipi_msg_type { IPI_RESCHEDULE, IPI_CALL_FUNC, - IPI_CALL_FUNC_SINGLE, IPI_CPU_STOP, IPI_TIMER, IPI_IRQ_WORK, @@ -151,6 +151,7 @@ asmlinkage void secondary_start_kernel(void) */ cpu_set_reserved_ttbr0(); flush_tlb_all(); + cpu_set_default_tcr_t0sz(); preempt_disable(); trace_hardirqs_off(); @@ -309,6 +310,7 @@ void cpu_die(void) void __init smp_cpus_done(unsigned int max_cpus) { pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); + do_post_cpus_up_work(); } void __init smp_prepare_boot_cpu(void) @@ -321,7 +323,7 @@ void __init smp_prepare_boot_cpu(void) * cpu logical map array containing MPIDR values related to logical * cpus. Assumes that cpu_logical_map(0) has already been initialized. */ -void __init smp_init_cpus(void) +void __init of_smp_init_cpus(void) { struct device_node *dn = NULL; unsigned int i, cpu = 1; @@ -481,7 +483,6 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = { #define S(x,s) [x] = s S(IPI_RESCHEDULE, "Rescheduling interrupts"), S(IPI_CALL_FUNC, "Function call interrupts"), - S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), S(IPI_CPU_STOP, "CPU stop interrupts"), S(IPI_TIMER, "Timer broadcast interrupts"), S(IPI_IRQ_WORK, "IRQ work interrupts"), @@ -525,7 +526,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask) void arch_send_call_function_single_ipi(int cpu) { - smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); + smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC); } #ifdef CONFIG_IRQ_WORK @@ -583,12 +584,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs) irq_exit(); break; - case IPI_CALL_FUNC_SINGLE: - irq_enter(); - generic_smp_call_function_single_interrupt(); - irq_exit(); - break; - case IPI_CPU_STOP: irq_enter(); ipi_cpu_stop(cpu); @@ -641,7 +636,7 @@ void smp_send_stop(void) cpumask_t mask; cpumask_copy(&mask, cpu_online_mask); - cpu_clear(smp_processor_id(), mask); + cpumask_clear_cpu(smp_processor_id(), &mask); smp_cross_call(&mask, IPI_CPU_STOP); } diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index 4f93c67e63de..14944e5b28da 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -25,6 +25,7 @@ #include <asm/cacheflush.h> #include <asm/cpu_ops.h> #include <asm/cputype.h> +#include <asm/io.h> #include <asm/smp_plat.h> extern void secondary_holding_pen(void); diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index 13ad4dbb1615..d7daf45ae7a2 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -1,10 +1,10 @@ #include <linux/percpu.h> #include <linux/slab.h> #include <asm/cacheflush.h> -#include <asm/cpu_ops.h> #include <asm/debug-monitors.h> #include <asm/pgtable.h> #include <asm/memory.h> +#include <asm/mmu_context.h> #include <asm/smp_plat.h> #include <asm/suspend.h> #include <asm/tlbflush.h> @@ -50,26 +50,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *)) hw_breakpoint_restore = hw_bp_restore; } -/** - * cpu_suspend() - function to enter a low-power state - * @arg: argument to pass to CPU suspend operations - * - * Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU - * operations back-end error code otherwise. - */ -int cpu_suspend(unsigned long arg) -{ - int cpu = smp_processor_id(); - - /* - * If cpu_ops have not been registered or suspend - * has not been initialized, cpu_suspend call fails early. - */ - if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend) - return -EOPNOTSUPP; - return cpu_ops[cpu]->cpu_suspend(arg); -} - /* * __cpu_suspend * @@ -98,7 +78,18 @@ int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ ret = __cpu_suspend_enter(arg, fn); if (ret == 0) { - cpu_switch_mm(mm->pgd, mm); + /* + * We are resuming from reset with TTBR0_EL1 set to the + * idmap to enable the MMU; restore the active_mm mappings in + * TTBR0_EL1 unless the active_mm == &init_mm, in which case + * the thread entered __cpu_suspend with TTBR0_EL1 set to + * reserved TTBR0 page tables and should be restored as such. + */ + if (mm == &init_mm) + cpu_set_reserved_ttbr0(); + else + cpu_switch_mm(mm->pgd, mm); + flush_tlb_all(); /* @@ -126,8 +117,8 @@ int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) return ret; } -extern struct sleep_save_sp sleep_save_sp; -extern phys_addr_t sleep_idmap_phys; +struct sleep_save_sp sleep_save_sp; +phys_addr_t sleep_idmap_phys; static int __init cpu_suspend_init(void) { diff --git a/arch/arm64/kernel/sys.c b/arch/arm64/kernel/sys.c index 3fa98ff14f0e..75151aaf1a52 100644 --- a/arch/arm64/kernel/sys.c +++ b/arch/arm64/kernel/sys.c @@ -39,10 +39,9 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len, /* * Wrappers to pass the pt_regs argument. */ +asmlinkage long sys_rt_sigreturn_wrapper(void); #define sys_rt_sigreturn sys_rt_sigreturn_wrapper -#include <asm/syscalls.h> - #undef __SYSCALL #define __SYSCALL(nr, sym) [nr] = sym, @@ -50,7 +49,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len, * The sys_call_table array must be 4K aligned to be accessible from * kernel/entry.S. */ -void *sys_call_table[__NR_syscalls] __aligned(4096) = { +void * const sys_call_table[__NR_syscalls] __aligned(4096) = { [0 ... __NR_syscalls - 1] = sys_ni_syscall, #include <asm/unistd.h> }; diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c new file mode 100644 index 000000000000..a40b1343b819 --- /dev/null +++ b/arch/arm64/kernel/sys32.c @@ -0,0 +1,52 @@ +/* + * arch/arm64/kernel/sys32.c + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software(void); you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http(void);//www.gnu.org/licenses/>. + */ + +/* + * Needed to avoid conflicting __NR_* macros between uapi/asm/unistd.h and + * asm/unistd32.h. + */ +#define __COMPAT_SYSCALL_NR + +#include <linux/compiler.h> +#include <linux/syscalls.h> + +asmlinkage long compat_sys_sigreturn_wrapper(void); +asmlinkage long compat_sys_rt_sigreturn_wrapper(void); +asmlinkage long compat_sys_statfs64_wrapper(void); +asmlinkage long compat_sys_fstatfs64_wrapper(void); +asmlinkage long compat_sys_pread64_wrapper(void); +asmlinkage long compat_sys_pwrite64_wrapper(void); +asmlinkage long compat_sys_truncate64_wrapper(void); +asmlinkage long compat_sys_ftruncate64_wrapper(void); +asmlinkage long compat_sys_readahead_wrapper(void); +asmlinkage long compat_sys_fadvise64_64_wrapper(void); +asmlinkage long compat_sys_sync_file_range2_wrapper(void); +asmlinkage long compat_sys_fallocate_wrapper(void); +asmlinkage long compat_sys_mmap2_wrapper(void); + +#undef __SYSCALL +#define __SYSCALL(nr, sym) [nr] = sym, + +/* + * The sys_call_table array must be 4K aligned to be accessible from + * kernel/entry.S. + */ +void * const compat_sys_call_table[__NR_compat_syscalls] __aligned(4096) = { + [0 ... __NR_compat_syscalls - 1] = sys_ni_syscall, +#include <asm/unistd32.h> +}; diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index dc47e53e9e28..28c511b06edf 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -28,29 +28,39 @@ #include <asm/cacheflush.h> #include <asm/unistd.h> -static inline void -do_compat_cache_op(unsigned long start, unsigned long end, int flags) +static long +__do_compat_cache_op(unsigned long start, unsigned long end) { - struct mm_struct *mm = current->active_mm; - struct vm_area_struct *vma; + long ret; - if (end < start || flags) - return; + do { + unsigned long chunk = min(PAGE_SIZE, end - start); - down_read(&mm->mmap_sem); - vma = find_vma(mm, start); - if (vma && vma->vm_start < end) { - if (start < vma->vm_start) - start = vma->vm_start; - if (end > vma->vm_end) - end = vma->vm_end; - up_read(&mm->mmap_sem); - __flush_cache_user_range(start & PAGE_MASK, PAGE_ALIGN(end)); - return; - } - up_read(&mm->mmap_sem); + if (fatal_signal_pending(current)) + return 0; + + ret = __flush_cache_user_range(start, start + chunk); + if (ret) + return ret; + + cond_resched(); + start += chunk; + } while (start < end); + + return 0; } +static inline long +do_compat_cache_op(unsigned long start, unsigned long end, int flags) +{ + if (end < start || flags) + return -EINVAL; + + if (!access_ok(VERIFY_READ, start, end - start)) + return -EFAULT; + + return __do_compat_cache_op(start, end); +} /* * Handle all unrecognised system calls. */ @@ -74,8 +84,7 @@ long compat_arm_syscall(struct pt_regs *regs) * the specified region). */ case __ARM_NR_compat_cacheflush: - do_compat_cache_op(regs->regs[0], regs->regs[1], regs->regs[2]); - return 0; + return do_compat_cache_op(regs->regs[0], regs->regs[1], regs->regs[2]); case __ARM_NR_compat_set_tls: current->thread.tp_value = regs->regs[0]; diff --git a/arch/arm64/kernel/time.c b/arch/arm64/kernel/time.c index 1a7125c3099b..42f9195cf2f8 100644 --- a/arch/arm64/kernel/time.c +++ b/arch/arm64/kernel/time.c @@ -35,6 +35,7 @@ #include <linux/delay.h> #include <linux/clocksource.h> #include <linux/clk-provider.h> +#include <linux/acpi.h> #include <clocksource/arm_arch_timer.h> @@ -72,6 +73,12 @@ void __init time_init(void) tick_setup_hrtimer_broadcast(); + /* + * Since ACPI or FDT will only one be available in the system, + * we can use acpi_generic_timer_init() here safely + */ + acpi_generic_timer_init(); + arch_timer_rate = arch_timer_get_rate(); if (!arch_timer_rate) panic("Unable to initialise architected timer.\n"); diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index b6ee26b0939a..fcb8f7b42271 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -255,12 +255,15 @@ void store_cpu_topology(unsigned int cpuid) /* Multiprocessor system : Multi-threads per core */ cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); - cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2); + cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 2) | + MPIDR_AFFINITY_LEVEL(mpidr, 3) << 8; } else { /* Multiprocessor system : Single-thread per core */ cpuid_topo->thread_id = -1; cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0); - cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1); + cpuid_topo->cluster_id = MPIDR_AFFINITY_LEVEL(mpidr, 1) | + MPIDR_AFFINITY_LEVEL(mpidr, 2) << 8 | + MPIDR_AFFINITY_LEVEL(mpidr, 3) << 16; } pr_debug("CPU%u: cluster %d core %d thread %d mpidr %#016llx\n", diff --git a/arch/arm64/kernel/trace-events-emulation.h b/arch/arm64/kernel/trace-events-emulation.h new file mode 100644 index 000000000000..ae1dd598ea65 --- /dev/null +++ b/arch/arm64/kernel/trace-events-emulation.h @@ -0,0 +1,35 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM emulation + +#if !defined(_TRACE_EMULATION_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_EMULATION_H + +#include <linux/tracepoint.h> + +TRACE_EVENT(instruction_emulation, + + TP_PROTO(const char *instr, u64 addr), + TP_ARGS(instr, addr), + + TP_STRUCT__entry( + __string(instr, instr) + __field(u64, addr) + ), + + TP_fast_assign( + __assign_str(instr, instr); + __entry->addr = addr; + ), + + TP_printk("instr=\"%s\" addr=0x%llx", __get_str(instr), __entry->addr) +); + +#endif /* _TRACE_EMULATION_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH . + +#define TRACE_INCLUDE_FILE trace-events-emulation +#include <trace/define_trace.h> diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index de1b085e7963..1ef2940df13c 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -33,6 +33,7 @@ #include <asm/atomic.h> #include <asm/debug-monitors.h> +#include <asm/esr.h> #include <asm/traps.h> #include <asm/stacktrace.h> #include <asm/exception.h> @@ -259,6 +260,69 @@ void arm64_notify_die(const char *str, struct pt_regs *regs, } } +static LIST_HEAD(undef_hook); +static DEFINE_RAW_SPINLOCK(undef_lock); + +void register_undef_hook(struct undef_hook *hook) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&undef_lock, flags); + list_add(&hook->node, &undef_hook); + raw_spin_unlock_irqrestore(&undef_lock, flags); +} + +void unregister_undef_hook(struct undef_hook *hook) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&undef_lock, flags); + list_del(&hook->node); + raw_spin_unlock_irqrestore(&undef_lock, flags); +} + +static int call_undef_hook(struct pt_regs *regs) +{ + struct undef_hook *hook; + unsigned long flags; + u32 instr; + int (*fn)(struct pt_regs *regs, u32 instr) = NULL; + void __user *pc = (void __user *)instruction_pointer(regs); + + if (!user_mode(regs)) + return 1; + + if (compat_thumb_mode(regs)) { + /* 16-bit Thumb instruction */ + if (get_user(instr, (u16 __user *)pc)) + goto exit; + instr = le16_to_cpu(instr); + if (aarch32_insn_is_wide(instr)) { + u32 instr2; + + if (get_user(instr2, (u16 __user *)(pc + 2))) + goto exit; + instr2 = le16_to_cpu(instr2); + instr = (instr << 16) | instr2; + } + } else { + /* 32-bit ARM instruction */ + if (get_user(instr, (u32 __user *)pc)) + goto exit; + instr = le32_to_cpu(instr); + } + + raw_spin_lock_irqsave(&undef_lock, flags); + list_for_each_entry(hook, &undef_hook, node) + if ((instr & hook->instr_mask) == hook->instr_val && + (regs->pstate & hook->pstate_mask) == hook->pstate_val) + fn = hook->fn; + + raw_spin_unlock_irqrestore(&undef_lock, flags); +exit: + return fn ? fn(regs, instr) : 1; +} + asmlinkage void __exception do_undefinstr(struct pt_regs *regs) { siginfo_t info; @@ -268,6 +332,9 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) if (!aarch32_break_handler(regs)) return; + if (call_undef_hook(regs) == 0) + return; + if (show_unhandled_signals && unhandled_signal(current, SIGILL) && printk_ratelimit()) { pr_info("%s[%d]: undefined instruction: pc=%p\n", @@ -307,6 +374,51 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs) return sys_ni_syscall(); } +static const char *esr_class_str[] = { + [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC", + [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized", + [ESR_ELx_EC_WFx] = "WFI/WFE", + [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC", + [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC", + [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC", + [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC", + [ESR_ELx_EC_FP_ASIMD] = "ASIMD", + [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS", + [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC", + [ESR_ELx_EC_ILL] = "PSTATE.IL", + [ESR_ELx_EC_SVC32] = "SVC (AArch32)", + [ESR_ELx_EC_HVC32] = "HVC (AArch32)", + [ESR_ELx_EC_SMC32] = "SMC (AArch32)", + [ESR_ELx_EC_SVC64] = "SVC (AArch64)", + [ESR_ELx_EC_HVC64] = "HVC (AArch64)", + [ESR_ELx_EC_SMC64] = "SMC (AArch64)", + [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)", + [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF", + [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)", + [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)", + [ESR_ELx_EC_PC_ALIGN] = "PC Alignment", + [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)", + [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)", + [ESR_ELx_EC_SP_ALIGN] = "SP Alignment", + [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)", + [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)", + [ESR_ELx_EC_SERROR] = "SError", + [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)", + [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)", + [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)", + [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)", + [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)", + [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)", + [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)", + [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)", + [ESR_ELx_EC_BRK64] = "BRK (AArch64)", +}; + +const char *esr_get_class_string(u32 esr) +{ + return esr_class_str[esr >> ESR_ELx_EC_SHIFT]; +} + /* * bad_mode handles the impossible case in the exception vector. */ @@ -316,8 +428,8 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr) void __user *pc = (void __user *)instruction_pointer(regs); console_verbose(); - pr_crit("Bad mode in %s handler detected, code 0x%08x\n", - handler[reason], esr); + pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n", + handler[reason], esr, esr_get_class_string(esr)); __show_regs(regs); info.si_signo = SIGILL; diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c index 32aeea083d93..ec37ab3f524f 100644 --- a/arch/arm64/kernel/vdso.c +++ b/arch/arm64/kernel/vdso.c @@ -200,7 +200,7 @@ up_fail: void update_vsyscall(struct timekeeper *tk) { struct timespec xtime_coarse; - u32 use_syscall = strcmp(tk->tkr.clock->name, "arch_sys_counter"); + u32 use_syscall = strcmp(tk->tkr_mono.clock->name, "arch_sys_counter"); ++vdso_data->tb_seq_count; smp_wmb(); @@ -213,11 +213,11 @@ void update_vsyscall(struct timekeeper *tk) vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; if (!use_syscall) { - vdso_data->cs_cycle_last = tk->tkr.cycle_last; + vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; vdso_data->xtime_clock_sec = tk->xtime_sec; - vdso_data->xtime_clock_nsec = tk->tkr.xtime_nsec; - vdso_data->cs_mult = tk->tkr.mult; - vdso_data->cs_shift = tk->tkr.shift; + vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec; + vdso_data->cs_mult = tk->tkr_mono.mult; + vdso_data->cs_shift = tk->tkr_mono.shift; } smp_wmb(); diff --git a/arch/arm64/kernel/vdso/gettimeofday.S b/arch/arm64/kernel/vdso/gettimeofday.S index fe652ffd34c2..efa79e8d4196 100644 --- a/arch/arm64/kernel/vdso/gettimeofday.S +++ b/arch/arm64/kernel/vdso/gettimeofday.S @@ -174,8 +174,6 @@ ENDPROC(__kernel_clock_gettime) /* int __kernel_clock_getres(clockid_t clock_id, struct timespec *res); */ ENTRY(__kernel_clock_getres) .cfi_startproc - cbz w1, 3f - cmp w0, #CLOCK_REALTIME ccmp w0, #CLOCK_MONOTONIC, #0x4, ne b.ne 1f @@ -188,6 +186,7 @@ ENTRY(__kernel_clock_getres) b.ne 4f ldr x2, 6f 2: + cbz w1, 3f stp xzr, x2, [x1] 3: /* res == NULL. */ diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index edf8715ba39b..a2c29865c3fe 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -8,11 +8,13 @@ #include <asm/thread_info.h> #include <asm/memory.h> #include <asm/page.h> +#include <asm/pgtable.h> #include "image.h" -#define ARM_EXIT_KEEP(x) -#define ARM_EXIT_DISCARD(x) x +/* .exit.text needed in case of alternative patching */ +#define ARM_EXIT_KEEP(x) x +#define ARM_EXIT_DISCARD(x) OUTPUT_ARCH(aarch64) ENTRY(_text) @@ -21,10 +23,14 @@ jiffies = jiffies_64; #define HYPERVISOR_TEXT \ /* \ - * Force the alignment to be compatible with \ - * the vectors requirements \ + * Align to 4 KB so that \ + * a) the HYP vector table is at its minimum \ + * alignment of 2048 bytes \ + * b) the HYP init code will not cross a page \ + * boundary if its size does not exceed \ + * 4 KB (see related ASSERT() below) \ */ \ - . = ALIGN(2048); \ + . = ALIGN(SZ_4K); \ VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \ *(.hyp.idmap.text) \ VMLINUX_SYMBOL(__hyp_idmap_text_end) = .; \ @@ -32,6 +38,30 @@ jiffies = jiffies_64; *(.hyp.text) \ VMLINUX_SYMBOL(__hyp_text_end) = .; +/* + * The size of the PE/COFF section that covers the kernel image, which + * runs from stext to _edata, must be a round multiple of the PE/COFF + * FileAlignment, which we set to its minimum value of 0x200. 'stext' + * itself is 4 KB aligned, so padding out _edata to a 0x200 aligned + * boundary should be sufficient. + */ +PECOFF_FILE_ALIGNMENT = 0x200; + +#ifdef CONFIG_EFI +#define PECOFF_EDATA_PADDING \ + .pecoff_edata_padding : { BYTE(0); . = ALIGN(PECOFF_FILE_ALIGNMENT); } +#else +#define PECOFF_EDATA_PADDING +#endif + +#ifdef CONFIG_DEBUG_ALIGN_RODATA +#define ALIGN_DEBUG_RO . = ALIGN(1<<SECTION_SHIFT); +#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO +#else +#define ALIGN_DEBUG_RO +#define ALIGN_DEBUG_RO_MIN(min) . = ALIGN(min); +#endif + SECTIONS { /* @@ -54,6 +84,7 @@ SECTIONS _text = .; HEAD_TEXT } + ALIGN_DEBUG_RO .text : { /* Real text segment */ _stext = .; /* Text and read-only data */ __exception_text_start = .; @@ -70,19 +101,22 @@ SECTIONS *(.got) /* Global offset table */ } + ALIGN_DEBUG_RO RO_DATA(PAGE_SIZE) EXCEPTION_TABLE(8) NOTES + ALIGN_DEBUG_RO _etext = .; /* End of text and rodata section */ - . = ALIGN(PAGE_SIZE); + ALIGN_DEBUG_RO_MIN(PAGE_SIZE) __init_begin = .; INIT_TEXT_SECTION(8) .exit.text : { ARM_EXIT_KEEP(EXIT_TEXT) } - . = ALIGN(16); + + ALIGN_DEBUG_RO_MIN(16) .init.data : { INIT_DATA INIT_SETUP(16) @@ -100,9 +134,21 @@ SECTIONS . = ALIGN(PAGE_SIZE); __init_end = .; + . = ALIGN(4); + .altinstructions : { + __alt_instructions = .; + *(.altinstructions) + __alt_instructions_end = .; + } + .altinstr_replacement : { + *(.altinstr_replacement) + } + + . = ALIGN(PAGE_SIZE); _data = .; _sdata = .; RW_DATA_SECTION(64, PAGE_SIZE, THREAD_SIZE) + PECOFF_EDATA_PADDING _edata = .; BSS_SECTION(0, 0, 0) @@ -121,10 +167,11 @@ SECTIONS } /* - * The HYP init code can't be more than a page long. + * The HYP init code can't be more than a page long, + * and should not cross a page boundary. */ -ASSERT(((__hyp_idmap_text_start + PAGE_SIZE) > __hyp_idmap_text_end), - "HYP init code too big") +ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K, + "HYP init code too big or misaligned") /* * If padding is applied before .head.text, virt<->phys conversions will fail. diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 8ba85e9ea388..5105e297ed5f 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -18,14 +18,18 @@ if VIRTUALIZATION config KVM bool "Kernel-based Virtual Machine (KVM) support" + depends on OF select MMU_NOTIFIER select PREEMPT_NOTIFIERS select ANON_INODES select HAVE_KVM_CPU_RELAX_INTERCEPT + select HAVE_KVM_ARCH_TLB_FLUSH_ALL select KVM_MMIO select KVM_ARM_HOST - select KVM_ARM_VGIC - select KVM_ARM_TIMER + select KVM_GENERIC_DIRTYLOG_READ_PROTECT + select SRCU + select HAVE_KVM_EVENTFD + select HAVE_KVM_IRQFD ---help--- Support hosting virtualized guest machines. @@ -47,17 +51,4 @@ config KVM_ARM_MAX_VCPUS large, so only choose a reasonable number that you expect to actually use. -config KVM_ARM_VGIC - bool - depends on KVM_ARM_HOST && OF - select HAVE_KVM_IRQCHIP - ---help--- - Adds support for a hardware assisted, in-kernel GIC emulation. - -config KVM_ARM_TIMER - bool - depends on KVM_ARM_VGIC - ---help--- - Adds support for the Architected Timers in virtual machines. - endif # VIRTUALIZATION diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 32a096174b94..d5904f876cdb 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -2,7 +2,7 @@ # Makefile for Kernel-based Virtual Machine module # -ccflags-y += -Ivirt/kvm -Iarch/arm64/kvm +ccflags-y += -Iarch/arm64/kvm CFLAGS_arm.o := -I. CFLAGS_mmu.o := -I. @@ -11,7 +11,7 @@ ARM=../../../arch/arm/kvm obj-$(CONFIG_KVM_ARM_HOST) += kvm.o -kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/arm.o $(ARM)/mmu.o $(ARM)/mmio.o kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o @@ -19,9 +19,11 @@ kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o -kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o -kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o -kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o -kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o -kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o -kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v2-emul.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v2-switch.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/vgic-v3-emul.o +kvm-$(CONFIG_KVM_ARM_HOST) += vgic-v3-switch.o +kvm-$(CONFIG_KVM_ARM_HOST) += $(KVM)/arm/arch_timer.o diff --git a/arch/arm64/kvm/emulate.c b/arch/arm64/kvm/emulate.c index 124418d17049..f87d8fbaa48d 100644 --- a/arch/arm64/kvm/emulate.c +++ b/arch/arm64/kvm/emulate.c @@ -22,6 +22,7 @@ */ #include <linux/kvm_host.h> +#include <asm/esr.h> #include <asm/kvm_emulate.h> /* @@ -55,8 +56,8 @@ static int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu) { u32 esr = kvm_vcpu_get_hsr(vcpu); - if (esr & ESR_EL2_CV) - return (esr & ESR_EL2_COND) >> ESR_EL2_COND_SHIFT; + if (esr & ESR_ELx_CV) + return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT; return -1; } diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 76794692c20b..9535bd555d1d 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -38,7 +38,6 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) { - vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS; return 0; } @@ -297,31 +296,6 @@ int __attribute_const__ kvm_target_cpu(void) return -EINVAL; } -int kvm_vcpu_set_target(struct kvm_vcpu *vcpu, - const struct kvm_vcpu_init *init) -{ - unsigned int i; - int phys_target = kvm_target_cpu(); - - if (init->target != phys_target) - return -EINVAL; - - vcpu->arch.target = phys_target; - bitmap_zero(vcpu->arch.features, KVM_VCPU_MAX_FEATURES); - - /* -ENOENT for unknown features, -EINVAL for invalid combinations. */ - for (i = 0; i < sizeof(init->features) * 8; i++) { - if (init->features[i / 32] & (1 << (i % 32))) { - if (i >= KVM_VCPU_MAX_FEATURES) - return -ENOENT; - set_bit(i, vcpu->arch.features); - } - } - - /* Now we know what it is, we can reset it. */ - return kvm_reset_vcpu(vcpu); -} - int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init) { int target = kvm_target_cpu(); diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 34b8bd0711e9..524fa25671fc 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -21,17 +21,25 @@ #include <linux/kvm.h> #include <linux/kvm_host.h> -#include <asm/kvm_emulate.h> + +#include <asm/esr.h> #include <asm/kvm_coproc.h> +#include <asm/kvm_emulate.h> #include <asm/kvm_mmu.h> #include <asm/kvm_psci.h> +#define CREATE_TRACE_POINTS +#include "trace.h" + typedef int (*exit_handle_fn)(struct kvm_vcpu *, struct kvm_run *); static int handle_hvc(struct kvm_vcpu *vcpu, struct kvm_run *run) { int ret; + trace_kvm_hvc_arm64(*vcpu_pc(vcpu), *vcpu_reg(vcpu, 0), + kvm_vcpu_hvc_get_imm(vcpu)); + ret = kvm_psci_call(vcpu); if (ret < 0) { kvm_inject_undefined(vcpu); @@ -61,10 +69,13 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run) */ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) { - if (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EC_WFI_ISS_WFE) + if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE) { + trace_kvm_wfx_arm64(*vcpu_pc(vcpu), true); kvm_vcpu_on_spin(vcpu); - else + } else { + trace_kvm_wfx_arm64(*vcpu_pc(vcpu), false); kvm_vcpu_block(vcpu); + } kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu)); @@ -72,29 +83,30 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run) } static exit_handle_fn arm_exit_handlers[] = { - [ESR_EL2_EC_WFI] = kvm_handle_wfx, - [ESR_EL2_EC_CP15_32] = kvm_handle_cp15_32, - [ESR_EL2_EC_CP15_64] = kvm_handle_cp15_64, - [ESR_EL2_EC_CP14_MR] = kvm_handle_cp14_32, - [ESR_EL2_EC_CP14_LS] = kvm_handle_cp14_load_store, - [ESR_EL2_EC_CP14_64] = kvm_handle_cp14_64, - [ESR_EL2_EC_HVC32] = handle_hvc, - [ESR_EL2_EC_SMC32] = handle_smc, - [ESR_EL2_EC_HVC64] = handle_hvc, - [ESR_EL2_EC_SMC64] = handle_smc, - [ESR_EL2_EC_SYS64] = kvm_handle_sys_reg, - [ESR_EL2_EC_IABT] = kvm_handle_guest_abort, - [ESR_EL2_EC_DABT] = kvm_handle_guest_abort, + [ESR_ELx_EC_WFx] = kvm_handle_wfx, + [ESR_ELx_EC_CP15_32] = kvm_handle_cp15_32, + [ESR_ELx_EC_CP15_64] = kvm_handle_cp15_64, + [ESR_ELx_EC_CP14_MR] = kvm_handle_cp14_32, + [ESR_ELx_EC_CP14_LS] = kvm_handle_cp14_load_store, + [ESR_ELx_EC_CP14_64] = kvm_handle_cp14_64, + [ESR_ELx_EC_HVC32] = handle_hvc, + [ESR_ELx_EC_SMC32] = handle_smc, + [ESR_ELx_EC_HVC64] = handle_hvc, + [ESR_ELx_EC_SMC64] = handle_smc, + [ESR_ELx_EC_SYS64] = kvm_handle_sys_reg, + [ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort, + [ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort, }; static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu) { - u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu); + u32 hsr = kvm_vcpu_get_hsr(vcpu); + u8 hsr_ec = hsr >> ESR_ELx_EC_SHIFT; if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) || !arm_exit_handlers[hsr_ec]) { - kvm_err("Unknown exception class: hsr: %#08x\n", - (unsigned int)kvm_vcpu_get_hsr(vcpu)); + kvm_err("Unknown exception class: hsr: %#08x -- %s\n", + hsr, esr_get_class_string(hsr)); BUG(); } diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S index c3191168a994..178ba2248a98 100644 --- a/arch/arm64/kvm/hyp-init.S +++ b/arch/arm64/kvm/hyp-init.S @@ -20,6 +20,7 @@ #include <asm/assembler.h> #include <asm/kvm_arm.h> #include <asm/kvm_mmu.h> +#include <asm/pgtable-hwdef.h> .text .pushsection .hyp.idmap.text, "ax" @@ -65,6 +66,25 @@ __do_hyp_init: and x4, x4, x5 ldr x5, =TCR_EL2_FLAGS orr x4, x4, x5 + +#ifndef CONFIG_ARM64_VA_BITS_48 + /* + * If we are running with VA_BITS < 48, we may be running with an extra + * level of translation in the ID map. This is only the case if system + * RAM is out of range for the currently configured page size and number + * of translation levels, in which case we will also need the extra + * level for the HYP ID map, or we won't be able to enable the EL2 MMU. + * + * However, at EL2, there is only one TTBR register, and we can't switch + * between translation tables *and* update TCR_EL2.T0SZ at the same + * time. Bottom line: we need the extra level in *both* our translation + * tables. + * + * So use the same T0SZ value we use for the ID map. + */ + ldr_l x5, idmap_t0sz + bfi x4, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH +#endif msr tcr_el2, x4 ldr x4, =VTCR_EL2_FLAGS @@ -91,6 +111,10 @@ __do_hyp_init: msr sctlr_el2, x4 isb + /* Skip the trampoline dance if we merged the boot and runtime PGDs */ + cmp x0, x1 + b.eq merged + /* MMU is now enabled. Get ready for the trampoline dance */ ldr x4, =TRAMPOLINE_VA adr x5, target @@ -105,6 +129,7 @@ target: /* We're now in the trampoline code, switch page tables */ tlbi alle2 dsb sy +merged: /* Set the stack and new vectors */ kern_hyp_va x2 mov sp, x2 diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S index b72aa9f9215c..5befd010e232 100644 --- a/arch/arm64/kvm/hyp.S +++ b/arch/arm64/kvm/hyp.S @@ -17,15 +17,16 @@ #include <linux/linkage.h> -#include <asm/assembler.h> -#include <asm/memory.h> #include <asm/asm-offsets.h> +#include <asm/assembler.h> #include <asm/debug-monitors.h> +#include <asm/esr.h> #include <asm/fpsimdmacros.h> #include <asm/kvm.h> -#include <asm/kvm_asm.h> #include <asm/kvm_arm.h> +#include <asm/kvm_asm.h> #include <asm/kvm_mmu.h> +#include <asm/memory.h> #define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x) #define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x) @@ -761,10 +762,10 @@ .macro activate_traps ldr x2, [x0, #VCPU_HCR_EL2] msr hcr_el2, x2 - ldr x2, =(CPTR_EL2_TTA) + mov x2, #CPTR_EL2_TTA msr cptr_el2, x2 - ldr x2, =(1 << 15) // Trap CP15 Cr=15 + mov x2, #(1 << 15) // Trap CP15 Cr=15 msr hstr_el2, x2 mrs x2, mdcr_el2 @@ -1014,6 +1015,7 @@ ENTRY(__kvm_tlb_flush_vmid_ipa) * Instead, we invalidate Stage-2 for this IPA, and the * whole of Stage-1. Weep... */ + lsr x1, x1, #12 tlbi ipas2e1is, x1 /* * We have to ensure completion of the invalidation at Stage-2, @@ -1030,6 +1032,28 @@ ENTRY(__kvm_tlb_flush_vmid_ipa) ret ENDPROC(__kvm_tlb_flush_vmid_ipa) +/** + * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs + * @struct kvm *kvm - pointer to kvm structure + * + * Invalidates all Stage 1 and 2 TLB entries for current VMID. + */ +ENTRY(__kvm_tlb_flush_vmid) + dsb ishst + + kern_hyp_va x0 + ldr x2, [x0, #KVM_VTTBR] + msr vttbr_el2, x2 + isb + + tlbi vmalls12e1is + dsb ish + isb + + msr vttbr_el2, xzr + ret +ENDPROC(__kvm_tlb_flush_vmid) + ENTRY(__kvm_flush_vm_context) dsb ishst tlbi alle1is @@ -1140,9 +1164,9 @@ el1_sync: // Guest trapped into EL2 push x2, x3 mrs x1, esr_el2 - lsr x2, x1, #ESR_EL2_EC_SHIFT + lsr x2, x1, #ESR_ELx_EC_SHIFT - cmp x2, #ESR_EL2_EC_HVC64 + cmp x2, #ESR_ELx_EC_HVC64 b.ne el1_trap mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest @@ -1177,13 +1201,13 @@ el1_trap: * x1: ESR * x2: ESR_EC */ - cmp x2, #ESR_EL2_EC_DABT - mov x0, #ESR_EL2_EC_IABT + cmp x2, #ESR_ELx_EC_DABT_LOW + mov x0, #ESR_ELx_EC_IABT_LOW ccmp x2, x0, #4, ne b.ne 1f // Not an abort we care about /* This is an abort. Check for permission fault */ - and x2, x1, #ESR_EL2_FSC_TYPE + and x2, x1, #ESR_ELx_FSC_TYPE cmp x2, #FSC_PERM b.ne 1f // Not a permission fault diff --git a/arch/arm64/kvm/inject_fault.c b/arch/arm64/kvm/inject_fault.c index 81a02a8762b0..f02530e726f6 100644 --- a/arch/arm64/kvm/inject_fault.c +++ b/arch/arm64/kvm/inject_fault.c @@ -118,27 +118,27 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr * instruction set. Report an external synchronous abort. */ if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_EL1_IL; + esr |= ESR_ELx_IL; /* * Here, the guest runs in AArch64 mode when in EL1. If we get * an AArch32 fault, it means we managed to trap an EL0 fault. */ if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t) - esr |= (ESR_EL1_EC_IABT_EL0 << ESR_EL1_EC_SHIFT); + esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT); else - esr |= (ESR_EL1_EC_IABT_EL1 << ESR_EL1_EC_SHIFT); + esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT); if (!is_iabt) - esr |= ESR_EL1_EC_DABT_EL0; + esr |= ESR_ELx_EC_DABT_LOW; - vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_EL2_EC_xABT_xFSR_EXTABT; + vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_ELx_FSC_EXTABT; } static void inject_undef64(struct kvm_vcpu *vcpu) { unsigned long cpsr = *vcpu_cpsr(vcpu); - u32 esr = (ESR_EL1_EC_UNKNOWN << ESR_EL1_EC_SHIFT); + u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT); *vcpu_spsr(vcpu) = cpsr; *vcpu_elr_el1(vcpu) = *vcpu_pc(vcpu); @@ -151,7 +151,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu) * set. */ if (kvm_vcpu_trap_il_is32bit(vcpu)) - esr |= ESR_EL1_IL; + esr |= ESR_ELx_IL; vcpu_sys_reg(vcpu, ESR_EL1) = esr; } diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index 70a7816535cd..0b4326578985 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -90,7 +90,6 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu) if (!cpu_has_32bit_el1()) return -EINVAL; cpu_reset = &default_regs_reset32; - vcpu->arch.hcr_el2 &= ~HCR_RW; } else { cpu_reset = &default_regs_reset; } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4cc3b719208e..c370b4014799 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -20,17 +20,20 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#include <linux/mm.h> #include <linux/kvm_host.h> +#include <linux/mm.h> #include <linux/uaccess.h> -#include <asm/kvm_arm.h> -#include <asm/kvm_host.h> -#include <asm/kvm_emulate.h> -#include <asm/kvm_coproc.h> -#include <asm/kvm_mmu.h> + #include <asm/cacheflush.h> #include <asm/cputype.h> #include <asm/debug-monitors.h> +#include <asm/esr.h> +#include <asm/kvm_arm.h> +#include <asm/kvm_coproc.h> +#include <asm/kvm_emulate.h> +#include <asm/kvm_host.h> +#include <asm/kvm_mmu.h> + #include <trace/events/kvm.h> #include "sys_regs.h" @@ -69,68 +72,31 @@ static u32 get_ccsidr(u32 csselr) return ccsidr; } -static void do_dc_cisw(u32 val) -{ - asm volatile("dc cisw, %x0" : : "r" (val)); - dsb(ish); -} - -static void do_dc_csw(u32 val) -{ - asm volatile("dc csw, %x0" : : "r" (val)); - dsb(ish); -} - -/* See note at ARM ARM B1.14.4 */ +/* + * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized). + */ static bool access_dcsw(struct kvm_vcpu *vcpu, const struct sys_reg_params *p, const struct sys_reg_desc *r) { - unsigned long val; - int cpu; - if (!p->is_write) return read_from_write_only(vcpu, p); - cpu = get_cpu(); - - cpumask_setall(&vcpu->arch.require_dcache_flush); - cpumask_clear_cpu(cpu, &vcpu->arch.require_dcache_flush); - - /* If we were already preempted, take the long way around */ - if (cpu != vcpu->arch.last_pcpu) { - flush_cache_all(); - goto done; - } - - val = *vcpu_reg(vcpu, p->Rt); - - switch (p->CRm) { - case 6: /* Upgrade DCISW to DCCISW, as per HCR.SWIO */ - case 14: /* DCCISW */ - do_dc_cisw(val); - break; - - case 10: /* DCCSW */ - do_dc_csw(val); - break; - } - -done: - put_cpu(); - + kvm_set_way_flush(vcpu); return true; } /* * Generic accessor for VM registers. Only called as long as HCR_TVM - * is set. + * is set. If the guest enables the MMU, we stop trapping the VM + * sys_regs and leave it in complete control of the caches. */ static bool access_vm_reg(struct kvm_vcpu *vcpu, const struct sys_reg_params *p, const struct sys_reg_desc *r) { unsigned long val; + bool was_enabled = vcpu_has_cache_enabled(vcpu); BUG_ON(!p->is_write); @@ -143,24 +109,27 @@ static bool access_vm_reg(struct kvm_vcpu *vcpu, vcpu_cp15_64_low(vcpu, r->reg) = val & 0xffffffffUL; } + kvm_toggle_cache(vcpu, was_enabled); return true; } /* - * SCTLR_EL1 accessor. Only called as long as HCR_TVM is set. If the - * guest enables the MMU, we stop trapping the VM sys_regs and leave - * it in complete control of the caches. + * Trap handler for the GICv3 SGI generation system register. + * Forward the request to the VGIC emulation. + * The cp15_64 code makes sure this automatically works + * for both AArch64 and AArch32 accesses. */ -static bool access_sctlr(struct kvm_vcpu *vcpu, - const struct sys_reg_params *p, - const struct sys_reg_desc *r) +static bool access_gic_sgi(struct kvm_vcpu *vcpu, + const struct sys_reg_params *p, + const struct sys_reg_desc *r) { - access_vm_reg(vcpu, p, r); + u64 val; - if (vcpu_has_cache_enabled(vcpu)) { /* MMU+Caches enabled? */ - vcpu->arch.hcr_el2 &= ~HCR_TVM; - stage2_flush_vm(vcpu->kvm); - } + if (!p->is_write) + return read_from_write_only(vcpu, p); + + val = *vcpu_reg(vcpu, p->Rt); + vgic_v3_dispatch_sgi(vcpu, val); return true; } @@ -252,10 +221,19 @@ static void reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) static void reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { + u64 mpidr; + /* - * Simply map the vcpu_id into the Aff0 field of the MPIDR. + * Map the vcpu_id into the first three affinity level fields of + * the MPIDR. We limit the number of VCPUs in level 0 due to a + * limitation to 16 CPUs in that level in the ICC_SGIxR registers + * of the GICv3 to be able to address each CPU directly when + * sending IPIs. */ - vcpu_sys_reg(vcpu, MPIDR_EL1) = (1UL << 31) | (vcpu->vcpu_id & 0xff); + mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); + mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); + mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); + vcpu_sys_reg(vcpu, MPIDR_EL1) = (1ULL << 31) | mpidr; } /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */ @@ -377,7 +355,7 @@ static const struct sys_reg_desc sys_reg_descs[] = { NULL, reset_mpidr, MPIDR_EL1 }, /* SCTLR_EL1 */ { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b000), - access_sctlr, reset_val, SCTLR_EL1, 0x00C50078 }, + access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 }, /* CPACR_EL1 */ { Op0(0b11), Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b010), NULL, reset_val, CPACR_EL1, 0 }, @@ -424,6 +402,14 @@ static const struct sys_reg_desc sys_reg_descs[] = { /* VBAR_EL1 */ { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b0000), Op2(0b000), NULL, reset_val, VBAR_EL1, 0 }, + + /* ICC_SGI1R_EL1 */ + { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1011), Op2(0b101), + access_gic_sgi }, + /* ICC_SRE_EL1 */ + { Op0(0b11), Op1(0b000), CRn(0b1100), CRm(0b1100), Op2(0b101), + trap_raz_wi }, + /* CONTEXTIDR_EL1 */ { Op0(0b11), Op1(0b000), CRn(0b1101), CRm(0b0000), Op2(0b001), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 }, @@ -652,7 +638,9 @@ static const struct sys_reg_desc cp14_64_regs[] = { * register). */ static const struct sys_reg_desc cp15_regs[] = { - { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_sctlr, NULL, c1_SCTLR }, + { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, + + { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, c1_SCTLR }, { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, c2_TTBR1 }, { Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, c2_TTBCR }, @@ -690,11 +678,16 @@ static const struct sys_reg_desc cp15_regs[] = { { Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, c10_NMRR }, { Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, c10_AMAIR0 }, { Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, c10_AMAIR1 }, + + /* ICC_SRE */ + { Op1( 0), CRn(12), CRm(12), Op2( 5), trap_raz_wi }, + { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, c13_CID }, }; static const struct sys_reg_desc cp15_64_regs[] = { { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR0 }, + { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, c2_TTBR1 }, }; @@ -806,12 +799,12 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu, int cp; switch(hsr_ec) { - case ESR_EL2_EC_CP15_32: - case ESR_EL2_EC_CP15_64: + case ESR_ELx_EC_CP15_32: + case ESR_ELx_EC_CP15_64: cp = 15; break; - case ESR_EL2_EC_CP14_MR: - case ESR_EL2_EC_CP14_64: + case ESR_ELx_EC_CP14_MR: + case ESR_ELx_EC_CP14_64: cp = 14; break; default: diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h new file mode 100644 index 000000000000..157416e963f2 --- /dev/null +++ b/arch/arm64/kvm/trace.h @@ -0,0 +1,55 @@ +#if !defined(_TRACE_ARM64_KVM_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ARM64_KVM_H + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM kvm + +TRACE_EVENT(kvm_wfx_arm64, + TP_PROTO(unsigned long vcpu_pc, bool is_wfe), + TP_ARGS(vcpu_pc, is_wfe), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(bool, is_wfe) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->is_wfe = is_wfe; + ), + + TP_printk("guest executed wf%c at: 0x%08lx", + __entry->is_wfe ? 'e' : 'i', __entry->vcpu_pc) +); + +TRACE_EVENT(kvm_hvc_arm64, + TP_PROTO(unsigned long vcpu_pc, unsigned long r0, unsigned long imm), + TP_ARGS(vcpu_pc, r0, imm), + + TP_STRUCT__entry( + __field(unsigned long, vcpu_pc) + __field(unsigned long, r0) + __field(unsigned long, imm) + ), + + TP_fast_assign( + __entry->vcpu_pc = vcpu_pc; + __entry->r0 = r0; + __entry->imm = imm; + ), + + TP_printk("HVC at 0x%08lx (r0: 0x%08lx, imm: 0x%lx)", + __entry->vcpu_pc, __entry->r0, __entry->imm) +); + +#endif /* _TRACE_ARM64_KVM_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/arch/arm64/kvm/vgic-v3-switch.S b/arch/arm64/kvm/vgic-v3-switch.S index d16046999e06..617a012a0107 100644 --- a/arch/arm64/kvm/vgic-v3-switch.S +++ b/arch/arm64/kvm/vgic-v3-switch.S @@ -148,17 +148,18 @@ * x0: Register pointing to VCPU struct */ .macro restore_vgic_v3_state - // Disable SRE_EL1 access. Necessary, otherwise - // ICH_VMCR_EL2.VFIQEn becomes one, and FIQ happens... - msr_s ICC_SRE_EL1, xzr - isb - // Compute the address of struct vgic_cpu add x3, x0, #VCPU_VGIC_CPU // Restore all interesting registers ldr w4, [x3, #VGIC_V3_CPU_HCR] ldr w5, [x3, #VGIC_V3_CPU_VMCR] + ldr w25, [x3, #VGIC_V3_CPU_SRE] + + msr_s ICC_SRE_EL1, x25 + + // make sure SRE is valid before writing the other registers + isb msr_s ICH_HCR_EL2, x4 msr_s ICH_VMCR_EL2, x5 @@ -244,9 +245,12 @@ dsb sy // Prevent the guest from touching the GIC system registers + // if SRE isn't enabled for GICv3 emulation + cbnz x25, 1f mrs_s x5, ICC_SRE_EL2 and x5, x5, #~ICC_SRE_EL2_ENABLE msr_s ICC_SRE_EL2, x5 +1: .endm ENTRY(__save_vgic_v3_state) diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile index c56179ed2c09..773d37a14039 100644 --- a/arch/arm64/mm/Makefile +++ b/arch/arm64/mm/Makefile @@ -3,3 +3,4 @@ obj-y := dma-mapping.o extable.o fault.o init.o \ ioremap.o mmap.o pgd.o mmu.o \ context.o proc.o pageattr.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o +obj-$(CONFIG_ARM64_PTDUMP) += dump.o diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 23663837acff..2560e1e1562e 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -17,9 +17,12 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ +#include <linux/errno.h> #include <linux/linkage.h> #include <linux/init.h> #include <asm/assembler.h> +#include <asm/cpufeature.h> +#include <asm/alternative-asm.h> #include "proc-macros.S" @@ -138,9 +141,12 @@ USER(9f, ic ivau, x4 ) // invalidate I line PoU add x4, x4, x2 cmp x4, x1 b.lo 1b -9: // ignore any faulting cache operation dsb ish isb + mov x0, #0 + ret +9: + mov x0, #-EFAULT ret ENDPROC(flush_icache_range) ENDPROC(__flush_cache_user_range) @@ -210,7 +216,7 @@ __dma_clean_range: dcache_line_size x2, x3 sub x3, x2, #1 bic x0, x0, x3 -1: dc cvac, x0 // clean D / U line +1: alternative_insn "dc cvac, x0", "dc civac, x0", ARM64_WORKAROUND_CLEAN_CACHE add x0, x0, x2 cmp x0, x1 b.lo 1b diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index d92094203913..b0bd4e5fd5cf 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -51,7 +51,7 @@ static int __init early_coherent_pool(char *p) } early_param("coherent_pool", early_coherent_pool); -static void *__alloc_from_pool(size_t size, struct page **ret_page) +static void *__alloc_from_pool(size_t size, struct page **ret_page, gfp_t flags) { unsigned long val; void *ptr = NULL; @@ -67,6 +67,7 @@ static void *__alloc_from_pool(size_t size, struct page **ret_page) *ret_page = phys_to_page(phys); ptr = (void *)val; + memset(ptr, 0, size); } return ptr; @@ -101,15 +102,17 @@ static void *__dma_alloc_coherent(struct device *dev, size_t size, flags |= GFP_DMA; if (IS_ENABLED(CONFIG_DMA_CMA) && (flags & __GFP_WAIT)) { struct page *page; + void *addr; - size = PAGE_ALIGN(size); page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT, get_order(size)); if (!page) return NULL; *dma_handle = phys_to_dma(dev, page_to_phys(page)); - return page_address(page); + addr = page_address(page); + memset(addr, 0, size); + return addr; } else { return swiotlb_alloc_coherent(dev, size, dma_handle, flags); } @@ -134,30 +137,34 @@ static void __dma_free_coherent(struct device *dev, size_t size, swiotlb_free_coherent(dev, size, vaddr, dma_handle); } -static void *__dma_alloc_noncoherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flags, - struct dma_attrs *attrs) +static void *__dma_alloc(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flags, + struct dma_attrs *attrs) { struct page *page; void *ptr, *coherent_ptr; + bool coherent = is_device_dma_coherent(dev); size = PAGE_ALIGN(size); - if (!(flags & __GFP_WAIT)) { + if (!coherent && !(flags & __GFP_WAIT)) { struct page *page = NULL; - void *addr = __alloc_from_pool(size, &page); + void *addr = __alloc_from_pool(size, &page, flags); if (addr) *dma_handle = phys_to_dma(dev, page_to_phys(page)); return addr; - } ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs); if (!ptr) goto no_mem; + /* no need for non-cacheable mapping if coherent */ + if (coherent) + return ptr; + /* remove any dirty cache lines on the kernel alias */ __dma_flush_range(ptr, ptr + size); @@ -179,15 +186,19 @@ no_mem: return NULL; } -static void __dma_free_noncoherent(struct device *dev, size_t size, - void *vaddr, dma_addr_t dma_handle, - struct dma_attrs *attrs) +static void __dma_free(struct device *dev, size_t size, + void *vaddr, dma_addr_t dma_handle, + struct dma_attrs *attrs) { void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle)); - if (__free_from_pool(vaddr, size)) - return; - vunmap(vaddr); + size = PAGE_ALIGN(size); + + if (!is_device_dma_coherent(dev)) { + if (__free_from_pool(vaddr, size)) + return; + vunmap(vaddr); + } __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs); } @@ -199,7 +210,8 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page, dma_addr_t dev_addr; dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs); - __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); + if (!is_device_dma_coherent(dev)) + __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); return dev_addr; } @@ -209,7 +221,8 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); + if (!is_device_dma_coherent(dev)) + __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); swiotlb_unmap_page(dev, dev_addr, size, dir, attrs); } @@ -221,9 +234,10 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl, int i, ret; ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs); - for_each_sg(sgl, sg, ret, i) - __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), - sg->length, dir); + if (!is_device_dma_coherent(dev)) + for_each_sg(sgl, sg, ret, i) + __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), + sg->length, dir); return ret; } @@ -236,9 +250,10 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev, struct scatterlist *sg; int i; - for_each_sg(sgl, sg, nelems, i) - __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), - sg->length, dir); + if (!is_device_dma_coherent(dev)) + for_each_sg(sgl, sg, nelems, i) + __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), + sg->length, dir); swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs); } @@ -246,7 +261,8 @@ static void __swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir) { - __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); + if (!is_device_dma_coherent(dev)) + __dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir); } @@ -255,7 +271,8 @@ static void __swiotlb_sync_single_for_device(struct device *dev, enum dma_data_direction dir) { swiotlb_sync_single_for_device(dev, dev_addr, size, dir); - __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); + if (!is_device_dma_coherent(dev)) + __dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir); } static void __swiotlb_sync_sg_for_cpu(struct device *dev, @@ -265,9 +282,10 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg; int i; - for_each_sg(sgl, sg, nelems, i) - __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), - sg->length, dir); + if (!is_device_dma_coherent(dev)) + for_each_sg(sgl, sg, nelems, i) + __dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), + sg->length, dir); swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir); } @@ -279,9 +297,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev, int i; swiotlb_sync_sg_for_device(dev, sgl, nelems, dir); - for_each_sg(sgl, sg, nelems, i) - __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), - sg->length, dir); + if (!is_device_dma_coherent(dev)) + for_each_sg(sgl, sg, nelems, i) + __dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)), + sg->length, dir); } /* vma->vm_page_prot must be set appropriately before calling this function */ @@ -308,28 +327,20 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma, return ret; } -static int __swiotlb_mmap_noncoherent(struct device *dev, - struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) -{ - vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false); - return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); -} - -static int __swiotlb_mmap_coherent(struct device *dev, - struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - struct dma_attrs *attrs) +static int __swiotlb_mmap(struct device *dev, + struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_attrs *attrs) { - /* Just use whatever page_prot attributes were specified */ + vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, + is_device_dma_coherent(dev)); return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size); } -struct dma_map_ops noncoherent_swiotlb_dma_ops = { - .alloc = __dma_alloc_noncoherent, - .free = __dma_free_noncoherent, - .mmap = __swiotlb_mmap_noncoherent, +static struct dma_map_ops swiotlb_dma_ops = { + .alloc = __dma_alloc, + .free = __dma_free, + .mmap = __swiotlb_mmap, .map_page = __swiotlb_map_page, .unmap_page = __swiotlb_unmap_page, .map_sg = __swiotlb_map_sg_attrs, @@ -341,26 +352,6 @@ struct dma_map_ops noncoherent_swiotlb_dma_ops = { .dma_supported = swiotlb_dma_supported, .mapping_error = swiotlb_dma_mapping_error, }; -EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops); - -struct dma_map_ops coherent_swiotlb_dma_ops = { - .alloc = __dma_alloc_coherent, - .free = __dma_free_coherent, - .mmap = __swiotlb_mmap_coherent, - .map_page = swiotlb_map_page, - .unmap_page = swiotlb_unmap_page, - .map_sg = swiotlb_map_sg_attrs, - .unmap_sg = swiotlb_unmap_sg_attrs, - .sync_single_for_cpu = swiotlb_sync_single_for_cpu, - .sync_single_for_device = swiotlb_sync_single_for_device, - .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, - .sync_sg_for_device = swiotlb_sync_sg_for_device, - .dma_supported = swiotlb_dma_supported, - .mapping_error = swiotlb_dma_mapping_error, -}; -EXPORT_SYMBOL(coherent_swiotlb_dma_ops); - -extern int swiotlb_late_init_with_default_size(size_t default_size); static int __init atomic_pool_init(void) { @@ -423,21 +414,13 @@ out: return -ENOMEM; } -static int __init swiotlb_late_init(void) -{ - size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT); - - dma_ops = &noncoherent_swiotlb_dma_ops; - - return swiotlb_late_init_with_default_size(swiotlb_size); -} - static int __init arm64_dma_init(void) { - int ret = 0; + int ret; + + dma_ops = &swiotlb_dma_ops; - ret |= swiotlb_late_init(); - ret |= atomic_pool_init(); + ret = atomic_pool_init(); return ret; } diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c new file mode 100644 index 000000000000..f3d6221cd5bd --- /dev/null +++ b/arch/arm64/mm/dump.c @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2014, The Linux Foundation. All rights reserved. + * Debug helper to dump the current kernel pagetables of the system + * so that we can see what the various memory ranges are set to. + * + * Derived from x86 and arm implementation: + * (C) Copyright 2008 Intel Corporation + * + * Author: Arjan van de Ven <arjan@linux.intel.com> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + */ +#include <linux/debugfs.h> +#include <linux/errno.h> +#include <linux/fs.h> +#include <linux/io.h> +#include <linux/init.h> +#include <linux/mm.h> +#include <linux/sched.h> +#include <linux/seq_file.h> + +#include <asm/fixmap.h> +#include <asm/memory.h> +#include <asm/pgtable.h> +#include <asm/pgtable-hwdef.h> + +#define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS) + +struct addr_marker { + unsigned long start_address; + const char *name; +}; + +enum address_markers_idx { + VMALLOC_START_NR = 0, + VMALLOC_END_NR, +#ifdef CONFIG_SPARSEMEM_VMEMMAP + VMEMMAP_START_NR, + VMEMMAP_END_NR, +#endif + FIXADDR_START_NR, + FIXADDR_END_NR, + PCI_START_NR, + PCI_END_NR, + MODULES_START_NR, + MODUELS_END_NR, + KERNEL_SPACE_NR, +}; + +static struct addr_marker address_markers[] = { + { VMALLOC_START, "vmalloc() Area" }, + { VMALLOC_END, "vmalloc() End" }, +#ifdef CONFIG_SPARSEMEM_VMEMMAP + { 0, "vmemmap start" }, + { 0, "vmemmap end" }, +#endif + { FIXADDR_START, "Fixmap start" }, + { FIXADDR_TOP, "Fixmap end" }, + { PCI_IO_START, "PCI I/O start" }, + { PCI_IO_END, "PCI I/O end" }, + { MODULES_VADDR, "Modules start" }, + { MODULES_END, "Modules end" }, + { PAGE_OFFSET, "Kernel Mapping" }, + { -1, NULL }, +}; + +struct pg_state { + struct seq_file *seq; + const struct addr_marker *marker; + unsigned long start_address; + unsigned level; + u64 current_prot; +}; + +struct prot_bits { + u64 mask; + u64 val; + const char *set; + const char *clear; +}; + +static const struct prot_bits pte_bits[] = { + { + .mask = PTE_USER, + .val = PTE_USER, + .set = "USR", + .clear = " ", + }, { + .mask = PTE_RDONLY, + .val = PTE_RDONLY, + .set = "ro", + .clear = "RW", + }, { + .mask = PTE_PXN, + .val = PTE_PXN, + .set = "NX", + .clear = "x ", + }, { + .mask = PTE_SHARED, + .val = PTE_SHARED, + .set = "SHD", + .clear = " ", + }, { + .mask = PTE_AF, + .val = PTE_AF, + .set = "AF", + .clear = " ", + }, { + .mask = PTE_NG, + .val = PTE_NG, + .set = "NG", + .clear = " ", + }, { + .mask = PTE_UXN, + .val = PTE_UXN, + .set = "UXN", + }, { + .mask = PTE_ATTRINDX_MASK, + .val = PTE_ATTRINDX(MT_DEVICE_nGnRnE), + .set = "DEVICE/nGnRnE", + }, { + .mask = PTE_ATTRINDX_MASK, + .val = PTE_ATTRINDX(MT_DEVICE_nGnRE), + .set = "DEVICE/nGnRE", + }, { + .mask = PTE_ATTRINDX_MASK, + .val = PTE_ATTRINDX(MT_DEVICE_GRE), + .set = "DEVICE/GRE", + }, { + .mask = PTE_ATTRINDX_MASK, + .val = PTE_ATTRINDX(MT_NORMAL_NC), + .set = "MEM/NORMAL-NC", + }, { + .mask = PTE_ATTRINDX_MASK, + .val = PTE_ATTRINDX(MT_NORMAL), + .set = "MEM/NORMAL", + } +}; + +struct pg_level { + const struct prot_bits *bits; + size_t num; + u64 mask; +}; + +static struct pg_level pg_level[] = { + { + }, { /* pgd */ + .bits = pte_bits, + .num = ARRAY_SIZE(pte_bits), + }, { /* pud */ + .bits = pte_bits, + .num = ARRAY_SIZE(pte_bits), + }, { /* pmd */ + .bits = pte_bits, + .num = ARRAY_SIZE(pte_bits), + }, { /* pte */ + .bits = pte_bits, + .num = ARRAY_SIZE(pte_bits), + }, +}; + +static void dump_prot(struct pg_state *st, const struct prot_bits *bits, + size_t num) +{ + unsigned i; + + for (i = 0; i < num; i++, bits++) { + const char *s; + + if ((st->current_prot & bits->mask) == bits->val) + s = bits->set; + else + s = bits->clear; + + if (s) + seq_printf(st->seq, " %s", s); + } +} + +static void note_page(struct pg_state *st, unsigned long addr, unsigned level, + u64 val) +{ + static const char units[] = "KMGTPE"; + u64 prot = val & pg_level[level].mask; + + if (!st->level) { + st->level = level; + st->current_prot = prot; + st->start_address = addr; + seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); + } else if (prot != st->current_prot || level != st->level || + addr >= st->marker[1].start_address) { + const char *unit = units; + unsigned long delta; + + if (st->current_prot) { + seq_printf(st->seq, "0x%16lx-0x%16lx ", + st->start_address, addr); + + delta = (addr - st->start_address) >> 10; + while (!(delta & 1023) && unit[1]) { + delta >>= 10; + unit++; + } + seq_printf(st->seq, "%9lu%c", delta, *unit); + if (pg_level[st->level].bits) + dump_prot(st, pg_level[st->level].bits, + pg_level[st->level].num); + seq_puts(st->seq, "\n"); + } + + if (addr >= st->marker[1].start_address) { + st->marker++; + seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); + } + + st->start_address = addr; + st->current_prot = prot; + st->level = level; + } + + if (addr >= st->marker[1].start_address) { + st->marker++; + seq_printf(st->seq, "---[ %s ]---\n", st->marker->name); + } + +} + +static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start) +{ + pte_t *pte = pte_offset_kernel(pmd, 0); + unsigned long addr; + unsigned i; + + for (i = 0; i < PTRS_PER_PTE; i++, pte++) { + addr = start + i * PAGE_SIZE; + note_page(st, addr, 4, pte_val(*pte)); + } +} + +static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start) +{ + pmd_t *pmd = pmd_offset(pud, 0); + unsigned long addr; + unsigned i; + + for (i = 0; i < PTRS_PER_PMD; i++, pmd++) { + addr = start + i * PMD_SIZE; + if (pmd_none(*pmd) || pmd_sect(*pmd)) { + note_page(st, addr, 3, pmd_val(*pmd)); + } else { + BUG_ON(pmd_bad(*pmd)); + walk_pte(st, pmd, addr); + } + } +} + +static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start) +{ + pud_t *pud = pud_offset(pgd, 0); + unsigned long addr; + unsigned i; + + for (i = 0; i < PTRS_PER_PUD; i++, pud++) { + addr = start + i * PUD_SIZE; + if (pud_none(*pud) || pud_sect(*pud)) { + note_page(st, addr, 2, pud_val(*pud)); + } else { + BUG_ON(pud_bad(*pud)); + walk_pmd(st, pud, addr); + } + } +} + +static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long start) +{ + pgd_t *pgd = pgd_offset(mm, 0UL); + unsigned i; + unsigned long addr; + + for (i = 0; i < PTRS_PER_PGD; i++, pgd++) { + addr = start + i * PGDIR_SIZE; + if (pgd_none(*pgd)) { + note_page(st, addr, 1, pgd_val(*pgd)); + } else { + BUG_ON(pgd_bad(*pgd)); + walk_pud(st, pgd, addr); + } + } +} + +static int ptdump_show(struct seq_file *m, void *v) +{ + struct pg_state st = { + .seq = m, + .marker = address_markers, + }; + + walk_pgd(&st, &init_mm, LOWEST_ADDR); + + note_page(&st, 0, 0, 0); + return 0; +} + +static int ptdump_open(struct inode *inode, struct file *file) +{ + return single_open(file, ptdump_show, NULL); +} + +static const struct file_operations ptdump_fops = { + .open = ptdump_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int ptdump_init(void) +{ + struct dentry *pe; + unsigned i, j; + + for (i = 0; i < ARRAY_SIZE(pg_level); i++) + if (pg_level[i].bits) + for (j = 0; j < pg_level[i].num; j++) + pg_level[i].mask |= pg_level[i].bits[j].mask; + +#ifdef CONFIG_SPARSEMEM_VMEMMAP + address_markers[VMEMMAP_START_NR].start_address = + (unsigned long)virt_to_page(PAGE_OFFSET); + address_markers[VMEMMAP_END_NR].start_address = + (unsigned long)virt_to_page(high_memory); +#endif + + pe = debugfs_create_file("kernel_page_tables", 0400, NULL, NULL, + &ptdump_fops); + return pe ? 0 : -ENOMEM; +} +device_initcall(ptdump_init); diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 41cb6d3d6075..0948d327d013 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -211,7 +211,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, * If we're in an interrupt or have no user context, we must not take * the fault. */ - if (in_atomic() || !mm) + if (faulthandler_disabled() || !mm) goto no_context; if (user_mode(regs)) @@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, if (esr & ESR_LNX_EXEC) { vm_flags = VM_EXEC; - } else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) { + } else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) { vm_flags = VM_WRITE; mm_flags |= FAULT_FLAG_WRITE; } @@ -380,7 +380,7 @@ static struct fault_info { { do_bad, SIGBUS, 0, "level 1 address size fault" }, { do_bad, SIGBUS, 0, "level 2 address size fault" }, { do_bad, SIGBUS, 0, "level 3 address size fault" }, - { do_translation_fault, SIGSEGV, SEGV_MAPERR, "input address range fault" }, + { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, { do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 023747bf4dd7..2de9d2e59d96 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -38,12 +38,6 @@ int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep) } #endif -struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, - int write) -{ - return ERR_PTR(-EINVAL); -} - int pmd_huge(pmd_t pmd) { return !(pmd_val(pmd) & PMD_TABLE_BIT); diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 494297c698ca..597831bdddf3 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -33,16 +33,20 @@ #include <linux/dma-mapping.h> #include <linux/dma-contiguous.h> #include <linux/efi.h> +#include <linux/swiotlb.h> #include <asm/fixmap.h> +#include <asm/memory.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/sizes.h> #include <asm/tlb.h> +#include <asm/alternative.h> #include "mm.h" phys_addr_t memstart_addr __read_mostly = 0; +phys_addr_t arm64_dma_phys_limit __read_mostly; #ifdef CONFIG_BLK_DEV_INITRD static int __init early_initrd(char *p) @@ -83,7 +87,7 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max) /* 4GB maximum for 32-bit only capable devices */ if (IS_ENABLED(CONFIG_ZONE_DMA)) { - max_dma = PFN_DOWN(max_zone_dma_phys()); + max_dma = PFN_DOWN(arm64_dma_phys_limit); zone_size[ZONE_DMA] = max_dma - min; } zone_size[ZONE_NORMAL] = max - max_dma; @@ -135,9 +139,26 @@ static void arm64_memory_present(void) } #endif +static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX; + +/* + * Limit the memory size that was specified via FDT. + */ +static int __init early_mem(char *p) +{ + if (!p) + return 1; + + memory_limit = memparse(p, &p) & PAGE_MASK; + pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); + + return 0; +} +early_param("mem", early_mem); + void __init arm64_memblock_init(void) { - phys_addr_t dma_phys_limit = 0; + memblock_enforce_memory_limit(memory_limit); /* * Register the kernel text, kernel data, initrd, and initial @@ -153,8 +174,10 @@ void __init arm64_memblock_init(void) /* 4GB maximum for 32-bit only capable devices */ if (IS_ENABLED(CONFIG_ZONE_DMA)) - dma_phys_limit = max_zone_dma_phys(); - dma_contiguous_reserve(dma_phys_limit); + arm64_dma_phys_limit = max_zone_dma_phys(); + else + arm64_dma_phys_limit = PHYS_MASK + 1; + dma_contiguous_reserve(arm64_dma_phys_limit); memblock_allow_resize(); memblock_dump_all(); @@ -167,6 +190,8 @@ void __init bootmem_init(void) min = PFN_UP(memblock_start_of_DRAM()); max = PFN_DOWN(memblock_end_of_DRAM()); + early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT); + /* * Sparsemem tries to allocate bootmem in memory_present(), so must be * done after the fixed reservations. @@ -255,6 +280,8 @@ static void __init free_unused_memmap(void) */ void __init mem_init(void) { + swiotlb_init(1); + set_max_mapnr(pfn_to_page(max_pfn) - mem_map); #ifndef CONFIG_SPARSEMEM_VMEMMAP @@ -276,8 +303,8 @@ void __init mem_init(void) " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" " 0x%16lx - 0x%16lx (%6ld MB actual)\n" #endif - " PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n" " fixed : 0x%16lx - 0x%16lx (%6ld KB)\n" + " PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n" " modules : 0x%16lx - 0x%16lx (%6ld MB)\n" " memory : 0x%16lx - 0x%16lx (%6ld MB)\n" " .init : 0x%p" " - 0x%p" " (%6ld KB)\n" @@ -290,8 +317,8 @@ void __init mem_init(void) MLM((unsigned long)virt_to_page(PAGE_OFFSET), (unsigned long)virt_to_page(high_memory)), #endif - MLM((unsigned long)PCI_IOBASE, (unsigned long)PCI_IOBASE + SZ_16M), MLK(FIXADDR_START, FIXADDR_TOP), + MLM(PCI_IO_START, PCI_IO_END), MLM(MODULES_VADDR, MODULES_END), MLM(PAGE_OFFSET, (unsigned long)high_memory), MLK_ROUNDUP(__init_begin, __init_end), @@ -324,7 +351,9 @@ void __init mem_init(void) void free_initmem(void) { + fixup_init(); free_initmem_default(0); + free_alternatives_memory(); } #ifdef CONFIG_BLK_DEV_INITRD @@ -333,14 +362,8 @@ static int keep_initrd; void free_initrd_mem(unsigned long start, unsigned long end) { - if (!keep_initrd) { - if (start == initrd_start) - start = round_down(start, PAGE_SIZE); - if (end == initrd_end) - end = round_up(end, PAGE_SIZE); - + if (!keep_initrd) free_reserved_area((void *)start, (void *)end, 0, "initrd"); - } } static int __init keepinitrd_setup(char *__unused) diff --git a/arch/arm64/mm/ioremap.c b/arch/arm64/mm/ioremap.c index 4a07630a6616..01e88c8bcab0 100644 --- a/arch/arm64/mm/ioremap.c +++ b/arch/arm64/mm/ioremap.c @@ -62,6 +62,7 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size, if (!area) return NULL; addr = (unsigned long)area->addr; + area->phys_addr = phys_addr; err = ioremap_page_range(addr, addr + size, phys_addr, prot); if (err) { @@ -103,97 +104,10 @@ void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size) } EXPORT_SYMBOL(ioremap_cache); -static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; -#if CONFIG_ARM64_PGTABLE_LEVELS > 2 -static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss; -#endif -#if CONFIG_ARM64_PGTABLE_LEVELS > 3 -static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss; -#endif - -static inline pud_t * __init early_ioremap_pud(unsigned long addr) -{ - pgd_t *pgd; - - pgd = pgd_offset_k(addr); - BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); - - return pud_offset(pgd, addr); -} - -static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) -{ - pud_t *pud = early_ioremap_pud(addr); - - BUG_ON(pud_none(*pud) || pud_bad(*pud)); - - return pmd_offset(pud, addr); -} - -static inline pte_t * __init early_ioremap_pte(unsigned long addr) -{ - pmd_t *pmd = early_ioremap_pmd(addr); - - BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd)); - - return pte_offset_kernel(pmd, addr); -} - +/* + * Must be called after early_fixmap_init + */ void __init early_ioremap_init(void) { - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; - unsigned long addr = fix_to_virt(FIX_BTMAP_BEGIN); - - pgd = pgd_offset_k(addr); - pgd_populate(&init_mm, pgd, bm_pud); - pud = pud_offset(pgd, addr); - pud_populate(&init_mm, pud, bm_pmd); - pmd = pmd_offset(pud, addr); - pmd_populate_kernel(&init_mm, pmd, bm_pte); - - /* - * The boot-ioremap range spans multiple pmds, for which - * we are not prepared: - */ - BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) - != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); - - if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) { - WARN_ON(1); - pr_warn("pmd %p != %p\n", - pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))); - pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", - fix_to_virt(FIX_BTMAP_BEGIN)); - pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", - fix_to_virt(FIX_BTMAP_END)); - - pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); - pr_warn("FIX_BTMAP_BEGIN: %d\n", - FIX_BTMAP_BEGIN); - } - early_ioremap_setup(); } - -void __init __early_set_fixmap(enum fixed_addresses idx, - phys_addr_t phys, pgprot_t flags) -{ - unsigned long addr = __fix_to_virt(idx); - pte_t *pte; - - if (idx >= __end_of_fixed_addresses) { - BUG(); - return; - } - - pte = early_ioremap_pte(addr); - - if (pgprot_val(flags)) - set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); - else { - pte_clear(&init_mm, addr, pte); - flush_tlb_kernel_range(addr, addr+PAGE_SIZE); - } -} diff --git a/arch/arm64/mm/mm.h b/arch/arm64/mm/mm.h index d519f4f50c8c..ef47d99b5cbc 100644 --- a/arch/arm64/mm/mm.h +++ b/arch/arm64/mm/mm.h @@ -1,2 +1,3 @@ extern void __init bootmem_init(void); -extern void __init arm64_swiotlb_init(void); + +void fixup_init(void); diff --git a/arch/arm64/mm/mmap.c b/arch/arm64/mm/mmap.c index 1d73662f00ff..ed177475dd8c 100644 --- a/arch/arm64/mm/mmap.c +++ b/arch/arm64/mm/mmap.c @@ -47,25 +47,16 @@ static int mmap_is_legacy(void) return sysctl_legacy_va_layout; } -/* - * Since get_random_int() returns the same value within a 1 jiffy window, we - * will almost always get the same randomisation for the stack and mmap - * region. This will mean the relative distance between stack and mmap will be - * the same. - * - * To avoid this we can shift the randomness by 1 bit. - */ -static unsigned long mmap_rnd(void) +unsigned long arch_mmap_rnd(void) { - unsigned long rnd = 0; + unsigned long rnd; - if (current->flags & PF_RANDOMIZE) - rnd = (long)get_random_int() & (STACK_RND_MASK >> 1); + rnd = (unsigned long)get_random_int() & STACK_RND_MASK; - return rnd << (PAGE_SHIFT + 1); + return rnd << PAGE_SHIFT; } -static unsigned long mmap_base(void) +static unsigned long mmap_base(unsigned long rnd) { unsigned long gap = rlimit(RLIMIT_STACK); @@ -74,7 +65,7 @@ static unsigned long mmap_base(void) else if (gap > MAX_GAP) gap = MAX_GAP; - return PAGE_ALIGN(STACK_TOP - gap - mmap_rnd()); + return PAGE_ALIGN(STACK_TOP - gap - rnd); } /* @@ -83,15 +74,20 @@ static unsigned long mmap_base(void) */ void arch_pick_mmap_layout(struct mm_struct *mm) { + unsigned long random_factor = 0UL; + + if (current->flags & PF_RANDOMIZE) + random_factor = arch_mmap_rnd(); + /* * Fall back to the standard layout if the personality bit is set, or * if the expected stack growth is unlimited: */ if (mmap_is_legacy()) { - mm->mmap_base = TASK_UNMAPPED_BASE; + mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; mm->get_unmapped_area = arch_get_unmapped_area; } else { - mm->mmap_base = mmap_base(); + mm->mmap_base = mmap_base(random_factor); mm->get_unmapped_area = arch_get_unmapped_area_topdown; } } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index f4f8b500f74c..5b8b664422d3 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -26,8 +26,11 @@ #include <linux/memblock.h> #include <linux/fs.h> #include <linux/io.h> +#include <linux/slab.h> +#include <linux/stop_machine.h> #include <asm/cputype.h> +#include <asm/fixmap.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/sizes.h> @@ -37,6 +40,8 @@ #include "mm.h" +u64 idmap_t0sz = TCR_T0SZ(VA_BITS); + /* * Empty_zero_page is a special page that is used for zero-initialized data * and COW. @@ -44,80 +49,6 @@ struct page *empty_zero_page; EXPORT_SYMBOL(empty_zero_page); -struct cachepolicy { - const char policy[16]; - u64 mair; - u64 tcr; -}; - -static struct cachepolicy cache_policies[] __initdata = { - { - .policy = "uncached", - .mair = 0x44, /* inner, outer non-cacheable */ - .tcr = TCR_IRGN_NC | TCR_ORGN_NC, - }, { - .policy = "writethrough", - .mair = 0xaa, /* inner, outer write-through, read-allocate */ - .tcr = TCR_IRGN_WT | TCR_ORGN_WT, - }, { - .policy = "writeback", - .mair = 0xee, /* inner, outer write-back, read-allocate */ - .tcr = TCR_IRGN_WBnWA | TCR_ORGN_WBnWA, - } -}; - -/* - * These are useful for identifying cache coherency problems by allowing the - * cache or the cache and writebuffer to be turned off. It changes the Normal - * memory caching attributes in the MAIR_EL1 register. - */ -static int __init early_cachepolicy(char *p) -{ - int i; - u64 tmp; - - for (i = 0; i < ARRAY_SIZE(cache_policies); i++) { - int len = strlen(cache_policies[i].policy); - - if (memcmp(p, cache_policies[i].policy, len) == 0) - break; - } - if (i == ARRAY_SIZE(cache_policies)) { - pr_err("ERROR: unknown or unsupported cache policy: %s\n", p); - return 0; - } - - flush_cache_all(); - - /* - * Modify MT_NORMAL attributes in MAIR_EL1. - */ - asm volatile( - " mrs %0, mair_el1\n" - " bfi %0, %1, %2, #8\n" - " msr mair_el1, %0\n" - " isb\n" - : "=&r" (tmp) - : "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8)); - - /* - * Modify TCR PTW cacheability attributes. - */ - asm volatile( - " mrs %0, tcr_el1\n" - " bic %0, %0, %2\n" - " orr %0, %0, %1\n" - " msr tcr_el1, %0\n" - " isb\n" - : "=&r" (tmp) - : "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK)); - - flush_cache_all(); - - return 0; -} -early_param("cachepolicy", early_cachepolicy); - pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot) { @@ -132,19 +63,42 @@ EXPORT_SYMBOL(phys_mem_access_prot); static void __init *early_alloc(unsigned long sz) { void *ptr = __va(memblock_alloc(sz, sz)); + BUG_ON(!ptr); memset(ptr, 0, sz); return ptr; } -static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, +/* + * remap a PMD into pages + */ +static void split_pmd(pmd_t *pmd, pte_t *pte) +{ + unsigned long pfn = pmd_pfn(*pmd); + int i = 0; + + do { + /* + * Need to have the least restrictive permissions available + * permissions will be fixed up later + */ + set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); + pfn++; + } while (pte++, i++, i < PTRS_PER_PTE); +} + +static void alloc_init_pte(pmd_t *pmd, unsigned long addr, unsigned long end, unsigned long pfn, - pgprot_t prot) + pgprot_t prot, + void *(*alloc)(unsigned long size)) { pte_t *pte; - if (pmd_none(*pmd)) { - pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t)); + if (pmd_none(*pmd) || pmd_sect(*pmd)) { + pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); + if (pmd_sect(*pmd)) + split_pmd(pmd, pte); __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); + flush_tlb_all(); } BUG_ON(pmd_bad(*pmd)); @@ -155,30 +109,42 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr, } while (pte++, addr += PAGE_SIZE, addr != end); } -static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, - unsigned long end, phys_addr_t phys, - int map_io) +void split_pud(pud_t *old_pud, pmd_t *pmd) +{ + unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; + pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); + int i = 0; + + do { + set_pmd(pmd, __pmd(addr | prot)); + addr += PMD_SIZE; + } while (pmd++, i++, i < PTRS_PER_PMD); +} + +static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, + unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + void *(*alloc)(unsigned long size)) { pmd_t *pmd; unsigned long next; - pmdval_t prot_sect; - pgprot_t prot_pte; - - if (map_io) { - prot_sect = PROT_SECT_DEVICE_nGnRE; - prot_pte = __pgprot(PROT_DEVICE_nGnRE); - } else { - prot_sect = PROT_SECT_NORMAL_EXEC; - prot_pte = PAGE_KERNEL_EXEC; - } /* * Check for initial section mappings in the pgd/pud and remove them. */ - if (pud_none(*pud) || pud_bad(*pud)) { - pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t)); - pud_populate(&init_mm, pud, pmd); + if (pud_none(*pud) || pud_sect(*pud)) { + pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t)); + if (pud_sect(*pud)) { + /* + * need to have the 1G of mappings continue to be + * present + */ + split_pud(pud, pmd); + } + pud_populate(mm, pud, pmd); + flush_tlb_all(); } + BUG_ON(pud_bad(*pud)); pmd = pmd_offset(pud, addr); do { @@ -186,31 +152,51 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, /* try section mapping first */ if (((addr | next | phys) & ~SECTION_MASK) == 0) { pmd_t old_pmd =*pmd; - set_pmd(pmd, __pmd(phys | prot_sect)); + set_pmd(pmd, __pmd(phys | + pgprot_val(mk_sect_prot(prot)))); /* * Check for previous table entries created during * boot (__create_page_tables) and flush them. */ - if (!pmd_none(old_pmd)) + if (!pmd_none(old_pmd)) { flush_tlb_all(); + if (pmd_table(old_pmd)) { + phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0)); + if (!WARN_ON_ONCE(slab_is_available())) + memblock_free(table, PAGE_SIZE); + } + } } else { alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), - prot_pte); + prot, alloc); } phys += next - addr; } while (pmd++, addr = next, addr != end); } -static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, - unsigned long end, phys_addr_t phys, - int map_io) +static inline bool use_1G_block(unsigned long addr, unsigned long next, + unsigned long phys) +{ + if (PAGE_SHIFT != 12) + return false; + + if (((addr | next | phys) & ~PUD_MASK) != 0) + return false; + + return true; +} + +static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr, unsigned long end, + phys_addr_t phys, pgprot_t prot, + void *(*alloc)(unsigned long size)) { pud_t *pud; unsigned long next; if (pgd_none(*pgd)) { - pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t)); - pgd_populate(&init_mm, pgd, pud); + pud = alloc(PTRS_PER_PUD * sizeof(pud_t)); + pgd_populate(mm, pgd, pud); } BUG_ON(pgd_bad(*pgd)); @@ -221,10 +207,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, /* * For 4K granule only, attempt to put down a 1GB block */ - if (!map_io && (PAGE_SHIFT == 12) && - ((addr | next | phys) & ~PUD_MASK) == 0) { + if (use_1G_block(addr, next, phys)) { pud_t old_pud = *pud; - set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC)); + set_pud(pud, __pud(phys | + pgprot_val(mk_sect_prot(prot)))); /* * If we have an old value for a pud, it will @@ -234,12 +220,15 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, * Look up the old pmd table and free it. */ if (!pud_none(old_pud)) { - phys_addr_t table = __pa(pmd_offset(&old_pud, 0)); - memblock_free(table, PAGE_SIZE); flush_tlb_all(); + if (pud_table(old_pud)) { + phys_addr_t table = __pa(pmd_offset(&old_pud, 0)); + if (!WARN_ON_ONCE(slab_is_available())) + memblock_free(table, PAGE_SIZE); + } } } else { - alloc_init_pmd(pud, addr, next, phys, map_io); + alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc); } phys += next - addr; } while (pud++, addr = next, addr != end); @@ -249,9 +238,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, * Create the page directory entries and any necessary page tables for the * mapping specified by 'md'. */ -static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys, - unsigned long virt, phys_addr_t size, - int map_io) +static void __create_mapping(struct mm_struct *mm, pgd_t *pgd, + phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot, + void *(*alloc)(unsigned long size)) { unsigned long addr, length, end, next; @@ -261,31 +251,95 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys, end = addr + length; do { next = pgd_addr_end(addr, end); - alloc_init_pud(pgd, addr, next, phys, map_io); + alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc); phys += next - addr; } while (pgd++, addr = next, addr != end); } -static void __init create_mapping(phys_addr_t phys, unsigned long virt, - phys_addr_t size) +static void *late_alloc(unsigned long size) +{ + void *ptr; + + BUG_ON(size > PAGE_SIZE); + ptr = (void *)__get_free_page(PGALLOC_GFP); + BUG_ON(!ptr); + return ptr; +} + +static void __ref create_mapping(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot) { if (virt < VMALLOC_START) { pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", &phys, virt); return; } - __create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0); + __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt, + size, prot, early_alloc); } -void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io) +void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, + unsigned long virt, phys_addr_t size, + pgprot_t prot) { - if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) { - pr_warn("BUG: not creating id mapping for %pa\n", &addr); + __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot, + late_alloc); +} + +static void create_mapping_late(phys_addr_t phys, unsigned long virt, + phys_addr_t size, pgprot_t prot) +{ + if (virt < VMALLOC_START) { + pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", + &phys, virt); return; } - __create_mapping(&idmap_pg_dir[pgd_index(addr)], - addr, addr, size, map_io); + + return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), + phys, virt, size, prot, late_alloc); +} + +#ifdef CONFIG_DEBUG_RODATA +static void __init __map_memblock(phys_addr_t start, phys_addr_t end) +{ + /* + * Set up the executable regions using the existing section mappings + * for now. This will get more fine grained later once all memory + * is mapped + */ + unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE); + unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); + + if (end < kernel_x_start) { + create_mapping(start, __phys_to_virt(start), + end - start, PAGE_KERNEL); + } else if (start >= kernel_x_end) { + create_mapping(start, __phys_to_virt(start), + end - start, PAGE_KERNEL); + } else { + if (start < kernel_x_start) + create_mapping(start, __phys_to_virt(start), + kernel_x_start - start, + PAGE_KERNEL); + create_mapping(kernel_x_start, + __phys_to_virt(kernel_x_start), + kernel_x_end - kernel_x_start, + PAGE_KERNEL_EXEC); + if (kernel_x_end < end) + create_mapping(kernel_x_end, + __phys_to_virt(kernel_x_end), + end - kernel_x_end, + PAGE_KERNEL); + } + } +#else +static void __init __map_memblock(phys_addr_t start, phys_addr_t end) +{ + create_mapping(start, __phys_to_virt(start), end - start, + PAGE_KERNEL_EXEC); +} +#endif static void __init map_mem(void) { @@ -331,14 +385,53 @@ static void __init map_mem(void) memblock_set_current_limit(limit); } #endif - - create_mapping(start, __phys_to_virt(start), end - start); + __map_memblock(start, end); } /* Limit no longer required. */ memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); } +void __init fixup_executable(void) +{ +#ifdef CONFIG_DEBUG_RODATA + /* now that we are actually fully mapped, make the start/end more fine grained */ + if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) { + unsigned long aligned_start = round_down(__pa(_stext), + SECTION_SIZE); + + create_mapping(aligned_start, __phys_to_virt(aligned_start), + __pa(_stext) - aligned_start, + PAGE_KERNEL); + } + + if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) { + unsigned long aligned_end = round_up(__pa(__init_end), + SECTION_SIZE); + create_mapping(__pa(__init_end), (unsigned long)__init_end, + aligned_end - __pa(__init_end), + PAGE_KERNEL); + } +#endif +} + +#ifdef CONFIG_DEBUG_RODATA +void mark_rodata_ro(void) +{ + create_mapping_late(__pa(_stext), (unsigned long)_stext, + (unsigned long)_etext - (unsigned long)_stext, + PAGE_KERNEL_EXEC | PTE_RDONLY); + +} +#endif + +void fixup_init(void) +{ + create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin, + (unsigned long)__init_end - (unsigned long)__init_begin, + PAGE_KERNEL); +} + /* * paging_init() sets up the page tables, initialises the zone memory * maps and sets up the zero page. @@ -348,13 +441,7 @@ void __init paging_init(void) void *zero_page; map_mem(); - - /* - * Finally flush the caches and tlb to ensure that we're in a - * consistent state. - */ - flush_cache_all(); - flush_tlb_all(); + fixup_executable(); /* allocate the zero page. */ zero_page = early_alloc(PAGE_SIZE); @@ -369,6 +456,7 @@ void __init paging_init(void) */ cpu_set_reserved_ttbr0(); flush_tlb_all(); + cpu_set_default_tcr_t0sz(); } /* @@ -376,8 +464,10 @@ void __init paging_init(void) */ void setup_mm_for_reboot(void) { - cpu_switch_mm(idmap_pg_dir, &init_mm); + cpu_set_reserved_ttbr0(); flush_tlb_all(); + cpu_set_idmap_tcr_t0sz(); + cpu_switch_mm(idmap_pg_dir, &init_mm); } /* @@ -463,3 +553,93 @@ void vmemmap_free(unsigned long start, unsigned long end) { } #endif /* CONFIG_SPARSEMEM_VMEMMAP */ + +static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; +#if CONFIG_PGTABLE_LEVELS > 2 +static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss; +#endif +#if CONFIG_PGTABLE_LEVELS > 3 +static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss; +#endif + +static inline pud_t * fixmap_pud(unsigned long addr) +{ + pgd_t *pgd = pgd_offset_k(addr); + + BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); + + return pud_offset(pgd, addr); +} + +static inline pmd_t * fixmap_pmd(unsigned long addr) +{ + pud_t *pud = fixmap_pud(addr); + + BUG_ON(pud_none(*pud) || pud_bad(*pud)); + + return pmd_offset(pud, addr); +} + +static inline pte_t * fixmap_pte(unsigned long addr) +{ + pmd_t *pmd = fixmap_pmd(addr); + + BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd)); + + return pte_offset_kernel(pmd, addr); +} + +void __init early_fixmap_init(void) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + unsigned long addr = FIXADDR_START; + + pgd = pgd_offset_k(addr); + pgd_populate(&init_mm, pgd, bm_pud); + pud = pud_offset(pgd, addr); + pud_populate(&init_mm, pud, bm_pmd); + pmd = pmd_offset(pud, addr); + pmd_populate_kernel(&init_mm, pmd, bm_pte); + + /* + * The boot-ioremap range spans multiple pmds, for which + * we are not preparted: + */ + BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) + != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); + + if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) + || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { + WARN_ON(1); + pr_warn("pmd %p != %p, %p\n", + pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), + fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); + pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", + fix_to_virt(FIX_BTMAP_BEGIN)); + pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", + fix_to_virt(FIX_BTMAP_END)); + + pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); + pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); + } +} + +void __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags) +{ + unsigned long addr = __fix_to_virt(idx); + pte_t *pte; + + BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); + + pte = fixmap_pte(addr); + + if (pgprot_val(flags)) { + set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); + } else { + pte_clear(&init_mm, addr, pte); + flush_tlb_kernel_range(addr, addr+PAGE_SIZE); + } +} diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index bb0ea94c4ba1..e47ed1c5dce1 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -51,7 +51,10 @@ static int change_memory_common(unsigned long addr, int numpages, WARN_ON_ONCE(1); } - if (!is_module_address(start) || !is_module_address(end - 1)) + if (start < MODULES_VADDR || start >= MODULES_END) + return -EINVAL; + + if (end < MODULES_VADDR || end >= MODULES_END) return -EINVAL; data.set_mask = set_mask; @@ -70,7 +73,6 @@ int set_memory_ro(unsigned long addr, int numpages) __pgprot(PTE_RDONLY), __pgprot(PTE_WRITE)); } -EXPORT_SYMBOL_GPL(set_memory_ro); int set_memory_rw(unsigned long addr, int numpages) { @@ -78,7 +80,6 @@ int set_memory_rw(unsigned long addr, int numpages) __pgprot(PTE_WRITE), __pgprot(PTE_RDONLY)); } -EXPORT_SYMBOL_GPL(set_memory_rw); int set_memory_nx(unsigned long addr, int numpages) { diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c index 6682b361d3ac..71ca104f97bd 100644 --- a/arch/arm64/mm/pgd.c +++ b/arch/arm64/mm/pgd.c @@ -35,9 +35,9 @@ static struct kmem_cache *pgd_cache; pgd_t *pgd_alloc(struct mm_struct *mm) { if (PGD_SIZE == PAGE_SIZE) - return (pgd_t *)get_zeroed_page(GFP_KERNEL); + return (pgd_t *)__get_free_page(PGALLOC_GFP); else - return kmem_cache_zalloc(pgd_cache, GFP_KERNEL); + return kmem_cache_alloc(pgd_cache, PGALLOC_GFP); } void pgd_free(struct mm_struct *mm, pgd_t *pgd) diff --git a/arch/arm64/mm/proc-macros.S b/arch/arm64/mm/proc-macros.S index 005d29e2977d..4c4d93c4bf65 100644 --- a/arch/arm64/mm/proc-macros.S +++ b/arch/arm64/mm/proc-macros.S @@ -52,3 +52,13 @@ mov \reg, #4 // bytes per word lsl \reg, \reg, \tmp // actual cache line size .endm + +/* + * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map + */ + .macro tcr_set_idmap_t0sz, valreg, tmpreg +#ifndef CONFIG_ARM64_VA_BITS_48 + ldr_l \tmpreg, idmap_t0sz + bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH +#endif + .endm diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 4e778b13291b..cdd754e19b9b 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -102,7 +102,7 @@ ENTRY(cpu_do_idle) ret ENDPROC(cpu_do_idle) -#ifdef CONFIG_ARM64_CPU_SUSPEND +#ifdef CONFIG_CPU_PM /** * cpu_do_suspend - save CPU registers context * @@ -156,6 +156,7 @@ ENTRY(cpu_do_resume) msr cpacr_el1, x6 msr ttbr0_el1, x1 msr ttbr1_el1, x7 + tcr_set_idmap_t0sz x8, x7 msr tcr_el1, x8 msr vbar_el1, x9 msr mdscr_el1, x10 @@ -233,6 +234,8 @@ ENTRY(__cpu_setup) */ ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 + tcr_set_idmap_t0sz x10, x9 + /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in * TCR_EL1. @@ -244,14 +247,18 @@ ENTRY(__cpu_setup) ENDPROC(__cpu_setup) /* + * We set the desired value explicitly, including those of the + * reserved bits. The values of bits EE & E0E were set early in + * el2_setup, which are left untouched below. + * * n n T * U E WT T UD US IHBS * CE0 XWHW CZ ME TEEA S * .... .IEE .... NEAI TE.I ..AD DEN0 ACAM - * 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved - * .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings + * 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved + * .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings */ .type crval, #object crval: - .word 0x000802e2 // clear - .word 0x0405d11d // set + .word 0xfcffffff // clear + .word 0x34d5d91d // set diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 41f1e3e2ea24..dc6a4842683a 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -60,7 +60,7 @@ struct jit_ctx { const struct bpf_prog *prog; int idx; int tmp_used; - int body_offset; + int epilogue_offset; int *offset; u32 *image; }; @@ -130,8 +130,8 @@ static void jit_fill_hole(void *area, unsigned int size) static inline int epilogue_offset(const struct jit_ctx *ctx) { - int to = ctx->offset[ctx->prog->len - 1]; - int from = ctx->idx - ctx->body_offset; + int to = ctx->epilogue_offset; + int from = ctx->idx; return to - from; } @@ -463,6 +463,8 @@ emit_cond_jmp: } /* function return */ case BPF_JMP | BPF_EXIT: + /* Optimization: when last instruction is EXIT, + simply fallthrough to epilogue. */ if (i == ctx->prog->len - 1) break; jmp_offset = epilogue_offset(ctx); @@ -485,7 +487,7 @@ emit_cond_jmp: return -EINVAL; } - imm64 = (u64)insn1.imm << 32 | imm; + imm64 = (u64)insn1.imm << 32 | (u32)imm; emit_a64_mov_i64(dst, imm64, ctx); return 1; @@ -685,11 +687,13 @@ void bpf_int_jit_compile(struct bpf_prog *prog) /* 1. Initial fake pass to compute ctx->idx. */ - /* Fake pass to fill in ctx->offset. */ + /* Fake pass to fill in ctx->offset and ctx->tmp_used. */ if (build_body(&ctx)) goto out; build_prologue(&ctx); + + ctx.epilogue_offset = ctx.idx; build_epilogue(&ctx); /* Now we know the actual image size. */ @@ -706,7 +710,6 @@ void bpf_int_jit_compile(struct bpf_prog *prog) build_prologue(&ctx); - ctx.body_offset = ctx.idx; if (build_body(&ctx)) { bpf_jit_binary_free(header); goto out; |