diff options
Diffstat (limited to 'arch')
737 files changed, 23836 insertions, 22109 deletions
diff --git a/arch/Kconfig b/arch/Kconfig index e1068987bad1..a65eafb24997 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -32,7 +32,7 @@ config HAVE_OPROFILE config OPROFILE_NMI_TIMER def_bool y - depends on PERF_EVENTS && HAVE_PERF_EVENTS_NMI + depends on PERF_EVENTS && HAVE_PERF_EVENTS_NMI && !PPC64 config KPROBES bool "Kprobes" diff --git a/arch/alpha/include/asm/processor.h b/arch/alpha/include/asm/processor.h index b4cf03690394..43a7559c448b 100644 --- a/arch/alpha/include/asm/processor.h +++ b/arch/alpha/include/asm/processor.h @@ -44,6 +44,7 @@ struct task_struct; extern unsigned long thread_saved_pc(struct task_struct *); /* Do necessary setup to start up a newly executed thread. */ +struct pt_regs; extern void start_thread(struct pt_regs *, unsigned long, unsigned long); /* Free all resources held by a thread. */ diff --git a/arch/alpha/include/asm/thread_info.h b/arch/alpha/include/asm/thread_info.h index d5b98ab514bb..32e920a83ae5 100644 --- a/arch/alpha/include/asm/thread_info.h +++ b/arch/alpha/include/asm/thread_info.h @@ -18,7 +18,6 @@ struct thread_info { unsigned int flags; /* low level flags */ unsigned int ieee_state; /* see fpu.h */ - struct exec_domain *exec_domain; /* execution domain */ mm_segment_t addr_limit; /* thread address space */ unsigned cpu; /* current CPU */ int preempt_count; /* 0 => preemptable, <0 => BUG */ @@ -35,7 +34,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .addr_limit = KERNEL_DS, \ .preempt_count = INIT_PREEMPT_COUNT, \ } diff --git a/arch/arc/include/asm/thread_info.h b/arch/arc/include/asm/thread_info.h index 1163a1838ac1..aca0d5a45c7b 100644 --- a/arch/arc/include/asm/thread_info.h +++ b/arch/arc/include/asm/thread_info.h @@ -43,7 +43,6 @@ struct thread_info { int preempt_count; /* 0 => preemptable, <0 => BUG */ struct task_struct *task; /* main task structure */ mm_segment_t addr_limit; /* thread address space */ - struct exec_domain *exec_domain;/* execution domain */ __u32 cpu; /* current CPU */ unsigned long thr_ptr; /* TLS ptr */ }; @@ -56,7 +55,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/arc/kernel/signal.c b/arch/arc/kernel/signal.c index edda76fae83f..2251fb4bbfd7 100644 --- a/arch/arc/kernel/signal.c +++ b/arch/arc/kernel/signal.c @@ -171,18 +171,6 @@ static inline void __user *get_sigframe(struct ksignal *ksig, return frame; } -/* - * translate the signal - */ -static inline int map_sig(int sig) -{ - struct thread_info *thread = current_thread_info(); - if (thread->exec_domain && thread->exec_domain->signal_invmap - && sig < 32) - sig = thread->exec_domain->signal_invmap[sig]; - return sig; -} - static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { @@ -231,7 +219,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) return err; /* #1 arg to the user Signal handler */ - regs->r0 = map_sig(ksig->sig); + regs->r0 = ksig->sig; /* setup PC of user space signal handler */ regs->ret = (unsigned long)ksig->ka.sa.sa_handler; diff --git a/arch/arc/kernel/troubleshoot.c b/arch/arc/kernel/troubleshoot.c index 1badf9b84b51..e00a01879025 100644 --- a/arch/arc/kernel/troubleshoot.c +++ b/arch/arc/kernel/troubleshoot.c @@ -52,7 +52,7 @@ static void show_callee_regs(struct callee_regs *cregs) print_reg_file(&(cregs->r13), 13); } -void print_task_path_n_nm(struct task_struct *tsk, char *buf) +static void print_task_path_n_nm(struct task_struct *tsk, char *buf) { struct path path; char *path_nm = NULL; @@ -77,7 +77,6 @@ void print_task_path_n_nm(struct task_struct *tsk, char *buf) done: pr_info("Path: %s\n", path_nm); } -EXPORT_SYMBOL(print_task_path_n_nm); static void show_faulting_vma(unsigned long address, char *buf) { diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 4b62f4caf0ce..392e7ae69452 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -21,6 +21,7 @@ config ARM select GENERIC_IDLE_POLL_SETUP select GENERIC_IRQ_PROBE select GENERIC_IRQ_SHOW + select GENERIC_IRQ_SHOW_LEVEL select GENERIC_PCI_IOMAP select GENERIC_SCHED_CLOCK select GENERIC_SMP_IDLE_THREAD @@ -1063,7 +1064,7 @@ config ARM_ERRATA_430973 depends on CPU_V7 help This option enables the workaround for the 430973 Cortex-A8 - (r1p0..r1p2) erratum. If a code sequence containing an ARM/Thumb + r1p* erratum. If a code sequence containing an ARM/Thumb interworking branch is replaced with another code sequence at the same virtual address, whether due to self-modifying code or virtual to physical address re-mapping, Cortex-A8 does not recover from the @@ -1132,6 +1133,7 @@ config ARM_ERRATA_742231 config ARM_ERRATA_643719 bool "ARM errata: LoUIS bit field in CLIDR register is incorrect" depends on CPU_V7 && SMP + default y help This option enables the workaround for the 643719 Cortex-A9 (prior to r1p0) erratum. On affected cores the LoUIS bit field of the CLIDR @@ -1349,7 +1351,7 @@ config SMP If you don't know what to do here, say N. config SMP_ON_UP - bool "Allow booting SMP kernel on uniprocessor systems (EXPERIMENTAL)" + bool "Allow booting SMP kernel on uniprocessor systems" depends on SMP && !XIP_KERNEL && MMU default y help @@ -2131,16 +2133,6 @@ menu "Userspace binary formats" source "fs/Kconfig.binfmt" -config ARTHUR - tristate "RISC OS personality" - depends on !AEABI - help - Say Y here to include the kernel code necessary if you want to run - Acorn RISC OS/Arthur binaries under Linux. This code is still very - experimental; if this sounds frightening, say N and sleep in peace. - You can also say M here to compile this support as a module (which - will be called arthur). - endmenu menu "Power management options" @@ -2173,6 +2165,9 @@ source "arch/arm/Kconfig.debug" source "security/Kconfig" source "crypto/Kconfig" +if CRYPTO +source "arch/arm/crypto/Kconfig" +endif source "lib/Kconfig" diff --git a/arch/arm/Makefile b/arch/arm/Makefile index eb7bb511f853..5575d9fa8806 100644 --- a/arch/arm/Makefile +++ b/arch/arm/Makefile @@ -13,7 +13,7 @@ # Ensure linker flags are correct LDFLAGS := -LDFLAGS_vmlinux :=-p --no-undefined -X +LDFLAGS_vmlinux :=-p --no-undefined -X --pic-veneer ifeq ($(CONFIG_CPU_ENDIAN_BE8),y) LDFLAGS_vmlinux += --be8 LDFLAGS_MODULE += --be8 @@ -264,6 +264,7 @@ core-$(CONFIG_FPE_FASTFPE) += $(FASTFPE_OBJ) core-$(CONFIG_VFP) += arch/arm/vfp/ core-$(CONFIG_XEN) += arch/arm/xen/ core-$(CONFIG_KVM_ARM_HOST) += arch/arm/kvm/ +core-$(CONFIG_VDSO) += arch/arm/vdso/ # If we have a machine-specific directory, then include it in the build. core-y += arch/arm/kernel/ arch/arm/mm/ arch/arm/common/ @@ -321,6 +322,12 @@ dtbs: prepare scripts dtbs_install: $(Q)$(MAKE) $(dtbinst)=$(boot)/dts +PHONY += vdso_install +vdso_install: +ifeq ($(CONFIG_VDSO),y) + $(Q)$(MAKE) $(build)=arch/arm/vdso $@ +endif + # We use MRPROPER_FILES and CLEAN_FILES now archclean: $(Q)$(MAKE) $(clean)=$(boot) @@ -345,4 +352,5 @@ define archhelp echo ' Install using (your) ~/bin/$(INSTALLKERNEL) or' echo ' (distribution) /sbin/$(INSTALLKERNEL) or' echo ' install to $$(INSTALL_PATH) and run lilo' + echo ' vdso_install - Install unstripped vdso.so to $$(INSTALL_MOD_PATH)/vdso' endef diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile index ec2f8065f955..9eca7aee927f 100644 --- a/arch/arm/boot/Makefile +++ b/arch/arm/boot/Makefile @@ -12,7 +12,7 @@ # ifneq ($(MACHINE),) -include $(srctree)/$(MACHINE)/Makefile.boot +include $(MACHINE)/Makefile.boot endif # Note: the following conditions must always be true: diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index c41a793b519c..2c45b5709fa4 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -10,8 +10,11 @@ */ #include <linux/linkage.h> #include <asm/assembler.h> +#include <asm/v7m.h> + + AR_CLASS( .arch armv7-a ) + M_CLASS( .arch armv7-m ) - .arch armv7-a /* * Debugging stuff * @@ -114,7 +117,12 @@ * sort out different calling conventions */ .align - .arm @ Always enter in ARM state + /* + * Always enter in ARM state for CPUs that support the ARM ISA. + * As of today (2014) that's exactly the members of the A and R + * classes. + */ + AR_CLASS( .arm ) start: .type start,#function .rept 7 @@ -132,14 +140,15 @@ start: THUMB( .thumb ) 1: - ARM_BE8( setend be ) @ go BE8 if compiled for BE8 - mrs r9, cpsr + ARM_BE8( setend be ) @ go BE8 if compiled for BE8 + AR_CLASS( mrs r9, cpsr ) #ifdef CONFIG_ARM_VIRT_EXT bl __hyp_stub_install @ get into SVC mode, reversibly #endif mov r7, r1 @ save architecture ID mov r8, r2 @ save atags pointer +#ifndef CONFIG_CPU_V7M /* * Booting from Angel - need to enter SVC mode and disable * FIQs/IRQs (numeric definitions from angel arm.h source). @@ -155,6 +164,7 @@ not_angel: safe_svcmode_maskall r0 msr spsr_cxsf, r9 @ Save the CPU boot mode in @ SPSR +#endif /* * Note that some cache flushing and other stuff may * be needed here - is there an Angel SWI call for this? @@ -168,9 +178,26 @@ not_angel: .text #ifdef CONFIG_AUTO_ZRELADDR - @ determine final kernel image address + /* + * Find the start of physical memory. As we are executing + * without the MMU on, we are in the physical address space. + * We just need to get rid of any offset by aligning the + * address. + * + * This alignment is a balance between the requirements of + * different platforms - we have chosen 128MB to allow + * platforms which align the start of their physical memory + * to 128MB to use this feature, while allowing the zImage + * to be placed within the first 128MB of memory on other + * platforms. Increasing the alignment means we place + * stricter alignment requirements on the start of physical + * memory, but relaxing it means that we break people who + * are already placing their zImage in (eg) the top 64MB + * of this range. + */ mov r4, pc and r4, r4, #0xf8000000 + /* Determine final kernel image address. */ add r4, r4, #TEXT_OFFSET #else ldr r4, =zreladdr @@ -810,6 +837,16 @@ __common_mmu_cache_on: call_cache_fn: adr r12, proc_types #ifdef CONFIG_CPU_CP15 mrc p15, 0, r9, c0, c0 @ get processor ID +#elif defined(CONFIG_CPU_V7M) + /* + * On v7-M the processor id is located in the V7M_SCB_CPUID + * register, but as cache handling is IMPLEMENTATION DEFINED on + * v7-M (if existant at all) we just return early here. + * If V7M_SCB_CPUID were used the cpu ID functions (i.e. + * __armv7_mmu_cache_{on,off,flush}) would be selected which + * use cp15 registers that are not implemented on v7-M. + */ + bx lr #else ldr r9, =CONFIG_PROCESSOR_ID #endif @@ -1310,8 +1347,9 @@ __hyp_reentry_vectors: __enter_kernel: mov r0, #0 @ must be 0 - ARM( mov pc, r4 ) @ call kernel - THUMB( bx r4 ) @ entry point is always ARM + ARM( mov pc, r4 ) @ call kernel + M_CLASS( add r4, r4, #1 ) @ enter in Thumb mode for M class + THUMB( bx r4 ) @ entry point is always ARM for A/R classes reloc_code_end: diff --git a/arch/arm/boot/dts/at91sam9260.dtsi b/arch/arm/boot/dts/at91sam9260.dtsi index e7f0a4ae271c..62d25b14deb8 100644 --- a/arch/arm/boot/dts/at91sam9260.dtsi +++ b/arch/arm/boot/dts/at91sam9260.dtsi @@ -842,7 +842,7 @@ }; macb0: ethernet@fffc4000 { - compatible = "cdns,at32ap7000-macb", "cdns,macb"; + compatible = "cdns,at91sam9260-macb", "cdns,macb"; reg = <0xfffc4000 0x100>; interrupts = <21 IRQ_TYPE_LEVEL_HIGH 3>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/at91sam9263.dtsi b/arch/arm/boot/dts/at91sam9263.dtsi index fce301c4e9d6..e4f61a979a57 100644 --- a/arch/arm/boot/dts/at91sam9263.dtsi +++ b/arch/arm/boot/dts/at91sam9263.dtsi @@ -845,7 +845,7 @@ }; macb0: ethernet@fffbc000 { - compatible = "cdns,at32ap7000-macb", "cdns,macb"; + compatible = "cdns,at91sam9260-macb", "cdns,macb"; reg = <0xfffbc000 0x100>; interrupts = <21 IRQ_TYPE_LEVEL_HIGH 3>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/at91sam9g45.dtsi b/arch/arm/boot/dts/at91sam9g45.dtsi index 488af63d5174..8ec05b11298a 100644 --- a/arch/arm/boot/dts/at91sam9g45.dtsi +++ b/arch/arm/boot/dts/at91sam9g45.dtsi @@ -956,7 +956,7 @@ }; macb0: ethernet@fffbc000 { - compatible = "cdns,at32ap7000-macb", "cdns,macb"; + compatible = "cdns,at91sam9260-macb", "cdns,macb"; reg = <0xfffbc000 0x100>; interrupts = <25 IRQ_TYPE_LEVEL_HIGH 3>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/at91sam9x5_macb0.dtsi b/arch/arm/boot/dts/at91sam9x5_macb0.dtsi index 57e89d1d0325..73d7e30965ba 100644 --- a/arch/arm/boot/dts/at91sam9x5_macb0.dtsi +++ b/arch/arm/boot/dts/at91sam9x5_macb0.dtsi @@ -53,7 +53,7 @@ }; macb0: ethernet@f802c000 { - compatible = "cdns,at32ap7000-macb", "cdns,macb"; + compatible = "cdns,at91sam9260-macb", "cdns,macb"; reg = <0xf802c000 0x100>; interrupts = <24 IRQ_TYPE_LEVEL_HIGH 3>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/at91sam9x5_macb1.dtsi b/arch/arm/boot/dts/at91sam9x5_macb1.dtsi index 663676c02861..d81980c40c7d 100644 --- a/arch/arm/boot/dts/at91sam9x5_macb1.dtsi +++ b/arch/arm/boot/dts/at91sam9x5_macb1.dtsi @@ -41,7 +41,7 @@ }; macb1: ethernet@f8030000 { - compatible = "cdns,at32ap7000-macb", "cdns,macb"; + compatible = "cdns,at91sam9260-macb", "cdns,macb"; reg = <0xf8030000 0x100>; interrupts = <27 IRQ_TYPE_LEVEL_HIGH 3>; pinctrl-names = "default"; diff --git a/arch/arm/boot/dts/sama5d3_emac.dtsi b/arch/arm/boot/dts/sama5d3_emac.dtsi index fe2af9276312..b4544cf11bad 100644 --- a/arch/arm/boot/dts/sama5d3_emac.dtsi +++ b/arch/arm/boot/dts/sama5d3_emac.dtsi @@ -41,7 +41,7 @@ }; macb1: ethernet@f802c000 { - compatible = "cdns,at32ap7000-macb", "cdns,macb"; + compatible = "cdns,at91sam9260-macb", "cdns,macb"; reg = <0xf802c000 0x100>; interrupts = <35 IRQ_TYPE_LEVEL_HIGH 3>; pinctrl-names = "default"; diff --git a/arch/arm/configs/badge4_defconfig b/arch/arm/configs/badge4_defconfig index 0494c8f229a2..d59009878312 100644 --- a/arch/arm/configs/badge4_defconfig +++ b/arch/arm/configs/badge4_defconfig @@ -12,7 +12,6 @@ CONFIG_CPU_FREQ_GOV_PERFORMANCE=y CONFIG_FPE_NWFPE=y CONFIG_BINFMT_AOUT=m CONFIG_BINFMT_MISC=m -CONFIG_ARTHUR=m CONFIG_NET=y CONFIG_PACKET=y CONFIG_UNIX=y diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig new file mode 100644 index 000000000000..8da2207b0072 --- /dev/null +++ b/arch/arm/crypto/Kconfig @@ -0,0 +1,130 @@ + +menuconfig ARM_CRYPTO + bool "ARM Accelerated Cryptographic Algorithms" + depends on ARM + help + Say Y here to choose from a selection of cryptographic algorithms + implemented using ARM specific CPU features or instructions. + +if ARM_CRYPTO + +config CRYPTO_SHA1_ARM + tristate "SHA1 digest algorithm (ARM-asm)" + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using optimized ARM assembler. + +config CRYPTO_SHA1_ARM_NEON + tristate "SHA1 digest algorithm (ARM NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_SHA1_ARM + select CRYPTO_SHA1 + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using optimized ARM NEON assembly, when NEON instructions are + available. + +config CRYPTO_SHA1_ARM_CE + tristate "SHA1 digest algorithm (ARM v8 Crypto Extensions)" + depends on KERNEL_MODE_NEON + select CRYPTO_SHA1_ARM + select CRYPTO_HASH + help + SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented + using special ARMv8 Crypto Extensions. + +config CRYPTO_SHA2_ARM_CE + tristate "SHA-224/256 digest algorithm (ARM v8 Crypto Extensions)" + depends on KERNEL_MODE_NEON + select CRYPTO_SHA256_ARM + select CRYPTO_HASH + help + SHA-256 secure hash standard (DFIPS 180-2) implemented + using special ARMv8 Crypto Extensions. + +config CRYPTO_SHA256_ARM + tristate "SHA-224/256 digest algorithm (ARM-asm and NEON)" + select CRYPTO_HASH + depends on !CPU_V7M + help + SHA-256 secure hash standard (DFIPS 180-2) implemented + using optimized ARM assembler and NEON, when available. + +config CRYPTO_SHA512_ARM_NEON + tristate "SHA384 and SHA512 digest algorithm (ARM NEON)" + depends on KERNEL_MODE_NEON + select CRYPTO_SHA512 + select CRYPTO_HASH + help + SHA-512 secure hash standard (DFIPS 180-2) implemented + using ARM NEON instructions, when available. + + This version of SHA implements a 512 bit hash with 256 bits of + security against collision attacks. + + This code also includes SHA-384, a 384 bit hash with 192 bits + of security against collision attacks. + +config CRYPTO_AES_ARM + tristate "AES cipher algorithms (ARM-asm)" + depends on ARM + select CRYPTO_ALGAPI + select CRYPTO_AES + help + Use optimized AES assembler routines for ARM platforms. + + AES cipher algorithms (FIPS-197). AES uses the Rijndael + algorithm. + + Rijndael appears to be consistently a very good performer in + both hardware and software across a wide range of computing + environments regardless of its use in feedback or non-feedback + modes. Its key setup time is excellent, and its key agility is + good. Rijndael's very low memory requirements make it very well + suited for restricted-space environments, in which it also + demonstrates excellent performance. Rijndael's operations are + among the easiest to defend against power and timing attacks. + + The AES specifies three key sizes: 128, 192 and 256 bits + + See <http://csrc.nist.gov/encryption/aes/> for more information. + +config CRYPTO_AES_ARM_BS + tristate "Bit sliced AES using NEON instructions" + depends on KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_AES_ARM + select CRYPTO_ABLK_HELPER + help + Use a faster and more secure NEON based implementation of AES in CBC, + CTR and XTS modes + + Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode + and for XTS mode encryption, CBC and XTS mode decryption speedup is + around 25%. (CBC encryption speed is not affected by this driver.) + This implementation does not rely on any lookup tables so it is + believed to be invulnerable to cache timing attacks. + +config CRYPTO_AES_ARM_CE + tristate "Accelerated AES using ARMv8 Crypto Extensions" + depends on KERNEL_MODE_NEON + select CRYPTO_ALGAPI + select CRYPTO_ABLK_HELPER + help + Use an implementation of AES in CBC, CTR and XTS modes that uses + ARMv8 Crypto Extensions + +config CRYPTO_GHASH_ARM_CE + tristate "PMULL-accelerated GHASH using ARMv8 Crypto Extensions" + depends on KERNEL_MODE_NEON + select CRYPTO_HASH + select CRYPTO_CRYPTD + help + Use an implementation of GHASH (used by the GCM AEAD chaining mode) + that uses the 64x64 to 128 bit polynomial multiplication (vmull.p64) + that is part of the ARMv8 Crypto Extensions + +endif diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index b48fa341648d..6ea828241fcb 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -6,13 +6,35 @@ obj-$(CONFIG_CRYPTO_AES_ARM) += aes-arm.o obj-$(CONFIG_CRYPTO_AES_ARM_BS) += aes-arm-bs.o obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o +obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o obj-$(CONFIG_CRYPTO_SHA512_ARM_NEON) += sha512-arm-neon.o +ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o +ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o +ce-obj-$(CONFIG_CRYPTO_SHA2_ARM_CE) += sha2-arm-ce.o +ce-obj-$(CONFIG_CRYPTO_GHASH_ARM_CE) += ghash-arm-ce.o + +ifneq ($(ce-obj-y)$(ce-obj-m),) +ifeq ($(call as-instr,.fpu crypto-neon-fp-armv8,y,n),y) +obj-y += $(ce-obj-y) +obj-m += $(ce-obj-m) +else +$(warning These ARMv8 Crypto Extensions modules need binutils 2.23 or higher) +$(warning $(ce-obj-y) $(ce-obj-m)) +endif +endif + aes-arm-y := aes-armv4.o aes_glue.o aes-arm-bs-y := aesbs-core.o aesbs-glue.o sha1-arm-y := sha1-armv4-large.o sha1_glue.o sha1-arm-neon-y := sha1-armv7-neon.o sha1_neon_glue.o +sha256-arm-neon-$(CONFIG_KERNEL_MODE_NEON) := sha256_neon_glue.o +sha256-arm-y := sha256-core.o sha256_glue.o $(sha256-arm-neon-y) sha512-arm-neon-y := sha512-armv7-neon.o sha512_neon_glue.o +sha1-arm-ce-y := sha1-ce-core.o sha1-ce-glue.o +sha2-arm-ce-y := sha2-ce-core.o sha2-ce-glue.o +aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o +ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) @@ -20,4 +42,7 @@ quiet_cmd_perl = PERL $@ $(src)/aesbs-core.S_shipped: $(src)/bsaes-armv7.pl $(call cmd,perl) -.PRECIOUS: $(obj)/aesbs-core.S +$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl + $(call cmd,perl) + +.PRECIOUS: $(obj)/aesbs-core.S $(obj)/sha256-core.S diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S new file mode 100644 index 000000000000..8cfa468ee570 --- /dev/null +++ b/arch/arm/crypto/aes-ce-core.S @@ -0,0 +1,518 @@ +/* + * aes-ce-core.S - AES in CBC/CTR/XTS mode using ARMv8 Crypto Extensions + * + * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .text + .fpu crypto-neon-fp-armv8 + .align 3 + + .macro enc_round, state, key + aese.8 \state, \key + aesmc.8 \state, \state + .endm + + .macro dec_round, state, key + aesd.8 \state, \key + aesimc.8 \state, \state + .endm + + .macro enc_dround, key1, key2 + enc_round q0, \key1 + enc_round q0, \key2 + .endm + + .macro dec_dround, key1, key2 + dec_round q0, \key1 + dec_round q0, \key2 + .endm + + .macro enc_fround, key1, key2, key3 + enc_round q0, \key1 + aese.8 q0, \key2 + veor q0, q0, \key3 + .endm + + .macro dec_fround, key1, key2, key3 + dec_round q0, \key1 + aesd.8 q0, \key2 + veor q0, q0, \key3 + .endm + + .macro enc_dround_3x, key1, key2 + enc_round q0, \key1 + enc_round q1, \key1 + enc_round q2, \key1 + enc_round q0, \key2 + enc_round q1, \key2 + enc_round q2, \key2 + .endm + + .macro dec_dround_3x, key1, key2 + dec_round q0, \key1 + dec_round q1, \key1 + dec_round q2, \key1 + dec_round q0, \key2 + dec_round q1, \key2 + dec_round q2, \key2 + .endm + + .macro enc_fround_3x, key1, key2, key3 + enc_round q0, \key1 + enc_round q1, \key1 + enc_round q2, \key1 + aese.8 q0, \key2 + aese.8 q1, \key2 + aese.8 q2, \key2 + veor q0, q0, \key3 + veor q1, q1, \key3 + veor q2, q2, \key3 + .endm + + .macro dec_fround_3x, key1, key2, key3 + dec_round q0, \key1 + dec_round q1, \key1 + dec_round q2, \key1 + aesd.8 q0, \key2 + aesd.8 q1, \key2 + aesd.8 q2, \key2 + veor q0, q0, \key3 + veor q1, q1, \key3 + veor q2, q2, \key3 + .endm + + .macro do_block, dround, fround + cmp r3, #12 @ which key size? + vld1.8 {q10-q11}, [ip]! + \dround q8, q9 + vld1.8 {q12-q13}, [ip]! + \dround q10, q11 + vld1.8 {q10-q11}, [ip]! + \dround q12, q13 + vld1.8 {q12-q13}, [ip]! + \dround q10, q11 + blo 0f @ AES-128: 10 rounds + vld1.8 {q10-q11}, [ip]! + beq 1f @ AES-192: 12 rounds + \dround q12, q13 + vld1.8 {q12-q13}, [ip] + \dround q10, q11 +0: \fround q12, q13, q14 + bx lr + +1: \dround q12, q13 + \fround q10, q11, q14 + bx lr + .endm + + /* + * Internal, non-AAPCS compliant functions that implement the core AES + * transforms. These should preserve all registers except q0 - q2 and ip + * Arguments: + * q0 : first in/output block + * q1 : second in/output block (_3x version only) + * q2 : third in/output block (_3x version only) + * q8 : first round key + * q9 : secound round key + * ip : address of 3rd round key + * q14 : final round key + * r3 : number of rounds + */ + .align 6 +aes_encrypt: + add ip, r2, #32 @ 3rd round key +.Laes_encrypt_tweak: + do_block enc_dround, enc_fround +ENDPROC(aes_encrypt) + + .align 6 +aes_decrypt: + add ip, r2, #32 @ 3rd round key + do_block dec_dround, dec_fround +ENDPROC(aes_decrypt) + + .align 6 +aes_encrypt_3x: + add ip, r2, #32 @ 3rd round key + do_block enc_dround_3x, enc_fround_3x +ENDPROC(aes_encrypt_3x) + + .align 6 +aes_decrypt_3x: + add ip, r2, #32 @ 3rd round key + do_block dec_dround_3x, dec_fround_3x +ENDPROC(aes_decrypt_3x) + + .macro prepare_key, rk, rounds + add ip, \rk, \rounds, lsl #4 + vld1.8 {q8-q9}, [\rk] @ load first 2 round keys + vld1.8 {q14}, [ip] @ load last round key + .endm + + /* + * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks) + * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks) + */ +ENTRY(ce_aes_ecb_encrypt) + push {r4, lr} + ldr r4, [sp, #8] + prepare_key r2, r3 +.Lecbencloop3x: + subs r4, r4, #3 + bmi .Lecbenc1x + vld1.8 {q0-q1}, [r1, :64]! + vld1.8 {q2}, [r1, :64]! + bl aes_encrypt_3x + vst1.8 {q0-q1}, [r0, :64]! + vst1.8 {q2}, [r0, :64]! + b .Lecbencloop3x +.Lecbenc1x: + adds r4, r4, #3 + beq .Lecbencout +.Lecbencloop: + vld1.8 {q0}, [r1, :64]! + bl aes_encrypt + vst1.8 {q0}, [r0, :64]! + subs r4, r4, #1 + bne .Lecbencloop +.Lecbencout: + pop {r4, pc} +ENDPROC(ce_aes_ecb_encrypt) + +ENTRY(ce_aes_ecb_decrypt) + push {r4, lr} + ldr r4, [sp, #8] + prepare_key r2, r3 +.Lecbdecloop3x: + subs r4, r4, #3 + bmi .Lecbdec1x + vld1.8 {q0-q1}, [r1, :64]! + vld1.8 {q2}, [r1, :64]! + bl aes_decrypt_3x + vst1.8 {q0-q1}, [r0, :64]! + vst1.8 {q2}, [r0, :64]! + b .Lecbdecloop3x +.Lecbdec1x: + adds r4, r4, #3 + beq .Lecbdecout +.Lecbdecloop: + vld1.8 {q0}, [r1, :64]! + bl aes_decrypt + vst1.8 {q0}, [r0, :64]! + subs r4, r4, #1 + bne .Lecbdecloop +.Lecbdecout: + pop {r4, pc} +ENDPROC(ce_aes_ecb_decrypt) + + /* + * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks, u8 iv[]) + * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks, u8 iv[]) + */ +ENTRY(ce_aes_cbc_encrypt) + push {r4-r6, lr} + ldrd r4, r5, [sp, #16] + vld1.8 {q0}, [r5] + prepare_key r2, r3 +.Lcbcencloop: + vld1.8 {q1}, [r1, :64]! @ get next pt block + veor q0, q0, q1 @ ..and xor with iv + bl aes_encrypt + vst1.8 {q0}, [r0, :64]! + subs r4, r4, #1 + bne .Lcbcencloop + vst1.8 {q0}, [r5] + pop {r4-r6, pc} +ENDPROC(ce_aes_cbc_encrypt) + +ENTRY(ce_aes_cbc_decrypt) + push {r4-r6, lr} + ldrd r4, r5, [sp, #16] + vld1.8 {q6}, [r5] @ keep iv in q6 + prepare_key r2, r3 +.Lcbcdecloop3x: + subs r4, r4, #3 + bmi .Lcbcdec1x + vld1.8 {q0-q1}, [r1, :64]! + vld1.8 {q2}, [r1, :64]! + vmov q3, q0 + vmov q4, q1 + vmov q5, q2 + bl aes_decrypt_3x + veor q0, q0, q6 + veor q1, q1, q3 + veor q2, q2, q4 + vmov q6, q5 + vst1.8 {q0-q1}, [r0, :64]! + vst1.8 {q2}, [r0, :64]! + b .Lcbcdecloop3x +.Lcbcdec1x: + adds r4, r4, #3 + beq .Lcbcdecout + vmov q15, q14 @ preserve last round key +.Lcbcdecloop: + vld1.8 {q0}, [r1, :64]! @ get next ct block + veor q14, q15, q6 @ combine prev ct with last key + vmov q6, q0 + bl aes_decrypt + vst1.8 {q0}, [r0, :64]! + subs r4, r4, #1 + bne .Lcbcdecloop +.Lcbcdecout: + vst1.8 {q6}, [r5] @ keep iv in q6 + pop {r4-r6, pc} +ENDPROC(ce_aes_cbc_decrypt) + + /* + * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, + * int blocks, u8 ctr[]) + */ +ENTRY(ce_aes_ctr_encrypt) + push {r4-r6, lr} + ldrd r4, r5, [sp, #16] + vld1.8 {q6}, [r5] @ load ctr + prepare_key r2, r3 + vmov r6, s27 @ keep swabbed ctr in r6 + rev r6, r6 + cmn r6, r4 @ 32 bit overflow? + bcs .Lctrloop +.Lctrloop3x: + subs r4, r4, #3 + bmi .Lctr1x + add r6, r6, #1 + vmov q0, q6 + vmov q1, q6 + rev ip, r6 + add r6, r6, #1 + vmov q2, q6 + vmov s7, ip + rev ip, r6 + add r6, r6, #1 + vmov s11, ip + vld1.8 {q3-q4}, [r1, :64]! + vld1.8 {q5}, [r1, :64]! + bl aes_encrypt_3x + veor q0, q0, q3 + veor q1, q1, q4 + veor q2, q2, q5 + rev ip, r6 + vst1.8 {q0-q1}, [r0, :64]! + vst1.8 {q2}, [r0, :64]! + vmov s27, ip + b .Lctrloop3x +.Lctr1x: + adds r4, r4, #3 + beq .Lctrout +.Lctrloop: + vmov q0, q6 + bl aes_encrypt + subs r4, r4, #1 + bmi .Lctrhalfblock @ blocks < 0 means 1/2 block + vld1.8 {q3}, [r1, :64]! + veor q3, q0, q3 + vst1.8 {q3}, [r0, :64]! + + adds r6, r6, #1 @ increment BE ctr + rev ip, r6 + vmov s27, ip + bcs .Lctrcarry + teq r4, #0 + bne .Lctrloop +.Lctrout: + vst1.8 {q6}, [r5] + pop {r4-r6, pc} + +.Lctrhalfblock: + vld1.8 {d1}, [r1, :64] + veor d0, d0, d1 + vst1.8 {d0}, [r0, :64] + pop {r4-r6, pc} + +.Lctrcarry: + .irp sreg, s26, s25, s24 + vmov ip, \sreg @ load next word of ctr + rev ip, ip @ ... to handle the carry + adds ip, ip, #1 + rev ip, ip + vmov \sreg, ip + bcc 0f + .endr +0: teq r4, #0 + beq .Lctrout + b .Lctrloop +ENDPROC(ce_aes_ctr_encrypt) + + /* + * aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, + * int blocks, u8 iv[], u8 const rk2[], int first) + * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, + * int blocks, u8 iv[], u8 const rk2[], int first) + */ + + .macro next_tweak, out, in, const, tmp + vshr.s64 \tmp, \in, #63 + vand \tmp, \tmp, \const + vadd.u64 \out, \in, \in + vext.8 \tmp, \tmp, \tmp, #8 + veor \out, \out, \tmp + .endm + + .align 3 +.Lxts_mul_x: + .quad 1, 0x87 + +ce_aes_xts_init: + vldr d14, .Lxts_mul_x + vldr d15, .Lxts_mul_x + 8 + + ldrd r4, r5, [sp, #16] @ load args + ldr r6, [sp, #28] + vld1.8 {q0}, [r5] @ load iv + teq r6, #1 @ start of a block? + bxne lr + + @ Encrypt the IV in q0 with the second AES key. This should only + @ be done at the start of a block. + ldr r6, [sp, #24] @ load AES key 2 + prepare_key r6, r3 + add ip, r6, #32 @ 3rd round key of key 2 + b .Laes_encrypt_tweak @ tail call +ENDPROC(ce_aes_xts_init) + +ENTRY(ce_aes_xts_encrypt) + push {r4-r6, lr} + + bl ce_aes_xts_init @ run shared prologue + prepare_key r2, r3 + vmov q3, q0 + + teq r6, #0 @ start of a block? + bne .Lxtsenc3x + +.Lxtsencloop3x: + next_tweak q3, q3, q7, q6 +.Lxtsenc3x: + subs r4, r4, #3 + bmi .Lxtsenc1x + vld1.8 {q0-q1}, [r1, :64]! @ get 3 pt blocks + vld1.8 {q2}, [r1, :64]! + next_tweak q4, q3, q7, q6 + veor q0, q0, q3 + next_tweak q5, q4, q7, q6 + veor q1, q1, q4 + veor q2, q2, q5 + bl aes_encrypt_3x + veor q0, q0, q3 + veor q1, q1, q4 + veor q2, q2, q5 + vst1.8 {q0-q1}, [r0, :64]! @ write 3 ct blocks + vst1.8 {q2}, [r0, :64]! + vmov q3, q5 + teq r4, #0 + beq .Lxtsencout + b .Lxtsencloop3x +.Lxtsenc1x: + adds r4, r4, #3 + beq .Lxtsencout +.Lxtsencloop: + vld1.8 {q0}, [r1, :64]! + veor q0, q0, q3 + bl aes_encrypt + veor q0, q0, q3 + vst1.8 {q0}, [r0, :64]! + subs r4, r4, #1 + beq .Lxtsencout + next_tweak q3, q3, q7, q6 + b .Lxtsencloop +.Lxtsencout: + vst1.8 {q3}, [r5] + pop {r4-r6, pc} +ENDPROC(ce_aes_xts_encrypt) + + +ENTRY(ce_aes_xts_decrypt) + push {r4-r6, lr} + + bl ce_aes_xts_init @ run shared prologue + prepare_key r2, r3 + vmov q3, q0 + + teq r6, #0 @ start of a block? + bne .Lxtsdec3x + +.Lxtsdecloop3x: + next_tweak q3, q3, q7, q6 +.Lxtsdec3x: + subs r4, r4, #3 + bmi .Lxtsdec1x + vld1.8 {q0-q1}, [r1, :64]! @ get 3 ct blocks + vld1.8 {q2}, [r1, :64]! + next_tweak q4, q3, q7, q6 + veor q0, q0, q3 + next_tweak q5, q4, q7, q6 + veor q1, q1, q4 + veor q2, q2, q5 + bl aes_decrypt_3x + veor q0, q0, q3 + veor q1, q1, q4 + veor q2, q2, q5 + vst1.8 {q0-q1}, [r0, :64]! @ write 3 pt blocks + vst1.8 {q2}, [r0, :64]! + vmov q3, q5 + teq r4, #0 + beq .Lxtsdecout + b .Lxtsdecloop3x +.Lxtsdec1x: + adds r4, r4, #3 + beq .Lxtsdecout +.Lxtsdecloop: + vld1.8 {q0}, [r1, :64]! + veor q0, q0, q3 + add ip, r2, #32 @ 3rd round key + bl aes_decrypt + veor q0, q0, q3 + vst1.8 {q0}, [r0, :64]! + subs r4, r4, #1 + beq .Lxtsdecout + next_tweak q3, q3, q7, q6 + b .Lxtsdecloop +.Lxtsdecout: + vst1.8 {q3}, [r5] + pop {r4-r6, pc} +ENDPROC(ce_aes_xts_decrypt) + + /* + * u32 ce_aes_sub(u32 input) - use the aese instruction to perform the + * AES sbox substitution on each byte in + * 'input' + */ +ENTRY(ce_aes_sub) + vdup.32 q1, r0 + veor q0, q0, q0 + aese.8 q0, q1 + vmov r0, s0 + bx lr +ENDPROC(ce_aes_sub) + + /* + * void ce_aes_invert(u8 *dst, u8 *src) - perform the Inverse MixColumns + * operation on round key *src + */ +ENTRY(ce_aes_invert) + vld1.8 {q0}, [r1] + aesimc.8 q0, q0 + vst1.8 {q0}, [r0] + bx lr +ENDPROC(ce_aes_invert) diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c new file mode 100644 index 000000000000..b445a5d56f43 --- /dev/null +++ b/arch/arm/crypto/aes-ce-glue.c @@ -0,0 +1,524 @@ +/* + * aes-ce-glue.c - wrapper code for ARMv8 AES + * + * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <asm/hwcap.h> +#include <asm/neon.h> +#include <asm/hwcap.h> +#include <crypto/aes.h> +#include <crypto/ablk_helper.h> +#include <crypto/algapi.h> +#include <linux/module.h> + +MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions"); +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); +MODULE_LICENSE("GPL v2"); + +/* defined in aes-ce-core.S */ +asmlinkage u32 ce_aes_sub(u32 input); +asmlinkage void ce_aes_invert(void *dst, void *src); + +asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], + int rounds, int blocks); +asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], + int rounds, int blocks); + +asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], + int rounds, int blocks, u8 iv[]); +asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], + int rounds, int blocks, u8 iv[]); + +asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], + int rounds, int blocks, u8 ctr[]); + +asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[], + int rounds, int blocks, u8 iv[], + u8 const rk2[], int first); +asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], + int rounds, int blocks, u8 iv[], + u8 const rk2[], int first); + +struct aes_block { + u8 b[AES_BLOCK_SIZE]; +}; + +static int num_rounds(struct crypto_aes_ctx *ctx) +{ + /* + * # of rounds specified by AES: + * 128 bit key 10 rounds + * 192 bit key 12 rounds + * 256 bit key 14 rounds + * => n byte key => 6 + (n/4) rounds + */ + return 6 + ctx->key_length / 4; +} + +static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + /* + * The AES key schedule round constants + */ + static u8 const rcon[] = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + }; + + u32 kwords = key_len / sizeof(u32); + struct aes_block *key_enc, *key_dec; + int i, j; + + if (key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256) + return -EINVAL; + + memcpy(ctx->key_enc, in_key, key_len); + ctx->key_length = key_len; + + kernel_neon_begin(); + for (i = 0; i < sizeof(rcon); i++) { + u32 *rki = ctx->key_enc + (i * kwords); + u32 *rko = rki + kwords; + + rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8); + rko[0] = rko[0] ^ rki[0] ^ rcon[i]; + rko[1] = rko[0] ^ rki[1]; + rko[2] = rko[1] ^ rki[2]; + rko[3] = rko[2] ^ rki[3]; + + if (key_len == AES_KEYSIZE_192) { + if (i >= 7) + break; + rko[4] = rko[3] ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + } else if (key_len == AES_KEYSIZE_256) { + if (i >= 6) + break; + rko[4] = ce_aes_sub(rko[3]) ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + rko[6] = rko[5] ^ rki[6]; + rko[7] = rko[6] ^ rki[7]; + } + } + + /* + * Generate the decryption keys for the Equivalent Inverse Cipher. + * This involves reversing the order of the round keys, and applying + * the Inverse Mix Columns transformation on all but the first and + * the last one. + */ + key_enc = (struct aes_block *)ctx->key_enc; + key_dec = (struct aes_block *)ctx->key_dec; + j = num_rounds(ctx); + + key_dec[0] = key_enc[j]; + for (i = 1, j--; j > 0; i++, j--) + ce_aes_invert(key_dec + i, key_enc + j); + key_dec[i] = key_enc[0]; + + kernel_neon_end(); + return 0; +} + +static int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = ce_aes_expandkey(ctx, in_key, key_len); + if (!ret) + return 0; + + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; +} + +struct crypto_aes_xts_ctx { + struct crypto_aes_ctx key1; + struct crypto_aes_ctx __aligned(8) key2; +}; + +static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + int ret; + + ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2); + if (!ret) + ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2], + key_len / 2); + if (!ret) + return 0; + + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; +} + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int blocks; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + kernel_neon_begin(); + while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { + ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr, + (u8 *)ctx->key_enc, num_rounds(ctx), blocks); + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % AES_BLOCK_SIZE); + } + kernel_neon_end(); + return err; +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int blocks; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + kernel_neon_begin(); + while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { + ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr, + (u8 *)ctx->key_dec, num_rounds(ctx), blocks); + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % AES_BLOCK_SIZE); + } + kernel_neon_end(); + return err; +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int blocks; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + kernel_neon_begin(); + while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { + ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr, + (u8 *)ctx->key_enc, num_rounds(ctx), blocks, + walk.iv); + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % AES_BLOCK_SIZE); + } + kernel_neon_end(); + return err; +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int blocks; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + kernel_neon_begin(); + while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { + ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr, + (u8 *)ctx->key_dec, num_rounds(ctx), blocks, + walk.iv); + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % AES_BLOCK_SIZE); + } + kernel_neon_end(); + return err; +} + +static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err, blocks; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); + + kernel_neon_begin(); + while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) { + ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr, + (u8 *)ctx->key_enc, num_rounds(ctx), blocks, + walk.iv); + nbytes -= blocks * AES_BLOCK_SIZE; + if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE) + break; + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % AES_BLOCK_SIZE); + } + if (nbytes) { + u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE; + u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE; + u8 __aligned(8) tail[AES_BLOCK_SIZE]; + + /* + * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need + * to tell aes_ctr_encrypt() to only read half a block. + */ + blocks = (nbytes <= 8) ? -1 : 1; + + ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, + num_rounds(ctx), blocks, walk.iv); + memcpy(tdst, tail, nbytes); + err = blkcipher_walk_done(desc, &walk, 0); + } + kernel_neon_end(); + + return err; +} + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + int err, first, rounds = num_rounds(&ctx->key1); + struct blkcipher_walk walk; + unsigned int blocks; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + kernel_neon_begin(); + for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { + ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr, + (u8 *)ctx->key1.key_enc, rounds, blocks, + walk.iv, (u8 *)ctx->key2.key_enc, first); + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % AES_BLOCK_SIZE); + } + kernel_neon_end(); + + return err; +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + int err, first, rounds = num_rounds(&ctx->key1); + struct blkcipher_walk walk; + unsigned int blocks; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + kernel_neon_begin(); + for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) { + ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr, + (u8 *)ctx->key1.key_dec, rounds, blocks, + walk.iv, (u8 *)ctx->key2.key_enc, first); + err = blkcipher_walk_done(desc, &walk, + walk.nbytes % AES_BLOCK_SIZE); + } + kernel_neon_end(); + + return err; +} + +static struct crypto_alg aes_algs[] = { { + .cra_name = "__ecb-aes-ce", + .cra_driver_name = "__driver-ecb-aes-ce", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto_aes_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ce_aes_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, +}, { + .cra_name = "__cbc-aes-ce", + .cra_driver_name = "__driver-cbc-aes-ce", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto_aes_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ce_aes_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, +}, { + .cra_name = "__ctr-aes-ce", + .cra_driver_name = "__driver-ctr-aes-ce", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto_aes_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ce_aes_setkey, + .encrypt = ctr_encrypt, + .decrypt = ctr_encrypt, + }, +}, { + .cra_name = "__xts-aes-ce", + .cra_driver_name = "__driver-xts-aes-ce", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_blkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = xts_set_key, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, +}, { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +}, { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_ablkcipher = { + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + } +} }; + +static int __init aes_init(void) +{ + if (!(elf_hwcap2 & HWCAP2_AES)) + return -ENODEV; + return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs)); +} + +static void __exit aes_exit(void) +{ + crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs)); +} + +module_init(aes_init); +module_exit(aes_exit); diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c index 15468fbbdea3..6d685298690e 100644 --- a/arch/arm/crypto/aesbs-glue.c +++ b/arch/arm/crypto/aesbs-glue.c @@ -301,7 +301,8 @@ static struct crypto_alg aesbs_algs[] = { { .cra_name = "__cbc-aes-neonbs", .cra_driver_name = "__driver-cbc-aes-neonbs", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aesbs_cbc_ctx), .cra_alignmask = 7, @@ -319,7 +320,8 @@ static struct crypto_alg aesbs_algs[] = { { .cra_name = "__ctr-aes-neonbs", .cra_driver_name = "__driver-ctr-aes-neonbs", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct aesbs_ctr_ctx), .cra_alignmask = 7, @@ -337,7 +339,8 @@ static struct crypto_alg aesbs_algs[] = { { .cra_name = "__xts-aes-neonbs", .cra_driver_name = "__driver-xts-aes-neonbs", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aesbs_xts_ctx), .cra_alignmask = 7, diff --git a/arch/arm/crypto/ghash-ce-core.S b/arch/arm/crypto/ghash-ce-core.S new file mode 100644 index 000000000000..f6ab8bcc9efe --- /dev/null +++ b/arch/arm/crypto/ghash-ce-core.S @@ -0,0 +1,94 @@ +/* + * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. + * + * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + SHASH .req q0 + SHASH2 .req q1 + T1 .req q2 + T2 .req q3 + MASK .req q4 + XL .req q5 + XM .req q6 + XH .req q7 + IN1 .req q7 + + SHASH_L .req d0 + SHASH_H .req d1 + SHASH2_L .req d2 + T1_L .req d4 + MASK_L .req d8 + XL_L .req d10 + XL_H .req d11 + XM_L .req d12 + XM_H .req d13 + XH_L .req d14 + + .text + .fpu crypto-neon-fp-armv8 + + /* + * void pmull_ghash_update(int blocks, u64 dg[], const char *src, + * struct ghash_key const *k, const char *head) + */ +ENTRY(pmull_ghash_update) + vld1.64 {SHASH}, [r3] + vld1.64 {XL}, [r1] + vmov.i8 MASK, #0xe1 + vext.8 SHASH2, SHASH, SHASH, #8 + vshl.u64 MASK, MASK, #57 + veor SHASH2, SHASH2, SHASH + + /* do the head block first, if supplied */ + ldr ip, [sp] + teq ip, #0 + beq 0f + vld1.64 {T1}, [ip] + teq r0, #0 + b 1f + +0: vld1.64 {T1}, [r2]! + subs r0, r0, #1 + +1: /* multiply XL by SHASH in GF(2^128) */ +#ifndef CONFIG_CPU_BIG_ENDIAN + vrev64.8 T1, T1 +#endif + vext.8 T2, XL, XL, #8 + vext.8 IN1, T1, T1, #8 + veor T1, T1, T2 + veor XL, XL, IN1 + + vmull.p64 XH, SHASH_H, XL_H @ a1 * b1 + veor T1, T1, XL + vmull.p64 XL, SHASH_L, XL_L @ a0 * b0 + vmull.p64 XM, SHASH2_L, T1_L @ (a1 + a0)(b1 + b0) + + vext.8 T1, XL, XH, #8 + veor T2, XL, XH + veor XM, XM, T1 + veor XM, XM, T2 + vmull.p64 T2, XL_L, MASK_L + + vmov XH_L, XM_H + vmov XM_H, XL_L + + veor XL, XM, T2 + vext.8 T2, XL, XL, #8 + vmull.p64 XL, XL_L, MASK_L + veor T2, T2, XH + veor XL, XL, T2 + + bne 0b + + vst1.64 {XL}, [r1] + bx lr +ENDPROC(pmull_ghash_update) diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c new file mode 100644 index 000000000000..03a39fe29246 --- /dev/null +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -0,0 +1,320 @@ +/* + * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. + * + * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include <asm/hwcap.h> +#include <asm/neon.h> +#include <asm/simd.h> +#include <asm/unaligned.h> +#include <crypto/cryptd.h> +#include <crypto/internal/hash.h> +#include <crypto/gf128mul.h> +#include <linux/crypto.h> +#include <linux/module.h> + +MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions"); +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); +MODULE_LICENSE("GPL v2"); + +#define GHASH_BLOCK_SIZE 16 +#define GHASH_DIGEST_SIZE 16 + +struct ghash_key { + u64 a; + u64 b; +}; + +struct ghash_desc_ctx { + u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; + u8 buf[GHASH_BLOCK_SIZE]; + u32 count; +}; + +struct ghash_async_ctx { + struct cryptd_ahash *cryptd_tfm; +}; + +asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src, + struct ghash_key const *k, const char *head); + +static int ghash_init(struct shash_desc *desc) +{ + struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + + *ctx = (struct ghash_desc_ctx){}; + return 0; +} + +static int ghash_update(struct shash_desc *desc, const u8 *src, + unsigned int len) +{ + struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + + ctx->count += len; + + if ((partial + len) >= GHASH_BLOCK_SIZE) { + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + int blocks; + + if (partial) { + int p = GHASH_BLOCK_SIZE - partial; + + memcpy(ctx->buf + partial, src, p); + src += p; + len -= p; + } + + blocks = len / GHASH_BLOCK_SIZE; + len %= GHASH_BLOCK_SIZE; + + kernel_neon_begin(); + pmull_ghash_update(blocks, ctx->digest, src, key, + partial ? ctx->buf : NULL); + kernel_neon_end(); + src += blocks * GHASH_BLOCK_SIZE; + partial = 0; + } + if (len) + memcpy(ctx->buf + partial, src, len); + return 0; +} + +static int ghash_final(struct shash_desc *desc, u8 *dst) +{ + struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); + unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; + + if (partial) { + struct ghash_key *key = crypto_shash_ctx(desc->tfm); + + memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); + kernel_neon_begin(); + pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL); + kernel_neon_end(); + } + put_unaligned_be64(ctx->digest[1], dst); + put_unaligned_be64(ctx->digest[0], dst + 8); + + *ctx = (struct ghash_desc_ctx){}; + return 0; +} + +static int ghash_setkey(struct crypto_shash *tfm, + const u8 *inkey, unsigned int keylen) +{ + struct ghash_key *key = crypto_shash_ctx(tfm); + u64 a, b; + + if (keylen != GHASH_BLOCK_SIZE) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + /* perform multiplication by 'x' in GF(2^128) */ + b = get_unaligned_be64(inkey); + a = get_unaligned_be64(inkey + 8); + + key->a = (a << 1) | (b >> 63); + key->b = (b << 1) | (a >> 63); + + if (b >> 63) + key->b ^= 0xc200000000000000UL; + + return 0; +} + +static struct shash_alg ghash_alg = { + .digestsize = GHASH_DIGEST_SIZE, + .init = ghash_init, + .update = ghash_update, + .final = ghash_final, + .setkey = ghash_setkey, + .descsize = sizeof(struct ghash_desc_ctx), + .base = { + .cra_name = "ghash", + .cra_driver_name = "__driver-ghash-ce", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_INTERNAL, + .cra_blocksize = GHASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ghash_key), + .cra_module = THIS_MODULE, + }, +}; + +static int ghash_async_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; + + if (!may_use_simd()) { + memcpy(cryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); + return crypto_ahash_init(cryptd_req); + } else { + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); + + desc->tfm = child; + desc->flags = req->base.flags; + return crypto_shash_init(desc); + } +} + +static int ghash_async_update(struct ahash_request *req) +{ + struct ahash_request *cryptd_req = ahash_request_ctx(req); + + if (!may_use_simd()) { + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; + + memcpy(cryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); + return crypto_ahash_update(cryptd_req); + } else { + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + return shash_ahash_update(req, desc); + } +} + +static int ghash_async_final(struct ahash_request *req) +{ + struct ahash_request *cryptd_req = ahash_request_ctx(req); + + if (!may_use_simd()) { + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; + + memcpy(cryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); + return crypto_ahash_final(cryptd_req); + } else { + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + return crypto_shash_final(desc, req->result); + } +} + +static int ghash_async_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct ahash_request *cryptd_req = ahash_request_ctx(req); + struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; + + if (!may_use_simd()) { + memcpy(cryptd_req, req, sizeof(*req)); + ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); + return crypto_ahash_digest(cryptd_req); + } else { + struct shash_desc *desc = cryptd_shash_desc(cryptd_req); + struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); + + desc->tfm = child; + desc->flags = req->base.flags; + return shash_ahash_digest(req, desc); + } +} + +static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); + struct crypto_ahash *child = &ctx->cryptd_tfm->base; + int err; + + crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) + & CRYPTO_TFM_REQ_MASK); + err = crypto_ahash_setkey(child, key, keylen); + crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child) + & CRYPTO_TFM_RES_MASK); + + return err; +} + +static int ghash_async_init_tfm(struct crypto_tfm *tfm) +{ + struct cryptd_ahash *cryptd_tfm; + struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); + + cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + ctx->cryptd_tfm = cryptd_tfm; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(&cryptd_tfm->base)); + + return 0; +} + +static void ghash_async_exit_tfm(struct crypto_tfm *tfm) +{ + struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); + + cryptd_free_ahash(ctx->cryptd_tfm); +} + +static struct ahash_alg ghash_async_alg = { + .init = ghash_async_init, + .update = ghash_async_update, + .final = ghash_async_final, + .setkey = ghash_async_setkey, + .digest = ghash_async_digest, + .halg.digestsize = GHASH_DIGEST_SIZE, + .halg.base = { + .cra_name = "ghash", + .cra_driver_name = "ghash-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, + .cra_blocksize = GHASH_BLOCK_SIZE, + .cra_type = &crypto_ahash_type, + .cra_ctxsize = sizeof(struct ghash_async_ctx), + .cra_module = THIS_MODULE, + .cra_init = ghash_async_init_tfm, + .cra_exit = ghash_async_exit_tfm, + }, +}; + +static int __init ghash_ce_mod_init(void) +{ + int err; + + if (!(elf_hwcap2 & HWCAP2_PMULL)) + return -ENODEV; + + err = crypto_register_shash(&ghash_alg); + if (err) + return err; + err = crypto_register_ahash(&ghash_async_alg); + if (err) + goto err_shash; + + return 0; + +err_shash: + crypto_unregister_shash(&ghash_alg); + return err; +} + +static void __exit ghash_ce_mod_exit(void) +{ + crypto_unregister_ahash(&ghash_async_alg); + crypto_unregister_shash(&ghash_alg); +} + +module_init(ghash_ce_mod_init); +module_exit(ghash_ce_mod_exit); diff --git a/arch/arm/crypto/sha1-ce-core.S b/arch/arm/crypto/sha1-ce-core.S new file mode 100644 index 000000000000..b623f51ccbcf --- /dev/null +++ b/arch/arm/crypto/sha1-ce-core.S @@ -0,0 +1,125 @@ +/* + * sha1-ce-core.S - SHA-1 secure hash using ARMv8 Crypto Extensions + * + * Copyright (C) 2015 Linaro Ltd. + * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .text + .fpu crypto-neon-fp-armv8 + + k0 .req q0 + k1 .req q1 + k2 .req q2 + k3 .req q3 + + ta0 .req q4 + ta1 .req q5 + tb0 .req q5 + tb1 .req q4 + + dga .req q6 + dgb .req q7 + dgbs .req s28 + + dg0 .req q12 + dg1a0 .req q13 + dg1a1 .req q14 + dg1b0 .req q14 + dg1b1 .req q13 + + .macro add_only, op, ev, rc, s0, dg1 + .ifnb \s0 + vadd.u32 tb\ev, q\s0, \rc + .endif + sha1h.32 dg1b\ev, dg0 + .ifb \dg1 + sha1\op\().32 dg0, dg1a\ev, ta\ev + .else + sha1\op\().32 dg0, \dg1, ta\ev + .endif + .endm + + .macro add_update, op, ev, rc, s0, s1, s2, s3, dg1 + sha1su0.32 q\s0, q\s1, q\s2 + add_only \op, \ev, \rc, \s1, \dg1 + sha1su1.32 q\s0, q\s3 + .endm + + .align 6 +.Lsha1_rcon: + .word 0x5a827999, 0x5a827999, 0x5a827999, 0x5a827999 + .word 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1, 0x6ed9eba1 + .word 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc, 0x8f1bbcdc + .word 0xca62c1d6, 0xca62c1d6, 0xca62c1d6, 0xca62c1d6 + + /* + * void sha1_ce_transform(struct sha1_state *sst, u8 const *src, + * int blocks); + */ +ENTRY(sha1_ce_transform) + /* load round constants */ + adr ip, .Lsha1_rcon + vld1.32 {k0-k1}, [ip, :128]! + vld1.32 {k2-k3}, [ip, :128] + + /* load state */ + vld1.32 {dga}, [r0] + vldr dgbs, [r0, #16] + + /* load input */ +0: vld1.32 {q8-q9}, [r1]! + vld1.32 {q10-q11}, [r1]! + subs r2, r2, #1 + +#ifndef CONFIG_CPU_BIG_ENDIAN + vrev32.8 q8, q8 + vrev32.8 q9, q9 + vrev32.8 q10, q10 + vrev32.8 q11, q11 +#endif + + vadd.u32 ta0, q8, k0 + vmov dg0, dga + + add_update c, 0, k0, 8, 9, 10, 11, dgb + add_update c, 1, k0, 9, 10, 11, 8 + add_update c, 0, k0, 10, 11, 8, 9 + add_update c, 1, k0, 11, 8, 9, 10 + add_update c, 0, k1, 8, 9, 10, 11 + + add_update p, 1, k1, 9, 10, 11, 8 + add_update p, 0, k1, 10, 11, 8, 9 + add_update p, 1, k1, 11, 8, 9, 10 + add_update p, 0, k1, 8, 9, 10, 11 + add_update p, 1, k2, 9, 10, 11, 8 + + add_update m, 0, k2, 10, 11, 8, 9 + add_update m, 1, k2, 11, 8, 9, 10 + add_update m, 0, k2, 8, 9, 10, 11 + add_update m, 1, k2, 9, 10, 11, 8 + add_update m, 0, k3, 10, 11, 8, 9 + + add_update p, 1, k3, 11, 8, 9, 10 + add_only p, 0, k3, 9 + add_only p, 1, k3, 10 + add_only p, 0, k3, 11 + add_only p, 1 + + /* update state */ + vadd.u32 dga, dga, dg0 + vadd.u32 dgb, dgb, dg1a0 + bne 0b + + /* store new state */ + vst1.32 {dga}, [r0] + vstr dgbs, [r0, #16] + bx lr +ENDPROC(sha1_ce_transform) diff --git a/arch/arm/crypto/sha1-ce-glue.c b/arch/arm/crypto/sha1-ce-glue.c new file mode 100644 index 000000000000..80bc2fcd241a --- /dev/null +++ b/arch/arm/crypto/sha1-ce-glue.c @@ -0,0 +1,96 @@ +/* + * sha1-ce-glue.c - SHA-1 secure hash using ARMv8 Crypto Extensions + * + * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/sha1_base.h> +#include <linux/crypto.h> +#include <linux/module.h> + +#include <asm/hwcap.h> +#include <asm/neon.h> +#include <asm/simd.h> + +#include "sha1.h" + +MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); +MODULE_LICENSE("GPL v2"); + +asmlinkage void sha1_ce_transform(struct sha1_state *sst, u8 const *src, + int blocks); + +static int sha1_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + if (!may_use_simd() || + (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) + return sha1_update_arm(desc, data, len); + + kernel_neon_begin(); + sha1_base_do_update(desc, data, len, sha1_ce_transform); + kernel_neon_end(); + + return 0; +} + +static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + if (!may_use_simd()) + return sha1_finup_arm(desc, data, len, out); + + kernel_neon_begin(); + if (len) + sha1_base_do_update(desc, data, len, sha1_ce_transform); + sha1_base_do_finalize(desc, sha1_ce_transform); + kernel_neon_end(); + + return sha1_base_finish(desc, out); +} + +static int sha1_ce_final(struct shash_desc *desc, u8 *out) +{ + return sha1_ce_finup(desc, NULL, 0, out); +} + +static struct shash_alg alg = { + .init = sha1_base_init, + .update = sha1_ce_update, + .final = sha1_ce_final, + .finup = sha1_ce_finup, + .descsize = sizeof(struct sha1_state), + .digestsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-ce", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init sha1_ce_mod_init(void) +{ + if (!(elf_hwcap2 & HWCAP2_SHA1)) + return -ENODEV; + return crypto_register_shash(&alg); +} + +static void __exit sha1_ce_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(sha1_ce_mod_init); +module_exit(sha1_ce_mod_fini); diff --git a/arch/arm/include/asm/crypto/sha1.h b/arch/arm/crypto/sha1.h index 75e6a417416b..ffd8bd08b1a7 100644 --- a/arch/arm/include/asm/crypto/sha1.h +++ b/arch/arm/crypto/sha1.h @@ -7,4 +7,7 @@ extern int sha1_update_arm(struct shash_desc *desc, const u8 *data, unsigned int len); +extern int sha1_finup_arm(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out); + #endif diff --git a/arch/arm/crypto/sha1_glue.c b/arch/arm/crypto/sha1_glue.c index e31b0440c613..6fc73bf8766d 100644 --- a/arch/arm/crypto/sha1_glue.c +++ b/arch/arm/crypto/sha1_glue.c @@ -22,127 +22,47 @@ #include <linux/cryptohash.h> #include <linux/types.h> #include <crypto/sha.h> +#include <crypto/sha1_base.h> #include <asm/byteorder.h> -#include <asm/crypto/sha1.h> +#include "sha1.h" asmlinkage void sha1_block_data_order(u32 *digest, const unsigned char *data, unsigned int rounds); - -static int sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - - -static int __sha1_update(struct sha1_state *sctx, const u8 *data, - unsigned int len, unsigned int partial) -{ - unsigned int done = 0; - - sctx->count += len; - - if (partial) { - done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->buffer + partial, data, done); - sha1_block_data_order(sctx->state, sctx->buffer, 1); - } - - if (len - done >= SHA1_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - sha1_block_data_order(sctx->state, data + done, rounds); - done += rounds * SHA1_BLOCK_SIZE; - } - - memcpy(sctx->buffer, data + done, len - done); - return 0; -} - - int sha1_update_arm(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - int res; + /* make sure casting to sha1_block_fn() is safe */ + BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); - /* Handle the fast case right here */ - if (partial + len < SHA1_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buffer + partial, data, len); - return 0; - } - res = __sha1_update(sctx, data, len, partial); - return res; + return sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_block_data_order); } EXPORT_SYMBOL_GPL(sha1_update_arm); - -/* Add padding and return the message digest. */ static int sha1_final(struct shash_desc *desc, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA1_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); - /* We need to fill a whole block for __sha1_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buffer + index, padding, padlen); - } else { - __sha1_update(sctx, padding, padlen, index); - } - __sha1_update(sctx, (const u8 *)&bits, sizeof(bits), 56); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - return 0; + sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_block_data_order); + return sha1_base_finish(desc, out); } - -static int sha1_export(struct shash_desc *desc, void *out) +int sha1_finup_arm(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - memcpy(out, sctx, sizeof(*sctx)); - return 0; + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_block_data_order); + return sha1_final(desc, out); } - - -static int sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - memcpy(sctx, in, sizeof(*sctx)); - return 0; -} - +EXPORT_SYMBOL_GPL(sha1_finup_arm); static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_init, + .init = sha1_base_init, .update = sha1_update_arm, .final = sha1_final, - .export = sha1_export, - .import = sha1_import, + .finup = sha1_finup_arm, .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name= "sha1-asm", diff --git a/arch/arm/crypto/sha1_neon_glue.c b/arch/arm/crypto/sha1_neon_glue.c index 0b0083757d47..4e22f122f966 100644 --- a/arch/arm/crypto/sha1_neon_glue.c +++ b/arch/arm/crypto/sha1_neon_glue.c @@ -25,147 +25,60 @@ #include <linux/cryptohash.h> #include <linux/types.h> #include <crypto/sha.h> -#include <asm/byteorder.h> +#include <crypto/sha1_base.h> #include <asm/neon.h> #include <asm/simd.h> -#include <asm/crypto/sha1.h> +#include "sha1.h" asmlinkage void sha1_transform_neon(void *state_h, const char *data, unsigned int rounds); - -static int sha1_neon_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - -static int __sha1_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len, unsigned int partial) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int done = 0; - - sctx->count += len; - - if (partial) { - done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->buffer + partial, data, done); - sha1_transform_neon(sctx->state, sctx->buffer, 1); - } - - if (len - done >= SHA1_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - - sha1_transform_neon(sctx->state, data + done, rounds); - done += rounds * SHA1_BLOCK_SIZE; - } - - memcpy(sctx->buffer, data + done, len - done); - - return 0; -} - static int sha1_neon_update(struct shash_desc *desc, const u8 *data, - unsigned int len) + unsigned int len) { struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - int res; - /* Handle the fast case right here */ - if (partial + len < SHA1_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buffer + partial, data, len); + if (!may_use_simd() || + (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) + return sha1_update_arm(desc, data, len); - return 0; - } - - if (!may_use_simd()) { - res = sha1_update_arm(desc, data, len); - } else { - kernel_neon_begin(); - res = __sha1_neon_update(desc, data, len, partial); - kernel_neon_end(); - } - - return res; -} - - -/* Add padding and return the message digest. */ -static int sha1_neon_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA1_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); - if (!may_use_simd()) { - sha1_update_arm(desc, padding, padlen); - sha1_update_arm(desc, (const u8 *)&bits, sizeof(bits)); - } else { - kernel_neon_begin(); - /* We need to fill a whole block for __sha1_neon_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buffer + index, padding, padlen); - } else { - __sha1_neon_update(desc, padding, padlen, index); - } - __sha1_neon_update(desc, (const u8 *)&bits, sizeof(bits), 56); - kernel_neon_end(); - } - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); + kernel_neon_begin(); + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_transform_neon); + kernel_neon_end(); return 0; } -static int sha1_neon_export(struct shash_desc *desc, void *out) +static int sha1_neon_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); + if (!may_use_simd()) + return sha1_finup_arm(desc, data, len, out); - memcpy(out, sctx, sizeof(*sctx)); + kernel_neon_begin(); + if (len) + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_transform_neon); + sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_neon); + kernel_neon_end(); - return 0; + return sha1_base_finish(desc, out); } -static int sha1_neon_import(struct shash_desc *desc, const void *in) +static int sha1_neon_final(struct shash_desc *desc, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; + return sha1_neon_finup(desc, NULL, 0, out); } static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_neon_init, + .init = sha1_base_init, .update = sha1_neon_update, .final = sha1_neon_final, - .export = sha1_neon_export, - .import = sha1_neon_import, + .finup = sha1_neon_finup, .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name = "sha1-neon", diff --git a/arch/arm/crypto/sha2-ce-core.S b/arch/arm/crypto/sha2-ce-core.S new file mode 100644 index 000000000000..87ec11a5f405 --- /dev/null +++ b/arch/arm/crypto/sha2-ce-core.S @@ -0,0 +1,125 @@ +/* + * sha2-ce-core.S - SHA-224/256 secure hash using ARMv8 Crypto Extensions + * + * Copyright (C) 2015 Linaro Ltd. + * Author: Ard Biesheuvel <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + + .text + .fpu crypto-neon-fp-armv8 + + k0 .req q7 + k1 .req q8 + rk .req r3 + + ta0 .req q9 + ta1 .req q10 + tb0 .req q10 + tb1 .req q9 + + dga .req q11 + dgb .req q12 + + dg0 .req q13 + dg1 .req q14 + dg2 .req q15 + + .macro add_only, ev, s0 + vmov dg2, dg0 + .ifnb \s0 + vld1.32 {k\ev}, [rk, :128]! + .endif + sha256h.32 dg0, dg1, tb\ev + sha256h2.32 dg1, dg2, tb\ev + .ifnb \s0 + vadd.u32 ta\ev, q\s0, k\ev + .endif + .endm + + .macro add_update, ev, s0, s1, s2, s3 + sha256su0.32 q\s0, q\s1 + add_only \ev, \s1 + sha256su1.32 q\s0, q\s2, q\s3 + .endm + + .align 6 +.Lsha256_rcon: + .word 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5 + .word 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5 + .word 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3 + .word 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174 + .word 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc + .word 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da + .word 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7 + .word 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967 + .word 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13 + .word 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85 + .word 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3 + .word 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070 + .word 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5 + .word 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3 + .word 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208 + .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 + + /* + * void sha2_ce_transform(struct sha256_state *sst, u8 const *src, + int blocks); + */ +ENTRY(sha2_ce_transform) + /* load state */ + vld1.32 {dga-dgb}, [r0] + + /* load input */ +0: vld1.32 {q0-q1}, [r1]! + vld1.32 {q2-q3}, [r1]! + subs r2, r2, #1 + +#ifndef CONFIG_CPU_BIG_ENDIAN + vrev32.8 q0, q0 + vrev32.8 q1, q1 + vrev32.8 q2, q2 + vrev32.8 q3, q3 +#endif + + /* load first round constant */ + adr rk, .Lsha256_rcon + vld1.32 {k0}, [rk, :128]! + + vadd.u32 ta0, q0, k0 + vmov dg0, dga + vmov dg1, dgb + + add_update 1, 0, 1, 2, 3 + add_update 0, 1, 2, 3, 0 + add_update 1, 2, 3, 0, 1 + add_update 0, 3, 0, 1, 2 + add_update 1, 0, 1, 2, 3 + add_update 0, 1, 2, 3, 0 + add_update 1, 2, 3, 0, 1 + add_update 0, 3, 0, 1, 2 + add_update 1, 0, 1, 2, 3 + add_update 0, 1, 2, 3, 0 + add_update 1, 2, 3, 0, 1 + add_update 0, 3, 0, 1, 2 + + add_only 1, 1 + add_only 0, 2 + add_only 1, 3 + add_only 0 + + /* update state */ + vadd.u32 dga, dga, dg0 + vadd.u32 dgb, dgb, dg1 + bne 0b + + /* store new state */ + vst1.32 {dga-dgb}, [r0] + bx lr +ENDPROC(sha2_ce_transform) diff --git a/arch/arm/crypto/sha2-ce-glue.c b/arch/arm/crypto/sha2-ce-glue.c new file mode 100644 index 000000000000..0755b2d657f3 --- /dev/null +++ b/arch/arm/crypto/sha2-ce-glue.c @@ -0,0 +1,114 @@ +/* + * sha2-ce-glue.c - SHA-224/SHA-256 using ARMv8 Crypto Extensions + * + * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/sha256_base.h> +#include <linux/crypto.h> +#include <linux/module.h> + +#include <asm/hwcap.h> +#include <asm/simd.h> +#include <asm/neon.h> +#include <asm/unaligned.h> + +#include "sha256_glue.h" + +MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); +MODULE_LICENSE("GPL v2"); + +asmlinkage void sha2_ce_transform(struct sha256_state *sst, u8 const *src, + int blocks); + +static int sha2_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + if (!may_use_simd() || + (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) + return crypto_sha256_arm_update(desc, data, len); + + kernel_neon_begin(); + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha2_ce_transform); + kernel_neon_end(); + + return 0; +} + +static int sha2_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + if (!may_use_simd()) + return crypto_sha256_arm_finup(desc, data, len, out); + + kernel_neon_begin(); + if (len) + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha2_ce_transform); + sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); + kernel_neon_end(); + + return sha256_base_finish(desc, out); +} + +static int sha2_ce_final(struct shash_desc *desc, u8 *out) +{ + return sha2_ce_finup(desc, NULL, 0, out); +} + +static struct shash_alg algs[] = { { + .init = sha224_base_init, + .update = sha2_ce_update, + .final = sha2_ce_final, + .finup = sha2_ce_finup, + .descsize = sizeof(struct sha256_state), + .digestsize = SHA224_DIGEST_SIZE, + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .init = sha256_base_init, + .update = sha2_ce_update, + .final = sha2_ce_final, + .finup = sha2_ce_finup, + .descsize = sizeof(struct sha256_state), + .digestsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-ce", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init sha2_ce_mod_init(void) +{ + if (!(elf_hwcap2 & HWCAP2_SHA2)) + return -ENODEV; + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} + +static void __exit sha2_ce_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} + +module_init(sha2_ce_mod_init); +module_exit(sha2_ce_mod_fini); diff --git a/arch/arm/crypto/sha256-armv4.pl b/arch/arm/crypto/sha256-armv4.pl new file mode 100644 index 000000000000..fac0533ea633 --- /dev/null +++ b/arch/arm/crypto/sha256-armv4.pl @@ -0,0 +1,716 @@ +#!/usr/bin/env perl + +# ==================================================================== +# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL +# project. The module is, however, dual licensed under OpenSSL and +# CRYPTOGAMS licenses depending on where you obtain it. For further +# details see http://www.openssl.org/~appro/cryptogams/. +# +# Permission to use under GPL terms is granted. +# ==================================================================== + +# SHA256 block procedure for ARMv4. May 2007. + +# Performance is ~2x better than gcc 3.4 generated code and in "abso- +# lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per +# byte [on single-issue Xscale PXA250 core]. + +# July 2010. +# +# Rescheduling for dual-issue pipeline resulted in 22% improvement on +# Cortex A8 core and ~20 cycles per processed byte. + +# February 2011. +# +# Profiler-assisted and platform-specific optimization resulted in 16% +# improvement on Cortex A8 core and ~15.4 cycles per processed byte. + +# September 2013. +# +# Add NEON implementation. On Cortex A8 it was measured to process one +# byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon +# S4 does it in 12.5 cycles too, but it's 50% faster than integer-only +# code (meaning that latter performs sub-optimally, nothing was done +# about it). + +# May 2014. +# +# Add ARMv8 code path performing at 2.0 cpb on Apple A7. + +while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} +open STDOUT,">$output"; + +$ctx="r0"; $t0="r0"; +$inp="r1"; $t4="r1"; +$len="r2"; $t1="r2"; +$T1="r3"; $t3="r3"; +$A="r4"; +$B="r5"; +$C="r6"; +$D="r7"; +$E="r8"; +$F="r9"; +$G="r10"; +$H="r11"; +@V=($A,$B,$C,$D,$E,$F,$G,$H); +$t2="r12"; +$Ktbl="r14"; + +@Sigma0=( 2,13,22); +@Sigma1=( 6,11,25); +@sigma0=( 7,18, 3); +@sigma1=(17,19,10); + +sub BODY_00_15 { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; + +$code.=<<___ if ($i<16); +#if __ARM_ARCH__>=7 + @ ldr $t1,[$inp],#4 @ $i +# if $i==15 + str $inp,[sp,#17*4] @ make room for $t4 +# endif + eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` + add $a,$a,$t2 @ h+=Maj(a,b,c) from the past + eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e) +# ifndef __ARMEB__ + rev $t1,$t1 +# endif +#else + @ ldrb $t1,[$inp,#3] @ $i + add $a,$a,$t2 @ h+=Maj(a,b,c) from the past + ldrb $t2,[$inp,#2] + ldrb $t0,[$inp,#1] + orr $t1,$t1,$t2,lsl#8 + ldrb $t2,[$inp],#4 + orr $t1,$t1,$t0,lsl#16 +# if $i==15 + str $inp,[sp,#17*4] @ make room for $t4 +# endif + eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` + orr $t1,$t1,$t2,lsl#24 + eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e) +#endif +___ +$code.=<<___; + ldr $t2,[$Ktbl],#4 @ *K256++ + add $h,$h,$t1 @ h+=X[i] + str $t1,[sp,#`$i%16`*4] + eor $t1,$f,$g + add $h,$h,$t0,ror#$Sigma1[0] @ h+=Sigma1(e) + and $t1,$t1,$e + add $h,$h,$t2 @ h+=K256[i] + eor $t1,$t1,$g @ Ch(e,f,g) + eor $t0,$a,$a,ror#`$Sigma0[1]-$Sigma0[0]` + add $h,$h,$t1 @ h+=Ch(e,f,g) +#if $i==31 + and $t2,$t2,#0xff + cmp $t2,#0xf2 @ done? +#endif +#if $i<15 +# if __ARM_ARCH__>=7 + ldr $t1,[$inp],#4 @ prefetch +# else + ldrb $t1,[$inp,#3] +# endif + eor $t2,$a,$b @ a^b, b^c in next round +#else + ldr $t1,[sp,#`($i+2)%16`*4] @ from future BODY_16_xx + eor $t2,$a,$b @ a^b, b^c in next round + ldr $t4,[sp,#`($i+15)%16`*4] @ from future BODY_16_xx +#endif + eor $t0,$t0,$a,ror#`$Sigma0[2]-$Sigma0[0]` @ Sigma0(a) + and $t3,$t3,$t2 @ (b^c)&=(a^b) + add $d,$d,$h @ d+=h + eor $t3,$t3,$b @ Maj(a,b,c) + add $h,$h,$t0,ror#$Sigma0[0] @ h+=Sigma0(a) + @ add $h,$h,$t3 @ h+=Maj(a,b,c) +___ + ($t2,$t3)=($t3,$t2); +} + +sub BODY_16_XX { +my ($i,$a,$b,$c,$d,$e,$f,$g,$h) = @_; + +$code.=<<___; + @ ldr $t1,[sp,#`($i+1)%16`*4] @ $i + @ ldr $t4,[sp,#`($i+14)%16`*4] + mov $t0,$t1,ror#$sigma0[0] + add $a,$a,$t2 @ h+=Maj(a,b,c) from the past + mov $t2,$t4,ror#$sigma1[0] + eor $t0,$t0,$t1,ror#$sigma0[1] + eor $t2,$t2,$t4,ror#$sigma1[1] + eor $t0,$t0,$t1,lsr#$sigma0[2] @ sigma0(X[i+1]) + ldr $t1,[sp,#`($i+0)%16`*4] + eor $t2,$t2,$t4,lsr#$sigma1[2] @ sigma1(X[i+14]) + ldr $t4,[sp,#`($i+9)%16`*4] + + add $t2,$t2,$t0 + eor $t0,$e,$e,ror#`$Sigma1[1]-$Sigma1[0]` @ from BODY_00_15 + add $t1,$t1,$t2 + eor $t0,$t0,$e,ror#`$Sigma1[2]-$Sigma1[0]` @ Sigma1(e) + add $t1,$t1,$t4 @ X[i] +___ + &BODY_00_15(@_); +} + +$code=<<___; +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ 7 +#endif + +.text +#if __ARM_ARCH__<7 +.code 32 +#else +.syntax unified +# ifdef __thumb2__ +# define adrl adr +.thumb +# else +.code 32 +# endif +#endif + +.type K256,%object +.align 5 +K256: +.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.size K256,.-K256 +.word 0 @ terminator +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-sha256_block_data_order +#endif +.align 5 + +.global sha256_block_data_order +.type sha256_block_data_order,%function +sha256_block_data_order: +#if __ARM_ARCH__<7 + sub r3,pc,#8 @ sha256_block_data_order +#else + adr r3,sha256_block_data_order +#endif +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + ldr r12,.LOPENSSL_armcap + ldr r12,[r3,r12] @ OPENSSL_armcap_P + tst r12,#ARMV8_SHA256 + bne .LARMv8 + tst r12,#ARMV7_NEON + bne .LNEON +#endif + add $len,$inp,$len,lsl#6 @ len to point at the end of inp + stmdb sp!,{$ctx,$inp,$len,r4-r11,lr} + ldmia $ctx,{$A,$B,$C,$D,$E,$F,$G,$H} + sub $Ktbl,r3,#256+32 @ K256 + sub sp,sp,#16*4 @ alloca(X[16]) +.Loop: +# if __ARM_ARCH__>=7 + ldr $t1,[$inp],#4 +# else + ldrb $t1,[$inp,#3] +# endif + eor $t3,$B,$C @ magic + eor $t2,$t2,$t2 +___ +for($i=0;$i<16;$i++) { &BODY_00_15($i,@V); unshift(@V,pop(@V)); } +$code.=".Lrounds_16_xx:\n"; +for (;$i<32;$i++) { &BODY_16_XX($i,@V); unshift(@V,pop(@V)); } +$code.=<<___; +#if __ARM_ARCH__>=7 + ite eq @ Thumb2 thing, sanity check in ARM +#endif + ldreq $t3,[sp,#16*4] @ pull ctx + bne .Lrounds_16_xx + + add $A,$A,$t2 @ h+=Maj(a,b,c) from the past + ldr $t0,[$t3,#0] + ldr $t1,[$t3,#4] + ldr $t2,[$t3,#8] + add $A,$A,$t0 + ldr $t0,[$t3,#12] + add $B,$B,$t1 + ldr $t1,[$t3,#16] + add $C,$C,$t2 + ldr $t2,[$t3,#20] + add $D,$D,$t0 + ldr $t0,[$t3,#24] + add $E,$E,$t1 + ldr $t1,[$t3,#28] + add $F,$F,$t2 + ldr $inp,[sp,#17*4] @ pull inp + ldr $t2,[sp,#18*4] @ pull inp+len + add $G,$G,$t0 + add $H,$H,$t1 + stmia $t3,{$A,$B,$C,$D,$E,$F,$G,$H} + cmp $inp,$t2 + sub $Ktbl,$Ktbl,#256 @ rewind Ktbl + bne .Loop + + add sp,sp,#`16+3`*4 @ destroy frame +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else + ldmia sp!,{r4-r11,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + bx lr @ interoperable with Thumb ISA:-) +#endif +.size sha256_block_data_order,.-sha256_block_data_order +___ +###################################################################### +# NEON stuff +# +{{{ +my @X=map("q$_",(0..3)); +my ($T0,$T1,$T2,$T3,$T4,$T5)=("q8","q9","q10","q11","d24","d25"); +my $Xfer=$t4; +my $j=0; + +sub Dlo() { shift=~m|q([1]?[0-9])|?"d".($1*2):""; } +sub Dhi() { shift=~m|q([1]?[0-9])|?"d".($1*2+1):""; } + +sub AUTOLOAD() # thunk [simplified] x86-style perlasm +{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://; $opcode =~ s/_/\./; + my $arg = pop; + $arg = "#$arg" if ($arg*1 eq $arg); + $code .= "\t$opcode\t".join(',',@_,$arg)."\n"; +} + +sub Xupdate() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e,$f,$g,$h); + + &vext_8 ($T0,@X[0],@X[1],4); # X[1..4] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vext_8 ($T1,@X[2],@X[3],4); # X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T2,$T0,$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += X[9..12] + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T1,$T0,$sigma0[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T2,$T0,32-$sigma0[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T3,$T0,$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T1,$T1,$T2); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T3,$T0,32-$sigma0[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T1,$T1,$T3); # sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T5,&Dhi(@X[3]),$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (@X[0],@X[0],$T1); # X[0..3] += sigma0(X[1..4]) + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dhi(@X[3]),$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dhi(@X[3]),32-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); # sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (&Dlo(@X[0]),&Dlo(@X[0]),$T5);# X[0..1] += sigma1(X[14..15]) + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[0]); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T5,&Dlo(@X[0]),$sigma1[2]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); + eval(shift(@insns)); + eval(shift(@insns)); + &vshr_u32 ($T4,&Dlo(@X[0]),$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &vld1_32 ("{$T0}","[$Ktbl,:128]!"); + eval(shift(@insns)); + eval(shift(@insns)); + &vsli_32 ($T4,&Dlo(@X[0]),32-$sigma1[1]); + eval(shift(@insns)); + eval(shift(@insns)); + &veor ($T5,$T5,$T4); # sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 (&Dhi(@X[0]),&Dhi(@X[0]),$T5);# X[2..3] += sigma1(X[16..17]) + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 ($T0,$T0,@X[0]); + while($#insns>=2) { eval(shift(@insns)); } + &vst1_32 ("{$T0}","[$Xfer,:128]!"); + eval(shift(@insns)); + eval(shift(@insns)); + + push(@X,shift(@X)); # "rotate" X[] +} + +sub Xpreload() +{ use integer; + my $body = shift; + my @insns = (&$body,&$body,&$body,&$body); + my ($a,$b,$c,$d,$e,$f,$g,$h); + + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vld1_32 ("{$T0}","[$Ktbl,:128]!"); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vrev32_8 (@X[0],@X[0]); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + eval(shift(@insns)); + &vadd_i32 ($T0,$T0,@X[0]); + foreach (@insns) { eval; } # remaining instructions + &vst1_32 ("{$T0}","[$Xfer,:128]!"); + + push(@X,shift(@X)); # "rotate" X[] +} + +sub body_00_15 () { + ( + '($a,$b,$c,$d,$e,$f,$g,$h)=@V;'. + '&add ($h,$h,$t1)', # h+=X[i]+K[i] + '&eor ($t1,$f,$g)', + '&eor ($t0,$e,$e,"ror#".($Sigma1[1]-$Sigma1[0]))', + '&add ($a,$a,$t2)', # h+=Maj(a,b,c) from the past + '&and ($t1,$t1,$e)', + '&eor ($t2,$t0,$e,"ror#".($Sigma1[2]-$Sigma1[0]))', # Sigma1(e) + '&eor ($t0,$a,$a,"ror#".($Sigma0[1]-$Sigma0[0]))', + '&eor ($t1,$t1,$g)', # Ch(e,f,g) + '&add ($h,$h,$t2,"ror#$Sigma1[0]")', # h+=Sigma1(e) + '&eor ($t2,$a,$b)', # a^b, b^c in next round + '&eor ($t0,$t0,$a,"ror#".($Sigma0[2]-$Sigma0[0]))', # Sigma0(a) + '&add ($h,$h,$t1)', # h+=Ch(e,f,g) + '&ldr ($t1,sprintf "[sp,#%d]",4*(($j+1)&15)) if (($j&15)!=15);'. + '&ldr ($t1,"[$Ktbl]") if ($j==15);'. + '&ldr ($t1,"[sp,#64]") if ($j==31)', + '&and ($t3,$t3,$t2)', # (b^c)&=(a^b) + '&add ($d,$d,$h)', # d+=h + '&add ($h,$h,$t0,"ror#$Sigma0[0]");'. # h+=Sigma0(a) + '&eor ($t3,$t3,$b)', # Maj(a,b,c) + '$j++; unshift(@V,pop(@V)); ($t2,$t3)=($t3,$t2);' + ) +} + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.global sha256_block_data_order_neon +.type sha256_block_data_order_neon,%function +.align 4 +sha256_block_data_order_neon: +.LNEON: + stmdb sp!,{r4-r12,lr} + + sub $H,sp,#16*4+16 + adrl $Ktbl,K256 + bic $H,$H,#15 @ align for 128-bit stores + mov $t2,sp + mov sp,$H @ alloca + add $len,$inp,$len,lsl#6 @ len to point at the end of inp + + vld1.8 {@X[0]},[$inp]! + vld1.8 {@X[1]},[$inp]! + vld1.8 {@X[2]},[$inp]! + vld1.8 {@X[3]},[$inp]! + vld1.32 {$T0},[$Ktbl,:128]! + vld1.32 {$T1},[$Ktbl,:128]! + vld1.32 {$T2},[$Ktbl,:128]! + vld1.32 {$T3},[$Ktbl,:128]! + vrev32.8 @X[0],@X[0] @ yes, even on + str $ctx,[sp,#64] + vrev32.8 @X[1],@X[1] @ big-endian + str $inp,[sp,#68] + mov $Xfer,sp + vrev32.8 @X[2],@X[2] + str $len,[sp,#72] + vrev32.8 @X[3],@X[3] + str $t2,[sp,#76] @ save original sp + vadd.i32 $T0,$T0,@X[0] + vadd.i32 $T1,$T1,@X[1] + vst1.32 {$T0},[$Xfer,:128]! + vadd.i32 $T2,$T2,@X[2] + vst1.32 {$T1},[$Xfer,:128]! + vadd.i32 $T3,$T3,@X[3] + vst1.32 {$T2},[$Xfer,:128]! + vst1.32 {$T3},[$Xfer,:128]! + + ldmia $ctx,{$A-$H} + sub $Xfer,$Xfer,#64 + ldr $t1,[sp,#0] + eor $t2,$t2,$t2 + eor $t3,$B,$C + b .L_00_48 + +.align 4 +.L_00_48: +___ + &Xupdate(\&body_00_15); + &Xupdate(\&body_00_15); + &Xupdate(\&body_00_15); + &Xupdate(\&body_00_15); +$code.=<<___; + teq $t1,#0 @ check for K256 terminator + ldr $t1,[sp,#0] + sub $Xfer,$Xfer,#64 + bne .L_00_48 + + ldr $inp,[sp,#68] + ldr $t0,[sp,#72] + sub $Ktbl,$Ktbl,#256 @ rewind $Ktbl + teq $inp,$t0 + it eq + subeq $inp,$inp,#64 @ avoid SEGV + vld1.8 {@X[0]},[$inp]! @ load next input block + vld1.8 {@X[1]},[$inp]! + vld1.8 {@X[2]},[$inp]! + vld1.8 {@X[3]},[$inp]! + it ne + strne $inp,[sp,#68] + mov $Xfer,sp +___ + &Xpreload(\&body_00_15); + &Xpreload(\&body_00_15); + &Xpreload(\&body_00_15); + &Xpreload(\&body_00_15); +$code.=<<___; + ldr $t0,[$t1,#0] + add $A,$A,$t2 @ h+=Maj(a,b,c) from the past + ldr $t2,[$t1,#4] + ldr $t3,[$t1,#8] + ldr $t4,[$t1,#12] + add $A,$A,$t0 @ accumulate + ldr $t0,[$t1,#16] + add $B,$B,$t2 + ldr $t2,[$t1,#20] + add $C,$C,$t3 + ldr $t3,[$t1,#24] + add $D,$D,$t4 + ldr $t4,[$t1,#28] + add $E,$E,$t0 + str $A,[$t1],#4 + add $F,$F,$t2 + str $B,[$t1],#4 + add $G,$G,$t3 + str $C,[$t1],#4 + add $H,$H,$t4 + str $D,[$t1],#4 + stmia $t1,{$E-$H} + + ittte ne + movne $Xfer,sp + ldrne $t1,[sp,#0] + eorne $t2,$t2,$t2 + ldreq sp,[sp,#76] @ restore original sp + itt ne + eorne $t3,$B,$C + bne .L_00_48 + + ldmia sp!,{r4-r12,pc} +.size sha256_block_data_order_neon,.-sha256_block_data_order_neon +#endif +___ +}}} +###################################################################### +# ARMv8 stuff +# +{{{ +my ($ABCD,$EFGH,$abcd)=map("q$_",(0..2)); +my @MSG=map("q$_",(8..11)); +my ($W0,$W1,$ABCD_SAVE,$EFGH_SAVE)=map("q$_",(12..15)); +my $Ktbl="r3"; + +$code.=<<___; +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + +# ifdef __thumb2__ +# define INST(a,b,c,d) .byte c,d|0xc,a,b +# else +# define INST(a,b,c,d) .byte a,b,c,d +# endif + +.type sha256_block_data_order_armv8,%function +.align 5 +sha256_block_data_order_armv8: +.LARMv8: + vld1.32 {$ABCD,$EFGH},[$ctx] +# ifdef __thumb2__ + adr $Ktbl,.LARMv8 + sub $Ktbl,$Ktbl,#.LARMv8-K256 +# else + adrl $Ktbl,K256 +# endif + add $len,$inp,$len,lsl#6 @ len to point at the end of inp + +.Loop_v8: + vld1.8 {@MSG[0]-@MSG[1]},[$inp]! + vld1.8 {@MSG[2]-@MSG[3]},[$inp]! + vld1.32 {$W0},[$Ktbl]! + vrev32.8 @MSG[0],@MSG[0] + vrev32.8 @MSG[1],@MSG[1] + vrev32.8 @MSG[2],@MSG[2] + vrev32.8 @MSG[3],@MSG[3] + vmov $ABCD_SAVE,$ABCD @ offload + vmov $EFGH_SAVE,$EFGH + teq $inp,$len +___ +for($i=0;$i<12;$i++) { +$code.=<<___; + vld1.32 {$W1},[$Ktbl]! + vadd.i32 $W0,$W0,@MSG[0] + sha256su0 @MSG[0],@MSG[1] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + sha256su1 @MSG[0],@MSG[2],@MSG[3] +___ + ($W0,$W1)=($W1,$W0); push(@MSG,shift(@MSG)); +} +$code.=<<___; + vld1.32 {$W1},[$Ktbl]! + vadd.i32 $W0,$W0,@MSG[0] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + + vld1.32 {$W0},[$Ktbl]! + vadd.i32 $W1,$W1,@MSG[1] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W1 + sha256h2 $EFGH,$abcd,$W1 + + vld1.32 {$W1},[$Ktbl] + vadd.i32 $W0,$W0,@MSG[2] + sub $Ktbl,$Ktbl,#256-16 @ rewind + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W0 + sha256h2 $EFGH,$abcd,$W0 + + vadd.i32 $W1,$W1,@MSG[3] + vmov $abcd,$ABCD + sha256h $ABCD,$EFGH,$W1 + sha256h2 $EFGH,$abcd,$W1 + + vadd.i32 $ABCD,$ABCD,$ABCD_SAVE + vadd.i32 $EFGH,$EFGH,$EFGH_SAVE + it ne + bne .Loop_v8 + + vst1.32 {$ABCD,$EFGH},[$ctx] + + ret @ bx lr +.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8 +#endif +___ +}}} +$code.=<<___; +.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro\@openssl.org>" +.align 2 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.comm OPENSSL_armcap_P,4,4 +#endif +___ + +open SELF,$0; +while(<SELF>) { + next if (/^#!/); + last if (!s/^#/@/ and !/^$/); + print; +} +close SELF; + +{ my %opcode = ( + "sha256h" => 0xf3000c40, "sha256h2" => 0xf3100c40, + "sha256su0" => 0xf3ba03c0, "sha256su1" => 0xf3200c40 ); + + sub unsha256 { + my ($mnemonic,$arg)=@_; + + if ($arg =~ m/q([0-9]+)(?:,\s*q([0-9]+))?,\s*q([0-9]+)/o) { + my $word = $opcode{$mnemonic}|(($1&7)<<13)|(($1&8)<<19) + |(($2&7)<<17)|(($2&8)<<4) + |(($3&7)<<1) |(($3&8)<<2); + # since ARMv7 instructions are always encoded little-endian. + # correct solution is to use .inst directive, but older + # assemblers don't implement it:-( + sprintf "INST(0x%02x,0x%02x,0x%02x,0x%02x)\t@ %s %s", + $word&0xff,($word>>8)&0xff, + ($word>>16)&0xff,($word>>24)&0xff, + $mnemonic,$arg; + } + } +} + +foreach (split($/,$code)) { + + s/\`([^\`]*)\`/eval $1/geo; + + s/\b(sha256\w+)\s+(q.*)/unsha256($1,$2)/geo; + + s/\bret\b/bx lr/go or + s/\bbx\s+lr\b/.word\t0xe12fff1e/go; # make it possible to compile with -march=armv4 + + print $_,"\n"; +} + +close STDOUT; # enforce flush diff --git a/arch/arm/crypto/sha256-core.S_shipped b/arch/arm/crypto/sha256-core.S_shipped new file mode 100644 index 000000000000..555a1a8eec90 --- /dev/null +++ b/arch/arm/crypto/sha256-core.S_shipped @@ -0,0 +1,2808 @@ + +@ ==================================================================== +@ Written by Andy Polyakov <appro@openssl.org> for the OpenSSL +@ project. The module is, however, dual licensed under OpenSSL and +@ CRYPTOGAMS licenses depending on where you obtain it. For further +@ details see http://www.openssl.org/~appro/cryptogams/. +@ +@ Permission to use under GPL terms is granted. +@ ==================================================================== + +@ SHA256 block procedure for ARMv4. May 2007. + +@ Performance is ~2x better than gcc 3.4 generated code and in "abso- +@ lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per +@ byte [on single-issue Xscale PXA250 core]. + +@ July 2010. +@ +@ Rescheduling for dual-issue pipeline resulted in 22% improvement on +@ Cortex A8 core and ~20 cycles per processed byte. + +@ February 2011. +@ +@ Profiler-assisted and platform-specific optimization resulted in 16% +@ improvement on Cortex A8 core and ~15.4 cycles per processed byte. + +@ September 2013. +@ +@ Add NEON implementation. On Cortex A8 it was measured to process one +@ byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon +@ S4 does it in 12.5 cycles too, but it's 50% faster than integer-only +@ code (meaning that latter performs sub-optimally, nothing was done +@ about it). + +@ May 2014. +@ +@ Add ARMv8 code path performing at 2.0 cpb on Apple A7. + +#ifndef __KERNEL__ +# include "arm_arch.h" +#else +# define __ARM_ARCH__ __LINUX_ARM_ARCH__ +# define __ARM_MAX_ARCH__ 7 +#endif + +.text +#if __ARM_ARCH__<7 +.code 32 +#else +.syntax unified +# ifdef __thumb2__ +# define adrl adr +.thumb +# else +.code 32 +# endif +#endif + +.type K256,%object +.align 5 +K256: +.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 +.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 +.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 +.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 +.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc +.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da +.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 +.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 +.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 +.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 +.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 +.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 +.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 +.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 +.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 +.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +.size K256,.-K256 +.word 0 @ terminator +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.LOPENSSL_armcap: +.word OPENSSL_armcap_P-sha256_block_data_order +#endif +.align 5 + +.global sha256_block_data_order +.type sha256_block_data_order,%function +sha256_block_data_order: +#if __ARM_ARCH__<7 + sub r3,pc,#8 @ sha256_block_data_order +#else + adr r3,sha256_block_data_order +#endif +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + ldr r12,.LOPENSSL_armcap + ldr r12,[r3,r12] @ OPENSSL_armcap_P + tst r12,#ARMV8_SHA256 + bne .LARMv8 + tst r12,#ARMV7_NEON + bne .LNEON +#endif + add r2,r1,r2,lsl#6 @ len to point at the end of inp + stmdb sp!,{r0,r1,r2,r4-r11,lr} + ldmia r0,{r4,r5,r6,r7,r8,r9,r10,r11} + sub r14,r3,#256+32 @ K256 + sub sp,sp,#16*4 @ alloca(X[16]) +.Loop: +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 +# else + ldrb r2,[r1,#3] +# endif + eor r3,r5,r6 @ magic + eor r12,r12,r12 +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 0 +# if 0==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r8,r8,ror#5 + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r8,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 0 + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 0==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r8,r8,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r8,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r11,r11,r2 @ h+=X[i] + str r2,[sp,#0*4] + eor r2,r9,r10 + add r11,r11,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r8 + add r11,r11,r12 @ h+=K256[i] + eor r2,r2,r10 @ Ch(e,f,g) + eor r0,r4,r4,ror#11 + add r11,r11,r2 @ h+=Ch(e,f,g) +#if 0==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 0<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r4,r5 @ a^b, b^c in next round +#else + ldr r2,[sp,#2*4] @ from future BODY_16_xx + eor r12,r4,r5 @ a^b, b^c in next round + ldr r1,[sp,#15*4] @ from future BODY_16_xx +#endif + eor r0,r0,r4,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r7,r7,r11 @ d+=h + eor r3,r3,r5 @ Maj(a,b,c) + add r11,r11,r0,ror#2 @ h+=Sigma0(a) + @ add r11,r11,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 1 +# if 1==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r7,r7,ror#5 + add r11,r11,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r7,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 1 + add r11,r11,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 1==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r7,r7,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r7,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r10,r10,r2 @ h+=X[i] + str r2,[sp,#1*4] + eor r2,r8,r9 + add r10,r10,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r7 + add r10,r10,r3 @ h+=K256[i] + eor r2,r2,r9 @ Ch(e,f,g) + eor r0,r11,r11,ror#11 + add r10,r10,r2 @ h+=Ch(e,f,g) +#if 1==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 1<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r11,r4 @ a^b, b^c in next round +#else + ldr r2,[sp,#3*4] @ from future BODY_16_xx + eor r3,r11,r4 @ a^b, b^c in next round + ldr r1,[sp,#0*4] @ from future BODY_16_xx +#endif + eor r0,r0,r11,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r6,r6,r10 @ d+=h + eor r12,r12,r4 @ Maj(a,b,c) + add r10,r10,r0,ror#2 @ h+=Sigma0(a) + @ add r10,r10,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 2 +# if 2==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r6,r6,ror#5 + add r10,r10,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r6,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 2 + add r10,r10,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 2==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r6,r6,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r6,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r9,r9,r2 @ h+=X[i] + str r2,[sp,#2*4] + eor r2,r7,r8 + add r9,r9,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r6 + add r9,r9,r12 @ h+=K256[i] + eor r2,r2,r8 @ Ch(e,f,g) + eor r0,r10,r10,ror#11 + add r9,r9,r2 @ h+=Ch(e,f,g) +#if 2==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 2<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r10,r11 @ a^b, b^c in next round +#else + ldr r2,[sp,#4*4] @ from future BODY_16_xx + eor r12,r10,r11 @ a^b, b^c in next round + ldr r1,[sp,#1*4] @ from future BODY_16_xx +#endif + eor r0,r0,r10,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r5,r5,r9 @ d+=h + eor r3,r3,r11 @ Maj(a,b,c) + add r9,r9,r0,ror#2 @ h+=Sigma0(a) + @ add r9,r9,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 3 +# if 3==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r5,r5,ror#5 + add r9,r9,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r5,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 3 + add r9,r9,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 3==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r5,r5,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r5,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r8,r8,r2 @ h+=X[i] + str r2,[sp,#3*4] + eor r2,r6,r7 + add r8,r8,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r5 + add r8,r8,r3 @ h+=K256[i] + eor r2,r2,r7 @ Ch(e,f,g) + eor r0,r9,r9,ror#11 + add r8,r8,r2 @ h+=Ch(e,f,g) +#if 3==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 3<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r9,r10 @ a^b, b^c in next round +#else + ldr r2,[sp,#5*4] @ from future BODY_16_xx + eor r3,r9,r10 @ a^b, b^c in next round + ldr r1,[sp,#2*4] @ from future BODY_16_xx +#endif + eor r0,r0,r9,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r4,r4,r8 @ d+=h + eor r12,r12,r10 @ Maj(a,b,c) + add r8,r8,r0,ror#2 @ h+=Sigma0(a) + @ add r8,r8,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 4 +# if 4==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r4,r4,ror#5 + add r8,r8,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r4,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 4 + add r8,r8,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 4==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r4,r4,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r4,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r7,r7,r2 @ h+=X[i] + str r2,[sp,#4*4] + eor r2,r5,r6 + add r7,r7,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r4 + add r7,r7,r12 @ h+=K256[i] + eor r2,r2,r6 @ Ch(e,f,g) + eor r0,r8,r8,ror#11 + add r7,r7,r2 @ h+=Ch(e,f,g) +#if 4==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 4<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r8,r9 @ a^b, b^c in next round +#else + ldr r2,[sp,#6*4] @ from future BODY_16_xx + eor r12,r8,r9 @ a^b, b^c in next round + ldr r1,[sp,#3*4] @ from future BODY_16_xx +#endif + eor r0,r0,r8,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r11,r11,r7 @ d+=h + eor r3,r3,r9 @ Maj(a,b,c) + add r7,r7,r0,ror#2 @ h+=Sigma0(a) + @ add r7,r7,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 5 +# if 5==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r11,r11,ror#5 + add r7,r7,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r11,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 5 + add r7,r7,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 5==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r11,r11,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r11,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r6,r6,r2 @ h+=X[i] + str r2,[sp,#5*4] + eor r2,r4,r5 + add r6,r6,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r11 + add r6,r6,r3 @ h+=K256[i] + eor r2,r2,r5 @ Ch(e,f,g) + eor r0,r7,r7,ror#11 + add r6,r6,r2 @ h+=Ch(e,f,g) +#if 5==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 5<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r7,r8 @ a^b, b^c in next round +#else + ldr r2,[sp,#7*4] @ from future BODY_16_xx + eor r3,r7,r8 @ a^b, b^c in next round + ldr r1,[sp,#4*4] @ from future BODY_16_xx +#endif + eor r0,r0,r7,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r10,r10,r6 @ d+=h + eor r12,r12,r8 @ Maj(a,b,c) + add r6,r6,r0,ror#2 @ h+=Sigma0(a) + @ add r6,r6,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 6 +# if 6==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r10,r10,ror#5 + add r6,r6,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r10,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 6 + add r6,r6,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 6==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r10,r10,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r10,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r5,r5,r2 @ h+=X[i] + str r2,[sp,#6*4] + eor r2,r11,r4 + add r5,r5,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r10 + add r5,r5,r12 @ h+=K256[i] + eor r2,r2,r4 @ Ch(e,f,g) + eor r0,r6,r6,ror#11 + add r5,r5,r2 @ h+=Ch(e,f,g) +#if 6==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 6<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r6,r7 @ a^b, b^c in next round +#else + ldr r2,[sp,#8*4] @ from future BODY_16_xx + eor r12,r6,r7 @ a^b, b^c in next round + ldr r1,[sp,#5*4] @ from future BODY_16_xx +#endif + eor r0,r0,r6,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r9,r9,r5 @ d+=h + eor r3,r3,r7 @ Maj(a,b,c) + add r5,r5,r0,ror#2 @ h+=Sigma0(a) + @ add r5,r5,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 7 +# if 7==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r9,r9,ror#5 + add r5,r5,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r9,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 7 + add r5,r5,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 7==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r9,r9,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r9,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r4,r4,r2 @ h+=X[i] + str r2,[sp,#7*4] + eor r2,r10,r11 + add r4,r4,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r9 + add r4,r4,r3 @ h+=K256[i] + eor r2,r2,r11 @ Ch(e,f,g) + eor r0,r5,r5,ror#11 + add r4,r4,r2 @ h+=Ch(e,f,g) +#if 7==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 7<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r5,r6 @ a^b, b^c in next round +#else + ldr r2,[sp,#9*4] @ from future BODY_16_xx + eor r3,r5,r6 @ a^b, b^c in next round + ldr r1,[sp,#6*4] @ from future BODY_16_xx +#endif + eor r0,r0,r5,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r8,r8,r4 @ d+=h + eor r12,r12,r6 @ Maj(a,b,c) + add r4,r4,r0,ror#2 @ h+=Sigma0(a) + @ add r4,r4,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 8 +# if 8==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r8,r8,ror#5 + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r8,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 8 + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 8==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r8,r8,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r8,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r11,r11,r2 @ h+=X[i] + str r2,[sp,#8*4] + eor r2,r9,r10 + add r11,r11,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r8 + add r11,r11,r12 @ h+=K256[i] + eor r2,r2,r10 @ Ch(e,f,g) + eor r0,r4,r4,ror#11 + add r11,r11,r2 @ h+=Ch(e,f,g) +#if 8==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 8<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r4,r5 @ a^b, b^c in next round +#else + ldr r2,[sp,#10*4] @ from future BODY_16_xx + eor r12,r4,r5 @ a^b, b^c in next round + ldr r1,[sp,#7*4] @ from future BODY_16_xx +#endif + eor r0,r0,r4,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r7,r7,r11 @ d+=h + eor r3,r3,r5 @ Maj(a,b,c) + add r11,r11,r0,ror#2 @ h+=Sigma0(a) + @ add r11,r11,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 9 +# if 9==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r7,r7,ror#5 + add r11,r11,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r7,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 9 + add r11,r11,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 9==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r7,r7,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r7,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r10,r10,r2 @ h+=X[i] + str r2,[sp,#9*4] + eor r2,r8,r9 + add r10,r10,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r7 + add r10,r10,r3 @ h+=K256[i] + eor r2,r2,r9 @ Ch(e,f,g) + eor r0,r11,r11,ror#11 + add r10,r10,r2 @ h+=Ch(e,f,g) +#if 9==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 9<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r11,r4 @ a^b, b^c in next round +#else + ldr r2,[sp,#11*4] @ from future BODY_16_xx + eor r3,r11,r4 @ a^b, b^c in next round + ldr r1,[sp,#8*4] @ from future BODY_16_xx +#endif + eor r0,r0,r11,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r6,r6,r10 @ d+=h + eor r12,r12,r4 @ Maj(a,b,c) + add r10,r10,r0,ror#2 @ h+=Sigma0(a) + @ add r10,r10,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 10 +# if 10==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r6,r6,ror#5 + add r10,r10,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r6,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 10 + add r10,r10,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 10==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r6,r6,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r6,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r9,r9,r2 @ h+=X[i] + str r2,[sp,#10*4] + eor r2,r7,r8 + add r9,r9,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r6 + add r9,r9,r12 @ h+=K256[i] + eor r2,r2,r8 @ Ch(e,f,g) + eor r0,r10,r10,ror#11 + add r9,r9,r2 @ h+=Ch(e,f,g) +#if 10==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 10<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r10,r11 @ a^b, b^c in next round +#else + ldr r2,[sp,#12*4] @ from future BODY_16_xx + eor r12,r10,r11 @ a^b, b^c in next round + ldr r1,[sp,#9*4] @ from future BODY_16_xx +#endif + eor r0,r0,r10,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r5,r5,r9 @ d+=h + eor r3,r3,r11 @ Maj(a,b,c) + add r9,r9,r0,ror#2 @ h+=Sigma0(a) + @ add r9,r9,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 11 +# if 11==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r5,r5,ror#5 + add r9,r9,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r5,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 11 + add r9,r9,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 11==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r5,r5,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r5,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r8,r8,r2 @ h+=X[i] + str r2,[sp,#11*4] + eor r2,r6,r7 + add r8,r8,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r5 + add r8,r8,r3 @ h+=K256[i] + eor r2,r2,r7 @ Ch(e,f,g) + eor r0,r9,r9,ror#11 + add r8,r8,r2 @ h+=Ch(e,f,g) +#if 11==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 11<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r9,r10 @ a^b, b^c in next round +#else + ldr r2,[sp,#13*4] @ from future BODY_16_xx + eor r3,r9,r10 @ a^b, b^c in next round + ldr r1,[sp,#10*4] @ from future BODY_16_xx +#endif + eor r0,r0,r9,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r4,r4,r8 @ d+=h + eor r12,r12,r10 @ Maj(a,b,c) + add r8,r8,r0,ror#2 @ h+=Sigma0(a) + @ add r8,r8,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 12 +# if 12==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r4,r4,ror#5 + add r8,r8,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r4,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 12 + add r8,r8,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 12==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r4,r4,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r4,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r7,r7,r2 @ h+=X[i] + str r2,[sp,#12*4] + eor r2,r5,r6 + add r7,r7,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r4 + add r7,r7,r12 @ h+=K256[i] + eor r2,r2,r6 @ Ch(e,f,g) + eor r0,r8,r8,ror#11 + add r7,r7,r2 @ h+=Ch(e,f,g) +#if 12==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 12<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r8,r9 @ a^b, b^c in next round +#else + ldr r2,[sp,#14*4] @ from future BODY_16_xx + eor r12,r8,r9 @ a^b, b^c in next round + ldr r1,[sp,#11*4] @ from future BODY_16_xx +#endif + eor r0,r0,r8,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r11,r11,r7 @ d+=h + eor r3,r3,r9 @ Maj(a,b,c) + add r7,r7,r0,ror#2 @ h+=Sigma0(a) + @ add r7,r7,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 13 +# if 13==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r11,r11,ror#5 + add r7,r7,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r11,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 13 + add r7,r7,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 13==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r11,r11,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r11,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r6,r6,r2 @ h+=X[i] + str r2,[sp,#13*4] + eor r2,r4,r5 + add r6,r6,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r11 + add r6,r6,r3 @ h+=K256[i] + eor r2,r2,r5 @ Ch(e,f,g) + eor r0,r7,r7,ror#11 + add r6,r6,r2 @ h+=Ch(e,f,g) +#if 13==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 13<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r7,r8 @ a^b, b^c in next round +#else + ldr r2,[sp,#15*4] @ from future BODY_16_xx + eor r3,r7,r8 @ a^b, b^c in next round + ldr r1,[sp,#12*4] @ from future BODY_16_xx +#endif + eor r0,r0,r7,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r10,r10,r6 @ d+=h + eor r12,r12,r8 @ Maj(a,b,c) + add r6,r6,r0,ror#2 @ h+=Sigma0(a) + @ add r6,r6,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 14 +# if 14==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r10,r10,ror#5 + add r6,r6,r12 @ h+=Maj(a,b,c) from the past + eor r0,r0,r10,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 14 + add r6,r6,r12 @ h+=Maj(a,b,c) from the past + ldrb r12,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r12,lsl#8 + ldrb r12,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 14==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r10,r10,ror#5 + orr r2,r2,r12,lsl#24 + eor r0,r0,r10,ror#19 @ Sigma1(e) +#endif + ldr r12,[r14],#4 @ *K256++ + add r5,r5,r2 @ h+=X[i] + str r2,[sp,#14*4] + eor r2,r11,r4 + add r5,r5,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r10 + add r5,r5,r12 @ h+=K256[i] + eor r2,r2,r4 @ Ch(e,f,g) + eor r0,r6,r6,ror#11 + add r5,r5,r2 @ h+=Ch(e,f,g) +#if 14==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 14<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r6,r7 @ a^b, b^c in next round +#else + ldr r2,[sp,#0*4] @ from future BODY_16_xx + eor r12,r6,r7 @ a^b, b^c in next round + ldr r1,[sp,#13*4] @ from future BODY_16_xx +#endif + eor r0,r0,r6,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r9,r9,r5 @ d+=h + eor r3,r3,r7 @ Maj(a,b,c) + add r5,r5,r0,ror#2 @ h+=Sigma0(a) + @ add r5,r5,r3 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + @ ldr r2,[r1],#4 @ 15 +# if 15==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r9,r9,ror#5 + add r5,r5,r3 @ h+=Maj(a,b,c) from the past + eor r0,r0,r9,ror#19 @ Sigma1(e) +# ifndef __ARMEB__ + rev r2,r2 +# endif +#else + @ ldrb r2,[r1,#3] @ 15 + add r5,r5,r3 @ h+=Maj(a,b,c) from the past + ldrb r3,[r1,#2] + ldrb r0,[r1,#1] + orr r2,r2,r3,lsl#8 + ldrb r3,[r1],#4 + orr r2,r2,r0,lsl#16 +# if 15==15 + str r1,[sp,#17*4] @ make room for r1 +# endif + eor r0,r9,r9,ror#5 + orr r2,r2,r3,lsl#24 + eor r0,r0,r9,ror#19 @ Sigma1(e) +#endif + ldr r3,[r14],#4 @ *K256++ + add r4,r4,r2 @ h+=X[i] + str r2,[sp,#15*4] + eor r2,r10,r11 + add r4,r4,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r9 + add r4,r4,r3 @ h+=K256[i] + eor r2,r2,r11 @ Ch(e,f,g) + eor r0,r5,r5,ror#11 + add r4,r4,r2 @ h+=Ch(e,f,g) +#if 15==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 15<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r5,r6 @ a^b, b^c in next round +#else + ldr r2,[sp,#1*4] @ from future BODY_16_xx + eor r3,r5,r6 @ a^b, b^c in next round + ldr r1,[sp,#14*4] @ from future BODY_16_xx +#endif + eor r0,r0,r5,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r8,r8,r4 @ d+=h + eor r12,r12,r6 @ Maj(a,b,c) + add r4,r4,r0,ror#2 @ h+=Sigma0(a) + @ add r4,r4,r12 @ h+=Maj(a,b,c) +.Lrounds_16_xx: + @ ldr r2,[sp,#1*4] @ 16 + @ ldr r1,[sp,#14*4] + mov r0,r2,ror#7 + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#0*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#9*4] + + add r12,r12,r0 + eor r0,r8,r8,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r8,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r11,r11,r2 @ h+=X[i] + str r2,[sp,#0*4] + eor r2,r9,r10 + add r11,r11,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r8 + add r11,r11,r12 @ h+=K256[i] + eor r2,r2,r10 @ Ch(e,f,g) + eor r0,r4,r4,ror#11 + add r11,r11,r2 @ h+=Ch(e,f,g) +#if 16==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 16<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r4,r5 @ a^b, b^c in next round +#else + ldr r2,[sp,#2*4] @ from future BODY_16_xx + eor r12,r4,r5 @ a^b, b^c in next round + ldr r1,[sp,#15*4] @ from future BODY_16_xx +#endif + eor r0,r0,r4,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r7,r7,r11 @ d+=h + eor r3,r3,r5 @ Maj(a,b,c) + add r11,r11,r0,ror#2 @ h+=Sigma0(a) + @ add r11,r11,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#2*4] @ 17 + @ ldr r1,[sp,#15*4] + mov r0,r2,ror#7 + add r11,r11,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#1*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#10*4] + + add r3,r3,r0 + eor r0,r7,r7,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r7,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r10,r10,r2 @ h+=X[i] + str r2,[sp,#1*4] + eor r2,r8,r9 + add r10,r10,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r7 + add r10,r10,r3 @ h+=K256[i] + eor r2,r2,r9 @ Ch(e,f,g) + eor r0,r11,r11,ror#11 + add r10,r10,r2 @ h+=Ch(e,f,g) +#if 17==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 17<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r11,r4 @ a^b, b^c in next round +#else + ldr r2,[sp,#3*4] @ from future BODY_16_xx + eor r3,r11,r4 @ a^b, b^c in next round + ldr r1,[sp,#0*4] @ from future BODY_16_xx +#endif + eor r0,r0,r11,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r6,r6,r10 @ d+=h + eor r12,r12,r4 @ Maj(a,b,c) + add r10,r10,r0,ror#2 @ h+=Sigma0(a) + @ add r10,r10,r12 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#3*4] @ 18 + @ ldr r1,[sp,#0*4] + mov r0,r2,ror#7 + add r10,r10,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#2*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#11*4] + + add r12,r12,r0 + eor r0,r6,r6,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r6,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r9,r9,r2 @ h+=X[i] + str r2,[sp,#2*4] + eor r2,r7,r8 + add r9,r9,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r6 + add r9,r9,r12 @ h+=K256[i] + eor r2,r2,r8 @ Ch(e,f,g) + eor r0,r10,r10,ror#11 + add r9,r9,r2 @ h+=Ch(e,f,g) +#if 18==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 18<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r10,r11 @ a^b, b^c in next round +#else + ldr r2,[sp,#4*4] @ from future BODY_16_xx + eor r12,r10,r11 @ a^b, b^c in next round + ldr r1,[sp,#1*4] @ from future BODY_16_xx +#endif + eor r0,r0,r10,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r5,r5,r9 @ d+=h + eor r3,r3,r11 @ Maj(a,b,c) + add r9,r9,r0,ror#2 @ h+=Sigma0(a) + @ add r9,r9,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#4*4] @ 19 + @ ldr r1,[sp,#1*4] + mov r0,r2,ror#7 + add r9,r9,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#3*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#12*4] + + add r3,r3,r0 + eor r0,r5,r5,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r5,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r8,r8,r2 @ h+=X[i] + str r2,[sp,#3*4] + eor r2,r6,r7 + add r8,r8,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r5 + add r8,r8,r3 @ h+=K256[i] + eor r2,r2,r7 @ Ch(e,f,g) + eor r0,r9,r9,ror#11 + add r8,r8,r2 @ h+=Ch(e,f,g) +#if 19==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 19<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r9,r10 @ a^b, b^c in next round +#else + ldr r2,[sp,#5*4] @ from future BODY_16_xx + eor r3,r9,r10 @ a^b, b^c in next round + ldr r1,[sp,#2*4] @ from future BODY_16_xx +#endif + eor r0,r0,r9,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r4,r4,r8 @ d+=h + eor r12,r12,r10 @ Maj(a,b,c) + add r8,r8,r0,ror#2 @ h+=Sigma0(a) + @ add r8,r8,r12 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#5*4] @ 20 + @ ldr r1,[sp,#2*4] + mov r0,r2,ror#7 + add r8,r8,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#4*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#13*4] + + add r12,r12,r0 + eor r0,r4,r4,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r4,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r7,r7,r2 @ h+=X[i] + str r2,[sp,#4*4] + eor r2,r5,r6 + add r7,r7,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r4 + add r7,r7,r12 @ h+=K256[i] + eor r2,r2,r6 @ Ch(e,f,g) + eor r0,r8,r8,ror#11 + add r7,r7,r2 @ h+=Ch(e,f,g) +#if 20==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 20<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r8,r9 @ a^b, b^c in next round +#else + ldr r2,[sp,#6*4] @ from future BODY_16_xx + eor r12,r8,r9 @ a^b, b^c in next round + ldr r1,[sp,#3*4] @ from future BODY_16_xx +#endif + eor r0,r0,r8,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r11,r11,r7 @ d+=h + eor r3,r3,r9 @ Maj(a,b,c) + add r7,r7,r0,ror#2 @ h+=Sigma0(a) + @ add r7,r7,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#6*4] @ 21 + @ ldr r1,[sp,#3*4] + mov r0,r2,ror#7 + add r7,r7,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#5*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#14*4] + + add r3,r3,r0 + eor r0,r11,r11,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r11,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r6,r6,r2 @ h+=X[i] + str r2,[sp,#5*4] + eor r2,r4,r5 + add r6,r6,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r11 + add r6,r6,r3 @ h+=K256[i] + eor r2,r2,r5 @ Ch(e,f,g) + eor r0,r7,r7,ror#11 + add r6,r6,r2 @ h+=Ch(e,f,g) +#if 21==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 21<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r7,r8 @ a^b, b^c in next round +#else + ldr r2,[sp,#7*4] @ from future BODY_16_xx + eor r3,r7,r8 @ a^b, b^c in next round + ldr r1,[sp,#4*4] @ from future BODY_16_xx +#endif + eor r0,r0,r7,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r10,r10,r6 @ d+=h + eor r12,r12,r8 @ Maj(a,b,c) + add r6,r6,r0,ror#2 @ h+=Sigma0(a) + @ add r6,r6,r12 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#7*4] @ 22 + @ ldr r1,[sp,#4*4] + mov r0,r2,ror#7 + add r6,r6,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#6*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#15*4] + + add r12,r12,r0 + eor r0,r10,r10,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r10,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r5,r5,r2 @ h+=X[i] + str r2,[sp,#6*4] + eor r2,r11,r4 + add r5,r5,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r10 + add r5,r5,r12 @ h+=K256[i] + eor r2,r2,r4 @ Ch(e,f,g) + eor r0,r6,r6,ror#11 + add r5,r5,r2 @ h+=Ch(e,f,g) +#if 22==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 22<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r6,r7 @ a^b, b^c in next round +#else + ldr r2,[sp,#8*4] @ from future BODY_16_xx + eor r12,r6,r7 @ a^b, b^c in next round + ldr r1,[sp,#5*4] @ from future BODY_16_xx +#endif + eor r0,r0,r6,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r9,r9,r5 @ d+=h + eor r3,r3,r7 @ Maj(a,b,c) + add r5,r5,r0,ror#2 @ h+=Sigma0(a) + @ add r5,r5,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#8*4] @ 23 + @ ldr r1,[sp,#5*4] + mov r0,r2,ror#7 + add r5,r5,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#7*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#0*4] + + add r3,r3,r0 + eor r0,r9,r9,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r9,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r4,r4,r2 @ h+=X[i] + str r2,[sp,#7*4] + eor r2,r10,r11 + add r4,r4,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r9 + add r4,r4,r3 @ h+=K256[i] + eor r2,r2,r11 @ Ch(e,f,g) + eor r0,r5,r5,ror#11 + add r4,r4,r2 @ h+=Ch(e,f,g) +#if 23==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 23<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r5,r6 @ a^b, b^c in next round +#else + ldr r2,[sp,#9*4] @ from future BODY_16_xx + eor r3,r5,r6 @ a^b, b^c in next round + ldr r1,[sp,#6*4] @ from future BODY_16_xx +#endif + eor r0,r0,r5,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r8,r8,r4 @ d+=h + eor r12,r12,r6 @ Maj(a,b,c) + add r4,r4,r0,ror#2 @ h+=Sigma0(a) + @ add r4,r4,r12 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#9*4] @ 24 + @ ldr r1,[sp,#6*4] + mov r0,r2,ror#7 + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#8*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#1*4] + + add r12,r12,r0 + eor r0,r8,r8,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r8,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r11,r11,r2 @ h+=X[i] + str r2,[sp,#8*4] + eor r2,r9,r10 + add r11,r11,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r8 + add r11,r11,r12 @ h+=K256[i] + eor r2,r2,r10 @ Ch(e,f,g) + eor r0,r4,r4,ror#11 + add r11,r11,r2 @ h+=Ch(e,f,g) +#if 24==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 24<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r4,r5 @ a^b, b^c in next round +#else + ldr r2,[sp,#10*4] @ from future BODY_16_xx + eor r12,r4,r5 @ a^b, b^c in next round + ldr r1,[sp,#7*4] @ from future BODY_16_xx +#endif + eor r0,r0,r4,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r7,r7,r11 @ d+=h + eor r3,r3,r5 @ Maj(a,b,c) + add r11,r11,r0,ror#2 @ h+=Sigma0(a) + @ add r11,r11,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#10*4] @ 25 + @ ldr r1,[sp,#7*4] + mov r0,r2,ror#7 + add r11,r11,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#9*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#2*4] + + add r3,r3,r0 + eor r0,r7,r7,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r7,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r10,r10,r2 @ h+=X[i] + str r2,[sp,#9*4] + eor r2,r8,r9 + add r10,r10,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r7 + add r10,r10,r3 @ h+=K256[i] + eor r2,r2,r9 @ Ch(e,f,g) + eor r0,r11,r11,ror#11 + add r10,r10,r2 @ h+=Ch(e,f,g) +#if 25==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 25<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r11,r4 @ a^b, b^c in next round +#else + ldr r2,[sp,#11*4] @ from future BODY_16_xx + eor r3,r11,r4 @ a^b, b^c in next round + ldr r1,[sp,#8*4] @ from future BODY_16_xx +#endif + eor r0,r0,r11,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r6,r6,r10 @ d+=h + eor r12,r12,r4 @ Maj(a,b,c) + add r10,r10,r0,ror#2 @ h+=Sigma0(a) + @ add r10,r10,r12 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#11*4] @ 26 + @ ldr r1,[sp,#8*4] + mov r0,r2,ror#7 + add r10,r10,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#10*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#3*4] + + add r12,r12,r0 + eor r0,r6,r6,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r6,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r9,r9,r2 @ h+=X[i] + str r2,[sp,#10*4] + eor r2,r7,r8 + add r9,r9,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r6 + add r9,r9,r12 @ h+=K256[i] + eor r2,r2,r8 @ Ch(e,f,g) + eor r0,r10,r10,ror#11 + add r9,r9,r2 @ h+=Ch(e,f,g) +#if 26==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 26<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r10,r11 @ a^b, b^c in next round +#else + ldr r2,[sp,#12*4] @ from future BODY_16_xx + eor r12,r10,r11 @ a^b, b^c in next round + ldr r1,[sp,#9*4] @ from future BODY_16_xx +#endif + eor r0,r0,r10,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r5,r5,r9 @ d+=h + eor r3,r3,r11 @ Maj(a,b,c) + add r9,r9,r0,ror#2 @ h+=Sigma0(a) + @ add r9,r9,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#12*4] @ 27 + @ ldr r1,[sp,#9*4] + mov r0,r2,ror#7 + add r9,r9,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#11*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#4*4] + + add r3,r3,r0 + eor r0,r5,r5,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r5,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r8,r8,r2 @ h+=X[i] + str r2,[sp,#11*4] + eor r2,r6,r7 + add r8,r8,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r5 + add r8,r8,r3 @ h+=K256[i] + eor r2,r2,r7 @ Ch(e,f,g) + eor r0,r9,r9,ror#11 + add r8,r8,r2 @ h+=Ch(e,f,g) +#if 27==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 27<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r9,r10 @ a^b, b^c in next round +#else + ldr r2,[sp,#13*4] @ from future BODY_16_xx + eor r3,r9,r10 @ a^b, b^c in next round + ldr r1,[sp,#10*4] @ from future BODY_16_xx +#endif + eor r0,r0,r9,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r4,r4,r8 @ d+=h + eor r12,r12,r10 @ Maj(a,b,c) + add r8,r8,r0,ror#2 @ h+=Sigma0(a) + @ add r8,r8,r12 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#13*4] @ 28 + @ ldr r1,[sp,#10*4] + mov r0,r2,ror#7 + add r8,r8,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#12*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#5*4] + + add r12,r12,r0 + eor r0,r4,r4,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r4,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r7,r7,r2 @ h+=X[i] + str r2,[sp,#12*4] + eor r2,r5,r6 + add r7,r7,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r4 + add r7,r7,r12 @ h+=K256[i] + eor r2,r2,r6 @ Ch(e,f,g) + eor r0,r8,r8,ror#11 + add r7,r7,r2 @ h+=Ch(e,f,g) +#if 28==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 28<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r8,r9 @ a^b, b^c in next round +#else + ldr r2,[sp,#14*4] @ from future BODY_16_xx + eor r12,r8,r9 @ a^b, b^c in next round + ldr r1,[sp,#11*4] @ from future BODY_16_xx +#endif + eor r0,r0,r8,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r11,r11,r7 @ d+=h + eor r3,r3,r9 @ Maj(a,b,c) + add r7,r7,r0,ror#2 @ h+=Sigma0(a) + @ add r7,r7,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#14*4] @ 29 + @ ldr r1,[sp,#11*4] + mov r0,r2,ror#7 + add r7,r7,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#13*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#6*4] + + add r3,r3,r0 + eor r0,r11,r11,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r11,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r6,r6,r2 @ h+=X[i] + str r2,[sp,#13*4] + eor r2,r4,r5 + add r6,r6,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r11 + add r6,r6,r3 @ h+=K256[i] + eor r2,r2,r5 @ Ch(e,f,g) + eor r0,r7,r7,ror#11 + add r6,r6,r2 @ h+=Ch(e,f,g) +#if 29==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 29<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r7,r8 @ a^b, b^c in next round +#else + ldr r2,[sp,#15*4] @ from future BODY_16_xx + eor r3,r7,r8 @ a^b, b^c in next round + ldr r1,[sp,#12*4] @ from future BODY_16_xx +#endif + eor r0,r0,r7,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r10,r10,r6 @ d+=h + eor r12,r12,r8 @ Maj(a,b,c) + add r6,r6,r0,ror#2 @ h+=Sigma0(a) + @ add r6,r6,r12 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#15*4] @ 30 + @ ldr r1,[sp,#12*4] + mov r0,r2,ror#7 + add r6,r6,r12 @ h+=Maj(a,b,c) from the past + mov r12,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r12,r12,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#14*4] + eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#7*4] + + add r12,r12,r0 + eor r0,r10,r10,ror#5 @ from BODY_00_15 + add r2,r2,r12 + eor r0,r0,r10,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r12,[r14],#4 @ *K256++ + add r5,r5,r2 @ h+=X[i] + str r2,[sp,#14*4] + eor r2,r11,r4 + add r5,r5,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r10 + add r5,r5,r12 @ h+=K256[i] + eor r2,r2,r4 @ Ch(e,f,g) + eor r0,r6,r6,ror#11 + add r5,r5,r2 @ h+=Ch(e,f,g) +#if 30==31 + and r12,r12,#0xff + cmp r12,#0xf2 @ done? +#endif +#if 30<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r12,r6,r7 @ a^b, b^c in next round +#else + ldr r2,[sp,#0*4] @ from future BODY_16_xx + eor r12,r6,r7 @ a^b, b^c in next round + ldr r1,[sp,#13*4] @ from future BODY_16_xx +#endif + eor r0,r0,r6,ror#20 @ Sigma0(a) + and r3,r3,r12 @ (b^c)&=(a^b) + add r9,r9,r5 @ d+=h + eor r3,r3,r7 @ Maj(a,b,c) + add r5,r5,r0,ror#2 @ h+=Sigma0(a) + @ add r5,r5,r3 @ h+=Maj(a,b,c) + @ ldr r2,[sp,#0*4] @ 31 + @ ldr r1,[sp,#13*4] + mov r0,r2,ror#7 + add r5,r5,r3 @ h+=Maj(a,b,c) from the past + mov r3,r1,ror#17 + eor r0,r0,r2,ror#18 + eor r3,r3,r1,ror#19 + eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) + ldr r2,[sp,#15*4] + eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) + ldr r1,[sp,#8*4] + + add r3,r3,r0 + eor r0,r9,r9,ror#5 @ from BODY_00_15 + add r2,r2,r3 + eor r0,r0,r9,ror#19 @ Sigma1(e) + add r2,r2,r1 @ X[i] + ldr r3,[r14],#4 @ *K256++ + add r4,r4,r2 @ h+=X[i] + str r2,[sp,#15*4] + eor r2,r10,r11 + add r4,r4,r0,ror#6 @ h+=Sigma1(e) + and r2,r2,r9 + add r4,r4,r3 @ h+=K256[i] + eor r2,r2,r11 @ Ch(e,f,g) + eor r0,r5,r5,ror#11 + add r4,r4,r2 @ h+=Ch(e,f,g) +#if 31==31 + and r3,r3,#0xff + cmp r3,#0xf2 @ done? +#endif +#if 31<15 +# if __ARM_ARCH__>=7 + ldr r2,[r1],#4 @ prefetch +# else + ldrb r2,[r1,#3] +# endif + eor r3,r5,r6 @ a^b, b^c in next round +#else + ldr r2,[sp,#1*4] @ from future BODY_16_xx + eor r3,r5,r6 @ a^b, b^c in next round + ldr r1,[sp,#14*4] @ from future BODY_16_xx +#endif + eor r0,r0,r5,ror#20 @ Sigma0(a) + and r12,r12,r3 @ (b^c)&=(a^b) + add r8,r8,r4 @ d+=h + eor r12,r12,r6 @ Maj(a,b,c) + add r4,r4,r0,ror#2 @ h+=Sigma0(a) + @ add r4,r4,r12 @ h+=Maj(a,b,c) +#if __ARM_ARCH__>=7 + ite eq @ Thumb2 thing, sanity check in ARM +#endif + ldreq r3,[sp,#16*4] @ pull ctx + bne .Lrounds_16_xx + + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + ldr r0,[r3,#0] + ldr r2,[r3,#4] + ldr r12,[r3,#8] + add r4,r4,r0 + ldr r0,[r3,#12] + add r5,r5,r2 + ldr r2,[r3,#16] + add r6,r6,r12 + ldr r12,[r3,#20] + add r7,r7,r0 + ldr r0,[r3,#24] + add r8,r8,r2 + ldr r2,[r3,#28] + add r9,r9,r12 + ldr r1,[sp,#17*4] @ pull inp + ldr r12,[sp,#18*4] @ pull inp+len + add r10,r10,r0 + add r11,r11,r2 + stmia r3,{r4,r5,r6,r7,r8,r9,r10,r11} + cmp r1,r12 + sub r14,r14,#256 @ rewind Ktbl + bne .Loop + + add sp,sp,#19*4 @ destroy frame +#if __ARM_ARCH__>=5 + ldmia sp!,{r4-r11,pc} +#else + ldmia sp!,{r4-r11,lr} + tst lr,#1 + moveq pc,lr @ be binary compatible with V4, yet + .word 0xe12fff1e @ interoperable with Thumb ISA:-) +#endif +.size sha256_block_data_order,.-sha256_block_data_order +#if __ARM_MAX_ARCH__>=7 +.arch armv7-a +.fpu neon + +.global sha256_block_data_order_neon +.type sha256_block_data_order_neon,%function +.align 4 +sha256_block_data_order_neon: +.LNEON: + stmdb sp!,{r4-r12,lr} + + sub r11,sp,#16*4+16 + adrl r14,K256 + bic r11,r11,#15 @ align for 128-bit stores + mov r12,sp + mov sp,r11 @ alloca + add r2,r1,r2,lsl#6 @ len to point at the end of inp + + vld1.8 {q0},[r1]! + vld1.8 {q1},[r1]! + vld1.8 {q2},[r1]! + vld1.8 {q3},[r1]! + vld1.32 {q8},[r14,:128]! + vld1.32 {q9},[r14,:128]! + vld1.32 {q10},[r14,:128]! + vld1.32 {q11},[r14,:128]! + vrev32.8 q0,q0 @ yes, even on + str r0,[sp,#64] + vrev32.8 q1,q1 @ big-endian + str r1,[sp,#68] + mov r1,sp + vrev32.8 q2,q2 + str r2,[sp,#72] + vrev32.8 q3,q3 + str r12,[sp,#76] @ save original sp + vadd.i32 q8,q8,q0 + vadd.i32 q9,q9,q1 + vst1.32 {q8},[r1,:128]! + vadd.i32 q10,q10,q2 + vst1.32 {q9},[r1,:128]! + vadd.i32 q11,q11,q3 + vst1.32 {q10},[r1,:128]! + vst1.32 {q11},[r1,:128]! + + ldmia r0,{r4-r11} + sub r1,r1,#64 + ldr r2,[sp,#0] + eor r12,r12,r12 + eor r3,r5,r6 + b .L_00_48 + +.align 4 +.L_00_48: + vext.8 q8,q0,q1,#4 + add r11,r11,r2 + eor r2,r9,r10 + eor r0,r8,r8,ror#5 + vext.8 q9,q2,q3,#4 + add r4,r4,r12 + and r2,r2,r8 + eor r12,r0,r8,ror#19 + vshr.u32 q10,q8,#7 + eor r0,r4,r4,ror#11 + eor r2,r2,r10 + vadd.i32 q0,q0,q9 + add r11,r11,r12,ror#6 + eor r12,r4,r5 + vshr.u32 q9,q8,#3 + eor r0,r0,r4,ror#20 + add r11,r11,r2 + vsli.32 q10,q8,#25 + ldr r2,[sp,#4] + and r3,r3,r12 + vshr.u32 q11,q8,#18 + add r7,r7,r11 + add r11,r11,r0,ror#2 + eor r3,r3,r5 + veor q9,q9,q10 + add r10,r10,r2 + vsli.32 q11,q8,#14 + eor r2,r8,r9 + eor r0,r7,r7,ror#5 + vshr.u32 d24,d7,#17 + add r11,r11,r3 + and r2,r2,r7 + veor q9,q9,q11 + eor r3,r0,r7,ror#19 + eor r0,r11,r11,ror#11 + vsli.32 d24,d7,#15 + eor r2,r2,r9 + add r10,r10,r3,ror#6 + vshr.u32 d25,d7,#10 + eor r3,r11,r4 + eor r0,r0,r11,ror#20 + vadd.i32 q0,q0,q9 + add r10,r10,r2 + ldr r2,[sp,#8] + veor d25,d25,d24 + and r12,r12,r3 + add r6,r6,r10 + vshr.u32 d24,d7,#19 + add r10,r10,r0,ror#2 + eor r12,r12,r4 + vsli.32 d24,d7,#13 + add r9,r9,r2 + eor r2,r7,r8 + veor d25,d25,d24 + eor r0,r6,r6,ror#5 + add r10,r10,r12 + vadd.i32 d0,d0,d25 + and r2,r2,r6 + eor r12,r0,r6,ror#19 + vshr.u32 d24,d0,#17 + eor r0,r10,r10,ror#11 + eor r2,r2,r8 + vsli.32 d24,d0,#15 + add r9,r9,r12,ror#6 + eor r12,r10,r11 + vshr.u32 d25,d0,#10 + eor r0,r0,r10,ror#20 + add r9,r9,r2 + veor d25,d25,d24 + ldr r2,[sp,#12] + and r3,r3,r12 + vshr.u32 d24,d0,#19 + add r5,r5,r9 + add r9,r9,r0,ror#2 + eor r3,r3,r11 + vld1.32 {q8},[r14,:128]! + add r8,r8,r2 + vsli.32 d24,d0,#13 + eor r2,r6,r7 + eor r0,r5,r5,ror#5 + veor d25,d25,d24 + add r9,r9,r3 + and r2,r2,r5 + vadd.i32 d1,d1,d25 + eor r3,r0,r5,ror#19 + eor r0,r9,r9,ror#11 + vadd.i32 q8,q8,q0 + eor r2,r2,r7 + add r8,r8,r3,ror#6 + eor r3,r9,r10 + eor r0,r0,r9,ror#20 + add r8,r8,r2 + ldr r2,[sp,#16] + and r12,r12,r3 + add r4,r4,r8 + vst1.32 {q8},[r1,:128]! + add r8,r8,r0,ror#2 + eor r12,r12,r10 + vext.8 q8,q1,q2,#4 + add r7,r7,r2 + eor r2,r5,r6 + eor r0,r4,r4,ror#5 + vext.8 q9,q3,q0,#4 + add r8,r8,r12 + and r2,r2,r4 + eor r12,r0,r4,ror#19 + vshr.u32 q10,q8,#7 + eor r0,r8,r8,ror#11 + eor r2,r2,r6 + vadd.i32 q1,q1,q9 + add r7,r7,r12,ror#6 + eor r12,r8,r9 + vshr.u32 q9,q8,#3 + eor r0,r0,r8,ror#20 + add r7,r7,r2 + vsli.32 q10,q8,#25 + ldr r2,[sp,#20] + and r3,r3,r12 + vshr.u32 q11,q8,#18 + add r11,r11,r7 + add r7,r7,r0,ror#2 + eor r3,r3,r9 + veor q9,q9,q10 + add r6,r6,r2 + vsli.32 q11,q8,#14 + eor r2,r4,r5 + eor r0,r11,r11,ror#5 + vshr.u32 d24,d1,#17 + add r7,r7,r3 + and r2,r2,r11 + veor q9,q9,q11 + eor r3,r0,r11,ror#19 + eor r0,r7,r7,ror#11 + vsli.32 d24,d1,#15 + eor r2,r2,r5 + add r6,r6,r3,ror#6 + vshr.u32 d25,d1,#10 + eor r3,r7,r8 + eor r0,r0,r7,ror#20 + vadd.i32 q1,q1,q9 + add r6,r6,r2 + ldr r2,[sp,#24] + veor d25,d25,d24 + and r12,r12,r3 + add r10,r10,r6 + vshr.u32 d24,d1,#19 + add r6,r6,r0,ror#2 + eor r12,r12,r8 + vsli.32 d24,d1,#13 + add r5,r5,r2 + eor r2,r11,r4 + veor d25,d25,d24 + eor r0,r10,r10,ror#5 + add r6,r6,r12 + vadd.i32 d2,d2,d25 + and r2,r2,r10 + eor r12,r0,r10,ror#19 + vshr.u32 d24,d2,#17 + eor r0,r6,r6,ror#11 + eor r2,r2,r4 + vsli.32 d24,d2,#15 + add r5,r5,r12,ror#6 + eor r12,r6,r7 + vshr.u32 d25,d2,#10 + eor r0,r0,r6,ror#20 + add r5,r5,r2 + veor d25,d25,d24 + ldr r2,[sp,#28] + and r3,r3,r12 + vshr.u32 d24,d2,#19 + add r9,r9,r5 + add r5,r5,r0,ror#2 + eor r3,r3,r7 + vld1.32 {q8},[r14,:128]! + add r4,r4,r2 + vsli.32 d24,d2,#13 + eor r2,r10,r11 + eor r0,r9,r9,ror#5 + veor d25,d25,d24 + add r5,r5,r3 + and r2,r2,r9 + vadd.i32 d3,d3,d25 + eor r3,r0,r9,ror#19 + eor r0,r5,r5,ror#11 + vadd.i32 q8,q8,q1 + eor r2,r2,r11 + add r4,r4,r3,ror#6 + eor r3,r5,r6 + eor r0,r0,r5,ror#20 + add r4,r4,r2 + ldr r2,[sp,#32] + and r12,r12,r3 + add r8,r8,r4 + vst1.32 {q8},[r1,:128]! + add r4,r4,r0,ror#2 + eor r12,r12,r6 + vext.8 q8,q2,q3,#4 + add r11,r11,r2 + eor r2,r9,r10 + eor r0,r8,r8,ror#5 + vext.8 q9,q0,q1,#4 + add r4,r4,r12 + and r2,r2,r8 + eor r12,r0,r8,ror#19 + vshr.u32 q10,q8,#7 + eor r0,r4,r4,ror#11 + eor r2,r2,r10 + vadd.i32 q2,q2,q9 + add r11,r11,r12,ror#6 + eor r12,r4,r5 + vshr.u32 q9,q8,#3 + eor r0,r0,r4,ror#20 + add r11,r11,r2 + vsli.32 q10,q8,#25 + ldr r2,[sp,#36] + and r3,r3,r12 + vshr.u32 q11,q8,#18 + add r7,r7,r11 + add r11,r11,r0,ror#2 + eor r3,r3,r5 + veor q9,q9,q10 + add r10,r10,r2 + vsli.32 q11,q8,#14 + eor r2,r8,r9 + eor r0,r7,r7,ror#5 + vshr.u32 d24,d3,#17 + add r11,r11,r3 + and r2,r2,r7 + veor q9,q9,q11 + eor r3,r0,r7,ror#19 + eor r0,r11,r11,ror#11 + vsli.32 d24,d3,#15 + eor r2,r2,r9 + add r10,r10,r3,ror#6 + vshr.u32 d25,d3,#10 + eor r3,r11,r4 + eor r0,r0,r11,ror#20 + vadd.i32 q2,q2,q9 + add r10,r10,r2 + ldr r2,[sp,#40] + veor d25,d25,d24 + and r12,r12,r3 + add r6,r6,r10 + vshr.u32 d24,d3,#19 + add r10,r10,r0,ror#2 + eor r12,r12,r4 + vsli.32 d24,d3,#13 + add r9,r9,r2 + eor r2,r7,r8 + veor d25,d25,d24 + eor r0,r6,r6,ror#5 + add r10,r10,r12 + vadd.i32 d4,d4,d25 + and r2,r2,r6 + eor r12,r0,r6,ror#19 + vshr.u32 d24,d4,#17 + eor r0,r10,r10,ror#11 + eor r2,r2,r8 + vsli.32 d24,d4,#15 + add r9,r9,r12,ror#6 + eor r12,r10,r11 + vshr.u32 d25,d4,#10 + eor r0,r0,r10,ror#20 + add r9,r9,r2 + veor d25,d25,d24 + ldr r2,[sp,#44] + and r3,r3,r12 + vshr.u32 d24,d4,#19 + add r5,r5,r9 + add r9,r9,r0,ror#2 + eor r3,r3,r11 + vld1.32 {q8},[r14,:128]! + add r8,r8,r2 + vsli.32 d24,d4,#13 + eor r2,r6,r7 + eor r0,r5,r5,ror#5 + veor d25,d25,d24 + add r9,r9,r3 + and r2,r2,r5 + vadd.i32 d5,d5,d25 + eor r3,r0,r5,ror#19 + eor r0,r9,r9,ror#11 + vadd.i32 q8,q8,q2 + eor r2,r2,r7 + add r8,r8,r3,ror#6 + eor r3,r9,r10 + eor r0,r0,r9,ror#20 + add r8,r8,r2 + ldr r2,[sp,#48] + and r12,r12,r3 + add r4,r4,r8 + vst1.32 {q8},[r1,:128]! + add r8,r8,r0,ror#2 + eor r12,r12,r10 + vext.8 q8,q3,q0,#4 + add r7,r7,r2 + eor r2,r5,r6 + eor r0,r4,r4,ror#5 + vext.8 q9,q1,q2,#4 + add r8,r8,r12 + and r2,r2,r4 + eor r12,r0,r4,ror#19 + vshr.u32 q10,q8,#7 + eor r0,r8,r8,ror#11 + eor r2,r2,r6 + vadd.i32 q3,q3,q9 + add r7,r7,r12,ror#6 + eor r12,r8,r9 + vshr.u32 q9,q8,#3 + eor r0,r0,r8,ror#20 + add r7,r7,r2 + vsli.32 q10,q8,#25 + ldr r2,[sp,#52] + and r3,r3,r12 + vshr.u32 q11,q8,#18 + add r11,r11,r7 + add r7,r7,r0,ror#2 + eor r3,r3,r9 + veor q9,q9,q10 + add r6,r6,r2 + vsli.32 q11,q8,#14 + eor r2,r4,r5 + eor r0,r11,r11,ror#5 + vshr.u32 d24,d5,#17 + add r7,r7,r3 + and r2,r2,r11 + veor q9,q9,q11 + eor r3,r0,r11,ror#19 + eor r0,r7,r7,ror#11 + vsli.32 d24,d5,#15 + eor r2,r2,r5 + add r6,r6,r3,ror#6 + vshr.u32 d25,d5,#10 + eor r3,r7,r8 + eor r0,r0,r7,ror#20 + vadd.i32 q3,q3,q9 + add r6,r6,r2 + ldr r2,[sp,#56] + veor d25,d25,d24 + and r12,r12,r3 + add r10,r10,r6 + vshr.u32 d24,d5,#19 + add r6,r6,r0,ror#2 + eor r12,r12,r8 + vsli.32 d24,d5,#13 + add r5,r5,r2 + eor r2,r11,r4 + veor d25,d25,d24 + eor r0,r10,r10,ror#5 + add r6,r6,r12 + vadd.i32 d6,d6,d25 + and r2,r2,r10 + eor r12,r0,r10,ror#19 + vshr.u32 d24,d6,#17 + eor r0,r6,r6,ror#11 + eor r2,r2,r4 + vsli.32 d24,d6,#15 + add r5,r5,r12,ror#6 + eor r12,r6,r7 + vshr.u32 d25,d6,#10 + eor r0,r0,r6,ror#20 + add r5,r5,r2 + veor d25,d25,d24 + ldr r2,[sp,#60] + and r3,r3,r12 + vshr.u32 d24,d6,#19 + add r9,r9,r5 + add r5,r5,r0,ror#2 + eor r3,r3,r7 + vld1.32 {q8},[r14,:128]! + add r4,r4,r2 + vsli.32 d24,d6,#13 + eor r2,r10,r11 + eor r0,r9,r9,ror#5 + veor d25,d25,d24 + add r5,r5,r3 + and r2,r2,r9 + vadd.i32 d7,d7,d25 + eor r3,r0,r9,ror#19 + eor r0,r5,r5,ror#11 + vadd.i32 q8,q8,q3 + eor r2,r2,r11 + add r4,r4,r3,ror#6 + eor r3,r5,r6 + eor r0,r0,r5,ror#20 + add r4,r4,r2 + ldr r2,[r14] + and r12,r12,r3 + add r8,r8,r4 + vst1.32 {q8},[r1,:128]! + add r4,r4,r0,ror#2 + eor r12,r12,r6 + teq r2,#0 @ check for K256 terminator + ldr r2,[sp,#0] + sub r1,r1,#64 + bne .L_00_48 + + ldr r1,[sp,#68] + ldr r0,[sp,#72] + sub r14,r14,#256 @ rewind r14 + teq r1,r0 + it eq + subeq r1,r1,#64 @ avoid SEGV + vld1.8 {q0},[r1]! @ load next input block + vld1.8 {q1},[r1]! + vld1.8 {q2},[r1]! + vld1.8 {q3},[r1]! + it ne + strne r1,[sp,#68] + mov r1,sp + add r11,r11,r2 + eor r2,r9,r10 + eor r0,r8,r8,ror#5 + add r4,r4,r12 + vld1.32 {q8},[r14,:128]! + and r2,r2,r8 + eor r12,r0,r8,ror#19 + eor r0,r4,r4,ror#11 + eor r2,r2,r10 + vrev32.8 q0,q0 + add r11,r11,r12,ror#6 + eor r12,r4,r5 + eor r0,r0,r4,ror#20 + add r11,r11,r2 + vadd.i32 q8,q8,q0 + ldr r2,[sp,#4] + and r3,r3,r12 + add r7,r7,r11 + add r11,r11,r0,ror#2 + eor r3,r3,r5 + add r10,r10,r2 + eor r2,r8,r9 + eor r0,r7,r7,ror#5 + add r11,r11,r3 + and r2,r2,r7 + eor r3,r0,r7,ror#19 + eor r0,r11,r11,ror#11 + eor r2,r2,r9 + add r10,r10,r3,ror#6 + eor r3,r11,r4 + eor r0,r0,r11,ror#20 + add r10,r10,r2 + ldr r2,[sp,#8] + and r12,r12,r3 + add r6,r6,r10 + add r10,r10,r0,ror#2 + eor r12,r12,r4 + add r9,r9,r2 + eor r2,r7,r8 + eor r0,r6,r6,ror#5 + add r10,r10,r12 + and r2,r2,r6 + eor r12,r0,r6,ror#19 + eor r0,r10,r10,ror#11 + eor r2,r2,r8 + add r9,r9,r12,ror#6 + eor r12,r10,r11 + eor r0,r0,r10,ror#20 + add r9,r9,r2 + ldr r2,[sp,#12] + and r3,r3,r12 + add r5,r5,r9 + add r9,r9,r0,ror#2 + eor r3,r3,r11 + add r8,r8,r2 + eor r2,r6,r7 + eor r0,r5,r5,ror#5 + add r9,r9,r3 + and r2,r2,r5 + eor r3,r0,r5,ror#19 + eor r0,r9,r9,ror#11 + eor r2,r2,r7 + add r8,r8,r3,ror#6 + eor r3,r9,r10 + eor r0,r0,r9,ror#20 + add r8,r8,r2 + ldr r2,[sp,#16] + and r12,r12,r3 + add r4,r4,r8 + add r8,r8,r0,ror#2 + eor r12,r12,r10 + vst1.32 {q8},[r1,:128]! + add r7,r7,r2 + eor r2,r5,r6 + eor r0,r4,r4,ror#5 + add r8,r8,r12 + vld1.32 {q8},[r14,:128]! + and r2,r2,r4 + eor r12,r0,r4,ror#19 + eor r0,r8,r8,ror#11 + eor r2,r2,r6 + vrev32.8 q1,q1 + add r7,r7,r12,ror#6 + eor r12,r8,r9 + eor r0,r0,r8,ror#20 + add r7,r7,r2 + vadd.i32 q8,q8,q1 + ldr r2,[sp,#20] + and r3,r3,r12 + add r11,r11,r7 + add r7,r7,r0,ror#2 + eor r3,r3,r9 + add r6,r6,r2 + eor r2,r4,r5 + eor r0,r11,r11,ror#5 + add r7,r7,r3 + and r2,r2,r11 + eor r3,r0,r11,ror#19 + eor r0,r7,r7,ror#11 + eor r2,r2,r5 + add r6,r6,r3,ror#6 + eor r3,r7,r8 + eor r0,r0,r7,ror#20 + add r6,r6,r2 + ldr r2,[sp,#24] + and r12,r12,r3 + add r10,r10,r6 + add r6,r6,r0,ror#2 + eor r12,r12,r8 + add r5,r5,r2 + eor r2,r11,r4 + eor r0,r10,r10,ror#5 + add r6,r6,r12 + and r2,r2,r10 + eor r12,r0,r10,ror#19 + eor r0,r6,r6,ror#11 + eor r2,r2,r4 + add r5,r5,r12,ror#6 + eor r12,r6,r7 + eor r0,r0,r6,ror#20 + add r5,r5,r2 + ldr r2,[sp,#28] + and r3,r3,r12 + add r9,r9,r5 + add r5,r5,r0,ror#2 + eor r3,r3,r7 + add r4,r4,r2 + eor r2,r10,r11 + eor r0,r9,r9,ror#5 + add r5,r5,r3 + and r2,r2,r9 + eor r3,r0,r9,ror#19 + eor r0,r5,r5,ror#11 + eor r2,r2,r11 + add r4,r4,r3,ror#6 + eor r3,r5,r6 + eor r0,r0,r5,ror#20 + add r4,r4,r2 + ldr r2,[sp,#32] + and r12,r12,r3 + add r8,r8,r4 + add r4,r4,r0,ror#2 + eor r12,r12,r6 + vst1.32 {q8},[r1,:128]! + add r11,r11,r2 + eor r2,r9,r10 + eor r0,r8,r8,ror#5 + add r4,r4,r12 + vld1.32 {q8},[r14,:128]! + and r2,r2,r8 + eor r12,r0,r8,ror#19 + eor r0,r4,r4,ror#11 + eor r2,r2,r10 + vrev32.8 q2,q2 + add r11,r11,r12,ror#6 + eor r12,r4,r5 + eor r0,r0,r4,ror#20 + add r11,r11,r2 + vadd.i32 q8,q8,q2 + ldr r2,[sp,#36] + and r3,r3,r12 + add r7,r7,r11 + add r11,r11,r0,ror#2 + eor r3,r3,r5 + add r10,r10,r2 + eor r2,r8,r9 + eor r0,r7,r7,ror#5 + add r11,r11,r3 + and r2,r2,r7 + eor r3,r0,r7,ror#19 + eor r0,r11,r11,ror#11 + eor r2,r2,r9 + add r10,r10,r3,ror#6 + eor r3,r11,r4 + eor r0,r0,r11,ror#20 + add r10,r10,r2 + ldr r2,[sp,#40] + and r12,r12,r3 + add r6,r6,r10 + add r10,r10,r0,ror#2 + eor r12,r12,r4 + add r9,r9,r2 + eor r2,r7,r8 + eor r0,r6,r6,ror#5 + add r10,r10,r12 + and r2,r2,r6 + eor r12,r0,r6,ror#19 + eor r0,r10,r10,ror#11 + eor r2,r2,r8 + add r9,r9,r12,ror#6 + eor r12,r10,r11 + eor r0,r0,r10,ror#20 + add r9,r9,r2 + ldr r2,[sp,#44] + and r3,r3,r12 + add r5,r5,r9 + add r9,r9,r0,ror#2 + eor r3,r3,r11 + add r8,r8,r2 + eor r2,r6,r7 + eor r0,r5,r5,ror#5 + add r9,r9,r3 + and r2,r2,r5 + eor r3,r0,r5,ror#19 + eor r0,r9,r9,ror#11 + eor r2,r2,r7 + add r8,r8,r3,ror#6 + eor r3,r9,r10 + eor r0,r0,r9,ror#20 + add r8,r8,r2 + ldr r2,[sp,#48] + and r12,r12,r3 + add r4,r4,r8 + add r8,r8,r0,ror#2 + eor r12,r12,r10 + vst1.32 {q8},[r1,:128]! + add r7,r7,r2 + eor r2,r5,r6 + eor r0,r4,r4,ror#5 + add r8,r8,r12 + vld1.32 {q8},[r14,:128]! + and r2,r2,r4 + eor r12,r0,r4,ror#19 + eor r0,r8,r8,ror#11 + eor r2,r2,r6 + vrev32.8 q3,q3 + add r7,r7,r12,ror#6 + eor r12,r8,r9 + eor r0,r0,r8,ror#20 + add r7,r7,r2 + vadd.i32 q8,q8,q3 + ldr r2,[sp,#52] + and r3,r3,r12 + add r11,r11,r7 + add r7,r7,r0,ror#2 + eor r3,r3,r9 + add r6,r6,r2 + eor r2,r4,r5 + eor r0,r11,r11,ror#5 + add r7,r7,r3 + and r2,r2,r11 + eor r3,r0,r11,ror#19 + eor r0,r7,r7,ror#11 + eor r2,r2,r5 + add r6,r6,r3,ror#6 + eor r3,r7,r8 + eor r0,r0,r7,ror#20 + add r6,r6,r2 + ldr r2,[sp,#56] + and r12,r12,r3 + add r10,r10,r6 + add r6,r6,r0,ror#2 + eor r12,r12,r8 + add r5,r5,r2 + eor r2,r11,r4 + eor r0,r10,r10,ror#5 + add r6,r6,r12 + and r2,r2,r10 + eor r12,r0,r10,ror#19 + eor r0,r6,r6,ror#11 + eor r2,r2,r4 + add r5,r5,r12,ror#6 + eor r12,r6,r7 + eor r0,r0,r6,ror#20 + add r5,r5,r2 + ldr r2,[sp,#60] + and r3,r3,r12 + add r9,r9,r5 + add r5,r5,r0,ror#2 + eor r3,r3,r7 + add r4,r4,r2 + eor r2,r10,r11 + eor r0,r9,r9,ror#5 + add r5,r5,r3 + and r2,r2,r9 + eor r3,r0,r9,ror#19 + eor r0,r5,r5,ror#11 + eor r2,r2,r11 + add r4,r4,r3,ror#6 + eor r3,r5,r6 + eor r0,r0,r5,ror#20 + add r4,r4,r2 + ldr r2,[sp,#64] + and r12,r12,r3 + add r8,r8,r4 + add r4,r4,r0,ror#2 + eor r12,r12,r6 + vst1.32 {q8},[r1,:128]! + ldr r0,[r2,#0] + add r4,r4,r12 @ h+=Maj(a,b,c) from the past + ldr r12,[r2,#4] + ldr r3,[r2,#8] + ldr r1,[r2,#12] + add r4,r4,r0 @ accumulate + ldr r0,[r2,#16] + add r5,r5,r12 + ldr r12,[r2,#20] + add r6,r6,r3 + ldr r3,[r2,#24] + add r7,r7,r1 + ldr r1,[r2,#28] + add r8,r8,r0 + str r4,[r2],#4 + add r9,r9,r12 + str r5,[r2],#4 + add r10,r10,r3 + str r6,[r2],#4 + add r11,r11,r1 + str r7,[r2],#4 + stmia r2,{r8-r11} + + ittte ne + movne r1,sp + ldrne r2,[sp,#0] + eorne r12,r12,r12 + ldreq sp,[sp,#76] @ restore original sp + itt ne + eorne r3,r5,r6 + bne .L_00_48 + + ldmia sp!,{r4-r12,pc} +.size sha256_block_data_order_neon,.-sha256_block_data_order_neon +#endif +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) + +# ifdef __thumb2__ +# define INST(a,b,c,d) .byte c,d|0xc,a,b +# else +# define INST(a,b,c,d) .byte a,b,c,d +# endif + +.type sha256_block_data_order_armv8,%function +.align 5 +sha256_block_data_order_armv8: +.LARMv8: + vld1.32 {q0,q1},[r0] +# ifdef __thumb2__ + adr r3,.LARMv8 + sub r3,r3,#.LARMv8-K256 +# else + adrl r3,K256 +# endif + add r2,r1,r2,lsl#6 @ len to point at the end of inp + +.Loop_v8: + vld1.8 {q8-q9},[r1]! + vld1.8 {q10-q11},[r1]! + vld1.32 {q12},[r3]! + vrev32.8 q8,q8 + vrev32.8 q9,q9 + vrev32.8 q10,q10 + vrev32.8 q11,q11 + vmov q14,q0 @ offload + vmov q15,q1 + teq r1,r2 + vld1.32 {q13},[r3]! + vadd.i32 q12,q12,q8 + INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9 + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11 + vld1.32 {q12},[r3]! + vadd.i32 q13,q13,q9 + INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8 + vld1.32 {q13},[r3]! + vadd.i32 q12,q12,q10 + INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11 + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9 + vld1.32 {q12},[r3]! + vadd.i32 q13,q13,q11 + INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10 + vld1.32 {q13},[r3]! + vadd.i32 q12,q12,q8 + INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9 + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11 + vld1.32 {q12},[r3]! + vadd.i32 q13,q13,q9 + INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8 + vld1.32 {q13},[r3]! + vadd.i32 q12,q12,q10 + INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11 + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9 + vld1.32 {q12},[r3]! + vadd.i32 q13,q13,q11 + INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10 + vld1.32 {q13},[r3]! + vadd.i32 q12,q12,q8 + INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9 + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11 + vld1.32 {q12},[r3]! + vadd.i32 q13,q13,q9 + INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8 + vld1.32 {q13},[r3]! + vadd.i32 q12,q12,q10 + INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11 + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9 + vld1.32 {q12},[r3]! + vadd.i32 q13,q13,q11 + INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10 + vld1.32 {q13},[r3]! + vadd.i32 q12,q12,q8 + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + + vld1.32 {q12},[r3]! + vadd.i32 q13,q13,q9 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + + vld1.32 {q13},[r3] + vadd.i32 q12,q12,q10 + sub r3,r3,#256-16 @ rewind + vmov q2,q0 + INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 + INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 + + vadd.i32 q13,q13,q11 + vmov q2,q0 + INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 + INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 + + vadd.i32 q0,q0,q14 + vadd.i32 q1,q1,q15 + it ne + bne .Loop_v8 + + vst1.32 {q0,q1},[r0] + + bx lr @ bx lr +.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8 +#endif +.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by <appro@openssl.org>" +.align 2 +#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) +.comm OPENSSL_armcap_P,4,4 +#endif diff --git a/arch/arm/crypto/sha256_glue.c b/arch/arm/crypto/sha256_glue.c new file mode 100644 index 000000000000..a84e869ef900 --- /dev/null +++ b/arch/arm/crypto/sha256_glue.c @@ -0,0 +1,128 @@ +/* + * Glue code for the SHA256 Secure Hash Algorithm assembly implementation + * using optimized ARM assembler and NEON instructions. + * + * Copyright © 2015 Google Inc. + * + * This file is based on sha256_ssse3_glue.c: + * Copyright (C) 2013 Intel Corporation + * Author: Tim Chen <tim.c.chen@linux.intel.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <crypto/internal/hash.h> +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <linux/string.h> +#include <crypto/sha.h> +#include <crypto/sha256_base.h> +#include <asm/simd.h> +#include <asm/neon.h> + +#include "sha256_glue.h" + +asmlinkage void sha256_block_data_order(u32 *digest, const void *data, + unsigned int num_blks); + +int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + /* make sure casting to sha256_block_fn() is safe */ + BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); + + return sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha256_block_data_order); +} +EXPORT_SYMBOL(crypto_sha256_arm_update); + +static int sha256_final(struct shash_desc *desc, u8 *out) +{ + sha256_base_do_finalize(desc, + (sha256_block_fn *)sha256_block_data_order); + return sha256_base_finish(desc, out); +} + +int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha256_block_data_order); + return sha256_final(desc, out); +} +EXPORT_SYMBOL(crypto_sha256_arm_finup); + +static struct shash_alg algs[] = { { + .digestsize = SHA256_DIGEST_SIZE, + .init = sha256_base_init, + .update = crypto_sha256_arm_update, + .final = sha256_final, + .finup = crypto_sha256_arm_finup, + .descsize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-asm", + .cra_priority = 150, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA224_DIGEST_SIZE, + .init = sha224_base_init, + .update = crypto_sha256_arm_update, + .final = sha256_final, + .finup = crypto_sha256_arm_finup, + .descsize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-asm", + .cra_priority = 150, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init sha256_mod_init(void) +{ + int res = crypto_register_shashes(algs, ARRAY_SIZE(algs)); + + if (res < 0) + return res; + + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) { + res = crypto_register_shashes(sha256_neon_algs, + ARRAY_SIZE(sha256_neon_algs)); + + if (res < 0) + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); + } + + return res; +} + +static void __exit sha256_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); + + if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && cpu_has_neon()) + crypto_unregister_shashes(sha256_neon_algs, + ARRAY_SIZE(sha256_neon_algs)); +} + +module_init(sha256_mod_init); +module_exit(sha256_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm (ARM), including NEON"); + +MODULE_ALIAS_CRYPTO("sha256"); diff --git a/arch/arm/crypto/sha256_glue.h b/arch/arm/crypto/sha256_glue.h new file mode 100644 index 000000000000..7cf0bf786ada --- /dev/null +++ b/arch/arm/crypto/sha256_glue.h @@ -0,0 +1,14 @@ +#ifndef _CRYPTO_SHA256_GLUE_H +#define _CRYPTO_SHA256_GLUE_H + +#include <linux/crypto.h> + +extern struct shash_alg sha256_neon_algs[2]; + +int crypto_sha256_arm_update(struct shash_desc *desc, const u8 *data, + unsigned int len); + +int crypto_sha256_arm_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *hash); + +#endif /* _CRYPTO_SHA256_GLUE_H */ diff --git a/arch/arm/crypto/sha256_neon_glue.c b/arch/arm/crypto/sha256_neon_glue.c new file mode 100644 index 000000000000..39ccd658817e --- /dev/null +++ b/arch/arm/crypto/sha256_neon_glue.c @@ -0,0 +1,101 @@ +/* + * Glue code for the SHA256 Secure Hash Algorithm assembly implementation + * using NEON instructions. + * + * Copyright © 2015 Google Inc. + * + * This file is based on sha512_neon_glue.c: + * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <crypto/internal/hash.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <linux/string.h> +#include <crypto/sha.h> +#include <crypto/sha256_base.h> +#include <asm/byteorder.h> +#include <asm/simd.h> +#include <asm/neon.h> + +#include "sha256_glue.h" + +asmlinkage void sha256_block_data_order_neon(u32 *digest, const void *data, + unsigned int num_blks); + +static int sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + if (!may_use_simd() || + (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) + return crypto_sha256_arm_update(desc, data, len); + + kernel_neon_begin(); + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha256_block_data_order_neon); + kernel_neon_end(); + + return 0; +} + +static int sha256_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + if (!may_use_simd()) + return crypto_sha256_arm_finup(desc, data, len, out); + + kernel_neon_begin(); + if (len) + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha256_block_data_order_neon); + sha256_base_do_finalize(desc, + (sha256_block_fn *)sha256_block_data_order_neon); + kernel_neon_end(); + + return sha256_base_finish(desc, out); +} + +static int sha256_final(struct shash_desc *desc, u8 *out) +{ + return sha256_finup(desc, NULL, 0, out); +} + +struct shash_alg sha256_neon_algs[] = { { + .digestsize = SHA256_DIGEST_SIZE, + .init = sha256_base_init, + .update = sha256_update, + .final = sha256_final, + .finup = sha256_finup, + .descsize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA224_DIGEST_SIZE, + .init = sha224_base_init, + .update = sha256_update, + .final = sha256_final, + .finup = sha256_finup, + .descsize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name = "sha224-neon", + .cra_priority = 250, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; diff --git a/arch/arm/include/asm/Kbuild b/arch/arm/include/asm/Kbuild index fe74c0d1e485..3c4596d0ce6c 100644 --- a/arch/arm/include/asm/Kbuild +++ b/arch/arm/include/asm/Kbuild @@ -1,6 +1,5 @@ -generic-y += auxvec.h generic-y += bitsperlong.h generic-y += cputime.h generic-y += current.h @@ -22,6 +21,7 @@ generic-y += preempt.h generic-y += resource.h generic-y += rwsem.h generic-y += scatterlist.h +generic-y += seccomp.h generic-y += sections.h generic-y += segment.h generic-y += sembuf.h diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index f67fd3afebdf..186270b3e194 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -237,6 +237,9 @@ .pushsection ".alt.smp.init", "a" ;\ .long 9998b ;\ 9997: instr ;\ + .if . - 9997b == 2 ;\ + nop ;\ + .endif ;\ .if . - 9997b != 4 ;\ .error "ALT_UP() content must assemble to exactly 4 bytes";\ .endif ;\ diff --git a/arch/arm/include/asm/auxvec.h b/arch/arm/include/asm/auxvec.h new file mode 100644 index 000000000000..fbd388c46299 --- /dev/null +++ b/arch/arm/include/asm/auxvec.h @@ -0,0 +1 @@ +#include <uapi/asm/auxvec.h> diff --git a/arch/arm/include/asm/cputype.h b/arch/arm/include/asm/cputype.h index 819777d0e91f..85e374f873ac 100644 --- a/arch/arm/include/asm/cputype.h +++ b/arch/arm/include/asm/cputype.h @@ -253,4 +253,20 @@ static inline int cpu_is_pj4(void) #else #define cpu_is_pj4() 0 #endif + +static inline int __attribute_const__ cpuid_feature_extract_field(u32 features, + int field) +{ + int feature = (features >> field) & 15; + + /* feature registers are signed values */ + if (feature > 8) + feature -= 16; + + return feature; +} + +#define cpuid_feature_extract(reg, field) \ + cpuid_feature_extract_field(read_cpuid_ext(reg), field) + #endif diff --git a/arch/arm/include/asm/elf.h b/arch/arm/include/asm/elf.h index c1ff8ab12914..d2315ffd8f12 100644 --- a/arch/arm/include/asm/elf.h +++ b/arch/arm/include/asm/elf.h @@ -1,7 +1,9 @@ #ifndef __ASMARM_ELF_H #define __ASMARM_ELF_H +#include <asm/auxvec.h> #include <asm/hwcap.h> +#include <asm/vdso_datapage.h> /* * ELF register definitions.. @@ -115,7 +117,7 @@ int dump_task_regs(struct task_struct *t, elf_gregset_t *elfregs); the loader. We need to make sure that it is out of the way of the program that it will "exec", and that there is sufficient room for the brk. */ -#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3) +#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2) /* When the program starts, a1 contains a pointer to a function to be registered with atexit, as per the SVR4 ABI. A value of 0 means we @@ -126,6 +128,13 @@ extern void elf_set_personality(const struct elf32_hdr *); #define SET_PERSONALITY(ex) elf_set_personality(&(ex)) #ifdef CONFIG_MMU +#ifdef CONFIG_VDSO +#define ARCH_DLINFO \ +do { \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, \ + (elf_addr_t)current->mm->context.vdso); \ +} while (0) +#endif #define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 struct linux_binprm; int arch_setup_additional_pages(struct linux_binprm *, int); diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h index 53e69dae796f..4e78065a16aa 100644 --- a/arch/arm/include/asm/futex.h +++ b/arch/arm/include/asm/futex.h @@ -13,7 +13,7 @@ " .align 3\n" \ " .long 1b, 4f, 2b, 4f\n" \ " .popsection\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "4: mov %0, " err_reg "\n" \ " b 3b\n" \ diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h index 4cf48c3aca13..405aa1883307 100644 --- a/arch/arm/include/asm/kvm_mmu.h +++ b/arch/arm/include/asm/kvm_mmu.h @@ -269,6 +269,16 @@ static inline void __kvm_flush_dcache_pud(pud_t pud) void kvm_set_way_flush(struct kvm_vcpu *vcpu); void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); +static inline bool __kvm_cpu_uses_extended_idmap(void) +{ + return false; +} + +static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, + pgd_t *hyp_pgd, + pgd_t *merged_hyp_pgd, + unsigned long hyp_idmap_start) { } + #endif /* !__ASSEMBLY__ */ #endif /* __ARM_KVM_MMU_H__ */ diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h index 64fd15159b7d..a5b47421059d 100644 --- a/arch/arm/include/asm/mmu.h +++ b/arch/arm/include/asm/mmu.h @@ -11,6 +11,9 @@ typedef struct { #endif unsigned int vmalloc_seq; unsigned long sigpage; +#ifdef CONFIG_VDSO + unsigned long vdso; +#endif } mm_context_t; #ifdef CONFIG_CPU_HAS_ASID diff --git a/arch/arm/include/asm/pmu.h b/arch/arm/include/asm/pmu.h index b1596bd59129..675e4ab79f68 100644 --- a/arch/arm/include/asm/pmu.h +++ b/arch/arm/include/asm/pmu.h @@ -92,6 +92,7 @@ struct pmu_hw_events { struct arm_pmu { struct pmu pmu; cpumask_t active_irqs; + int *irq_affinity; char *name; irqreturn_t (*handle_irq)(int irq_num, void *dev); void (*enable)(struct perf_event *event); diff --git a/arch/arm/include/asm/seccomp.h b/arch/arm/include/asm/seccomp.h deleted file mode 100644 index 52b156b341f5..000000000000 --- a/arch/arm/include/asm/seccomp.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef _ASM_ARM_SECCOMP_H -#define _ASM_ARM_SECCOMP_H - -#include <linux/unistd.h> - -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - -#endif /* _ASM_ARM_SECCOMP_H */ diff --git a/arch/arm/include/asm/smp_plat.h b/arch/arm/include/asm/smp_plat.h index 0ad7d490ee6f..993e5224d8f7 100644 --- a/arch/arm/include/asm/smp_plat.h +++ b/arch/arm/include/asm/smp_plat.h @@ -104,6 +104,7 @@ static inline u32 mpidr_hash_size(void) return 1 << mpidr_hash.bits; } +extern int platform_can_secondary_boot(void); extern int platform_can_cpu_hotplug(void); #endif diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 72812a1f3d1c..bd32eded3e50 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -23,7 +23,6 @@ #ifndef __ASSEMBLY__ struct task_struct; -struct exec_domain; #include <asm/types.h> #include <asm/domain.h> @@ -53,7 +52,6 @@ struct thread_info { int preempt_count; /* 0 => preemptable, <0 => bug */ mm_segment_t addr_limit; /* address limit */ struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ __u32 cpu; /* cpu */ __u32 cpu_domain; /* cpu domain */ struct cpu_context_save cpu_context; /* cpu context */ @@ -73,7 +71,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index ce0786efd26c..74b17d09ef7a 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -315,7 +315,7 @@ do { \ __asm__ __volatile__( \ "1: " TUSER(ldrb) " %1,[%2],#0\n" \ "2:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %0, %3\n" \ " mov %1, #0\n" \ @@ -351,7 +351,7 @@ do { \ __asm__ __volatile__( \ "1: " TUSER(ldr) " %1,[%2],#0\n" \ "2:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %0, %3\n" \ " mov %1, #0\n" \ @@ -397,7 +397,7 @@ do { \ __asm__ __volatile__( \ "1: " TUSER(strb) " %1,[%2],#0\n" \ "2:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %0, %3\n" \ " b 2b\n" \ @@ -430,7 +430,7 @@ do { \ __asm__ __volatile__( \ "1: " TUSER(str) " %1,[%2],#0\n" \ "2:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %0, %3\n" \ " b 2b\n" \ @@ -458,7 +458,7 @@ do { \ THUMB( "1: " TUSER(str) " " __reg_oper1 ", [%1]\n" ) \ THUMB( "2: " TUSER(str) " " __reg_oper0 ", [%1, #4]\n" ) \ "3:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "4: mov %0, %3\n" \ " b 3b\n" \ diff --git a/arch/arm/include/asm/unified.h b/arch/arm/include/asm/unified.h index b88beaba6b4a..200f9a7cd623 100644 --- a/arch/arm/include/asm/unified.h +++ b/arch/arm/include/asm/unified.h @@ -24,6 +24,14 @@ .syntax unified #endif +#ifdef CONFIG_CPU_V7M +#define AR_CLASS(x...) +#define M_CLASS(x...) x +#else +#define AR_CLASS(x...) x +#define M_CLASS(x...) +#endif + #ifdef CONFIG_THUMB2_KERNEL #if __GNUC__ < 4 diff --git a/arch/arm/include/asm/vdso.h b/arch/arm/include/asm/vdso.h new file mode 100644 index 000000000000..d0295f1dd1a3 --- /dev/null +++ b/arch/arm/include/asm/vdso.h @@ -0,0 +1,32 @@ +#ifndef __ASM_VDSO_H +#define __ASM_VDSO_H + +#ifdef __KERNEL__ + +#ifndef __ASSEMBLY__ + +struct mm_struct; + +#ifdef CONFIG_VDSO + +void arm_install_vdso(struct mm_struct *mm, unsigned long addr); + +extern char vdso_start, vdso_end; + +extern unsigned int vdso_total_pages; + +#else /* CONFIG_VDSO */ + +static inline void arm_install_vdso(struct mm_struct *mm, unsigned long addr) +{ +} + +#define vdso_total_pages 0 + +#endif /* CONFIG_VDSO */ + +#endif /* __ASSEMBLY__ */ + +#endif /* __KERNEL__ */ + +#endif /* __ASM_VDSO_H */ diff --git a/arch/arm/include/asm/vdso_datapage.h b/arch/arm/include/asm/vdso_datapage.h new file mode 100644 index 000000000000..9be259442fca --- /dev/null +++ b/arch/arm/include/asm/vdso_datapage.h @@ -0,0 +1,60 @@ +/* + * Adapted from arm64 version. + * + * Copyright (C) 2012 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +#ifndef __ASM_VDSO_DATAPAGE_H +#define __ASM_VDSO_DATAPAGE_H + +#ifdef __KERNEL__ + +#ifndef __ASSEMBLY__ + +#include <asm/page.h> + +/* Try to be cache-friendly on systems that don't implement the + * generic timer: fit the unconditionally updated fields in the first + * 32 bytes. + */ +struct vdso_data { + u32 seq_count; /* sequence count - odd during updates */ + u16 tk_is_cntvct; /* fall back to syscall if false */ + u16 cs_shift; /* clocksource shift */ + u32 xtime_coarse_sec; /* coarse time */ + u32 xtime_coarse_nsec; + + u32 wtm_clock_sec; /* wall to monotonic offset */ + u32 wtm_clock_nsec; + u32 xtime_clock_sec; /* CLOCK_REALTIME - seconds */ + u32 cs_mult; /* clocksource multiplier */ + + u64 cs_cycle_last; /* last cycle value */ + u64 cs_mask; /* clocksource mask */ + + u64 xtime_clock_snsec; /* CLOCK_REALTIME sub-ns base */ + u32 tz_minuteswest; /* timezone info for gettimeofday(2) */ + u32 tz_dsttime; +}; + +union vdso_data_store { + struct vdso_data data; + u8 page[PAGE_SIZE]; +}; + +#endif /* !__ASSEMBLY__ */ + +#endif /* __KERNEL__ */ + +#endif /* __ASM_VDSO_DATAPAGE_H */ diff --git a/arch/arm/include/asm/word-at-a-time.h b/arch/arm/include/asm/word-at-a-time.h index a6d0a29861e7..5831dce4b51c 100644 --- a/arch/arm/include/asm/word-at-a-time.h +++ b/arch/arm/include/asm/word-at-a-time.h @@ -71,7 +71,7 @@ static inline unsigned long load_unaligned_zeropad(const void *addr) asm( "1: ldr %0, [%2]\n" "2:\n" - " .pushsection .fixup,\"ax\"\n" + " .pushsection .text.fixup,\"ax\"\n" " .align 2\n" "3: and %1, %2, #0x3\n" " bic %2, %2, #0x3\n" diff --git a/arch/arm/include/uapi/asm/Kbuild b/arch/arm/include/uapi/asm/Kbuild index 70a1c9da30ca..a1c05f93d920 100644 --- a/arch/arm/include/uapi/asm/Kbuild +++ b/arch/arm/include/uapi/asm/Kbuild @@ -1,6 +1,7 @@ # UAPI Header export list include include/uapi/asm-generic/Kbuild.asm +header-y += auxvec.h header-y += byteorder.h header-y += fcntl.h header-y += hwcap.h diff --git a/arch/arm/include/uapi/asm/auxvec.h b/arch/arm/include/uapi/asm/auxvec.h new file mode 100644 index 000000000000..cb02a767a500 --- /dev/null +++ b/arch/arm/include/uapi/asm/auxvec.h @@ -0,0 +1,7 @@ +#ifndef __ASM_AUXVEC_H +#define __ASM_AUXVEC_H + +/* VDSO location */ +#define AT_SYSINFO_EHDR 33 + +#endif diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 902397dd1000..752725dcbf42 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -16,7 +16,7 @@ CFLAGS_REMOVE_return_address.o = -pg # Object file lists. obj-y := elf.o entry-common.o irq.o opcodes.o \ - process.o ptrace.o return_address.o \ + process.o ptrace.o reboot.o return_address.o \ setup.o signal.o sigreturn_codes.o \ stacktrace.o sys_arm.o time.o traps.o @@ -34,7 +34,6 @@ obj-$(CONFIG_CPU_IDLE) += cpuidle.o obj-$(CONFIG_ISA_DMA_API) += dma.o obj-$(CONFIG_FIQ) += fiq.o fiqasm.o obj-$(CONFIG_MODULES) += armksyms.o module.o -obj-$(CONFIG_ARTHUR) += arthur.o obj-$(CONFIG_ISA_DMA) += dma-isa.o obj-$(CONFIG_PCI) += bios32.o isa.o obj-$(CONFIG_ARM_CPU_SUSPEND) += sleep.o suspend.o @@ -75,6 +74,7 @@ obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o perf_event_cpu.o CFLAGS_pj4-cp0.o := -marm AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt obj-$(CONFIG_ARM_CPU_TOPOLOGY) += topology.o +obj-$(CONFIG_VDSO) += vdso.o ifneq ($(CONFIG_ARCH_EBSA110),y) obj-y += io.o @@ -86,7 +86,7 @@ obj-$(CONFIG_EARLY_PRINTK) += early_printk.o obj-$(CONFIG_ARM_VIRT_EXT) += hyp-stub.o ifeq ($(CONFIG_ARM_PSCI),y) -obj-y += psci.o +obj-y += psci.o psci-call.o obj-$(CONFIG_SMP) += psci_smp.o endif diff --git a/arch/arm/kernel/arthur.c b/arch/arm/kernel/arthur.c deleted file mode 100644 index 321c5291d05f..000000000000 --- a/arch/arm/kernel/arthur.c +++ /dev/null @@ -1,94 +0,0 @@ -/* - * linux/arch/arm/kernel/arthur.c - * - * Copyright (C) 1998, 1999, 2000, 2001 Philip Blundell - * - * Arthur personality - */ - -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include <linux/module.h> -#include <linux/personality.h> -#include <linux/stddef.h> -#include <linux/signal.h> -#include <linux/init.h> -#include <linux/sched.h> - -#include <asm/ptrace.h> - -/* Arthur doesn't have many signals, and a lot of those that it does - have don't map easily to any Linux equivalent. Never mind. */ - -#define ARTHUR_SIGABRT 1 -#define ARTHUR_SIGFPE 2 -#define ARTHUR_SIGILL 3 -#define ARTHUR_SIGINT 4 -#define ARTHUR_SIGSEGV 5 -#define ARTHUR_SIGTERM 6 -#define ARTHUR_SIGSTAK 7 -#define ARTHUR_SIGUSR1 8 -#define ARTHUR_SIGUSR2 9 -#define ARTHUR_SIGOSERROR 10 - -static unsigned long arthur_to_linux_signals[32] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31 -}; - -static unsigned long linux_to_arthur_signals[32] = { - 0, -1, ARTHUR_SIGINT, -1, - ARTHUR_SIGILL, 5, ARTHUR_SIGABRT, 7, - ARTHUR_SIGFPE, 9, ARTHUR_SIGUSR1, ARTHUR_SIGSEGV, - ARTHUR_SIGUSR2, 13, 14, ARTHUR_SIGTERM, - 16, 17, 18, 19, - 20, 21, 22, 23, - 24, 25, 26, 27, - 28, 29, 30, 31 -}; - -static void arthur_lcall7(int nr, struct pt_regs *regs) -{ - struct siginfo info; - info.si_signo = SIGSWI; - info.si_errno = nr; - /* Bounce it to the emulator */ - send_sig_info(SIGSWI, &info, current); -} - -static struct exec_domain arthur_exec_domain = { - .name = "Arthur", - .handler = arthur_lcall7, - .pers_low = PER_RISCOS, - .pers_high = PER_RISCOS, - .signal_map = arthur_to_linux_signals, - .signal_invmap = linux_to_arthur_signals, - .module = THIS_MODULE, -}; - -/* - * We could do with some locking to stop Arthur being removed while - * processes are using it. - */ - -static int __init arthur_init(void) -{ - return register_exec_domain(&arthur_exec_domain); -} - -static void __exit arthur_exit(void) -{ - unregister_exec_domain(&arthur_exec_domain); -} - -module_init(arthur_init); -module_exit(arthur_exit); - -MODULE_LICENSE("GPL"); diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 488eaac56028..871b8267d211 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -25,6 +25,7 @@ #include <asm/memory.h> #include <asm/procinfo.h> #include <asm/suspend.h> +#include <asm/vdso_datapage.h> #include <asm/hardware/cache-l2x0.h> #include <linux/kbuild.h> @@ -66,7 +67,6 @@ int main(void) DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); @@ -206,5 +206,9 @@ int main(void) DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base)); DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); #endif + BLANK(); +#ifdef CONFIG_VDSO + DEFINE(VDSO_DATA_SIZE, sizeof(union vdso_data_store)); +#endif return 0; } diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c index ab19b7c03423..fcbbbb1b9e95 100644 --- a/arch/arm/kernel/bios32.c +++ b/arch/arm/kernel/bios32.c @@ -618,21 +618,15 @@ int pcibios_enable_device(struct pci_dev *dev, int mask) int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, enum pci_mmap_state mmap_state, int write_combine) { - struct pci_sys_data *root = dev->sysdata; - unsigned long phys; - - if (mmap_state == pci_mmap_io) { + if (mmap_state == pci_mmap_io) return -EINVAL; - } else { - phys = vma->vm_pgoff + (root->mem_offset >> PAGE_SHIFT); - } /* * Mark this as IO */ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - if (remap_pfn_range(vma, vma->vm_start, phys, + if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, vma->vm_end - vma->vm_start, vma->vm_page_prot)) return -EAGAIN; diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 672b21942fff..570306c49406 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -545,7 +545,7 @@ ENDPROC(__und_usr) /* * The out of line fixup for the ldrt instructions above. */ - .pushsection .fixup, "ax" + .pushsection .text.fixup, "ax" .align 2 4: str r4, [sp, #S_PC] @ retry current instruction ret r9 diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 01963273c07a..3637973a9708 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -138,9 +138,9 @@ ENTRY(stext) @ mmu has been enabled adr lr, BSYM(1f) @ return (PIC) address mov r8, r4 @ set TTBR1 to swapper_pg_dir - ARM( add pc, r10, #PROCINFO_INITFUNC ) - THUMB( add r12, r10, #PROCINFO_INITFUNC ) - THUMB( ret r12 ) + ldr r12, [r10, #PROCINFO_INITFUNC] + add r12, r12, r10 + ret r12 1: b __enable_mmu ENDPROC(stext) .ltorg @@ -386,10 +386,10 @@ ENTRY(secondary_startup) ldr r8, [r7, lr] @ get secondary_data.swapper_pg_dir adr lr, BSYM(__enable_mmu) @ return address mov r13, r12 @ __secondary_switched address - ARM( add pc, r10, #PROCINFO_INITFUNC ) @ initialise processor - @ (return control reg) - THUMB( add r12, r10, #PROCINFO_INITFUNC ) - THUMB( ret r12 ) + ldr r12, [r10, #PROCINFO_INITFUNC] + add r12, r12, r10 @ initialise processor + @ (return control reg) + ret r12 ENDPROC(secondary_startup) ENDPROC(secondary_startup_arm) diff --git a/arch/arm/kernel/hibernate.c b/arch/arm/kernel/hibernate.c index c4cc50e58c13..a71501ff6f18 100644 --- a/arch/arm/kernel/hibernate.c +++ b/arch/arm/kernel/hibernate.c @@ -22,6 +22,7 @@ #include <asm/suspend.h> #include <asm/memory.h> #include <asm/sections.h> +#include "reboot.h" int pfn_is_nosave(unsigned long pfn) { @@ -61,7 +62,7 @@ static int notrace arch_save_image(unsigned long unused) ret = swsusp_save(); if (ret == 0) - soft_restart(virt_to_phys(cpu_resume)); + _soft_restart(virt_to_phys(cpu_resume), false); return ret; } @@ -86,7 +87,7 @@ static void notrace arch_restore_image(void *unused) for (pbe = restore_pblist; pbe; pbe = pbe->next) copy_page(pbe->orig_address, pbe->address); - soft_restart(virt_to_phys(cpu_resume)); + _soft_restart(virt_to_phys(cpu_resume), false); } static u64 resume_stack[PAGE_SIZE/2/sizeof(u64)] __nosavedata; @@ -99,7 +100,6 @@ static u64 resume_stack[PAGE_SIZE/2/sizeof(u64)] __nosavedata; */ int swsusp_arch_resume(void) { - extern void call_with_stack(void (*fn)(void *), void *arg, void *sp); call_with_stack(arch_restore_image, 0, resume_stack + ARRAY_SIZE(resume_stack)); return 0; diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c index de2b085ad753..8bf3b7c09888 100644 --- a/arch/arm/kernel/machine_kexec.c +++ b/arch/arm/kernel/machine_kexec.c @@ -46,7 +46,8 @@ int machine_kexec_prepare(struct kimage *image) * and implements CPU hotplug for the current HW. If not, we won't be * able to kexec reliably, so fail the prepare operation. */ - if (num_possible_cpus() > 1 && !platform_can_cpu_hotplug()) + if (num_possible_cpus() > 1 && platform_can_secondary_boot() && + !platform_can_cpu_hotplug()) return -EINVAL; /* diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index 2e11961f65ae..af791f4a6205 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -98,14 +98,19 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, case R_ARM_PC24: case R_ARM_CALL: case R_ARM_JUMP24: + if (sym->st_value & 3) { + pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (ARM -> Thumb)\n", + module->name, relindex, i, symname); + return -ENOEXEC; + } + offset = __mem_to_opcode_arm(*(u32 *)loc); offset = (offset & 0x00ffffff) << 2; if (offset & 0x02000000) offset -= 0x04000000; offset += sym->st_value - loc; - if (offset & 3 || - offset <= (s32)0xfe000000 || + if (offset <= (s32)0xfe000000 || offset >= (s32)0x02000000) { pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", module->name, relindex, i, symname, @@ -155,6 +160,22 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, #ifdef CONFIG_THUMB2_KERNEL case R_ARM_THM_CALL: case R_ARM_THM_JUMP24: + /* + * For function symbols, only Thumb addresses are + * allowed (no interworking). + * + * For non-function symbols, the destination + * has no specific ARM/Thumb disposition, so + * the branch is resolved under the assumption + * that interworking is not required. + */ + if (ELF32_ST_TYPE(sym->st_info) == STT_FUNC && + !(sym->st_value & 1)) { + pr_err("%s: section %u reloc %u sym '%s': unsupported interworking call (Thumb -> ARM)\n", + module->name, relindex, i, symname); + return -ENOEXEC; + } + upper = __mem_to_opcode_thumb16(*(u16 *)loc); lower = __mem_to_opcode_thumb16(*(u16 *)(loc + 2)); @@ -182,18 +203,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, offset -= 0x02000000; offset += sym->st_value - loc; - /* - * For function symbols, only Thumb addresses are - * allowed (no interworking). - * - * For non-function symbols, the destination - * has no specific ARM/Thumb disposition, so - * the branch is resolved under the assumption - * that interworking is not required. - */ - if ((ELF32_ST_TYPE(sym->st_info) == STT_FUNC && - !(offset & 1)) || - offset <= (s32)0xff000000 || + if (offset <= (s32)0xff000000 || offset >= (s32)0x01000000) { pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", module->name, relindex, i, symname, diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c index 557e128e4df0..4a86a0133ac3 100644 --- a/arch/arm/kernel/perf_event.c +++ b/arch/arm/kernel/perf_event.c @@ -259,20 +259,29 @@ out: } static int -validate_event(struct pmu_hw_events *hw_events, - struct perf_event *event) +validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, + struct perf_event *event) { - struct arm_pmu *armpmu = to_arm_pmu(event->pmu); + struct arm_pmu *armpmu; if (is_software_event(event)) return 1; + /* + * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The + * core perf code won't check that the pmu->ctx == leader->ctx + * until after pmu->event_init(event). + */ + if (event->pmu != pmu) + return 0; + if (event->state < PERF_EVENT_STATE_OFF) return 1; if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) return 1; + armpmu = to_arm_pmu(event->pmu); return armpmu->get_event_idx(hw_events, event) >= 0; } @@ -288,15 +297,15 @@ validate_group(struct perf_event *event) */ memset(&fake_pmu.used_mask, 0, sizeof(fake_pmu.used_mask)); - if (!validate_event(&fake_pmu, leader)) + if (!validate_event(event->pmu, &fake_pmu, leader)) return -EINVAL; list_for_each_entry(sibling, &leader->sibling_list, group_entry) { - if (!validate_event(&fake_pmu, sibling)) + if (!validate_event(event->pmu, &fake_pmu, sibling)) return -EINVAL; } - if (!validate_event(&fake_pmu, event)) + if (!validate_event(event->pmu, &fake_pmu, event)) return -EINVAL; return 0; diff --git a/arch/arm/kernel/perf_event_cpu.c b/arch/arm/kernel/perf_event_cpu.c index 61b53c46edfa..91c7ba182dcd 100644 --- a/arch/arm/kernel/perf_event_cpu.c +++ b/arch/arm/kernel/perf_event_cpu.c @@ -92,11 +92,16 @@ static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu) free_percpu_irq(irq, &hw_events->percpu_pmu); } else { for (i = 0; i < irqs; ++i) { - if (!cpumask_test_and_clear_cpu(i, &cpu_pmu->active_irqs)) + int cpu = i; + + if (cpu_pmu->irq_affinity) + cpu = cpu_pmu->irq_affinity[i]; + + if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs)) continue; irq = platform_get_irq(pmu_device, i); if (irq >= 0) - free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, i)); + free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu)); } } } @@ -128,32 +133,37 @@ static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler) on_each_cpu(cpu_pmu_enable_percpu_irq, &irq, 1); } else { for (i = 0; i < irqs; ++i) { + int cpu = i; + err = 0; irq = platform_get_irq(pmu_device, i); if (irq < 0) continue; + if (cpu_pmu->irq_affinity) + cpu = cpu_pmu->irq_affinity[i]; + /* * If we have a single PMU interrupt that we can't shift, * assume that we're running on a uniprocessor machine and * continue. Otherwise, continue without this interrupt. */ - if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { + if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) { pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n", - irq, i); + irq, cpu); continue; } err = request_irq(irq, handler, IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu", - per_cpu_ptr(&hw_events->percpu_pmu, i)); + per_cpu_ptr(&hw_events->percpu_pmu, cpu)); if (err) { pr_err("unable to request IRQ%d for ARM PMU counters\n", irq); return err; } - cpumask_set_cpu(i, &cpu_pmu->active_irqs); + cpumask_set_cpu(cpu, &cpu_pmu->active_irqs); } } @@ -243,6 +253,8 @@ static const struct of_device_id cpu_pmu_of_device_ids[] = { {.compatible = "arm,arm1176-pmu", .data = armv6_1176_pmu_init}, {.compatible = "arm,arm1136-pmu", .data = armv6_1136_pmu_init}, {.compatible = "qcom,krait-pmu", .data = krait_pmu_init}, + {.compatible = "qcom,scorpion-pmu", .data = scorpion_pmu_init}, + {.compatible = "qcom,scorpion-mp-pmu", .data = scorpion_mp_pmu_init}, {}, }; @@ -289,6 +301,48 @@ static int probe_current_pmu(struct arm_pmu *pmu) return ret; } +static int of_pmu_irq_cfg(struct platform_device *pdev) +{ + int i; + int *irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL); + + if (!irqs) + return -ENOMEM; + + for (i = 0; i < pdev->num_resources; ++i) { + struct device_node *dn; + int cpu; + + dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity", + i); + if (!dn) { + pr_warn("Failed to parse %s/interrupt-affinity[%d]\n", + of_node_full_name(dn), i); + break; + } + + for_each_possible_cpu(cpu) + if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL)) + break; + + of_node_put(dn); + if (cpu >= nr_cpu_ids) { + pr_warn("Failed to find logical CPU for %s\n", + dn->name); + break; + } + + irqs[i] = cpu; + } + + if (i == pdev->num_resources) + cpu_pmu->irq_affinity = irqs; + else + kfree(irqs); + + return 0; +} + static int cpu_pmu_device_probe(struct platform_device *pdev) { const struct of_device_id *of_id; @@ -313,7 +367,10 @@ static int cpu_pmu_device_probe(struct platform_device *pdev) if (node && (of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node))) { init_fn = of_id->data; - ret = init_fn(pmu); + + ret = of_pmu_irq_cfg(pdev); + if (!ret) + ret = init_fn(pmu); } else { ret = probe_current_pmu(pmu); } diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 8993770c47de..f4207a4dcb01 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -140,6 +140,23 @@ enum krait_perf_types { KRAIT_PERFCTR_L1_DTLB_ACCESS = 0x12210, }; +/* ARMv7 Scorpion specific event types */ +enum scorpion_perf_types { + SCORPION_LPM0_GROUP0 = 0x4c, + SCORPION_LPM1_GROUP0 = 0x50, + SCORPION_LPM2_GROUP0 = 0x54, + SCORPION_L2LPM_GROUP0 = 0x58, + SCORPION_VLPM_GROUP0 = 0x5c, + + SCORPION_ICACHE_ACCESS = 0x10053, + SCORPION_ICACHE_MISS = 0x10052, + + SCORPION_DTLB_ACCESS = 0x12013, + SCORPION_DTLB_MISS = 0x12012, + + SCORPION_ITLB_MISS = 0x12021, +}; + /* * Cortex-A8 HW events mapping * @@ -482,6 +499,49 @@ static const unsigned krait_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] }; /* + * Scorpion HW events mapping + */ +static const unsigned scorpion_perf_map[PERF_COUNT_HW_MAX] = { + PERF_MAP_ALL_UNSUPPORTED, + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, +}; + +static const unsigned scorpion_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + PERF_CACHE_MAP_ALL_UNSUPPORTED, + /* + * The performance counters don't differentiate between read and write + * accesses/misses so this isn't strictly correct, but it's the best we + * can do. Writes and reads get combined. + */ + [C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + [C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_DCACHE_ACCESS, + [C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_L1_DCACHE_REFILL, + [C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_ICACHE_ACCESS, + [C(L1I)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ICACHE_MISS, + /* + * Only ITLB misses and DTLB refills are supported. If users want the + * DTLB refills misses a raw counter must be used. + */ + [C(DTLB)][C(OP_READ)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, + [C(DTLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, + [C(DTLB)][C(OP_WRITE)][C(RESULT_ACCESS)] = SCORPION_DTLB_ACCESS, + [C(DTLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_DTLB_MISS, + [C(ITLB)][C(OP_READ)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, + [C(ITLB)][C(OP_WRITE)][C(RESULT_MISS)] = SCORPION_ITLB_MISS, + [C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_BRANCH_PRED, + [C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, +}; + +/* * Perf Events' indices */ #define ARMV7_IDX_CYCLE_COUNTER 0 @@ -976,6 +1036,12 @@ static int krait_map_event_no_branch(struct perf_event *event) &krait_perf_cache_map, 0xFFFFF); } +static int scorpion_map_event(struct perf_event *event) +{ + return armpmu_map_event(event, &scorpion_perf_map, + &scorpion_perf_cache_map, 0xFFFFF); +} + static void armv7pmu_init(struct arm_pmu *cpu_pmu) { cpu_pmu->handle_irq = armv7pmu_handle_irq; @@ -1103,6 +1169,12 @@ static int armv7_a17_pmu_init(struct arm_pmu *cpu_pmu) #define KRAIT_EVENT_MASK (KRAIT_EVENT | VENUM_EVENT) #define PMRESRn_EN BIT(31) +#define EVENT_REGION(event) (((event) >> 12) & 0xf) /* R */ +#define EVENT_GROUP(event) ((event) & 0xf) /* G */ +#define EVENT_CODE(event) (((event) >> 4) & 0xff) /* CC */ +#define EVENT_VENUM(event) (!!(event & VENUM_EVENT)) /* N=2 */ +#define EVENT_CPU(event) (!!(event & KRAIT_EVENT)) /* N=1 */ + static u32 krait_read_pmresrn(int n) { u32 val; @@ -1141,19 +1213,19 @@ static void krait_write_pmresrn(int n, u32 val) } } -static u32 krait_read_vpmresr0(void) +static u32 venum_read_pmresr(void) { u32 val; asm volatile("mrc p10, 7, %0, c11, c0, 0" : "=r" (val)); return val; } -static void krait_write_vpmresr0(u32 val) +static void venum_write_pmresr(u32 val) { asm volatile("mcr p10, 7, %0, c11, c0, 0" : : "r" (val)); } -static void krait_pre_vpmresr0(u32 *venum_orig_val, u32 *fp_orig_val) +static void venum_pre_pmresr(u32 *venum_orig_val, u32 *fp_orig_val) { u32 venum_new_val; u32 fp_new_val; @@ -1170,7 +1242,7 @@ static void krait_pre_vpmresr0(u32 *venum_orig_val, u32 *fp_orig_val) fmxr(FPEXC, fp_new_val); } -static void krait_post_vpmresr0(u32 venum_orig_val, u32 fp_orig_val) +static void venum_post_pmresr(u32 venum_orig_val, u32 fp_orig_val) { BUG_ON(preemptible()); /* Restore FPEXC */ @@ -1193,16 +1265,11 @@ static void krait_evt_setup(int idx, u32 config_base) u32 val; u32 mask; u32 vval, fval; - unsigned int region; - unsigned int group; - unsigned int code; + unsigned int region = EVENT_REGION(config_base); + unsigned int group = EVENT_GROUP(config_base); + unsigned int code = EVENT_CODE(config_base); unsigned int group_shift; - bool venum_event; - - venum_event = !!(config_base & VENUM_EVENT); - region = (config_base >> 12) & 0xf; - code = (config_base >> 4) & 0xff; - group = (config_base >> 0) & 0xf; + bool venum_event = EVENT_VENUM(config_base); group_shift = group * 8; mask = 0xff << group_shift; @@ -1217,16 +1284,14 @@ static void krait_evt_setup(int idx, u32 config_base) val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); armv7_pmnc_write_evtsel(idx, val); - asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); - if (venum_event) { - krait_pre_vpmresr0(&vval, &fval); - val = krait_read_vpmresr0(); + venum_pre_pmresr(&vval, &fval); + val = venum_read_pmresr(); val &= ~mask; val |= code << group_shift; val |= PMRESRn_EN; - krait_write_vpmresr0(val); - krait_post_vpmresr0(vval, fval); + venum_write_pmresr(val); + venum_post_pmresr(vval, fval); } else { val = krait_read_pmresrn(region); val &= ~mask; @@ -1236,7 +1301,7 @@ static void krait_evt_setup(int idx, u32 config_base) } } -static u32 krait_clear_pmresrn_group(u32 val, int group) +static u32 clear_pmresrn_group(u32 val, int group) { u32 mask; int group_shift; @@ -1256,23 +1321,19 @@ static void krait_clearpmu(u32 config_base) { u32 val; u32 vval, fval; - unsigned int region; - unsigned int group; - bool venum_event; - - venum_event = !!(config_base & VENUM_EVENT); - region = (config_base >> 12) & 0xf; - group = (config_base >> 0) & 0xf; + unsigned int region = EVENT_REGION(config_base); + unsigned int group = EVENT_GROUP(config_base); + bool venum_event = EVENT_VENUM(config_base); if (venum_event) { - krait_pre_vpmresr0(&vval, &fval); - val = krait_read_vpmresr0(); - val = krait_clear_pmresrn_group(val, group); - krait_write_vpmresr0(val); - krait_post_vpmresr0(vval, fval); + venum_pre_pmresr(&vval, &fval); + val = venum_read_pmresr(); + val = clear_pmresrn_group(val, group); + venum_write_pmresr(val); + venum_post_pmresr(vval, fval); } else { val = krait_read_pmresrn(region); - val = krait_clear_pmresrn_group(val, group); + val = clear_pmresrn_group(val, group); krait_write_pmresrn(region, val); } } @@ -1342,6 +1403,8 @@ static void krait_pmu_enable_event(struct perf_event *event) static void krait_pmu_reset(void *info) { u32 vval, fval; + struct arm_pmu *cpu_pmu = info; + u32 idx, nb_cnt = cpu_pmu->num_events; armv7pmu_reset(info); @@ -1350,9 +1413,16 @@ static void krait_pmu_reset(void *info) krait_write_pmresrn(1, 0); krait_write_pmresrn(2, 0); - krait_pre_vpmresr0(&vval, &fval); - krait_write_vpmresr0(0); - krait_post_vpmresr0(vval, fval); + venum_pre_pmresr(&vval, &fval); + venum_write_pmresr(0); + venum_post_pmresr(vval, fval); + + /* Reset PMxEVNCTCR to sane default */ + for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { + armv7_pmnc_select_counter(idx); + asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); + } + } static int krait_event_to_bit(struct perf_event *event, unsigned int region, @@ -1386,26 +1456,18 @@ static int krait_pmu_get_event_idx(struct pmu_hw_events *cpuc, { int idx; int bit = -1; - unsigned int prefix; - unsigned int region; - unsigned int code; - unsigned int group; - bool krait_event; struct hw_perf_event *hwc = &event->hw; + unsigned int region = EVENT_REGION(hwc->config_base); + unsigned int code = EVENT_CODE(hwc->config_base); + unsigned int group = EVENT_GROUP(hwc->config_base); + bool venum_event = EVENT_VENUM(hwc->config_base); + bool krait_event = EVENT_CPU(hwc->config_base); - region = (hwc->config_base >> 12) & 0xf; - code = (hwc->config_base >> 4) & 0xff; - group = (hwc->config_base >> 0) & 0xf; - krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK); - - if (krait_event) { + if (venum_event || krait_event) { /* Ignore invalid events */ if (group > 3 || region > 2) return -EINVAL; - prefix = hwc->config_base & KRAIT_EVENT_MASK; - if (prefix != KRAIT_EVENT && prefix != VENUM_EVENT) - return -EINVAL; - if (prefix == VENUM_EVENT && (code & 0xe0)) + if (venum_event && (code & 0xe0)) return -EINVAL; bit = krait_event_to_bit(event, region, group); @@ -1425,15 +1487,12 @@ static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, { int bit; struct hw_perf_event *hwc = &event->hw; - unsigned int region; - unsigned int group; - bool krait_event; + unsigned int region = EVENT_REGION(hwc->config_base); + unsigned int group = EVENT_GROUP(hwc->config_base); + bool venum_event = EVENT_VENUM(hwc->config_base); + bool krait_event = EVENT_CPU(hwc->config_base); - region = (hwc->config_base >> 12) & 0xf; - group = (hwc->config_base >> 0) & 0xf; - krait_event = !!(hwc->config_base & KRAIT_EVENT_MASK); - - if (krait_event) { + if (venum_event || krait_event) { bit = krait_event_to_bit(event, region, group); clear_bit(bit, cpuc->used_mask); } @@ -1458,6 +1517,344 @@ static int krait_pmu_init(struct arm_pmu *cpu_pmu) cpu_pmu->clear_event_idx = krait_pmu_clear_event_idx; return 0; } + +/* + * Scorpion Local Performance Monitor Register (LPMn) + * + * 31 30 24 16 8 0 + * +--------------------------------+ + * LPM0 | EN | CC | CC | CC | CC | N = 1, R = 0 + * +--------------------------------+ + * LPM1 | EN | CC | CC | CC | CC | N = 1, R = 1 + * +--------------------------------+ + * LPM2 | EN | CC | CC | CC | CC | N = 1, R = 2 + * +--------------------------------+ + * L2LPM | EN | CC | CC | CC | CC | N = 1, R = 3 + * +--------------------------------+ + * VLPM | EN | CC | CC | CC | CC | N = 2, R = ? + * +--------------------------------+ + * EN | G=3 | G=2 | G=1 | G=0 + * + * + * Event Encoding: + * + * hwc->config_base = 0xNRCCG + * + * N = prefix, 1 for Scorpion CPU (LPMn/L2LPM), 2 for Venum VFP (VLPM) + * R = region register + * CC = class of events the group G is choosing from + * G = group or particular event + * + * Example: 0x12021 is a Scorpion CPU event in LPM2's group 1 with code 2 + * + * A region (R) corresponds to a piece of the CPU (execution unit, instruction + * unit, etc.) while the event code (CC) corresponds to a particular class of + * events (interrupts for example). An event code is broken down into + * groups (G) that can be mapped into the PMU (irq, fiqs, and irq+fiqs for + * example). + */ + +static u32 scorpion_read_pmresrn(int n) +{ + u32 val; + + switch (n) { + case 0: + asm volatile("mrc p15, 0, %0, c15, c0, 0" : "=r" (val)); + break; + case 1: + asm volatile("mrc p15, 1, %0, c15, c0, 0" : "=r" (val)); + break; + case 2: + asm volatile("mrc p15, 2, %0, c15, c0, 0" : "=r" (val)); + break; + case 3: + asm volatile("mrc p15, 3, %0, c15, c2, 0" : "=r" (val)); + break; + default: + BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ + } + + return val; +} + +static void scorpion_write_pmresrn(int n, u32 val) +{ + switch (n) { + case 0: + asm volatile("mcr p15, 0, %0, c15, c0, 0" : : "r" (val)); + break; + case 1: + asm volatile("mcr p15, 1, %0, c15, c0, 0" : : "r" (val)); + break; + case 2: + asm volatile("mcr p15, 2, %0, c15, c0, 0" : : "r" (val)); + break; + case 3: + asm volatile("mcr p15, 3, %0, c15, c2, 0" : : "r" (val)); + break; + default: + BUG(); /* Should be validated in scorpion_pmu_get_event_idx() */ + } +} + +static u32 scorpion_get_pmresrn_event(unsigned int region) +{ + static const u32 pmresrn_table[] = { SCORPION_LPM0_GROUP0, + SCORPION_LPM1_GROUP0, + SCORPION_LPM2_GROUP0, + SCORPION_L2LPM_GROUP0 }; + return pmresrn_table[region]; +} + +static void scorpion_evt_setup(int idx, u32 config_base) +{ + u32 val; + u32 mask; + u32 vval, fval; + unsigned int region = EVENT_REGION(config_base); + unsigned int group = EVENT_GROUP(config_base); + unsigned int code = EVENT_CODE(config_base); + unsigned int group_shift; + bool venum_event = EVENT_VENUM(config_base); + + group_shift = group * 8; + mask = 0xff << group_shift; + + /* Configure evtsel for the region and group */ + if (venum_event) + val = SCORPION_VLPM_GROUP0; + else + val = scorpion_get_pmresrn_event(region); + val += group; + /* Mix in mode-exclusion bits */ + val |= config_base & (ARMV7_EXCLUDE_USER | ARMV7_EXCLUDE_PL1); + armv7_pmnc_write_evtsel(idx, val); + + asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); + + if (venum_event) { + venum_pre_pmresr(&vval, &fval); + val = venum_read_pmresr(); + val &= ~mask; + val |= code << group_shift; + val |= PMRESRn_EN; + venum_write_pmresr(val); + venum_post_pmresr(vval, fval); + } else { + val = scorpion_read_pmresrn(region); + val &= ~mask; + val |= code << group_shift; + val |= PMRESRn_EN; + scorpion_write_pmresrn(region, val); + } +} + +static void scorpion_clearpmu(u32 config_base) +{ + u32 val; + u32 vval, fval; + unsigned int region = EVENT_REGION(config_base); + unsigned int group = EVENT_GROUP(config_base); + bool venum_event = EVENT_VENUM(config_base); + + if (venum_event) { + venum_pre_pmresr(&vval, &fval); + val = venum_read_pmresr(); + val = clear_pmresrn_group(val, group); + venum_write_pmresr(val); + venum_post_pmresr(vval, fval); + } else { + val = scorpion_read_pmresrn(region); + val = clear_pmresrn_group(val, group); + scorpion_write_pmresrn(region, val); + } +} + +static void scorpion_pmu_disable_event(struct perf_event *event) +{ + unsigned long flags; + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); + struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); + + /* Disable counter and interrupt */ + raw_spin_lock_irqsave(&events->pmu_lock, flags); + + /* Disable counter */ + armv7_pmnc_disable_counter(idx); + + /* + * Clear pmresr code (if destined for PMNx counters) + */ + if (hwc->config_base & KRAIT_EVENT_MASK) + scorpion_clearpmu(hwc->config_base); + + /* Disable interrupt for this counter */ + armv7_pmnc_disable_intens(idx); + + raw_spin_unlock_irqrestore(&events->pmu_lock, flags); +} + +static void scorpion_pmu_enable_event(struct perf_event *event) +{ + unsigned long flags; + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); + struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events); + + /* + * Enable counter and interrupt, and set the counter to count + * the event that we're interested in. + */ + raw_spin_lock_irqsave(&events->pmu_lock, flags); + + /* Disable counter */ + armv7_pmnc_disable_counter(idx); + + /* + * Set event (if destined for PMNx counters) + * We don't set the event for the cycle counter because we + * don't have the ability to perform event filtering. + */ + if (hwc->config_base & KRAIT_EVENT_MASK) + scorpion_evt_setup(idx, hwc->config_base); + else if (idx != ARMV7_IDX_CYCLE_COUNTER) + armv7_pmnc_write_evtsel(idx, hwc->config_base); + + /* Enable interrupt for this counter */ + armv7_pmnc_enable_intens(idx); + + /* Enable counter */ + armv7_pmnc_enable_counter(idx); + + raw_spin_unlock_irqrestore(&events->pmu_lock, flags); +} + +static void scorpion_pmu_reset(void *info) +{ + u32 vval, fval; + struct arm_pmu *cpu_pmu = info; + u32 idx, nb_cnt = cpu_pmu->num_events; + + armv7pmu_reset(info); + + /* Clear all pmresrs */ + scorpion_write_pmresrn(0, 0); + scorpion_write_pmresrn(1, 0); + scorpion_write_pmresrn(2, 0); + scorpion_write_pmresrn(3, 0); + + venum_pre_pmresr(&vval, &fval); + venum_write_pmresr(0); + venum_post_pmresr(vval, fval); + + /* Reset PMxEVNCTCR to sane default */ + for (idx = ARMV7_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) { + armv7_pmnc_select_counter(idx); + asm volatile("mcr p15, 0, %0, c9, c15, 0" : : "r" (0)); + } +} + +static int scorpion_event_to_bit(struct perf_event *event, unsigned int region, + unsigned int group) +{ + int bit; + struct hw_perf_event *hwc = &event->hw; + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); + + if (hwc->config_base & VENUM_EVENT) + bit = SCORPION_VLPM_GROUP0; + else + bit = scorpion_get_pmresrn_event(region); + bit -= scorpion_get_pmresrn_event(0); + bit += group; + /* + * Lower bits are reserved for use by the counters (see + * armv7pmu_get_event_idx() for more info) + */ + bit += ARMV7_IDX_COUNTER_LAST(cpu_pmu) + 1; + + return bit; +} + +/* + * We check for column exclusion constraints here. + * Two events cant use the same group within a pmresr register. + */ +static int scorpion_pmu_get_event_idx(struct pmu_hw_events *cpuc, + struct perf_event *event) +{ + int idx; + int bit = -1; + struct hw_perf_event *hwc = &event->hw; + unsigned int region = EVENT_REGION(hwc->config_base); + unsigned int group = EVENT_GROUP(hwc->config_base); + bool venum_event = EVENT_VENUM(hwc->config_base); + bool scorpion_event = EVENT_CPU(hwc->config_base); + + if (venum_event || scorpion_event) { + /* Ignore invalid events */ + if (group > 3 || region > 3) + return -EINVAL; + + bit = scorpion_event_to_bit(event, region, group); + if (test_and_set_bit(bit, cpuc->used_mask)) + return -EAGAIN; + } + + idx = armv7pmu_get_event_idx(cpuc, event); + if (idx < 0 && bit >= 0) + clear_bit(bit, cpuc->used_mask); + + return idx; +} + +static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc, + struct perf_event *event) +{ + int bit; + struct hw_perf_event *hwc = &event->hw; + unsigned int region = EVENT_REGION(hwc->config_base); + unsigned int group = EVENT_GROUP(hwc->config_base); + bool venum_event = EVENT_VENUM(hwc->config_base); + bool scorpion_event = EVENT_CPU(hwc->config_base); + + if (venum_event || scorpion_event) { + bit = scorpion_event_to_bit(event, region, group); + clear_bit(bit, cpuc->used_mask); + } +} + +static int scorpion_pmu_init(struct arm_pmu *cpu_pmu) +{ + armv7pmu_init(cpu_pmu); + cpu_pmu->name = "armv7_scorpion"; + cpu_pmu->map_event = scorpion_map_event; + cpu_pmu->num_events = armv7_read_num_pmnc_events(); + cpu_pmu->reset = scorpion_pmu_reset; + cpu_pmu->enable = scorpion_pmu_enable_event; + cpu_pmu->disable = scorpion_pmu_disable_event; + cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; + cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; + return 0; +} + +static int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu) +{ + armv7pmu_init(cpu_pmu); + cpu_pmu->name = "armv7_scorpion_mp"; + cpu_pmu->map_event = scorpion_map_event; + cpu_pmu->num_events = armv7_read_num_pmnc_events(); + cpu_pmu->reset = scorpion_pmu_reset; + cpu_pmu->enable = scorpion_pmu_enable_event; + cpu_pmu->disable = scorpion_pmu_disable_event; + cpu_pmu->get_event_idx = scorpion_pmu_get_event_idx; + cpu_pmu->clear_event_idx = scorpion_pmu_clear_event_idx; + return 0; +} #else static inline int armv7_a8_pmu_init(struct arm_pmu *cpu_pmu) { @@ -1498,4 +1895,14 @@ static inline int krait_pmu_init(struct arm_pmu *cpu_pmu) { return -ENODEV; } + +static inline int scorpion_pmu_init(struct arm_pmu *cpu_pmu) +{ + return -ENODEV; +} + +static inline int scorpion_mp_pmu_init(struct arm_pmu *cpu_pmu) +{ + return -ENODEV; +} #endif /* CONFIG_CPU_V7 */ diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index fdfa3a78ec8c..f192a2a41719 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -17,12 +17,9 @@ #include <linux/stddef.h> #include <linux/unistd.h> #include <linux/user.h> -#include <linux/delay.h> -#include <linux/reboot.h> #include <linux/interrupt.h> #include <linux/kallsyms.h> #include <linux/init.h> -#include <linux/cpu.h> #include <linux/elfcore.h> #include <linux/pm.h> #include <linux/tick.h> @@ -31,16 +28,14 @@ #include <linux/random.h> #include <linux/hw_breakpoint.h> #include <linux/leds.h> -#include <linux/reboot.h> -#include <asm/cacheflush.h> -#include <asm/idmap.h> #include <asm/processor.h> #include <asm/thread_notify.h> #include <asm/stacktrace.h> #include <asm/system_misc.h> #include <asm/mach/time.h> #include <asm/tls.h> +#include <asm/vdso.h> #ifdef CONFIG_CC_STACKPROTECTOR #include <linux/stackprotector.h> @@ -59,69 +54,6 @@ static const char *isa_modes[] __maybe_unused = { "ARM" , "Thumb" , "Jazelle", "ThumbEE" }; -extern void call_with_stack(void (*fn)(void *), void *arg, void *sp); -typedef void (*phys_reset_t)(unsigned long); - -/* - * A temporary stack to use for CPU reset. This is static so that we - * don't clobber it with the identity mapping. When running with this - * stack, any references to the current task *will not work* so you - * should really do as little as possible before jumping to your reset - * code. - */ -static u64 soft_restart_stack[16]; - -static void __soft_restart(void *addr) -{ - phys_reset_t phys_reset; - - /* Take out a flat memory mapping. */ - setup_mm_for_reboot(); - - /* Clean and invalidate caches */ - flush_cache_all(); - - /* Turn off caching */ - cpu_proc_fin(); - - /* Push out any further dirty data, and ensure cache is empty */ - flush_cache_all(); - - /* Switch to the identity mapping. */ - phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset); - phys_reset((unsigned long)addr); - - /* Should never get here. */ - BUG(); -} - -void soft_restart(unsigned long addr) -{ - u64 *stack = soft_restart_stack + ARRAY_SIZE(soft_restart_stack); - - /* Disable interrupts first */ - raw_local_irq_disable(); - local_fiq_disable(); - - /* Disable the L2 if we're the last man standing. */ - if (num_online_cpus() == 1) - outer_disable(); - - /* Change to the new stack and continue with the reset. */ - call_with_stack(__soft_restart, (void *)addr, (void *)stack); - - /* Should never get here. */ - BUG(); -} - -/* - * Function pointers to optional machine specific functions - */ -void (*pm_power_off)(void); -EXPORT_SYMBOL(pm_power_off); - -void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); - /* * This is our default idle handler. */ @@ -166,79 +98,6 @@ void arch_cpu_idle_dead(void) } #endif -/* - * Called by kexec, immediately prior to machine_kexec(). - * - * This must completely disable all secondary CPUs; simply causing those CPUs - * to execute e.g. a RAM-based pin loop is not sufficient. This allows the - * kexec'd kernel to use any and all RAM as it sees fit, without having to - * avoid any code or data used by any SW CPU pin loop. The CPU hotplug - * functionality embodied in disable_nonboot_cpus() to achieve this. - */ -void machine_shutdown(void) -{ - disable_nonboot_cpus(); -} - -/* - * Halting simply requires that the secondary CPUs stop performing any - * activity (executing tasks, handling interrupts). smp_send_stop() - * achieves this. - */ -void machine_halt(void) -{ - local_irq_disable(); - smp_send_stop(); - - local_irq_disable(); - while (1); -} - -/* - * Power-off simply requires that the secondary CPUs stop performing any - * activity (executing tasks, handling interrupts). smp_send_stop() - * achieves this. When the system power is turned off, it will take all CPUs - * with it. - */ -void machine_power_off(void) -{ - local_irq_disable(); - smp_send_stop(); - - if (pm_power_off) - pm_power_off(); -} - -/* - * Restart requires that the secondary CPUs stop performing any activity - * while the primary CPU resets the system. Systems with a single CPU can - * use soft_restart() as their machine descriptor's .restart hook, since that - * will cause the only available CPU to reset. Systems with multiple CPUs must - * provide a HW restart implementation, to ensure that all CPUs reset at once. - * This is required so that any code running after reset on the primary CPU - * doesn't have to co-ordinate with other CPUs to ensure they aren't still - * executing pre-reset code, and using RAM that the primary CPU's code wishes - * to use. Implementing such co-ordination would be essentially impossible. - */ -void machine_restart(char *cmd) -{ - local_irq_disable(); - smp_send_stop(); - - if (arm_pm_restart) - arm_pm_restart(reboot_mode, cmd); - else - do_kernel_restart(cmd); - - /* Give a grace period for failure to restart of 1s */ - mdelay(1000); - - /* Whoops - the platform was unable to reboot. Tell the user! */ - printk("Reboot failed -- System halted\n"); - local_irq_disable(); - while (1); -} - void __show_regs(struct pt_regs *regs) { unsigned long flags; @@ -475,7 +334,7 @@ const char *arch_vma_name(struct vm_area_struct *vma) } /* If possible, provide a placement hint at a random offset from the - * stack for the signal page. + * stack for the sigpage and vdso pages. */ static unsigned long sigpage_addr(const struct mm_struct *mm, unsigned int npages) @@ -519,6 +378,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; + unsigned long npages; unsigned long addr; unsigned long hint; int ret = 0; @@ -528,9 +388,12 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) if (!signal_page) return -ENOMEM; + npages = 1; /* for sigpage */ + npages += vdso_total_pages; + down_write(&mm->mmap_sem); - hint = sigpage_addr(mm, 1); - addr = get_unmapped_area(NULL, hint, PAGE_SIZE, 0, 0); + hint = sigpage_addr(mm, npages); + addr = get_unmapped_area(NULL, hint, npages << PAGE_SHIFT, 0, 0); if (IS_ERR_VALUE(addr)) { ret = addr; goto up_fail; @@ -547,6 +410,12 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) mm->context.sigpage = addr; + /* Unlike the sigpage, failure to install the vdso is unlikely + * to be fatal to the process, so no error check needed + * here. + */ + arm_install_vdso(mm, addr + PAGE_SIZE); + up_fail: up_write(&mm->mmap_sem); return ret; diff --git a/arch/arm/kernel/psci-call.S b/arch/arm/kernel/psci-call.S new file mode 100644 index 000000000000..a78e9e1e206d --- /dev/null +++ b/arch/arm/kernel/psci-call.S @@ -0,0 +1,31 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Copyright (C) 2015 ARM Limited + * + * Author: Mark Rutland <mark.rutland@arm.com> + */ + +#include <linux/linkage.h> + +#include <asm/opcodes-sec.h> +#include <asm/opcodes-virt.h> + +/* int __invoke_psci_fn_hvc(u32 function_id, u32 arg0, u32 arg1, u32 arg2) */ +ENTRY(__invoke_psci_fn_hvc) + __HVC(0) + bx lr +ENDPROC(__invoke_psci_fn_hvc) + +/* int __invoke_psci_fn_smc(u32 function_id, u32 arg0, u32 arg1, u32 arg2) */ +ENTRY(__invoke_psci_fn_smc) + __SMC(0) + bx lr +ENDPROC(__invoke_psci_fn_smc) diff --git a/arch/arm/kernel/psci.c b/arch/arm/kernel/psci.c index f73891b6b730..f90fdf4ce7c7 100644 --- a/arch/arm/kernel/psci.c +++ b/arch/arm/kernel/psci.c @@ -23,8 +23,6 @@ #include <asm/compiler.h> #include <asm/errno.h> -#include <asm/opcodes-sec.h> -#include <asm/opcodes-virt.h> #include <asm/psci.h> #include <asm/system_misc.h> @@ -33,6 +31,9 @@ struct psci_operations psci_ops; static int (*invoke_psci_fn)(u32, u32, u32, u32); typedef int (*psci_initcall_t)(const struct device_node *); +asmlinkage int __invoke_psci_fn_hvc(u32, u32, u32, u32); +asmlinkage int __invoke_psci_fn_smc(u32, u32, u32, u32); + enum psci_function { PSCI_FN_CPU_SUSPEND, PSCI_FN_CPU_ON, @@ -71,40 +72,6 @@ static u32 psci_power_state_pack(struct psci_power_state state) & PSCI_0_2_POWER_STATE_AFFL_MASK); } -/* - * The following two functions are invoked via the invoke_psci_fn pointer - * and will not be inlined, allowing us to piggyback on the AAPCS. - */ -static noinline int __invoke_psci_fn_hvc(u32 function_id, u32 arg0, u32 arg1, - u32 arg2) -{ - asm volatile( - __asmeq("%0", "r0") - __asmeq("%1", "r1") - __asmeq("%2", "r2") - __asmeq("%3", "r3") - __HVC(0) - : "+r" (function_id) - : "r" (arg0), "r" (arg1), "r" (arg2)); - - return function_id; -} - -static noinline int __invoke_psci_fn_smc(u32 function_id, u32 arg0, u32 arg1, - u32 arg2) -{ - asm volatile( - __asmeq("%0", "r0") - __asmeq("%1", "r1") - __asmeq("%2", "r2") - __asmeq("%3", "r3") - __SMC(0) - : "+r" (function_id) - : "r" (arg0), "r" (arg1), "r" (arg2)); - - return function_id; -} - static int psci_get_version(void) { int err; diff --git a/arch/arm/kernel/reboot.c b/arch/arm/kernel/reboot.c new file mode 100644 index 000000000000..1a4d232796be --- /dev/null +++ b/arch/arm/kernel/reboot.c @@ -0,0 +1,155 @@ +/* + * Copyright (C) 1996-2000 Russell King - Converted to ARM. + * Original Copyright (C) 1995 Linus Torvalds + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/cpu.h> +#include <linux/delay.h> +#include <linux/reboot.h> + +#include <asm/cacheflush.h> +#include <asm/idmap.h> + +#include "reboot.h" + +typedef void (*phys_reset_t)(unsigned long); + +/* + * Function pointers to optional machine specific functions + */ +void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd); +void (*pm_power_off)(void); +EXPORT_SYMBOL(pm_power_off); + +/* + * A temporary stack to use for CPU reset. This is static so that we + * don't clobber it with the identity mapping. When running with this + * stack, any references to the current task *will not work* so you + * should really do as little as possible before jumping to your reset + * code. + */ +static u64 soft_restart_stack[16]; + +static void __soft_restart(void *addr) +{ + phys_reset_t phys_reset; + + /* Take out a flat memory mapping. */ + setup_mm_for_reboot(); + + /* Clean and invalidate caches */ + flush_cache_all(); + + /* Turn off caching */ + cpu_proc_fin(); + + /* Push out any further dirty data, and ensure cache is empty */ + flush_cache_all(); + + /* Switch to the identity mapping. */ + phys_reset = (phys_reset_t)(unsigned long)virt_to_phys(cpu_reset); + phys_reset((unsigned long)addr); + + /* Should never get here. */ + BUG(); +} + +void _soft_restart(unsigned long addr, bool disable_l2) +{ + u64 *stack = soft_restart_stack + ARRAY_SIZE(soft_restart_stack); + + /* Disable interrupts first */ + raw_local_irq_disable(); + local_fiq_disable(); + + /* Disable the L2 if we're the last man standing. */ + if (disable_l2) + outer_disable(); + + /* Change to the new stack and continue with the reset. */ + call_with_stack(__soft_restart, (void *)addr, (void *)stack); + + /* Should never get here. */ + BUG(); +} + +void soft_restart(unsigned long addr) +{ + _soft_restart(addr, num_online_cpus() == 1); +} + +/* + * Called by kexec, immediately prior to machine_kexec(). + * + * This must completely disable all secondary CPUs; simply causing those CPUs + * to execute e.g. a RAM-based pin loop is not sufficient. This allows the + * kexec'd kernel to use any and all RAM as it sees fit, without having to + * avoid any code or data used by any SW CPU pin loop. The CPU hotplug + * functionality embodied in disable_nonboot_cpus() to achieve this. + */ +void machine_shutdown(void) +{ + disable_nonboot_cpus(); +} + +/* + * Halting simply requires that the secondary CPUs stop performing any + * activity (executing tasks, handling interrupts). smp_send_stop() + * achieves this. + */ +void machine_halt(void) +{ + local_irq_disable(); + smp_send_stop(); + + local_irq_disable(); + while (1); +} + +/* + * Power-off simply requires that the secondary CPUs stop performing any + * activity (executing tasks, handling interrupts). smp_send_stop() + * achieves this. When the system power is turned off, it will take all CPUs + * with it. + */ +void machine_power_off(void) +{ + local_irq_disable(); + smp_send_stop(); + + if (pm_power_off) + pm_power_off(); +} + +/* + * Restart requires that the secondary CPUs stop performing any activity + * while the primary CPU resets the system. Systems with a single CPU can + * use soft_restart() as their machine descriptor's .restart hook, since that + * will cause the only available CPU to reset. Systems with multiple CPUs must + * provide a HW restart implementation, to ensure that all CPUs reset at once. + * This is required so that any code running after reset on the primary CPU + * doesn't have to co-ordinate with other CPUs to ensure they aren't still + * executing pre-reset code, and using RAM that the primary CPU's code wishes + * to use. Implementing such co-ordination would be essentially impossible. + */ +void machine_restart(char *cmd) +{ + local_irq_disable(); + smp_send_stop(); + + if (arm_pm_restart) + arm_pm_restart(reboot_mode, cmd); + else + do_kernel_restart(cmd); + + /* Give a grace period for failure to restart of 1s */ + mdelay(1000); + + /* Whoops - the platform was unable to reboot. Tell the user! */ + printk("Reboot failed -- System halted\n"); + local_irq_disable(); + while (1); +} diff --git a/arch/arm/kernel/reboot.h b/arch/arm/kernel/reboot.h new file mode 100644 index 000000000000..bf7a0b1f076e --- /dev/null +++ b/arch/arm/kernel/reboot.h @@ -0,0 +1,7 @@ +#ifndef REBOOT_H +#define REBOOT_H + +extern void call_with_stack(void (*fn)(void *), void *arg, void *sp); +extern void _soft_restart(unsigned long addr, bool disable_l2); + +#endif diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c index 24b4a04846eb..36ed35073289 100644 --- a/arch/arm/kernel/return_address.c +++ b/arch/arm/kernel/return_address.c @@ -56,8 +56,6 @@ void *return_address(unsigned int level) return NULL; } -#else /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) */ - -#endif /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) / else */ +#endif /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) */ EXPORT_SYMBOL_GPL(return_address); diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 1d60bebea4b8..6c777e908a24 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -372,30 +372,48 @@ void __init early_print(const char *str, ...) static void __init cpuid_init_hwcaps(void) { - unsigned int divide_instrs, vmsa; + int block; + u32 isar5; if (cpu_architecture() < CPU_ARCH_ARMv7) return; - divide_instrs = (read_cpuid_ext(CPUID_EXT_ISAR0) & 0x0f000000) >> 24; - - switch (divide_instrs) { - case 2: + block = cpuid_feature_extract(CPUID_EXT_ISAR0, 24); + if (block >= 2) elf_hwcap |= HWCAP_IDIVA; - case 1: + if (block >= 1) elf_hwcap |= HWCAP_IDIVT; - } /* LPAE implies atomic ldrd/strd instructions */ - vmsa = (read_cpuid_ext(CPUID_EXT_MMFR0) & 0xf) >> 0; - if (vmsa >= 5) + block = cpuid_feature_extract(CPUID_EXT_MMFR0, 0); + if (block >= 5) elf_hwcap |= HWCAP_LPAE; + + /* check for supported v8 Crypto instructions */ + isar5 = read_cpuid_ext(CPUID_EXT_ISAR5); + + block = cpuid_feature_extract_field(isar5, 4); + if (block >= 2) + elf_hwcap2 |= HWCAP2_PMULL; + if (block >= 1) + elf_hwcap2 |= HWCAP2_AES; + + block = cpuid_feature_extract_field(isar5, 8); + if (block >= 1) + elf_hwcap2 |= HWCAP2_SHA1; + + block = cpuid_feature_extract_field(isar5, 12); + if (block >= 1) + elf_hwcap2 |= HWCAP2_SHA2; + + block = cpuid_feature_extract_field(isar5, 16); + if (block >= 1) + elf_hwcap2 |= HWCAP2_CRC32; } static void __init elf_hwcap_fixup(void) { unsigned id = read_cpuid_id(); - unsigned sync_prim; /* * HWCAP_TLS is available only on 1136 r1p0 and later, @@ -416,9 +434,9 @@ static void __init elf_hwcap_fixup(void) * avoid advertising SWP; it may not be atomic with * multiprocessing cores. */ - sync_prim = ((read_cpuid_ext(CPUID_EXT_ISAR3) >> 8) & 0xf0) | - ((read_cpuid_ext(CPUID_EXT_ISAR4) >> 20) & 0x0f); - if (sync_prim >= 0x13) + if (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) > 1 || + (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) == 1 && + cpuid_feature_extract(CPUID_EXT_ISAR3, 20) >= 3)) elf_hwcap &= ~HWCAP_SWP; } diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index 023ac905e4c3..423663e23791 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -318,17 +318,6 @@ get_sigframe(struct ksignal *ksig, struct pt_regs *regs, int framesize) return frame; } -/* - * translate the signal - */ -static inline int map_sig(int sig) -{ - struct thread_info *thread = current_thread_info(); - if (sig < 32 && thread->exec_domain && thread->exec_domain->signal_invmap) - sig = thread->exec_domain->signal_invmap[sig]; - return sig; -} - static int setup_return(struct pt_regs *regs, struct ksignal *ksig, unsigned long __user *rc, void __user *frame) @@ -412,7 +401,7 @@ setup_return(struct pt_regs *regs, struct ksignal *ksig, } } - regs->ARM_r0 = map_sig(ksig->sig); + regs->ARM_r0 = ksig->sig; regs->ARM_sp = (unsigned long)frame; regs->ARM_lr = retcode; regs->ARM_pc = handler; diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index e1e60e5a7a27..7d37bfc50830 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -116,14 +116,7 @@ cpu_resume_after_mmu: ldmfd sp!, {r4 - r11, pc} ENDPROC(cpu_resume_after_mmu) -/* - * Note: Yes, part of the following code is located into the .data section. - * This is to allow sleep_save_sp to be accessed with a relative load - * while we can't rely on any MMU translation. We could have put - * sleep_save_sp in the .text section as well, but some setups might - * insist on it to be truly read-only. - */ - .data + .text .align ENTRY(cpu_resume) ARM_BE8(setend be) @ ensure we are in BE mode @@ -145,6 +138,8 @@ ARM_BE8(setend be) @ ensure we are in BE mode compute_mpidr_hash r1, r4, r5, r6, r0, r3 1: adr r0, _sleep_save_sp + ldr r2, [r0] + add r0, r0, r2 ldr r0, [r0, #SLEEP_SAVE_SP_PHYS] ldr r0, [r0, r1, lsl #2] @@ -156,10 +151,12 @@ THUMB( bx r3 ) ENDPROC(cpu_resume) .align 2 +_sleep_save_sp: + .long sleep_save_sp - . mpidr_hash_ptr: .long mpidr_hash - . @ mpidr_hash struct offset + .data .type sleep_save_sp, #object ENTRY(sleep_save_sp) -_sleep_save_sp: .space SLEEP_SAVE_SP_SZ @ struct sleep_save_sp diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 86ef244c5a24..cca5b8758185 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -145,6 +145,11 @@ void __init smp_init_cpus(void) smp_ops.smp_init_cpus(); } +int platform_can_secondary_boot(void) +{ + return !!smp_ops.smp_boot_secondary; +} + int platform_can_cpu_hotplug(void) { #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c index afdd51e30bec..1361756782c7 100644 --- a/arch/arm/kernel/swp_emulate.c +++ b/arch/arm/kernel/swp_emulate.c @@ -42,7 +42,7 @@ " cmp %0, #0\n" \ " movne %0, %4\n" \ "2:\n" \ - " .section .fixup,\"ax\"\n" \ + " .section .text.fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %0, %5\n" \ " b 2b\n" \ diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 788e23fe64d8..3dce1a342030 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -505,12 +505,10 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason) static int bad_syscall(int n, struct pt_regs *regs) { - struct thread_info *thread = current_thread_info(); siginfo_t info; - if ((current->personality & PER_MASK) != PER_LINUX && - thread->exec_domain->handler) { - thread->exec_domain->handler(n, regs); + if ((current->personality & PER_MASK) != PER_LINUX) { + send_sig(SIGSEGV, current, 1); return regs->ARM_r0; } diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c new file mode 100644 index 000000000000..efe17dd9b921 --- /dev/null +++ b/arch/arm/kernel/vdso.c @@ -0,0 +1,337 @@ +/* + * Adapted from arm64 version. + * + * Copyright (C) 2012 ARM Limited + * Copyright (C) 2015 Mentor Graphics Corporation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/elf.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/of.h> +#include <linux/printk.h> +#include <linux/slab.h> +#include <linux/timekeeper_internal.h> +#include <linux/vmalloc.h> +#include <asm/arch_timer.h> +#include <asm/barrier.h> +#include <asm/cacheflush.h> +#include <asm/page.h> +#include <asm/vdso.h> +#include <asm/vdso_datapage.h> +#include <clocksource/arm_arch_timer.h> + +#define MAX_SYMNAME 64 + +static struct page **vdso_text_pagelist; + +/* Total number of pages needed for the data and text portions of the VDSO. */ +unsigned int vdso_total_pages __read_mostly; + +/* + * The VDSO data page. + */ +static union vdso_data_store vdso_data_store __page_aligned_data; +static struct vdso_data *vdso_data = &vdso_data_store.data; + +static struct page *vdso_data_page; +static struct vm_special_mapping vdso_data_mapping = { + .name = "[vvar]", + .pages = &vdso_data_page, +}; + +static struct vm_special_mapping vdso_text_mapping = { + .name = "[vdso]", +}; + +struct elfinfo { + Elf32_Ehdr *hdr; /* ptr to ELF */ + Elf32_Sym *dynsym; /* ptr to .dynsym section */ + unsigned long dynsymsize; /* size of .dynsym section */ + char *dynstr; /* ptr to .dynstr section */ +}; + +/* Cached result of boot-time check for whether the arch timer exists, + * and if so, whether the virtual counter is useable. + */ +static bool cntvct_ok __read_mostly; + +static bool __init cntvct_functional(void) +{ + struct device_node *np; + bool ret = false; + + if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER)) + goto out; + + /* The arm_arch_timer core should export + * arch_timer_use_virtual or similar so we don't have to do + * this. + */ + np = of_find_compatible_node(NULL, NULL, "arm,armv7-timer"); + if (!np) + goto out_put; + + if (of_property_read_bool(np, "arm,cpu-registers-not-fw-configured")) + goto out_put; + + ret = true; + +out_put: + of_node_put(np); +out: + return ret; +} + +static void * __init find_section(Elf32_Ehdr *ehdr, const char *name, + unsigned long *size) +{ + Elf32_Shdr *sechdrs; + unsigned int i; + char *secnames; + + /* Grab section headers and strings so we can tell who is who */ + sechdrs = (void *)ehdr + ehdr->e_shoff; + secnames = (void *)ehdr + sechdrs[ehdr->e_shstrndx].sh_offset; + + /* Find the section they want */ + for (i = 1; i < ehdr->e_shnum; i++) { + if (strcmp(secnames + sechdrs[i].sh_name, name) == 0) { + if (size) + *size = sechdrs[i].sh_size; + return (void *)ehdr + sechdrs[i].sh_offset; + } + } + + if (size) + *size = 0; + return NULL; +} + +static Elf32_Sym * __init find_symbol(struct elfinfo *lib, const char *symname) +{ + unsigned int i; + + for (i = 0; i < (lib->dynsymsize / sizeof(Elf32_Sym)); i++) { + char name[MAX_SYMNAME], *c; + + if (lib->dynsym[i].st_name == 0) + continue; + strlcpy(name, lib->dynstr + lib->dynsym[i].st_name, + MAX_SYMNAME); + c = strchr(name, '@'); + if (c) + *c = 0; + if (strcmp(symname, name) == 0) + return &lib->dynsym[i]; + } + return NULL; +} + +static void __init vdso_nullpatch_one(struct elfinfo *lib, const char *symname) +{ + Elf32_Sym *sym; + + sym = find_symbol(lib, symname); + if (!sym) + return; + + sym->st_name = 0; +} + +static void __init patch_vdso(void *ehdr) +{ + struct elfinfo einfo; + + einfo = (struct elfinfo) { + .hdr = ehdr, + }; + + einfo.dynsym = find_section(einfo.hdr, ".dynsym", &einfo.dynsymsize); + einfo.dynstr = find_section(einfo.hdr, ".dynstr", NULL); + + /* If the virtual counter is absent or non-functional we don't + * want programs to incur the slight additional overhead of + * dispatching through the VDSO only to fall back to syscalls. + */ + if (!cntvct_ok) { + vdso_nullpatch_one(&einfo, "__vdso_gettimeofday"); + vdso_nullpatch_one(&einfo, "__vdso_clock_gettime"); + } +} + +static int __init vdso_init(void) +{ + unsigned int text_pages; + int i; + + if (memcmp(&vdso_start, "\177ELF", 4)) { + pr_err("VDSO is not a valid ELF object!\n"); + return -ENOEXEC; + } + + text_pages = (&vdso_end - &vdso_start) >> PAGE_SHIFT; + pr_debug("vdso: %i text pages at base %p\n", text_pages, &vdso_start); + + /* Allocate the VDSO text pagelist */ + vdso_text_pagelist = kcalloc(text_pages, sizeof(struct page *), + GFP_KERNEL); + if (vdso_text_pagelist == NULL) + return -ENOMEM; + + /* Grab the VDSO data page. */ + vdso_data_page = virt_to_page(vdso_data); + + /* Grab the VDSO text pages. */ + for (i = 0; i < text_pages; i++) { + struct page *page; + + page = virt_to_page(&vdso_start + i * PAGE_SIZE); + vdso_text_pagelist[i] = page; + } + + vdso_text_mapping.pages = vdso_text_pagelist; + + vdso_total_pages = 1; /* for the data/vvar page */ + vdso_total_pages += text_pages; + + cntvct_ok = cntvct_functional(); + + patch_vdso(&vdso_start); + + return 0; +} +arch_initcall(vdso_init); + +static int install_vvar(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + + vma = _install_special_mapping(mm, addr, PAGE_SIZE, + VM_READ | VM_MAYREAD, + &vdso_data_mapping); + + return IS_ERR(vma) ? PTR_ERR(vma) : 0; +} + +/* assumes mmap_sem is write-locked */ +void arm_install_vdso(struct mm_struct *mm, unsigned long addr) +{ + struct vm_area_struct *vma; + unsigned long len; + + mm->context.vdso = 0; + + if (vdso_text_pagelist == NULL) + return; + + if (install_vvar(mm, addr)) + return; + + /* Account for vvar page. */ + addr += PAGE_SIZE; + len = (vdso_total_pages - 1) << PAGE_SHIFT; + + vma = _install_special_mapping(mm, addr, len, + VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC, + &vdso_text_mapping); + + if (!IS_ERR(vma)) + mm->context.vdso = addr; +} + +static void vdso_write_begin(struct vdso_data *vdata) +{ + ++vdso_data->seq_count; + smp_wmb(); /* Pairs with smp_rmb in vdso_read_retry */ +} + +static void vdso_write_end(struct vdso_data *vdata) +{ + smp_wmb(); /* Pairs with smp_rmb in vdso_read_begin */ + ++vdso_data->seq_count; +} + +static bool tk_is_cntvct(const struct timekeeper *tk) +{ + if (!IS_ENABLED(CONFIG_ARM_ARCH_TIMER)) + return false; + + if (strcmp(tk->tkr_mono.clock->name, "arch_sys_counter") != 0) + return false; + + return true; +} + +/** + * update_vsyscall - update the vdso data page + * + * Increment the sequence counter, making it odd, indicating to + * userspace that an update is in progress. Update the fields used + * for coarse clocks and, if the architected system timer is in use, + * the fields used for high precision clocks. Increment the sequence + * counter again, making it even, indicating to userspace that the + * update is finished. + * + * Userspace is expected to sample seq_count before reading any other + * fields from the data page. If seq_count is odd, userspace is + * expected to wait until it becomes even. After copying data from + * the page, userspace must sample seq_count again; if it has changed + * from its previous value, userspace must retry the whole sequence. + * + * Calls to update_vsyscall are serialized by the timekeeping core. + */ +void update_vsyscall(struct timekeeper *tk) +{ + struct timespec xtime_coarse; + struct timespec64 *wtm = &tk->wall_to_monotonic; + + if (!cntvct_ok) { + /* The entry points have been zeroed, so there is no + * point in updating the data page. + */ + return; + } + + vdso_write_begin(vdso_data); + + xtime_coarse = __current_kernel_time(); + vdso_data->tk_is_cntvct = tk_is_cntvct(tk); + vdso_data->xtime_coarse_sec = xtime_coarse.tv_sec; + vdso_data->xtime_coarse_nsec = xtime_coarse.tv_nsec; + vdso_data->wtm_clock_sec = wtm->tv_sec; + vdso_data->wtm_clock_nsec = wtm->tv_nsec; + + if (vdso_data->tk_is_cntvct) { + vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; + vdso_data->xtime_clock_sec = tk->xtime_sec; + vdso_data->xtime_clock_snsec = tk->tkr_mono.xtime_nsec; + vdso_data->cs_mult = tk->tkr_mono.mult; + vdso_data->cs_shift = tk->tkr_mono.shift; + vdso_data->cs_mask = tk->tkr_mono.mask; + } + + vdso_write_end(vdso_data); + + flush_dcache_page(virt_to_page(vdso_data)); +} + +void update_vsyscall_tz(void) +{ + vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; + vdso_data->tz_dsttime = sys_tz.tz_dsttime; + flush_dcache_page(virt_to_page(vdso_data)); +} diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index b31aa73e8076..8b60fde5ce48 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -11,7 +11,7 @@ #ifdef CONFIG_ARM_KERNMEM_PERMS #include <asm/pgtable.h> #endif - + #define PROC_INFO \ . = ALIGN(4); \ VMLINUX_SYMBOL(__proc_info_begin) = .; \ @@ -23,7 +23,7 @@ VMLINUX_SYMBOL(__idmap_text_start) = .; \ *(.idmap.text) \ VMLINUX_SYMBOL(__idmap_text_end) = .; \ - . = ALIGN(32); \ + . = ALIGN(PAGE_SIZE); \ VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \ *(.hyp.idmap.text) \ VMLINUX_SYMBOL(__hyp_idmap_text_end) = .; @@ -74,7 +74,7 @@ SECTIONS ARM_EXIT_DISCARD(EXIT_DATA) EXIT_CALL #ifndef CONFIG_MMU - *(.fixup) + *(.text.fixup) *(__ex_table) #endif #ifndef CONFIG_SMP_ON_UP @@ -100,6 +100,7 @@ SECTIONS .text : { /* Real text segment */ _stext = .; /* Text and read-only data */ + IDMAP_TEXT __exception_text_start = .; *(.exception.text) __exception_text_end = .; @@ -108,10 +109,6 @@ SECTIONS SCHED_TEXT LOCK_TEXT KPROBES_TEXT - IDMAP_TEXT -#ifdef CONFIG_MMU - *(.fixup) -#endif *(.gnu.warning) *(.glue_7) *(.glue_7t) @@ -346,8 +343,11 @@ SECTIONS */ ASSERT((__proc_info_end - __proc_info_begin), "missing CPU support") ASSERT((__arch_info_end - __arch_info_begin), "no machine record defined") + /* - * The HYP init code can't be more than a page long. + * The HYP init code can't be more than a page long, + * and should not cross a page boundary. * The above comment applies as well. */ -ASSERT(((__hyp_idmap_text_end - __hyp_idmap_text_start) <= PAGE_SIZE), "HYP init code too big") +ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & PAGE_MASK) <= PAGE_SIZE, + "HYP init code too big or misaligned") diff --git a/arch/arm/kvm/mmu.c b/arch/arm/kvm/mmu.c index 15b050d46fc9..1d5accbd3dcf 100644 --- a/arch/arm/kvm/mmu.c +++ b/arch/arm/kvm/mmu.c @@ -35,9 +35,9 @@ extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[]; static pgd_t *boot_hyp_pgd; static pgd_t *hyp_pgd; +static pgd_t *merged_hyp_pgd; static DEFINE_MUTEX(kvm_hyp_pgd_mutex); -static void *init_bounce_page; static unsigned long hyp_idmap_start; static unsigned long hyp_idmap_end; static phys_addr_t hyp_idmap_vector; @@ -405,9 +405,6 @@ void free_boot_hyp_pgd(void) if (hyp_pgd) unmap_range(NULL, hyp_pgd, TRAMPOLINE_VA, PAGE_SIZE); - free_page((unsigned long)init_bounce_page); - init_bounce_page = NULL; - mutex_unlock(&kvm_hyp_pgd_mutex); } @@ -438,6 +435,11 @@ void free_hyp_pgds(void) free_pages((unsigned long)hyp_pgd, hyp_pgd_order); hyp_pgd = NULL; } + if (merged_hyp_pgd) { + clear_page(merged_hyp_pgd); + free_page((unsigned long)merged_hyp_pgd); + merged_hyp_pgd = NULL; + } mutex_unlock(&kvm_hyp_pgd_mutex); } @@ -1622,12 +1624,18 @@ void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu) phys_addr_t kvm_mmu_get_httbr(void) { - return virt_to_phys(hyp_pgd); + if (__kvm_cpu_uses_extended_idmap()) + return virt_to_phys(merged_hyp_pgd); + else + return virt_to_phys(hyp_pgd); } phys_addr_t kvm_mmu_get_boot_httbr(void) { - return virt_to_phys(boot_hyp_pgd); + if (__kvm_cpu_uses_extended_idmap()) + return virt_to_phys(merged_hyp_pgd); + else + return virt_to_phys(boot_hyp_pgd); } phys_addr_t kvm_get_idmap_vector(void) @@ -1643,39 +1651,11 @@ int kvm_mmu_init(void) hyp_idmap_end = kvm_virt_to_phys(__hyp_idmap_text_end); hyp_idmap_vector = kvm_virt_to_phys(__kvm_hyp_init); - if ((hyp_idmap_start ^ hyp_idmap_end) & PAGE_MASK) { - /* - * Our init code is crossing a page boundary. Allocate - * a bounce page, copy the code over and use that. - */ - size_t len = __hyp_idmap_text_end - __hyp_idmap_text_start; - phys_addr_t phys_base; - - init_bounce_page = (void *)__get_free_page(GFP_KERNEL); - if (!init_bounce_page) { - kvm_err("Couldn't allocate HYP init bounce page\n"); - err = -ENOMEM; - goto out; - } - - memcpy(init_bounce_page, __hyp_idmap_text_start, len); - /* - * Warning: the code we just copied to the bounce page - * must be flushed to the point of coherency. - * Otherwise, the data may be sitting in L2, and HYP - * mode won't be able to observe it as it runs with - * caches off at that point. - */ - kvm_flush_dcache_to_poc(init_bounce_page, len); - - phys_base = kvm_virt_to_phys(init_bounce_page); - hyp_idmap_vector += phys_base - hyp_idmap_start; - hyp_idmap_start = phys_base; - hyp_idmap_end = phys_base + len; - - kvm_info("Using HYP init bounce page @%lx\n", - (unsigned long)phys_base); - } + /* + * We rely on the linker script to ensure at build time that the HYP + * init code does not cross a page boundary. + */ + BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK); hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); boot_hyp_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, hyp_pgd_order); @@ -1698,6 +1678,17 @@ int kvm_mmu_init(void) goto out; } + if (__kvm_cpu_uses_extended_idmap()) { + merged_hyp_pgd = (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + if (!merged_hyp_pgd) { + kvm_err("Failed to allocate extra HYP pgd\n"); + goto out; + } + __kvm_extend_hypmap(boot_hyp_pgd, hyp_pgd, merged_hyp_pgd, + hyp_idmap_start); + return 0; + } + /* Map the very same page at the trampoline VA */ err = __create_hyp_mappings(boot_hyp_pgd, TRAMPOLINE_VA, TRAMPOLINE_VA + PAGE_SIZE, diff --git a/arch/arm/lib/clear_user.S b/arch/arm/lib/clear_user.S index 14a0d988c82c..1710fd7db2d5 100644 --- a/arch/arm/lib/clear_user.S +++ b/arch/arm/lib/clear_user.S @@ -47,7 +47,7 @@ USER( strnebt r2, [r0]) ENDPROC(__clear_user) ENDPROC(__clear_user_std) - .pushsection .fixup,"ax" + .pushsection .text.fixup,"ax" .align 0 9001: ldmfd sp!, {r0, pc} .popsection diff --git a/arch/arm/lib/copy_to_user.S b/arch/arm/lib/copy_to_user.S index a9d3db16ecb5..9648b0675a3e 100644 --- a/arch/arm/lib/copy_to_user.S +++ b/arch/arm/lib/copy_to_user.S @@ -100,7 +100,7 @@ WEAK(__copy_to_user) ENDPROC(__copy_to_user) ENDPROC(__copy_to_user_std) - .pushsection .fixup,"ax" + .pushsection .text.fixup,"ax" .align 0 copy_abort_preamble ldmfd sp!, {r1, r2, r3} diff --git a/arch/arm/lib/csumpartialcopyuser.S b/arch/arm/lib/csumpartialcopyuser.S index 7d08b43d2c0e..1d0957e61f89 100644 --- a/arch/arm/lib/csumpartialcopyuser.S +++ b/arch/arm/lib/csumpartialcopyuser.S @@ -68,7 +68,7 @@ * so properly, we would have to add in whatever registers were loaded before * the fault, which, with the current asm above is not predictable. */ - .pushsection .fixup,"ax" + .pushsection .text.fixup,"ax" .align 4 9001: mov r4, #-EFAULT ldr r5, [sp, #8*4] @ *err_ptr diff --git a/arch/arm/lib/delay.c b/arch/arm/lib/delay.c index 312d43eb686a..8044591dca72 100644 --- a/arch/arm/lib/delay.c +++ b/arch/arm/lib/delay.c @@ -83,6 +83,12 @@ void __init register_current_timer_delay(const struct delay_timer *timer) NSEC_PER_SEC, 3600); res = cyc_to_ns(1ULL, new_mult, new_shift); + if (res > 1000) { + pr_err("Ignoring delay timer %ps, which has insufficient resolution of %lluns\n", + timer, res); + return; + } + if (!delay_calibrated && (!delay_res || (res < delay_res))) { pr_info("Switching to timer-based delay loop, resolution %lluns\n", res); delay_timer = timer; diff --git a/arch/arm/mach-exynos/sleep.S b/arch/arm/mach-exynos/sleep.S index 31d25834b9c4..cf950790fbdc 100644 --- a/arch/arm/mach-exynos/sleep.S +++ b/arch/arm/mach-exynos/sleep.S @@ -23,14 +23,7 @@ #define CPU_MASK 0xff0ffff0 #define CPU_CORTEX_A9 0x410fc090 - /* - * The following code is located into the .data section. This is to - * allow l2x0_regs_phys to be accessed with a relative load while we - * can't rely on any MMU translation. We could have put l2x0_regs_phys - * in the .text section as well, but some setups might insist on it to - * be truly read-only. (Reference from: arch/arm/kernel/sleep.S) - */ - .data + .text .align /* @@ -69,10 +62,12 @@ ENTRY(exynos_cpu_resume_ns) cmp r0, r1 bne skip_cp15 - adr r0, cp15_save_power + adr r0, _cp15_save_power ldr r1, [r0] - adr r0, cp15_save_diag + ldr r1, [r0, r1] + adr r0, _cp15_save_diag ldr r2, [r0] + ldr r2, [r0, r2] mov r0, #SMC_CMD_C15RESUME dsb smc #0 @@ -118,14 +113,20 @@ skip_l2x0: skip_cp15: b cpu_resume ENDPROC(exynos_cpu_resume_ns) + + .align +_cp15_save_power: + .long cp15_save_power - . +_cp15_save_diag: + .long cp15_save_diag - . +#ifdef CONFIG_CACHE_L2X0 +1: .long l2x0_saved_regs - . +#endif /* CONFIG_CACHE_L2X0 */ + + .data .globl cp15_save_diag cp15_save_diag: .long 0 @ cp15 diagnostic .globl cp15_save_power cp15_save_power: .long 0 @ cp15 power control - -#ifdef CONFIG_CACHE_L2X0 - .align -1: .long l2x0_saved_regs - . -#endif /* CONFIG_CACHE_L2X0 */ diff --git a/arch/arm/mach-s5pv210/sleep.S b/arch/arm/mach-s5pv210/sleep.S index 7c43ddd33ba8..dfbfc0f7f8b8 100644 --- a/arch/arm/mach-s5pv210/sleep.S +++ b/arch/arm/mach-s5pv210/sleep.S @@ -14,7 +14,7 @@ #include <linux/linkage.h> - .data + .text .align /* diff --git a/arch/arm/mach-shmobile/board-armadillo800eva.c b/arch/arm/mach-shmobile/board-armadillo800eva.c index 6d949f1c850b..36aaeb12e1a5 100644 --- a/arch/arm/mach-shmobile/board-armadillo800eva.c +++ b/arch/arm/mach-shmobile/board-armadillo800eva.c @@ -1015,7 +1015,6 @@ static struct asoc_simple_card_info fsi_wm8978_info = { .platform = "sh_fsi2", .daifmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBM_CFM, .cpu_dai = { - .fmt = SND_SOC_DAIFMT_IB_NF, .name = "fsia-dai", }, .codec_dai = { @@ -1040,9 +1039,9 @@ static struct asoc_simple_card_info fsi2_hdmi_info = { .card = "FSI2B-HDMI", .codec = "sh-mobile-hdmi", .platform = "sh_fsi2", + .daifmt = SND_SOC_DAIFMT_CBS_CFS, .cpu_dai = { .name = "fsib-dai", - .fmt = SND_SOC_DAIFMT_CBS_CFS, }, .codec_dai = { .name = "sh_mobile_hdmi-hifi", diff --git a/arch/arm/mach-vexpress/Kconfig b/arch/arm/mach-vexpress/Kconfig index 3c2509b4b694..4be537977040 100644 --- a/arch/arm/mach-vexpress/Kconfig +++ b/arch/arm/mach-vexpress/Kconfig @@ -42,6 +42,7 @@ if ARCH_VEXPRESS config ARCH_VEXPRESS_CORTEX_A5_A9_ERRATA bool "Enable A5 and A9 only errata work-arounds" default y + select ARM_ERRATA_643719 if SMP select ARM_ERRATA_720789 select PL310_ERRATA_753970 if CACHE_L2X0 help diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 9b4f29e595a4..b7644310236b 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -738,7 +738,7 @@ config CPU_ICACHE_DISABLE config CPU_DCACHE_DISABLE bool "Disable D-Cache (C-bit)" - depends on CPU_CP15 + depends on CPU_CP15 && !SMP help Say Y here to disable the processor data cache. Unless you have a reason not to or are unsure, say N. @@ -825,6 +825,20 @@ config KUSER_HELPERS Say N here only if you are absolutely certain that you do not need these helpers; otherwise, the safe option is to say Y. +config VDSO + bool "Enable VDSO for acceleration of some system calls" + depends on AEABI && MMU + default y if ARM_ARCH_TIMER + select GENERIC_TIME_VSYSCALL + help + Place in the process address space an ELF shared object + providing fast implementations of gettimeofday and + clock_gettime. Systems that implement the ARM architected + timer will receive maximum benefit. + + You must have glibc 2.22 or later for programs to seamlessly + take advantage of this. + config DMA_CACHE_RWFO bool "Enable read/write for ownership DMA cache maintenance" depends on CPU_V6K && SMP diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index 2c0c541c60ca..9769f1eefe3b 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -201,7 +201,7 @@ union offset_union { THUMB( "1: "ins" %1, [%2]\n" ) \ THUMB( " add %2, %2, #1\n" ) \ "2:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "3: mov %0, #1\n" \ " b 2b\n" \ @@ -261,7 +261,7 @@ union offset_union { " mov %1, %1, "NEXT_BYTE"\n" \ "2: "ins" %1, [%2]\n" \ "3:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "4: mov %0, #1\n" \ " b 3b\n" \ @@ -301,7 +301,7 @@ union offset_union { " mov %1, %1, "NEXT_BYTE"\n" \ "4: "ins" %1, [%2]\n" \ "5:\n" \ - " .pushsection .fixup,\"ax\"\n" \ + " .pushsection .text.fixup,\"ax\"\n" \ " .align 2\n" \ "6: mov %0, #1\n" \ " b 5b\n" \ diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 8f15f70622a6..e309c8f35af5 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -1647,6 +1647,7 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask) struct device_node *np; struct resource res; u32 cache_id, old_aux; + u32 cache_level = 2; np = of_find_matching_node(NULL, l2x0_ids); if (!np) @@ -1679,6 +1680,12 @@ int __init l2x0_of_init(u32 aux_val, u32 aux_mask) if (!of_property_read_bool(np, "cache-unified")) pr_err("L2C: device tree omits to specify unified cache\n"); + if (of_property_read_u32(np, "cache-level", &cache_level)) + pr_err("L2C: device tree omits to specify cache-level\n"); + + if (cache_level != 2) + pr_err("L2C: device tree specifies invalid cache level\n"); + /* Read back current (default) hardware configuration */ if (data->save) data->save(l2x0_base); diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index b966656d2c2d..a134d8a13d00 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -36,10 +36,10 @@ ENTRY(v7_invalidate_l1) mcr p15, 2, r0, c0, c0, 0 mrc p15, 1, r0, c0, c0, 0 - ldr r1, =0x7fff + movw r1, #0x7fff and r2, r1, r0, lsr #13 - ldr r1, =0x3ff + movw r1, #0x3ff and r3, r1, r0, lsr #3 @ NumWays - 1 add r2, r2, #1 @ NumSets @@ -90,21 +90,20 @@ ENDPROC(v7_flush_icache_all) ENTRY(v7_flush_dcache_louis) dmb @ ensure ordering with previous memory accesses mrc p15, 1, r0, c0, c0, 1 @ read clidr, r0 = clidr - ALT_SMP(ands r3, r0, #(7 << 21)) @ extract LoUIS from clidr - ALT_UP(ands r3, r0, #(7 << 27)) @ extract LoUU from clidr +ALT_SMP(mov r3, r0, lsr #20) @ move LoUIS into position +ALT_UP( mov r3, r0, lsr #26) @ move LoUU into position + ands r3, r3, #7 << 1 @ extract LoU*2 field from clidr + bne start_flush_levels @ LoU != 0, start flushing #ifdef CONFIG_ARM_ERRATA_643719 - ALT_SMP(mrceq p15, 0, r2, c0, c0, 0) @ read main ID register - ALT_UP(reteq lr) @ LoUU is zero, so nothing to do - ldreq r1, =0x410fc090 @ ID of ARM Cortex A9 r0p? - biceq r2, r2, #0x0000000f @ clear minor revision number - teqeq r2, r1 @ test for errata affected core and if so... - orreqs r3, #(1 << 21) @ fix LoUIS value (and set flags state to 'ne') +ALT_SMP(mrc p15, 0, r2, c0, c0, 0) @ read main ID register +ALT_UP( ret lr) @ LoUU is zero, so nothing to do + movw r1, #:lower16:(0x410fc090 >> 4) @ ID of ARM Cortex A9 r0p? + movt r1, #:upper16:(0x410fc090 >> 4) + teq r1, r2, lsr #4 @ test for errata affected core and if so... + moveq r3, #1 << 1 @ fix LoUIS value + beq start_flush_levels @ start flushing cache levels #endif - ALT_SMP(mov r3, r3, lsr #20) @ r3 = LoUIS * 2 - ALT_UP(mov r3, r3, lsr #26) @ r3 = LoUU * 2 - reteq lr @ return if level == 0 - mov r10, #0 @ r10 (starting level) = 0 - b flush_levels @ start flushing cache levels + ret lr ENDPROC(v7_flush_dcache_louis) /* @@ -119,9 +118,10 @@ ENDPROC(v7_flush_dcache_louis) ENTRY(v7_flush_dcache_all) dmb @ ensure ordering with previous memory accesses mrc p15, 1, r0, c0, c0, 1 @ read clidr - ands r3, r0, #0x7000000 @ extract loc from clidr - mov r3, r3, lsr #23 @ left align loc bit field + mov r3, r0, lsr #23 @ move LoC into position + ands r3, r3, #7 << 1 @ extract LoC*2 from clidr beq finished @ if loc is 0, then no need to clean +start_flush_levels: mov r10, #0 @ start clean at cache level 0 flush_levels: add r2, r10, r10, lsr #1 @ work out 3x current cache level @@ -140,10 +140,10 @@ flush_levels: #endif and r2, r1, #7 @ extract the length of the cache lines add r2, r2, #4 @ add 4 (line length offset) - ldr r4, =0x3ff + movw r4, #0x3ff ands r4, r4, r1, lsr #3 @ find maximum number on the way size clz r5, r4 @ find bit position of way size increment - ldr r7, =0x7fff + movw r7, #0x7fff ands r7, r7, r1, lsr #13 @ extract max number of the index size loop1: mov r9, r7 @ create working copy of max index diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index e315dfe3af1b..09c5fe3d30c2 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -289,11 +289,11 @@ static void __dma_free_buffer(struct page *page, size_t size) static void *__alloc_from_contiguous(struct device *dev, size_t size, pgprot_t prot, struct page **ret_page, - const void *caller); + const void *caller, bool want_vaddr); static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, pgprot_t prot, struct page **ret_page, - const void *caller); + const void *caller, bool want_vaddr); static void * __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, @@ -357,10 +357,10 @@ static int __init atomic_pool_init(void) if (dev_get_cma_area(NULL)) ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, - &page, atomic_pool_init); + &page, atomic_pool_init, true); else ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, - &page, atomic_pool_init); + &page, atomic_pool_init, true); if (ptr) { int ret; @@ -467,13 +467,15 @@ static void __dma_remap(struct page *page, size_t size, pgprot_t prot) static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, pgprot_t prot, struct page **ret_page, - const void *caller) + const void *caller, bool want_vaddr) { struct page *page; - void *ptr; + void *ptr = NULL; page = __dma_alloc_buffer(dev, size, gfp); if (!page) return NULL; + if (!want_vaddr) + goto out; ptr = __dma_alloc_remap(page, size, gfp, prot, caller); if (!ptr) { @@ -481,6 +483,7 @@ static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, return NULL; } + out: *ret_page = page; return ptr; } @@ -523,12 +526,12 @@ static int __free_from_pool(void *start, size_t size) static void *__alloc_from_contiguous(struct device *dev, size_t size, pgprot_t prot, struct page **ret_page, - const void *caller) + const void *caller, bool want_vaddr) { unsigned long order = get_order(size); size_t count = size >> PAGE_SHIFT; struct page *page; - void *ptr; + void *ptr = NULL; page = dma_alloc_from_contiguous(dev, count, order); if (!page) @@ -536,6 +539,9 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size, __dma_clear_buffer(page, size); + if (!want_vaddr) + goto out; + if (PageHighMem(page)) { ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); if (!ptr) { @@ -546,17 +552,21 @@ static void *__alloc_from_contiguous(struct device *dev, size_t size, __dma_remap(page, size, prot); ptr = page_address(page); } + + out: *ret_page = page; return ptr; } static void __free_from_contiguous(struct device *dev, struct page *page, - void *cpu_addr, size_t size) + void *cpu_addr, size_t size, bool want_vaddr) { - if (PageHighMem(page)) - __dma_free_remap(cpu_addr, size); - else - __dma_remap(page, size, PAGE_KERNEL); + if (want_vaddr) { + if (PageHighMem(page)) + __dma_free_remap(cpu_addr, size); + else + __dma_remap(page, size, PAGE_KERNEL); + } dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); } @@ -574,12 +584,12 @@ static inline pgprot_t __get_dma_pgprot(struct dma_attrs *attrs, pgprot_t prot) #define nommu() 1 -#define __get_dma_pgprot(attrs, prot) __pgprot(0) -#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c) NULL +#define __get_dma_pgprot(attrs, prot) __pgprot(0) +#define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL #define __alloc_from_pool(size, ret_page) NULL -#define __alloc_from_contiguous(dev, size, prot, ret, c) NULL +#define __alloc_from_contiguous(dev, size, prot, ret, c, wv) NULL #define __free_from_pool(cpu_addr, size) 0 -#define __free_from_contiguous(dev, page, cpu_addr, size) do { } while (0) +#define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0) #define __dma_free_remap(cpu_addr, size) do { } while (0) #endif /* CONFIG_MMU */ @@ -599,11 +609,13 @@ static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, pgprot_t prot, bool is_coherent, const void *caller) + gfp_t gfp, pgprot_t prot, bool is_coherent, + struct dma_attrs *attrs, const void *caller) { u64 mask = get_coherent_dma_mask(dev); struct page *page = NULL; void *addr; + bool want_vaddr; #ifdef CONFIG_DMA_API_DEBUG u64 limit = (mask + 1) & ~mask; @@ -631,20 +643,21 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, *handle = DMA_ERROR_CODE; size = PAGE_ALIGN(size); + want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); if (is_coherent || nommu()) addr = __alloc_simple_buffer(dev, size, gfp, &page); else if (!(gfp & __GFP_WAIT)) addr = __alloc_from_pool(size, &page); else if (!dev_get_cma_area(dev)) - addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller); + addr = __alloc_remap_buffer(dev, size, gfp, prot, &page, caller, want_vaddr); else - addr = __alloc_from_contiguous(dev, size, prot, &page, caller); + addr = __alloc_from_contiguous(dev, size, prot, &page, caller, want_vaddr); - if (addr) + if (page) *handle = pfn_to_dma(dev, page_to_pfn(page)); - return addr; + return want_vaddr ? addr : page; } /* @@ -661,7 +674,7 @@ void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, return memory; return __dma_alloc(dev, size, handle, gfp, prot, false, - __builtin_return_address(0)); + attrs, __builtin_return_address(0)); } static void *arm_coherent_dma_alloc(struct device *dev, size_t size, @@ -674,7 +687,7 @@ static void *arm_coherent_dma_alloc(struct device *dev, size_t size, return memory; return __dma_alloc(dev, size, handle, gfp, prot, true, - __builtin_return_address(0)); + attrs, __builtin_return_address(0)); } /* @@ -715,6 +728,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, bool is_coherent) { struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); + bool want_vaddr = !dma_get_attr(DMA_ATTR_NO_KERNEL_MAPPING, attrs); if (dma_release_from_coherent(dev, get_order(size), cpu_addr)) return; @@ -726,14 +740,15 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, } else if (__free_from_pool(cpu_addr, size)) { return; } else if (!dev_get_cma_area(dev)) { - __dma_free_remap(cpu_addr, size); + if (want_vaddr) + __dma_free_remap(cpu_addr, size); __dma_free_buffer(page, size); } else { /* * Non-atomic allocations cannot be freed with IRQs disabled */ WARN_ON(irqs_disabled()); - __free_from_contiguous(dev, page, cpu_addr, size); + __free_from_contiguous(dev, page, cpu_addr, size, want_vaddr); } } @@ -1135,13 +1150,28 @@ static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, gfp |= __GFP_NOWARN | __GFP_HIGHMEM; while (count) { - int j, order = __fls(count); + int j, order; + + for (order = __fls(count); order > 0; --order) { + /* + * We do not want OOM killer to be invoked as long + * as we can fall back to single pages, so we force + * __GFP_NORETRY for orders higher than zero. + */ + pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); + if (pages[i]) + break; + } - pages[i] = alloc_pages(gfp, order); - while (!pages[i] && order) - pages[i] = alloc_pages(gfp, --order); - if (!pages[i]) - goto error; + if (!pages[i]) { + /* + * Fall back to single page allocation. + * Might invoke OOM killer as last resort. + */ + pages[i] = alloc_pages(gfp, 0); + if (!pages[i]) + goto error; + } if (order) { split_page(pages[i], order); @@ -1206,7 +1236,7 @@ __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, static dma_addr_t __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; dma_addr_t dma_addr, iova; int i, ret = DMA_ERROR_CODE; @@ -1242,7 +1272,7 @@ fail: static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); /* * add optional in-page offset from iova to size and align @@ -1457,7 +1487,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, enum dma_data_direction dir, struct dma_attrs *attrs, bool is_coherent) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova, iova_base; int ret = 0; unsigned int count; @@ -1678,7 +1708,7 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *p unsigned long offset, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t dma_addr; int ret, prot, len = PAGE_ALIGN(size + offset); @@ -1731,7 +1761,7 @@ static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; int offset = handle & ~PAGE_MASK; int len = PAGE_ALIGN(size + offset); @@ -1756,7 +1786,7 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); int offset = handle & ~PAGE_MASK; @@ -1775,7 +1805,7 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, static void arm_iommu_sync_single_for_cpu(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); unsigned int offset = handle & ~PAGE_MASK; @@ -1789,7 +1819,7 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev, static void arm_iommu_sync_single_for_device(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); unsigned int offset = handle & ~PAGE_MASK; @@ -1950,7 +1980,7 @@ static int __arm_iommu_attach_device(struct device *dev, return err; kref_get(&mapping->kref); - dev->archdata.mapping = mapping; + to_dma_iommu_mapping(dev) = mapping; pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); return 0; @@ -1995,7 +2025,7 @@ static void __arm_iommu_detach_device(struct device *dev) iommu_detach_device(mapping->domain, dev); kref_put(&mapping->kref, release_iommu_mapping); - dev->archdata.mapping = NULL; + to_dma_iommu_mapping(dev) = NULL; pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); } @@ -2053,7 +2083,7 @@ static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, static void arm_teardown_iommu_dma_ops(struct device *dev) { - struct dma_iommu_mapping *mapping = dev->archdata.mapping; + struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); if (!mapping) return; diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 3d0e9aed4b40..be92fa0f2f35 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -86,55 +86,6 @@ static int __init parse_tag_initrd2(const struct tag *tag) __tagtable(ATAG_INITRD2, parse_tag_initrd2); -/* - * This keeps memory configuration data used by a couple memory - * initialization functions, as well as show_mem() for the skipping - * of holes in the memory map. It is populated by arm_add_memory(). - */ -void show_mem(unsigned int filter) -{ - int free = 0, total = 0, reserved = 0; - int shared = 0, cached = 0, slab = 0; - struct memblock_region *reg; - - printk("Mem-info:\n"); - show_free_areas(filter); - - for_each_memblock (memory, reg) { - unsigned int pfn1, pfn2; - struct page *page, *end; - - pfn1 = memblock_region_memory_base_pfn(reg); - pfn2 = memblock_region_memory_end_pfn(reg); - - page = pfn_to_page(pfn1); - end = pfn_to_page(pfn2 - 1) + 1; - - do { - total++; - if (PageReserved(page)) - reserved++; - else if (PageSwapCache(page)) - cached++; - else if (PageSlab(page)) - slab++; - else if (!page_count(page)) - free++; - else - shared += page_count(page) - 1; - pfn1++; - page = pfn_to_page(pfn1); - } while (pfn1 < pfn2); - } - - printk("%d pages of RAM\n", total); - printk("%d free pages\n", free); - printk("%d reserved pages\n", reserved); - printk("%d slab pages\n", slab); - printk("%d pages shared\n", shared); - printk("%d pages swap cached\n", cached); -} - static void __init find_limits(unsigned long *min, unsigned long *max_low, unsigned long *max_high) { diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index 86ee5d47ce3c..aa0519eed698 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -507,7 +507,7 @@ cpu_arm1020_name: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm1020_proc_info,#object __arm1020_proc_info: @@ -519,7 +519,7 @@ __arm1020_proc_info: .long PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm1020_setup + initfn __arm1020_setup, __arm1020_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S index a6331d78601f..bff4c7f70fd6 100644 --- a/arch/arm/mm/proc-arm1020e.S +++ b/arch/arm/mm/proc-arm1020e.S @@ -465,7 +465,7 @@ arm1020e_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm1020e_proc_info,#object __arm1020e_proc_info: @@ -479,7 +479,7 @@ __arm1020e_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm1020e_setup + initfn __arm1020e_setup, __arm1020e_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_EDSP diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S index a126b7a59928..dbb2413fe04d 100644 --- a/arch/arm/mm/proc-arm1022.S +++ b/arch/arm/mm/proc-arm1022.S @@ -448,7 +448,7 @@ arm1022_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm1022_proc_info,#object __arm1022_proc_info: @@ -462,7 +462,7 @@ __arm1022_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm1022_setup + initfn __arm1022_setup, __arm1022_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_EDSP diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S index fc294067e977..0b37b2cef9d3 100644 --- a/arch/arm/mm/proc-arm1026.S +++ b/arch/arm/mm/proc-arm1026.S @@ -442,7 +442,7 @@ arm1026_crval: string cpu_arm1026_name, "ARM1026EJ-S" .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm1026_proc_info,#object __arm1026_proc_info: @@ -456,7 +456,7 @@ __arm1026_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm1026_setup + initfn __arm1026_setup, __arm1026_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_JAVA diff --git a/arch/arm/mm/proc-arm720.S b/arch/arm/mm/proc-arm720.S index 2baa66b3ac9b..3651cd70e418 100644 --- a/arch/arm/mm/proc-arm720.S +++ b/arch/arm/mm/proc-arm720.S @@ -186,7 +186,7 @@ arm720_crval: * See <asm/procinfo.h> for a definition of this structure. */ - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro arm720_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cpu_flush:req .type __\name\()_proc_info,#object @@ -203,7 +203,7 @@ __\name\()_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b \cpu_flush @ cpu_flush + initfn \cpu_flush, __\name\()_proc_info @ cpu_flush .long cpu_arch_name @ arch_name .long cpu_elf_name @ elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB @ elf_hwcap diff --git a/arch/arm/mm/proc-arm740.S b/arch/arm/mm/proc-arm740.S index ac1ea6b3bce4..024fb7732407 100644 --- a/arch/arm/mm/proc-arm740.S +++ b/arch/arm/mm/proc-arm740.S @@ -132,14 +132,14 @@ __arm740_setup: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm740_proc_info,#object __arm740_proc_info: .long 0x41807400 .long 0xfffffff0 .long 0 .long 0 - b __arm740_setup + initfn __arm740_setup, __arm740_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_26BIT diff --git a/arch/arm/mm/proc-arm7tdmi.S b/arch/arm/mm/proc-arm7tdmi.S index bf6ba4bc30ff..25472d94426d 100644 --- a/arch/arm/mm/proc-arm7tdmi.S +++ b/arch/arm/mm/proc-arm7tdmi.S @@ -76,7 +76,7 @@ __arm7tdmi_setup: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro arm7tdmi_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, \ extra_hwcaps=0 @@ -86,7 +86,7 @@ __\name\()_proc_info: .long \cpu_mask .long 0 .long 0 - b __arm7tdmi_setup + initfn __arm7tdmi_setup, __\name\()_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_26BIT | ( \extra_hwcaps ) diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 22bf8dde4f84..7a14bd4414c9 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -448,7 +448,7 @@ arm920_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm920_proc_info,#object __arm920_proc_info: @@ -464,7 +464,7 @@ __arm920_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm920_setup + initfn __arm920_setup, __arm920_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index 0c6d5ac5a6d4..edccfcdcd551 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -426,7 +426,7 @@ arm922_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm922_proc_info,#object __arm922_proc_info: @@ -442,7 +442,7 @@ __arm922_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm922_setup + initfn __arm922_setup, __arm922_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S index c32d073282ea..ede8c54ab4aa 100644 --- a/arch/arm/mm/proc-arm925.S +++ b/arch/arm/mm/proc-arm925.S @@ -494,7 +494,7 @@ arm925_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro arm925_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache .type __\name\()_proc_info,#object @@ -510,7 +510,7 @@ __\name\()_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm925_setup + initfn __arm925_setup, __\name\()_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 252b2503038d..fb827c633693 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -474,7 +474,7 @@ arm926_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm926_proc_info,#object __arm926_proc_info: @@ -490,7 +490,7 @@ __arm926_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __arm926_setup + initfn __arm926_setup, __arm926_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP|HWCAP_JAVA diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S index e5212d489377..ee5b66f847c4 100644 --- a/arch/arm/mm/proc-arm940.S +++ b/arch/arm/mm/proc-arm940.S @@ -297,26 +297,16 @@ __arm940_setup: mcr p15, 0, r0, c6, c0, 1 ldr r0, =(CONFIG_DRAM_BASE & 0xFFFFF000) @ base[31:12] of RAM - ldr r1, =(CONFIG_DRAM_SIZE >> 12) @ size of RAM (must be >= 4KB) - mov r2, #10 @ 11 is the minimum (4KB) -1: add r2, r2, #1 @ area size *= 2 - mov r1, r1, lsr #1 - bne 1b @ count not zero r-shift - orr r0, r0, r2, lsl #1 @ the area register value - orr r0, r0, #1 @ set enable bit - mcr p15, 0, r0, c6, c1, 0 @ set area 1, RAM - mcr p15, 0, r0, c6, c1, 1 + ldr r7, =CONFIG_DRAM_SIZE >> 12 @ size of RAM (must be >= 4KB) + pr_val r3, r0, r7, #1 + mcr p15, 0, r3, c6, c1, 0 @ set area 1, RAM + mcr p15, 0, r3, c6, c1, 1 ldr r0, =(CONFIG_FLASH_MEM_BASE & 0xFFFFF000) @ base[31:12] of FLASH - ldr r1, =(CONFIG_FLASH_SIZE >> 12) @ size of FLASH (must be >= 4KB) - mov r2, #10 @ 11 is the minimum (4KB) -1: add r2, r2, #1 @ area size *= 2 - mov r1, r1, lsr #1 - bne 1b @ count not zero r-shift - orr r0, r0, r2, lsl #1 @ the area register value - orr r0, r0, #1 @ set enable bit - mcr p15, 0, r0, c6, c2, 0 @ set area 2, ROM/FLASH - mcr p15, 0, r0, c6, c2, 1 + ldr r7, =CONFIG_FLASH_SIZE @ size of FLASH (must be >= 4KB) + pr_val r3, r0, r6, #1 + mcr p15, 0, r3, c6, c2, 0 @ set area 2, ROM/FLASH + mcr p15, 0, r3, c6, c2, 1 mov r0, #0x06 mcr p15, 0, r0, c2, c0, 0 @ Region 1&2 cacheable @@ -354,14 +344,14 @@ __arm940_setup: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm940_proc_info,#object __arm940_proc_info: .long 0x41009400 .long 0xff00fff0 .long 0 - b __arm940_setup + initfn __arm940_setup, __arm940_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S index b3dd9b2d0b8e..7361837edc31 100644 --- a/arch/arm/mm/proc-arm946.S +++ b/arch/arm/mm/proc-arm946.S @@ -343,24 +343,14 @@ __arm946_setup: mcr p15, 0, r0, c6, c0, 0 @ set region 0, default ldr r0, =(CONFIG_DRAM_BASE & 0xFFFFF000) @ base[31:12] of RAM - ldr r1, =(CONFIG_DRAM_SIZE >> 12) @ size of RAM (must be >= 4KB) - mov r2, #10 @ 11 is the minimum (4KB) -1: add r2, r2, #1 @ area size *= 2 - mov r1, r1, lsr #1 - bne 1b @ count not zero r-shift - orr r0, r0, r2, lsl #1 @ the region register value - orr r0, r0, #1 @ set enable bit - mcr p15, 0, r0, c6, c1, 0 @ set region 1, RAM + ldr r7, =CONFIG_DRAM_SIZE @ size of RAM (must be >= 4KB) + pr_val r3, r0, r7, #1 + mcr p15, 0, r3, c6, c1, 0 ldr r0, =(CONFIG_FLASH_MEM_BASE & 0xFFFFF000) @ base[31:12] of FLASH - ldr r1, =(CONFIG_FLASH_SIZE >> 12) @ size of FLASH (must be >= 4KB) - mov r2, #10 @ 11 is the minimum (4KB) -1: add r2, r2, #1 @ area size *= 2 - mov r1, r1, lsr #1 - bne 1b @ count not zero r-shift - orr r0, r0, r2, lsl #1 @ the region register value - orr r0, r0, #1 @ set enable bit - mcr p15, 0, r0, c6, c2, 0 @ set region 2, ROM/FLASH + ldr r7, =CONFIG_FLASH_SIZE @ size of FLASH (must be >= 4KB) + pr_val r3, r0, r7, #1 + mcr p15, 0, r3, c6, c2, 0 mov r0, #0x06 mcr p15, 0, r0, c2, c0, 0 @ region 1,2 d-cacheable @@ -409,14 +399,14 @@ __arm946_setup: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __arm946_proc_info,#object __arm946_proc_info: .long 0x41009460 .long 0xff00fff0 .long 0 .long 0 - b __arm946_setup + initfn __arm946_setup, __arm946_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB diff --git a/arch/arm/mm/proc-arm9tdmi.S b/arch/arm/mm/proc-arm9tdmi.S index 8227322bbb8f..7fac8c612134 100644 --- a/arch/arm/mm/proc-arm9tdmi.S +++ b/arch/arm/mm/proc-arm9tdmi.S @@ -70,7 +70,7 @@ __arm9tdmi_setup: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro arm9tdmi_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req .type __\name\()_proc_info, #object @@ -79,7 +79,7 @@ __\name\()_proc_info: .long \cpu_mask .long 0 .long 0 - b __arm9tdmi_setup + initfn __arm9tdmi_setup, __\name\()_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_THUMB | HWCAP_26BIT diff --git a/arch/arm/mm/proc-fa526.S b/arch/arm/mm/proc-fa526.S index c494886892ba..4001b73af4ee 100644 --- a/arch/arm/mm/proc-fa526.S +++ b/arch/arm/mm/proc-fa526.S @@ -190,7 +190,7 @@ fa526_cr1_set: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __fa526_proc_info,#object __fa526_proc_info: @@ -206,7 +206,7 @@ __fa526_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __fa526_setup + initfn __fa526_setup, __fa526_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index 03a1b75f2e16..e494d6d6acbe 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -584,7 +584,7 @@ feroceon_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro feroceon_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache:req .type __\name\()_proc_info,#object @@ -601,7 +601,8 @@ __\name\()_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __feroceon_setup + initfn __feroceon_setup, __\name\()_proc_info + .long __feroceon_setup .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index 082b9f2f7e90..c671f345266a 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -331,3 +331,31 @@ ENTRY(\name\()_tlb_fns) .globl \x .equ \x, \y .endm + +.macro initfn, func, base + .long \func - \base +.endm + + /* + * Macro to calculate the log2 size for the protection region + * registers. This calculates rd = log2(size) - 1. tmp must + * not be the same register as rd. + */ +.macro pr_sz, rd, size, tmp + mov \tmp, \size, lsr #12 + mov \rd, #11 +1: movs \tmp, \tmp, lsr #1 + addne \rd, \rd, #1 + bne 1b +.endm + + /* + * Macro to generate a protection region register value + * given a pre-masked address, size, and enable bit. + * Corrupts size. + */ +.macro pr_val, dest, addr, size, enable + pr_sz \dest, \size, \size @ calculate log2(size) - 1 + orr \dest, \addr, \dest, lsl #1 @ mask in the region size + orr \dest, \dest, \enable +.endm diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S index 53d393455f13..d65edf717bf7 100644 --- a/arch/arm/mm/proc-mohawk.S +++ b/arch/arm/mm/proc-mohawk.S @@ -427,7 +427,7 @@ mohawk_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __88sv331x_proc_info,#object __88sv331x_proc_info: @@ -443,7 +443,7 @@ __88sv331x_proc_info: PMD_BIT4 | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __mohawk_setup + initfn __mohawk_setup, __88sv331x_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP diff --git a/arch/arm/mm/proc-sa110.S b/arch/arm/mm/proc-sa110.S index 8008a0461cf5..ee2ce496239f 100644 --- a/arch/arm/mm/proc-sa110.S +++ b/arch/arm/mm/proc-sa110.S @@ -199,7 +199,7 @@ sa110_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .type __sa110_proc_info,#object __sa110_proc_info: @@ -213,7 +213,7 @@ __sa110_proc_info: .long PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __sa110_setup + initfn __sa110_setup, __sa110_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_26BIT | HWCAP_FAST_MULT diff --git a/arch/arm/mm/proc-sa1100.S b/arch/arm/mm/proc-sa1100.S index 89f97ac648a9..222d5836f666 100644 --- a/arch/arm/mm/proc-sa1100.S +++ b/arch/arm/mm/proc-sa1100.S @@ -242,7 +242,7 @@ sa1100_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro sa1100_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req .type __\name\()_proc_info,#object @@ -257,7 +257,7 @@ __\name\()_proc_info: .long PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __sa1100_setup + initfn __sa1100_setup, __\name\()_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_26BIT | HWCAP_FAST_MULT diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S index d0390f4b3f18..06d890a2342b 100644 --- a/arch/arm/mm/proc-v6.S +++ b/arch/arm/mm/proc-v6.S @@ -264,7 +264,7 @@ v6_crval: string cpu_elf_name, "v6" .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc /* * Match any ARMv6 processor core. @@ -287,7 +287,7 @@ __v6_proc_info: PMD_SECT_XN | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __v6_setup + initfn __v6_setup, __v6_proc_info .long cpu_arch_name .long cpu_elf_name /* See also feat_v6_fixup() for HWCAP_TLS */ diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S index ed448d8a596b..10405b8d31af 100644 --- a/arch/arm/mm/proc-v7-2level.S +++ b/arch/arm/mm/proc-v7-2level.S @@ -37,15 +37,18 @@ * It is assumed that: * - we are not using split page tables */ -ENTRY(cpu_v7_switch_mm) +ENTRY(cpu_ca8_switch_mm) #ifdef CONFIG_MMU mov r2, #0 - mmid r1, r1 @ get mm->context.id - ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP) - ALT_UP(orr r0, r0, #TTB_FLAGS_UP) #ifdef CONFIG_ARM_ERRATA_430973 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB #endif +#endif +ENTRY(cpu_v7_switch_mm) +#ifdef CONFIG_MMU + mmid r1, r1 @ get mm->context.id + ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP) + ALT_UP(orr r0, r0, #TTB_FLAGS_UP) #ifdef CONFIG_PID_IN_CONTEXTIDR mrc p15, 0, r2, c13, c0, 1 @ read current context ID lsr r2, r2, #8 @ extract the PID @@ -61,6 +64,7 @@ ENTRY(cpu_v7_switch_mm) #endif bx lr ENDPROC(cpu_v7_switch_mm) +ENDPROC(cpu_ca8_switch_mm) /* * cpu_v7_set_pte_ext(ptep, pte) diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 8b4ee5e81c14..3d1054f11a8a 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -153,6 +153,21 @@ ENDPROC(cpu_v7_do_resume) #endif /* + * Cortex-A8 + */ + globl_equ cpu_ca8_proc_init, cpu_v7_proc_init + globl_equ cpu_ca8_proc_fin, cpu_v7_proc_fin + globl_equ cpu_ca8_reset, cpu_v7_reset + globl_equ cpu_ca8_do_idle, cpu_v7_do_idle + globl_equ cpu_ca8_dcache_clean_area, cpu_v7_dcache_clean_area + globl_equ cpu_ca8_set_pte_ext, cpu_v7_set_pte_ext + globl_equ cpu_ca8_suspend_size, cpu_v7_suspend_size +#ifdef CONFIG_ARM_CPU_SUSPEND + globl_equ cpu_ca8_do_suspend, cpu_v7_do_suspend + globl_equ cpu_ca8_do_resume, cpu_v7_do_resume +#endif + +/* * Cortex-A9 processor functions */ globl_equ cpu_ca9mp_proc_init, cpu_v7_proc_init @@ -451,7 +466,10 @@ __v7_setup_stack: @ define struct processor (see <asm/proc-fns.h> and proc-macros.S) define_processor_functions v7, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 +#ifndef CONFIG_ARM_LPAE + define_processor_functions ca8, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 define_processor_functions ca9mp, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 +#endif #ifdef CONFIG_CPU_PJ4B define_processor_functions pj4b, dabort=v7_early_abort, pabort=v7_pabort, suspend=1 #endif @@ -462,19 +480,19 @@ __v7_setup_stack: string cpu_elf_name, "v7" .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc /* * Standard v7 proc info content */ -.macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0, proc_fns = v7_processor_functions +.macro __v7_proc name, initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0, proc_fns = v7_processor_functions ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags) ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags) .long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags - W(b) \initfunc + initfn \initfunc, \name .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP | HWCAP_HALF | HWCAP_THUMB | HWCAP_FAST_MULT | \ @@ -494,7 +512,7 @@ __v7_setup_stack: __v7_ca5mp_proc_info: .long 0x410fc050 .long 0xff0ffff0 - __v7_proc __v7_ca5mp_setup + __v7_proc __v7_ca5mp_proc_info, __v7_ca5mp_setup .size __v7_ca5mp_proc_info, . - __v7_ca5mp_proc_info /* @@ -504,9 +522,19 @@ __v7_ca5mp_proc_info: __v7_ca9mp_proc_info: .long 0x410fc090 .long 0xff0ffff0 - __v7_proc __v7_ca9mp_setup, proc_fns = ca9mp_processor_functions + __v7_proc __v7_ca9mp_proc_info, __v7_ca9mp_setup, proc_fns = ca9mp_processor_functions .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info + /* + * ARM Ltd. Cortex A8 processor. + */ + .type __v7_ca8_proc_info, #object +__v7_ca8_proc_info: + .long 0x410fc080 + .long 0xff0ffff0 + __v7_proc __v7_ca8_proc_info, __v7_setup, proc_fns = ca8_processor_functions + .size __v7_ca8_proc_info, . - __v7_ca8_proc_info + #endif /* CONFIG_ARM_LPAE */ /* @@ -517,7 +545,7 @@ __v7_ca9mp_proc_info: __v7_pj4b_proc_info: .long 0x560f5800 .long 0xff0fff00 - __v7_proc __v7_pj4b_setup, proc_fns = pj4b_processor_functions + __v7_proc __v7_pj4b_proc_info, __v7_pj4b_setup, proc_fns = pj4b_processor_functions .size __v7_pj4b_proc_info, . - __v7_pj4b_proc_info #endif @@ -528,7 +556,7 @@ __v7_pj4b_proc_info: __v7_cr7mp_proc_info: .long 0x410fc170 .long 0xff0ffff0 - __v7_proc __v7_cr7mp_setup + __v7_proc __v7_cr7mp_proc_info, __v7_cr7mp_setup .size __v7_cr7mp_proc_info, . - __v7_cr7mp_proc_info /* @@ -538,7 +566,7 @@ __v7_cr7mp_proc_info: __v7_ca7mp_proc_info: .long 0x410fc070 .long 0xff0ffff0 - __v7_proc __v7_ca7mp_setup + __v7_proc __v7_ca7mp_proc_info, __v7_ca7mp_setup .size __v7_ca7mp_proc_info, . - __v7_ca7mp_proc_info /* @@ -548,7 +576,7 @@ __v7_ca7mp_proc_info: __v7_ca12mp_proc_info: .long 0x410fc0d0 .long 0xff0ffff0 - __v7_proc __v7_ca12mp_setup + __v7_proc __v7_ca12mp_proc_info, __v7_ca12mp_setup .size __v7_ca12mp_proc_info, . - __v7_ca12mp_proc_info /* @@ -558,7 +586,7 @@ __v7_ca12mp_proc_info: __v7_ca15mp_proc_info: .long 0x410fc0f0 .long 0xff0ffff0 - __v7_proc __v7_ca15mp_setup + __v7_proc __v7_ca15mp_proc_info, __v7_ca15mp_setup .size __v7_ca15mp_proc_info, . - __v7_ca15mp_proc_info /* @@ -568,7 +596,7 @@ __v7_ca15mp_proc_info: __v7_b15mp_proc_info: .long 0x420f00f0 .long 0xff0ffff0 - __v7_proc __v7_b15mp_setup + __v7_proc __v7_b15mp_proc_info, __v7_b15mp_setup .size __v7_b15mp_proc_info, . - __v7_b15mp_proc_info /* @@ -578,7 +606,7 @@ __v7_b15mp_proc_info: __v7_ca17mp_proc_info: .long 0x410fc0e0 .long 0xff0ffff0 - __v7_proc __v7_ca17mp_setup + __v7_proc __v7_ca17mp_proc_info, __v7_ca17mp_setup .size __v7_ca17mp_proc_info, . - __v7_ca17mp_proc_info /* @@ -594,7 +622,7 @@ __krait_proc_info: * do support them. They also don't indicate support for fused multiply * instructions even though they actually do support them. */ - __v7_proc __v7_setup, hwcaps = HWCAP_IDIV | HWCAP_VFPv4 + __v7_proc __krait_proc_info, __v7_setup, hwcaps = HWCAP_IDIV | HWCAP_VFPv4 .size __krait_proc_info, . - __krait_proc_info /* @@ -604,5 +632,5 @@ __krait_proc_info: __v7_proc_info: .long 0x000f0000 @ Required ID value .long 0x000f0000 @ Mask for ID - __v7_proc __v7_setup + __v7_proc __v7_proc_info, __v7_setup .size __v7_proc_info, . - __v7_proc_info diff --git a/arch/arm/mm/proc-v7m.S b/arch/arm/mm/proc-v7m.S index d1e68b553d3b..e08e1f2bab76 100644 --- a/arch/arm/mm/proc-v7m.S +++ b/arch/arm/mm/proc-v7m.S @@ -135,7 +135,7 @@ __v7m_setup_stack_top: string cpu_elf_name "v7m" string cpu_v7m_name "ARMv7-M" - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc /* * Match any ARMv7-M processor core. @@ -146,7 +146,7 @@ __v7m_proc_info: .long 0x000f0000 @ Mask for ID .long 0 @ proc_info_list.__cpu_mm_mmu_flags .long 0 @ proc_info_list.__cpu_io_mmu_flags - b __v7m_setup @ proc_info_list.__cpu_flush + initfn __v7m_setup, __v7m_proc_info @ proc_info_list.__cpu_flush .long cpu_arch_name .long cpu_elf_name .long HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index f8acdfece036..293dcc2c441f 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -499,7 +499,7 @@ xsc3_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro xsc3_proc_info name:req, cpu_val:req, cpu_mask:req .type __\name\()_proc_info,#object @@ -514,7 +514,7 @@ __\name\()_proc_info: .long PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __xsc3_setup + initfn __xsc3_setup, __\name\()_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index afa2b3c4df4a..b6bbfdb6dfdc 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -612,7 +612,7 @@ xscale_crval: .align - .section ".proc.info.init", #alloc, #execinstr + .section ".proc.info.init", #alloc .macro xscale_proc_info name:req, cpu_val:req, cpu_mask:req, cpu_name:req, cache .type __\name\()_proc_info,#object @@ -627,7 +627,7 @@ __\name\()_proc_info: .long PMD_TYPE_SECT | \ PMD_SECT_AP_WRITE | \ PMD_SECT_AP_READ - b __xscale_setup + initfn __xscale_setup, __\name\()_proc_info .long cpu_arch_name .long cpu_elf_name .long HWCAP_SWP|HWCAP_HALF|HWCAP_THUMB|HWCAP_FAST_MULT|HWCAP_EDSP diff --git a/arch/arm/nwfpe/entry.S b/arch/arm/nwfpe/entry.S index 5d65be1f1e8a..71df43547659 100644 --- a/arch/arm/nwfpe/entry.S +++ b/arch/arm/nwfpe/entry.S @@ -113,7 +113,7 @@ next: @ to fault. Emit the appropriate exception gunk to fix things up. @ ??? For some reason, faults can happen at .Lx2 even with a @ plain LDR instruction. Weird, but it seems harmless. - .pushsection .fixup,"ax" + .pushsection .text.fixup,"ax" .align 2 .Lfix: ret r9 @ let the user eat segfaults .popsection diff --git a/arch/arm/plat-pxa/dma.c b/arch/arm/plat-pxa/dma.c index 054fc5a1a11c..d92f07f6ecfb 100644 --- a/arch/arm/plat-pxa/dma.c +++ b/arch/arm/plat-pxa/dma.c @@ -51,19 +51,19 @@ static struct dentry *dbgfs_root, *dbgfs_state, **dbgfs_chan; static int dbg_show_requester_chan(struct seq_file *s, void *p) { - int pos = 0; int chan = (int)s->private; int i; u32 drcmr; - pos += seq_printf(s, "DMA channel %d requesters list :\n", chan); + seq_printf(s, "DMA channel %d requesters list :\n", chan); for (i = 0; i < DMA_MAX_REQUESTERS; i++) { drcmr = DRCMR(i); if ((drcmr & DRCMR_CHLNUM) == chan) - pos += seq_printf(s, "\tRequester %d (MAPVLD=%d)\n", i, - !!(drcmr & DRCMR_MAPVLD)); + seq_printf(s, "\tRequester %d (MAPVLD=%d)\n", + i, !!(drcmr & DRCMR_MAPVLD)); } - return pos; + + return 0; } static inline int dbg_burst_from_dcmd(u32 dcmd) @@ -83,7 +83,6 @@ static int is_phys_valid(unsigned long addr) static int dbg_show_descriptors(struct seq_file *s, void *p) { - int pos = 0; int chan = (int)s->private; int i, max_show = 20, burst, width; u32 dcmd; @@ -94,44 +93,43 @@ static int dbg_show_descriptors(struct seq_file *s, void *p) spin_lock_irqsave(&dma_channels[chan].lock, flags); phys_desc = DDADR(chan); - pos += seq_printf(s, "DMA channel %d descriptors :\n", chan); - pos += seq_printf(s, "[%03d] First descriptor unknown\n", 0); + seq_printf(s, "DMA channel %d descriptors :\n", chan); + seq_printf(s, "[%03d] First descriptor unknown\n", 0); for (i = 1; i < max_show && is_phys_valid(phys_desc); i++) { desc = phys_to_virt(phys_desc); dcmd = desc->dcmd; burst = dbg_burst_from_dcmd(dcmd); width = (1 << ((dcmd >> 14) & 0x3)) >> 1; - pos += seq_printf(s, "[%03d] Desc at %08lx(virt %p)\n", - i, phys_desc, desc); - pos += seq_printf(s, "\tDDADR = %08x\n", desc->ddadr); - pos += seq_printf(s, "\tDSADR = %08x\n", desc->dsadr); - pos += seq_printf(s, "\tDTADR = %08x\n", desc->dtadr); - pos += seq_printf(s, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d" - " width=%d len=%d)\n", - dcmd, - DCMD_STR(INCSRCADDR), DCMD_STR(INCTRGADDR), - DCMD_STR(FLOWSRC), DCMD_STR(FLOWTRG), - DCMD_STR(STARTIRQEN), DCMD_STR(ENDIRQEN), - DCMD_STR(ENDIAN), burst, width, - dcmd & DCMD_LENGTH); + seq_printf(s, "[%03d] Desc at %08lx(virt %p)\n", + i, phys_desc, desc); + seq_printf(s, "\tDDADR = %08x\n", desc->ddadr); + seq_printf(s, "\tDSADR = %08x\n", desc->dsadr); + seq_printf(s, "\tDTADR = %08x\n", desc->dtadr); + seq_printf(s, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d width=%d len=%d)\n", + dcmd, + DCMD_STR(INCSRCADDR), DCMD_STR(INCTRGADDR), + DCMD_STR(FLOWSRC), DCMD_STR(FLOWTRG), + DCMD_STR(STARTIRQEN), DCMD_STR(ENDIRQEN), + DCMD_STR(ENDIAN), burst, width, + dcmd & DCMD_LENGTH); phys_desc = desc->ddadr; } if (i == max_show) - pos += seq_printf(s, "[%03d] Desc at %08lx ... max display reached\n", - i, phys_desc); + seq_printf(s, "[%03d] Desc at %08lx ... max display reached\n", + i, phys_desc); else - pos += seq_printf(s, "[%03d] Desc at %08lx is %s\n", - i, phys_desc, phys_desc == DDADR_STOP ? - "DDADR_STOP" : "invalid"); + seq_printf(s, "[%03d] Desc at %08lx is %s\n", + i, phys_desc, phys_desc == DDADR_STOP ? + "DDADR_STOP" : "invalid"); spin_unlock_irqrestore(&dma_channels[chan].lock, flags); - return pos; + + return 0; } static int dbg_show_chan_state(struct seq_file *s, void *p) { - int pos = 0; int chan = (int)s->private; u32 dcsr, dcmd; int burst, width; @@ -142,42 +140,39 @@ static int dbg_show_chan_state(struct seq_file *s, void *p) burst = dbg_burst_from_dcmd(dcmd); width = (1 << ((dcmd >> 14) & 0x3)) >> 1; - pos += seq_printf(s, "DMA channel %d\n", chan); - pos += seq_printf(s, "\tPriority : %s\n", - str_prio[dma_channels[chan].prio]); - pos += seq_printf(s, "\tUnaligned transfer bit: %s\n", - DALGN & (1 << chan) ? "yes" : "no"); - pos += seq_printf(s, "\tDCSR = %08x (%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", - dcsr, DCSR_STR(RUN), DCSR_STR(NODESC), - DCSR_STR(STOPIRQEN), DCSR_STR(EORIRQEN), - DCSR_STR(EORJMPEN), DCSR_STR(EORSTOPEN), - DCSR_STR(SETCMPST), DCSR_STR(CLRCMPST), - DCSR_STR(CMPST), DCSR_STR(EORINTR), DCSR_STR(REQPEND), - DCSR_STR(STOPSTATE), DCSR_STR(ENDINTR), - DCSR_STR(STARTINTR), DCSR_STR(BUSERR)); - - pos += seq_printf(s, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d width=%d" - " len=%d)\n", - dcmd, - DCMD_STR(INCSRCADDR), DCMD_STR(INCTRGADDR), - DCMD_STR(FLOWSRC), DCMD_STR(FLOWTRG), - DCMD_STR(STARTIRQEN), DCMD_STR(ENDIRQEN), - DCMD_STR(ENDIAN), burst, width, dcmd & DCMD_LENGTH); - pos += seq_printf(s, "\tDSADR = %08x\n", DSADR(chan)); - pos += seq_printf(s, "\tDTADR = %08x\n", DTADR(chan)); - pos += seq_printf(s, "\tDDADR = %08x\n", DDADR(chan)); - return pos; + seq_printf(s, "DMA channel %d\n", chan); + seq_printf(s, "\tPriority : %s\n", str_prio[dma_channels[chan].prio]); + seq_printf(s, "\tUnaligned transfer bit: %s\n", + DALGN & (1 << chan) ? "yes" : "no"); + seq_printf(s, "\tDCSR = %08x (%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", + dcsr, DCSR_STR(RUN), DCSR_STR(NODESC), + DCSR_STR(STOPIRQEN), DCSR_STR(EORIRQEN), + DCSR_STR(EORJMPEN), DCSR_STR(EORSTOPEN), + DCSR_STR(SETCMPST), DCSR_STR(CLRCMPST), + DCSR_STR(CMPST), DCSR_STR(EORINTR), DCSR_STR(REQPEND), + DCSR_STR(STOPSTATE), DCSR_STR(ENDINTR), + DCSR_STR(STARTINTR), DCSR_STR(BUSERR)); + + seq_printf(s, "\tDCMD = %08x (%s%s%s%s%s%s%sburst=%d width=%d len=%d)\n", + dcmd, + DCMD_STR(INCSRCADDR), DCMD_STR(INCTRGADDR), + DCMD_STR(FLOWSRC), DCMD_STR(FLOWTRG), + DCMD_STR(STARTIRQEN), DCMD_STR(ENDIRQEN), + DCMD_STR(ENDIAN), burst, width, dcmd & DCMD_LENGTH); + seq_printf(s, "\tDSADR = %08x\n", DSADR(chan)); + seq_printf(s, "\tDTADR = %08x\n", DTADR(chan)); + seq_printf(s, "\tDDADR = %08x\n", DDADR(chan)); + + return 0; } static int dbg_show_state(struct seq_file *s, void *p) { - int pos = 0; - /* basic device status */ - pos += seq_printf(s, "DMA engine status\n"); - pos += seq_printf(s, "\tChannel number: %d\n", num_dma_channels); + seq_puts(s, "DMA engine status\n"); + seq_printf(s, "\tChannel number: %d\n", num_dma_channels); - return pos; + return 0; } #define DBGFS_FUNC_DECL(name) \ diff --git a/arch/arm/vdso/.gitignore b/arch/arm/vdso/.gitignore new file mode 100644 index 000000000000..f8b69d84238e --- /dev/null +++ b/arch/arm/vdso/.gitignore @@ -0,0 +1 @@ +vdso.lds diff --git a/arch/arm/vdso/Makefile b/arch/arm/vdso/Makefile new file mode 100644 index 000000000000..bab0a8be7924 --- /dev/null +++ b/arch/arm/vdso/Makefile @@ -0,0 +1,74 @@ +hostprogs-y := vdsomunge + +obj-vdso := vgettimeofday.o datapage.o + +# Build rules +targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.so.raw vdso.lds +obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) + +ccflags-y := -shared -fPIC -fno-common -fno-builtin -fno-stack-protector +ccflags-y += -nostdlib -Wl,-soname=linux-vdso.so.1 -DDISABLE_BRANCH_PROFILING +ccflags-y += -Wl,--no-undefined $(call cc-ldoption, -Wl$(comma)--hash-style=sysv) + +obj-y += vdso.o +extra-y += vdso.lds +CPPFLAGS_vdso.lds += -P -C -U$(ARCH) + +CFLAGS_REMOVE_vdso.o = -pg + +# Force -O2 to avoid libgcc dependencies +CFLAGS_REMOVE_vgettimeofday.o = -pg -Os +CFLAGS_vgettimeofday.o = -O2 + +# Disable gcov profiling for VDSO code +GCOV_PROFILE := n + +# Force dependency +$(obj)/vdso.o : $(obj)/vdso.so + +# Link rule for the .so file +$(obj)/vdso.so.raw: $(src)/vdso.lds $(obj-vdso) FORCE + $(call if_changed,vdsold) + +$(obj)/vdso.so.dbg: $(obj)/vdso.so.raw $(obj)/vdsomunge FORCE + $(call if_changed,vdsomunge) + +# Strip rule for the .so file +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +# Actual build commands +quiet_cmd_vdsold = VDSO $@ + cmd_vdsold = $(CC) $(c_flags) -Wl,-T $(filter %.lds,$^) $(filter %.o,$^) \ + $(call cc-ldoption, -Wl$(comma)--build-id) \ + -Wl,-Bsymbolic -Wl,-z,max-page-size=4096 \ + -Wl,-z,common-page-size=4096 -o $@ + +quiet_cmd_vdsomunge = MUNGE $@ + cmd_vdsomunge = $(objtree)/$(obj)/vdsomunge $< $@ + +# +# Install the unstripped copy of vdso.so.dbg. If our toolchain +# supports build-id, install .build-id links as well. +# +# Cribbed from arch/x86/vdso/Makefile. +# +quiet_cmd_vdso_install = INSTALL $< +define cmd_vdso_install + cp $< "$(MODLIB)/vdso/vdso.so"; \ + if readelf -n $< | grep -q 'Build ID'; then \ + buildid=`readelf -n $< |grep 'Build ID' |sed -e 's/^.*Build ID: \(.*\)$$/\1/'`; \ + first=`echo $$buildid | cut -b-2`; \ + last=`echo $$buildid | cut -b3-`; \ + mkdir -p "$(MODLIB)/vdso/.build-id/$$first"; \ + ln -sf "../../vdso.so" "$(MODLIB)/vdso/.build-id/$$first/$$last.debug"; \ + fi +endef + +$(MODLIB)/vdso: FORCE + @mkdir -p $(MODLIB)/vdso + +PHONY += vdso_install +vdso_install: $(obj)/vdso.so.dbg $(MODLIB)/vdso FORCE + $(call cmd,vdso_install) diff --git a/arch/arm/vdso/datapage.S b/arch/arm/vdso/datapage.S new file mode 100644 index 000000000000..a2e60367931b --- /dev/null +++ b/arch/arm/vdso/datapage.S @@ -0,0 +1,15 @@ +#include <linux/linkage.h> +#include <asm/asm-offsets.h> + + .align 2 +.L_vdso_data_ptr: + .long _start - . - VDSO_DATA_SIZE + +ENTRY(__get_datapage) + .fnstart + adr r0, .L_vdso_data_ptr + ldr r1, [r0] + add r0, r0, r1 + bx lr + .fnend +ENDPROC(__get_datapage) diff --git a/arch/arm64/kernel/cputable.c b/arch/arm/vdso/vdso.S index fd3993cb060f..b2b97e3e7bab 100644 --- a/arch/arm64/kernel/cputable.c +++ b/arch/arm/vdso/vdso.S @@ -1,9 +1,9 @@ /* - * arch/arm64/kernel/cputable.c + * Adapted from arm64 version. * - * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2012 ARM Limited * - * This program is free software: you can redistribute it and/or modify + * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * @@ -14,20 +14,22 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * Author: Will Deacon <will.deacon@arm.com> */ #include <linux/init.h> +#include <linux/linkage.h> +#include <linux/const.h> +#include <asm/page.h> -#include <asm/cputable.h> + __PAGE_ALIGNED_DATA -extern unsigned long __cpu_setup(void); + .globl vdso_start, vdso_end + .balign PAGE_SIZE +vdso_start: + .incbin "arch/arm/vdso/vdso.so" + .balign PAGE_SIZE +vdso_end: -struct cpu_info cpu_table[] = { - { - .cpu_id_val = 0x000f0000, - .cpu_id_mask = 0x000f0000, - .cpu_name = "AArch64 Processor", - .cpu_setup = __cpu_setup, - }, - { /* Empty */ }, -}; + .previous diff --git a/arch/arm/vdso/vdso.lds.S b/arch/arm/vdso/vdso.lds.S new file mode 100644 index 000000000000..89ca89f12d23 --- /dev/null +++ b/arch/arm/vdso/vdso.lds.S @@ -0,0 +1,87 @@ +/* + * Adapted from arm64 version. + * + * GNU linker script for the VDSO library. + * + * Copyright (C) 2012 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * Author: Will Deacon <will.deacon@arm.com> + * Heavily based on the vDSO linker scripts for other archs. + */ + +#include <linux/const.h> +#include <asm/page.h> +#include <asm/vdso.h> + +OUTPUT_FORMAT("elf32-littlearm", "elf32-bigarm", "elf32-littlearm") +OUTPUT_ARCH(arm) + +SECTIONS +{ + PROVIDE(_start = .); + + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .note : { *(.note.*) } :text :note + + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + .dynamic : { *(.dynamic) } :text :dynamic + + .rodata : { *(.rodata*) } :text + + .text : { *(.text*) } :text =0xe7f001f2 + + .got : { *(.got) } + .rel.plt : { *(.rel.plt) } + + /DISCARD/ : { + *(.note.GNU-stack) + *(.data .data.* .gnu.linkonce.d.* .sdata*) + *(.bss .sbss .dynbss .dynsbss) + } +} + +/* + * We must supply the ELF program headers explicitly to get just one + * PT_LOAD segment, and set the flags explicitly to make segments read-only. + */ +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} + +VERSION +{ + LINUX_2.6 { + global: + __vdso_clock_gettime; + __vdso_gettimeofday; + local: *; + }; +} diff --git a/arch/arm/vdso/vdsomunge.c b/arch/arm/vdso/vdsomunge.c new file mode 100644 index 000000000000..9005b07296c8 --- /dev/null +++ b/arch/arm/vdso/vdsomunge.c @@ -0,0 +1,201 @@ +/* + * Copyright 2015 Mentor Graphics Corporation. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * + * vdsomunge - Host program which produces a shared object + * architecturally specified to be usable by both soft- and hard-float + * programs. + * + * The Procedure Call Standard for the ARM Architecture (ARM IHI + * 0042E) says: + * + * 6.4.1 VFP and Base Standard Compatibility + * + * Code compiled for the VFP calling standard is compatible with + * the base standard (and vice-versa) if no floating-point or + * containerized vector arguments or results are used. + * + * And ELF for the ARM Architecture (ARM IHI 0044E) (Table 4-2) says: + * + * If both EF_ARM_ABI_FLOAT_XXXX bits are clear, conformance to the + * base procedure-call standard is implied. + * + * The VDSO is built with -msoft-float, as with the rest of the ARM + * kernel, and uses no floating point arguments or results. The build + * process will produce a shared object that may or may not have the + * EF_ARM_ABI_FLOAT_SOFT flag set (it seems to depend on the binutils + * version; binutils starting with 2.24 appears to set it). The + * EF_ARM_ABI_FLOAT_HARD flag should definitely not be set, and this + * program will error out if it is. + * + * If the soft-float flag is set, this program clears it. That's all + * it does. + */ + +#define _GNU_SOURCE + +#include <byteswap.h> +#include <elf.h> +#include <errno.h> +#include <error.h> +#include <fcntl.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define HOST_ORDER ELFDATA2LSB +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define HOST_ORDER ELFDATA2MSB +#endif + +/* Some of the ELF constants we'd like to use were added to <elf.h> + * relatively recently. + */ +#ifndef EF_ARM_EABI_VER5 +#define EF_ARM_EABI_VER5 0x05000000 +#endif + +#ifndef EF_ARM_ABI_FLOAT_SOFT +#define EF_ARM_ABI_FLOAT_SOFT 0x200 +#endif + +#ifndef EF_ARM_ABI_FLOAT_HARD +#define EF_ARM_ABI_FLOAT_HARD 0x400 +#endif + +static const char *outfile; + +static void cleanup(void) +{ + if (error_message_count > 0 && outfile != NULL) + unlink(outfile); +} + +static Elf32_Word read_elf_word(Elf32_Word word, bool swap) +{ + return swap ? bswap_32(word) : word; +} + +static Elf32_Half read_elf_half(Elf32_Half half, bool swap) +{ + return swap ? bswap_16(half) : half; +} + +static void write_elf_word(Elf32_Word val, Elf32_Word *dst, bool swap) +{ + *dst = swap ? bswap_32(val) : val; +} + +int main(int argc, char **argv) +{ + const Elf32_Ehdr *inhdr; + bool clear_soft_float; + const char *infile; + Elf32_Word e_flags; + const void *inbuf; + struct stat stat; + void *outbuf; + bool swap; + int outfd; + int infd; + + atexit(cleanup); + + if (argc != 3) + error(EXIT_FAILURE, 0, "Usage: %s [infile] [outfile]", argv[0]); + + infile = argv[1]; + outfile = argv[2]; + + infd = open(infile, O_RDONLY); + if (infd < 0) + error(EXIT_FAILURE, errno, "Cannot open %s", infile); + + if (fstat(infd, &stat) != 0) + error(EXIT_FAILURE, errno, "Failed stat for %s", infile); + + inbuf = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, infd, 0); + if (inbuf == MAP_FAILED) + error(EXIT_FAILURE, errno, "Failed to map %s", infile); + + close(infd); + + inhdr = inbuf; + + if (memcmp(&inhdr->e_ident, ELFMAG, SELFMAG) != 0) + error(EXIT_FAILURE, 0, "Not an ELF file"); + + if (inhdr->e_ident[EI_CLASS] != ELFCLASS32) + error(EXIT_FAILURE, 0, "Unsupported ELF class"); + + swap = inhdr->e_ident[EI_DATA] != HOST_ORDER; + + if (read_elf_half(inhdr->e_type, swap) != ET_DYN) + error(EXIT_FAILURE, 0, "Not a shared object"); + + if (read_elf_half(inhdr->e_machine, swap) != EM_ARM) { + error(EXIT_FAILURE, 0, "Unsupported architecture %#x", + inhdr->e_machine); + } + + e_flags = read_elf_word(inhdr->e_flags, swap); + + if (EF_ARM_EABI_VERSION(e_flags) != EF_ARM_EABI_VER5) { + error(EXIT_FAILURE, 0, "Unsupported EABI version %#x", + EF_ARM_EABI_VERSION(e_flags)); + } + + if (e_flags & EF_ARM_ABI_FLOAT_HARD) + error(EXIT_FAILURE, 0, + "Unexpected hard-float flag set in e_flags"); + + clear_soft_float = !!(e_flags & EF_ARM_ABI_FLOAT_SOFT); + + outfd = open(outfile, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (outfd < 0) + error(EXIT_FAILURE, errno, "Cannot open %s", outfile); + + if (ftruncate(outfd, stat.st_size) != 0) + error(EXIT_FAILURE, errno, "Cannot truncate %s", outfile); + + outbuf = mmap(NULL, stat.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, + outfd, 0); + if (outbuf == MAP_FAILED) + error(EXIT_FAILURE, errno, "Failed to map %s", outfile); + + close(outfd); + + memcpy(outbuf, inbuf, stat.st_size); + + if (clear_soft_float) { + Elf32_Ehdr *outhdr; + + outhdr = outbuf; + e_flags &= ~EF_ARM_ABI_FLOAT_SOFT; + write_elf_word(e_flags, &outhdr->e_flags, swap); + } + + if (msync(outbuf, stat.st_size, MS_SYNC) != 0) + error(EXIT_FAILURE, errno, "Failed to sync %s", outfile); + + return EXIT_SUCCESS; +} diff --git a/arch/arm/vdso/vgettimeofday.c b/arch/arm/vdso/vgettimeofday.c new file mode 100644 index 000000000000..79214d5ff097 --- /dev/null +++ b/arch/arm/vdso/vgettimeofday.c @@ -0,0 +1,282 @@ +/* + * Copyright 2015 Mentor Graphics Corporation. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 of the + * License. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/compiler.h> +#include <linux/hrtimer.h> +#include <linux/time.h> +#include <asm/arch_timer.h> +#include <asm/barrier.h> +#include <asm/bug.h> +#include <asm/page.h> +#include <asm/unistd.h> +#include <asm/vdso_datapage.h> + +#ifndef CONFIG_AEABI +#error This code depends on AEABI system call conventions +#endif + +extern struct vdso_data *__get_datapage(void); + +static notrace u32 __vdso_read_begin(const struct vdso_data *vdata) +{ + u32 seq; +repeat: + seq = ACCESS_ONCE(vdata->seq_count); + if (seq & 1) { + cpu_relax(); + goto repeat; + } + return seq; +} + +static notrace u32 vdso_read_begin(const struct vdso_data *vdata) +{ + u32 seq; + + seq = __vdso_read_begin(vdata); + + smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */ + return seq; +} + +static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start) +{ + smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */ + return vdata->seq_count != start; +} + +static notrace long clock_gettime_fallback(clockid_t _clkid, + struct timespec *_ts) +{ + register struct timespec *ts asm("r1") = _ts; + register clockid_t clkid asm("r0") = _clkid; + register long ret asm ("r0"); + register long nr asm("r7") = __NR_clock_gettime; + + asm volatile( + " swi #0\n" + : "=r" (ret) + : "r" (clkid), "r" (ts), "r" (nr) + : "memory"); + + return ret; +} + +static notrace int do_realtime_coarse(struct timespec *ts, + struct vdso_data *vdata) +{ + u32 seq; + + do { + seq = vdso_read_begin(vdata); + + ts->tv_sec = vdata->xtime_coarse_sec; + ts->tv_nsec = vdata->xtime_coarse_nsec; + + } while (vdso_read_retry(vdata, seq)); + + return 0; +} + +static notrace int do_monotonic_coarse(struct timespec *ts, + struct vdso_data *vdata) +{ + struct timespec tomono; + u32 seq; + + do { + seq = vdso_read_begin(vdata); + + ts->tv_sec = vdata->xtime_coarse_sec; + ts->tv_nsec = vdata->xtime_coarse_nsec; + + tomono.tv_sec = vdata->wtm_clock_sec; + tomono.tv_nsec = vdata->wtm_clock_nsec; + + } while (vdso_read_retry(vdata, seq)); + + ts->tv_sec += tomono.tv_sec; + timespec_add_ns(ts, tomono.tv_nsec); + + return 0; +} + +#ifdef CONFIG_ARM_ARCH_TIMER + +static notrace u64 get_ns(struct vdso_data *vdata) +{ + u64 cycle_delta; + u64 cycle_now; + u64 nsec; + + cycle_now = arch_counter_get_cntvct(); + + cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask; + + nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec; + nsec >>= vdata->cs_shift; + + return nsec; +} + +static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata) +{ + u64 nsecs; + u32 seq; + + do { + seq = vdso_read_begin(vdata); + + if (!vdata->tk_is_cntvct) + return -1; + + ts->tv_sec = vdata->xtime_clock_sec; + nsecs = get_ns(vdata); + + } while (vdso_read_retry(vdata, seq)); + + ts->tv_nsec = 0; + timespec_add_ns(ts, nsecs); + + return 0; +} + +static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata) +{ + struct timespec tomono; + u64 nsecs; + u32 seq; + + do { + seq = vdso_read_begin(vdata); + + if (!vdata->tk_is_cntvct) + return -1; + + ts->tv_sec = vdata->xtime_clock_sec; + nsecs = get_ns(vdata); + + tomono.tv_sec = vdata->wtm_clock_sec; + tomono.tv_nsec = vdata->wtm_clock_nsec; + + } while (vdso_read_retry(vdata, seq)); + + ts->tv_sec += tomono.tv_sec; + ts->tv_nsec = 0; + timespec_add_ns(ts, nsecs + tomono.tv_nsec); + + return 0; +} + +#else /* CONFIG_ARM_ARCH_TIMER */ + +static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata) +{ + return -1; +} + +static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata) +{ + return -1; +} + +#endif /* CONFIG_ARM_ARCH_TIMER */ + +notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts) +{ + struct vdso_data *vdata; + int ret = -1; + + vdata = __get_datapage(); + + switch (clkid) { + case CLOCK_REALTIME_COARSE: + ret = do_realtime_coarse(ts, vdata); + break; + case CLOCK_MONOTONIC_COARSE: + ret = do_monotonic_coarse(ts, vdata); + break; + case CLOCK_REALTIME: + ret = do_realtime(ts, vdata); + break; + case CLOCK_MONOTONIC: + ret = do_monotonic(ts, vdata); + break; + default: + break; + } + + if (ret) + ret = clock_gettime_fallback(clkid, ts); + + return ret; +} + +static notrace long gettimeofday_fallback(struct timeval *_tv, + struct timezone *_tz) +{ + register struct timezone *tz asm("r1") = _tz; + register struct timeval *tv asm("r0") = _tv; + register long ret asm ("r0"); + register long nr asm("r7") = __NR_gettimeofday; + + asm volatile( + " swi #0\n" + : "=r" (ret) + : "r" (tv), "r" (tz), "r" (nr) + : "memory"); + + return ret; +} + +notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) +{ + struct timespec ts; + struct vdso_data *vdata; + int ret; + + vdata = __get_datapage(); + + ret = do_realtime(&ts, vdata); + if (ret) + return gettimeofday_fallback(tv, tz); + + if (tv) { + tv->tv_sec = ts.tv_sec; + tv->tv_usec = ts.tv_nsec / 1000; + } + if (tz) { + tz->tz_minuteswest = vdata->tz_minuteswest; + tz->tz_dsttime = vdata->tz_dsttime; + } + + return ret; +} + +/* Avoid unresolved references emitted by GCC */ + +void __aeabi_unwind_cpp_pr0(void) +{ +} + +void __aeabi_unwind_cpp_pr1(void) +{ +} + +void __aeabi_unwind_cpp_pr2(void) +{ +} diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index 263a2044c65b..224081ccc92f 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c @@ -53,105 +53,33 @@ EXPORT_SYMBOL_GPL(xen_platform_pci_unplug); static __read_mostly int xen_events_irq = -1; -/* map fgmfn of domid to lpfn in the current domain */ -static int map_foreign_page(unsigned long lpfn, unsigned long fgmfn, - unsigned int domid) +int xen_remap_domain_mfn_array(struct vm_area_struct *vma, + unsigned long addr, + xen_pfn_t *mfn, int nr, + int *err_ptr, pgprot_t prot, + unsigned domid, + struct page **pages) { - int rc; - struct xen_add_to_physmap_range xatp = { - .domid = DOMID_SELF, - .foreign_domid = domid, - .size = 1, - .space = XENMAPSPACE_gmfn_foreign, - }; - xen_ulong_t idx = fgmfn; - xen_pfn_t gpfn = lpfn; - int err = 0; - - set_xen_guest_handle(xatp.idxs, &idx); - set_xen_guest_handle(xatp.gpfns, &gpfn); - set_xen_guest_handle(xatp.errs, &err); - - rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp); - if (rc || err) { - pr_warn("Failed to map pfn to mfn rc:%d:%d pfn:%lx mfn:%lx\n", - rc, err, lpfn, fgmfn); - return 1; - } - return 0; -} - -struct remap_data { - xen_pfn_t fgmfn; /* foreign domain's gmfn */ - pgprot_t prot; - domid_t domid; - struct vm_area_struct *vma; - int index; - struct page **pages; - struct xen_remap_mfn_info *info; -}; - -static int remap_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr, - void *data) -{ - struct remap_data *info = data; - struct page *page = info->pages[info->index++]; - unsigned long pfn = page_to_pfn(page); - pte_t pte = pte_mkspecial(pfn_pte(pfn, info->prot)); - - if (map_foreign_page(pfn, info->fgmfn, info->domid)) - return -EFAULT; - set_pte_at(info->vma->vm_mm, addr, ptep, pte); - - return 0; + return xen_xlate_remap_gfn_array(vma, addr, mfn, nr, err_ptr, + prot, domid, pages); } +EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array); +/* Not used by XENFEAT_auto_translated guests. */ int xen_remap_domain_mfn_range(struct vm_area_struct *vma, - unsigned long addr, - xen_pfn_t mfn, int nr, - pgprot_t prot, unsigned domid, - struct page **pages) + unsigned long addr, + xen_pfn_t mfn, int nr, + pgprot_t prot, unsigned domid, + struct page **pages) { - int err; - struct remap_data data; - - /* TBD: Batching, current sole caller only does page at a time */ - if (nr > 1) - return -EINVAL; - - data.fgmfn = mfn; - data.prot = prot; - data.domid = domid; - data.vma = vma; - data.index = 0; - data.pages = pages; - err = apply_to_page_range(vma->vm_mm, addr, nr << PAGE_SHIFT, - remap_pte_fn, &data); - return err; + return -ENOSYS; } EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range); int xen_unmap_domain_mfn_range(struct vm_area_struct *vma, int nr, struct page **pages) { - int i; - - for (i = 0; i < nr; i++) { - struct xen_remove_from_physmap xrp; - unsigned long rc, pfn; - - pfn = page_to_pfn(pages[i]); - - xrp.domid = DOMID_SELF; - xrp.gpfn = pfn; - rc = HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp); - if (rc) { - pr_warn("Failed to unmap pfn:%lx rc:%ld\n", - pfn, rc); - return rc; - } - } - return 0; + return xen_xlate_unmap_gfn_range(vma, nr, pages); } EXPORT_SYMBOL_GPL(xen_unmap_domain_mfn_range); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 34f487d5d84e..b8d96f1554af 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -368,6 +368,27 @@ config ARM64_ERRATUM_832075 If unsure, say Y. +config ARM64_ERRATUM_845719 + bool "Cortex-A53: 845719: a load might read incorrect data" + depends on COMPAT + default y + help + This option adds an alternative code sequence to work around ARM + erratum 845719 on Cortex-A53 parts up to r0p4. + + When running a compat (AArch32) userspace on an affected Cortex-A53 + part, a load at EL0 from a virtual address that matches the bottom 32 + bits of the virtual address used by a recent load at (AArch64) EL1 + might return incorrect data. + + The workaround is to write the contextidr_el1 register on exception + return to a 32-bit task. + Please note that this does not necessarily enable the workaround, + as it depends on the alternative framework, which will only patch + the kernel if an affected CPU is detected. + + If unsure, say Y. + endmenu @@ -455,8 +476,8 @@ config SCHED_SMT places. If unsure say N here. config NR_CPUS - int "Maximum number of CPUs (2-64)" - range 2 64 + int "Maximum number of CPUs (2-4096)" + range 2 4096 depends on SMP # These have to remain sorted largest to smallest default "64" @@ -470,6 +491,10 @@ config HOTPLUG_CPU source kernel/Kconfig.preempt +config UP_LATE_INIT + def_bool y + depends on !SMP + config HZ int default 100 @@ -670,7 +695,7 @@ source "fs/Kconfig.binfmt" config COMPAT bool "Kernel support for 32-bit EL0" - depends on !ARM64_64K_PAGES + depends on !ARM64_64K_PAGES || EXPERT select COMPAT_BINFMT_ELF select HAVE_UID16 select OLD_SIGSUSPEND3 @@ -681,6 +706,10 @@ config COMPAT the user helper functions, VFP support and the ptrace interface are handled appropriately by the kernel. + If you also enabled CONFIG_ARM64_64K_PAGES, please be aware that you + will only be able to execute AArch32 binaries that were compiled with + 64k aligned segments. + If you want to execute 32-bit userspace applications, say Y. config SYSVIPC_COMPAT diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 69ceedc982a5..4d2a925998f9 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -48,7 +48,7 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/ core-$(CONFIG_XEN) += arch/arm64/xen/ core-$(CONFIG_CRYPTO) += arch/arm64/crypto/ libs-y := arch/arm64/lib/ $(libs-y) -libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/ +core-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a # Default target when executing plain make KBUILD_IMAGE := Image.gz diff --git a/arch/arm64/boot/dts/apm/apm-mustang.dts b/arch/arm64/boot/dts/apm/apm-mustang.dts index 2e25de0800b9..83578e766b94 100644 --- a/arch/arm64/boot/dts/apm/apm-mustang.dts +++ b/arch/arm64/boot/dts/apm/apm-mustang.dts @@ -45,6 +45,10 @@ status = "ok"; }; +&sgenet1 { + status = "ok"; +}; + &xgenet { status = "ok"; }; diff --git a/arch/arm64/boot/dts/apm/apm-storm.dtsi b/arch/arm64/boot/dts/apm/apm-storm.dtsi index a857794432d6..e74f6e0a208c 100644 --- a/arch/arm64/boot/dts/apm/apm-storm.dtsi +++ b/arch/arm64/boot/dts/apm/apm-storm.dtsi @@ -186,6 +186,16 @@ clock-output-names = "sge0clk"; }; + sge1clk: sge1clk@1f21c000 { + compatible = "apm,xgene-device-clock"; + #clock-cells = <1>; + clocks = <&socplldiv2 0>; + reg = <0x0 0x1f21c000 0x0 0x1000>; + reg-names = "csr-reg"; + csr-mask = <0xc>; + clock-output-names = "sge1clk"; + }; + xge0clk: xge0clk@1f61c000 { compatible = "apm,xgene-device-clock"; #clock-cells = <1>; @@ -628,13 +638,30 @@ <0x0 0x1f200000 0x0 0Xc300>, <0x0 0x1B000000 0x0 0X200>; reg-names = "enet_csr", "ring_csr", "ring_cmd"; - interrupts = <0x0 0xA0 0x4>; + interrupts = <0x0 0xA0 0x4>, + <0x0 0xA1 0x4>; dma-coherent; clocks = <&sge0clk 0>; local-mac-address = [00 00 00 00 00 00]; phy-connection-type = "sgmii"; }; + sgenet1: ethernet@1f210030 { + compatible = "apm,xgene1-sgenet"; + status = "disabled"; + reg = <0x0 0x1f210030 0x0 0xd100>, + <0x0 0x1f200000 0x0 0Xc300>, + <0x0 0x1B000000 0x0 0X8000>; + reg-names = "enet_csr", "ring_csr", "ring_cmd"; + interrupts = <0x0 0xAC 0x4>, + <0x0 0xAD 0x4>; + port-id = <1>; + dma-coherent; + clocks = <&sge1clk 0>; + local-mac-address = [00 00 00 00 00 00]; + phy-connection-type = "sgmii"; + }; + xgenet: ethernet@1f610000 { compatible = "apm,xgene1-xgenet"; status = "disabled"; @@ -642,7 +669,8 @@ <0x0 0x1f600000 0x0 0Xc300>, <0x0 0x18000000 0x0 0X200>; reg-names = "enet_csr", "ring_csr", "ring_cmd"; - interrupts = <0x0 0x60 0x4>; + interrupts = <0x0 0x60 0x4>, + <0x0 0x61 0x4>; dma-coherent; clocks = <&xge0clk 0>; /* mac address will be overwritten by the bootloader */ diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index af6a452b1aac..4e03d8dd23f6 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -31,8 +31,12 @@ CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y # CONFIG_BLK_DEV_BSG is not set # CONFIG_IOSCHED_DEADLINE is not set +CONFIG_ARCH_EXYNOS7=y CONFIG_ARCH_FSL_LS2085A=y CONFIG_ARCH_MEDIATEK=y +CONFIG_ARCH_SEATTLE=y +CONFIG_ARCH_TEGRA=y +CONFIG_ARCH_TEGRA_132_SOC=y CONFIG_ARCH_THUNDER=y CONFIG_ARCH_VEXPRESS=y CONFIG_ARCH_XGENE=y @@ -62,6 +66,7 @@ CONFIG_BPF_JIT=y # CONFIG_WIRELESS is not set CONFIG_NET_9P=y CONFIG_NET_9P_VIRTIO=y +# CONFIG_TEGRA_AHB is not set CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" CONFIG_DEVTMPFS=y CONFIG_DEVTMPFS_MOUNT=y @@ -81,6 +86,7 @@ CONFIG_NETDEVICES=y CONFIG_TUN=y CONFIG_VIRTIO_NET=y CONFIG_NET_XGENE=y +CONFIG_SKY2=y CONFIG_SMC91X=y CONFIG_SMSC911X=y # CONFIG_WLAN is not set @@ -100,6 +106,8 @@ CONFIG_SPI=y CONFIG_SPI_PL022=y CONFIG_GPIO_PL061=y CONFIG_GPIO_XGENE=y +CONFIG_POWER_RESET_XGENE=y +CONFIG_POWER_RESET_SYSCON=y # CONFIG_HWMON is not set CONFIG_REGULATOR=y CONFIG_REGULATOR_FIXED_VOLTAGE=y @@ -112,10 +120,10 @@ CONFIG_LOGO=y CONFIG_USB=y CONFIG_USB_EHCI_HCD=y CONFIG_USB_EHCI_HCD_PLATFORM=y -CONFIG_USB_ISP1760_HCD=y CONFIG_USB_OHCI_HCD=y CONFIG_USB_OHCI_HCD_PLATFORM=y CONFIG_USB_STORAGE=y +CONFIG_USB_ISP1760=y CONFIG_USB_ULPI=y CONFIG_MMC=y CONFIG_MMC_ARMMMCI=y @@ -125,6 +133,7 @@ CONFIG_MMC_SPI=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_EFI=y CONFIG_RTC_DRV_XGENE=y +CONFIG_VIRTIO_PCI=y CONFIG_VIRTIO_BALLOON=y CONFIG_VIRTIO_MMIO=y # CONFIG_IOMMU_SUPPORT is not set @@ -143,8 +152,10 @@ CONFIG_CUSE=y CONFIG_VFAT_FS=y CONFIG_TMPFS=y CONFIG_HUGETLBFS=y +CONFIG_EFIVAR_FS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y +CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y CONFIG_9P_FS=y CONFIG_NLS_CODEPAGE_437=y @@ -159,7 +170,6 @@ CONFIG_LOCKUP_DETECTOR=y # CONFIG_SCHED_DEBUG is not set # CONFIG_DEBUG_PREEMPT is not set # CONFIG_FTRACE is not set -CONFIG_KEYS=y CONFIG_SECURITY=y CONFIG_CRYPTO_ANSI_CPRNG=y CONFIG_ARM64_CRYPTO=y diff --git a/arch/arm64/crypto/aes-ce-ccm-core.S b/arch/arm64/crypto/aes-ce-ccm-core.S index 432e4841cd81..a2a7fbcacc14 100644 --- a/arch/arm64/crypto/aes-ce-ccm-core.S +++ b/arch/arm64/crypto/aes-ce-ccm-core.S @@ -101,19 +101,19 @@ ENTRY(ce_aes_ccm_final) 0: mov v4.16b, v3.16b 1: ld1 {v5.2d}, [x2], #16 /* load next round key */ aese v0.16b, v4.16b - aese v1.16b, v4.16b aesmc v0.16b, v0.16b + aese v1.16b, v4.16b aesmc v1.16b, v1.16b 2: ld1 {v3.2d}, [x2], #16 /* load next round key */ aese v0.16b, v5.16b - aese v1.16b, v5.16b aesmc v0.16b, v0.16b + aese v1.16b, v5.16b aesmc v1.16b, v1.16b 3: ld1 {v4.2d}, [x2], #16 /* load next round key */ subs w3, w3, #3 aese v0.16b, v3.16b - aese v1.16b, v3.16b aesmc v0.16b, v0.16b + aese v1.16b, v3.16b aesmc v1.16b, v1.16b bpl 1b aese v0.16b, v4.16b @@ -146,19 +146,19 @@ ENDPROC(ce_aes_ccm_final) ld1 {v5.2d}, [x10], #16 /* load 2nd round key */ 2: /* inner loop: 3 rounds, 2x interleaved */ aese v0.16b, v4.16b - aese v1.16b, v4.16b aesmc v0.16b, v0.16b + aese v1.16b, v4.16b aesmc v1.16b, v1.16b 3: ld1 {v3.2d}, [x10], #16 /* load next round key */ aese v0.16b, v5.16b - aese v1.16b, v5.16b aesmc v0.16b, v0.16b + aese v1.16b, v5.16b aesmc v1.16b, v1.16b 4: ld1 {v4.2d}, [x10], #16 /* load next round key */ subs w7, w7, #3 aese v0.16b, v3.16b - aese v1.16b, v3.16b aesmc v0.16b, v0.16b + aese v1.16b, v3.16b aesmc v1.16b, v1.16b ld1 {v5.2d}, [x10], #16 /* load next round key */ bpl 2b diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S index 685a18f731eb..78f3cfe92c08 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/arch/arm64/crypto/aes-ce.S @@ -45,18 +45,14 @@ .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3 aes\de \i0\().16b, \k\().16b - .ifnb \i1 - aes\de \i1\().16b, \k\().16b - .ifnb \i3 - aes\de \i2\().16b, \k\().16b - aes\de \i3\().16b, \k\().16b - .endif - .endif aes\mc \i0\().16b, \i0\().16b .ifnb \i1 + aes\de \i1\().16b, \k\().16b aes\mc \i1\().16b, \i1\().16b .ifnb \i3 + aes\de \i2\().16b, \k\().16b aes\mc \i2\().16b, \i2\().16b + aes\de \i3\().16b, \k\().16b aes\mc \i3\().16b, \i3\().16b .endif .endif diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c index b1b5b893eb20..05d9e16c0dfd 100644 --- a/arch/arm64/crypto/aes-glue.c +++ b/arch/arm64/crypto/aes-glue.c @@ -284,7 +284,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__ecb-aes-" MODE, .cra_driver_name = "__driver-ecb-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, @@ -302,7 +303,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__cbc-aes-" MODE, .cra_driver_name = "__driver-cbc-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, @@ -320,7 +322,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__ctr-aes-" MODE, .cra_driver_name = "__driver-ctr-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto_aes_ctx), .cra_alignmask = 7, @@ -338,7 +341,8 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "__xts-aes-" MODE, .cra_driver_name = "__driver-xts-aes-" MODE, .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_xts_ctx), .cra_alignmask = 7, diff --git a/arch/arm64/crypto/sha1-ce-core.S b/arch/arm64/crypto/sha1-ce-core.S index 09d57d98609c..033aae6d732a 100644 --- a/arch/arm64/crypto/sha1-ce-core.S +++ b/arch/arm64/crypto/sha1-ce-core.S @@ -66,8 +66,8 @@ .word 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6 /* - * void sha1_ce_transform(int blocks, u8 const *src, u32 *state, - * u8 *head, long bytes) + * void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + * int blocks) */ ENTRY(sha1_ce_transform) /* load round constants */ @@ -78,25 +78,22 @@ ENTRY(sha1_ce_transform) ld1r {k3.4s}, [x6] /* load state */ - ldr dga, [x2] - ldr dgb, [x2, #16] + ldr dga, [x0] + ldr dgb, [x0, #16] - /* load partial state (if supplied) */ - cbz x3, 0f - ld1 {v8.4s-v11.4s}, [x3] - b 1f + /* load sha1_ce_state::finalize */ + ldr w4, [x0, #:lo12:sha1_ce_offsetof_finalize] /* load input */ 0: ld1 {v8.4s-v11.4s}, [x1], #64 - sub w0, w0, #1 + sub w2, w2, #1 -1: CPU_LE( rev32 v8.16b, v8.16b ) CPU_LE( rev32 v9.16b, v9.16b ) CPU_LE( rev32 v10.16b, v10.16b ) CPU_LE( rev32 v11.16b, v11.16b ) -2: add t0.4s, v8.4s, k0.4s +1: add t0.4s, v8.4s, k0.4s mov dg0v.16b, dgav.16b add_update c, ev, k0, 8, 9, 10, 11, dgb @@ -127,15 +124,15 @@ CPU_LE( rev32 v11.16b, v11.16b ) add dgbv.2s, dgbv.2s, dg1v.2s add dgav.4s, dgav.4s, dg0v.4s - cbnz w0, 0b + cbnz w2, 0b /* * Final block: add padding and total bit count. - * Skip if we have no total byte count in x4. In that case, the input - * size was not a round multiple of the block size, and the padding is - * handled by the C code. + * Skip if the input size was not a round multiple of the block size, + * the padding is handled by the C code in that case. */ cbz x4, 3f + ldr x4, [x0, #:lo12:sha1_ce_offsetof_count] movi v9.2d, #0 mov x8, #0x80000000 movi v10.2d, #0 @@ -144,10 +141,10 @@ CPU_LE( rev32 v11.16b, v11.16b ) mov x4, #0 mov v11.d[0], xzr mov v11.d[1], x7 - b 2b + b 1b /* store new state */ -3: str dga, [x2] - str dgb, [x2, #16] +3: str dga, [x0] + str dgb, [x0, #16] ret ENDPROC(sha1_ce_transform) diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index 6fe83f37a750..114e7cc5de8c 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -12,144 +12,81 @@ #include <asm/unaligned.h> #include <crypto/internal/hash.h> #include <crypto/sha.h> +#include <crypto/sha1_base.h> #include <linux/cpufeature.h> #include <linux/crypto.h> #include <linux/module.h> +#define ASM_EXPORT(sym, val) \ + asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); + MODULE_DESCRIPTION("SHA1 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -asmlinkage void sha1_ce_transform(int blocks, u8 const *src, u32 *state, - u8 *head, long bytes); +struct sha1_ce_state { + struct sha1_state sst; + u32 finalize; +}; -static int sha1_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); +asmlinkage void sha1_ce_transform(struct sha1_ce_state *sst, u8 const *src, + int blocks); - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - return 0; -} - -static int sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha1_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - - sctx->count += len; - - if ((partial + len) >= SHA1_BLOCK_SIZE) { - int blocks; - - if (partial) { - int p = SHA1_BLOCK_SIZE - partial; + struct sha1_ce_state *sctx = shash_desc_ctx(desc); - memcpy(sctx->buffer + partial, data, p); - data += p; - len -= p; - } - - blocks = len / SHA1_BLOCK_SIZE; - len %= SHA1_BLOCK_SIZE; - - kernel_neon_begin_partial(16); - sha1_ce_transform(blocks, data, sctx->state, - partial ? sctx->buffer : NULL, 0); - kernel_neon_end(); + sctx->finalize = 0; + kernel_neon_begin_partial(16); + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_ce_transform); + kernel_neon_end(); - data += blocks * SHA1_BLOCK_SIZE; - partial = 0; - } - if (len) - memcpy(sctx->buffer + partial, data, len); return 0; } -static int sha1_final(struct shash_desc *desc, u8 *out) +static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; + struct sha1_ce_state *sctx = shash_desc_ctx(desc); + bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); - struct sha1_state *sctx = shash_desc_ctx(desc); - __be64 bits = cpu_to_be64(sctx->count << 3); - __be32 *dst = (__be32 *)out; - int i; - - u32 padlen = SHA1_BLOCK_SIZE - - ((sctx->count + sizeof(bits)) % SHA1_BLOCK_SIZE); - - sha1_update(desc, padding, padlen); - sha1_update(desc, (const u8 *)&bits, sizeof(bits)); - - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha1_state){}; - return 0; -} - -static int sha1_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int blocks; - int i; - - if (sctx->count || !len || (len % SHA1_BLOCK_SIZE)) { - sha1_update(desc, data, len); - return sha1_final(desc, out); - } + ASM_EXPORT(sha1_ce_offsetof_count, + offsetof(struct sha1_ce_state, sst.count)); + ASM_EXPORT(sha1_ce_offsetof_finalize, + offsetof(struct sha1_ce_state, finalize)); /* - * Use a fast path if the input is a multiple of 64 bytes. In - * this case, there is no need to copy data around, and we can - * perform the entire digest calculation in a single invocation - * of sha1_ce_transform() + * Allow the asm code to perform the finalization if there is no + * partial data and the input is a round multiple of the block size. */ - blocks = len / SHA1_BLOCK_SIZE; + sctx->finalize = finalize; kernel_neon_begin_partial(16); - sha1_ce_transform(blocks, data, sctx->state, NULL, len); + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_ce_transform); + if (!finalize) + sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); kernel_neon_end(); - - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha1_state){}; - return 0; + return sha1_base_finish(desc, out); } -static int sha1_export(struct shash_desc *desc, void *out) +static int sha1_ce_final(struct shash_desc *desc, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - struct sha1_state *dst = out; - - *dst = *sctx; - return 0; -} - -static int sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - struct sha1_state const *src = in; - - *sctx = *src; - return 0; + kernel_neon_begin_partial(16); + sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_ce_transform); + kernel_neon_end(); + return sha1_base_finish(desc, out); } static struct shash_alg alg = { - .init = sha1_init, - .update = sha1_update, - .final = sha1_final, - .finup = sha1_finup, - .export = sha1_export, - .import = sha1_import, - .descsize = sizeof(struct sha1_state), + .init = sha1_base_init, + .update = sha1_ce_update, + .final = sha1_ce_final, + .finup = sha1_ce_finup, + .descsize = sizeof(struct sha1_ce_state), .digestsize = SHA1_DIGEST_SIZE, - .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name = "sha1-ce", diff --git a/arch/arm64/crypto/sha2-ce-core.S b/arch/arm64/crypto/sha2-ce-core.S index 7f29fc031ea8..5df9d9d470ad 100644 --- a/arch/arm64/crypto/sha2-ce-core.S +++ b/arch/arm64/crypto/sha2-ce-core.S @@ -73,8 +73,8 @@ .word 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 /* - * void sha2_ce_transform(int blocks, u8 const *src, u32 *state, - * u8 *head, long bytes) + * void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, + * int blocks) */ ENTRY(sha2_ce_transform) /* load round constants */ @@ -85,24 +85,21 @@ ENTRY(sha2_ce_transform) ld1 {v12.4s-v15.4s}, [x8] /* load state */ - ldp dga, dgb, [x2] + ldp dga, dgb, [x0] - /* load partial input (if supplied) */ - cbz x3, 0f - ld1 {v16.4s-v19.4s}, [x3] - b 1f + /* load sha256_ce_state::finalize */ + ldr w4, [x0, #:lo12:sha256_ce_offsetof_finalize] /* load input */ 0: ld1 {v16.4s-v19.4s}, [x1], #64 - sub w0, w0, #1 + sub w2, w2, #1 -1: CPU_LE( rev32 v16.16b, v16.16b ) CPU_LE( rev32 v17.16b, v17.16b ) CPU_LE( rev32 v18.16b, v18.16b ) CPU_LE( rev32 v19.16b, v19.16b ) -2: add t0.4s, v16.4s, v0.4s +1: add t0.4s, v16.4s, v0.4s mov dg0v.16b, dgav.16b mov dg1v.16b, dgbv.16b @@ -131,15 +128,15 @@ CPU_LE( rev32 v19.16b, v19.16b ) add dgbv.4s, dgbv.4s, dg1v.4s /* handled all input blocks? */ - cbnz w0, 0b + cbnz w2, 0b /* * Final block: add padding and total bit count. - * Skip if we have no total byte count in x4. In that case, the input - * size was not a round multiple of the block size, and the padding is - * handled by the C code. + * Skip if the input size was not a round multiple of the block size, + * the padding is handled by the C code in that case. */ cbz x4, 3f + ldr x4, [x0, #:lo12:sha256_ce_offsetof_count] movi v17.2d, #0 mov x8, #0x80000000 movi v18.2d, #0 @@ -148,9 +145,9 @@ CPU_LE( rev32 v19.16b, v19.16b ) mov x4, #0 mov v19.d[0], xzr mov v19.d[1], x7 - b 2b + b 1b /* store new state */ -3: stp dga, dgb, [x2] +3: stp dga, dgb, [x0] ret ENDPROC(sha2_ce_transform) diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index ae67e88c28b9..1340e44c048b 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -12,206 +12,82 @@ #include <asm/unaligned.h> #include <crypto/internal/hash.h> #include <crypto/sha.h> +#include <crypto/sha256_base.h> #include <linux/cpufeature.h> #include <linux/crypto.h> #include <linux/module.h> +#define ASM_EXPORT(sym, val) \ + asm(".globl " #sym "; .set " #sym ", %0" :: "I"(val)); + MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash using ARMv8 Crypto Extensions"); MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); MODULE_LICENSE("GPL v2"); -asmlinkage int sha2_ce_transform(int blocks, u8 const *src, u32 *state, - u8 *head, long bytes); - -static int sha224_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha256_state){ - .state = { - SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, - SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, - } - }; - return 0; -} - -static int sha256_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha256_state){ - .state = { - SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, - } - }; - return 0; -} - -static int sha2_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->count += len; - - if ((partial + len) >= SHA256_BLOCK_SIZE) { - int blocks; - - if (partial) { - int p = SHA256_BLOCK_SIZE - partial; - - memcpy(sctx->buf + partial, data, p); - data += p; - len -= p; - } +struct sha256_ce_state { + struct sha256_state sst; + u32 finalize; +}; - blocks = len / SHA256_BLOCK_SIZE; - len %= SHA256_BLOCK_SIZE; +asmlinkage void sha2_ce_transform(struct sha256_ce_state *sst, u8 const *src, + int blocks); - kernel_neon_begin_partial(28); - sha2_ce_transform(blocks, data, sctx->state, - partial ? sctx->buf : NULL, 0); - kernel_neon_end(); - - data += blocks * SHA256_BLOCK_SIZE; - partial = 0; - } - if (len) - memcpy(sctx->buf + partial, data, len); - return 0; -} - -static void sha2_final(struct shash_desc *desc) +static int sha256_ce_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { - static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; - - struct sha256_state *sctx = shash_desc_ctx(desc); - __be64 bits = cpu_to_be64(sctx->count << 3); - u32 padlen = SHA256_BLOCK_SIZE - - ((sctx->count + sizeof(bits)) % SHA256_BLOCK_SIZE); - - sha2_update(desc, padding, padlen); - sha2_update(desc, (const u8 *)&bits, sizeof(bits)); -} - -static int sha224_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_final(desc); - - for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha256_state){}; - return 0; -} + struct sha256_ce_state *sctx = shash_desc_ctx(desc); -static int sha256_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_final(desc); - - for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); + sctx->finalize = 0; + kernel_neon_begin_partial(28); + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha2_ce_transform); + kernel_neon_end(); - *sctx = (struct sha256_state){}; return 0; } -static void sha2_finup(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - int blocks; + struct sha256_ce_state *sctx = shash_desc_ctx(desc); + bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); - if (sctx->count || !len || (len % SHA256_BLOCK_SIZE)) { - sha2_update(desc, data, len); - sha2_final(desc); - return; - } + ASM_EXPORT(sha256_ce_offsetof_count, + offsetof(struct sha256_ce_state, sst.count)); + ASM_EXPORT(sha256_ce_offsetof_finalize, + offsetof(struct sha256_ce_state, finalize)); /* - * Use a fast path if the input is a multiple of 64 bytes. In - * this case, there is no need to copy data around, and we can - * perform the entire digest calculation in a single invocation - * of sha2_ce_transform() + * Allow the asm code to perform the finalization if there is no + * partial data and the input is a round multiple of the block size. */ - blocks = len / SHA256_BLOCK_SIZE; + sctx->finalize = finalize; kernel_neon_begin_partial(28); - sha2_ce_transform(blocks, data, sctx->state, NULL, len); + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha2_ce_transform); + if (!finalize) + sha256_base_do_finalize(desc, + (sha256_block_fn *)sha2_ce_transform); kernel_neon_end(); + return sha256_base_finish(desc, out); } -static int sha224_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) +static int sha256_ce_final(struct shash_desc *desc, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_finup(desc, data, len); - - for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha256_state){}; - return 0; -} - -static int sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - int i; - - sha2_finup(desc, data, len); - - for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(__be32); i++) - put_unaligned_be32(sctx->state[i], dst++); - - *sctx = (struct sha256_state){}; - return 0; -} - -static int sha2_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - struct sha256_state *dst = out; - - *dst = *sctx; - return 0; -} - -static int sha2_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - struct sha256_state const *src = in; - - *sctx = *src; - return 0; + kernel_neon_begin_partial(28); + sha256_base_do_finalize(desc, (sha256_block_fn *)sha2_ce_transform); + kernel_neon_end(); + return sha256_base_finish(desc, out); } static struct shash_alg algs[] = { { - .init = sha224_init, - .update = sha2_update, - .final = sha224_final, - .finup = sha224_finup, - .export = sha2_export, - .import = sha2_import, - .descsize = sizeof(struct sha256_state), + .init = sha224_base_init, + .update = sha256_ce_update, + .final = sha256_ce_final, + .finup = sha256_ce_finup, + .descsize = sizeof(struct sha256_ce_state), .digestsize = SHA224_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ce", @@ -221,15 +97,12 @@ static struct shash_alg algs[] = { { .cra_module = THIS_MODULE, } }, { - .init = sha256_init, - .update = sha2_update, - .final = sha256_final, - .finup = sha256_finup, - .export = sha2_export, - .import = sha2_import, - .descsize = sizeof(struct sha256_state), + .init = sha256_base_init, + .update = sha256_ce_update, + .final = sha256_ce_final, + .finup = sha256_ce_finup, + .descsize = sizeof(struct sha256_ce_state), .digestsize = SHA256_DIGEST_SIZE, - .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ce", diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 750bac4e637e..144b64ad96c3 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -159,4 +159,52 @@ lr .req x30 // link register orr \rd, \lbits, \hbits, lsl #32 .endm +/* + * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where + * <symbol> is within the range +/- 4 GB of the PC. + */ + /* + * @dst: destination register (64 bit wide) + * @sym: name of the symbol + * @tmp: optional scratch register to be used if <dst> == sp, which + * is not allowed in an adrp instruction + */ + .macro adr_l, dst, sym, tmp= + .ifb \tmp + adrp \dst, \sym + add \dst, \dst, :lo12:\sym + .else + adrp \tmp, \sym + add \dst, \tmp, :lo12:\sym + .endif + .endm + + /* + * @dst: destination register (32 or 64 bit wide) + * @sym: name of the symbol + * @tmp: optional 64-bit scratch register to be used if <dst> is a + * 32-bit wide register, in which case it cannot be used to hold + * the address + */ + .macro ldr_l, dst, sym, tmp= + .ifb \tmp + adrp \dst, \sym + ldr \dst, [\dst, :lo12:\sym] + .else + adrp \tmp, \sym + ldr \dst, [\tmp, :lo12:\sym] + .endif + .endm + + /* + * @src: source register (32 or 64 bit wide) + * @sym: name of the symbol + * @tmp: mandatory 64-bit scratch register to calculate the address + * while <src> needs to be preserved. + */ + .macro str_l, src, sym, tmp + adrp \tmp, \sym + str \src, [\tmp, :lo12:\sym] + .endm + #endif /* __ASM_ASSEMBLER_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index b6c16d5f622f..82cb9f98ba1a 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -23,11 +23,24 @@ #define ARM64_WORKAROUND_CLEAN_CACHE 0 #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 +#define ARM64_WORKAROUND_845719 2 -#define ARM64_NCAPS 2 +#define ARM64_NCAPS 3 #ifndef __ASSEMBLY__ +struct arm64_cpu_capabilities { + const char *desc; + u16 capability; + bool (*matches)(const struct arm64_cpu_capabilities *); + union { + struct { /* To be used for erratum handling only */ + u32 midr_model; + u32 midr_range_min, midr_range_max; + }; + }; +}; + extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); static inline bool cpu_have_feature(unsigned int num) @@ -51,7 +64,10 @@ static inline void cpus_set_cap(unsigned int num) __set_bit(num, cpu_hwcaps); } +void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, + const char *info); void check_local_cpu_errata(void); +void check_local_cpu_features(void); bool cpu_supports_mixed_endian_el0(void); bool system_supports_mixed_endian_el0(void); diff --git a/arch/arm64/include/asm/cputable.h b/arch/arm64/include/asm/cputable.h deleted file mode 100644 index e3bd983d3661..000000000000 --- a/arch/arm64/include/asm/cputable.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * arch/arm64/include/asm/cputable.h - * - * Copyright (C) 2012 ARM Ltd. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - */ -#ifndef __ASM_CPUTABLE_H -#define __ASM_CPUTABLE_H - -struct cpu_info { - unsigned int cpu_id_val; - unsigned int cpu_id_mask; - const char *cpu_name; - unsigned long (*cpu_setup)(void); -}; - -extern struct cpu_info *lookup_processor_type(unsigned int); - -#endif diff --git a/arch/arm64/include/asm/dma-mapping.h b/arch/arm64/include/asm/dma-mapping.h index 6932bb57dba0..9437e3dc5833 100644 --- a/arch/arm64/include/asm/dma-mapping.h +++ b/arch/arm64/include/asm/dma-mapping.h @@ -97,7 +97,7 @@ static inline int dma_set_mask(struct device *dev, u64 mask) static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { if (!dev->dma_mask) - return 0; + return false; return addr + size - 1 <= *dev->dma_mask; } diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index defa0ff98250..926495686554 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -33,6 +33,7 @@ enum fixed_addresses { FIX_HOLE, FIX_EARLYCON_MEM_BASE, + FIX_TEXT_POKE0, __end_of_permanent_fixed_addresses, /* @@ -49,7 +50,6 @@ enum fixed_addresses { FIX_BTMAP_END = __end_of_permanent_fixed_addresses, FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1, - FIX_TEXT_POKE0, __end_of_fixed_addresses }; diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index d2f49423c5dc..f81b328d9cf4 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -285,6 +285,7 @@ bool aarch64_insn_is_nop(u32 insn); int aarch64_insn_read(void *addr, u32 *insnp); int aarch64_insn_write(void *addr, u32 insn); enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); +u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn); u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, u32 insn, u64 imm); u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr, diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 36250705dc4c..61505676d085 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -68,6 +68,8 @@ #include <asm/pgalloc.h> #include <asm/cachetype.h> #include <asm/cacheflush.h> +#include <asm/mmu_context.h> +#include <asm/pgtable.h> #define KERN_TO_HYP(kva) ((unsigned long)kva - PAGE_OFFSET + HYP_PAGE_OFFSET) @@ -269,5 +271,36 @@ static inline void __kvm_flush_dcache_pud(pud_t pud) void kvm_set_way_flush(struct kvm_vcpu *vcpu); void kvm_toggle_cache(struct kvm_vcpu *vcpu, bool was_enabled); +static inline bool __kvm_cpu_uses_extended_idmap(void) +{ + return __cpu_uses_extended_idmap(); +} + +static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd, + pgd_t *hyp_pgd, + pgd_t *merged_hyp_pgd, + unsigned long hyp_idmap_start) +{ + int idmap_idx; + + /* + * Use the first entry to access the HYP mappings. It is + * guaranteed to be free, otherwise we wouldn't use an + * extended idmap. + */ + VM_BUG_ON(pgd_val(merged_hyp_pgd[0])); + merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE); + + /* + * Create another extended level entry that points to the boot HYP map, + * which contains an ID mapping of the HYP init code. We essentially + * merge the boot and runtime HYP maps by doing so, but they don't + * overlap anyway, so this is fine. + */ + idmap_idx = hyp_idmap_start >> VA_BITS; + VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx])); + merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE); +} + #endif /* __ASSEMBLY__ */ #endif /* __ARM64_KVM_MMU_H__ */ diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index 101a42bde728..8ec41e5f56f0 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -64,6 +64,49 @@ static inline void cpu_set_reserved_ttbr0(void) : "r" (ttbr)); } +/* + * TCR.T0SZ value to use when the ID map is active. Usually equals + * TCR_T0SZ(VA_BITS), unless system RAM is positioned very high in + * physical memory, in which case it will be smaller. + */ +extern u64 idmap_t0sz; + +static inline bool __cpu_uses_extended_idmap(void) +{ + return (!IS_ENABLED(CONFIG_ARM64_VA_BITS_48) && + unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS))); +} + +static inline void __cpu_set_tcr_t0sz(u64 t0sz) +{ + unsigned long tcr; + + if (__cpu_uses_extended_idmap()) + asm volatile ( + " mrs %0, tcr_el1 ;" + " bfi %0, %1, %2, %3 ;" + " msr tcr_el1, %0 ;" + " isb" + : "=&r" (tcr) + : "r"(t0sz), "I"(TCR_T0SZ_OFFSET), "I"(TCR_TxSZ_WIDTH)); +} + +/* + * Set TCR.T0SZ to the value appropriate for activating the identity map. + */ +static inline void cpu_set_idmap_tcr_t0sz(void) +{ + __cpu_set_tcr_t0sz(idmap_t0sz); +} + +/* + * Set TCR.T0SZ to its default value (based on VA_BITS) + */ +static inline void cpu_set_default_tcr_t0sz(void) +{ + __cpu_set_tcr_t0sz(TCR_T0SZ(VA_BITS)); +} + static inline void switch_new_context(struct mm_struct *mm) { unsigned long flags; diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 8fc8fa280e92..7d9c7e4a424b 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -33,7 +33,9 @@ * image. Both require pgd, pud (4 levels only) and pmd tables to (section) * map the kernel. With the 64K page configuration, swapper and idmap need to * map to pte level. The swapper also maps the FDT (see __create_page_tables - * for more information). + * for more information). Note that the number of ID map translation levels + * could be increased on the fly if system RAM is out of reach for the default + * VA range, so 3 pages are reserved in all cases. */ #ifdef CONFIG_ARM64_64K_PAGES #define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS) @@ -42,7 +44,7 @@ #endif #define SWAPPER_DIR_SIZE (SWAPPER_PGTABLE_LEVELS * PAGE_SIZE) -#define IDMAP_DIR_SIZE (SWAPPER_DIR_SIZE) +#define IDMAP_DIR_SIZE (3 * PAGE_SIZE) #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index 80f3d241cff8..59bfae75dc98 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -143,7 +143,12 @@ /* * TCR flags. */ -#define TCR_TxSZ(x) (((UL(64) - (x)) << 16) | ((UL(64) - (x)) << 0)) +#define TCR_T0SZ_OFFSET 0 +#define TCR_T1SZ_OFFSET 16 +#define TCR_T0SZ(x) ((UL(64) - (x)) << TCR_T0SZ_OFFSET) +#define TCR_T1SZ(x) ((UL(64) - (x)) << TCR_T1SZ_OFFSET) +#define TCR_TxSZ(x) (TCR_T0SZ(x) | TCR_T1SZ(x)) +#define TCR_TxSZ_WIDTH 6 #define TCR_IRGN_NC ((UL(0) << 8) | (UL(0) << 24)) #define TCR_IRGN_WBWA ((UL(1) << 8) | (UL(1) << 24)) #define TCR_IRGN_WT ((UL(2) << 8) | (UL(2) << 24)) diff --git a/arch/arm64/include/asm/pmu.h b/arch/arm64/include/asm/pmu.h index e6f087806aaf..b7710a59672c 100644 --- a/arch/arm64/include/asm/pmu.h +++ b/arch/arm64/include/asm/pmu.h @@ -44,6 +44,7 @@ struct pmu_hw_events { struct arm_pmu { struct pmu pmu; cpumask_t active_irqs; + int *irq_affinity; const char *name; irqreturn_t (*handle_irq)(int irq_num, void *dev); void (*enable)(struct hw_perf_event *evt, int idx); diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h index 941c375616e2..220633b791b8 100644 --- a/arch/arm64/include/asm/proc-fns.h +++ b/arch/arm64/include/asm/proc-fns.h @@ -45,15 +45,6 @@ do { \ cpu_do_switch_mm(virt_to_phys(pgd),mm); \ } while (0) -#define cpu_get_pgd() \ -({ \ - unsigned long pg; \ - asm("mrs %0, ttbr0_el1\n" \ - : "=r" (pg)); \ - pg &= ~0xffff000000003ffful; \ - (pgd_t *)phys_to_virt(pg); \ -}) - #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* __ASM_PROCFNS_H */ diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 20e9591a60cf..d2c37a1df0eb 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -127,7 +127,11 @@ extern void release_thread(struct task_struct *); unsigned long get_wchan(struct task_struct *p); -#define cpu_relax() barrier() +static inline void cpu_relax(void) +{ + asm volatile("yield" ::: "memory"); +} + #define cpu_relax_lowlatency() cpu_relax() /* Thread switching */ diff --git a/arch/arm64/include/asm/smp_plat.h b/arch/arm64/include/asm/smp_plat.h index 59e282311b58..8dcd61e32176 100644 --- a/arch/arm64/include/asm/smp_plat.h +++ b/arch/arm64/include/asm/smp_plat.h @@ -40,4 +40,6 @@ static inline u32 mpidr_hash_size(void) extern u64 __cpu_logical_map[NR_CPUS]; #define cpu_logical_map(cpu) __cpu_logical_map[cpu] +void __init do_post_cpus_up_work(void); + #endif /* __ASM_SMP_PLAT_H */ diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h index 702e1e6a0d80..dcd06d18a42a 100644 --- a/arch/arm64/include/asm/thread_info.h +++ b/arch/arm64/include/asm/thread_info.h @@ -33,7 +33,6 @@ #ifndef __ASSEMBLY__ struct task_struct; -struct exec_domain; #include <asm/types.h> @@ -47,7 +46,6 @@ struct thread_info { unsigned long flags; /* low level flags */ mm_segment_t addr_limit; /* address limit */ struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ int preempt_count; /* 0 => preemptable, <0 => bug */ int cpu; /* cpu */ }; @@ -55,7 +53,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index 27224426e0bf..cef934a90f17 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -406,7 +406,7 @@ __SYSCALL(__NR_vfork, sys_vfork) #define __NR_ugetrlimit 191 /* SuS compliant getrlimit */ __SYSCALL(__NR_ugetrlimit, compat_sys_getrlimit) /* SuS compliant getrlimit */ #define __NR_mmap2 192 -__SYSCALL(__NR_mmap2, sys_mmap_pgoff) +__SYSCALL(__NR_mmap2, compat_sys_mmap2_wrapper) #define __NR_truncate64 193 __SYSCALL(__NR_truncate64, compat_sys_truncate64_wrapper) #define __NR_ftruncate64 194 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 5ee07eee80c2..b12e15b80516 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -12,12 +12,12 @@ CFLAGS_REMOVE_insn.o = -pg CFLAGS_REMOVE_return_address.o = -pg # Object file lists. -arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \ +arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ entry-fpsimd.o process.o ptrace.o setup.o signal.o \ sys.o stacktrace.o time.o traps.o io.o vdso.o \ hyp-stub.o psci.o psci-call.o cpu_ops.o insn.o \ return_address.o cpuinfo.o cpu_errata.o \ - alternative.o cacheinfo.o + cpufeature.o alternative.o cacheinfo.o arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \ sys_compat.o entry32.o \ diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index ad7821d64a1d..21033bba9390 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -24,6 +24,7 @@ #include <asm/cacheflush.h> #include <asm/alternative.h> #include <asm/cpufeature.h> +#include <asm/insn.h> #include <linux/stop_machine.h> extern struct alt_instr __alt_instructions[], __alt_instructions_end[]; @@ -33,6 +34,48 @@ struct alt_region { struct alt_instr *end; }; +/* + * Decode the imm field of a b/bl instruction, and return the byte + * offset as a signed value (so it can be used when computing a new + * branch target). + */ +static s32 get_branch_offset(u32 insn) +{ + s32 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn); + + /* sign-extend the immediate before turning it into a byte offset */ + return (imm << 6) >> 4; +} + +static u32 get_alt_insn(u8 *insnptr, u8 *altinsnptr) +{ + u32 insn; + + aarch64_insn_read(altinsnptr, &insn); + + /* Stop the world on instructions we don't support... */ + BUG_ON(aarch64_insn_is_cbz(insn)); + BUG_ON(aarch64_insn_is_cbnz(insn)); + BUG_ON(aarch64_insn_is_bcond(insn)); + /* ... and there is probably more. */ + + if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) { + enum aarch64_insn_branch_type type; + unsigned long target; + + if (aarch64_insn_is_b(insn)) + type = AARCH64_INSN_BRANCH_NOLINK; + else + type = AARCH64_INSN_BRANCH_LINK; + + target = (unsigned long)altinsnptr + get_branch_offset(insn); + insn = aarch64_insn_gen_branch_imm((unsigned long)insnptr, + target, type); + } + + return insn; +} + static int __apply_alternatives(void *alt_region) { struct alt_instr *alt; @@ -40,16 +83,24 @@ static int __apply_alternatives(void *alt_region) u8 *origptr, *replptr; for (alt = region->begin; alt < region->end; alt++) { + u32 insn; + int i; + if (!cpus_have_cap(alt->cpufeature)) continue; - BUG_ON(alt->alt_len > alt->orig_len); + BUG_ON(alt->alt_len != alt->orig_len); pr_info_once("patching kernel code\n"); origptr = (u8 *)&alt->orig_offset + alt->orig_offset; replptr = (u8 *)&alt->alt_offset + alt->alt_offset; - memcpy(origptr, replptr, alt->alt_len); + + for (i = 0; i < alt->alt_len; i += sizeof(insn)) { + insn = get_alt_insn(origptr + i, replptr + i); + aarch64_insn_write(origptr + i, insn); + } + flush_icache_range((uintptr_t)origptr, (uintptr_t)(origptr + alt->alt_len)); } diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index f7fa65d4c352..da675cc5dfae 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -24,7 +24,6 @@ #include <linux/kvm_host.h> #include <asm/thread_info.h> #include <asm/memory.h> -#include <asm/cputable.h> #include <asm/smp_plat.h> #include <asm/suspend.h> #include <asm/vdso_datapage.h> @@ -38,7 +37,6 @@ int main(void) DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); BLANK(); DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context)); @@ -71,9 +69,6 @@ int main(void) BLANK(); DEFINE(PAGE_SZ, PAGE_SIZE); BLANK(); - DEFINE(CPU_INFO_SZ, sizeof(struct cpu_info)); - DEFINE(CPU_INFO_SETUP, offsetof(struct cpu_info, cpu_setup)); - BLANK(); DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL); DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index fa62637e63a8..6ffd91438560 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -16,8 +16,6 @@ * along with this program. If not, see <http://www.gnu.org/licenses/>. */ -#define pr_fmt(fmt) "alternatives: " fmt - #include <linux/types.h> #include <asm/cpu.h> #include <asm/cputype.h> @@ -26,27 +24,11 @@ #define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) -/* - * Add a struct or another datatype to the union below if you need - * different means to detect an affected CPU. - */ -struct arm64_cpu_capabilities { - const char *desc; - u16 capability; - bool (*is_affected)(struct arm64_cpu_capabilities *); - union { - struct { - u32 midr_model; - u32 midr_range_min, midr_range_max; - }; - }; -}; - #define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \ MIDR_ARCHITECTURE_MASK) static bool __maybe_unused -is_affected_midr_range(struct arm64_cpu_capabilities *entry) +is_affected_midr_range(const struct arm64_cpu_capabilities *entry) { u32 midr = read_cpuid_id(); @@ -59,12 +41,12 @@ is_affected_midr_range(struct arm64_cpu_capabilities *entry) } #define MIDR_RANGE(model, min, max) \ - .is_affected = is_affected_midr_range, \ + .matches = is_affected_midr_range, \ .midr_model = model, \ .midr_range_min = min, \ .midr_range_max = max -struct arm64_cpu_capabilities arm64_errata[] = { +const struct arm64_cpu_capabilities arm64_errata[] = { #if defined(CONFIG_ARM64_ERRATUM_826319) || \ defined(CONFIG_ARM64_ERRATUM_827319) || \ defined(CONFIG_ARM64_ERRATUM_824069) @@ -88,7 +70,16 @@ struct arm64_cpu_capabilities arm64_errata[] = { /* Cortex-A57 r0p0 - r1p2 */ .desc = "ARM erratum 832075", .capability = ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE, - MIDR_RANGE(MIDR_CORTEX_A57, 0x00, 0x12), + MIDR_RANGE(MIDR_CORTEX_A57, 0x00, + (1 << MIDR_VARIANT_SHIFT) | 2), + }, +#endif +#ifdef CONFIG_ARM64_ERRATUM_845719 + { + /* Cortex-A53 r0p[01234] */ + .desc = "ARM erratum 845719", + .capability = ARM64_WORKAROUND_845719, + MIDR_RANGE(MIDR_CORTEX_A53, 0x00, 0x04), }, #endif { @@ -97,15 +88,5 @@ struct arm64_cpu_capabilities arm64_errata[] = { void check_local_cpu_errata(void) { - struct arm64_cpu_capabilities *cpus = arm64_errata; - int i; - - for (i = 0; cpus[i].desc; i++) { - if (!cpus[i].is_affected(&cpus[i])) - continue; - - if (!cpus_have_cap(cpus[i].capability)) - pr_info("enabling workaround for %s\n", cpus[i].desc); - cpus_set_cap(cpus[i].capability); - } + check_cpu_capabilities(arm64_errata, "enabling workaround for"); } diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c new file mode 100644 index 000000000000..3d9967e43d89 --- /dev/null +++ b/arch/arm64/kernel/cpufeature.c @@ -0,0 +1,47 @@ +/* + * Contains CPU feature definitions + * + * Copyright (C) 2015 ARM Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#define pr_fmt(fmt) "alternatives: " fmt + +#include <linux/types.h> +#include <asm/cpu.h> +#include <asm/cpufeature.h> + +static const struct arm64_cpu_capabilities arm64_features[] = { + {}, +}; + +void check_cpu_capabilities(const struct arm64_cpu_capabilities *caps, + const char *info) +{ + int i; + + for (i = 0; caps[i].desc; i++) { + if (!caps[i].matches(&caps[i])) + continue; + + if (!cpus_have_cap(caps[i].capability)) + pr_info("%s %s\n", info, caps[i].desc); + cpus_set_cap(caps[i].capability); + } +} + +void check_local_cpu_features(void) +{ + check_cpu_capabilities(arm64_features, "detected feature"); +} diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 929855691dae..75d5a867e7fb 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -236,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) cpuinfo_detect_icache_policy(info); check_local_cpu_errata(); + check_local_cpu_features(); update_cpu_features(info); } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index cf21bb3bf752..959fe8733560 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -21,8 +21,10 @@ #include <linux/init.h> #include <linux/linkage.h> +#include <asm/alternative-asm.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> +#include <asm/cpufeature.h> #include <asm/errno.h> #include <asm/esr.h> #include <asm/thread_info.h> @@ -120,6 +122,24 @@ ct_user_enter ldr x23, [sp, #S_SP] // load return stack pointer msr sp_el0, x23 + +#ifdef CONFIG_ARM64_ERRATUM_845719 + alternative_insn \ + "nop", \ + "tbz x22, #4, 1f", \ + ARM64_WORKAROUND_845719 +#ifdef CONFIG_PID_IN_CONTEXTIDR + alternative_insn \ + "nop; nop", \ + "mrs x29, contextidr_el1; msr contextidr_el1, x29; 1:", \ + ARM64_WORKAROUND_845719 +#else + alternative_insn \ + "nop", \ + "msr contextidr_el1, xzr; 1:", \ + ARM64_WORKAROUND_845719 +#endif +#endif .endif msr elr_el1, x21 // set up the return data msr spsr_el1, x22 diff --git a/arch/arm64/kernel/entry32.S b/arch/arm64/kernel/entry32.S index 9a8f6ae2530e..bd9bfaa9269b 100644 --- a/arch/arm64/kernel/entry32.S +++ b/arch/arm64/kernel/entry32.S @@ -19,9 +19,12 @@ */ #include <linux/linkage.h> +#include <linux/const.h> #include <asm/assembler.h> #include <asm/asm-offsets.h> +#include <asm/errno.h> +#include <asm/page.h> /* * System call wrappers for the AArch32 compatibility layer. @@ -54,6 +57,21 @@ ENTRY(compat_sys_fstatfs64_wrapper) ENDPROC(compat_sys_fstatfs64_wrapper) /* + * Note: off_4k (w5) is always in units of 4K. If we can't do the + * requested offset because it is not page-aligned, we return -EINVAL. + */ +ENTRY(compat_sys_mmap2_wrapper) +#if PAGE_SHIFT > 12 + tst w5, #~PAGE_MASK >> 12 + b.ne 1f + lsr w5, w5, #PAGE_SHIFT - 12 +#endif + b sys_mmap_pgoff +1: mov x0, #-EINVAL + ret +ENDPROC(compat_sys_mmap2_wrapper) + +/* * Wrappers for AArch32 syscalls that either take 64-bit parameters * in registers or that take 32-bit parameters which require sign * extension. diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 07f930540f4a..19f915e8f6e0 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -36,7 +36,7 @@ #include <asm/page.h> #include <asm/virt.h> -#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) +#define __PHYS_OFFSET (KERNEL_START - TEXT_OFFSET) #if (TEXT_OFFSET & 0xfff) != 0 #error TEXT_OFFSET must be at least 4KB aligned @@ -46,13 +46,6 @@ #error TEXT_OFFSET must be less than 2MB #endif - .macro pgtbl, ttb0, ttb1, virt_to_phys - ldr \ttb1, =swapper_pg_dir - ldr \ttb0, =idmap_pg_dir - add \ttb1, \ttb1, \virt_to_phys - add \ttb0, \ttb0, \virt_to_phys - .endm - #ifdef CONFIG_ARM64_64K_PAGES #define BLOCK_SHIFT PAGE_SHIFT #define BLOCK_SIZE PAGE_SIZE @@ -63,7 +56,7 @@ #define TABLE_SHIFT PUD_SHIFT #endif -#define KERNEL_START KERNEL_RAM_VADDR +#define KERNEL_START _text #define KERNEL_END _end /* @@ -240,40 +233,43 @@ section_table: #endif ENTRY(stext) - mov x21, x0 // x21=FDT + bl preserve_boot_args bl el2_setup // Drop to EL1, w20=cpu_boot_mode - bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET + adrp x24, __PHYS_OFFSET bl set_cpu_boot_mode_flag - mrs x22, midr_el1 // x22=cpuid - mov x0, x22 - bl lookup_processor_type - mov x23, x0 // x23=current cpu_table - /* - * __error_p may end up out of range for cbz if text areas are - * aligned up to section sizes. - */ - cbnz x23, 1f // invalid processor (x23=0)? - b __error_p -1: + bl __vet_fdt bl __create_page_tables // x25=TTBR0, x26=TTBR1 /* - * The following calls CPU specific code in a position independent - * manner. See arch/arm64/mm/proc.S for details. x23 = base of - * cpu_info structure selected by lookup_processor_type above. + * The following calls CPU setup code, see arch/arm64/mm/proc.S for + * details. * On return, the CPU will be ready for the MMU to be turned on and * the TCR will have been set. */ - ldr x27, __switch_data // address to jump to after + ldr x27, =__mmap_switched // address to jump to after // MMU has been enabled - adrp lr, __enable_mmu // return (PIC) address - add lr, lr, #:lo12:__enable_mmu - ldr x12, [x23, #CPU_INFO_SETUP] - add x12, x12, x28 // __virt_to_phys - br x12 // initialise processor + adr_l lr, __enable_mmu // return (PIC) address + b __cpu_setup // initialise processor ENDPROC(stext) /* + * Preserve the arguments passed by the bootloader in x0 .. x3 + */ +preserve_boot_args: + mov x21, x0 // x21=FDT + + adr_l x0, boot_args // record the contents of + stp x21, x1, [x0] // x0 .. x3 at kernel entry + stp x2, x3, [x0, #16] + + dmb sy // needed before dc ivac with + // MMU off + + add x1, x0, #0x20 // 4 x 8 bytes + b __inval_cache_range // tail call +ENDPROC(preserve_boot_args) + +/* * Determine validity of the x21 FDT pointer. * The dtb must be 8-byte aligned and live in the first 512M of memory. */ @@ -356,7 +352,8 @@ ENDPROC(__vet_fdt) * - pgd entry for fixed mappings (TTBR1) */ __create_page_tables: - pgtbl x25, x26, x28 // idmap_pg_dir and swapper_pg_dir addresses + adrp x25, idmap_pg_dir + adrp x26, swapper_pg_dir mov x27, lr /* @@ -385,12 +382,50 @@ __create_page_tables: * Create the identity mapping. */ mov x0, x25 // idmap_pg_dir - ldr x3, =KERNEL_START - add x3, x3, x28 // __pa(KERNEL_START) + adrp x3, KERNEL_START // __pa(KERNEL_START) + +#ifndef CONFIG_ARM64_VA_BITS_48 +#define EXTRA_SHIFT (PGDIR_SHIFT + PAGE_SHIFT - 3) +#define EXTRA_PTRS (1 << (48 - EXTRA_SHIFT)) + + /* + * If VA_BITS < 48, it may be too small to allow for an ID mapping to be + * created that covers system RAM if that is located sufficiently high + * in the physical address space. So for the ID map, use an extended + * virtual range in that case, by configuring an additional translation + * level. + * First, we have to verify our assumption that the current value of + * VA_BITS was chosen such that all translation levels are fully + * utilised, and that lowering T0SZ will always result in an additional + * translation level to be configured. + */ +#if VA_BITS != EXTRA_SHIFT +#error "Mismatch between VA_BITS and page size/number of translation levels" +#endif + + /* + * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the + * entire kernel image can be ID mapped. As T0SZ == (64 - #bits used), + * this number conveniently equals the number of leading zeroes in + * the physical address of KERNEL_END. + */ + adrp x5, KERNEL_END + clz x5, x5 + cmp x5, TCR_T0SZ(VA_BITS) // default T0SZ small enough? + b.ge 1f // .. then skip additional level + + adr_l x6, idmap_t0sz + str x5, [x6] + dmb sy + dc ivac, x6 // Invalidate potentially stale cache line + + create_table_entry x0, x3, EXTRA_SHIFT, EXTRA_PTRS, x5, x6 +1: +#endif + create_pgd_entry x0, x3, x5, x6 - ldr x6, =KERNEL_END mov x5, x3 // __pa(KERNEL_START) - add x6, x6, x28 // __pa(KERNEL_END) + adr_l x6, KERNEL_END // __pa(KERNEL_END) create_block_map x0, x7, x3, x5, x6 /* @@ -399,7 +434,7 @@ __create_page_tables: mov x0, x26 // swapper_pg_dir mov x5, #PAGE_OFFSET create_pgd_entry x0, x5, x3, x6 - ldr x6, =KERNEL_END + ldr x6, =KERNEL_END // __va(KERNEL_END) mov x3, x24 // phys offset create_block_map x0, x7, x3, x5, x6 @@ -426,6 +461,7 @@ __create_page_tables: */ mov x0, x25 add x1, x26, #SWAPPER_DIR_SIZE + dmb sy bl __inval_cache_range mov lr, x27 @@ -433,37 +469,22 @@ __create_page_tables: ENDPROC(__create_page_tables) .ltorg - .align 3 - .type __switch_data, %object -__switch_data: - .quad __mmap_switched - .quad __bss_start // x6 - .quad __bss_stop // x7 - .quad processor_id // x4 - .quad __fdt_pointer // x5 - .quad memstart_addr // x6 - .quad init_thread_union + THREAD_START_SP // sp - /* - * The following fragment of code is executed with the MMU on in MMU mode, and - * uses absolute addresses; this is not position independent. + * The following fragment of code is executed with the MMU enabled. */ + .set initial_sp, init_thread_union + THREAD_START_SP __mmap_switched: - adr x3, __switch_data + 8 + adr_l x6, __bss_start + adr_l x7, __bss_stop - ldp x6, x7, [x3], #16 1: cmp x6, x7 b.hs 2f str xzr, [x6], #8 // Clear BSS b 1b 2: - ldp x4, x5, [x3], #16 - ldr x6, [x3], #8 - ldr x16, [x3] - mov sp, x16 - str x22, [x4] // Save processor ID - str x21, [x5] // Save FDT pointer - str x24, [x6] // Save PHYS_OFFSET + adr_l sp, initial_sp, x4 + str_l x21, __fdt_pointer, x5 // Save FDT pointer + str_l x24, memstart_addr, x6 // Save PHYS_OFFSET mov x29, #0 b start_kernel ENDPROC(__mmap_switched) @@ -566,8 +587,7 @@ ENDPROC(el2_setup) * in x20. See arch/arm64/include/asm/virt.h for more info. */ ENTRY(set_cpu_boot_mode_flag) - ldr x1, =__boot_cpu_mode // Compute __boot_cpu_mode - add x1, x1, x28 + adr_l x1, __boot_cpu_mode cmp w20, #BOOT_CPU_MODE_EL2 b.ne 1f add x1, x1, #4 @@ -588,29 +608,21 @@ ENDPROC(set_cpu_boot_mode_flag) .align L1_CACHE_SHIFT ENTRY(__boot_cpu_mode) .long BOOT_CPU_MODE_EL2 - .long 0 + .long BOOT_CPU_MODE_EL1 .popsection #ifdef CONFIG_SMP - .align 3 -1: .quad . - .quad secondary_holding_pen_release - /* * This provides a "holding pen" for platforms to hold all secondary * cores are held until we're ready for them to initialise. */ ENTRY(secondary_holding_pen) bl el2_setup // Drop to EL1, w20=cpu_boot_mode - bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET bl set_cpu_boot_mode_flag mrs x0, mpidr_el1 ldr x1, =MPIDR_HWID_BITMASK and x0, x0, x1 - adr x1, 1b - ldp x2, x3, [x1] - sub x1, x1, x2 - add x3, x3, x1 + adr_l x3, secondary_holding_pen_release pen: ldr x4, [x3] cmp x4, x0 b.eq secondary_startup @@ -624,7 +636,6 @@ ENDPROC(secondary_holding_pen) */ ENTRY(secondary_entry) bl el2_setup // Drop to EL1 - bl __calc_phys_offset // x24=PHYS_OFFSET, x28=PHYS_OFFSET-PAGE_OFFSET bl set_cpu_boot_mode_flag b secondary_startup ENDPROC(secondary_entry) @@ -633,16 +644,9 @@ ENTRY(secondary_startup) /* * Common entry point for secondary CPUs. */ - mrs x22, midr_el1 // x22=cpuid - mov x0, x22 - bl lookup_processor_type - mov x23, x0 // x23=current cpu_table - cbz x23, __error_p // invalid processor (x23=0)? - - pgtbl x25, x26, x28 // x25=TTBR0, x26=TTBR1 - ldr x12, [x23, #CPU_INFO_SETUP] - add x12, x12, x28 // __virt_to_phys - blr x12 // initialise processor + adrp x25, idmap_pg_dir + adrp x26, swapper_pg_dir + bl __cpu_setup // initialise processor ldr x21, =secondary_data ldr x27, =__secondary_switched // address to jump to after enabling the MMU @@ -658,11 +662,12 @@ ENDPROC(__secondary_switched) #endif /* CONFIG_SMP */ /* - * Setup common bits before finally enabling the MMU. Essentially this is just - * loading the page table pointer and vector base registers. + * Enable the MMU. * - * On entry to this code, x0 must contain the SCTLR_EL1 value for turning on - * the MMU. + * x0 = SCTLR_EL1 value for turning on the MMU. + * x27 = *virtual* address to jump to upon completion + * + * other registers depend on the function called upon completion */ __enable_mmu: ldr x5, =vectors @@ -670,89 +675,7 @@ __enable_mmu: msr ttbr0_el1, x25 // load TTBR0 msr ttbr1_el1, x26 // load TTBR1 isb - b __turn_mmu_on -ENDPROC(__enable_mmu) - -/* - * Enable the MMU. This completely changes the structure of the visible memory - * space. You will not be able to trace execution through this. - * - * x0 = system control register - * x27 = *virtual* address to jump to upon completion - * - * other registers depend on the function called upon completion - * - * We align the entire function to the smallest power of two larger than it to - * ensure it fits within a single block map entry. Otherwise were PHYS_OFFSET - * close to the end of a 512MB or 1GB block we might require an additional - * table to map the entire function. - */ - .align 4 -__turn_mmu_on: msr sctlr_el1, x0 isb br x27 -ENDPROC(__turn_mmu_on) - -/* - * Calculate the start of physical memory. - */ -__calc_phys_offset: - adr x0, 1f - ldp x1, x2, [x0] - sub x28, x0, x1 // x28 = PHYS_OFFSET - PAGE_OFFSET - add x24, x2, x28 // x24 = PHYS_OFFSET - ret -ENDPROC(__calc_phys_offset) - - .align 3 -1: .quad . - .quad PAGE_OFFSET - -/* - * Exception handling. Something went wrong and we can't proceed. We ought to - * tell the user, but since we don't have any guarantee that we're even - * running on the right architecture, we do virtually nothing. - */ -__error_p: -ENDPROC(__error_p) - -__error: -1: nop - b 1b -ENDPROC(__error) - -/* - * This function gets the processor ID in w0 and searches the cpu_table[] for - * a match. It returns a pointer to the struct cpu_info it found. The - * cpu_table[] must end with an empty (all zeros) structure. - * - * This routine can be called via C code and it needs to work with the MMU - * both disabled and enabled (the offset is calculated automatically). - */ -ENTRY(lookup_processor_type) - adr x1, __lookup_processor_type_data - ldp x2, x3, [x1] - sub x1, x1, x2 // get offset between VA and PA - add x3, x3, x1 // convert VA to PA -1: - ldp w5, w6, [x3] // load cpu_id_val and cpu_id_mask - cbz w5, 2f // end of list? - and w6, w6, w0 - cmp w5, w6 - b.eq 3f - add x3, x3, #CPU_INFO_SZ - b 1b -2: - mov x3, #0 // unknown processor -3: - mov x0, x3 - ret -ENDPROC(lookup_processor_type) - - .align 3 - .type __lookup_processor_type_data, %object -__lookup_processor_type_data: - .quad . - .quad cpu_table - .size __lookup_processor_type_data, . - __lookup_processor_type_data +ENDPROC(__enable_mmu) diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/kernel/insn.c index c8eca88f12e6..924902083e47 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/kernel/insn.c @@ -265,23 +265,13 @@ int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) return aarch64_insn_patch_text_sync(addrs, insns, cnt); } -u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, - u32 insn, u64 imm) +static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, + u32 *maskp, int *shiftp) { - u32 immlo, immhi, lomask, himask, mask; + u32 mask; int shift; switch (type) { - case AARCH64_INSN_IMM_ADR: - lomask = 0x3; - himask = 0x7ffff; - immlo = imm & lomask; - imm >>= 2; - immhi = imm & himask; - imm = (immlo << 24) | (immhi); - mask = (lomask << 24) | (himask); - shift = 5; - break; case AARCH64_INSN_IMM_26: mask = BIT(26) - 1; shift = 0; @@ -320,9 +310,68 @@ u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, shift = 16; break; default: - pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", - type); - return 0; + return -EINVAL; + } + + *maskp = mask; + *shiftp = shift; + + return 0; +} + +#define ADR_IMM_HILOSPLIT 2 +#define ADR_IMM_SIZE SZ_2M +#define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1) +#define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1) +#define ADR_IMM_LOSHIFT 29 +#define ADR_IMM_HISHIFT 5 + +u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn) +{ + u32 immlo, immhi, mask; + int shift; + + switch (type) { + case AARCH64_INSN_IMM_ADR: + shift = 0; + immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK; + immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK; + insn = (immhi << ADR_IMM_HILOSPLIT) | immlo; + mask = ADR_IMM_SIZE - 1; + break; + default: + if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { + pr_err("aarch64_insn_decode_immediate: unknown immediate encoding %d\n", + type); + return 0; + } + } + + return (insn >> shift) & mask; +} + +u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type, + u32 insn, u64 imm) +{ + u32 immlo, immhi, mask; + int shift; + + switch (type) { + case AARCH64_INSN_IMM_ADR: + shift = 0; + immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT; + imm >>= ADR_IMM_HILOSPLIT; + immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT; + imm = immlo | immhi; + mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) | + (ADR_IMM_HIMASK << ADR_IMM_HISHIFT)); + break; + default: + if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) { + pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n", + type); + return 0; + } } /* Update the immediate field. */ diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index 25a5308744b1..195991dadc37 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -25,8 +25,10 @@ #include <linux/irq.h> #include <linux/kernel.h> #include <linux/export.h> +#include <linux/of.h> #include <linux/perf_event.h> #include <linux/platform_device.h> +#include <linux/slab.h> #include <linux/spinlock.h> #include <linux/uaccess.h> @@ -322,22 +324,31 @@ out: } static int -validate_event(struct pmu_hw_events *hw_events, - struct perf_event *event) +validate_event(struct pmu *pmu, struct pmu_hw_events *hw_events, + struct perf_event *event) { - struct arm_pmu *armpmu = to_arm_pmu(event->pmu); + struct arm_pmu *armpmu; struct hw_perf_event fake_event = event->hw; struct pmu *leader_pmu = event->group_leader->pmu; if (is_software_event(event)) return 1; + /* + * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The + * core perf code won't check that the pmu->ctx == leader->ctx + * until after pmu->event_init(event). + */ + if (event->pmu != pmu) + return 0; + if (event->pmu != leader_pmu || event->state < PERF_EVENT_STATE_OFF) return 1; if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) return 1; + armpmu = to_arm_pmu(event->pmu); return armpmu->get_event_idx(hw_events, &fake_event) >= 0; } @@ -355,15 +366,15 @@ validate_group(struct perf_event *event) memset(fake_used_mask, 0, sizeof(fake_used_mask)); fake_pmu.used_mask = fake_used_mask; - if (!validate_event(&fake_pmu, leader)) + if (!validate_event(event->pmu, &fake_pmu, leader)) return -EINVAL; list_for_each_entry(sibling, &leader->sibling_list, group_entry) { - if (!validate_event(&fake_pmu, sibling)) + if (!validate_event(event->pmu, &fake_pmu, sibling)) return -EINVAL; } - if (!validate_event(&fake_pmu, event)) + if (!validate_event(event->pmu, &fake_pmu, event)) return -EINVAL; return 0; @@ -396,7 +407,12 @@ armpmu_release_hardware(struct arm_pmu *armpmu) free_percpu_irq(irq, &cpu_hw_events); } else { for (i = 0; i < irqs; ++i) { - if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs)) + int cpu = i; + + if (armpmu->irq_affinity) + cpu = armpmu->irq_affinity[i]; + + if (!cpumask_test_and_clear_cpu(cpu, &armpmu->active_irqs)) continue; irq = platform_get_irq(pmu_device, i); if (irq > 0) @@ -450,19 +466,24 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) on_each_cpu(armpmu_enable_percpu_irq, &irq, 1); } else { for (i = 0; i < irqs; ++i) { + int cpu = i; + err = 0; irq = platform_get_irq(pmu_device, i); if (irq <= 0) continue; + if (armpmu->irq_affinity) + cpu = armpmu->irq_affinity[i]; + /* * If we have a single PMU interrupt that we can't shift, * assume that we're running on a uniprocessor machine and * continue. Otherwise, continue without this interrupt. */ - if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) { + if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) { pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", - irq, i); + irq, cpu); continue; } @@ -476,7 +497,7 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu) return err; } - cpumask_set_cpu(i, &armpmu->active_irqs); + cpumask_set_cpu(cpu, &armpmu->active_irqs); } } @@ -1289,9 +1310,46 @@ static const struct of_device_id armpmu_of_device_ids[] = { static int armpmu_device_probe(struct platform_device *pdev) { + int i, *irqs; + if (!cpu_pmu) return -ENODEV; + irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL); + if (!irqs) + return -ENOMEM; + + for (i = 0; i < pdev->num_resources; ++i) { + struct device_node *dn; + int cpu; + + dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity", + i); + if (!dn) { + pr_warn("Failed to parse %s/interrupt-affinity[%d]\n", + of_node_full_name(dn), i); + break; + } + + for_each_possible_cpu(cpu) + if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL)) + break; + + of_node_put(dn); + if (cpu >= nr_cpu_ids) { + pr_warn("Failed to find logical CPU for %s\n", + dn->name); + break; + } + + irqs[i] = cpu; + } + + if (i == pdev->num_resources) + cpu_pmu->irq_affinity = irqs; + else + kfree(irqs); + cpu_pmu->plat_device = pdev; return 0; } diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index e8420f635bd4..51ef97274b52 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -50,7 +50,6 @@ #include <asm/cpu.h> #include <asm/cputype.h> #include <asm/elf.h> -#include <asm/cputable.h> #include <asm/cpufeature.h> #include <asm/cpu_ops.h> #include <asm/sections.h> @@ -62,9 +61,7 @@ #include <asm/memblock.h> #include <asm/psci.h> #include <asm/efi.h> - -unsigned int processor_id; -EXPORT_SYMBOL(processor_id); +#include <asm/virt.h> unsigned long elf_hwcap __read_mostly; EXPORT_SYMBOL_GPL(elf_hwcap); @@ -83,7 +80,6 @@ unsigned int compat_elf_hwcap2 __read_mostly; DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); -static const char *cpu_name; phys_addr_t __fdt_pointer __initdata; /* @@ -119,6 +115,11 @@ void __init early_print(const char *str, ...) printk("%s", buf); } +/* + * The recorded values of x0 .. x3 upon kernel entry. + */ +u64 __cacheline_aligned boot_args[4]; + void __init smp_setup_processor_id(void) { u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; @@ -207,24 +208,38 @@ static void __init smp_build_mpidr_hash(void) } #endif +static void __init hyp_mode_check(void) +{ + if (is_hyp_mode_available()) + pr_info("CPU: All CPU(s) started at EL2\n"); + else if (is_hyp_mode_mismatched()) + WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC, + "CPU: CPUs started in inconsistent modes"); + else + pr_info("CPU: All CPU(s) started at EL1\n"); +} + +void __init do_post_cpus_up_work(void) +{ + hyp_mode_check(); + apply_alternatives_all(); +} + +#ifdef CONFIG_UP_LATE_INIT +void __init up_late_init(void) +{ + do_post_cpus_up_work(); +} +#endif /* CONFIG_UP_LATE_INIT */ + static void __init setup_processor(void) { - struct cpu_info *cpu_info; u64 features, block; u32 cwg; int cls; - cpu_info = lookup_processor_type(read_cpuid_id()); - if (!cpu_info) { - printk("CPU configuration botched (ID %08x), unable to continue.\n", - read_cpuid_id()); - while (1); - } - - cpu_name = cpu_info->cpu_name; - - printk("CPU: %s [%08x] revision %d\n", - cpu_name, read_cpuid_id(), read_cpuid_id() & 15); + printk("CPU: AArch64 Processor [%08x] revision %d\n", + read_cpuid_id(), read_cpuid_id() & 15); sprintf(init_utsname()->machine, ELF_PLATFORM); elf_hwcap = 0; @@ -402,6 +417,12 @@ void __init setup_arch(char **cmdline_p) conswitchp = &dummy_con; #endif #endif + if (boot_args[1] || boot_args[2] || boot_args[3]) { + pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n" + "\tx1: %016llx\n\tx2: %016llx\n\tx3: %016llx\n" + "This indicates a broken bootloader or old kernel\n", + boot_args[1], boot_args[2], boot_args[3]); + } } static int __init arm64_device_init(void) diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 660ccf9f7524..e18c48cb6db1 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -287,19 +287,12 @@ static void setup_restart_syscall(struct pt_regs *regs) */ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs) { - struct thread_info *thread = current_thread_info(); struct task_struct *tsk = current; sigset_t *oldset = sigmask_to_save(); int usig = ksig->sig; int ret; /* - * translate the signal - */ - if (usig < 32 && thread->exec_domain && thread->exec_domain->signal_invmap) - usig = thread->exec_domain->signal_invmap[usig]; - - /* * Set up the stack frame */ if (is_compat_task()) { diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 328b8ce4b007..ffe8e1b814e0 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -151,6 +151,7 @@ asmlinkage void secondary_start_kernel(void) */ cpu_set_reserved_ttbr0(); flush_tlb_all(); + cpu_set_default_tcr_t0sz(); preempt_disable(); trace_hardirqs_off(); @@ -309,7 +310,7 @@ void cpu_die(void) void __init smp_cpus_done(unsigned int max_cpus) { pr_info("SMP: Total of %d processors activated.\n", num_online_cpus()); - apply_alternatives_all(); + do_post_cpus_up_work(); } void __init smp_prepare_boot_cpu(void) diff --git a/arch/arm64/kernel/sys32.c b/arch/arm64/kernel/sys32.c index 2d5ab3c90b82..a40b1343b819 100644 --- a/arch/arm64/kernel/sys32.c +++ b/arch/arm64/kernel/sys32.c @@ -37,6 +37,7 @@ asmlinkage long compat_sys_readahead_wrapper(void); asmlinkage long compat_sys_fadvise64_64_wrapper(void); asmlinkage long compat_sys_sync_file_range2_wrapper(void); asmlinkage long compat_sys_fallocate_wrapper(void); +asmlinkage long compat_sys_mmap2_wrapper(void); #undef __SYSCALL #define __SYSCALL(nr, sym) [nr] = sym, diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S index 5d9d2dca530d..a2c29865c3fe 100644 --- a/arch/arm64/kernel/vmlinux.lds.S +++ b/arch/arm64/kernel/vmlinux.lds.S @@ -23,10 +23,14 @@ jiffies = jiffies_64; #define HYPERVISOR_TEXT \ /* \ - * Force the alignment to be compatible with \ - * the vectors requirements \ + * Align to 4 KB so that \ + * a) the HYP vector table is at its minimum \ + * alignment of 2048 bytes \ + * b) the HYP init code will not cross a page \ + * boundary if its size does not exceed \ + * 4 KB (see related ASSERT() below) \ */ \ - . = ALIGN(2048); \ + . = ALIGN(SZ_4K); \ VMLINUX_SYMBOL(__hyp_idmap_text_start) = .; \ *(.hyp.idmap.text) \ VMLINUX_SYMBOL(__hyp_idmap_text_end) = .; \ @@ -163,10 +167,11 @@ SECTIONS } /* - * The HYP init code can't be more than a page long. + * The HYP init code can't be more than a page long, + * and should not cross a page boundary. */ -ASSERT(((__hyp_idmap_text_start + PAGE_SIZE) > __hyp_idmap_text_end), - "HYP init code too big") +ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K, + "HYP init code too big or misaligned") /* * If padding is applied before .head.text, virt<->phys conversions will fail. diff --git a/arch/arm64/kvm/hyp-init.S b/arch/arm64/kvm/hyp-init.S index c3191168a994..178ba2248a98 100644 --- a/arch/arm64/kvm/hyp-init.S +++ b/arch/arm64/kvm/hyp-init.S @@ -20,6 +20,7 @@ #include <asm/assembler.h> #include <asm/kvm_arm.h> #include <asm/kvm_mmu.h> +#include <asm/pgtable-hwdef.h> .text .pushsection .hyp.idmap.text, "ax" @@ -65,6 +66,25 @@ __do_hyp_init: and x4, x4, x5 ldr x5, =TCR_EL2_FLAGS orr x4, x4, x5 + +#ifndef CONFIG_ARM64_VA_BITS_48 + /* + * If we are running with VA_BITS < 48, we may be running with an extra + * level of translation in the ID map. This is only the case if system + * RAM is out of range for the currently configured page size and number + * of translation levels, in which case we will also need the extra + * level for the HYP ID map, or we won't be able to enable the EL2 MMU. + * + * However, at EL2, there is only one TTBR register, and we can't switch + * between translation tables *and* update TCR_EL2.T0SZ at the same + * time. Bottom line: we need the extra level in *both* our translation + * tables. + * + * So use the same T0SZ value we use for the ID map. + */ + ldr_l x5, idmap_t0sz + bfi x4, x5, TCR_T0SZ_OFFSET, TCR_TxSZ_WIDTH +#endif msr tcr_el2, x4 ldr x4, =VTCR_EL2_FLAGS @@ -91,6 +111,10 @@ __do_hyp_init: msr sctlr_el2, x4 isb + /* Skip the trampoline dance if we merged the boot and runtime PGDs */ + cmp x0, x1 + b.eq merged + /* MMU is now enabled. Get ready for the trampoline dance */ ldr x4, =TRAMPOLINE_VA adr x5, target @@ -105,6 +129,7 @@ target: /* We're now in the trampoline code, switch page tables */ tlbi alle2 dsb sy +merged: /* Set the stack and new vectors */ kern_hyp_va x2 mov sp, x2 diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 79e01163a981..5b8b664422d3 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -40,6 +40,8 @@ #include "mm.h" +u64 idmap_t0sz = TCR_T0SZ(VA_BITS); + /* * Empty_zero_page is a special page that is used for zero-initialized data * and COW. @@ -454,6 +456,7 @@ void __init paging_init(void) */ cpu_set_reserved_ttbr0(); flush_tlb_all(); + cpu_set_default_tcr_t0sz(); } /* @@ -461,8 +464,10 @@ void __init paging_init(void) */ void setup_mm_for_reboot(void) { - cpu_switch_mm(idmap_pg_dir, &init_mm); + cpu_set_reserved_ttbr0(); flush_tlb_all(); + cpu_set_idmap_tcr_t0sz(); + cpu_switch_mm(idmap_pg_dir, &init_mm); } /* @@ -627,10 +632,7 @@ void __set_fixmap(enum fixed_addresses idx, unsigned long addr = __fix_to_virt(idx); pte_t *pte; - if (idx >= __end_of_fixed_addresses) { - BUG(); - return; - } + BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); pte = fixmap_pte(addr); diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index 1d3ec3ddd84b..e47ed1c5dce1 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -73,7 +73,6 @@ int set_memory_ro(unsigned long addr, int numpages) __pgprot(PTE_RDONLY), __pgprot(PTE_WRITE)); } -EXPORT_SYMBOL_GPL(set_memory_ro); int set_memory_rw(unsigned long addr, int numpages) { @@ -81,7 +80,6 @@ int set_memory_rw(unsigned long addr, int numpages) __pgprot(PTE_WRITE), __pgprot(PTE_RDONLY)); } -EXPORT_SYMBOL_GPL(set_memory_rw); int set_memory_nx(unsigned long addr, int numpages) { diff --git a/arch/arm64/mm/proc-macros.S b/arch/arm64/mm/proc-macros.S index 005d29e2977d..4c4d93c4bf65 100644 --- a/arch/arm64/mm/proc-macros.S +++ b/arch/arm64/mm/proc-macros.S @@ -52,3 +52,13 @@ mov \reg, #4 // bytes per word lsl \reg, \reg, \tmp // actual cache line size .endm + +/* + * tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map + */ + .macro tcr_set_idmap_t0sz, valreg, tmpreg +#ifndef CONFIG_ARM64_VA_BITS_48 + ldr_l \tmpreg, idmap_t0sz + bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH +#endif + .endm diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 28eebfb6af76..cdd754e19b9b 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -156,6 +156,7 @@ ENTRY(cpu_do_resume) msr cpacr_el1, x6 msr ttbr0_el1, x1 msr ttbr1_el1, x7 + tcr_set_idmap_t0sz x8, x7 msr tcr_el1, x8 msr vbar_el1, x9 msr mdscr_el1, x10 @@ -233,6 +234,8 @@ ENTRY(__cpu_setup) */ ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 + tcr_set_idmap_t0sz x10, x9 + /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in * TCR_EL1. diff --git a/arch/avr32/include/asm/thread_info.h b/arch/avr32/include/asm/thread_info.h index d56afa99a514..d4d3079541ea 100644 --- a/arch/avr32/include/asm/thread_info.h +++ b/arch/avr32/include/asm/thread_info.h @@ -17,11 +17,9 @@ #include <asm/types.h> struct task_struct; -struct exec_domain; struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ __u32 cpu; __s32 preempt_count; /* 0 => preemptable, <0 => BUG */ @@ -36,7 +34,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/avr32/kernel/asm-offsets.c b/arch/avr32/kernel/asm-offsets.c index e41c84516e5d..2c9764fe3532 100644 --- a/arch/avr32/kernel/asm-offsets.c +++ b/arch/avr32/kernel/asm-offsets.c @@ -12,7 +12,6 @@ void foo(void) { OFFSET(TI_task, thread_info, task); - OFFSET(TI_exec_domain, thread_info, exec_domain); OFFSET(TI_flags, thread_info, flags); OFFSET(TI_cpu, thread_info, cpu); OFFSET(TI_preempt_count, thread_info, preempt_count); diff --git a/arch/blackfin/include/asm/thread_info.h b/arch/blackfin/include/asm/thread_info.h index 57c3a8bd583d..2966b93850a1 100644 --- a/arch/blackfin/include/asm/thread_info.h +++ b/arch/blackfin/include/asm/thread_info.h @@ -37,7 +37,6 @@ typedef unsigned long mm_segment_t; struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, <0 => BUG */ @@ -53,7 +52,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ @@ -76,15 +74,6 @@ static inline struct thread_info *current_thread_info(void) #endif /* __ASSEMBLY__ */ /* - * Offsets in thread_info structure, used in assembly code - */ -#define TI_TASK 0 -#define TI_EXECDOMAIN 4 -#define TI_FLAGS 8 -#define TI_CPU 12 -#define TI_PREEMPT 16 - -/* * thread information flag bit numbers */ #define TIF_SYSCALL_TRACE 0 /* syscall trace active */ diff --git a/arch/blackfin/kernel/asm-offsets.c b/arch/blackfin/kernel/asm-offsets.c index 37fcae950216..486560aea050 100644 --- a/arch/blackfin/kernel/asm-offsets.c +++ b/arch/blackfin/kernel/asm-offsets.c @@ -42,6 +42,12 @@ int main(void) DEFINE(THREAD_PC, offsetof(struct thread_struct, pc)); DEFINE(KERNEL_STACK_SIZE, THREAD_SIZE); + /* offsets in thread_info struct */ + OFFSET(TI_TASK, thread_info, task); + OFFSET(TI_FLAGS, thread_info, flags); + OFFSET(TI_CPU, thread_info, cpu); + OFFSET(TI_PREEMPT, thread_info, preempt_count); + /* offsets into the pt_regs */ DEFINE(PT_ORIG_R0, offsetof(struct pt_regs, orig_r0)); DEFINE(PT_ORIG_P0, offsetof(struct pt_regs, orig_p0)); diff --git a/arch/blackfin/kernel/signal.c b/arch/blackfin/kernel/signal.c index f2a8b5493bd3..ea570db598e5 100644 --- a/arch/blackfin/kernel/signal.c +++ b/arch/blackfin/kernel/signal.c @@ -151,11 +151,7 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) frame = get_sigframe(ksig, sizeof(*frame)); - err |= __put_user((current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && ksig->sig < 32 - ? current_thread_info()->exec_domain-> - signal_invmap[ksig->sig] : ksig->sig), &frame->sig); + err |= __put_user(ksig->sig, &frame->sig); err |= __put_user(&frame->info, &frame->pinfo); err |= __put_user(&frame->uc, &frame->puc); diff --git a/arch/blackfin/kernel/traps.c b/arch/blackfin/kernel/traps.c index de5c2c3ebd9b..1ed85ddadc0d 100644 --- a/arch/blackfin/kernel/traps.c +++ b/arch/blackfin/kernel/traps.c @@ -18,6 +18,7 @@ #include <asm/fixed_code.h> #include <asm/pseudo_instructions.h> #include <asm/pda.h> +#include <asm/asm-offsets.h> #ifdef CONFIG_KGDB # include <linux/kgdb.h> diff --git a/arch/c6x/Makefile b/arch/c6x/Makefile index e72eb3417239..6b0be670ddfa 100644 --- a/arch/c6x/Makefile +++ b/arch/c6x/Makefile @@ -8,7 +8,7 @@ KBUILD_DEFCONFIG := dsk6455_defconfig -cflags-y += -mno-dsbt -msdata=none +cflags-y += -mno-dsbt -msdata=none -D__linux__ cflags-$(CONFIG_C6X_BIG_KERNEL) += -mlong-calls diff --git a/arch/c6x/include/asm/Kbuild b/arch/c6x/include/asm/Kbuild index 2de73391b81e..ae0a51f5376c 100644 --- a/arch/c6x/include/asm/Kbuild +++ b/arch/c6x/include/asm/Kbuild @@ -41,6 +41,7 @@ generic-y += resource.h generic-y += scatterlist.h generic-y += segment.h generic-y += sembuf.h +generic-y += serial.h generic-y += shmbuf.h generic-y += shmparam.h generic-y += siginfo.h diff --git a/arch/c6x/include/asm/dma-mapping.h b/arch/c6x/include/asm/dma-mapping.h index 88bd0d899bdb..bbd7774e4d4e 100644 --- a/arch/c6x/include/asm/dma-mapping.h +++ b/arch/c6x/include/asm/dma-mapping.h @@ -17,6 +17,14 @@ #define dma_supported(d, m) 1 +static inline void dma_sync_single_range_for_device(struct device *dev, + dma_addr_t addr, + unsigned long offset, + size_t size, + enum dma_data_direction dir) +{ +} + static inline int dma_set_mask(struct device *dev, u64 dma_mask) { if (!dev->dma_mask || !dma_supported(dev, dma_mask)) diff --git a/arch/c6x/include/asm/flat.h b/arch/c6x/include/asm/flat.h new file mode 100644 index 000000000000..a1858bd5f6c8 --- /dev/null +++ b/arch/c6x/include/asm/flat.h @@ -0,0 +1,12 @@ +#ifndef __ASM_C6X_FLAT_H +#define __ASM_C6X_FLAT_H + +#define flat_argvp_envp_on_stack() 0 +#define flat_old_ram_flag(flags) (flags) +#define flat_reloc_valid(reloc, size) ((reloc) <= (size)) +#define flat_get_addr_from_rp(rp, relval, flags, p) get_unaligned(rp) +#define flat_put_addr_at_rp(rp, val, relval) put_unaligned(val, rp) +#define flat_get_relocate_addr(rel) (rel) +#define flat_set_persistent(relval, p) 0 + +#endif /* __ASM_C6X_FLAT_H */ diff --git a/arch/c6x/include/asm/setup.h b/arch/c6x/include/asm/setup.h index 696804475f55..852afb209afb 100644 --- a/arch/c6x/include/asm/setup.h +++ b/arch/c6x/include/asm/setup.h @@ -12,6 +12,7 @@ #define _ASM_C6X_SETUP_H #include <uapi/asm/setup.h> +#include <linux/types.h> #ifndef __ASSEMBLY__ extern int c6x_add_memory(phys_addr_t start, unsigned long size); diff --git a/arch/c6x/include/asm/thread_info.h b/arch/c6x/include/asm/thread_info.h index 584e253f3217..acc70c135ab8 100644 --- a/arch/c6x/include/asm/thread_info.h +++ b/arch/c6x/include/asm/thread_info.h @@ -40,7 +40,6 @@ typedef struct { */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ int cpu; /* cpu we're on */ int preempt_count; /* 0 = preemptable, <0 = BUG */ @@ -55,7 +54,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/c6x/kernel/setup.c b/arch/c6x/kernel/setup.c index 757128868d43..72e17f7ebd6f 100644 --- a/arch/c6x/kernel/setup.c +++ b/arch/c6x/kernel/setup.c @@ -26,7 +26,8 @@ #include <linux/cpu.h> #include <linux/fs.h> #include <linux/of.h> - +#include <linux/console.h> +#include <linux/screen_info.h> #include <asm/sections.h> #include <asm/div64.h> @@ -38,6 +39,8 @@ static const char *c6x_soc_name; +struct screen_info screen_info; + int c6x_num_cores; EXPORT_SYMBOL_GPL(c6x_num_cores); @@ -60,6 +63,7 @@ unsigned char c6x_fuse_mac[6]; unsigned long memory_start; unsigned long memory_end; +EXPORT_SYMBOL(memory_end); unsigned long ram_start; unsigned long ram_end; @@ -265,8 +269,8 @@ int __init c6x_add_memory(phys_addr_t start, unsigned long size) */ notrace void __init machine_init(unsigned long dt_ptr) { - const void *dtb = __va(dt_ptr); - const void *fdt = _fdt_start; + void *dtb = __va(dt_ptr); + void *fdt = _fdt_start; /* interrupts must be masked */ set_creg(IER, 2); diff --git a/arch/c6x/kernel/time.c b/arch/c6x/kernel/time.c index 356ee84cad95..04845aaf5985 100644 --- a/arch/c6x/kernel/time.c +++ b/arch/c6x/kernel/time.c @@ -49,7 +49,7 @@ u64 sched_clock(void) return (tsc * sched_clock_multiplier) >> SCHED_CLOCK_SHIFT; } -void time_init(void) +void __init time_init(void) { u64 tmp = (u64)NSEC_PER_SEC << SCHED_CLOCK_SHIFT; diff --git a/arch/c6x/platforms/cache.c b/arch/c6x/platforms/cache.c index 86318a16a252..46fd2d530271 100644 --- a/arch/c6x/platforms/cache.c +++ b/arch/c6x/platforms/cache.c @@ -350,6 +350,7 @@ void L1P_cache_block_invalidate(unsigned int start, unsigned int end) (unsigned int *) end, IMCR_L1PIBAR, IMCR_L1PIWC); } +EXPORT_SYMBOL(L1P_cache_block_invalidate); void L1D_cache_block_invalidate(unsigned int start, unsigned int end) { @@ -371,6 +372,7 @@ void L1D_cache_block_writeback(unsigned int start, unsigned int end) (unsigned int *) end, IMCR_L1DWBAR, IMCR_L1DWWC); } +EXPORT_SYMBOL(L1D_cache_block_writeback); /* * L2 block operations diff --git a/arch/cris/arch-v10/kernel/fasttimer.c b/arch/cris/arch-v10/kernel/fasttimer.c index 48a59afbeeb1..e9298739d72e 100644 --- a/arch/cris/arch-v10/kernel/fasttimer.c +++ b/arch/cris/arch-v10/kernel/fasttimer.c @@ -527,7 +527,8 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) i = debug_log_cnt; while (i != end_i || debug_log_cnt_wrapped) { - if (seq_printf(m, debug_log_string[i], debug_log_value[i]) < 0) + seq_printf(m, debug_log_string[i], debug_log_value[i]); + if (seq_has_overflowed(m)) return 0; i = (i+1) % DEBUG_LOG_MAX; } @@ -542,24 +543,22 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) int cur = (fast_timers_started - i - 1) % NUM_TIMER_STATS; #if 1 //ndef FAST_TIMER_LOG - seq_printf(m, "div: %i freq: %i delay: %i" - "\n", + seq_printf(m, "div: %i freq: %i delay: %i\n", timer_div_settings[cur], timer_freq_settings[cur], timer_delay_settings[cur]); #endif #ifdef FAST_TIMER_LOG t = &timer_started_log[cur]; - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; #endif } @@ -571,16 +570,15 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) seq_printf(m, "Timers added: %i\n", fast_timers_added); for (i = 0; i < num_to_show; i++) { t = &timer_added_log[(fast_timers_added - i - 1) % NUM_TIMER_STATS]; - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; } seq_putc(m, '\n'); @@ -590,16 +588,15 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) seq_printf(m, "Timers expired: %i\n", fast_timers_expired); for (i = 0; i < num_to_show; i++) { t = &timer_expired_log[(fast_timers_expired - i - 1) % NUM_TIMER_STATS]; - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; } seq_putc(m, '\n'); @@ -611,19 +608,15 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) while (t) { nextt = t->next; local_irq_restore(flags); - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" -/* " func: 0x%08lX" */ - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data -/* , t->function */ - ) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; local_irq_save(flags); if (t->next != nextt) diff --git a/arch/cris/arch-v10/kernel/setup.c b/arch/cris/arch-v10/kernel/setup.c index 4f96d71b5154..7ab31f1c7540 100644 --- a/arch/cris/arch-v10/kernel/setup.c +++ b/arch/cris/arch-v10/kernel/setup.c @@ -63,35 +63,37 @@ int show_cpuinfo(struct seq_file *m, void *v) else info = &cpu_info[revision]; - return seq_printf(m, - "processor\t: 0\n" - "cpu\t\t: CRIS\n" - "cpu revision\t: %lu\n" - "cpu model\t: %s\n" - "cache size\t: %d kB\n" - "fpu\t\t: %s\n" - "mmu\t\t: %s\n" - "mmu DMA bug\t: %s\n" - "ethernet\t: %s Mbps\n" - "token ring\t: %s\n" - "scsi\t\t: %s\n" - "ata\t\t: %s\n" - "usb\t\t: %s\n" - "bogomips\t: %lu.%02lu\n", + seq_printf(m, + "processor\t: 0\n" + "cpu\t\t: CRIS\n" + "cpu revision\t: %lu\n" + "cpu model\t: %s\n" + "cache size\t: %d kB\n" + "fpu\t\t: %s\n" + "mmu\t\t: %s\n" + "mmu DMA bug\t: %s\n" + "ethernet\t: %s Mbps\n" + "token ring\t: %s\n" + "scsi\t\t: %s\n" + "ata\t\t: %s\n" + "usb\t\t: %s\n" + "bogomips\t: %lu.%02lu\n", - revision, - info->model, - info->cache, - info->flags & HAS_FPU ? "yes" : "no", - info->flags & HAS_MMU ? "yes" : "no", - info->flags & HAS_MMU_BUG ? "yes" : "no", - info->flags & HAS_ETHERNET100 ? "10/100" : "10", - info->flags & HAS_TOKENRING ? "4/16 Mbps" : "no", - info->flags & HAS_SCSI ? "yes" : "no", - info->flags & HAS_ATA ? "yes" : "no", - info->flags & HAS_USB ? "yes" : "no", - (loops_per_jiffy * HZ + 500) / 500000, - ((loops_per_jiffy * HZ + 500) / 5000) % 100); + revision, + info->model, + info->cache, + info->flags & HAS_FPU ? "yes" : "no", + info->flags & HAS_MMU ? "yes" : "no", + info->flags & HAS_MMU_BUG ? "yes" : "no", + info->flags & HAS_ETHERNET100 ? "10/100" : "10", + info->flags & HAS_TOKENRING ? "4/16 Mbps" : "no", + info->flags & HAS_SCSI ? "yes" : "no", + info->flags & HAS_ATA ? "yes" : "no", + info->flags & HAS_USB ? "yes" : "no", + (loops_per_jiffy * HZ + 500) / 500000, + ((loops_per_jiffy * HZ + 500) / 5000) % 100); + + return 0; } #endif /* CONFIG_PROC_FS */ diff --git a/arch/cris/arch-v10/kernel/signal.c b/arch/cris/arch-v10/kernel/signal.c index 74d7ba35120d..7122d9773b13 100644 --- a/arch/cris/arch-v10/kernel/signal.c +++ b/arch/cris/arch-v10/kernel/signal.c @@ -321,8 +321,6 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, if (err) return -EFAULT; - /* TODO what is the current->exec_domain stuff and invmap ? */ - /* Set up registers for signal handler */ /* What we enter NOW */ diff --git a/arch/cris/arch-v32/kernel/fasttimer.c b/arch/cris/arch-v32/kernel/fasttimer.c index b130c2c5fdd8..5c84dbb99f30 100644 --- a/arch/cris/arch-v32/kernel/fasttimer.c +++ b/arch/cris/arch-v32/kernel/fasttimer.c @@ -501,7 +501,8 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) i = debug_log_cnt; while ((i != end_i || debug_log_cnt_wrapped)) { - if (seq_printf(m, debug_log_string[i], debug_log_value[i]) < 0) + seq_printf(m, debug_log_string[i], debug_log_value[i]); + if (seq_has_overflowed(m)) return 0; i = (i+1) % DEBUG_LOG_MAX; } @@ -516,23 +517,21 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) int cur = (fast_timers_started - i - 1) % NUM_TIMER_STATS; #if 1 //ndef FAST_TIMER_LOG - seq_printf(m, "div: %i delay: %i" - "\n", + seq_printf(m, "div: %i delay: %i\n", timer_div_settings[cur], timer_delay_settings[cur]); #endif #ifdef FAST_TIMER_LOG t = &timer_started_log[cur]; - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; #endif } @@ -544,16 +543,15 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) seq_printf(m, "Timers added: %i\n", fast_timers_added); for (i = 0; i < num_to_show; i++) { t = &timer_added_log[(fast_timers_added - i - 1) % NUM_TIMER_STATS]; - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; } seq_putc(m, '\n'); @@ -563,16 +561,15 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) seq_printf(m, "Timers expired: %i\n", fast_timers_expired); for (i = 0; i < num_to_show; i++){ t = &timer_expired_log[(fast_timers_expired - i - 1) % NUM_TIMER_STATS]; - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; } seq_putc(m, '\n'); @@ -584,19 +581,15 @@ static int proc_fasttimer_show(struct seq_file *m, void *v) while (t != NULL){ nextt = t->next; local_irq_restore(flags); - if (seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu " - "d: %6li us data: 0x%08lX" -/* " func: 0x%08lX" */ - "\n", - t->name, - (unsigned long)t->tv_set.tv_jiff, - (unsigned long)t->tv_set.tv_usec, - (unsigned long)t->tv_expires.tv_jiff, - (unsigned long)t->tv_expires.tv_usec, - t->delay_us, - t->data -/* , t->function */ - ) < 0) + seq_printf(m, "%-14s s: %6lu.%06lu e: %6lu.%06lu d: %6li us data: 0x%08lX\n", + t->name, + (unsigned long)t->tv_set.tv_jiff, + (unsigned long)t->tv_set.tv_usec, + (unsigned long)t->tv_expires.tv_jiff, + (unsigned long)t->tv_expires.tv_usec, + t->delay_us, + t->data); + if (seq_has_overflowed(m)) return 0; local_irq_save(flags); if (t->next != nextt) diff --git a/arch/cris/arch-v32/kernel/setup.c b/arch/cris/arch-v32/kernel/setup.c index 61e10ae65296..81715c683baf 100644 --- a/arch/cris/arch-v32/kernel/setup.c +++ b/arch/cris/arch-v32/kernel/setup.c @@ -77,36 +77,38 @@ int show_cpuinfo(struct seq_file *m, void *v) } } - return seq_printf(m, - "processor\t: %d\n" - "cpu\t\t: CRIS\n" - "cpu revision\t: %lu\n" - "cpu model\t: %s\n" - "cache size\t: %d KB\n" - "fpu\t\t: %s\n" - "mmu\t\t: %s\n" - "mmu DMA bug\t: %s\n" - "ethernet\t: %s Mbps\n" - "token ring\t: %s\n" - "scsi\t\t: %s\n" - "ata\t\t: %s\n" - "usb\t\t: %s\n" - "bogomips\t: %lu.%02lu\n\n", - - cpu, - revision, - info->cpu_model, - info->cache_size, - info->flags & HAS_FPU ? "yes" : "no", - info->flags & HAS_MMU ? "yes" : "no", - info->flags & HAS_MMU_BUG ? "yes" : "no", - info->flags & HAS_ETHERNET100 ? "10/100" : "10", - info->flags & HAS_TOKENRING ? "4/16 Mbps" : "no", - info->flags & HAS_SCSI ? "yes" : "no", - info->flags & HAS_ATA ? "yes" : "no", - info->flags & HAS_USB ? "yes" : "no", - (loops_per_jiffy * HZ + 500) / 500000, - ((loops_per_jiffy * HZ + 500) / 5000) % 100); + seq_printf(m, + "processor\t: %d\n" + "cpu\t\t: CRIS\n" + "cpu revision\t: %lu\n" + "cpu model\t: %s\n" + "cache size\t: %d KB\n" + "fpu\t\t: %s\n" + "mmu\t\t: %s\n" + "mmu DMA bug\t: %s\n" + "ethernet\t: %s Mbps\n" + "token ring\t: %s\n" + "scsi\t\t: %s\n" + "ata\t\t: %s\n" + "usb\t\t: %s\n" + "bogomips\t: %lu.%02lu\n\n", + + cpu, + revision, + info->cpu_model, + info->cache_size, + info->flags & HAS_FPU ? "yes" : "no", + info->flags & HAS_MMU ? "yes" : "no", + info->flags & HAS_MMU_BUG ? "yes" : "no", + info->flags & HAS_ETHERNET100 ? "10/100" : "10", + info->flags & HAS_TOKENRING ? "4/16 Mbps" : "no", + info->flags & HAS_SCSI ? "yes" : "no", + info->flags & HAS_ATA ? "yes" : "no", + info->flags & HAS_USB ? "yes" : "no", + (loops_per_jiffy * HZ + 500) / 500000, + ((loops_per_jiffy * HZ + 500) / 5000) % 100); + + return 0; } #endif /* CONFIG_PROC_FS */ diff --git a/arch/cris/arch-v32/kernel/signal.c b/arch/cris/arch-v32/kernel/signal.c index 870e3e069318..0c9ce9eac614 100644 --- a/arch/cris/arch-v32/kernel/signal.c +++ b/arch/cris/arch-v32/kernel/signal.c @@ -287,8 +287,6 @@ setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - /* TODO: what is the current->exec_domain stuff and invmap ? */ - err |= __put_user(&frame->info, &frame->pinfo); err |= __put_user(&frame->uc, &frame->puc); err |= copy_siginfo_to_user(&frame->info, &ksig->info); diff --git a/arch/cris/include/asm/thread_info.h b/arch/cris/include/asm/thread_info.h index 7286db5ed90e..4ead1b40d2d7 100644 --- a/arch/cris/include/asm/thread_info.h +++ b/arch/cris/include/asm/thread_info.h @@ -28,7 +28,6 @@ #ifndef __ASSEMBLY__ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ __u32 cpu; /* current CPU */ int preempt_count; /* 0 => preemptable, <0 => BUG */ @@ -50,7 +49,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/frv/include/asm/thread_info.h b/arch/frv/include/asm/thread_info.h index 6b917f1c2955..ccba3b6ce918 100644 --- a/arch/frv/include/asm/thread_info.h +++ b/arch/frv/include/asm/thread_info.h @@ -31,7 +31,6 @@ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ __u32 cpu; /* current CPU */ @@ -59,7 +58,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/frv/kernel/asm-offsets.c b/arch/frv/kernel/asm-offsets.c index 446e89d500cc..8414293f213a 100644 --- a/arch/frv/kernel/asm-offsets.c +++ b/arch/frv/kernel/asm-offsets.c @@ -34,7 +34,6 @@ void foo(void) { /* offsets into the thread_info structure */ OFFSET(TI_TASK, thread_info, task); - OFFSET(TI_EXEC_DOMAIN, thread_info, exec_domain); OFFSET(TI_FLAGS, thread_info, flags); OFFSET(TI_STATUS, thread_info, status); OFFSET(TI_CPU, thread_info, cpu); diff --git a/arch/frv/kernel/signal.c b/arch/frv/kernel/signal.c index 85ca6727ca07..82d5e914dc15 100644 --- a/arch/frv/kernel/signal.c +++ b/arch/frv/kernel/signal.c @@ -174,20 +174,14 @@ static inline void __user *get_sigframe(struct ksignal *ksig, static int setup_frame(struct ksignal *ksig, sigset_t *set) { struct sigframe __user *frame; - int rsig, sig = ksig->sig; + int sig = ksig->sig; frame = get_sigframe(ksig, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - rsig = sig; - if (sig < 32 && - __current_thread_info->exec_domain && - __current_thread_info->exec_domain->signal_invmap) - rsig = __current_thread_info->exec_domain->signal_invmap[sig]; - - if (__put_user(rsig, &frame->sig) < 0) + if (__put_user(sig, &frame->sig) < 0) return -EFAULT; if (setup_sigcontext(&frame->sc, set->sig[0])) @@ -253,20 +247,14 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set) static int setup_rt_frame(struct ksignal *ksig, sigset_t *set) { struct rt_sigframe __user *frame; - int rsig, sig = ksig->sig; + int sig = ksig->sig; frame = get_sigframe(ksig, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - rsig = sig; - if (sig < 32 && - __current_thread_info->exec_domain && - __current_thread_info->exec_domain->signal_invmap) - rsig = __current_thread_info->exec_domain->signal_invmap[sig]; - - if (__put_user(rsig, &frame->sig) || + if (__put_user(sig, &frame->sig) || __put_user(&frame->info, &frame->pinfo) || __put_user(&frame->uc, &frame->puc)) return -EFAULT; diff --git a/arch/hexagon/include/asm/thread_info.h b/arch/hexagon/include/asm/thread_info.h index bacd3d6030c5..b80fe1db7b64 100644 --- a/arch/hexagon/include/asm/thread_info.h +++ b/arch/hexagon/include/asm/thread_info.h @@ -47,7 +47,6 @@ typedef struct { struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ __u32 cpu; /* current cpu */ int preempt_count; /* 0=>preemptible,<0=>BUG */ @@ -77,7 +76,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = 1, \ diff --git a/arch/ia64/include/asm/thread_info.h b/arch/ia64/include/asm/thread_info.h index c16f21a068ff..aa995b67c3f5 100644 --- a/arch/ia64/include/asm/thread_info.h +++ b/arch/ia64/include/asm/thread_info.h @@ -20,7 +20,6 @@ */ struct thread_info { struct task_struct *task; /* XXX not really needed, except for dup_task_struct() */ - struct exec_domain *exec_domain;/* execution domain */ __u32 flags; /* thread_info flags (see TIF_*) */ __u32 cpu; /* current CPU */ __u32 last_cpu; /* Last CPU thread ran on */ @@ -40,7 +39,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .addr_limit = KERNEL_DS, \ diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index 20678a9ed11a..d68b5cf81e31 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile @@ -51,7 +51,7 @@ obj-$(CONFIG_BINFMT_ELF) += elfcore.o CFLAGS_traps.o += -mfixed-range=f2-f5,f16-f31 # The gate DSO image is built using a special linker script. -include $(srctree)/arch/ia64/kernel/Makefile.gate +include $(src)/Makefile.gate # tell compiled for native CPPFLAGS_gate.lds += -D__IA64_GATE_PARAVIRTUALIZED_NATIVE diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c index 6b3345758d3e..a9b65cf7b34a 100644 --- a/arch/ia64/mm/init.c +++ b/arch/ia64/mm/init.c @@ -692,31 +692,6 @@ int arch_remove_memory(u64 start, u64 size) #endif #endif -/* - * Even when CONFIG_IA32_SUPPORT is not enabled it is - * useful to have the Linux/x86 domain registered to - * avoid an attempted module load when emulators call - * personality(PER_LINUX32). This saves several milliseconds - * on each such call. - */ -static struct exec_domain ia32_exec_domain; - -static int __init -per_linux32_init(void) -{ - ia32_exec_domain.name = "Linux/x86"; - ia32_exec_domain.handler = NULL; - ia32_exec_domain.pers_low = PER_LINUX32; - ia32_exec_domain.pers_high = PER_LINUX32; - ia32_exec_domain.signal_map = default_exec_domain.signal_map; - ia32_exec_domain.signal_invmap = default_exec_domain.signal_invmap; - register_exec_domain(&ia32_exec_domain); - - return 0; -} - -__initcall(per_linux32_init); - /** * show_mem - give short summary of memory stats * diff --git a/arch/m32r/include/asm/asm-offsets.h b/arch/m32r/include/asm/asm-offsets.h new file mode 100644 index 000000000000..d370ee36a182 --- /dev/null +++ b/arch/m32r/include/asm/asm-offsets.h @@ -0,0 +1 @@ +#include <generated/asm-offsets.h> diff --git a/arch/m32r/include/asm/thread_info.h b/arch/m32r/include/asm/thread_info.h index 32422d0211c3..f630d9c30b28 100644 --- a/arch/m32r/include/asm/thread_info.h +++ b/arch/m32r/include/asm/thread_info.h @@ -24,7 +24,6 @@ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ __u32 cpu; /* current CPU */ @@ -38,18 +37,7 @@ struct thread_info { __u8 supervisor_stack[0]; }; -#else /* !__ASSEMBLY__ */ - -/* offsets into the thread_info struct for assembly code access */ -#define TI_TASK 0x00000000 -#define TI_EXEC_DOMAIN 0x00000004 -#define TI_FLAGS 0x00000008 -#define TI_STATUS 0x0000000C -#define TI_CPU 0x00000010 -#define TI_PRE_COUNT 0x00000014 -#define TI_ADDR_LIMIT 0x00000018 - -#endif +#endif /* !__ASSEMBLY__ */ #define THREAD_SIZE (PAGE_SIZE << 1) #define THREAD_SIZE_ORDER 1 @@ -61,7 +49,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/m32r/kernel/asm-offsets.c b/arch/m32r/kernel/asm-offsets.c index 9e263112a6e2..cd3d2fc9c8df 100644 --- a/arch/m32r/kernel/asm-offsets.c +++ b/arch/m32r/kernel/asm-offsets.c @@ -1 +1,14 @@ -/* Dummy asm-offsets.c file. Required by kbuild and ready to be used - hint! */ +#include <linux/thread_info.h> +#include <linux/kbuild.h> + +int foo(void) +{ + OFFSET(TI_TASK, thread_info, task); + OFFSET(TI_FLAGS, thread_info, flags); + OFFSET(TI_STATUS, thread_info, status); + OFFSET(TI_CPU, thread_info, cpu); + OFFSET(TI_PRE_COUNT, thread_info, preempt_count); + OFFSET(TI_ADDR_LIMIT, thread_info, addr_limit); + + return 0; +} diff --git a/arch/m32r/kernel/entry.S b/arch/m32r/kernel/entry.S index 7c3db9940ce1..c639bfa32232 100644 --- a/arch/m32r/kernel/entry.S +++ b/arch/m32r/kernel/entry.S @@ -65,6 +65,7 @@ #include <asm/page.h> #include <asm/m32r.h> #include <asm/mmu_context.h> +#include <asm/asm-offsets.h> #if !defined(CONFIG_MMU) #define sys_madvise sys_ni_syscall diff --git a/arch/m32r/kernel/signal.c b/arch/m32r/kernel/signal.c index 8c25e0c8f6a5..1c81e24fd006 100644 --- a/arch/m32r/kernel/signal.c +++ b/arch/m32r/kernel/signal.c @@ -172,20 +172,14 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, { struct rt_sigframe __user *frame; int err = 0; - int signal, sig = ksig->sig; + int sig = ksig->sig; frame = get_sigframe(ksig, regs->spu, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - signal = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - - err |= __put_user(signal, &frame->sig); + err |= __put_user(sig, &frame->sig); if (err) return -EFAULT; @@ -209,7 +203,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, /* Set up registers for signal handler */ regs->spu = (unsigned long)frame; - regs->r0 = signal; /* Arg for signal handler */ + regs->r0 = sig; /* Arg for signal handler */ regs->r1 = (unsigned long)&frame->info; regs->r2 = (unsigned long)&frame->uc; regs->bpc = (unsigned long)ksig->ka.sa.sa_handler; diff --git a/arch/m68k/include/asm/thread_info.h b/arch/m68k/include/asm/thread_info.h index c54256e69e64..cee13c2e5161 100644 --- a/arch/m68k/include/asm/thread_info.h +++ b/arch/m68k/include/asm/thread_info.h @@ -26,7 +26,6 @@ struct thread_info { struct task_struct *task; /* main task structure */ unsigned long flags; - struct exec_domain *exec_domain; /* execution domain */ mm_segment_t addr_limit; /* thread address space */ int preempt_count; /* 0 => preemptable, <0 => BUG */ __u32 cpu; /* should always be 0 on m68k */ @@ -37,7 +36,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .addr_limit = KERNEL_DS, \ .preempt_count = INIT_PREEMPT_COUNT, \ } diff --git a/arch/m68k/kernel/signal.c b/arch/m68k/kernel/signal.c index d7179281e74a..af1c4f330aef 100644 --- a/arch/m68k/kernel/signal.c +++ b/arch/m68k/kernel/signal.c @@ -863,12 +863,7 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, if (fsize) err |= copy_to_user (frame + 1, regs + 1, fsize); - err |= __put_user((current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig), - &frame->sig); + err |= __put_user(sig, &frame->sig); err |= __put_user(regs->vector, &frame->code); err |= __put_user(&frame->sc, &frame->psc); @@ -948,12 +943,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, if (fsize) err |= copy_to_user (&frame->uc.uc_extra, regs + 1, fsize); - err |= __put_user((current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig), - &frame->sig); + err |= __put_user(sig, &frame->sig); err |= __put_user(&frame->info, &frame->pinfo); err |= __put_user(&frame->uc, &frame->puc); err |= copy_siginfo_to_user(&frame->info, &ksig->info); diff --git a/arch/metag/include/asm/thread_info.h b/arch/metag/include/asm/thread_info.h index afb3ca4776d1..32677cc278aa 100644 --- a/arch/metag/include/asm/thread_info.h +++ b/arch/metag/include/asm/thread_info.h @@ -28,7 +28,6 @@ /* This must be 8 byte aligned so we can ensure stack alignment. */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ u32 cpu; /* current CPU */ @@ -68,7 +67,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/metag/kernel/irq.c b/arch/metag/kernel/irq.c index 5385dd1216b7..4f8f1f87ef11 100644 --- a/arch/metag/kernel/irq.c +++ b/arch/metag/kernel/irq.c @@ -132,7 +132,6 @@ void irq_ctx_init(int cpu) irqctx = (union irq_ctx *) &hardirq_stack[cpu * THREAD_SIZE]; irqctx->tinfo.task = NULL; - irqctx->tinfo.exec_domain = NULL; irqctx->tinfo.cpu = cpu; irqctx->tinfo.preempt_count = HARDIRQ_OFFSET; irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); @@ -141,7 +140,6 @@ void irq_ctx_init(int cpu) irqctx = (union irq_ctx *) &softirq_stack[cpu * THREAD_SIZE]; irqctx->tinfo.task = NULL; - irqctx->tinfo.exec_domain = NULL; irqctx->tinfo.cpu = cpu; irqctx->tinfo.preempt_count = 0; irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); diff --git a/arch/microblaze/include/asm/seccomp.h b/arch/microblaze/include/asm/seccomp.h index 0d912758a0d7..204618a2ce84 100644 --- a/arch/microblaze/include/asm/seccomp.h +++ b/arch/microblaze/include/asm/seccomp.h @@ -3,14 +3,8 @@ #include <linux/unistd.h> -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit #define __NR_seccomp_sigreturn __NR_sigreturn -#define __NR_seccomp_read_32 __NR_read -#define __NR_seccomp_write_32 __NR_write -#define __NR_seccomp_exit_32 __NR_exit -#define __NR_seccomp_sigreturn_32 __NR_sigreturn +#include <asm-generic/seccomp.h> #endif /* _ASM_MICROBLAZE_SECCOMP_H */ diff --git a/arch/microblaze/include/asm/thread_info.h b/arch/microblaze/include/asm/thread_info.h index b699fbd7de4a..383f387b4eee 100644 --- a/arch/microblaze/include/asm/thread_info.h +++ b/arch/microblaze/include/asm/thread_info.h @@ -65,7 +65,6 @@ typedef struct { struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ __u32 cpu; /* current CPU */ @@ -81,7 +80,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/microblaze/kernel/cpu/mb.c b/arch/microblaze/kernel/cpu/mb.c index 7b5dca7ed39d..9581d194d9e4 100644 --- a/arch/microblaze/kernel/cpu/mb.c +++ b/arch/microblaze/kernel/cpu/mb.c @@ -27,7 +27,6 @@ static int show_cpuinfo(struct seq_file *m, void *v) { - int count = 0; char *fpga_family = "Unknown"; char *cpu_ver = "Unknown"; int i; @@ -48,91 +47,89 @@ static int show_cpuinfo(struct seq_file *m, void *v) } } - count = seq_printf(m, - "CPU-Family: MicroBlaze\n" - "FPGA-Arch: %s\n" - "CPU-Ver: %s, %s endian\n" - "CPU-MHz: %d.%02d\n" - "BogoMips: %lu.%02lu\n", - fpga_family, - cpu_ver, - cpuinfo.endian ? "little" : "big", - cpuinfo.cpu_clock_freq / - 1000000, - cpuinfo.cpu_clock_freq % - 1000000, - loops_per_jiffy / (500000 / HZ), - (loops_per_jiffy / (5000 / HZ)) % 100); - - count += seq_printf(m, - "HW:\n Shift:\t\t%s\n" - " MSR:\t\t%s\n" - " PCMP:\t\t%s\n" - " DIV:\t\t%s\n", - (cpuinfo.use_instr & PVR0_USE_BARREL_MASK) ? "yes" : "no", - (cpuinfo.use_instr & PVR2_USE_MSR_INSTR) ? "yes" : "no", - (cpuinfo.use_instr & PVR2_USE_PCMP_INSTR) ? "yes" : "no", - (cpuinfo.use_instr & PVR0_USE_DIV_MASK) ? "yes" : "no"); - - count += seq_printf(m, - " MMU:\t\t%x\n", - cpuinfo.mmu); - - count += seq_printf(m, - " MUL:\t\t%s\n" - " FPU:\t\t%s\n", - (cpuinfo.use_mult & PVR2_USE_MUL64_MASK) ? "v2" : - (cpuinfo.use_mult & PVR0_USE_HW_MUL_MASK) ? "v1" : "no", - (cpuinfo.use_fpu & PVR2_USE_FPU2_MASK) ? "v2" : - (cpuinfo.use_fpu & PVR0_USE_FPU_MASK) ? "v1" : "no"); - - count += seq_printf(m, - " Exc:\t\t%s%s%s%s%s%s%s%s\n", - (cpuinfo.use_exc & PVR2_OPCODE_0x0_ILL_MASK) ? "op0x0 " : "", - (cpuinfo.use_exc & PVR2_UNALIGNED_EXC_MASK) ? "unal " : "", - (cpuinfo.use_exc & PVR2_ILL_OPCODE_EXC_MASK) ? "ill " : "", - (cpuinfo.use_exc & PVR2_IOPB_BUS_EXC_MASK) ? "iopb " : "", - (cpuinfo.use_exc & PVR2_DOPB_BUS_EXC_MASK) ? "dopb " : "", - (cpuinfo.use_exc & PVR2_DIV_ZERO_EXC_MASK) ? "zero " : "", - (cpuinfo.use_exc & PVR2_FPU_EXC_MASK) ? "fpu " : "", - (cpuinfo.use_exc & PVR2_USE_FSL_EXC) ? "fsl " : ""); - - count += seq_printf(m, - "Stream-insns:\t%sprivileged\n", - cpuinfo.mmu_privins ? "un" : ""); + seq_printf(m, + "CPU-Family: MicroBlaze\n" + "FPGA-Arch: %s\n" + "CPU-Ver: %s, %s endian\n" + "CPU-MHz: %d.%02d\n" + "BogoMips: %lu.%02lu\n", + fpga_family, + cpu_ver, + cpuinfo.endian ? "little" : "big", + cpuinfo.cpu_clock_freq / 1000000, + cpuinfo.cpu_clock_freq % 1000000, + loops_per_jiffy / (500000 / HZ), + (loops_per_jiffy / (5000 / HZ)) % 100); + + seq_printf(m, + "HW:\n Shift:\t\t%s\n" + " MSR:\t\t%s\n" + " PCMP:\t\t%s\n" + " DIV:\t\t%s\n", + (cpuinfo.use_instr & PVR0_USE_BARREL_MASK) ? "yes" : "no", + (cpuinfo.use_instr & PVR2_USE_MSR_INSTR) ? "yes" : "no", + (cpuinfo.use_instr & PVR2_USE_PCMP_INSTR) ? "yes" : "no", + (cpuinfo.use_instr & PVR0_USE_DIV_MASK) ? "yes" : "no"); + + seq_printf(m, " MMU:\t\t%x\n", cpuinfo.mmu); + + seq_printf(m, + " MUL:\t\t%s\n" + " FPU:\t\t%s\n", + (cpuinfo.use_mult & PVR2_USE_MUL64_MASK) ? "v2" : + (cpuinfo.use_mult & PVR0_USE_HW_MUL_MASK) ? "v1" : "no", + (cpuinfo.use_fpu & PVR2_USE_FPU2_MASK) ? "v2" : + (cpuinfo.use_fpu & PVR0_USE_FPU_MASK) ? "v1" : "no"); + + seq_printf(m, + " Exc:\t\t%s%s%s%s%s%s%s%s\n", + (cpuinfo.use_exc & PVR2_OPCODE_0x0_ILL_MASK) ? "op0x0 " : "", + (cpuinfo.use_exc & PVR2_UNALIGNED_EXC_MASK) ? "unal " : "", + (cpuinfo.use_exc & PVR2_ILL_OPCODE_EXC_MASK) ? "ill " : "", + (cpuinfo.use_exc & PVR2_IOPB_BUS_EXC_MASK) ? "iopb " : "", + (cpuinfo.use_exc & PVR2_DOPB_BUS_EXC_MASK) ? "dopb " : "", + (cpuinfo.use_exc & PVR2_DIV_ZERO_EXC_MASK) ? "zero " : "", + (cpuinfo.use_exc & PVR2_FPU_EXC_MASK) ? "fpu " : "", + (cpuinfo.use_exc & PVR2_USE_FSL_EXC) ? "fsl " : ""); + + seq_printf(m, + "Stream-insns:\t%sprivileged\n", + cpuinfo.mmu_privins ? "un" : ""); if (cpuinfo.use_icache) - count += seq_printf(m, - "Icache:\t\t%ukB\tline length:\t%dB\n", - cpuinfo.icache_size >> 10, - cpuinfo.icache_line_length); + seq_printf(m, + "Icache:\t\t%ukB\tline length:\t%dB\n", + cpuinfo.icache_size >> 10, + cpuinfo.icache_line_length); else - count += seq_printf(m, "Icache:\t\tno\n"); + seq_puts(m, "Icache:\t\tno\n"); if (cpuinfo.use_dcache) { - count += seq_printf(m, - "Dcache:\t\t%ukB\tline length:\t%dB\n", - cpuinfo.dcache_size >> 10, - cpuinfo.dcache_line_length); - seq_printf(m, "Dcache-Policy:\t"); + seq_printf(m, + "Dcache:\t\t%ukB\tline length:\t%dB\n", + cpuinfo.dcache_size >> 10, + cpuinfo.dcache_line_length); + seq_puts(m, "Dcache-Policy:\t"); if (cpuinfo.dcache_wb) - count += seq_printf(m, "write-back\n"); + seq_puts(m, "write-back\n"); else - count += seq_printf(m, "write-through\n"); - } else - count += seq_printf(m, "Dcache:\t\tno\n"); + seq_puts(m, "write-through\n"); + } else { + seq_puts(m, "Dcache:\t\tno\n"); + } + + seq_printf(m, + "HW-Debug:\t%s\n", + cpuinfo.hw_debug ? "yes" : "no"); - count += seq_printf(m, - "HW-Debug:\t%s\n", - cpuinfo.hw_debug ? "yes" : "no"); + seq_printf(m, + "PVR-USR1:\t%02x\n" + "PVR-USR2:\t%08x\n", + cpuinfo.pvr_user1, + cpuinfo.pvr_user2); - count += seq_printf(m, - "PVR-USR1:\t%02x\n" - "PVR-USR2:\t%08x\n", - cpuinfo.pvr_user1, - cpuinfo.pvr_user2); + seq_printf(m, "Page size:\t%lu\n", PAGE_SIZE); - count += seq_printf(m, "Page size:\t%lu\n", PAGE_SIZE); return 0; } diff --git a/arch/microblaze/kernel/signal.c b/arch/microblaze/kernel/signal.c index 20ccd4e2baa5..97001524ca2d 100644 --- a/arch/microblaze/kernel/signal.c +++ b/arch/microblaze/kernel/signal.c @@ -158,7 +158,6 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, { struct rt_sigframe __user *frame; int err = 0, sig = ksig->sig; - unsigned long signal; unsigned long address = 0; #ifdef CONFIG_MMU pmd_t *pmdp; @@ -170,12 +169,6 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - signal = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : (unsigned long)sig; - if (ksig->ka.sa.sa_flags & SA_SIGINFO) err |= copy_siginfo_to_user(&frame->info, &ksig->info); @@ -230,7 +223,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, regs->r1 = (unsigned long) frame; /* Signal handler args: */ - regs->r5 = signal; /* arg 0: signum */ + regs->r5 = sig; /* arg 0: signum */ regs->r6 = (unsigned long) &frame->info; /* arg 1: siginfo */ regs->r7 = (unsigned long) &frame->uc; /* arg2: ucontext */ /* Offset to handle microblaze rtid r14, 0 */ diff --git a/arch/mips/Makefile b/arch/mips/Makefile index 8f57fc72d62c..d152dfbc360d 100644 --- a/arch/mips/Makefile +++ b/arch/mips/Makefile @@ -225,7 +225,7 @@ endif # # Board-dependent options and extra files # -include $(srctree)/arch/mips/Kbuild.platforms +include arch/mips/Kbuild.platforms ifdef CONFIG_PHYSICAL_START load-y = $(CONFIG_PHYSICAL_START) diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile index a74f76d85a2f..f7aa9d5d3b87 100644 --- a/arch/mips/cavium-octeon/crypto/Makefile +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -4,4 +4,7 @@ obj-y += octeon-crypto.o -obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o +obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o +obj-$(CONFIG_CRYPTO_SHA1_OCTEON) += octeon-sha1.o +obj-$(CONFIG_CRYPTO_SHA256_OCTEON) += octeon-sha256.o +obj-$(CONFIG_CRYPTO_SHA512_OCTEON) += octeon-sha512.o diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.c b/arch/mips/cavium-octeon/crypto/octeon-crypto.c index 7c82ff463b65..f66bd1adc7ff 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.c +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.c @@ -17,7 +17,7 @@ * crypto operations in calls to octeon_crypto_enable/disable in order to make * sure the state of COP2 isn't corrupted if userspace is also performing * hardware crypto operations. Allocate the state parameter on the stack. - * Preemption must be disabled to prevent context switches. + * Returns with preemption disabled. * * @state: Pointer to state structure to store current COP2 state in. * @@ -28,6 +28,7 @@ unsigned long octeon_crypto_enable(struct octeon_cop2_state *state) int status; unsigned long flags; + preempt_disable(); local_irq_save(flags); status = read_c0_status(); write_c0_status(status | ST0_CU2); @@ -62,5 +63,6 @@ void octeon_crypto_disable(struct octeon_cop2_state *state, else write_c0_status(read_c0_status() & ~ST0_CU2); local_irq_restore(flags); + preempt_enable(); } EXPORT_SYMBOL_GPL(octeon_crypto_disable); diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h index e2a4aece9c24..355072535110 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-crypto.h +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -5,7 +5,8 @@ * * Copyright (C) 2012-2013 Cavium Inc., All Rights Reserved. * - * MD5 instruction definitions added by Aaro Koskinen <aaro.koskinen@iki.fi>. + * MD5/SHA1/SHA256/SHA512 instruction definitions added by + * Aaro Koskinen <aaro.koskinen@iki.fi>. * */ #ifndef __LINUX_OCTEON_CRYPTO_H @@ -21,11 +22,11 @@ extern void octeon_crypto_disable(struct octeon_cop2_state *state, unsigned long flags); /* - * Macros needed to implement MD5: + * Macros needed to implement MD5/SHA1/SHA256: */ /* - * The index can be 0-1. + * The index can be 0-1 (MD5) or 0-2 (SHA1), 0-3 (SHA256). */ #define write_octeon_64bit_hash_dword(value, index) \ do { \ @@ -36,7 +37,7 @@ do { \ } while (0) /* - * The index can be 0-1. + * The index can be 0-1 (MD5) or 0-2 (SHA1), 0-3 (SHA256). */ #define read_octeon_64bit_hash_dword(index) \ ({ \ @@ -72,4 +73,78 @@ do { \ : [rt] "d" (value)); \ } while (0) +/* + * The value is the final block dword (64-bit). + */ +#define octeon_sha1_start(value) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x4057" \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * The value is the final block dword (64-bit). + */ +#define octeon_sha256_start(value) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x404f" \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * Macros needed to implement SHA512: + */ + +/* + * The index can be 0-7. + */ +#define write_octeon_64bit_hash_sha512(value, index) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x0250+" STR(index) \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * The index can be 0-7. + */ +#define read_octeon_64bit_hash_sha512(index) \ +({ \ + u64 __value; \ + \ + __asm__ __volatile__ ( \ + "dmfc2 %[rt],0x0250+" STR(index) \ + : [rt] "=d" (__value) \ + : ); \ + \ + __value; \ +}) + +/* + * The index can be 0-14. + */ +#define write_octeon_64bit_block_sha512(value, index) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x0240+" STR(index) \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * The value is the final block word (64-bit). + */ +#define octeon_sha512_start(value) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x424f" \ + : \ + : [rt] "d" (value)); \ +} while (0) + #endif /* __LINUX_OCTEON_CRYPTO_H */ diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c index b909881ba6c1..12dccdb38286 100644 --- a/arch/mips/cavium-octeon/crypto/octeon-md5.c +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -97,8 +97,6 @@ static int octeon_md5_update(struct shash_desc *desc, const u8 *data, memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), data, avail); - local_bh_disable(); - preempt_disable(); flags = octeon_crypto_enable(&state); octeon_md5_store_hash(mctx); @@ -114,8 +112,6 @@ static int octeon_md5_update(struct shash_desc *desc, const u8 *data, octeon_md5_read_hash(mctx); octeon_crypto_disable(&state, flags); - preempt_enable(); - local_bh_enable(); memcpy(mctx->block, data, len); @@ -133,8 +129,6 @@ static int octeon_md5_final(struct shash_desc *desc, u8 *out) *p++ = 0x80; - local_bh_disable(); - preempt_disable(); flags = octeon_crypto_enable(&state); octeon_md5_store_hash(mctx); @@ -152,8 +146,6 @@ static int octeon_md5_final(struct shash_desc *desc, u8 *out) octeon_md5_read_hash(mctx); octeon_crypto_disable(&state, flags); - preempt_enable(); - local_bh_enable(); memcpy(out, mctx->hash, sizeof(mctx->hash)); memset(mctx, 0, sizeof(*mctx)); diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha1.c b/arch/mips/cavium-octeon/crypto/octeon-sha1.c new file mode 100644 index 000000000000..2b74b5b67cae --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-sha1.c @@ -0,0 +1,241 @@ +/* + * Cryptographic API. + * + * SHA1 Secure Hash Algorithm. + * + * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>. + * + * Based on crypto/sha1_generic.c, which is: + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include <linux/mm.h> +#include <crypto/sha.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/module.h> +#include <asm/byteorder.h> +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> + +#include "octeon-crypto.h" + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void octeon_sha1_store_hash(struct sha1_state *sctx) +{ + u64 *hash = (u64 *)sctx->state; + union { + u32 word[2]; + u64 dword; + } hash_tail = { { sctx->state[4], } }; + + write_octeon_64bit_hash_dword(hash[0], 0); + write_octeon_64bit_hash_dword(hash[1], 1); + write_octeon_64bit_hash_dword(hash_tail.dword, 2); + memzero_explicit(&hash_tail.word[0], sizeof(hash_tail.word[0])); +} + +static void octeon_sha1_read_hash(struct sha1_state *sctx) +{ + u64 *hash = (u64 *)sctx->state; + union { + u32 word[2]; + u64 dword; + } hash_tail; + + hash[0] = read_octeon_64bit_hash_dword(0); + hash[1] = read_octeon_64bit_hash_dword(1); + hash_tail.dword = read_octeon_64bit_hash_dword(2); + sctx->state[4] = hash_tail.word[0]; + memzero_explicit(&hash_tail.dword, sizeof(hash_tail.dword)); +} + +static void octeon_sha1_transform(const void *_block) +{ + const u64 *block = _block; + + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_sha1_start(block[7]); +} + +static int octeon_sha1_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA1_H0; + sctx->state[1] = SHA1_H1; + sctx->state[2] = SHA1_H2; + sctx->state[3] = SHA1_H3; + sctx->state[4] = SHA1_H4; + sctx->count = 0; + + return 0; +} + +static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data, + unsigned int len) +{ + unsigned int partial; + unsigned int done; + const u8 *src; + + partial = sctx->count % SHA1_BLOCK_SIZE; + sctx->count += len; + done = 0; + src = data; + + if ((partial + len) >= SHA1_BLOCK_SIZE) { + if (partial) { + done = -partial; + memcpy(sctx->buffer + partial, data, + done + SHA1_BLOCK_SIZE); + src = sctx->buffer; + } + + do { + octeon_sha1_transform(src); + done += SHA1_BLOCK_SIZE; + src = data + done; + } while (done + SHA1_BLOCK_SIZE <= len); + + partial = 0; + } + memcpy(sctx->buffer + partial, src, len - done); +} + +static int octeon_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + struct octeon_cop2_state state; + unsigned long flags; + + /* + * Small updates never reach the crypto engine, so the generic sha1 is + * faster because of the heavyweight octeon_crypto_enable() / + * octeon_crypto_disable(). + */ + if ((sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) + return crypto_sha1_update(desc, data, len); + + flags = octeon_crypto_enable(&state); + octeon_sha1_store_hash(sctx); + + __octeon_sha1_update(sctx, data, len); + + octeon_sha1_read_hash(sctx); + octeon_crypto_disable(&state, flags); + + return 0; +} + +static int octeon_sha1_final(struct shash_desc *desc, u8 *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + static const u8 padding[64] = { 0x80, }; + struct octeon_cop2_state state; + __be32 *dst = (__be32 *)out; + unsigned int pad_len; + unsigned long flags; + unsigned int index; + __be64 bits; + int i; + + /* Save number of bits. */ + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64. */ + index = sctx->count & 0x3f; + pad_len = (index < 56) ? (56 - index) : ((64+56) - index); + + flags = octeon_crypto_enable(&state); + octeon_sha1_store_hash(sctx); + + __octeon_sha1_update(sctx, padding, pad_len); + + /* Append length (before padding). */ + __octeon_sha1_update(sctx, (const u8 *)&bits, sizeof(bits)); + + octeon_sha1_read_hash(sctx); + octeon_crypto_disable(&state, flags); + + /* Store state in digest */ + for (i = 0; i < 5; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Zeroize sensitive information. */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int octeon_sha1_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int octeon_sha1_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + +static struct shash_alg octeon_sha1_alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = octeon_sha1_init, + .update = octeon_sha1_update, + .final = octeon_sha1_final, + .export = octeon_sha1_export, + .import = octeon_sha1_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name= "octeon-sha1", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init octeon_sha1_mod_init(void) +{ + if (!octeon_has_crypto()) + return -ENOTSUPP; + return crypto_register_shash(&octeon_sha1_alg); +} + +static void __exit octeon_sha1_mod_fini(void) +{ + crypto_unregister_shash(&octeon_sha1_alg); +} + +module_init(octeon_sha1_mod_init); +module_exit(octeon_sha1_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm (OCTEON)"); +MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>"); diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha256.c b/arch/mips/cavium-octeon/crypto/octeon-sha256.c new file mode 100644 index 000000000000..97e96fead08a --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-sha256.c @@ -0,0 +1,280 @@ +/* + * Cryptographic API. + * + * SHA-224 and SHA-256 Secure Hash Algorithm. + * + * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>. + * + * Based on crypto/sha256_generic.c, which is: + * + * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com> + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> + * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include <linux/mm.h> +#include <crypto/sha.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/module.h> +#include <asm/byteorder.h> +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> + +#include "octeon-crypto.h" + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void octeon_sha256_store_hash(struct sha256_state *sctx) +{ + u64 *hash = (u64 *)sctx->state; + + write_octeon_64bit_hash_dword(hash[0], 0); + write_octeon_64bit_hash_dword(hash[1], 1); + write_octeon_64bit_hash_dword(hash[2], 2); + write_octeon_64bit_hash_dword(hash[3], 3); +} + +static void octeon_sha256_read_hash(struct sha256_state *sctx) +{ + u64 *hash = (u64 *)sctx->state; + + hash[0] = read_octeon_64bit_hash_dword(0); + hash[1] = read_octeon_64bit_hash_dword(1); + hash[2] = read_octeon_64bit_hash_dword(2); + hash[3] = read_octeon_64bit_hash_dword(3); +} + +static void octeon_sha256_transform(const void *_block) +{ + const u64 *block = _block; + + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_sha256_start(block[7]); +} + +static int octeon_sha224_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; + + return 0; +} + +static int octeon_sha256_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; + + return 0; +} + +static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data, + unsigned int len) +{ + unsigned int partial; + unsigned int done; + const u8 *src; + + partial = sctx->count % SHA256_BLOCK_SIZE; + sctx->count += len; + done = 0; + src = data; + + if ((partial + len) >= SHA256_BLOCK_SIZE) { + if (partial) { + done = -partial; + memcpy(sctx->buf + partial, data, + done + SHA256_BLOCK_SIZE); + src = sctx->buf; + } + + do { + octeon_sha256_transform(src); + done += SHA256_BLOCK_SIZE; + src = data + done; + } while (done + SHA256_BLOCK_SIZE <= len); + + partial = 0; + } + memcpy(sctx->buf + partial, src, len - done); +} + +static int octeon_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + struct octeon_cop2_state state; + unsigned long flags; + + /* + * Small updates never reach the crypto engine, so the generic sha256 is + * faster because of the heavyweight octeon_crypto_enable() / + * octeon_crypto_disable(). + */ + if ((sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) + return crypto_sha256_update(desc, data, len); + + flags = octeon_crypto_enable(&state); + octeon_sha256_store_hash(sctx); + + __octeon_sha256_update(sctx, data, len); + + octeon_sha256_read_hash(sctx); + octeon_crypto_disable(&state, flags); + + return 0; +} + +static int octeon_sha256_final(struct shash_desc *desc, u8 *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + static const u8 padding[64] = { 0x80, }; + struct octeon_cop2_state state; + __be32 *dst = (__be32 *)out; + unsigned int pad_len; + unsigned long flags; + unsigned int index; + __be64 bits; + int i; + + /* Save number of bits. */ + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64. */ + index = sctx->count & 0x3f; + pad_len = (index < 56) ? (56 - index) : ((64+56) - index); + + flags = octeon_crypto_enable(&state); + octeon_sha256_store_hash(sctx); + + __octeon_sha256_update(sctx, padding, pad_len); + + /* Append length (before padding). */ + __octeon_sha256_update(sctx, (const u8 *)&bits, sizeof(bits)); + + octeon_sha256_read_hash(sctx); + octeon_crypto_disable(&state, flags); + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Zeroize sensitive information. */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int octeon_sha224_final(struct shash_desc *desc, u8 *hash) +{ + u8 D[SHA256_DIGEST_SIZE]; + + octeon_sha256_final(desc, D); + + memcpy(hash, D, SHA224_DIGEST_SIZE); + memzero_explicit(D, SHA256_DIGEST_SIZE); + + return 0; +} + +static int octeon_sha256_export(struct shash_desc *desc, void *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int octeon_sha256_import(struct shash_desc *desc, const void *in) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + +static struct shash_alg octeon_sha256_algs[2] = { { + .digestsize = SHA256_DIGEST_SIZE, + .init = octeon_sha256_init, + .update = octeon_sha256_update, + .final = octeon_sha256_final, + .export = octeon_sha256_export, + .import = octeon_sha256_import, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name= "octeon-sha256", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA224_DIGEST_SIZE, + .init = octeon_sha224_init, + .update = octeon_sha256_update, + .final = octeon_sha224_final, + .descsize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name= "octeon-sha224", + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init octeon_sha256_mod_init(void) +{ + if (!octeon_has_crypto()) + return -ENOTSUPP; + return crypto_register_shashes(octeon_sha256_algs, + ARRAY_SIZE(octeon_sha256_algs)); +} + +static void __exit octeon_sha256_mod_fini(void) +{ + crypto_unregister_shashes(octeon_sha256_algs, + ARRAY_SIZE(octeon_sha256_algs)); +} + +module_init(octeon_sha256_mod_init); +module_exit(octeon_sha256_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm (OCTEON)"); +MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>"); diff --git a/arch/mips/cavium-octeon/crypto/octeon-sha512.c b/arch/mips/cavium-octeon/crypto/octeon-sha512.c new file mode 100644 index 000000000000..d5fb3c6f22ae --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-sha512.c @@ -0,0 +1,277 @@ +/* + * Cryptographic API. + * + * SHA-512 and SHA-384 Secure Hash Algorithm. + * + * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>. + * + * Based on crypto/sha512_generic.c, which is: + * + * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com> + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) 2003 Kyle McMartin <kyle@debian.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + */ + +#include <linux/mm.h> +#include <crypto/sha.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/module.h> +#include <asm/byteorder.h> +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> + +#include "octeon-crypto.h" + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void octeon_sha512_store_hash(struct sha512_state *sctx) +{ + write_octeon_64bit_hash_sha512(sctx->state[0], 0); + write_octeon_64bit_hash_sha512(sctx->state[1], 1); + write_octeon_64bit_hash_sha512(sctx->state[2], 2); + write_octeon_64bit_hash_sha512(sctx->state[3], 3); + write_octeon_64bit_hash_sha512(sctx->state[4], 4); + write_octeon_64bit_hash_sha512(sctx->state[5], 5); + write_octeon_64bit_hash_sha512(sctx->state[6], 6); + write_octeon_64bit_hash_sha512(sctx->state[7], 7); +} + +static void octeon_sha512_read_hash(struct sha512_state *sctx) +{ + sctx->state[0] = read_octeon_64bit_hash_sha512(0); + sctx->state[1] = read_octeon_64bit_hash_sha512(1); + sctx->state[2] = read_octeon_64bit_hash_sha512(2); + sctx->state[3] = read_octeon_64bit_hash_sha512(3); + sctx->state[4] = read_octeon_64bit_hash_sha512(4); + sctx->state[5] = read_octeon_64bit_hash_sha512(5); + sctx->state[6] = read_octeon_64bit_hash_sha512(6); + sctx->state[7] = read_octeon_64bit_hash_sha512(7); +} + +static void octeon_sha512_transform(const void *_block) +{ + const u64 *block = _block; + + write_octeon_64bit_block_sha512(block[0], 0); + write_octeon_64bit_block_sha512(block[1], 1); + write_octeon_64bit_block_sha512(block[2], 2); + write_octeon_64bit_block_sha512(block[3], 3); + write_octeon_64bit_block_sha512(block[4], 4); + write_octeon_64bit_block_sha512(block[5], 5); + write_octeon_64bit_block_sha512(block[6], 6); + write_octeon_64bit_block_sha512(block[7], 7); + write_octeon_64bit_block_sha512(block[8], 8); + write_octeon_64bit_block_sha512(block[9], 9); + write_octeon_64bit_block_sha512(block[10], 10); + write_octeon_64bit_block_sha512(block[11], 11); + write_octeon_64bit_block_sha512(block[12], 12); + write_octeon_64bit_block_sha512(block[13], 13); + write_octeon_64bit_block_sha512(block[14], 14); + octeon_sha512_start(block[15]); +} + +static int octeon_sha512_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA512_H0; + sctx->state[1] = SHA512_H1; + sctx->state[2] = SHA512_H2; + sctx->state[3] = SHA512_H3; + sctx->state[4] = SHA512_H4; + sctx->state[5] = SHA512_H5; + sctx->state[6] = SHA512_H6; + sctx->state[7] = SHA512_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int octeon_sha384_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA384_H0; + sctx->state[1] = SHA384_H1; + sctx->state[2] = SHA384_H2; + sctx->state[3] = SHA384_H3; + sctx->state[4] = SHA384_H4; + sctx->state[5] = SHA384_H5; + sctx->state[6] = SHA384_H6; + sctx->state[7] = SHA384_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data, + unsigned int len) +{ + unsigned int part_len; + unsigned int index; + unsigned int i; + + /* Compute number of bytes mod 128. */ + index = sctx->count[0] % SHA512_BLOCK_SIZE; + + /* Update number of bytes. */ + if ((sctx->count[0] += len) < len) + sctx->count[1]++; + + part_len = SHA512_BLOCK_SIZE - index; + + /* Transform as many times as possible. */ + if (len >= part_len) { + memcpy(&sctx->buf[index], data, part_len); + octeon_sha512_transform(sctx->buf); + + for (i = part_len; i + SHA512_BLOCK_SIZE <= len; + i += SHA512_BLOCK_SIZE) + octeon_sha512_transform(&data[i]); + + index = 0; + } else { + i = 0; + } + + /* Buffer remaining input. */ + memcpy(&sctx->buf[index], &data[i], len - i); +} + +static int octeon_sha512_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + struct octeon_cop2_state state; + unsigned long flags; + + /* + * Small updates never reach the crypto engine, so the generic sha512 is + * faster because of the heavyweight octeon_crypto_enable() / + * octeon_crypto_disable(). + */ + if ((sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) + return crypto_sha512_update(desc, data, len); + + flags = octeon_crypto_enable(&state); + octeon_sha512_store_hash(sctx); + + __octeon_sha512_update(sctx, data, len); + + octeon_sha512_read_hash(sctx); + octeon_crypto_disable(&state, flags); + + return 0; +} + +static int octeon_sha512_final(struct shash_desc *desc, u8 *hash) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + static u8 padding[128] = { 0x80, }; + struct octeon_cop2_state state; + __be64 *dst = (__be64 *)hash; + unsigned int pad_len; + unsigned long flags; + unsigned int index; + __be64 bits[2]; + int i; + + /* Save number of bits. */ + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); + + /* Pad out to 112 mod 128. */ + index = sctx->count[0] & 0x7f; + pad_len = (index < 112) ? (112 - index) : ((128+112) - index); + + flags = octeon_crypto_enable(&state); + octeon_sha512_store_hash(sctx); + + __octeon_sha512_update(sctx, padding, pad_len); + + /* Append length (before padding). */ + __octeon_sha512_update(sctx, (const u8 *)bits, sizeof(bits)); + + octeon_sha512_read_hash(sctx); + octeon_crypto_disable(&state, flags); + + /* Store state in digest. */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be64(sctx->state[i]); + + /* Zeroize sensitive information. */ + memset(sctx, 0, sizeof(struct sha512_state)); + + return 0; +} + +static int octeon_sha384_final(struct shash_desc *desc, u8 *hash) +{ + u8 D[64]; + + octeon_sha512_final(desc, D); + + memcpy(hash, D, 48); + memzero_explicit(D, 64); + + return 0; +} + +static struct shash_alg octeon_sha512_algs[2] = { { + .digestsize = SHA512_DIGEST_SIZE, + .init = octeon_sha512_init, + .update = octeon_sha512_update, + .final = octeon_sha512_final, + .descsize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name= "octeon-sha512", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA384_DIGEST_SIZE, + .init = octeon_sha384_init, + .update = octeon_sha512_update, + .final = octeon_sha384_final, + .descsize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name= "octeon-sha384", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init octeon_sha512_mod_init(void) +{ + if (!octeon_has_crypto()) + return -ENOTSUPP; + return crypto_register_shashes(octeon_sha512_algs, + ARRAY_SIZE(octeon_sha512_algs)); +} + +static void __exit octeon_sha512_mod_fini(void) +{ + crypto_unregister_shashes(octeon_sha512_algs, + ARRAY_SIZE(octeon_sha512_algs)); +} + +module_init(octeon_sha512_mod_init); +module_exit(octeon_sha512_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-512 and SHA-384 Secure Hash Algorithms (OCTEON)"); +MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>"); diff --git a/arch/mips/configs/ip32_defconfig b/arch/mips/configs/ip32_defconfig index 70ffe9b55829..fe48220157a9 100644 --- a/arch/mips/configs/ip32_defconfig +++ b/arch/mips/configs/ip32_defconfig @@ -105,7 +105,8 @@ CONFIG_RTC_CLASS=y # CONFIG_RTC_HCTOSYS is not set # CONFIG_RTC_INTF_SYSFS is not set # CONFIG_RTC_INTF_PROC is not set -CONFIG_RTC_DRV_CMOS=y +CONFIG_RTC_DRV_DS1685_FAMILY=y +CONFIG_RTC_DRV_DS1685=y CONFIG_EXT2_FS=y CONFIG_EXT2_FS_XATTR=y CONFIG_EXT2_FS_POSIX_ACL=y diff --git a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h index 4794067cb5a7..5035f09c5427 100644 --- a/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h +++ b/arch/mips/include/asm/mach-bcm63xx/bcm63xx_regs.h @@ -1259,20 +1259,6 @@ #define M2M_DSTID_REG(x) ((x) * 0x40 + 0x18) /************************************************************************* - * _REG relative to RSET_RNG - *************************************************************************/ - -#define RNG_CTRL 0x00 -#define RNG_EN (1 << 0) - -#define RNG_STAT 0x04 -#define RNG_AVAIL_MASK (0xff000000) - -#define RNG_DATA 0x08 -#define RNG_THRES 0x0c -#define RNG_MASK 0x10 - -/************************************************************************* * _REG relative to RSET_SPI *************************************************************************/ diff --git a/arch/mips/include/asm/mach-ip32/mc146818rtc.h b/arch/mips/include/asm/mach-ip32/mc146818rtc.h deleted file mode 100644 index 6b6bab43d5c1..000000000000 --- a/arch/mips/include/asm/mach-ip32/mc146818rtc.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1998, 2001, 03 by Ralf Baechle - * Copyright (C) 2000 Harald Koerfgen - * - * RTC routines for IP32 style attached Dallas chip. - */ -#ifndef __ASM_MACH_IP32_MC146818RTC_H -#define __ASM_MACH_IP32_MC146818RTC_H - -#include <asm/ip32/mace.h> - -#define RTC_PORT(x) (0x70 + (x)) - -static unsigned char CMOS_READ(unsigned long addr) -{ - return mace->isa.rtc[addr << 8]; -} - -static inline void CMOS_WRITE(unsigned char data, unsigned long addr) -{ - mace->isa.rtc[addr << 8] = data; -} - -/* - * FIXME: Do it right. For now just assume that no one lives in 20th century - * and no O2 user in 22th century ;-) - */ -#define mc146818_decode_year(year) ((year) + 2000) - -#define RTC_ALWAYS_BCD 0 - -#endif /* __ASM_MACH_IP32_MC146818RTC_H */ diff --git a/arch/mips/include/asm/seccomp.h b/arch/mips/include/asm/seccomp.h index f29c75cf83c6..1d8a2e2c75c1 100644 --- a/arch/mips/include/asm/seccomp.h +++ b/arch/mips/include/asm/seccomp.h @@ -2,11 +2,6 @@ #include <linux/unistd.h> -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - /* * Kludge alert: * @@ -29,4 +24,6 @@ #endif /* CONFIG_MIPS32_O32 */ +#include <asm-generic/seccomp.h> + #endif /* __ASM_SECCOMP_H */ diff --git a/arch/mips/include/asm/thread_info.h b/arch/mips/include/asm/thread_info.h index 55ed6602204c..8408a30c47f3 100644 --- a/arch/mips/include/asm/thread_info.h +++ b/arch/mips/include/asm/thread_info.h @@ -23,7 +23,6 @@ */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long tp_value; /* thread pointer */ __u32 cpu; /* current CPU */ @@ -44,7 +43,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = _TIF_FIXADE, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c index e59fd7cfac9e..beabe19ff8e5 100644 --- a/arch/mips/kernel/asm-offsets.c +++ b/arch/mips/kernel/asm-offsets.c @@ -92,7 +92,6 @@ void output_thread_info_defines(void) { COMMENT("MIPS thread_info offsets."); OFFSET(TI_TASK, thread_info, task); - OFFSET(TI_EXEC_DOMAIN, thread_info, exec_domain); OFFSET(TI_FLAGS, thread_info, flags); OFFSET(TI_TP_VALUE, thread_info, tp_value); OFFSET(TI_CPU, thread_info, cpu); diff --git a/arch/mips/sgi-ip32/ip32-platform.c b/arch/mips/sgi-ip32/ip32-platform.c index 511e9ff2acfd..b522477129a5 100644 --- a/arch/mips/sgi-ip32/ip32-platform.c +++ b/arch/mips/sgi-ip32/ip32-platform.c @@ -9,10 +9,13 @@ #include <linux/init.h> #include <linux/platform_device.h> #include <linux/serial_8250.h> +#include <linux/rtc/ds1685.h> #include <asm/ip32/mace.h> #include <asm/ip32/ip32_ints.h> +extern void ip32_prepare_poweroff(void); + #define MACEISA_SERIAL1_OFFS offsetof(struct sgi_mace, isa.serial1) #define MACEISA_SERIAL2_OFFS offsetof(struct sgi_mace, isa.serial2) @@ -90,18 +93,47 @@ static __init int sgio2btns_devinit(void) device_initcall(sgio2btns_devinit); -static struct resource sgio2_cmos_rsrc[] = { +#define MACE_RTC_RES_START (MACE_BASE + offsetof(struct sgi_mace, isa.rtc)) +#define MACE_RTC_RES_END (MACE_RTC_RES_START + 32767) + +static struct resource ip32_rtc_resources[] = { { - .start = 0x70, - .end = 0x71, - .flags = IORESOURCE_IO + .start = MACEISA_RTC_IRQ, + .end = MACEISA_RTC_IRQ, + .flags = IORESOURCE_IRQ + }, { + .start = MACE_RTC_RES_START, + .end = MACE_RTC_RES_END, + .flags = IORESOURCE_MEM, } }; -static __init int sgio2_cmos_devinit(void) +/* RTC registers on IP32 are each padded by 256 bytes (0x100). */ +static struct ds1685_rtc_platform_data +ip32_rtc_platform_data[] = { + { + .regstep = 0x100, + .bcd_mode = true, + .no_irq = false, + .uie_unsupported = false, + .alloc_io_resources = true, + .plat_prepare_poweroff = ip32_prepare_poweroff, + }, +}; + +struct platform_device ip32_rtc_device = { + .name = "rtc-ds1685", + .id = -1, + .dev = { + .platform_data = ip32_rtc_platform_data, + }, + .num_resources = ARRAY_SIZE(ip32_rtc_resources), + .resource = ip32_rtc_resources, +}; + ++static int __init sgio2_rtc_devinit(void) { - return IS_ERR(platform_device_register_simple("rtc_cmos", -1, - sgio2_cmos_rsrc, 1)); + return platform_device_register(&ip32_rtc_device); } device_initcall(sgio2_cmos_devinit); diff --git a/arch/mips/sgi-ip32/ip32-reset.c b/arch/mips/sgi-ip32/ip32-reset.c index 44b3470a0bbb..8bd415c8729f 100644 --- a/arch/mips/sgi-ip32/ip32-reset.c +++ b/arch/mips/sgi-ip32/ip32-reset.c @@ -11,10 +11,11 @@ #include <linux/compiler.h> #include <linux/init.h> #include <linux/kernel.h> +#include <linux/module.h> #include <linux/sched.h> #include <linux/notifier.h> #include <linux/delay.h> -#include <linux/ds17287rtc.h> +#include <linux/rtc/ds1685.h> #include <linux/interrupt.h> #include <linux/pm.h> @@ -33,53 +34,40 @@ #define POWERDOWN_FREQ (HZ / 4) #define PANIC_FREQ (HZ / 8) -static struct timer_list power_timer, blink_timer, debounce_timer; -static int has_panicked, shuting_down; +extern struct platform_device ip32_rtc_device; -static void ip32_machine_restart(char *command) __noreturn; -static void ip32_machine_halt(void) __noreturn; -static void ip32_machine_power_off(void) __noreturn; +static struct timer_list power_timer, blink_timer; +static int has_panicked, shutting_down; -static void ip32_machine_restart(char *cmd) +static __noreturn void ip32_poweroff(void *data) { - crime->control = CRIME_CONTROL_HARD_RESET; - while (1); -} + void (*poweroff_func)(struct platform_device *) = + symbol_get(ds1685_rtc_poweroff); + +#ifdef CONFIG_MODULES + /* If the first __symbol_get failed, our module wasn't loaded. */ + if (!poweroff_func) { + request_module("rtc-ds1685"); + poweroff_func = symbol_get(ds1685_rtc_poweroff); + } +#endif -static inline void ip32_machine_halt(void) -{ - ip32_machine_power_off(); -} + if (!poweroff_func) + pr_emerg("RTC not available for power-off. Spinning forever ...\n"); + else { + (*poweroff_func)((struct platform_device *)data); + symbol_put(ds1685_rtc_poweroff); + } -static void ip32_machine_power_off(void) -{ - unsigned char reg_a, xctrl_a, xctrl_b; - - disable_irq(MACEISA_RTC_IRQ); - reg_a = CMOS_READ(RTC_REG_A); - - /* setup for kickstart & wake-up (DS12287 Ref. Man. p. 19) */ - reg_a &= ~DS_REGA_DV2; - reg_a |= DS_REGA_DV1; - - CMOS_WRITE(reg_a | DS_REGA_DV0, RTC_REG_A); - wbflush(); - xctrl_b = CMOS_READ(DS_B1_XCTRL4B) - | DS_XCTRL4B_ABE | DS_XCTRL4B_KFE; - CMOS_WRITE(xctrl_b, DS_B1_XCTRL4B); - xctrl_a = CMOS_READ(DS_B1_XCTRL4A) & ~DS_XCTRL4A_IFS; - CMOS_WRITE(xctrl_a, DS_B1_XCTRL4A); - wbflush(); - /* adios amigos... */ - CMOS_WRITE(xctrl_a | DS_XCTRL4A_PAB, DS_B1_XCTRL4A); - CMOS_WRITE(reg_a, RTC_REG_A); - wbflush(); - while (1); + unreachable(); } -static void power_timeout(unsigned long data) +static void ip32_machine_restart(char *cmd) __noreturn; +static void ip32_machine_restart(char *cmd) { - ip32_machine_power_off(); + msleep(20); + crime->control = CRIME_CONTROL_HARD_RESET; + unreachable(); } static void blink_timeout(unsigned long data) @@ -89,44 +77,27 @@ static void blink_timeout(unsigned long data) mod_timer(&blink_timer, jiffies + data); } -static void debounce(unsigned long data) +static void ip32_machine_halt(void) { - unsigned char reg_a, reg_c, xctrl_a; - - reg_c = CMOS_READ(RTC_INTR_FLAGS); - reg_a = CMOS_READ(RTC_REG_A); - CMOS_WRITE(reg_a | DS_REGA_DV0, RTC_REG_A); - wbflush(); - xctrl_a = CMOS_READ(DS_B1_XCTRL4A); - if ((xctrl_a & DS_XCTRL4A_IFS) || (reg_c & RTC_IRQF )) { - /* Interrupt still being sent. */ - debounce_timer.expires = jiffies + 50; - add_timer(&debounce_timer); - - /* clear interrupt source */ - CMOS_WRITE(xctrl_a & ~DS_XCTRL4A_IFS, DS_B1_XCTRL4A); - CMOS_WRITE(reg_a & ~DS_REGA_DV0, RTC_REG_A); - return; - } - CMOS_WRITE(reg_a & ~DS_REGA_DV0, RTC_REG_A); - - if (has_panicked) - ip32_machine_restart(NULL); + ip32_poweroff(&ip32_rtc_device); +} - enable_irq(MACEISA_RTC_IRQ); +static void power_timeout(unsigned long data) +{ + ip32_poweroff(&ip32_rtc_device); } -static inline void ip32_power_button(void) +void ip32_prepare_poweroff(void) { if (has_panicked) return; - if (shuting_down || kill_cad_pid(SIGINT, 1)) { + if (shutting_down || kill_cad_pid(SIGINT, 1)) { /* No init process or button pressed twice. */ - ip32_machine_power_off(); + ip32_poweroff(&ip32_rtc_device); } - shuting_down = 1; + shutting_down = 1; blink_timer.data = POWERDOWN_FREQ; blink_timeout(POWERDOWN_FREQ); @@ -136,27 +107,6 @@ static inline void ip32_power_button(void) add_timer(&power_timer); } -static irqreturn_t ip32_rtc_int(int irq, void *dev_id) -{ - unsigned char reg_c; - - reg_c = CMOS_READ(RTC_INTR_FLAGS); - if (!(reg_c & RTC_IRQF)) { - printk(KERN_WARNING - "%s: RTC IRQ without RTC_IRQF\n", __func__); - } - /* Wait until interrupt goes away */ - disable_irq_nosync(MACEISA_RTC_IRQ); - init_timer(&debounce_timer); - debounce_timer.function = debounce; - debounce_timer.expires = jiffies + 50; - add_timer(&debounce_timer); - - printk(KERN_DEBUG "Power button pressed\n"); - ip32_power_button(); - return IRQ_HANDLED; -} - static int panic_event(struct notifier_block *this, unsigned long event, void *ptr) { @@ -190,15 +140,12 @@ static __init int ip32_reboot_setup(void) _machine_restart = ip32_machine_restart; _machine_halt = ip32_machine_halt; - pm_power_off = ip32_machine_power_off; + pm_power_off = ip32_machine_halt; init_timer(&blink_timer); blink_timer.function = blink_timeout; atomic_notifier_chain_register(&panic_notifier_list, &panic_block); - if (request_irq(MACEISA_RTC_IRQ, ip32_rtc_int, 0, "rtc", NULL)) - panic("Can't allocate MACEISA RTC IRQ"); - return 0; } diff --git a/arch/mn10300/include/asm/thread_info.h b/arch/mn10300/include/asm/thread_info.h index c1c374f0ec12..4861a78c7160 100644 --- a/arch/mn10300/include/asm/thread_info.h +++ b/arch/mn10300/include/asm/thread_info.h @@ -40,7 +40,6 @@ typedef struct { struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ struct pt_regs *frame; /* current exception frame */ unsigned long flags; /* low level flags */ __u32 cpu; /* current CPU */ @@ -74,7 +73,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/mn10300/kernel/asm-offsets.c b/arch/mn10300/kernel/asm-offsets.c index d780670cbaf3..e5a61c659b5a 100644 --- a/arch/mn10300/kernel/asm-offsets.c +++ b/arch/mn10300/kernel/asm-offsets.c @@ -22,7 +22,6 @@ void foo(void) BLANK(); OFFSET(TI_task, thread_info, task); - OFFSET(TI_exec_domain, thread_info, exec_domain); OFFSET(TI_frame, thread_info, frame); OFFSET(TI_flags, thread_info, flags); OFFSET(TI_cpu, thread_info, cpu); @@ -85,7 +84,6 @@ void foo(void) DEFINE(SIGCHLD_asm, SIGCHLD); BLANK(); - OFFSET(EXEC_DOMAIN_handler, exec_domain, handler); OFFSET(RT_SIGFRAME_sigcontext, rt_sigframe, uc.uc_mcontext); DEFINE(PAGE_SIZE_asm, PAGE_SIZE); diff --git a/arch/mn10300/kernel/signal.c b/arch/mn10300/kernel/signal.c index 8609845f12c5..dfd0301cf200 100644 --- a/arch/mn10300/kernel/signal.c +++ b/arch/mn10300/kernel/signal.c @@ -202,20 +202,14 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { struct sigframe __user *frame; - int rsig, sig = ksig->sig; + int sig = ksig->sig; frame = get_sigframe(ksig, regs, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - rsig = sig; - if (sig < 32 && - current_thread_info()->exec_domain && - current_thread_info()->exec_domain->signal_invmap) - rsig = current_thread_info()->exec_domain->signal_invmap[sig]; - - if (__put_user(rsig, &frame->sig) < 0 || + if (__put_user(sig, &frame->sig) < 0 || __put_user(&frame->sc, &frame->psc) < 0) return -EFAULT; @@ -270,20 +264,14 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { struct rt_sigframe __user *frame; - int rsig, sig = ksig->sig; + int sig = ksig->sig; frame = get_sigframe(ksig, regs, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - rsig = sig; - if (sig < 32 && - current_thread_info()->exec_domain && - current_thread_info()->exec_domain->signal_invmap) - rsig = current_thread_info()->exec_domain->signal_invmap[sig]; - - if (__put_user(rsig, &frame->sig) || + if (__put_user(sig, &frame->sig) || __put_user(&frame->info, &frame->pinfo) || __put_user(&frame->uc, &frame->puc) || copy_siginfo_to_user(&frame->info, &ksig->info)) diff --git a/arch/nios2/include/asm/thread_info.h b/arch/nios2/include/asm/thread_info.h index a16e55cbd8ad..d69c338bd19c 100644 --- a/arch/nios2/include/asm/thread_info.h +++ b/arch/nios2/include/asm/thread_info.h @@ -39,7 +39,6 @@ typedef struct { */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ __u32 cpu; /* current CPU */ int preempt_count; /* 0 => preemptable,<0 => BUG */ @@ -58,7 +57,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/nios2/kernel/cpuinfo.c b/arch/nios2/kernel/cpuinfo.c index a223691dff4f..1d96de0bd4aa 100644 --- a/arch/nios2/kernel/cpuinfo.c +++ b/arch/nios2/kernel/cpuinfo.c @@ -126,47 +126,46 @@ void __init setup_cpuinfo(void) */ static int show_cpuinfo(struct seq_file *m, void *v) { - int count = 0; const u32 clockfreq = cpuinfo.cpu_clock_freq; - count = seq_printf(m, - "CPU:\t\tNios II/%s\n" - "MMU:\t\t%s\n" - "FPU:\t\tnone\n" - "Clocking:\t%u.%02u MHz\n" - "BogoMips:\t%lu.%02lu\n" - "Calibration:\t%lu loops\n", - cpuinfo.cpu_impl, - cpuinfo.mmu ? "present" : "none", - clockfreq / 1000000, (clockfreq / 100000) % 10, - (loops_per_jiffy * HZ) / 500000, - ((loops_per_jiffy * HZ) / 5000) % 100, - (loops_per_jiffy * HZ)); - - count += seq_printf(m, - "HW:\n" - " MUL:\t\t%s\n" - " MULX:\t\t%s\n" - " DIV:\t\t%s\n", - cpuinfo.has_mul ? "yes" : "no", - cpuinfo.has_mulx ? "yes" : "no", - cpuinfo.has_div ? "yes" : "no"); - - count += seq_printf(m, - "Icache:\t\t%ukB, line length: %u\n", - cpuinfo.icache_size >> 10, - cpuinfo.icache_line_size); - - count += seq_printf(m, - "Dcache:\t\t%ukB, line length: %u\n", - cpuinfo.dcache_size >> 10, - cpuinfo.dcache_line_size); - - count += seq_printf(m, - "TLB:\t\t%u ways, %u entries, %u PID bits\n", - cpuinfo.tlb_num_ways, - cpuinfo.tlb_num_entries, - cpuinfo.tlb_pid_num_bits); + seq_printf(m, + "CPU:\t\tNios II/%s\n" + "MMU:\t\t%s\n" + "FPU:\t\tnone\n" + "Clocking:\t%u.%02u MHz\n" + "BogoMips:\t%lu.%02lu\n" + "Calibration:\t%lu loops\n", + cpuinfo.cpu_impl, + cpuinfo.mmu ? "present" : "none", + clockfreq / 1000000, (clockfreq / 100000) % 10, + (loops_per_jiffy * HZ) / 500000, + ((loops_per_jiffy * HZ) / 5000) % 100, + (loops_per_jiffy * HZ)); + + seq_printf(m, + "HW:\n" + " MUL:\t\t%s\n" + " MULX:\t\t%s\n" + " DIV:\t\t%s\n", + cpuinfo.has_mul ? "yes" : "no", + cpuinfo.has_mulx ? "yes" : "no", + cpuinfo.has_div ? "yes" : "no"); + + seq_printf(m, + "Icache:\t\t%ukB, line length: %u\n", + cpuinfo.icache_size >> 10, + cpuinfo.icache_line_size); + + seq_printf(m, + "Dcache:\t\t%ukB, line length: %u\n", + cpuinfo.dcache_size >> 10, + cpuinfo.dcache_line_size); + + seq_printf(m, + "TLB:\t\t%u ways, %u entries, %u PID bits\n", + cpuinfo.tlb_num_ways, + cpuinfo.tlb_num_entries, + cpuinfo.tlb_pid_num_bits); return 0; } diff --git a/arch/openrisc/include/asm/thread_info.h b/arch/openrisc/include/asm/thread_info.h index 875f0845a707..6e619a79a401 100644 --- a/arch/openrisc/include/asm/thread_info.h +++ b/arch/openrisc/include/asm/thread_info.h @@ -48,7 +48,6 @@ typedef unsigned long mm_segment_t; struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ __u32 cpu; /* current CPU */ __s32 preempt_count; /* 0 => preemptable, <0 => BUG */ @@ -73,7 +72,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = 1, \ diff --git a/arch/openrisc/kernel/setup.c b/arch/openrisc/kernel/setup.c index 4fc7ccc0a2cf..b4ed8b36e078 100644 --- a/arch/openrisc/kernel/setup.c +++ b/arch/openrisc/kernel/setup.c @@ -329,30 +329,32 @@ static int show_cpuinfo(struct seq_file *m, void *v) version = (vr & SPR_VR_VER) >> 24; revision = vr & SPR_VR_REV; - return seq_printf(m, - "cpu\t\t: OpenRISC-%x\n" - "revision\t: %d\n" - "frequency\t: %ld\n" - "dcache size\t: %d bytes\n" - "dcache block size\t: %d bytes\n" - "icache size\t: %d bytes\n" - "icache block size\t: %d bytes\n" - "immu\t\t: %d entries, %lu ways\n" - "dmmu\t\t: %d entries, %lu ways\n" - "bogomips\t: %lu.%02lu\n", - version, - revision, - loops_per_jiffy * HZ, - cpuinfo.dcache_size, - cpuinfo.dcache_block_size, - cpuinfo.icache_size, - cpuinfo.icache_block_size, - 1 << ((mfspr(SPR_DMMUCFGR) & SPR_DMMUCFGR_NTS) >> 2), - 1 + (mfspr(SPR_DMMUCFGR) & SPR_DMMUCFGR_NTW), - 1 << ((mfspr(SPR_IMMUCFGR) & SPR_IMMUCFGR_NTS) >> 2), - 1 + (mfspr(SPR_IMMUCFGR) & SPR_IMMUCFGR_NTW), - (loops_per_jiffy * HZ) / 500000, - ((loops_per_jiffy * HZ) / 5000) % 100); + seq_printf(m, + "cpu\t\t: OpenRISC-%x\n" + "revision\t: %d\n" + "frequency\t: %ld\n" + "dcache size\t: %d bytes\n" + "dcache block size\t: %d bytes\n" + "icache size\t: %d bytes\n" + "icache block size\t: %d bytes\n" + "immu\t\t: %d entries, %lu ways\n" + "dmmu\t\t: %d entries, %lu ways\n" + "bogomips\t: %lu.%02lu\n", + version, + revision, + loops_per_jiffy * HZ, + cpuinfo.dcache_size, + cpuinfo.dcache_block_size, + cpuinfo.icache_size, + cpuinfo.icache_block_size, + 1 << ((mfspr(SPR_DMMUCFGR) & SPR_DMMUCFGR_NTS) >> 2), + 1 + (mfspr(SPR_DMMUCFGR) & SPR_DMMUCFGR_NTW), + 1 << ((mfspr(SPR_IMMUCFGR) & SPR_IMMUCFGR_NTS) >> 2), + 1 + (mfspr(SPR_IMMUCFGR) & SPR_IMMUCFGR_NTW), + (loops_per_jiffy * HZ) / 500000, + ((loops_per_jiffy * HZ) / 5000) % 100); + + return 0; } static void *c_start(struct seq_file *m, loff_t * pos) diff --git a/arch/openrisc/kernel/signal.c b/arch/openrisc/kernel/signal.c index 4112175bf803..c82be69b43c6 100644 --- a/arch/openrisc/kernel/signal.c +++ b/arch/openrisc/kernel/signal.c @@ -193,8 +193,6 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, if (err) return -EFAULT; - /* TODO what is the current->exec_domain stuff and invmap ? */ - /* Set up registers for signal handler */ regs->pc = (unsigned long)ksig->ka.sa.sa_handler; /* what we enter NOW */ regs->gpr[9] = (unsigned long)return_ip; /* what we enter LATER */ diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild index 8686237a3c3c..12b341d04f88 100644 --- a/arch/parisc/include/asm/Kbuild +++ b/arch/parisc/include/asm/Kbuild @@ -20,6 +20,7 @@ generic-y += param.h generic-y += percpu.h generic-y += poll.h generic-y += preempt.h +generic-y += seccomp.h generic-y += segment.h generic-y += topology.h generic-y += trace_clock.h diff --git a/arch/parisc/include/asm/seccomp.h b/arch/parisc/include/asm/seccomp.h deleted file mode 100644 index 015f7887aa29..000000000000 --- a/arch/parisc/include/asm/seccomp.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef _ASM_PARISC_SECCOMP_H -#define _ASM_PARISC_SECCOMP_H - -#include <linux/unistd.h> - -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - -#define __NR_seccomp_read_32 __NR_read -#define __NR_seccomp_write_32 __NR_write -#define __NR_seccomp_exit_32 __NR_exit -#define __NR_seccomp_sigreturn_32 __NR_rt_sigreturn - -#endif /* _ASM_PARISC_SECCOMP_H */ diff --git a/arch/parisc/include/asm/thread_info.h b/arch/parisc/include/asm/thread_info.h index fb13e3865563..e96e693fd58c 100644 --- a/arch/parisc/include/asm/thread_info.h +++ b/arch/parisc/include/asm/thread_info.h @@ -9,7 +9,6 @@ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain;/* execution domain */ unsigned long flags; /* thread_info flags (see TIF_*) */ mm_segment_t addr_limit; /* user-level address space limit */ __u32 cpu; /* current CPU */ @@ -19,7 +18,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .addr_limit = KERNEL_DS, \ diff --git a/arch/parisc/kernel/asm-offsets.c b/arch/parisc/kernel/asm-offsets.c index dcd55103a4bb..59001cea13f9 100644 --- a/arch/parisc/kernel/asm-offsets.c +++ b/arch/parisc/kernel/asm-offsets.c @@ -242,7 +242,6 @@ int main(void) DEFINE(PT_SZ_ALGN, align_frame(sizeof(struct pt_regs), FRAME_ALIGN)); BLANK(); DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain)); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_SEGMENT, offsetof(struct thread_info, addr_limit)); diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index e99014adf017..190cc48abc0c 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -126,7 +126,7 @@ config PPC select IRQ_FORCED_THREADING select HAVE_RCU_TABLE_FREE if SMP select HAVE_SYSCALL_TRACEPOINTS - select HAVE_BPF_JIT if PPC64 + select HAVE_BPF_JIT select HAVE_ARCH_JUMP_LABEL select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_HAS_GCOV_PROFILE_ALL @@ -152,6 +152,7 @@ config PPC select DCACHE_WORD_ACCESS if PPC64 && CPU_LITTLE_ENDIAN select NO_BOOTMEM select HAVE_GENERIC_RCU_GUP + select HAVE_PERF_EVENTS_NMI if PPC64 config GENERIC_CSUM def_bool CPU_LITTLE_ENDIAN @@ -189,9 +190,6 @@ config ARCH_MAY_HAVE_PC_FDC bool default PCI -config PPC_OF - def_bool y - config PPC_UDBG_16550 bool default n diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index ec2e40f2cc11..0efa8f90a8f1 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -117,7 +117,7 @@ config BDI_SWITCH config BOOTX_TEXT bool "Support for early boot text console (BootX or OpenFirmware only)" - depends on PPC_OF && PPC_BOOK3S + depends on PPC_BOOK3S help Say Y here to see progress messages from the boot firmware in text mode. Requires either BootX or Open Firmware. @@ -193,13 +193,6 @@ config PPC_EARLY_DEBUG_PAS_REALMODE Select this to enable early debugging for PA Semi. Output will be on UART0. -config PPC_EARLY_DEBUG_BEAT - bool "Beat HV Console" - depends on PPC_CELLEB - select PPC_UDBG_BEAT - help - Select this to enable early debugging for Celleb with Beat. - config PPC_EARLY_DEBUG_44x bool "Early serial debugging for IBM/AMCC 44x CPUs" depends on 44x diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index fc502e042438..07a480861f78 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -248,10 +248,10 @@ boot := arch/$(ARCH)/boot ifeq ($(CONFIG_RELOCATABLE),y) quiet_cmd_relocs_check = CALL $< - cmd_relocs_check = perl $< "$(OBJDUMP)" "$(obj)/vmlinux" + cmd_relocs_check = $(CONFIG_SHELL) $< "$(OBJDUMP)" "$(obj)/vmlinux" PHONY += relocs_check -relocs_check: arch/powerpc/relocs_check.pl vmlinux +relocs_check: arch/powerpc/relocs_check.sh vmlinux $(call cmd,relocs_check) zImage: relocs_check diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index 8a5bc1cfc6aa..73eddda53b8e 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -110,7 +110,6 @@ src-plat-$(CONFIG_EPAPR_BOOT) += epapr.c epapr-wrapper.c src-plat-$(CONFIG_PPC_PSERIES) += pseries-head.S src-plat-$(CONFIG_PPC_POWERNV) += pseries-head.S src-plat-$(CONFIG_PPC_IBM_CELL_BLADE) += pseries-head.S -src-plat-$(CONFIG_PPC_CELLEB) += pseries-head.S src-plat-$(CONFIG_PPC_CELL_QPACE) += pseries-head.S src-wlib := $(sort $(src-wlib-y)) @@ -215,7 +214,6 @@ image-$(CONFIG_PPC_POWERNV) += zImage.pseries image-$(CONFIG_PPC_MAPLE) += zImage.maple image-$(CONFIG_PPC_IBM_CELL_BLADE) += zImage.pseries image-$(CONFIG_PPC_PS3) += dtbImage.ps3 -image-$(CONFIG_PPC_CELLEB) += zImage.pseries image-$(CONFIG_PPC_CELL_QPACE) += zImage.pseries image-$(CONFIG_PPC_CHRP) += zImage.chrp image-$(CONFIG_PPC_EFIKA) += zImage.chrp @@ -317,7 +315,7 @@ endif # Allow extra targets to be added to the defconfig image-y += $(subst ",,$(CONFIG_EXTRA_TARGETS)) -initrd- := $(patsubst zImage%, zImage.initrd%, $(image-n) $(image-)) +initrd- := $(patsubst zImage%, zImage.initrd%, $(image-)) initrd-y := $(patsubst zImage%, zImage.initrd%, \ $(patsubst dtbImage%, dtbImage.initrd%, \ $(patsubst simpleImage%, simpleImage.initrd%, \ diff --git a/arch/powerpc/boot/crt0.S b/arch/powerpc/boot/crt0.S index 14de4f8778a7..12866ccb5694 100644 --- a/arch/powerpc/boot/crt0.S +++ b/arch/powerpc/boot/crt0.S @@ -155,29 +155,29 @@ p_base: mflr r10 /* r10 now points to runtime addr of p_base */ ld r9,(p_rela-p_base)(r10) add r9,r9,r10 - li r7,0 + li r13,0 li r8,0 -9: ld r6,0(r11) /* get tag */ - cmpdi r6,0 +9: ld r12,0(r11) /* get tag */ + cmpdi r12,0 beq 12f /* end of list */ - cmpdi r6,RELA + cmpdi r12,RELA bne 10f - ld r7,8(r11) /* get RELA pointer in r7 */ + ld r13,8(r11) /* get RELA pointer in r13 */ b 11f -10: addis r6,r6,(-RELACOUNT)@ha - cmpdi r6,RELACOUNT@l +10: addis r12,r12,(-RELACOUNT)@ha + cmpdi r12,RELACOUNT@l bne 11f ld r8,8(r11) /* get RELACOUNT value in r8 */ 11: addi r11,r11,16 b 9b 12: - cmpdi r7,0 /* check we have both RELA and RELACOUNT */ + cmpdi r13,0 /* check we have both RELA and RELACOUNT */ cmpdi cr1,r8,0 beq 3f beq cr1,3f /* Calcuate the runtime offset. */ - subf r7,r7,r9 + subf r13,r13,r9 /* Run through the list of relocations and process the * R_PPC64_RELATIVE ones. */ @@ -185,10 +185,10 @@ p_base: mflr r10 /* r10 now points to runtime addr of p_base */ 13: ld r0,8(r9) /* ELF64_R_TYPE(reloc->r_info) */ cmpdi r0,22 /* R_PPC64_RELATIVE */ bne 3f - ld r6,0(r9) /* reloc->r_offset */ + ld r12,0(r9) /* reloc->r_offset */ ld r0,16(r9) /* reloc->r_addend */ - add r0,r0,r7 - stdx r0,r7,r6 + add r0,r0,r13 + stdx r0,r13,r12 addi r9,r9,24 bdnz 13b @@ -218,7 +218,7 @@ p_base: mflr r10 /* r10 now points to runtime addr of p_base */ beq 6f ld r1,0(r8) li r0,0 - stdu r0,-16(r1) /* establish a stack frame */ + stdu r0,-112(r1) /* establish a stack frame */ 6: #endif /* __powerpc64__ */ /* Call platform_init() */ diff --git a/arch/powerpc/boot/dts/b4860emu.dts b/arch/powerpc/boot/dts/b4860emu.dts deleted file mode 100644 index 2aa5cd318ce8..000000000000 --- a/arch/powerpc/boot/dts/b4860emu.dts +++ /dev/null @@ -1,223 +0,0 @@ -/* - * B4860 emulator Device Tree Source - * - * Copyright 2013 Freescale Semiconductor Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Freescale Semiconductor nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * - * ALTERNATIVELY, this software may be distributed under the terms of the - * GNU General Public License ("GPL") as published by the Free Software - * Foundation, either version 2 of that License or (at your option) any - * later version. - * - * This software is provided by Freescale Semiconductor "as is" and any - * express or implied warranties, including, but not limited to, the implied - * warranties of merchantability and fitness for a particular purpose are - * disclaimed. In no event shall Freescale Semiconductor be liable for any - * direct, indirect, incidental, special, exemplary, or consequential damages - * (including, but not limited to, procurement of substitute goods or services; - * loss of use, data, or profits; or business interruption) however caused and - * on any theory of liability, whether in contract, strict liability, or tort - * (including negligence or otherwise) arising in any way out of the use of - * this software, even if advised of the possibility of such damage. - */ - -/dts-v1/; - -/include/ "fsl/e6500_power_isa.dtsi" - -/ { - compatible = "fsl,B4860"; - #address-cells = <2>; - #size-cells = <2>; - interrupt-parent = <&mpic>; - - aliases { - ccsr = &soc; - - serial0 = &serial0; - serial1 = &serial1; - serial2 = &serial2; - serial3 = &serial3; - dma0 = &dma0; - dma1 = &dma1; - }; - - cpus { - #address-cells = <1>; - #size-cells = <0>; - - cpu0: PowerPC,e6500@0 { - device_type = "cpu"; - reg = <0 1>; - next-level-cache = <&L2>; - fsl,portid-mapping = <0x80000000>; - }; - cpu1: PowerPC,e6500@2 { - device_type = "cpu"; - reg = <2 3>; - next-level-cache = <&L2>; - fsl,portid-mapping = <0x80000000>; - }; - cpu2: PowerPC,e6500@4 { - device_type = "cpu"; - reg = <4 5>; - next-level-cache = <&L2>; - fsl,portid-mapping = <0x80000000>; - }; - cpu3: PowerPC,e6500@6 { - device_type = "cpu"; - reg = <6 7>; - next-level-cache = <&L2>; - fsl,portid-mapping = <0x80000000>; - }; - }; -}; - -/ { - model = "fsl,B4860QDS"; - compatible = "fsl,B4860EMU", "fsl,B4860QDS"; - #address-cells = <2>; - #size-cells = <2>; - interrupt-parent = <&mpic>; - - ifc: localbus@ffe124000 { - reg = <0xf 0xfe124000 0 0x2000>; - ranges = <0 0 0xf 0xe8000000 0x08000000 - 2 0 0xf 0xff800000 0x00010000 - 3 0 0xf 0xffdf0000 0x00008000>; - - nor@0,0 { - #address-cells = <1>; - #size-cells = <1>; - compatible = "cfi-flash"; - reg = <0x0 0x0 0x8000000>; - bank-width = <2>; - device-width = <1>; - }; - }; - - memory { - device_type = "memory"; - }; - - soc: soc@ffe000000 { - ranges = <0x00000000 0xf 0xfe000000 0x1000000>; - reg = <0xf 0xfe000000 0 0x00001000>; - }; -}; - -&ifc { - #address-cells = <2>; - #size-cells = <1>; - compatible = "fsl,ifc", "simple-bus"; - interrupts = <25 2 0 0>; -}; - -&soc { - #address-cells = <1>; - #size-cells = <1>; - device_type = "soc"; - compatible = "simple-bus"; - - soc-sram-error { - compatible = "fsl,soc-sram-error"; - interrupts = <16 2 1 2>; - }; - - corenet-law@0 { - compatible = "fsl,corenet-law"; - reg = <0x0 0x1000>; - fsl,num-laws = <32>; - }; - - ddr1: memory-controller@8000 { - compatible = "fsl,qoriq-memory-controller-v4.5", "fsl,qoriq-memory-controller"; - reg = <0x8000 0x1000>; - interrupts = <16 2 1 8>; - }; - - ddr2: memory-controller@9000 { - compatible = "fsl,qoriq-memory-controller-v4.5","fsl,qoriq-memory-controller"; - reg = <0x9000 0x1000>; - interrupts = <16 2 1 9>; - }; - - cpc: l3-cache-controller@10000 { - compatible = "fsl,b4-l3-cache-controller", "cache"; - reg = <0x10000 0x1000 - 0x11000 0x1000>; - interrupts = <16 2 1 4>; - }; - - corenet-cf@18000 { - compatible = "fsl,corenet2-cf", "fsl,corenet-cf"; - reg = <0x18000 0x1000>; - interrupts = <16 2 1 0>; - fsl,ccf-num-csdids = <32>; - fsl,ccf-num-snoopids = <32>; - }; - - iommu@20000 { - compatible = "fsl,pamu-v1.0", "fsl,pamu"; - reg = <0x20000 0x4000>; - fsl,portid-mapping = <0x8000>; - #address-cells = <1>; - #size-cells = <1>; - interrupts = < - 24 2 0 0 - 16 2 1 1>; - pamu0: pamu@0 { - reg = <0 0x1000>; - fsl,primary-cache-geometry = <8 1>; - fsl,secondary-cache-geometry = <32 2>; - }; - }; - -/include/ "fsl/qoriq-mpic.dtsi" - - guts: global-utilities@e0000 { - compatible = "fsl,b4-device-config"; - reg = <0xe0000 0xe00>; - fsl,has-rstcr; - fsl,liodn-bits = <12>; - }; - -/include/ "fsl/qoriq-clockgen2.dtsi" - global-utilities@e1000 { - compatible = "fsl,b4-clockgen", "fsl,qoriq-clockgen-2.0"; - }; - -/include/ "fsl/qoriq-dma-0.dtsi" - dma@100300 { - fsl,iommu-parent = <&pamu0>; - fsl,liodn-reg = <&guts 0x580>; /* DMA1LIODNR */ - }; - -/include/ "fsl/qoriq-dma-1.dtsi" - dma@101300 { - fsl,iommu-parent = <&pamu0>; - fsl,liodn-reg = <&guts 0x584>; /* DMA2LIODNR */ - }; - -/include/ "fsl/qoriq-i2c-0.dtsi" -/include/ "fsl/qoriq-i2c-1.dtsi" -/include/ "fsl/qoriq-duart-0.dtsi" -/include/ "fsl/qoriq-duart-1.dtsi" - - L2: l2-cache-controller@c20000 { - compatible = "fsl,b4-l2-cache-controller"; - reg = <0xc20000 0x1000>; - next-level-cache = <&cpc>; - }; -}; diff --git a/arch/powerpc/boot/dts/b4qds.dtsi b/arch/powerpc/boot/dts/b4qds.dtsi index e5bde0b85135..24ed80dc2120 100644 --- a/arch/powerpc/boot/dts/b4qds.dtsi +++ b/arch/powerpc/boot/dts/b4qds.dtsi @@ -1,7 +1,7 @@ /* * B4420DS Device Tree Source * - * Copyright 2012 Freescale Semiconductor, Inc. + * Copyright 2012 - 2014 Freescale Semiconductor, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -97,10 +97,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01052000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/fsl/b4860si-post.dtsi b/arch/powerpc/boot/dts/fsl/b4860si-post.dtsi index 65100b9636b7..f35e9e0a5445 100644 --- a/arch/powerpc/boot/dts/fsl/b4860si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/b4860si-post.dtsi @@ -1,7 +1,7 @@ /* * B4860 Silicon/SoC Device Tree Source (post include) * - * Copyright 2012 Freescale Semiconductor Inc. + * Copyright 2012 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -109,6 +109,64 @@ }; }; +&bportals { + bman-portal@38000 { + compatible = "fsl,bman-portal"; + reg = <0x38000 0x4000>, <0x100e000 0x1000>; + interrupts = <133 2 0 0>; + }; + bman-portal@3c000 { + compatible = "fsl,bman-portal"; + reg = <0x3c000 0x4000>, <0x100f000 0x1000>; + interrupts = <135 2 0 0>; + }; + bman-portal@40000 { + compatible = "fsl,bman-portal"; + reg = <0x40000 0x4000>, <0x1010000 0x1000>; + interrupts = <137 2 0 0>; + }; + bman-portal@44000 { + compatible = "fsl,bman-portal"; + reg = <0x44000 0x4000>, <0x1011000 0x1000>; + interrupts = <139 2 0 0>; + }; + bman-portal@48000 { + compatible = "fsl,bman-portal"; + reg = <0x48000 0x4000>, <0x1012000 0x1000>; + interrupts = <141 2 0 0>; + }; + bman-portal@4c000 { + compatible = "fsl,bman-portal"; + reg = <0x4c000 0x4000>, <0x1013000 0x1000>; + interrupts = <143 2 0 0>; + }; + bman-portal@50000 { + compatible = "fsl,bman-portal"; + reg = <0x50000 0x4000>, <0x1014000 0x1000>; + interrupts = <145 2 0 0>; + }; + bman-portal@54000 { + compatible = "fsl,bman-portal"; + reg = <0x54000 0x4000>, <0x1015000 0x1000>; + interrupts = <147 2 0 0>; + }; + bman-portal@58000 { + compatible = "fsl,bman-portal"; + reg = <0x58000 0x4000>, <0x1016000 0x1000>; + interrupts = <149 2 0 0>; + }; + bman-portal@5c000 { + compatible = "fsl,bman-portal"; + reg = <0x5c000 0x4000>, <0x1017000 0x1000>; + interrupts = <151 2 0 0>; + }; + bman-portal@60000 { + compatible = "fsl,bman-portal"; + reg = <0x60000 0x4000>, <0x1018000 0x1000>; + interrupts = <153 2 0 0>; + }; +}; + &soc { ddr2: memory-controller@9000 { compatible = "fsl,qoriq-memory-controller-v4.5", "fsl,qoriq-memory-controller"; diff --git a/arch/powerpc/boot/dts/fsl/b4si-post.dtsi b/arch/powerpc/boot/dts/fsl/b4si-post.dtsi index 1a54ba71f685..73136c0029d2 100644 --- a/arch/powerpc/boot/dts/fsl/b4si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/b4si-post.dtsi @@ -1,7 +1,7 @@ /* * B4420 Silicon/SoC Device Tree Source (post include) * - * Copyright 2012 Freescale Semiconductor, Inc. + * Copyright 2012 - 2014 Freescale Semiconductor, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * this software, even if advised of the possibility of such damage. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10000 0>; +}; + &ifc { #address-cells = <2>; #size-cells = <1>; @@ -128,6 +133,83 @@ }; }; +&bportals { + #address-cells = <0x1>; + #size-cells = <0x1>; + compatible = "simple-bus"; + + bman-portal@0 { + compatible = "fsl,bman-portal"; + reg = <0x0 0x4000>, <0x1000000 0x1000>; + interrupts = <105 2 0 0>; + }; + bman-portal@4000 { + compatible = "fsl,bman-portal"; + reg = <0x4000 0x4000>, <0x1001000 0x1000>; + interrupts = <107 2 0 0>; + }; + bman-portal@8000 { + compatible = "fsl,bman-portal"; + reg = <0x8000 0x4000>, <0x1002000 0x1000>; + interrupts = <109 2 0 0>; + }; + bman-portal@c000 { + compatible = "fsl,bman-portal"; + reg = <0xc000 0x4000>, <0x1003000 0x1000>; + interrupts = <111 2 0 0>; + }; + bman-portal@10000 { + compatible = "fsl,bman-portal"; + reg = <0x10000 0x4000>, <0x1004000 0x1000>; + interrupts = <113 2 0 0>; + }; + bman-portal@14000 { + compatible = "fsl,bman-portal"; + reg = <0x14000 0x4000>, <0x1005000 0x1000>; + interrupts = <115 2 0 0>; + }; + bman-portal@18000 { + compatible = "fsl,bman-portal"; + reg = <0x18000 0x4000>, <0x1006000 0x1000>; + interrupts = <117 2 0 0>; + }; + bman-portal@1c000 { + compatible = "fsl,bman-portal"; + reg = <0x1c000 0x4000>, <0x1007000 0x1000>; + interrupts = <119 2 0 0>; + }; + bman-portal@20000 { + compatible = "fsl,bman-portal"; + reg = <0x20000 0x4000>, <0x1008000 0x1000>; + interrupts = <121 2 0 0>; + }; + bman-portal@24000 { + compatible = "fsl,bman-portal"; + reg = <0x24000 0x4000>, <0x1009000 0x1000>; + interrupts = <123 2 0 0>; + }; + bman-portal@28000 { + compatible = "fsl,bman-portal"; + reg = <0x28000 0x4000>, <0x100a000 0x1000>; + interrupts = <125 2 0 0>; + }; + bman-portal@2c000 { + compatible = "fsl,bman-portal"; + reg = <0x2c000 0x4000>, <0x100b000 0x1000>; + interrupts = <127 2 0 0>; + }; + bman-portal@30000 { + compatible = "fsl,bman-portal"; + reg = <0x30000 0x4000>, <0x100c000 0x1000>; + interrupts = <129 2 0 0>; + }; + bman-portal@34000 { + compatible = "fsl,bman-portal"; + reg = <0x34000 0x4000>, <0x100d000 0x1000>; + interrupts = <131 2 0 0>; + }; +}; + &soc { #address-cells = <1>; #size-cells = <1>; @@ -261,6 +343,11 @@ /include/ "qoriq-duart-1.dtsi" /include/ "qoriq-sec5.3-0.dtsi" +/include/ "qoriq-bman1.dtsi" + bman: bman@31a000 { + interrupts = <16 2 1 29>; + }; + L2: l2-cache-controller@c20000 { compatible = "fsl,b4-l2-cache-controller"; reg = <0xc20000 0x1000>; diff --git a/arch/powerpc/boot/dts/fsl/p1023si-post.dtsi b/arch/powerpc/boot/dts/fsl/p1023si-post.dtsi index 81437fdf1db4..7780f21430cb 100644 --- a/arch/powerpc/boot/dts/fsl/p1023si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/p1023si-post.dtsi @@ -1,7 +1,7 @@ /* * P1023/P1017 Silicon/SoC Device Tree Source (post include) * - * Copyright 2011 Freescale Semiconductor Inc. + * Copyright 2011 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10 0>; +}; + &lbc { #address-cells = <2>; #size-cells = <1>; @@ -97,6 +102,28 @@ }; }; +&bportals { + #address-cells = <1>; + #size-cells = <1>; + compatible = "simple-bus"; + + bman-portal@0 { + compatible = "fsl,bman-portal"; + reg = <0x0 0x4000>, <0x100000 0x1000>; + interrupts = <30 2 0 0>; + }; + bman-portal@4000 { + compatible = "fsl,bman-portal"; + reg = <0x4000 0x4000>, <0x101000 0x1000>; + interrupts = <32 2 0 0>; + }; + bman-portal@8000 { + compatible = "fsl,bman-portal"; + reg = <0x8000 0x4000>, <0x102000 0x1000>; + interrupts = <34 2 0 0>; + }; +}; + &soc { #address-cells = <1>; #size-cells = <1>; @@ -221,6 +248,14 @@ /include/ "pq3-mpic.dtsi" /include/ "pq3-mpic-timer-B.dtsi" + bman: bman@8a000 { + compatible = "fsl,bman"; + reg = <0x8a000 0x1000>; + interrupts = <16 2 0 0>; + fsl,bman-portals = <&bportals>; + memory-region = <&bman_fbpr>; + }; + global-utilities@e0000 { compatible = "fsl,p1023-guts"; reg = <0xe0000 0x1000>; diff --git a/arch/powerpc/boot/dts/fsl/p2041si-post.dtsi b/arch/powerpc/boot/dts/fsl/p2041si-post.dtsi index efd74db4f9b0..f2feacfd9a25 100644 --- a/arch/powerpc/boot/dts/fsl/p2041si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/p2041si-post.dtsi @@ -1,7 +1,7 @@ /* * P2041/P2040 Silicon/SoC Device Tree Source (post include) * - * Copyright 2011 Freescale Semiconductor Inc. + * Copyright 2011 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10 0>; +}; + &lbc { compatible = "fsl,p2041-elbc", "fsl,elbc", "simple-bus"; interrupts = <25 2 0 0>; @@ -216,6 +221,8 @@ }; }; +/include/ "qoriq-bman1-portals.dtsi" + &soc { #address-cells = <1>; #size-cells = <1>; @@ -407,4 +414,6 @@ crypto: crypto@300000 { fsl,iommu-parent = <&pamu1>; }; + +/include/ "qoriq-bman1.dtsi" }; diff --git a/arch/powerpc/boot/dts/fsl/p3041si-post.dtsi b/arch/powerpc/boot/dts/fsl/p3041si-post.dtsi index d7425ef1ae41..d6fea37395ad 100644 --- a/arch/powerpc/boot/dts/fsl/p3041si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/p3041si-post.dtsi @@ -1,7 +1,7 @@ /* * P3041 Silicon/SoC Device Tree Source (post include) * - * Copyright 2011 Freescale Semiconductor Inc. + * Copyright 2011 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10 0>; +}; + &lbc { compatible = "fsl,p3041-elbc", "fsl,elbc", "simple-bus"; interrupts = <25 2 0 0>; @@ -243,6 +248,8 @@ }; }; +/include/ "qoriq-bman1-portals.dtsi" + &soc { #address-cells = <1>; #size-cells = <1>; @@ -434,4 +441,6 @@ crypto: crypto@300000 { fsl,iommu-parent = <&pamu1>; }; + +/include/ "qoriq-bman1.dtsi" }; diff --git a/arch/powerpc/boot/dts/fsl/p4080si-post.dtsi b/arch/powerpc/boot/dts/fsl/p4080si-post.dtsi index 7005a4a4cef0..89482c9b2301 100644 --- a/arch/powerpc/boot/dts/fsl/p4080si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/p4080si-post.dtsi @@ -1,7 +1,7 @@ /* * P4080/P4040 Silicon/SoC Device Tree Source (post include) * - * Copyright 2011 Freescale Semiconductor Inc. + * Copyright 2011 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10 0>; +}; + &lbc { compatible = "fsl,p4080-elbc", "fsl,elbc", "simple-bus"; interrupts = <25 2 0 0>; @@ -243,6 +248,8 @@ }; +/include/ "qoriq-bman1-portals.dtsi" + &soc { #address-cells = <1>; #size-cells = <1>; @@ -490,4 +497,6 @@ crypto: crypto@300000 { fsl,iommu-parent = <&pamu1>; }; + +/include/ "qoriq-bman1.dtsi" }; diff --git a/arch/powerpc/boot/dts/fsl/p5020si-post.dtsi b/arch/powerpc/boot/dts/fsl/p5020si-post.dtsi index 55834211bd28..6e04851e2fc9 100644 --- a/arch/powerpc/boot/dts/fsl/p5020si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/p5020si-post.dtsi @@ -1,7 +1,7 @@ /* * P5020/5010 Silicon/SoC Device Tree Source (post include) * - * Copyright 2011 Freescale Semiconductor Inc. + * Copyright 2011 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10000 0>; +}; + &lbc { compatible = "fsl,p5020-elbc", "fsl,elbc", "simple-bus"; interrupts = <25 2 0 0>; @@ -240,6 +245,8 @@ }; }; +/include/ "qoriq-bman1-portals.dtsi" + &soc { #address-cells = <1>; #size-cells = <1>; @@ -421,6 +428,8 @@ fsl,iommu-parent = <&pamu1>; }; +/include/ "qoriq-bman1.dtsi" + /include/ "qoriq-raid1.0-0.dtsi" raideng@320000 { fsl,iommu-parent = <&pamu1>; diff --git a/arch/powerpc/boot/dts/fsl/p5040si-post.dtsi b/arch/powerpc/boot/dts/fsl/p5040si-post.dtsi index 6e4cd6ce363c..5e44dfa1e1a5 100644 --- a/arch/powerpc/boot/dts/fsl/p5040si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/p5040si-post.dtsi @@ -1,7 +1,7 @@ /* * P5040 Silicon/SoC Device Tree Source (post include) * - * Copyright 2012 Freescale Semiconductor Inc. + * Copyright 2012 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * software, even if advised of the possibility of such damage. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10000 0>; +}; + &lbc { compatible = "fsl,p5040-elbc", "fsl,elbc", "simple-bus"; interrupts = <25 2 0 0>; @@ -195,6 +200,8 @@ }; }; +/include/ "qoriq-bman1-portals.dtsi" + &soc { #address-cells = <1>; #size-cells = <1>; @@ -399,4 +406,6 @@ crypto@300000 { fsl,iommu-parent = <&pamu4>; }; + +/include/ "qoriq-bman1.dtsi" }; diff --git a/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi b/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi index 15ae462e758f..5cc01be5b152 100644 --- a/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/t1040si-post.dtsi @@ -1,7 +1,7 @@ /* * T1040 Silicon/SoC Device Tree Source (post include) * - * Copyright 2013 Freescale Semiconductor Inc. + * Copyright 2013 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10000 0>; +}; + &ifc { #address-cells = <2>; #size-cells = <1>; @@ -218,6 +223,63 @@ }; }; +&bportals { + #address-cells = <0x1>; + #size-cells = <0x1>; + compatible = "simple-bus"; + + bman-portal@0 { + compatible = "fsl,bman-portal"; + reg = <0x0 0x4000>, <0x1000000 0x1000>; + interrupts = <105 2 0 0>; + }; + bman-portal@4000 { + compatible = "fsl,bman-portal"; + reg = <0x4000 0x4000>, <0x1001000 0x1000>; + interrupts = <107 2 0 0>; + }; + bman-portal@8000 { + compatible = "fsl,bman-portal"; + reg = <0x8000 0x4000>, <0x1002000 0x1000>; + interrupts = <109 2 0 0>; + }; + bman-portal@c000 { + compatible = "fsl,bman-portal"; + reg = <0xc000 0x4000>, <0x1003000 0x1000>; + interrupts = <111 2 0 0>; + }; + bman-portal@10000 { + compatible = "fsl,bman-portal"; + reg = <0x10000 0x4000>, <0x1004000 0x1000>; + interrupts = <113 2 0 0>; + }; + bman-portal@14000 { + compatible = "fsl,bman-portal"; + reg = <0x14000 0x4000>, <0x1005000 0x1000>; + interrupts = <115 2 0 0>; + }; + bman-portal@18000 { + compatible = "fsl,bman-portal"; + reg = <0x18000 0x4000>, <0x1006000 0x1000>; + interrupts = <117 2 0 0>; + }; + bman-portal@1c000 { + compatible = "fsl,bman-portal"; + reg = <0x1c000 0x4000>, <0x1007000 0x1000>; + interrupts = <119 2 0 0>; + }; + bman-portal@20000 { + compatible = "fsl,bman-portal"; + reg = <0x20000 0x4000>, <0x1008000 0x1000>; + interrupts = <121 2 0 0>; + }; + bman-portal@24000 { + compatible = "fsl,bman-portal"; + reg = <0x24000 0x4000>, <0x1009000 0x1000>; + interrupts = <123 2 0 0>; + }; +}; + &soc { #address-cells = <1>; #size-cells = <1>; @@ -401,4 +463,5 @@ fsl,liodn-reg = <&guts 0x554>; /* SATA2LIODNR */ }; /include/ "qoriq-sec5.0-0.dtsi" +/include/ "qoriq-bman1.dtsi" }; diff --git a/arch/powerpc/boot/dts/fsl/t2081si-post.dtsi b/arch/powerpc/boot/dts/fsl/t2081si-post.dtsi index 1ce91e3485a9..86bdaf6cbd14 100644 --- a/arch/powerpc/boot/dts/fsl/t2081si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/t2081si-post.dtsi @@ -1,7 +1,7 @@ /* * T2081 Silicon/SoC Device Tree Source (post include) * - * Copyright 2013 Freescale Semiconductor Inc. + * Copyright 2013 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10000 0>; +}; + &ifc { #address-cells = <2>; #size-cells = <1>; @@ -224,6 +229,103 @@ }; }; +&bportals { + #address-cells = <0x1>; + #size-cells = <0x1>; + compatible = "simple-bus"; + + bman-portal@0 { + compatible = "fsl,bman-portal"; + reg = <0x0 0x4000>, <0x1000000 0x1000>; + interrupts = <105 2 0 0>; + }; + bman-portal@4000 { + compatible = "fsl,bman-portal"; + reg = <0x4000 0x4000>, <0x1001000 0x1000>; + interrupts = <107 2 0 0>; + }; + bman-portal@8000 { + compatible = "fsl,bman-portal"; + reg = <0x8000 0x4000>, <0x1002000 0x1000>; + interrupts = <109 2 0 0>; + }; + bman-portal@c000 { + compatible = "fsl,bman-portal"; + reg = <0xc000 0x4000>, <0x1003000 0x1000>; + interrupts = <111 2 0 0>; + }; + bman-portal@10000 { + compatible = "fsl,bman-portal"; + reg = <0x10000 0x4000>, <0x1004000 0x1000>; + interrupts = <113 2 0 0>; + }; + bman-portal@14000 { + compatible = "fsl,bman-portal"; + reg = <0x14000 0x4000>, <0x1005000 0x1000>; + interrupts = <115 2 0 0>; + }; + bman-portal@18000 { + compatible = "fsl,bman-portal"; + reg = <0x18000 0x4000>, <0x1006000 0x1000>; + interrupts = <117 2 0 0>; + }; + bman-portal@1c000 { + compatible = "fsl,bman-portal"; + reg = <0x1c000 0x4000>, <0x1007000 0x1000>; + interrupts = <119 2 0 0>; + }; + bman-portal@20000 { + compatible = "fsl,bman-portal"; + reg = <0x20000 0x4000>, <0x1008000 0x1000>; + interrupts = <121 2 0 0>; + }; + bman-portal@24000 { + compatible = "fsl,bman-portal"; + reg = <0x24000 0x4000>, <0x1009000 0x1000>; + interrupts = <123 2 0 0>; + }; + bman-portal@28000 { + compatible = "fsl,bman-portal"; + reg = <0x28000 0x4000>, <0x100a000 0x1000>; + interrupts = <125 2 0 0>; + }; + bman-portal@2c000 { + compatible = "fsl,bman-portal"; + reg = <0x2c000 0x4000>, <0x100b000 0x1000>; + interrupts = <127 2 0 0>; + }; + bman-portal@30000 { + compatible = "fsl,bman-portal"; + reg = <0x30000 0x4000>, <0x100c000 0x1000>; + interrupts = <129 2 0 0>; + }; + bman-portal@34000 { + compatible = "fsl,bman-portal"; + reg = <0x34000 0x4000>, <0x100d000 0x1000>; + interrupts = <131 2 0 0>; + }; + bman-portal@38000 { + compatible = "fsl,bman-portal"; + reg = <0x38000 0x4000>, <0x100e000 0x1000>; + interrupts = <133 2 0 0>; + }; + bman-portal@3c000 { + compatible = "fsl,bman-portal"; + reg = <0x3c000 0x4000>, <0x100f000 0x1000>; + interrupts = <135 2 0 0>; + }; + bman-portal@40000 { + compatible = "fsl,bman-portal"; + reg = <0x40000 0x4000>, <0x1010000 0x1000>; + interrupts = <137 2 0 0>; + }; + bman-portal@44000 { + compatible = "fsl,bman-portal"; + reg = <0x44000 0x4000>, <0x1011000 0x1000>; + interrupts = <139 2 0 0>; + }; +}; + &soc { #address-cells = <1>; #size-cells = <1>; @@ -400,6 +502,7 @@ phy_type = "utmi"; }; /include/ "qoriq-sec5.2-0.dtsi" +/include/ "qoriq-bman1.dtsi" L2_1: l2-cache-controller@c20000 { /* Cluster 0 L2 cache */ diff --git a/arch/powerpc/boot/dts/fsl/t4240si-post.dtsi b/arch/powerpc/boot/dts/fsl/t4240si-post.dtsi index 0e96fcabe812..4d4f25895d8c 100644 --- a/arch/powerpc/boot/dts/fsl/t4240si-post.dtsi +++ b/arch/powerpc/boot/dts/fsl/t4240si-post.dtsi @@ -1,7 +1,7 @@ /* * T4240 Silicon/SoC Device Tree Source (post include) * - * Copyright 2012 Freescale Semiconductor Inc. + * Copyright 2012 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,11 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +&bman_fbpr { + compatible = "fsl,bman-fbpr"; + alloc-ranges = <0 0 0x10000 0>; +}; + &ifc { #address-cells = <2>; #size-cells = <1>; @@ -294,6 +299,263 @@ }; }; +&bportals { + #address-cells = <0x1>; + #size-cells = <0x1>; + compatible = "simple-bus"; + + bman-portal@0 { + compatible = "fsl,bman-portal"; + reg = <0x0 0x4000>, <0x1000000 0x1000>; + interrupts = <105 2 0 0>; + }; + bman-portal@4000 { + compatible = "fsl,bman-portal"; + reg = <0x4000 0x4000>, <0x1001000 0x1000>; + interrupts = <107 2 0 0>; + }; + bman-portal@8000 { + compatible = "fsl,bman-portal"; + reg = <0x8000 0x4000>, <0x1002000 0x1000>; + interrupts = <109 2 0 0>; + }; + bman-portal@c000 { + compatible = "fsl,bman-portal"; + reg = <0xc000 0x4000>, <0x1003000 0x1000>; + interrupts = <111 2 0 0>; + }; + bman-portal@10000 { + compatible = "fsl,bman-portal"; + reg = <0x10000 0x4000>, <0x1004000 0x1000>; + interrupts = <113 2 0 0>; + }; + bman-portal@14000 { + compatible = "fsl,bman-portal"; + reg = <0x14000 0x4000>, <0x1005000 0x1000>; + interrupts = <115 2 0 0>; + }; + bman-portal@18000 { + compatible = "fsl,bman-portal"; + reg = <0x18000 0x4000>, <0x1006000 0x1000>; + interrupts = <117 2 0 0>; + }; + bman-portal@1c000 { + compatible = "fsl,bman-portal"; + reg = <0x1c000 0x4000>, <0x1007000 0x1000>; + interrupts = <119 2 0 0>; + }; + bman-portal@20000 { + compatible = "fsl,bman-portal"; + reg = <0x20000 0x4000>, <0x1008000 0x1000>; + interrupts = <121 2 0 0>; + }; + bman-portal@24000 { + compatible = "fsl,bman-portal"; + reg = <0x24000 0x4000>, <0x1009000 0x1000>; + interrupts = <123 2 0 0>; + }; + bman-portal@28000 { + compatible = "fsl,bman-portal"; + reg = <0x28000 0x4000>, <0x100a000 0x1000>; + interrupts = <125 2 0 0>; + }; + bman-portal@2c000 { + compatible = "fsl,bman-portal"; + reg = <0x2c000 0x4000>, <0x100b000 0x1000>; + interrupts = <127 2 0 0>; + }; + bman-portal@30000 { + compatible = "fsl,bman-portal"; + reg = <0x30000 0x4000>, <0x100c000 0x1000>; + interrupts = <129 2 0 0>; + }; + bman-portal@34000 { + compatible = "fsl,bman-portal"; + reg = <0x34000 0x4000>, <0x100d000 0x1000>; + interrupts = <131 2 0 0>; + }; + bman-portal@38000 { + compatible = "fsl,bman-portal"; + reg = <0x38000 0x4000>, <0x100e000 0x1000>; + interrupts = <133 2 0 0>; + }; + bman-portal@3c000 { + compatible = "fsl,bman-portal"; + reg = <0x3c000 0x4000>, <0x100f000 0x1000>; + interrupts = <135 2 0 0>; + }; + bman-portal@40000 { + compatible = "fsl,bman-portal"; + reg = <0x40000 0x4000>, <0x1010000 0x1000>; + interrupts = <137 2 0 0>; + }; + bman-portal@44000 { + compatible = "fsl,bman-portal"; + reg = <0x44000 0x4000>, <0x1011000 0x1000>; + interrupts = <139 2 0 0>; + }; + bman-portal@48000 { + compatible = "fsl,bman-portal"; + reg = <0x48000 0x4000>, <0x1012000 0x1000>; + interrupts = <141 2 0 0>; + }; + bman-portal@4c000 { + compatible = "fsl,bman-portal"; + reg = <0x4c000 0x4000>, <0x1013000 0x1000>; + interrupts = <143 2 0 0>; + }; + bman-portal@50000 { + compatible = "fsl,bman-portal"; + reg = <0x50000 0x4000>, <0x1014000 0x1000>; + interrupts = <145 2 0 0>; + }; + bman-portal@54000 { + compatible = "fsl,bman-portal"; + reg = <0x54000 0x4000>, <0x1015000 0x1000>; + interrupts = <147 2 0 0>; + }; + bman-portal@58000 { + compatible = "fsl,bman-portal"; + reg = <0x58000 0x4000>, <0x1016000 0x1000>; + interrupts = <149 2 0 0>; + }; + bman-portal@5c000 { + compatible = "fsl,bman-portal"; + reg = <0x5c000 0x4000>, <0x1017000 0x1000>; + interrupts = <151 2 0 0>; + }; + bman-portal@60000 { + compatible = "fsl,bman-portal"; + reg = <0x60000 0x4000>, <0x1018000 0x1000>; + interrupts = <153 2 0 0>; + }; + bman-portal@64000 { + compatible = "fsl,bman-portal"; + reg = <0x64000 0x4000>, <0x1019000 0x1000>; + interrupts = <155 2 0 0>; + }; + bman-portal@68000 { + compatible = "fsl,bman-portal"; + reg = <0x68000 0x4000>, <0x101a000 0x1000>; + interrupts = <157 2 0 0>; + }; + bman-portal@6c000 { + compatible = "fsl,bman-portal"; + reg = <0x6c000 0x4000>, <0x101b000 0x1000>; + interrupts = <159 2 0 0>; + }; + bman-portal@70000 { + compatible = "fsl,bman-portal"; + reg = <0x70000 0x4000>, <0x101c000 0x1000>; + interrupts = <161 2 0 0>; + }; + bman-portal@74000 { + compatible = "fsl,bman-portal"; + reg = <0x74000 0x4000>, <0x101d000 0x1000>; + interrupts = <163 2 0 0>; + }; + bman-portal@78000 { + compatible = "fsl,bman-portal"; + reg = <0x78000 0x4000>, <0x101e000 0x1000>; + interrupts = <165 2 0 0>; + }; + bman-portal@7c000 { + compatible = "fsl,bman-portal"; + reg = <0x7c000 0x4000>, <0x101f000 0x1000>; + interrupts = <167 2 0 0>; + }; + bman-portal@80000 { + compatible = "fsl,bman-portal"; + reg = <0x80000 0x4000>, <0x1020000 0x1000>; + interrupts = <169 2 0 0>; + }; + bman-portal@84000 { + compatible = "fsl,bman-portal"; + reg = <0x84000 0x4000>, <0x1021000 0x1000>; + interrupts = <171 2 0 0>; + }; + bman-portal@88000 { + compatible = "fsl,bman-portal"; + reg = <0x88000 0x4000>, <0x1022000 0x1000>; + interrupts = <173 2 0 0>; + }; + bman-portal@8c000 { + compatible = "fsl,bman-portal"; + reg = <0x8c000 0x4000>, <0x1023000 0x1000>; + interrupts = <175 2 0 0>; + }; + bman-portal@90000 { + compatible = "fsl,bman-portal"; + reg = <0x90000 0x4000>, <0x1024000 0x1000>; + interrupts = <385 2 0 0>; + }; + bman-portal@94000 { + compatible = "fsl,bman-portal"; + reg = <0x94000 0x4000>, <0x1025000 0x1000>; + interrupts = <387 2 0 0>; + }; + bman-portal@98000 { + compatible = "fsl,bman-portal"; + reg = <0x98000 0x4000>, <0x1026000 0x1000>; + interrupts = <389 2 0 0>; + }; + bman-portal@9c000 { + compatible = "fsl,bman-portal"; + reg = <0x9c000 0x4000>, <0x1027000 0x1000>; + interrupts = <391 2 0 0>; + }; + bman-portal@a0000 { + compatible = "fsl,bman-portal"; + reg = <0xa0000 0x4000>, <0x1028000 0x1000>; + interrupts = <393 2 0 0>; + }; + bman-portal@a4000 { + compatible = "fsl,bman-portal"; + reg = <0xa4000 0x4000>, <0x1029000 0x1000>; + interrupts = <395 2 0 0>; + }; + bman-portal@a8000 { + compatible = "fsl,bman-portal"; + reg = <0xa8000 0x4000>, <0x102a000 0x1000>; + interrupts = <397 2 0 0>; + }; + bman-portal@ac000 { + compatible = "fsl,bman-portal"; + reg = <0xac000 0x4000>, <0x102b000 0x1000>; + interrupts = <399 2 0 0>; + }; + bman-portal@b0000 { + compatible = "fsl,bman-portal"; + reg = <0xb0000 0x4000>, <0x102c000 0x1000>; + interrupts = <401 2 0 0>; + }; + bman-portal@b4000 { + compatible = "fsl,bman-portal"; + reg = <0xb4000 0x4000>, <0x102d000 0x1000>; + interrupts = <403 2 0 0>; + }; + bman-portal@b8000 { + compatible = "fsl,bman-portal"; + reg = <0xb8000 0x4000>, <0x102e000 0x1000>; + interrupts = <405 2 0 0>; + }; + bman-portal@bc000 { + compatible = "fsl,bman-portal"; + reg = <0xbc000 0x4000>, <0x102f000 0x1000>; + interrupts = <407 2 0 0>; + }; + bman-portal@c0000 { + compatible = "fsl,bman-portal"; + reg = <0xc0000 0x4000>, <0x1030000 0x1000>; + interrupts = <409 2 0 0>; + }; + bman-portal@c4000 { + compatible = "fsl,bman-portal"; + reg = <0xc4000 0x4000>, <0x1031000 0x1000>; + interrupts = <411 2 0 0>; + }; +}; + &soc { #address-cells = <1>; #size-cells = <1>; @@ -486,6 +748,7 @@ /include/ "qoriq-sata2-0.dtsi" /include/ "qoriq-sata2-1.dtsi" /include/ "qoriq-sec5.0-0.dtsi" +/include/ "qoriq-bman1.dtsi" L2_1: l2-cache-controller@c20000 { compatible = "fsl,t4240-l2-cache-controller"; diff --git a/arch/powerpc/boot/dts/kmcoge4.dts b/arch/powerpc/boot/dts/kmcoge4.dts index 89b4119f3b19..97e6d11d1e6d 100644 --- a/arch/powerpc/boot/dts/kmcoge4.dts +++ b/arch/powerpc/boot/dts/kmcoge4.dts @@ -25,10 +25,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01008000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x200000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/oca4080.dts b/arch/powerpc/boot/dts/oca4080.dts index 3d4c751d1608..eb76caae11d9 100644 --- a/arch/powerpc/boot/dts/oca4080.dts +++ b/arch/powerpc/boot/dts/oca4080.dts @@ -49,10 +49,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01008000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x200000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/p1023rdb.dts b/arch/powerpc/boot/dts/p1023rdb.dts index 0a06a88ddbd5..9236e3742a23 100644 --- a/arch/powerpc/boot/dts/p1023rdb.dts +++ b/arch/powerpc/boot/dts/p1023rdb.dts @@ -1,7 +1,7 @@ /* * P1023 RDB Device Tree Source * - * Copyright 2013 Freescale Semiconductor Inc. + * Copyright 2013 - 2014 Freescale Semiconductor Inc. * * Author: Chunhe Lan <Chunhe.Lan@freescale.com> * @@ -47,6 +47,21 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + + bportals: bman-portals@ff200000 { + ranges = <0x0 0xf 0xff200000 0x200000>; + }; + soc: soc@ff600000 { ranges = <0x0 0x0 0xff600000 0x200000>; @@ -228,7 +243,6 @@ 0x0 0x100000>; }; }; - }; /include/ "fsl/p1023si-post.dtsi" diff --git a/arch/powerpc/boot/dts/p2041rdb.dts b/arch/powerpc/boot/dts/p2041rdb.dts index d97ad74c7279..c1e69dc7188e 100644 --- a/arch/powerpc/boot/dts/p2041rdb.dts +++ b/arch/powerpc/boot/dts/p2041rdb.dts @@ -1,7 +1,7 @@ /* * P2041RDB Device Tree Source * - * Copyright 2011 Freescale Semiconductor Inc. + * Copyright 2011 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,10 +45,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01008000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x200000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/p3041ds.dts b/arch/powerpc/boot/dts/p3041ds.dts index 394ea9c943c9..2192fe94866d 100644 --- a/arch/powerpc/boot/dts/p3041ds.dts +++ b/arch/powerpc/boot/dts/p3041ds.dts @@ -1,7 +1,7 @@ /* * P3041DS Device Tree Source * - * Copyright 2010-2011 Freescale Semiconductor Inc. + * Copyright 2010 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,10 +45,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01008000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x200000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/p4080ds.dts b/arch/powerpc/boot/dts/p4080ds.dts index 1cf6148b8b05..fad441654642 100644 --- a/arch/powerpc/boot/dts/p4080ds.dts +++ b/arch/powerpc/boot/dts/p4080ds.dts @@ -1,7 +1,7 @@ /* * P4080DS Device Tree Source * - * Copyright 2009-2011 Freescale Semiconductor Inc. + * Copyright 2009 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,10 +45,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01008000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x200000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/p5020ds.dts b/arch/powerpc/boot/dts/p5020ds.dts index b7f3057cd894..7382636dc560 100644 --- a/arch/powerpc/boot/dts/p5020ds.dts +++ b/arch/powerpc/boot/dts/p5020ds.dts @@ -1,7 +1,7 @@ /* * P5020DS Device Tree Source * - * Copyright 2010-2011 Freescale Semiconductor Inc. + * Copyright 2010 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,10 +45,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01008000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x200000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/p5040ds.dts b/arch/powerpc/boot/dts/p5040ds.dts index 7e04bf487c04..35dabf5b6098 100644 --- a/arch/powerpc/boot/dts/p5040ds.dts +++ b/arch/powerpc/boot/dts/p5040ds.dts @@ -1,7 +1,7 @@ /* * P5040DS Device Tree Source * - * Copyright 2012 Freescale Semiconductor Inc. + * Copyright 2012 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,10 +45,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01008000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x200000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/t104xqds.dtsi b/arch/powerpc/boot/dts/t104xqds.dtsi index 234f4b596c5b..f7e9bfbeefc7 100644 --- a/arch/powerpc/boot/dts/t104xqds.dtsi +++ b/arch/powerpc/boot/dts/t104xqds.dtsi @@ -1,7 +1,7 @@ /* * T104xQDS Device Tree Source * - * Copyright 2013 Freescale Semiconductor Inc. + * Copyright 2013 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,6 +38,17 @@ #size-cells = <2>; interrupt-parent = <&mpic>; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + ifc: localbus@ffe124000 { reg = <0xf 0xfe124000 0 0x2000>; ranges = <0 0 0xf 0xe8000000 0x08000000 @@ -77,6 +88,10 @@ ranges = <0x00000000 0xf 0x00000000 0x01072000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/t104xrdb.dtsi b/arch/powerpc/boot/dts/t104xrdb.dtsi index 187add885cae..76e07a3f2ca8 100644 --- a/arch/powerpc/boot/dts/t104xrdb.dtsi +++ b/arch/powerpc/boot/dts/t104xrdb.dtsi @@ -33,6 +33,16 @@ */ / { + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; ifc: localbus@ffe124000 { reg = <0xf 0xfe124000 0 0x2000>; @@ -69,6 +79,10 @@ ranges = <0x00000000 0xf 0x00000000 0x01072000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/t208xqds.dtsi b/arch/powerpc/boot/dts/t208xqds.dtsi index 59061834d54e..c42e07f4f648 100644 --- a/arch/powerpc/boot/dts/t208xqds.dtsi +++ b/arch/powerpc/boot/dts/t208xqds.dtsi @@ -1,7 +1,7 @@ /* * T2080/T2081 QDS Device Tree Source * - * Copyright 2013 Freescale Semiconductor Inc. + * Copyright 2013 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,6 +39,17 @@ #size-cells = <2>; interrupt-parent = <&mpic>; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + ifc: localbus@ffe124000 { reg = <0xf 0xfe124000 0 0x2000>; ranges = <0 0 0xf 0xe8000000 0x08000000 @@ -78,6 +89,10 @@ ranges = <0x00000000 0xf 0x00000000 0x01072000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; @@ -137,7 +152,7 @@ rtc@68 { compatible = "dallas,ds3232"; reg = <0x68>; - interrupts = <0x1 0x1 0 0>; + interrupts = <0xb 0x1 0 0>; }; }; diff --git a/arch/powerpc/boot/dts/t208xrdb.dtsi b/arch/powerpc/boot/dts/t208xrdb.dtsi index 1481e192e783..e1463b165d0e 100644 --- a/arch/powerpc/boot/dts/t208xrdb.dtsi +++ b/arch/powerpc/boot/dts/t208xrdb.dtsi @@ -39,6 +39,17 @@ #size-cells = <2>; interrupt-parent = <&mpic>; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + ifc: localbus@ffe124000 { reg = <0xf 0xfe124000 0 0x2000>; ranges = <0 0 0xf 0xe8000000 0x08000000 @@ -79,6 +90,10 @@ ranges = <0x00000000 0xf 0x00000000 0x01072000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/t4240qds.dts b/arch/powerpc/boot/dts/t4240qds.dts index 97683f6a2936..6df77766410b 100644 --- a/arch/powerpc/boot/dts/t4240qds.dts +++ b/arch/powerpc/boot/dts/t4240qds.dts @@ -1,7 +1,7 @@ /* * T4240QDS Device Tree Source * - * Copyright 2012 Freescale Semiconductor Inc. + * Copyright 2012 - 2014 Freescale Semiconductor Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -100,10 +100,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01072000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/dts/t4240rdb.dts b/arch/powerpc/boot/dts/t4240rdb.dts index 53761d4e8c51..46049cf37f02 100644 --- a/arch/powerpc/boot/dts/t4240rdb.dts +++ b/arch/powerpc/boot/dts/t4240rdb.dts @@ -69,10 +69,25 @@ device_type = "memory"; }; + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + bman_fbpr: bman-fbpr { + size = <0 0x1000000>; + alignment = <0 0x1000000>; + }; + }; + dcsr: dcsr@f00000000 { ranges = <0x00000000 0xf 0x00000000 0x01072000>; }; + bportals: bman-portals@ff4000000 { + ranges = <0x0 0xf 0xf4000000 0x2000000>; + }; + soc: soc@ffe000000 { ranges = <0x00000000 0xf 0xfe000000 0x1000000>; reg = <0xf 0xfe000000 0 0x00001000>; diff --git a/arch/powerpc/boot/libfdt-wrapper.c b/arch/powerpc/boot/libfdt-wrapper.c index bb8b9b3505ee..535e8fd8900d 100644 --- a/arch/powerpc/boot/libfdt-wrapper.c +++ b/arch/powerpc/boot/libfdt-wrapper.c @@ -44,12 +44,12 @@ #define offset_devp(off) \ ({ \ - int _offset = (off); \ + unsigned long _offset = (off); \ check_err(_offset) ? NULL : (void *)(_offset+1); \ }) -#define devp_offset_find(devp) (((int)(devp))-1) -#define devp_offset(devp) (devp ? ((int)(devp))-1 : 0) +#define devp_offset_find(devp) (((unsigned long)(devp))-1) +#define devp_offset(devp) (devp ? ((unsigned long)(devp))-1 : 0) static void *fdt; static void *buf; /* = NULL */ diff --git a/arch/powerpc/boot/libfdt_env.h b/arch/powerpc/boot/libfdt_env.h index c89fdb1b80e1..8dcd744e5728 100644 --- a/arch/powerpc/boot/libfdt_env.h +++ b/arch/powerpc/boot/libfdt_env.h @@ -4,15 +4,17 @@ #include <types.h> #include <string.h> +#include "of.h" + typedef u32 uint32_t; typedef u64 uint64_t; typedef unsigned long uintptr_t; -#define fdt16_to_cpu(x) (x) -#define cpu_to_fdt16(x) (x) -#define fdt32_to_cpu(x) (x) -#define cpu_to_fdt32(x) (x) -#define fdt64_to_cpu(x) (x) -#define cpu_to_fdt64(x) (x) +#define fdt16_to_cpu(x) be16_to_cpu(x) +#define cpu_to_fdt16(x) cpu_to_be16(x) +#define fdt32_to_cpu(x) be32_to_cpu(x) +#define cpu_to_fdt32(x) cpu_to_be32(x) +#define fdt64_to_cpu(x) be64_to_cpu(x) +#define cpu_to_fdt64(x) cpu_to_be64(x) #endif /* _ARCH_POWERPC_BOOT_LIBFDT_ENV_H */ diff --git a/arch/powerpc/boot/of.h b/arch/powerpc/boot/of.h index c8c1750aba0c..5603320dce07 100644 --- a/arch/powerpc/boot/of.h +++ b/arch/powerpc/boot/of.h @@ -24,11 +24,19 @@ void of_console_init(void); typedef u32 __be32; #ifdef __LITTLE_ENDIAN__ +#define cpu_to_be16(x) swab16(x) +#define be16_to_cpu(x) swab16(x) #define cpu_to_be32(x) swab32(x) #define be32_to_cpu(x) swab32(x) +#define cpu_to_be64(x) swab64(x) +#define be64_to_cpu(x) swab64(x) #else +#define cpu_to_be16(x) (x) +#define be16_to_cpu(x) (x) #define cpu_to_be32(x) (x) #define be32_to_cpu(x) (x) +#define cpu_to_be64(x) (x) +#define be64_to_cpu(x) (x) #endif #define PROM_ERROR (-1u) diff --git a/arch/powerpc/boot/planetcore.c b/arch/powerpc/boot/planetcore.c index 0d8558a475bb..75117e63e6db 100644 --- a/arch/powerpc/boot/planetcore.c +++ b/arch/powerpc/boot/planetcore.c @@ -131,36 +131,3 @@ void planetcore_set_stdout_path(const char *table) setprop_str(chosen, "linux,stdout-path", path); } - -void planetcore_set_serial_speed(const char *table) -{ - void *chosen, *stdout; - u64 baud; - u32 baud32; - int len; - - chosen = finddevice("/chosen"); - if (!chosen) - return; - - len = getprop(chosen, "linux,stdout-path", prop_buf, MAX_PROP_LEN); - if (len <= 0) - return; - - stdout = finddevice(prop_buf); - if (!stdout) { - printf("planetcore_set_serial_speed: " - "Bad /chosen/linux,stdout-path.\r\n"); - - return; - } - - if (!planetcore_get_decimal(table, PLANETCORE_KEY_SERIAL_BAUD, - &baud)) { - printf("planetcore_set_serial_speed: No SB tag.\r\n"); - return; - } - - baud32 = baud; - setprop(stdout, "current-speed", &baud32, 4); -} diff --git a/arch/powerpc/boot/planetcore.h b/arch/powerpc/boot/planetcore.h index 0d4094f1771c..d53c733cc463 100644 --- a/arch/powerpc/boot/planetcore.h +++ b/arch/powerpc/boot/planetcore.h @@ -43,7 +43,4 @@ void planetcore_set_mac_addrs(const char *table); */ void planetcore_set_stdout_path(const char *table); -/* Sets the current-speed property in the serial node. */ -void planetcore_set_serial_speed(const char *table); - #endif diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper index ae0f88ec4a32..3f50c27ed8f8 100755 --- a/arch/powerpc/boot/wrapper +++ b/arch/powerpc/boot/wrapper @@ -277,7 +277,7 @@ treeboot-iss4xx-mpic) platformo="$object/treeboot-iss4xx.o" ;; epapr) - platformo="$object/epapr.o $object/epapr-wrapper.o" + platformo="$object/pseries-head.o $object/epapr.o $object/epapr-wrapper.o" link_address='0x20000000' pie=-pie ;; diff --git a/arch/powerpc/configs/cell_defconfig b/arch/powerpc/configs/cell_defconfig index 9788b3c2d563..9227b517560a 100644 --- a/arch/powerpc/configs/cell_defconfig +++ b/arch/powerpc/configs/cell_defconfig @@ -28,7 +28,6 @@ CONFIG_PS3_ROM=m CONFIG_PS3_FLASH=m CONFIG_PS3_LPM=m CONFIG_PPC_IBM_CELL_BLADE=y -CONFIG_PPC_CELLEB=y CONFIG_RTAS_FLASH=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y @@ -113,7 +112,6 @@ CONFIG_IDE=y CONFIG_BLK_DEV_GENERIC=y CONFIG_BLK_DEV_AEC62XX=y CONFIG_BLK_DEV_SIIMAGE=y -CONFIG_BLK_DEV_CELLEB=y CONFIG_BLK_DEV_SD=y CONFIG_BLK_DEV_SR=m CONFIG_CHR_DEV_SG=y @@ -156,7 +154,6 @@ CONFIG_SERIAL_TXX9_NR_UARTS=2 CONFIG_SERIAL_TXX9_CONSOLE=y CONFIG_SERIAL_OF_PLATFORM=y CONFIG_HVC_RTAS=y -CONFIG_HVC_BEAT=y CONFIG_IPMI_HANDLER=m CONFIG_IPMI_DEVICE_INTERFACE=m CONFIG_IPMI_SI=m diff --git a/arch/powerpc/configs/celleb_defconfig b/arch/powerpc/configs/celleb_defconfig deleted file mode 100644 index ff454dcd2dd3..000000000000 --- a/arch/powerpc/configs/celleb_defconfig +++ /dev/null @@ -1,152 +0,0 @@ -CONFIG_PPC64=y -CONFIG_TUNE_CELL=y -CONFIG_ALTIVEC=y -CONFIG_SMP=y -CONFIG_NR_CPUS=4 -CONFIG_SYSVIPC=y -CONFIG_FHANDLE=y -CONFIG_NO_HZ=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_LOG_BUF_SHIFT=15 -CONFIG_BLK_DEV_INITRD=y -# CONFIG_COMPAT_BRK is not set -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_MODVERSIONS=y -CONFIG_MODULE_SRCVERSION_ALL=y -CONFIG_PARTITION_ADVANCED=y -# CONFIG_PPC_POWERNV is not set -# CONFIG_PPC_PSERIES is not set -# CONFIG_PPC_PMAC is not set -CONFIG_PPC_CELLEB=y -CONFIG_SPU_FS=y -# CONFIG_CBE_THERM is not set -CONFIG_UDBG_RTAS_CONSOLE=y -# CONFIG_RTAS_PROC is not set -CONFIG_BINFMT_MISC=m -CONFIG_KEXEC=y -CONFIG_NUMA=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_UNIX=y -CONFIG_INET=y -CONFIG_IP_MULTICAST=y -CONFIG_SYN_COOKIES=y -CONFIG_IPV6=y -CONFIG_INET6_AH=m -CONFIG_INET6_ESP=m -CONFIG_INET6_IPCOMP=m -CONFIG_IPV6_TUNNEL=m -CONFIG_NETFILTER=y -CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug" -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_RAM=y -CONFIG_BLK_DEV_RAM_SIZE=131072 -CONFIG_IDE=y -CONFIG_BLK_DEV_IDECD=m -CONFIG_BLK_DEV_GENERIC=y -CONFIG_BLK_DEV_CELLEB=y -CONFIG_SCSI=m -# CONFIG_SCSI_PROC_FS is not set -CONFIG_BLK_DEV_SD=m -CONFIG_BLK_DEV_SR=m -CONFIG_CHR_DEV_SG=m -CONFIG_MD=y -CONFIG_BLK_DEV_MD=m -CONFIG_MD_LINEAR=m -CONFIG_MD_RAID0=m -CONFIG_MD_RAID1=m -CONFIG_BLK_DEV_DM=m -CONFIG_DM_CRYPT=m -CONFIG_DM_SNAPSHOT=m -CONFIG_DM_MIRROR=m -CONFIG_DM_ZERO=m -CONFIG_DM_MULTIPATH=m -CONFIG_NETDEVICES=y -CONFIG_SPIDER_NET=y -# CONFIG_INPUT_MOUSEDEV_PSAUX is not set -# CONFIG_INPUT_KEYBOARD is not set -# CONFIG_INPUT_MOUSE is not set -# CONFIG_SERIO_I8042 is not set -# CONFIG_LEGACY_PTYS is not set -CONFIG_SERIAL_NONSTANDARD=y -CONFIG_SERIAL_TXX9_NR_UARTS=3 -CONFIG_SERIAL_TXX9_CONSOLE=y -CONFIG_HVC_RTAS=y -CONFIG_HVC_BEAT=y -# CONFIG_HW_RANDOM is not set -CONFIG_GEN_RTC=y -CONFIG_I2C=y -# CONFIG_HWMON is not set -CONFIG_WATCHDOG=y -# CONFIG_VGA_CONSOLE is not set -CONFIG_USB_HIDDEV=y -CONFIG_USB=y -CONFIG_USB_MON=y -CONFIG_USB_EHCI_HCD=m -# CONFIG_USB_EHCI_HCD_PPC_OF is not set -CONFIG_USB_OHCI_HCD=m -CONFIG_USB_STORAGE=m -CONFIG_EXT2_FS=y -CONFIG_EXT2_FS_XATTR=y -CONFIG_EXT2_FS_POSIX_ACL=y -CONFIG_EXT2_FS_SECURITY=y -CONFIG_EXT2_FS_XIP=y -CONFIG_EXT3_FS=y -CONFIG_EXT3_FS_POSIX_ACL=y -CONFIG_EXT3_FS_SECURITY=y -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_UDF_FS=m -CONFIG_MSDOS_FS=m -CONFIG_VFAT_FS=m -CONFIG_PROC_KCORE=y -CONFIG_TMPFS=y -CONFIG_HUGETLBFS=y -CONFIG_NFS_FS=m -CONFIG_NFS_V3_ACL=y -CONFIG_NFSD=m -CONFIG_NFSD_V3=y -CONFIG_NFSD_V3_ACL=y -CONFIG_NLS_ISO8859_1=m -CONFIG_NLS_ISO8859_2=m -CONFIG_NLS_ISO8859_3=m -CONFIG_NLS_ISO8859_4=m -CONFIG_NLS_ISO8859_5=m -CONFIG_NLS_ISO8859_6=m -CONFIG_NLS_ISO8859_7=m -CONFIG_NLS_ISO8859_9=m -CONFIG_NLS_ISO8859_13=m -CONFIG_NLS_ISO8859_14=m -CONFIG_NLS_ISO8859_15=m -CONFIG_LIBCRC32C=m -CONFIG_DEBUG_FS=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_MUTEXES=y -CONFIG_XMON=y -CONFIG_XMON_DEFAULT=y -CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_TEST=m -CONFIG_CRYPTO_ECB=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_SHA256=m -CONFIG_CRYPTO_SHA512=m -CONFIG_CRYPTO_TGR192=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_ANUBIS=m -CONFIG_CRYPTO_ARC4=m -CONFIG_CRYPTO_BLOWFISH=m -CONFIG_CRYPTO_CAST5=m -CONFIG_CRYPTO_CAST6=m -CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_TEA=m -CONFIG_CRYPTO_TWOFISH=m -# CONFIG_CRYPTO_HW is not set diff --git a/arch/powerpc/configs/corenet32_smp_defconfig b/arch/powerpc/configs/corenet32_smp_defconfig index ca7957b09a3c..37659937bd12 100644 --- a/arch/powerpc/configs/corenet32_smp_defconfig +++ b/arch/powerpc/configs/corenet32_smp_defconfig @@ -99,6 +99,8 @@ CONFIG_E1000E=y CONFIG_AT803X_PHY=y CONFIG_VITESSE_PHY=y CONFIG_FIXED_PHY=y +CONFIG_MDIO_BUS_MUX_GPIO=y +CONFIG_MDIO_BUS_MUX_MMIOREG=y # CONFIG_INPUT_MOUSEDEV is not set # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set @@ -114,11 +116,14 @@ CONFIG_NVRAM=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_MPC=y +CONFIG_I2C_MUX=y +CONFIG_I2C_MUX_PCA954x=y CONFIG_SPI=y CONFIG_SPI_GPIO=y CONFIG_SPI_FSL_SPI=y CONFIG_SPI_FSL_ESPI=y -# CONFIG_HWMON is not set +CONFIG_SENSORS_LM90=y +CONFIG_SENSORS_INA2XX=y CONFIG_USB_HID=m CONFIG_USB=y CONFIG_USB_MON=y diff --git a/arch/powerpc/configs/corenet64_smp_defconfig b/arch/powerpc/configs/corenet64_smp_defconfig index 04737aaa8b6b..33cd1df818ad 100644 --- a/arch/powerpc/configs/corenet64_smp_defconfig +++ b/arch/powerpc/configs/corenet64_smp_defconfig @@ -12,6 +12,10 @@ CONFIG_BSD_PROCESS_ACCT=y CONFIG_IKCONFIG=y CONFIG_IKCONFIG_PROC=y CONFIG_LOG_BUF_SHIFT=14 +CONFIG_CGROUPS=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_CGROUP_SCHED=y CONFIG_BLK_DEV_INITRD=y CONFIG_EXPERT=y CONFIG_KALLSYMS_ALL=y @@ -75,6 +79,10 @@ CONFIG_BLK_DEV_LOOP=y CONFIG_BLK_DEV_RAM=y CONFIG_BLK_DEV_RAM_SIZE=131072 CONFIG_EEPROM_LEGACY=y +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=y +CONFIG_BLK_DEV_SR_VENDOR=y +CONFIG_CHR_DEV_SG=y CONFIG_ATA=y CONFIG_SATA_FSL=y CONFIG_SATA_SIL24=y @@ -85,6 +93,8 @@ CONFIG_FSL_XGMAC_MDIO=y CONFIG_E1000E=y CONFIG_VITESSE_PHY=y CONFIG_FIXED_PHY=y +CONFIG_MDIO_BUS_MUX_GPIO=y +CONFIG_MDIO_BUS_MUX_MMIOREG=y CONFIG_INPUT_FF_MEMLESS=m # CONFIG_INPUT_MOUSEDEV is not set # CONFIG_INPUT_KEYBOARD is not set @@ -99,11 +109,14 @@ CONFIG_SERIAL_8250_RSA=y CONFIG_I2C=y CONFIG_I2C_CHARDEV=y CONFIG_I2C_MPC=y +CONFIG_I2C_MUX=y +CONFIG_I2C_MUX_PCA954x=y CONFIG_SPI=y CONFIG_SPI_GPIO=y CONFIG_SPI_FSL_SPI=y CONFIG_SPI_FSL_ESPI=y -# CONFIG_HWMON is not set +CONFIG_SENSORS_LM90=y +CONFIG_SENSORS_INA2XX=y CONFIG_USB_HID=m CONFIG_USB=y CONFIG_USB_MON=y diff --git a/arch/powerpc/configs/mpc85xx_defconfig b/arch/powerpc/configs/mpc85xx_defconfig index 8535c343dd57..6ecf7bdbc2f9 100644 --- a/arch/powerpc/configs/mpc85xx_defconfig +++ b/arch/powerpc/configs/mpc85xx_defconfig @@ -150,8 +150,7 @@ CONFIG_SPI=y CONFIG_SPI_FSL_SPI=y CONFIG_SPI_FSL_ESPI=y CONFIG_GPIO_MPC8XXX=y -CONFIG_HWMON=m -CONFIG_SENSORS_LM90=m +CONFIG_SENSORS_LM90=y CONFIG_FB=y CONFIG_FB_FSL_DIU=y # CONFIG_VGA_CONSOLE is not set diff --git a/arch/powerpc/configs/mpc85xx_smp_defconfig b/arch/powerpc/configs/mpc85xx_smp_defconfig index c45ad2e01b0c..b6c7111ea913 100644 --- a/arch/powerpc/configs/mpc85xx_smp_defconfig +++ b/arch/powerpc/configs/mpc85xx_smp_defconfig @@ -143,7 +143,7 @@ CONFIG_SPI=y CONFIG_SPI_FSL_SPI=y CONFIG_SPI_FSL_ESPI=y CONFIG_GPIO_MPC8XXX=y -# CONFIG_HWMON is not set +CONFIG_SENSORS_LM90=y CONFIG_FB=y CONFIG_FB_FSL_DIU=y # CONFIG_VGA_CONSOLE is not set diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 3315c9f0828a..aad501ae3834 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -36,7 +36,6 @@ CONFIG_PS3_ROM=m CONFIG_PS3_FLASH=m CONFIG_PS3_LPM=m CONFIG_PPC_IBM_CELL_BLADE=y -CONFIG_PPC_CELLEB=y CONFIG_PPC_CELL_QPACE=y CONFIG_RTAS_FLASH=m CONFIG_IBMEBUS=y @@ -89,7 +88,6 @@ CONFIG_IDE=y CONFIG_BLK_DEV_IDECD=y CONFIG_BLK_DEV_GENERIC=y CONFIG_BLK_DEV_AMD74XX=y -CONFIG_BLK_DEV_CELLEB=y CONFIG_BLK_DEV_IDE_PMAC=y CONFIG_BLK_DEV_IDE_PMAC_ATA100FIRST=y CONFIG_BLK_DEV_SD=y @@ -196,7 +194,6 @@ CONFIG_SERIAL_TXX9_CONSOLE=y CONFIG_SERIAL_JSM=m CONFIG_HVC_CONSOLE=y CONFIG_HVC_RTAS=y -CONFIG_HVC_BEAT=y CONFIG_HVCS=m CONFIG_VIRTIO_CONSOLE=m CONFIG_IBM_BSR=m diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 2926fb9c570a..9c221b69c181 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -4,6 +4,14 @@ # Arch-specific CryptoAPI modules. # +obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o +obj-$(CONFIG_CRYPTO_MD5_PPC) += md5-ppc.o obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o +obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o +obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o +aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o +md5-ppc-y := md5-asm.o md5-glue.o sha1-powerpc-y := sha1-powerpc-asm.o sha1.o +sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o +sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o diff --git a/arch/powerpc/crypto/aes-spe-core.S b/arch/powerpc/crypto/aes-spe-core.S new file mode 100644 index 000000000000..5dc6bce90a77 --- /dev/null +++ b/arch/powerpc/crypto/aes-spe-core.S @@ -0,0 +1,351 @@ +/* + * Fast AES implementation for SPE instruction set (PPC) + * + * This code makes use of the SPE SIMD instruction set as defined in + * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf + * Implementation is based on optimization guide notes from + * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <asm/ppc_asm.h> +#include "aes-spe-regs.h" + +#define EAD(in, bpos) \ + rlwimi rT0,in,28-((bpos+3)%4)*8,20,27; + +#define DAD(in, bpos) \ + rlwimi rT1,in,24-((bpos+3)%4)*8,24,31; + +#define LWH(out, off) \ + evlwwsplat out,off(rT0); /* load word high */ + +#define LWL(out, off) \ + lwz out,off(rT0); /* load word low */ + +#define LBZ(out, tab, off) \ + lbz out,off(tab); /* load byte */ + +#define LAH(out, in, bpos, off) \ + EAD(in, bpos) /* calc addr + load word high */ \ + LWH(out, off) + +#define LAL(out, in, bpos, off) \ + EAD(in, bpos) /* calc addr + load word low */ \ + LWL(out, off) + +#define LAE(out, in, bpos) \ + EAD(in, bpos) /* calc addr + load enc byte */ \ + LBZ(out, rT0, 8) + +#define LBE(out) \ + LBZ(out, rT0, 8) /* load enc byte */ + +#define LAD(out, in, bpos) \ + DAD(in, bpos) /* calc addr + load dec byte */ \ + LBZ(out, rT1, 0) + +#define LBD(out) \ + LBZ(out, rT1, 0) + +/* + * ppc_encrypt_block: The central encryption function for a single 16 bytes + * block. It does no stack handling or register saving to support fast calls + * via bl/blr. It expects that caller has pre-xored input data with first + * 4 words of encryption key into rD0-rD3. Pointer/counter registers must + * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3 + * and rW0-rW3 and caller must execute a final xor on the ouput registers. + * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing. + * + */ +_GLOBAL(ppc_encrypt_block) + LAH(rW4, rD1, 2, 4) + LAH(rW6, rD0, 3, 0) + LAH(rW3, rD0, 1, 8) +ppc_encrypt_block_loop: + LAH(rW0, rD3, 0, 12) + LAL(rW0, rD0, 0, 12) + LAH(rW1, rD1, 0, 12) + LAH(rW2, rD2, 1, 8) + LAL(rW2, rD3, 1, 8) + LAL(rW3, rD1, 1, 8) + LAL(rW4, rD2, 2, 4) + LAL(rW6, rD1, 3, 0) + LAH(rW5, rD3, 2, 4) + LAL(rW5, rD0, 2, 4) + LAH(rW7, rD2, 3, 0) + evldw rD1,16(rKP) + EAD(rD3, 3) + evxor rW2,rW2,rW4 + LWL(rW7, 0) + evxor rW2,rW2,rW6 + EAD(rD2, 0) + evxor rD1,rD1,rW2 + LWL(rW1, 12) + evxor rD1,rD1,rW0 + evldw rD3,24(rKP) + evmergehi rD0,rD0,rD1 + EAD(rD1, 2) + evxor rW3,rW3,rW5 + LWH(rW4, 4) + evxor rW3,rW3,rW7 + EAD(rD0, 3) + evxor rD3,rD3,rW3 + LWH(rW6, 0) + evxor rD3,rD3,rW1 + EAD(rD0, 1) + evmergehi rD2,rD2,rD3 + LWH(rW3, 8) + LAH(rW0, rD3, 0, 12) + LAL(rW0, rD0, 0, 12) + LAH(rW1, rD1, 0, 12) + LAH(rW2, rD2, 1, 8) + LAL(rW2, rD3, 1, 8) + LAL(rW3, rD1, 1, 8) + LAL(rW4, rD2, 2, 4) + LAL(rW6, rD1, 3, 0) + LAH(rW5, rD3, 2, 4) + LAL(rW5, rD0, 2, 4) + LAH(rW7, rD2, 3, 0) + evldw rD1,32(rKP) + EAD(rD3, 3) + evxor rW2,rW2,rW4 + LWL(rW7, 0) + evxor rW2,rW2,rW6 + EAD(rD2, 0) + evxor rD1,rD1,rW2 + LWL(rW1, 12) + evxor rD1,rD1,rW0 + evldw rD3,40(rKP) + evmergehi rD0,rD0,rD1 + EAD(rD1, 2) + evxor rW3,rW3,rW5 + LWH(rW4, 4) + evxor rW3,rW3,rW7 + EAD(rD0, 3) + evxor rD3,rD3,rW3 + LWH(rW6, 0) + evxor rD3,rD3,rW1 + EAD(rD0, 1) + evmergehi rD2,rD2,rD3 + LWH(rW3, 8) + addi rKP,rKP,32 + bdnz ppc_encrypt_block_loop + LAH(rW0, rD3, 0, 12) + LAL(rW0, rD0, 0, 12) + LAH(rW1, rD1, 0, 12) + LAH(rW2, rD2, 1, 8) + LAL(rW2, rD3, 1, 8) + LAL(rW3, rD1, 1, 8) + LAL(rW4, rD2, 2, 4) + LAH(rW5, rD3, 2, 4) + LAL(rW6, rD1, 3, 0) + LAL(rW5, rD0, 2, 4) + LAH(rW7, rD2, 3, 0) + evldw rD1,16(rKP) + EAD(rD3, 3) + evxor rW2,rW2,rW4 + LWL(rW7, 0) + evxor rW2,rW2,rW6 + EAD(rD2, 0) + evxor rD1,rD1,rW2 + LWL(rW1, 12) + evxor rD1,rD1,rW0 + evldw rD3,24(rKP) + evmergehi rD0,rD0,rD1 + EAD(rD1, 0) + evxor rW3,rW3,rW5 + LBE(rW2) + evxor rW3,rW3,rW7 + EAD(rD0, 1) + evxor rD3,rD3,rW3 + LBE(rW6) + evxor rD3,rD3,rW1 + EAD(rD0, 0) + evmergehi rD2,rD2,rD3 + LBE(rW1) + LAE(rW0, rD3, 0) + LAE(rW1, rD0, 0) + LAE(rW4, rD2, 1) + LAE(rW5, rD3, 1) + LAE(rW3, rD2, 0) + LAE(rW7, rD1, 1) + rlwimi rW0,rW4,8,16,23 + rlwimi rW1,rW5,8,16,23 + LAE(rW4, rD1, 2) + LAE(rW5, rD2, 2) + rlwimi rW2,rW6,8,16,23 + rlwimi rW3,rW7,8,16,23 + LAE(rW6, rD3, 2) + LAE(rW7, rD0, 2) + rlwimi rW0,rW4,16,8,15 + rlwimi rW1,rW5,16,8,15 + LAE(rW4, rD0, 3) + LAE(rW5, rD1, 3) + rlwimi rW2,rW6,16,8,15 + lwz rD0,32(rKP) + rlwimi rW3,rW7,16,8,15 + lwz rD1,36(rKP) + LAE(rW6, rD2, 3) + LAE(rW7, rD3, 3) + rlwimi rW0,rW4,24,0,7 + lwz rD2,40(rKP) + rlwimi rW1,rW5,24,0,7 + lwz rD3,44(rKP) + rlwimi rW2,rW6,24,0,7 + rlwimi rW3,rW7,24,0,7 + blr + +/* + * ppc_decrypt_block: The central decryption function for a single 16 bytes + * block. It does no stack handling or register saving to support fast calls + * via bl/blr. It expects that caller has pre-xored input data with first + * 4 words of encryption key into rD0-rD3. Pointer/counter registers must + * have also been set up before (rT0, rKP, CTR). Output is stored in rD0-rD3 + * and rW0-rW3 and caller must execute a final xor on the ouput registers. + * All working registers rD0-rD3 & rW0-rW7 are overwritten during processing. + * + */ +_GLOBAL(ppc_decrypt_block) + LAH(rW0, rD1, 0, 12) + LAH(rW6, rD0, 3, 0) + LAH(rW3, rD0, 1, 8) +ppc_decrypt_block_loop: + LAH(rW1, rD3, 0, 12) + LAL(rW0, rD2, 0, 12) + LAH(rW2, rD2, 1, 8) + LAL(rW2, rD3, 1, 8) + LAH(rW4, rD3, 2, 4) + LAL(rW4, rD0, 2, 4) + LAL(rW6, rD1, 3, 0) + LAH(rW5, rD1, 2, 4) + LAH(rW7, rD2, 3, 0) + LAL(rW7, rD3, 3, 0) + LAL(rW3, rD1, 1, 8) + evldw rD1,16(rKP) + EAD(rD0, 0) + evxor rW4,rW4,rW6 + LWL(rW1, 12) + evxor rW0,rW0,rW4 + EAD(rD2, 2) + evxor rW0,rW0,rW2 + LWL(rW5, 4) + evxor rD1,rD1,rW0 + evldw rD3,24(rKP) + evmergehi rD0,rD0,rD1 + EAD(rD1, 0) + evxor rW3,rW3,rW7 + LWH(rW0, 12) + evxor rW3,rW3,rW1 + EAD(rD0, 3) + evxor rD3,rD3,rW3 + LWH(rW6, 0) + evxor rD3,rD3,rW5 + EAD(rD0, 1) + evmergehi rD2,rD2,rD3 + LWH(rW3, 8) + LAH(rW1, rD3, 0, 12) + LAL(rW0, rD2, 0, 12) + LAH(rW2, rD2, 1, 8) + LAL(rW2, rD3, 1, 8) + LAH(rW4, rD3, 2, 4) + LAL(rW4, rD0, 2, 4) + LAL(rW6, rD1, 3, 0) + LAH(rW5, rD1, 2, 4) + LAH(rW7, rD2, 3, 0) + LAL(rW7, rD3, 3, 0) + LAL(rW3, rD1, 1, 8) + evldw rD1,32(rKP) + EAD(rD0, 0) + evxor rW4,rW4,rW6 + LWL(rW1, 12) + evxor rW0,rW0,rW4 + EAD(rD2, 2) + evxor rW0,rW0,rW2 + LWL(rW5, 4) + evxor rD1,rD1,rW0 + evldw rD3,40(rKP) + evmergehi rD0,rD0,rD1 + EAD(rD1, 0) + evxor rW3,rW3,rW7 + LWH(rW0, 12) + evxor rW3,rW3,rW1 + EAD(rD0, 3) + evxor rD3,rD3,rW3 + LWH(rW6, 0) + evxor rD3,rD3,rW5 + EAD(rD0, 1) + evmergehi rD2,rD2,rD3 + LWH(rW3, 8) + addi rKP,rKP,32 + bdnz ppc_decrypt_block_loop + LAH(rW1, rD3, 0, 12) + LAL(rW0, rD2, 0, 12) + LAH(rW2, rD2, 1, 8) + LAL(rW2, rD3, 1, 8) + LAH(rW4, rD3, 2, 4) + LAL(rW4, rD0, 2, 4) + LAL(rW6, rD1, 3, 0) + LAH(rW5, rD1, 2, 4) + LAH(rW7, rD2, 3, 0) + LAL(rW7, rD3, 3, 0) + LAL(rW3, rD1, 1, 8) + evldw rD1,16(rKP) + EAD(rD0, 0) + evxor rW4,rW4,rW6 + LWL(rW1, 12) + evxor rW0,rW0,rW4 + EAD(rD2, 2) + evxor rW0,rW0,rW2 + LWL(rW5, 4) + evxor rD1,rD1,rW0 + evldw rD3,24(rKP) + evmergehi rD0,rD0,rD1 + DAD(rD1, 0) + evxor rW3,rW3,rW7 + LBD(rW0) + evxor rW3,rW3,rW1 + DAD(rD0, 1) + evxor rD3,rD3,rW3 + LBD(rW6) + evxor rD3,rD3,rW5 + DAD(rD0, 0) + evmergehi rD2,rD2,rD3 + LBD(rW3) + LAD(rW2, rD3, 0) + LAD(rW1, rD2, 0) + LAD(rW4, rD2, 1) + LAD(rW5, rD3, 1) + LAD(rW7, rD1, 1) + rlwimi rW0,rW4,8,16,23 + rlwimi rW1,rW5,8,16,23 + LAD(rW4, rD3, 2) + LAD(rW5, rD0, 2) + rlwimi rW2,rW6,8,16,23 + rlwimi rW3,rW7,8,16,23 + LAD(rW6, rD1, 2) + LAD(rW7, rD2, 2) + rlwimi rW0,rW4,16,8,15 + rlwimi rW1,rW5,16,8,15 + LAD(rW4, rD0, 3) + LAD(rW5, rD1, 3) + rlwimi rW2,rW6,16,8,15 + lwz rD0,32(rKP) + rlwimi rW3,rW7,16,8,15 + lwz rD1,36(rKP) + LAD(rW6, rD2, 3) + LAD(rW7, rD3, 3) + rlwimi rW0,rW4,24,0,7 + lwz rD2,40(rKP) + rlwimi rW1,rW5,24,0,7 + lwz rD3,44(rKP) + rlwimi rW2,rW6,24,0,7 + rlwimi rW3,rW7,24,0,7 + blr diff --git a/arch/powerpc/crypto/aes-spe-glue.c b/arch/powerpc/crypto/aes-spe-glue.c new file mode 100644 index 000000000000..bd5e63f72ad4 --- /dev/null +++ b/arch/powerpc/crypto/aes-spe-glue.c @@ -0,0 +1,512 @@ +/* + * Glue code for AES implementation for SPE instructions (PPC) + * + * Based on generic implementation. The assembler module takes care + * about the SPE registers so it can run from interrupt context. + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <crypto/aes.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/crypto.h> +#include <asm/byteorder.h> +#include <asm/switch_to.h> +#include <crypto/algapi.h> + +/* + * MAX_BYTES defines the number of bytes that are allowed to be processed + * between preempt_disable() and preempt_enable(). e500 cores can issue two + * instructions per clock cycle using one 32/64 bit unit (SU1) and one 32 + * bit unit (SU2). One of these can be a memory access that is executed via + * a single load and store unit (LSU). XTS-AES-256 takes ~780 operations per + * 16 byte block block or 25 cycles per byte. Thus 768 bytes of input data + * will need an estimated maximum of 20,000 cycles. Headroom for cache misses + * included. Even with the low end model clocked at 667 MHz this equals to a + * critical time window of less than 30us. The value has been choosen to + * process a 512 byte disk block in one or a large 1400 bytes IPsec network + * packet in two runs. + * + */ +#define MAX_BYTES 768 + +struct ppc_aes_ctx { + u32 key_enc[AES_MAX_KEYLENGTH_U32]; + u32 key_dec[AES_MAX_KEYLENGTH_U32]; + u32 rounds; +}; + +struct ppc_xts_ctx { + u32 key_enc[AES_MAX_KEYLENGTH_U32]; + u32 key_dec[AES_MAX_KEYLENGTH_U32]; + u32 key_twk[AES_MAX_KEYLENGTH_U32]; + u32 rounds; +}; + +extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds); +extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds); +extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, + u32 bytes); +extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, + u32 bytes); +extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, + u32 bytes, u8 *iv); +extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, + u32 bytes, u8 *iv); +extern void ppc_crypt_ctr (u8 *out, const u8 *in, u32 *key_enc, u32 rounds, + u32 bytes, u8 *iv); +extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, + u32 bytes, u8 *iv, u32 *key_twk); +extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, + u32 bytes, u8 *iv, u32 *key_twk); + +extern void ppc_expand_key_128(u32 *key_enc, const u8 *key); +extern void ppc_expand_key_192(u32 *key_enc, const u8 *key); +extern void ppc_expand_key_256(u32 *key_enc, const u8 *key); + +extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc, + unsigned int key_len); + +static void spe_begin(void) +{ + /* disable preemption and save users SPE registers if required */ + preempt_disable(); + enable_kernel_spe(); +} + +static void spe_end(void) +{ + /* reenable preemption */ + preempt_enable(); +} + +static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + if (key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->rounds = 4; + ppc_expand_key_128(ctx->key_enc, in_key); + break; + case AES_KEYSIZE_192: + ctx->rounds = 5; + ppc_expand_key_192(ctx->key_enc, in_key); + break; + case AES_KEYSIZE_256: + ctx->rounds = 6; + ppc_expand_key_256(ctx->key_enc, in_key); + break; + } + + ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len); + + return 0; +} + +static int ppc_xts_setkey(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct ppc_xts_ctx *ctx = crypto_tfm_ctx(tfm); + + key_len >>= 1; + + if (key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256) { + tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->rounds = 4; + ppc_expand_key_128(ctx->key_enc, in_key); + ppc_expand_key_128(ctx->key_twk, in_key + AES_KEYSIZE_128); + break; + case AES_KEYSIZE_192: + ctx->rounds = 5; + ppc_expand_key_192(ctx->key_enc, in_key); + ppc_expand_key_192(ctx->key_twk, in_key + AES_KEYSIZE_192); + break; + case AES_KEYSIZE_256: + ctx->rounds = 6; + ppc_expand_key_256(ctx->key_enc, in_key); + ppc_expand_key_256(ctx->key_twk, in_key + AES_KEYSIZE_256); + break; + } + + ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len); + + return 0; +} + +static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + spe_begin(); + ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds); + spe_end(); +} + +static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) +{ + struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + spe_begin(); + ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds); + spe_end(); +} + +static int ppc_ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int ubytes; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + ubytes = nbytes > MAX_BYTES ? + nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1); + nbytes -= ubytes; + + spe_begin(); + ppc_encrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr, + ctx->key_enc, ctx->rounds, nbytes); + spe_end(); + + err = blkcipher_walk_done(desc, &walk, ubytes); + } + + return err; +} + +static int ppc_ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int ubytes; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + ubytes = nbytes > MAX_BYTES ? + nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1); + nbytes -= ubytes; + + spe_begin(); + ppc_decrypt_ecb(walk.dst.virt.addr, walk.src.virt.addr, + ctx->key_dec, ctx->rounds, nbytes); + spe_end(); + + err = blkcipher_walk_done(desc, &walk, ubytes); + } + + return err; +} + +static int ppc_cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int ubytes; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + ubytes = nbytes > MAX_BYTES ? + nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1); + nbytes -= ubytes; + + spe_begin(); + ppc_encrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr, + ctx->key_enc, ctx->rounds, nbytes, walk.iv); + spe_end(); + + err = blkcipher_walk_done(desc, &walk, ubytes); + } + + return err; +} + +static int ppc_cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int ubytes; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + ubytes = nbytes > MAX_BYTES ? + nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1); + nbytes -= ubytes; + + spe_begin(); + ppc_decrypt_cbc(walk.dst.virt.addr, walk.src.virt.addr, + ctx->key_dec, ctx->rounds, nbytes, walk.iv); + spe_end(); + + err = blkcipher_walk_done(desc, &walk, ubytes); + } + + return err; +} + +static int ppc_ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct ppc_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int pbytes, ubytes; + int err; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); + + while ((pbytes = walk.nbytes)) { + pbytes = pbytes > MAX_BYTES ? MAX_BYTES : pbytes; + pbytes = pbytes == nbytes ? + nbytes : pbytes & ~(AES_BLOCK_SIZE - 1); + ubytes = walk.nbytes - pbytes; + + spe_begin(); + ppc_crypt_ctr(walk.dst.virt.addr, walk.src.virt.addr, + ctx->key_enc, ctx->rounds, pbytes , walk.iv); + spe_end(); + + nbytes -= pbytes; + err = blkcipher_walk_done(desc, &walk, ubytes); + } + + return err; +} + +static int ppc_xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct ppc_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int ubytes; + int err; + u32 *twk; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + twk = ctx->key_twk; + + while ((nbytes = walk.nbytes)) { + ubytes = nbytes > MAX_BYTES ? + nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1); + nbytes -= ubytes; + + spe_begin(); + ppc_encrypt_xts(walk.dst.virt.addr, walk.src.virt.addr, + ctx->key_enc, ctx->rounds, nbytes, walk.iv, twk); + spe_end(); + + twk = NULL; + err = blkcipher_walk_done(desc, &walk, ubytes); + } + + return err; +} + +static int ppc_xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct ppc_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + unsigned int ubytes; + int err; + u32 *twk; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + twk = ctx->key_twk; + + while ((nbytes = walk.nbytes)) { + ubytes = nbytes > MAX_BYTES ? + nbytes - MAX_BYTES : nbytes & (AES_BLOCK_SIZE - 1); + nbytes -= ubytes; + + spe_begin(); + ppc_decrypt_xts(walk.dst.virt.addr, walk.src.virt.addr, + ctx->key_dec, ctx->rounds, nbytes, walk.iv, twk); + spe_end(); + + twk = NULL; + err = blkcipher_walk_done(desc, &walk, ubytes); + } + + return err; +} + +/* + * Algorithm definitions. Disabling alignment (cra_alignmask=0) was chosen + * because the e500 platform can handle unaligned reads/writes very efficently. + * This improves IPsec thoughput by another few percent. Additionally we assume + * that AES context is always aligned to at least 8 bytes because it is created + * with kmalloc() in the crypto infrastructure + * + */ +static struct crypto_alg aes_algs[] = { { + .cra_name = "aes", + .cra_driver_name = "aes-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ppc_aes_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = AES_MIN_KEY_SIZE, + .cia_max_keysize = AES_MAX_KEY_SIZE, + .cia_setkey = ppc_aes_setkey, + .cia_encrypt = ppc_aes_encrypt, + .cia_decrypt = ppc_aes_decrypt + } + } +}, { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ppc_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ppc_aes_setkey, + .encrypt = ppc_ecb_encrypt, + .decrypt = ppc_ecb_decrypt, + } + } +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ppc_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ppc_aes_setkey, + .encrypt = ppc_cbc_encrypt, + .decrypt = ppc_cbc_decrypt, + } + } +}, { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct ppc_aes_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = ppc_aes_setkey, + .encrypt = ppc_ctr_crypt, + .decrypt = ppc_ctr_crypt, + } + } +}, { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ppc_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE * 2, + .max_keysize = AES_MAX_KEY_SIZE * 2, + .ivsize = AES_BLOCK_SIZE, + .setkey = ppc_xts_setkey, + .encrypt = ppc_xts_encrypt, + .decrypt = ppc_xts_decrypt, + } + } +} }; + +static int __init ppc_aes_mod_init(void) +{ + return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs)); +} + +static void __exit ppc_aes_mod_fini(void) +{ + crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs)); +} + +module_init(ppc_aes_mod_init); +module_exit(ppc_aes_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS, SPE optimized"); + +MODULE_ALIAS_CRYPTO("aes"); +MODULE_ALIAS_CRYPTO("ecb(aes)"); +MODULE_ALIAS_CRYPTO("cbc(aes)"); +MODULE_ALIAS_CRYPTO("ctr(aes)"); +MODULE_ALIAS_CRYPTO("xts(aes)"); +MODULE_ALIAS_CRYPTO("aes-ppc-spe"); diff --git a/arch/powerpc/crypto/aes-spe-keys.S b/arch/powerpc/crypto/aes-spe-keys.S new file mode 100644 index 000000000000..be8090f3d700 --- /dev/null +++ b/arch/powerpc/crypto/aes-spe-keys.S @@ -0,0 +1,283 @@ +/* + * Key handling functions for PPC AES implementation + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <asm/ppc_asm.h> + +#ifdef __BIG_ENDIAN__ +#define LOAD_KEY(d, s, off) \ + lwz d,off(s); +#else +#define LOAD_KEY(d, s, off) \ + li r0,off; \ + lwbrx d,s,r0; +#endif + +#define INITIALIZE_KEY \ + stwu r1,-32(r1); /* create stack frame */ \ + stw r14,8(r1); /* save registers */ \ + stw r15,12(r1); \ + stw r16,16(r1); + +#define FINALIZE_KEY \ + lwz r14,8(r1); /* restore registers */ \ + lwz r15,12(r1); \ + lwz r16,16(r1); \ + xor r5,r5,r5; /* clear sensitive data */ \ + xor r6,r6,r6; \ + xor r7,r7,r7; \ + xor r8,r8,r8; \ + xor r9,r9,r9; \ + xor r10,r10,r10; \ + xor r11,r11,r11; \ + xor r12,r12,r12; \ + addi r1,r1,32; /* cleanup stack */ + +#define LS_BOX(r, t1, t2) \ + lis t2,PPC_AES_4K_ENCTAB@h; \ + ori t2,t2,PPC_AES_4K_ENCTAB@l; \ + rlwimi t2,r,4,20,27; \ + lbz t1,8(t2); \ + rlwimi r,t1,0,24,31; \ + rlwimi t2,r,28,20,27; \ + lbz t1,8(t2); \ + rlwimi r,t1,8,16,23; \ + rlwimi t2,r,20,20,27; \ + lbz t1,8(t2); \ + rlwimi r,t1,16,8,15; \ + rlwimi t2,r,12,20,27; \ + lbz t1,8(t2); \ + rlwimi r,t1,24,0,7; + +#define GF8_MUL(out, in, t1, t2) \ + lis t1,0x8080; /* multiplication in GF8 */ \ + ori t1,t1,0x8080; \ + and t1,t1,in; \ + srwi t1,t1,7; \ + mulli t1,t1,0x1b; \ + lis t2,0x7f7f; \ + ori t2,t2,0x7f7f; \ + and t2,t2,in; \ + slwi t2,t2,1; \ + xor out,t1,t2; + +/* + * ppc_expand_key_128(u32 *key_enc, const u8 *key) + * + * Expand 128 bit key into 176 bytes encryption key. It consists of + * key itself plus 10 rounds with 16 bytes each + * + */ +_GLOBAL(ppc_expand_key_128) + INITIALIZE_KEY + LOAD_KEY(r5,r4,0) + LOAD_KEY(r6,r4,4) + LOAD_KEY(r7,r4,8) + LOAD_KEY(r8,r4,12) + stw r5,0(r3) /* key[0..3] = input data */ + stw r6,4(r3) + stw r7,8(r3) + stw r8,12(r3) + li r16,10 /* 10 expansion rounds */ + lis r0,0x0100 /* RCO(1) */ +ppc_expand_128_loop: + addi r3,r3,16 + mr r14,r8 /* apply LS_BOX to 4th temp */ + rotlwi r14,r14,8 + LS_BOX(r14, r15, r4) + xor r14,r14,r0 + xor r5,r5,r14 /* xor next 4 keys */ + xor r6,r6,r5 + xor r7,r7,r6 + xor r8,r8,r7 + stw r5,0(r3) /* store next 4 keys */ + stw r6,4(r3) + stw r7,8(r3) + stw r8,12(r3) + GF8_MUL(r0, r0, r4, r14) /* multiply RCO by 2 in GF */ + subi r16,r16,1 + cmpwi r16,0 + bt eq,ppc_expand_128_end + b ppc_expand_128_loop +ppc_expand_128_end: + FINALIZE_KEY + blr + +/* + * ppc_expand_key_192(u32 *key_enc, const u8 *key) + * + * Expand 192 bit key into 208 bytes encryption key. It consists of key + * itself plus 12 rounds with 16 bytes each + * + */ +_GLOBAL(ppc_expand_key_192) + INITIALIZE_KEY + LOAD_KEY(r5,r4,0) + LOAD_KEY(r6,r4,4) + LOAD_KEY(r7,r4,8) + LOAD_KEY(r8,r4,12) + LOAD_KEY(r9,r4,16) + LOAD_KEY(r10,r4,20) + stw r5,0(r3) + stw r6,4(r3) + stw r7,8(r3) + stw r8,12(r3) + stw r9,16(r3) + stw r10,20(r3) + li r16,8 /* 8 expansion rounds */ + lis r0,0x0100 /* RCO(1) */ +ppc_expand_192_loop: + addi r3,r3,24 + mr r14,r10 /* apply LS_BOX to 6th temp */ + rotlwi r14,r14,8 + LS_BOX(r14, r15, r4) + xor r14,r14,r0 + xor r5,r5,r14 /* xor next 6 keys */ + xor r6,r6,r5 + xor r7,r7,r6 + xor r8,r8,r7 + xor r9,r9,r8 + xor r10,r10,r9 + stw r5,0(r3) + stw r6,4(r3) + stw r7,8(r3) + stw r8,12(r3) + subi r16,r16,1 + cmpwi r16,0 /* last round early kick out */ + bt eq,ppc_expand_192_end + stw r9,16(r3) + stw r10,20(r3) + GF8_MUL(r0, r0, r4, r14) /* multiply RCO GF8 */ + b ppc_expand_192_loop +ppc_expand_192_end: + FINALIZE_KEY + blr + +/* + * ppc_expand_key_256(u32 *key_enc, const u8 *key) + * + * Expand 256 bit key into 240 bytes encryption key. It consists of key + * itself plus 14 rounds with 16 bytes each + * + */ +_GLOBAL(ppc_expand_key_256) + INITIALIZE_KEY + LOAD_KEY(r5,r4,0) + LOAD_KEY(r6,r4,4) + LOAD_KEY(r7,r4,8) + LOAD_KEY(r8,r4,12) + LOAD_KEY(r9,r4,16) + LOAD_KEY(r10,r4,20) + LOAD_KEY(r11,r4,24) + LOAD_KEY(r12,r4,28) + stw r5,0(r3) + stw r6,4(r3) + stw r7,8(r3) + stw r8,12(r3) + stw r9,16(r3) + stw r10,20(r3) + stw r11,24(r3) + stw r12,28(r3) + li r16,7 /* 7 expansion rounds */ + lis r0,0x0100 /* RCO(1) */ +ppc_expand_256_loop: + addi r3,r3,32 + mr r14,r12 /* apply LS_BOX to 8th temp */ + rotlwi r14,r14,8 + LS_BOX(r14, r15, r4) + xor r14,r14,r0 + xor r5,r5,r14 /* xor 4 keys */ + xor r6,r6,r5 + xor r7,r7,r6 + xor r8,r8,r7 + mr r14,r8 + LS_BOX(r14, r15, r4) /* apply LS_BOX to 4th temp */ + xor r9,r9,r14 /* xor 4 keys */ + xor r10,r10,r9 + xor r11,r11,r10 + xor r12,r12,r11 + stw r5,0(r3) + stw r6,4(r3) + stw r7,8(r3) + stw r8,12(r3) + subi r16,r16,1 + cmpwi r16,0 /* last round early kick out */ + bt eq,ppc_expand_256_end + stw r9,16(r3) + stw r10,20(r3) + stw r11,24(r3) + stw r12,28(r3) + GF8_MUL(r0, r0, r4, r14) + b ppc_expand_256_loop +ppc_expand_256_end: + FINALIZE_KEY + blr + +/* + * ppc_generate_decrypt_key: derive decryption key from encryption key + * number of bytes to handle are calculated from length of key (16/24/32) + * + */ +_GLOBAL(ppc_generate_decrypt_key) + addi r6,r5,24 + slwi r6,r6,2 + lwzx r7,r4,r6 /* first/last 4 words are same */ + stw r7,0(r3) + lwz r7,0(r4) + stwx r7,r3,r6 + addi r6,r6,4 + lwzx r7,r4,r6 + stw r7,4(r3) + lwz r7,4(r4) + stwx r7,r3,r6 + addi r6,r6,4 + lwzx r7,r4,r6 + stw r7,8(r3) + lwz r7,8(r4) + stwx r7,r3,r6 + addi r6,r6,4 + lwzx r7,r4,r6 + stw r7,12(r3) + lwz r7,12(r4) + stwx r7,r3,r6 + addi r3,r3,16 + add r4,r4,r6 + subi r4,r4,28 + addi r5,r5,20 + srwi r5,r5,2 +ppc_generate_decrypt_block: + li r6,4 + mtctr r6 +ppc_generate_decrypt_word: + lwz r6,0(r4) + GF8_MUL(r7, r6, r0, r7) + GF8_MUL(r8, r7, r0, r8) + GF8_MUL(r9, r8, r0, r9) + xor r10,r9,r6 + xor r11,r7,r8 + xor r11,r11,r9 + xor r12,r7,r10 + rotrwi r12,r12,24 + xor r11,r11,r12 + xor r12,r8,r10 + rotrwi r12,r12,16 + xor r11,r11,r12 + rotrwi r12,r10,8 + xor r11,r11,r12 + stw r11,0(r3) + addi r3,r3,4 + addi r4,r4,4 + bdnz ppc_generate_decrypt_word + subi r4,r4,32 + subi r5,r5,1 + cmpwi r5,0 + bt gt,ppc_generate_decrypt_block + blr diff --git a/arch/powerpc/crypto/aes-spe-modes.S b/arch/powerpc/crypto/aes-spe-modes.S new file mode 100644 index 000000000000..ad48032ca8e0 --- /dev/null +++ b/arch/powerpc/crypto/aes-spe-modes.S @@ -0,0 +1,630 @@ +/* + * AES modes (ECB/CBC/CTR/XTS) for PPC AES implementation + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <asm/ppc_asm.h> +#include "aes-spe-regs.h" + +#ifdef __BIG_ENDIAN__ /* Macros for big endian builds */ + +#define LOAD_DATA(reg, off) \ + lwz reg,off(rSP); /* load with offset */ +#define SAVE_DATA(reg, off) \ + stw reg,off(rDP); /* save with offset */ +#define NEXT_BLOCK \ + addi rSP,rSP,16; /* increment pointers per bloc */ \ + addi rDP,rDP,16; +#define LOAD_IV(reg, off) \ + lwz reg,off(rIP); /* IV loading with offset */ +#define SAVE_IV(reg, off) \ + stw reg,off(rIP); /* IV saving with offset */ +#define START_IV /* nothing to reset */ +#define CBC_DEC 16 /* CBC decrement per block */ +#define CTR_DEC 1 /* CTR decrement one byte */ + +#else /* Macros for little endian */ + +#define LOAD_DATA(reg, off) \ + lwbrx reg,0,rSP; /* load reversed */ \ + addi rSP,rSP,4; /* and increment pointer */ +#define SAVE_DATA(reg, off) \ + stwbrx reg,0,rDP; /* save reversed */ \ + addi rDP,rDP,4; /* and increment pointer */ +#define NEXT_BLOCK /* nothing todo */ +#define LOAD_IV(reg, off) \ + lwbrx reg,0,rIP; /* load reversed */ \ + addi rIP,rIP,4; /* and increment pointer */ +#define SAVE_IV(reg, off) \ + stwbrx reg,0,rIP; /* load reversed */ \ + addi rIP,rIP,4; /* and increment pointer */ +#define START_IV \ + subi rIP,rIP,16; /* must reset pointer */ +#define CBC_DEC 32 /* 2 blocks because of incs */ +#define CTR_DEC 17 /* 1 block because of incs */ + +#endif + +#define SAVE_0_REGS +#define LOAD_0_REGS + +#define SAVE_4_REGS \ + stw rI0,96(r1); /* save 32 bit registers */ \ + stw rI1,100(r1); \ + stw rI2,104(r1); \ + stw rI3,108(r1); + +#define LOAD_4_REGS \ + lwz rI0,96(r1); /* restore 32 bit registers */ \ + lwz rI1,100(r1); \ + lwz rI2,104(r1); \ + lwz rI3,108(r1); + +#define SAVE_8_REGS \ + SAVE_4_REGS \ + stw rG0,112(r1); /* save 32 bit registers */ \ + stw rG1,116(r1); \ + stw rG2,120(r1); \ + stw rG3,124(r1); + +#define LOAD_8_REGS \ + LOAD_4_REGS \ + lwz rG0,112(r1); /* restore 32 bit registers */ \ + lwz rG1,116(r1); \ + lwz rG2,120(r1); \ + lwz rG3,124(r1); + +#define INITIALIZE_CRYPT(tab,nr32bitregs) \ + mflr r0; \ + stwu r1,-160(r1); /* create stack frame */ \ + lis rT0,tab@h; /* en-/decryption table pointer */ \ + stw r0,8(r1); /* save link register */ \ + ori rT0,rT0,tab@l; \ + evstdw r14,16(r1); \ + mr rKS,rKP; \ + evstdw r15,24(r1); /* We must save non volatile */ \ + evstdw r16,32(r1); /* registers. Take the chance */ \ + evstdw r17,40(r1); /* and save the SPE part too */ \ + evstdw r18,48(r1); \ + evstdw r19,56(r1); \ + evstdw r20,64(r1); \ + evstdw r21,72(r1); \ + evstdw r22,80(r1); \ + evstdw r23,88(r1); \ + SAVE_##nr32bitregs##_REGS + +#define FINALIZE_CRYPT(nr32bitregs) \ + lwz r0,8(r1); \ + evldw r14,16(r1); /* restore SPE registers */ \ + evldw r15,24(r1); \ + evldw r16,32(r1); \ + evldw r17,40(r1); \ + evldw r18,48(r1); \ + evldw r19,56(r1); \ + evldw r20,64(r1); \ + evldw r21,72(r1); \ + evldw r22,80(r1); \ + evldw r23,88(r1); \ + LOAD_##nr32bitregs##_REGS \ + mtlr r0; /* restore link register */ \ + xor r0,r0,r0; \ + stw r0,16(r1); /* delete sensitive data */ \ + stw r0,24(r1); /* that we might have pushed */ \ + stw r0,32(r1); /* from other context that runs */ \ + stw r0,40(r1); /* the same code */ \ + stw r0,48(r1); \ + stw r0,56(r1); \ + stw r0,64(r1); \ + stw r0,72(r1); \ + stw r0,80(r1); \ + stw r0,88(r1); \ + addi r1,r1,160; /* cleanup stack frame */ + +#define ENDIAN_SWAP(t0, t1, s0, s1) \ + rotrwi t0,s0,8; /* swap endianness for 2 GPRs */ \ + rotrwi t1,s1,8; \ + rlwimi t0,s0,8,8,15; \ + rlwimi t1,s1,8,8,15; \ + rlwimi t0,s0,8,24,31; \ + rlwimi t1,s1,8,24,31; + +#define GF128_MUL(d0, d1, d2, d3, t0) \ + li t0,0x87; /* multiplication in GF128 */ \ + cmpwi d3,-1; \ + iselgt t0,0,t0; \ + rlwimi d3,d2,0,0,0; /* propagate "carry" bits */ \ + rotlwi d3,d3,1; \ + rlwimi d2,d1,0,0,0; \ + rotlwi d2,d2,1; \ + rlwimi d1,d0,0,0,0; \ + slwi d0,d0,1; /* shift left 128 bit */ \ + rotlwi d1,d1,1; \ + xor d0,d0,t0; + +#define START_KEY(d0, d1, d2, d3) \ + lwz rW0,0(rKP); \ + mtctr rRR; \ + lwz rW1,4(rKP); \ + lwz rW2,8(rKP); \ + lwz rW3,12(rKP); \ + xor rD0,d0,rW0; \ + xor rD1,d1,rW1; \ + xor rD2,d2,rW2; \ + xor rD3,d3,rW3; + +/* + * ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, + * u32 rounds) + * + * called from glue layer to encrypt a single 16 byte block + * round values are AES128 = 4, AES192 = 5, AES256 = 6 + * + */ +_GLOBAL(ppc_encrypt_aes) + INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0) + LOAD_DATA(rD0, 0) + LOAD_DATA(rD1, 4) + LOAD_DATA(rD2, 8) + LOAD_DATA(rD3, 12) + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_encrypt_block + xor rD0,rD0,rW0 + SAVE_DATA(rD0, 0) + xor rD1,rD1,rW1 + SAVE_DATA(rD1, 4) + xor rD2,rD2,rW2 + SAVE_DATA(rD2, 8) + xor rD3,rD3,rW3 + SAVE_DATA(rD3, 12) + FINALIZE_CRYPT(0) + blr + +/* + * ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, + * u32 rounds) + * + * called from glue layer to decrypt a single 16 byte block + * round values are AES128 = 4, AES192 = 5, AES256 = 6 + * + */ +_GLOBAL(ppc_decrypt_aes) + INITIALIZE_CRYPT(PPC_AES_4K_DECTAB,0) + LOAD_DATA(rD0, 0) + addi rT1,rT0,4096 + LOAD_DATA(rD1, 4) + LOAD_DATA(rD2, 8) + LOAD_DATA(rD3, 12) + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_decrypt_block + xor rD0,rD0,rW0 + SAVE_DATA(rD0, 0) + xor rD1,rD1,rW1 + SAVE_DATA(rD1, 4) + xor rD2,rD2,rW2 + SAVE_DATA(rD2, 8) + xor rD3,rD3,rW3 + SAVE_DATA(rD3, 12) + FINALIZE_CRYPT(0) + blr + +/* + * ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, + * u32 rounds, u32 bytes); + * + * called from glue layer to encrypt multiple blocks via ECB + * Bytes must be larger or equal 16 and only whole blocks are + * processed. round values are AES128 = 4, AES192 = 5 and + * AES256 = 6 + * + */ +_GLOBAL(ppc_encrypt_ecb) + INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 0) +ppc_encrypt_ecb_loop: + LOAD_DATA(rD0, 0) + mr rKP,rKS + LOAD_DATA(rD1, 4) + subi rLN,rLN,16 + LOAD_DATA(rD2, 8) + cmpwi rLN,15 + LOAD_DATA(rD3, 12) + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_encrypt_block + xor rD0,rD0,rW0 + SAVE_DATA(rD0, 0) + xor rD1,rD1,rW1 + SAVE_DATA(rD1, 4) + xor rD2,rD2,rW2 + SAVE_DATA(rD2, 8) + xor rD3,rD3,rW3 + SAVE_DATA(rD3, 12) + NEXT_BLOCK + bt gt,ppc_encrypt_ecb_loop + FINALIZE_CRYPT(0) + blr + +/* + * ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, + * u32 rounds, u32 bytes); + * + * called from glue layer to decrypt multiple blocks via ECB + * Bytes must be larger or equal 16 and only whole blocks are + * processed. round values are AES128 = 4, AES192 = 5 and + * AES256 = 6 + * + */ +_GLOBAL(ppc_decrypt_ecb) + INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 0) + addi rT1,rT0,4096 +ppc_decrypt_ecb_loop: + LOAD_DATA(rD0, 0) + mr rKP,rKS + LOAD_DATA(rD1, 4) + subi rLN,rLN,16 + LOAD_DATA(rD2, 8) + cmpwi rLN,15 + LOAD_DATA(rD3, 12) + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_decrypt_block + xor rD0,rD0,rW0 + SAVE_DATA(rD0, 0) + xor rD1,rD1,rW1 + SAVE_DATA(rD1, 4) + xor rD2,rD2,rW2 + SAVE_DATA(rD2, 8) + xor rD3,rD3,rW3 + SAVE_DATA(rD3, 12) + NEXT_BLOCK + bt gt,ppc_decrypt_ecb_loop + FINALIZE_CRYPT(0) + blr + +/* + * ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, + * 32 rounds, u32 bytes, u8 *iv); + * + * called from glue layer to encrypt multiple blocks via CBC + * Bytes must be larger or equal 16 and only whole blocks are + * processed. round values are AES128 = 4, AES192 = 5 and + * AES256 = 6 + * + */ +_GLOBAL(ppc_encrypt_cbc) + INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4) + LOAD_IV(rI0, 0) + LOAD_IV(rI1, 4) + LOAD_IV(rI2, 8) + LOAD_IV(rI3, 12) +ppc_encrypt_cbc_loop: + LOAD_DATA(rD0, 0) + mr rKP,rKS + LOAD_DATA(rD1, 4) + subi rLN,rLN,16 + LOAD_DATA(rD2, 8) + cmpwi rLN,15 + LOAD_DATA(rD3, 12) + xor rD0,rD0,rI0 + xor rD1,rD1,rI1 + xor rD2,rD2,rI2 + xor rD3,rD3,rI3 + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_encrypt_block + xor rI0,rD0,rW0 + SAVE_DATA(rI0, 0) + xor rI1,rD1,rW1 + SAVE_DATA(rI1, 4) + xor rI2,rD2,rW2 + SAVE_DATA(rI2, 8) + xor rI3,rD3,rW3 + SAVE_DATA(rI3, 12) + NEXT_BLOCK + bt gt,ppc_encrypt_cbc_loop + START_IV + SAVE_IV(rI0, 0) + SAVE_IV(rI1, 4) + SAVE_IV(rI2, 8) + SAVE_IV(rI3, 12) + FINALIZE_CRYPT(4) + blr + +/* + * ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, + * u32 rounds, u32 bytes, u8 *iv); + * + * called from glue layer to decrypt multiple blocks via CBC + * round values are AES128 = 4, AES192 = 5, AES256 = 6 + * + */ +_GLOBAL(ppc_decrypt_cbc) + INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 4) + li rT1,15 + LOAD_IV(rI0, 0) + andc rLN,rLN,rT1 + LOAD_IV(rI1, 4) + subi rLN,rLN,16 + LOAD_IV(rI2, 8) + add rSP,rSP,rLN /* reverse processing */ + LOAD_IV(rI3, 12) + add rDP,rDP,rLN + LOAD_DATA(rD0, 0) + addi rT1,rT0,4096 + LOAD_DATA(rD1, 4) + LOAD_DATA(rD2, 8) + LOAD_DATA(rD3, 12) + START_IV + SAVE_IV(rD0, 0) + SAVE_IV(rD1, 4) + SAVE_IV(rD2, 8) + cmpwi rLN,16 + SAVE_IV(rD3, 12) + bt lt,ppc_decrypt_cbc_end +ppc_decrypt_cbc_loop: + mr rKP,rKS + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_decrypt_block + subi rLN,rLN,16 + subi rSP,rSP,CBC_DEC + xor rW0,rD0,rW0 + LOAD_DATA(rD0, 0) + xor rW1,rD1,rW1 + LOAD_DATA(rD1, 4) + xor rW2,rD2,rW2 + LOAD_DATA(rD2, 8) + xor rW3,rD3,rW3 + LOAD_DATA(rD3, 12) + xor rW0,rW0,rD0 + SAVE_DATA(rW0, 0) + xor rW1,rW1,rD1 + SAVE_DATA(rW1, 4) + xor rW2,rW2,rD2 + SAVE_DATA(rW2, 8) + xor rW3,rW3,rD3 + SAVE_DATA(rW3, 12) + cmpwi rLN,15 + subi rDP,rDP,CBC_DEC + bt gt,ppc_decrypt_cbc_loop +ppc_decrypt_cbc_end: + mr rKP,rKS + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_decrypt_block + xor rW0,rW0,rD0 + xor rW1,rW1,rD1 + xor rW2,rW2,rD2 + xor rW3,rW3,rD3 + xor rW0,rW0,rI0 /* decrypt with initial IV */ + SAVE_DATA(rW0, 0) + xor rW1,rW1,rI1 + SAVE_DATA(rW1, 4) + xor rW2,rW2,rI2 + SAVE_DATA(rW2, 8) + xor rW3,rW3,rI3 + SAVE_DATA(rW3, 12) + FINALIZE_CRYPT(4) + blr + +/* + * ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc, + * u32 rounds, u32 bytes, u8 *iv); + * + * called from glue layer to encrypt/decrypt multiple blocks + * via CTR. Number of bytes does not need to be a multiple of + * 16. Round values are AES128 = 4, AES192 = 5, AES256 = 6 + * + */ +_GLOBAL(ppc_crypt_ctr) + INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 4) + LOAD_IV(rI0, 0) + LOAD_IV(rI1, 4) + LOAD_IV(rI2, 8) + cmpwi rLN,16 + LOAD_IV(rI3, 12) + START_IV + bt lt,ppc_crypt_ctr_partial +ppc_crypt_ctr_loop: + mr rKP,rKS + START_KEY(rI0, rI1, rI2, rI3) + bl ppc_encrypt_block + xor rW0,rD0,rW0 + xor rW1,rD1,rW1 + xor rW2,rD2,rW2 + xor rW3,rD3,rW3 + LOAD_DATA(rD0, 0) + subi rLN,rLN,16 + LOAD_DATA(rD1, 4) + LOAD_DATA(rD2, 8) + LOAD_DATA(rD3, 12) + xor rD0,rD0,rW0 + SAVE_DATA(rD0, 0) + xor rD1,rD1,rW1 + SAVE_DATA(rD1, 4) + xor rD2,rD2,rW2 + SAVE_DATA(rD2, 8) + xor rD3,rD3,rW3 + SAVE_DATA(rD3, 12) + addic rI3,rI3,1 /* increase counter */ + addze rI2,rI2 + addze rI1,rI1 + addze rI0,rI0 + NEXT_BLOCK + cmpwi rLN,15 + bt gt,ppc_crypt_ctr_loop +ppc_crypt_ctr_partial: + cmpwi rLN,0 + bt eq,ppc_crypt_ctr_end + mr rKP,rKS + START_KEY(rI0, rI1, rI2, rI3) + bl ppc_encrypt_block + xor rW0,rD0,rW0 + SAVE_IV(rW0, 0) + xor rW1,rD1,rW1 + SAVE_IV(rW1, 4) + xor rW2,rD2,rW2 + SAVE_IV(rW2, 8) + xor rW3,rD3,rW3 + SAVE_IV(rW3, 12) + mtctr rLN + subi rIP,rIP,CTR_DEC + subi rSP,rSP,1 + subi rDP,rDP,1 +ppc_crypt_ctr_xorbyte: + lbzu rW4,1(rIP) /* bytewise xor for partial block */ + lbzu rW5,1(rSP) + xor rW4,rW4,rW5 + stbu rW4,1(rDP) + bdnz ppc_crypt_ctr_xorbyte + subf rIP,rLN,rIP + addi rIP,rIP,1 + addic rI3,rI3,1 + addze rI2,rI2 + addze rI1,rI1 + addze rI0,rI0 +ppc_crypt_ctr_end: + SAVE_IV(rI0, 0) + SAVE_IV(rI1, 4) + SAVE_IV(rI2, 8) + SAVE_IV(rI3, 12) + FINALIZE_CRYPT(4) + blr + +/* + * ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, + * u32 rounds, u32 bytes, u8 *iv, u32 *key_twk); + * + * called from glue layer to encrypt multiple blocks via XTS + * If key_twk is given, the initial IV encryption will be + * processed too. Round values are AES128 = 4, AES192 = 5, + * AES256 = 6 + * + */ +_GLOBAL(ppc_encrypt_xts) + INITIALIZE_CRYPT(PPC_AES_4K_ENCTAB, 8) + LOAD_IV(rI0, 0) + LOAD_IV(rI1, 4) + LOAD_IV(rI2, 8) + cmpwi rKT,0 + LOAD_IV(rI3, 12) + bt eq,ppc_encrypt_xts_notweak + mr rKP,rKT + START_KEY(rI0, rI1, rI2, rI3) + bl ppc_encrypt_block + xor rI0,rD0,rW0 + xor rI1,rD1,rW1 + xor rI2,rD2,rW2 + xor rI3,rD3,rW3 +ppc_encrypt_xts_notweak: + ENDIAN_SWAP(rG0, rG1, rI0, rI1) + ENDIAN_SWAP(rG2, rG3, rI2, rI3) +ppc_encrypt_xts_loop: + LOAD_DATA(rD0, 0) + mr rKP,rKS + LOAD_DATA(rD1, 4) + subi rLN,rLN,16 + LOAD_DATA(rD2, 8) + LOAD_DATA(rD3, 12) + xor rD0,rD0,rI0 + xor rD1,rD1,rI1 + xor rD2,rD2,rI2 + xor rD3,rD3,rI3 + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_encrypt_block + xor rD0,rD0,rW0 + xor rD1,rD1,rW1 + xor rD2,rD2,rW2 + xor rD3,rD3,rW3 + xor rD0,rD0,rI0 + SAVE_DATA(rD0, 0) + xor rD1,rD1,rI1 + SAVE_DATA(rD1, 4) + xor rD2,rD2,rI2 + SAVE_DATA(rD2, 8) + xor rD3,rD3,rI3 + SAVE_DATA(rD3, 12) + GF128_MUL(rG0, rG1, rG2, rG3, rW0) + ENDIAN_SWAP(rI0, rI1, rG0, rG1) + ENDIAN_SWAP(rI2, rI3, rG2, rG3) + cmpwi rLN,0 + NEXT_BLOCK + bt gt,ppc_encrypt_xts_loop + START_IV + SAVE_IV(rI0, 0) + SAVE_IV(rI1, 4) + SAVE_IV(rI2, 8) + SAVE_IV(rI3, 12) + FINALIZE_CRYPT(8) + blr + +/* + * ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, + * u32 rounds, u32 blocks, u8 *iv, u32 *key_twk); + * + * called from glue layer to decrypt multiple blocks via XTS + * If key_twk is given, the initial IV encryption will be + * processed too. Round values are AES128 = 4, AES192 = 5, + * AES256 = 6 + * + */ +_GLOBAL(ppc_decrypt_xts) + INITIALIZE_CRYPT(PPC_AES_4K_DECTAB, 8) + LOAD_IV(rI0, 0) + addi rT1,rT0,4096 + LOAD_IV(rI1, 4) + LOAD_IV(rI2, 8) + cmpwi rKT,0 + LOAD_IV(rI3, 12) + bt eq,ppc_decrypt_xts_notweak + subi rT0,rT0,4096 + mr rKP,rKT + START_KEY(rI0, rI1, rI2, rI3) + bl ppc_encrypt_block + xor rI0,rD0,rW0 + xor rI1,rD1,rW1 + xor rI2,rD2,rW2 + xor rI3,rD3,rW3 + addi rT0,rT0,4096 +ppc_decrypt_xts_notweak: + ENDIAN_SWAP(rG0, rG1, rI0, rI1) + ENDIAN_SWAP(rG2, rG3, rI2, rI3) +ppc_decrypt_xts_loop: + LOAD_DATA(rD0, 0) + mr rKP,rKS + LOAD_DATA(rD1, 4) + subi rLN,rLN,16 + LOAD_DATA(rD2, 8) + LOAD_DATA(rD3, 12) + xor rD0,rD0,rI0 + xor rD1,rD1,rI1 + xor rD2,rD2,rI2 + xor rD3,rD3,rI3 + START_KEY(rD0, rD1, rD2, rD3) + bl ppc_decrypt_block + xor rD0,rD0,rW0 + xor rD1,rD1,rW1 + xor rD2,rD2,rW2 + xor rD3,rD3,rW3 + xor rD0,rD0,rI0 + SAVE_DATA(rD0, 0) + xor rD1,rD1,rI1 + SAVE_DATA(rD1, 4) + xor rD2,rD2,rI2 + SAVE_DATA(rD2, 8) + xor rD3,rD3,rI3 + SAVE_DATA(rD3, 12) + GF128_MUL(rG0, rG1, rG2, rG3, rW0) + ENDIAN_SWAP(rI0, rI1, rG0, rG1) + ENDIAN_SWAP(rI2, rI3, rG2, rG3) + cmpwi rLN,0 + NEXT_BLOCK + bt gt,ppc_decrypt_xts_loop + START_IV + SAVE_IV(rI0, 0) + SAVE_IV(rI1, 4) + SAVE_IV(rI2, 8) + SAVE_IV(rI3, 12) + FINALIZE_CRYPT(8) + blr diff --git a/arch/powerpc/crypto/aes-spe-regs.h b/arch/powerpc/crypto/aes-spe-regs.h new file mode 100644 index 000000000000..30d217b399c3 --- /dev/null +++ b/arch/powerpc/crypto/aes-spe-regs.h @@ -0,0 +1,42 @@ +/* + * Common registers for PPC AES implementation + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#define rKS r0 /* copy of en-/decryption key pointer */ +#define rDP r3 /* destination pointer */ +#define rSP r4 /* source pointer */ +#define rKP r5 /* pointer to en-/decryption key pointer */ +#define rRR r6 /* en-/decryption rounds */ +#define rLN r7 /* length of data to be processed */ +#define rIP r8 /* potiner to IV (CBC/CTR/XTS modes) */ +#define rKT r9 /* pointer to tweak key (XTS mode) */ +#define rT0 r11 /* pointers to en-/decrpytion tables */ +#define rT1 r10 +#define rD0 r9 /* data */ +#define rD1 r14 +#define rD2 r12 +#define rD3 r15 +#define rW0 r16 /* working registers */ +#define rW1 r17 +#define rW2 r18 +#define rW3 r19 +#define rW4 r20 +#define rW5 r21 +#define rW6 r22 +#define rW7 r23 +#define rI0 r24 /* IV */ +#define rI1 r25 +#define rI2 r26 +#define rI3 r27 +#define rG0 r28 /* endian reversed tweak (XTS mode) */ +#define rG1 r29 +#define rG2 r30 +#define rG3 r31 diff --git a/arch/powerpc/crypto/aes-tab-4k.S b/arch/powerpc/crypto/aes-tab-4k.S new file mode 100644 index 000000000000..701e60240dc3 --- /dev/null +++ b/arch/powerpc/crypto/aes-tab-4k.S @@ -0,0 +1,331 @@ +/* + * 4K AES tables for PPC AES implementation + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +/* + * These big endian AES encryption/decryption tables have been taken from + * crypto/aes_generic.c and are designed to be simply accessed by a combination + * of rlwimi/lwz instructions with a minimum of table registers (usually only + * one required). Thus they are aligned to 4K. The locality of rotated values + * is derived from the reduced offsets that are available in the SPE load + * instructions. E.g. evldw, evlwwsplat, ... + * + * For the safety-conscious it has to be noted that they might be vulnerable + * to cache timing attacks because of their size. Nevertheless in contrast to + * the generic tables they have been reduced from 16KB to 8KB + 256 bytes. + * This is a quite good tradeoff for low power devices (e.g. routers) without + * dedicated encryption hardware where we usually have no multiuser + * environment. + * + */ + +#define R(a, b, c, d) \ + 0x##a##b##c##d, 0x##d##a##b##c, 0x##c##d##a##b, 0x##b##c##d##a + +.data +.align 12 +.globl PPC_AES_4K_ENCTAB +PPC_AES_4K_ENCTAB: +/* encryption table, same as crypto_ft_tab in crypto/aes-generic.c */ + .long R(c6, 63, 63, a5), R(f8, 7c, 7c, 84) + .long R(ee, 77, 77, 99), R(f6, 7b, 7b, 8d) + .long R(ff, f2, f2, 0d), R(d6, 6b, 6b, bd) + .long R(de, 6f, 6f, b1), R(91, c5, c5, 54) + .long R(60, 30, 30, 50), R(02, 01, 01, 03) + .long R(ce, 67, 67, a9), R(56, 2b, 2b, 7d) + .long R(e7, fe, fe, 19), R(b5, d7, d7, 62) + .long R(4d, ab, ab, e6), R(ec, 76, 76, 9a) + .long R(8f, ca, ca, 45), R(1f, 82, 82, 9d) + .long R(89, c9, c9, 40), R(fa, 7d, 7d, 87) + .long R(ef, fa, fa, 15), R(b2, 59, 59, eb) + .long R(8e, 47, 47, c9), R(fb, f0, f0, 0b) + .long R(41, ad, ad, ec), R(b3, d4, d4, 67) + .long R(5f, a2, a2, fd), R(45, af, af, ea) + .long R(23, 9c, 9c, bf), R(53, a4, a4, f7) + .long R(e4, 72, 72, 96), R(9b, c0, c0, 5b) + .long R(75, b7, b7, c2), R(e1, fd, fd, 1c) + .long R(3d, 93, 93, ae), R(4c, 26, 26, 6a) + .long R(6c, 36, 36, 5a), R(7e, 3f, 3f, 41) + .long R(f5, f7, f7, 02), R(83, cc, cc, 4f) + .long R(68, 34, 34, 5c), R(51, a5, a5, f4) + .long R(d1, e5, e5, 34), R(f9, f1, f1, 08) + .long R(e2, 71, 71, 93), R(ab, d8, d8, 73) + .long R(62, 31, 31, 53), R(2a, 15, 15, 3f) + .long R(08, 04, 04, 0c), R(95, c7, c7, 52) + .long R(46, 23, 23, 65), R(9d, c3, c3, 5e) + .long R(30, 18, 18, 28), R(37, 96, 96, a1) + .long R(0a, 05, 05, 0f), R(2f, 9a, 9a, b5) + .long R(0e, 07, 07, 09), R(24, 12, 12, 36) + .long R(1b, 80, 80, 9b), R(df, e2, e2, 3d) + .long R(cd, eb, eb, 26), R(4e, 27, 27, 69) + .long R(7f, b2, b2, cd), R(ea, 75, 75, 9f) + .long R(12, 09, 09, 1b), R(1d, 83, 83, 9e) + .long R(58, 2c, 2c, 74), R(34, 1a, 1a, 2e) + .long R(36, 1b, 1b, 2d), R(dc, 6e, 6e, b2) + .long R(b4, 5a, 5a, ee), R(5b, a0, a0, fb) + .long R(a4, 52, 52, f6), R(76, 3b, 3b, 4d) + .long R(b7, d6, d6, 61), R(7d, b3, b3, ce) + .long R(52, 29, 29, 7b), R(dd, e3, e3, 3e) + .long R(5e, 2f, 2f, 71), R(13, 84, 84, 97) + .long R(a6, 53, 53, f5), R(b9, d1, d1, 68) + .long R(00, 00, 00, 00), R(c1, ed, ed, 2c) + .long R(40, 20, 20, 60), R(e3, fc, fc, 1f) + .long R(79, b1, b1, c8), R(b6, 5b, 5b, ed) + .long R(d4, 6a, 6a, be), R(8d, cb, cb, 46) + .long R(67, be, be, d9), R(72, 39, 39, 4b) + .long R(94, 4a, 4a, de), R(98, 4c, 4c, d4) + .long R(b0, 58, 58, e8), R(85, cf, cf, 4a) + .long R(bb, d0, d0, 6b), R(c5, ef, ef, 2a) + .long R(4f, aa, aa, e5), R(ed, fb, fb, 16) + .long R(86, 43, 43, c5), R(9a, 4d, 4d, d7) + .long R(66, 33, 33, 55), R(11, 85, 85, 94) + .long R(8a, 45, 45, cf), R(e9, f9, f9, 10) + .long R(04, 02, 02, 06), R(fe, 7f, 7f, 81) + .long R(a0, 50, 50, f0), R(78, 3c, 3c, 44) + .long R(25, 9f, 9f, ba), R(4b, a8, a8, e3) + .long R(a2, 51, 51, f3), R(5d, a3, a3, fe) + .long R(80, 40, 40, c0), R(05, 8f, 8f, 8a) + .long R(3f, 92, 92, ad), R(21, 9d, 9d, bc) + .long R(70, 38, 38, 48), R(f1, f5, f5, 04) + .long R(63, bc, bc, df), R(77, b6, b6, c1) + .long R(af, da, da, 75), R(42, 21, 21, 63) + .long R(20, 10, 10, 30), R(e5, ff, ff, 1a) + .long R(fd, f3, f3, 0e), R(bf, d2, d2, 6d) + .long R(81, cd, cd, 4c), R(18, 0c, 0c, 14) + .long R(26, 13, 13, 35), R(c3, ec, ec, 2f) + .long R(be, 5f, 5f, e1), R(35, 97, 97, a2) + .long R(88, 44, 44, cc), R(2e, 17, 17, 39) + .long R(93, c4, c4, 57), R(55, a7, a7, f2) + .long R(fc, 7e, 7e, 82), R(7a, 3d, 3d, 47) + .long R(c8, 64, 64, ac), R(ba, 5d, 5d, e7) + .long R(32, 19, 19, 2b), R(e6, 73, 73, 95) + .long R(c0, 60, 60, a0), R(19, 81, 81, 98) + .long R(9e, 4f, 4f, d1), R(a3, dc, dc, 7f) + .long R(44, 22, 22, 66), R(54, 2a, 2a, 7e) + .long R(3b, 90, 90, ab), R(0b, 88, 88, 83) + .long R(8c, 46, 46, ca), R(c7, ee, ee, 29) + .long R(6b, b8, b8, d3), R(28, 14, 14, 3c) + .long R(a7, de, de, 79), R(bc, 5e, 5e, e2) + .long R(16, 0b, 0b, 1d), R(ad, db, db, 76) + .long R(db, e0, e0, 3b), R(64, 32, 32, 56) + .long R(74, 3a, 3a, 4e), R(14, 0a, 0a, 1e) + .long R(92, 49, 49, db), R(0c, 06, 06, 0a) + .long R(48, 24, 24, 6c), R(b8, 5c, 5c, e4) + .long R(9f, c2, c2, 5d), R(bd, d3, d3, 6e) + .long R(43, ac, ac, ef), R(c4, 62, 62, a6) + .long R(39, 91, 91, a8), R(31, 95, 95, a4) + .long R(d3, e4, e4, 37), R(f2, 79, 79, 8b) + .long R(d5, e7, e7, 32), R(8b, c8, c8, 43) + .long R(6e, 37, 37, 59), R(da, 6d, 6d, b7) + .long R(01, 8d, 8d, 8c), R(b1, d5, d5, 64) + .long R(9c, 4e, 4e, d2), R(49, a9, a9, e0) + .long R(d8, 6c, 6c, b4), R(ac, 56, 56, fa) + .long R(f3, f4, f4, 07), R(cf, ea, ea, 25) + .long R(ca, 65, 65, af), R(f4, 7a, 7a, 8e) + .long R(47, ae, ae, e9), R(10, 08, 08, 18) + .long R(6f, ba, ba, d5), R(f0, 78, 78, 88) + .long R(4a, 25, 25, 6f), R(5c, 2e, 2e, 72) + .long R(38, 1c, 1c, 24), R(57, a6, a6, f1) + .long R(73, b4, b4, c7), R(97, c6, c6, 51) + .long R(cb, e8, e8, 23), R(a1, dd, dd, 7c) + .long R(e8, 74, 74, 9c), R(3e, 1f, 1f, 21) + .long R(96, 4b, 4b, dd), R(61, bd, bd, dc) + .long R(0d, 8b, 8b, 86), R(0f, 8a, 8a, 85) + .long R(e0, 70, 70, 90), R(7c, 3e, 3e, 42) + .long R(71, b5, b5, c4), R(cc, 66, 66, aa) + .long R(90, 48, 48, d8), R(06, 03, 03, 05) + .long R(f7, f6, f6, 01), R(1c, 0e, 0e, 12) + .long R(c2, 61, 61, a3), R(6a, 35, 35, 5f) + .long R(ae, 57, 57, f9), R(69, b9, b9, d0) + .long R(17, 86, 86, 91), R(99, c1, c1, 58) + .long R(3a, 1d, 1d, 27), R(27, 9e, 9e, b9) + .long R(d9, e1, e1, 38), R(eb, f8, f8, 13) + .long R(2b, 98, 98, b3), R(22, 11, 11, 33) + .long R(d2, 69, 69, bb), R(a9, d9, d9, 70) + .long R(07, 8e, 8e, 89), R(33, 94, 94, a7) + .long R(2d, 9b, 9b, b6), R(3c, 1e, 1e, 22) + .long R(15, 87, 87, 92), R(c9, e9, e9, 20) + .long R(87, ce, ce, 49), R(aa, 55, 55, ff) + .long R(50, 28, 28, 78), R(a5, df, df, 7a) + .long R(03, 8c, 8c, 8f), R(59, a1, a1, f8) + .long R(09, 89, 89, 80), R(1a, 0d, 0d, 17) + .long R(65, bf, bf, da), R(d7, e6, e6, 31) + .long R(84, 42, 42, c6), R(d0, 68, 68, b8) + .long R(82, 41, 41, c3), R(29, 99, 99, b0) + .long R(5a, 2d, 2d, 77), R(1e, 0f, 0f, 11) + .long R(7b, b0, b0, cb), R(a8, 54, 54, fc) + .long R(6d, bb, bb, d6), R(2c, 16, 16, 3a) +.globl PPC_AES_4K_DECTAB +PPC_AES_4K_DECTAB: +/* decryption table, same as crypto_it_tab in crypto/aes-generic.c */ + .long R(51, f4, a7, 50), R(7e, 41, 65, 53) + .long R(1a, 17, a4, c3), R(3a, 27, 5e, 96) + .long R(3b, ab, 6b, cb), R(1f, 9d, 45, f1) + .long R(ac, fa, 58, ab), R(4b, e3, 03, 93) + .long R(20, 30, fa, 55), R(ad, 76, 6d, f6) + .long R(88, cc, 76, 91), R(f5, 02, 4c, 25) + .long R(4f, e5, d7, fc), R(c5, 2a, cb, d7) + .long R(26, 35, 44, 80), R(b5, 62, a3, 8f) + .long R(de, b1, 5a, 49), R(25, ba, 1b, 67) + .long R(45, ea, 0e, 98), R(5d, fe, c0, e1) + .long R(c3, 2f, 75, 02), R(81, 4c, f0, 12) + .long R(8d, 46, 97, a3), R(6b, d3, f9, c6) + .long R(03, 8f, 5f, e7), R(15, 92, 9c, 95) + .long R(bf, 6d, 7a, eb), R(95, 52, 59, da) + .long R(d4, be, 83, 2d), R(58, 74, 21, d3) + .long R(49, e0, 69, 29), R(8e, c9, c8, 44) + .long R(75, c2, 89, 6a), R(f4, 8e, 79, 78) + .long R(99, 58, 3e, 6b), R(27, b9, 71, dd) + .long R(be, e1, 4f, b6), R(f0, 88, ad, 17) + .long R(c9, 20, ac, 66), R(7d, ce, 3a, b4) + .long R(63, df, 4a, 18), R(e5, 1a, 31, 82) + .long R(97, 51, 33, 60), R(62, 53, 7f, 45) + .long R(b1, 64, 77, e0), R(bb, 6b, ae, 84) + .long R(fe, 81, a0, 1c), R(f9, 08, 2b, 94) + .long R(70, 48, 68, 58), R(8f, 45, fd, 19) + .long R(94, de, 6c, 87), R(52, 7b, f8, b7) + .long R(ab, 73, d3, 23), R(72, 4b, 02, e2) + .long R(e3, 1f, 8f, 57), R(66, 55, ab, 2a) + .long R(b2, eb, 28, 07), R(2f, b5, c2, 03) + .long R(86, c5, 7b, 9a), R(d3, 37, 08, a5) + .long R(30, 28, 87, f2), R(23, bf, a5, b2) + .long R(02, 03, 6a, ba), R(ed, 16, 82, 5c) + .long R(8a, cf, 1c, 2b), R(a7, 79, b4, 92) + .long R(f3, 07, f2, f0), R(4e, 69, e2, a1) + .long R(65, da, f4, cd), R(06, 05, be, d5) + .long R(d1, 34, 62, 1f), R(c4, a6, fe, 8a) + .long R(34, 2e, 53, 9d), R(a2, f3, 55, a0) + .long R(05, 8a, e1, 32), R(a4, f6, eb, 75) + .long R(0b, 83, ec, 39), R(40, 60, ef, aa) + .long R(5e, 71, 9f, 06), R(bd, 6e, 10, 51) + .long R(3e, 21, 8a, f9), R(96, dd, 06, 3d) + .long R(dd, 3e, 05, ae), R(4d, e6, bd, 46) + .long R(91, 54, 8d, b5), R(71, c4, 5d, 05) + .long R(04, 06, d4, 6f), R(60, 50, 15, ff) + .long R(19, 98, fb, 24), R(d6, bd, e9, 97) + .long R(89, 40, 43, cc), R(67, d9, 9e, 77) + .long R(b0, e8, 42, bd), R(07, 89, 8b, 88) + .long R(e7, 19, 5b, 38), R(79, c8, ee, db) + .long R(a1, 7c, 0a, 47), R(7c, 42, 0f, e9) + .long R(f8, 84, 1e, c9), R(00, 00, 00, 00) + .long R(09, 80, 86, 83), R(32, 2b, ed, 48) + .long R(1e, 11, 70, ac), R(6c, 5a, 72, 4e) + .long R(fd, 0e, ff, fb), R(0f, 85, 38, 56) + .long R(3d, ae, d5, 1e), R(36, 2d, 39, 27) + .long R(0a, 0f, d9, 64), R(68, 5c, a6, 21) + .long R(9b, 5b, 54, d1), R(24, 36, 2e, 3a) + .long R(0c, 0a, 67, b1), R(93, 57, e7, 0f) + .long R(b4, ee, 96, d2), R(1b, 9b, 91, 9e) + .long R(80, c0, c5, 4f), R(61, dc, 20, a2) + .long R(5a, 77, 4b, 69), R(1c, 12, 1a, 16) + .long R(e2, 93, ba, 0a), R(c0, a0, 2a, e5) + .long R(3c, 22, e0, 43), R(12, 1b, 17, 1d) + .long R(0e, 09, 0d, 0b), R(f2, 8b, c7, ad) + .long R(2d, b6, a8, b9), R(14, 1e, a9, c8) + .long R(57, f1, 19, 85), R(af, 75, 07, 4c) + .long R(ee, 99, dd, bb), R(a3, 7f, 60, fd) + .long R(f7, 01, 26, 9f), R(5c, 72, f5, bc) + .long R(44, 66, 3b, c5), R(5b, fb, 7e, 34) + .long R(8b, 43, 29, 76), R(cb, 23, c6, dc) + .long R(b6, ed, fc, 68), R(b8, e4, f1, 63) + .long R(d7, 31, dc, ca), R(42, 63, 85, 10) + .long R(13, 97, 22, 40), R(84, c6, 11, 20) + .long R(85, 4a, 24, 7d), R(d2, bb, 3d, f8) + .long R(ae, f9, 32, 11), R(c7, 29, a1, 6d) + .long R(1d, 9e, 2f, 4b), R(dc, b2, 30, f3) + .long R(0d, 86, 52, ec), R(77, c1, e3, d0) + .long R(2b, b3, 16, 6c), R(a9, 70, b9, 99) + .long R(11, 94, 48, fa), R(47, e9, 64, 22) + .long R(a8, fc, 8c, c4), R(a0, f0, 3f, 1a) + .long R(56, 7d, 2c, d8), R(22, 33, 90, ef) + .long R(87, 49, 4e, c7), R(d9, 38, d1, c1) + .long R(8c, ca, a2, fe), R(98, d4, 0b, 36) + .long R(a6, f5, 81, cf), R(a5, 7a, de, 28) + .long R(da, b7, 8e, 26), R(3f, ad, bf, a4) + .long R(2c, 3a, 9d, e4), R(50, 78, 92, 0d) + .long R(6a, 5f, cc, 9b), R(54, 7e, 46, 62) + .long R(f6, 8d, 13, c2), R(90, d8, b8, e8) + .long R(2e, 39, f7, 5e), R(82, c3, af, f5) + .long R(9f, 5d, 80, be), R(69, d0, 93, 7c) + .long R(6f, d5, 2d, a9), R(cf, 25, 12, b3) + .long R(c8, ac, 99, 3b), R(10, 18, 7d, a7) + .long R(e8, 9c, 63, 6e), R(db, 3b, bb, 7b) + .long R(cd, 26, 78, 09), R(6e, 59, 18, f4) + .long R(ec, 9a, b7, 01), R(83, 4f, 9a, a8) + .long R(e6, 95, 6e, 65), R(aa, ff, e6, 7e) + .long R(21, bc, cf, 08), R(ef, 15, e8, e6) + .long R(ba, e7, 9b, d9), R(4a, 6f, 36, ce) + .long R(ea, 9f, 09, d4), R(29, b0, 7c, d6) + .long R(31, a4, b2, af), R(2a, 3f, 23, 31) + .long R(c6, a5, 94, 30), R(35, a2, 66, c0) + .long R(74, 4e, bc, 37), R(fc, 82, ca, a6) + .long R(e0, 90, d0, b0), R(33, a7, d8, 15) + .long R(f1, 04, 98, 4a), R(41, ec, da, f7) + .long R(7f, cd, 50, 0e), R(17, 91, f6, 2f) + .long R(76, 4d, d6, 8d), R(43, ef, b0, 4d) + .long R(cc, aa, 4d, 54), R(e4, 96, 04, df) + .long R(9e, d1, b5, e3), R(4c, 6a, 88, 1b) + .long R(c1, 2c, 1f, b8), R(46, 65, 51, 7f) + .long R(9d, 5e, ea, 04), R(01, 8c, 35, 5d) + .long R(fa, 87, 74, 73), R(fb, 0b, 41, 2e) + .long R(b3, 67, 1d, 5a), R(92, db, d2, 52) + .long R(e9, 10, 56, 33), R(6d, d6, 47, 13) + .long R(9a, d7, 61, 8c), R(37, a1, 0c, 7a) + .long R(59, f8, 14, 8e), R(eb, 13, 3c, 89) + .long R(ce, a9, 27, ee), R(b7, 61, c9, 35) + .long R(e1, 1c, e5, ed), R(7a, 47, b1, 3c) + .long R(9c, d2, df, 59), R(55, f2, 73, 3f) + .long R(18, 14, ce, 79), R(73, c7, 37, bf) + .long R(53, f7, cd, ea), R(5f, fd, aa, 5b) + .long R(df, 3d, 6f, 14), R(78, 44, db, 86) + .long R(ca, af, f3, 81), R(b9, 68, c4, 3e) + .long R(38, 24, 34, 2c), R(c2, a3, 40, 5f) + .long R(16, 1d, c3, 72), R(bc, e2, 25, 0c) + .long R(28, 3c, 49, 8b), R(ff, 0d, 95, 41) + .long R(39, a8, 01, 71), R(08, 0c, b3, de) + .long R(d8, b4, e4, 9c), R(64, 56, c1, 90) + .long R(7b, cb, 84, 61), R(d5, 32, b6, 70) + .long R(48, 6c, 5c, 74), R(d0, b8, 57, 42) +.globl PPC_AES_4K_DECTAB2 +PPC_AES_4K_DECTAB2: +/* decryption table, same as crypto_il_tab in crypto/aes-generic.c */ + .byte 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38 + .byte 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb + .byte 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87 + .byte 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb + .byte 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d + .byte 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e + .byte 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2 + .byte 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25 + .byte 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16 + .byte 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92 + .byte 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda + .byte 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84 + .byte 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a + .byte 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06 + .byte 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02 + .byte 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b + .byte 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea + .byte 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73 + .byte 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85 + .byte 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e + .byte 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89 + .byte 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b + .byte 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20 + .byte 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4 + .byte 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31 + .byte 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f + .byte 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d + .byte 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef + .byte 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0 + .byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61 + .byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26 + .byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d diff --git a/arch/powerpc/crypto/md5-asm.S b/arch/powerpc/crypto/md5-asm.S new file mode 100644 index 000000000000..10cdf5bceebb --- /dev/null +++ b/arch/powerpc/crypto/md5-asm.S @@ -0,0 +1,243 @@ +/* + * Fast MD5 implementation for PPC + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#include <asm/ppc_asm.h> +#include <asm/asm-offsets.h> + +#define rHP r3 +#define rWP r4 + +#define rH0 r0 +#define rH1 r6 +#define rH2 r7 +#define rH3 r5 + +#define rW00 r8 +#define rW01 r9 +#define rW02 r10 +#define rW03 r11 +#define rW04 r12 +#define rW05 r14 +#define rW06 r15 +#define rW07 r16 +#define rW08 r17 +#define rW09 r18 +#define rW10 r19 +#define rW11 r20 +#define rW12 r21 +#define rW13 r22 +#define rW14 r23 +#define rW15 r24 + +#define rT0 r25 +#define rT1 r26 + +#define INITIALIZE \ + PPC_STLU r1,-INT_FRAME_SIZE(r1); \ + SAVE_8GPRS(14, r1); /* push registers onto stack */ \ + SAVE_4GPRS(22, r1); \ + SAVE_GPR(26, r1) + +#define FINALIZE \ + REST_8GPRS(14, r1); /* pop registers from stack */ \ + REST_4GPRS(22, r1); \ + REST_GPR(26, r1); \ + addi r1,r1,INT_FRAME_SIZE; + +#ifdef __BIG_ENDIAN__ +#define LOAD_DATA(reg, off) \ + lwbrx reg,0,rWP; /* load data */ +#define INC_PTR \ + addi rWP,rWP,4; /* increment per word */ +#define NEXT_BLOCK /* nothing to do */ +#else +#define LOAD_DATA(reg, off) \ + lwz reg,off(rWP); /* load data */ +#define INC_PTR /* nothing to do */ +#define NEXT_BLOCK \ + addi rWP,rWP,64; /* increment per block */ +#endif + +#define R_00_15(a, b, c, d, w0, w1, p, q, off, k0h, k0l, k1h, k1l) \ + LOAD_DATA(w0, off) /* W */ \ + and rT0,b,c; /* 1: f = b and c */ \ + INC_PTR /* ptr++ */ \ + andc rT1,d,b; /* 1: f' = ~b and d */ \ + LOAD_DATA(w1, off+4) /* W */ \ + or rT0,rT0,rT1; /* 1: f = f or f' */ \ + addi w0,w0,k0l; /* 1: wk = w + k */ \ + add a,a,rT0; /* 1: a = a + f */ \ + addis w0,w0,k0h; /* 1: wk = w + k' */ \ + addis w1,w1,k1h; /* 2: wk = w + k */ \ + add a,a,w0; /* 1: a = a + wk */ \ + addi w1,w1,k1l; /* 2: wk = w + k' */ \ + rotrwi a,a,p; /* 1: a = a rotl x */ \ + add d,d,w1; /* 2: a = a + wk */ \ + add a,a,b; /* 1: a = a + b */ \ + and rT0,a,b; /* 2: f = b and c */ \ + andc rT1,c,a; /* 2: f' = ~b and d */ \ + or rT0,rT0,rT1; /* 2: f = f or f' */ \ + add d,d,rT0; /* 2: a = a + f */ \ + INC_PTR /* ptr++ */ \ + rotrwi d,d,q; /* 2: a = a rotl x */ \ + add d,d,a; /* 2: a = a + b */ + +#define R_16_31(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ + andc rT0,c,d; /* 1: f = c and ~d */ \ + and rT1,b,d; /* 1: f' = b and d */ \ + addi w0,w0,k0l; /* 1: wk = w + k */ \ + or rT0,rT0,rT1; /* 1: f = f or f' */ \ + addis w0,w0,k0h; /* 1: wk = w + k' */ \ + add a,a,rT0; /* 1: a = a + f */ \ + addi w1,w1,k1l; /* 2: wk = w + k */ \ + add a,a,w0; /* 1: a = a + wk */ \ + addis w1,w1,k1h; /* 2: wk = w + k' */ \ + andc rT0,b,c; /* 2: f = c and ~d */ \ + rotrwi a,a,p; /* 1: a = a rotl x */ \ + add a,a,b; /* 1: a = a + b */ \ + add d,d,w1; /* 2: a = a + wk */ \ + and rT1,a,c; /* 2: f' = b and d */ \ + or rT0,rT0,rT1; /* 2: f = f or f' */ \ + add d,d,rT0; /* 2: a = a + f */ \ + rotrwi d,d,q; /* 2: a = a rotl x */ \ + add d,d,a; /* 2: a = a +b */ + +#define R_32_47(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ + xor rT0,b,c; /* 1: f' = b xor c */ \ + addi w0,w0,k0l; /* 1: wk = w + k */ \ + xor rT1,rT0,d; /* 1: f = f xor f' */ \ + addis w0,w0,k0h; /* 1: wk = w + k' */ \ + add a,a,rT1; /* 1: a = a + f */ \ + addi w1,w1,k1l; /* 2: wk = w + k */ \ + add a,a,w0; /* 1: a = a + wk */ \ + addis w1,w1,k1h; /* 2: wk = w + k' */ \ + rotrwi a,a,p; /* 1: a = a rotl x */ \ + add d,d,w1; /* 2: a = a + wk */ \ + add a,a,b; /* 1: a = a + b */ \ + xor rT1,rT0,a; /* 2: f = b xor f' */ \ + add d,d,rT1; /* 2: a = a + f */ \ + rotrwi d,d,q; /* 2: a = a rotl x */ \ + add d,d,a; /* 2: a = a + b */ + +#define R_48_63(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \ + addi w0,w0,k0l; /* 1: w = w + k */ \ + orc rT0,b,d; /* 1: f = b or ~d */ \ + addis w0,w0,k0h; /* 1: w = w + k' */ \ + xor rT0,rT0,c; /* 1: f = f xor c */ \ + add a,a,w0; /* 1: a = a + wk */ \ + addi w1,w1,k1l; /* 2: w = w + k */ \ + add a,a,rT0; /* 1: a = a + f */ \ + addis w1,w1,k1h; /* 2: w = w + k' */ \ + rotrwi a,a,p; /* 1: a = a rotl x */ \ + add a,a,b; /* 1: a = a + b */ \ + orc rT0,a,c; /* 2: f = b or ~d */ \ + add d,d,w1; /* 2: a = a + wk */ \ + xor rT0,rT0,b; /* 2: f = f xor c */ \ + add d,d,rT0; /* 2: a = a + f */ \ + rotrwi d,d,q; /* 2: a = a rotl x */ \ + add d,d,a; /* 2: a = a + b */ + +_GLOBAL(ppc_md5_transform) + INITIALIZE + + mtctr r5 + lwz rH0,0(rHP) + lwz rH1,4(rHP) + lwz rH2,8(rHP) + lwz rH3,12(rHP) + +ppc_md5_main: + R_00_15(rH0, rH1, rH2, rH3, rW00, rW01, 25, 20, 0, + 0xd76b, -23432, 0xe8c8, -18602) + R_00_15(rH2, rH3, rH0, rH1, rW02, rW03, 15, 10, 8, + 0x2420, 0x70db, 0xc1be, -12562) + R_00_15(rH0, rH1, rH2, rH3, rW04, rW05, 25, 20, 16, + 0xf57c, 0x0faf, 0x4788, -14806) + R_00_15(rH2, rH3, rH0, rH1, rW06, rW07, 15, 10, 24, + 0xa830, 0x4613, 0xfd47, -27391) + R_00_15(rH0, rH1, rH2, rH3, rW08, rW09, 25, 20, 32, + 0x6981, -26408, 0x8b45, -2129) + R_00_15(rH2, rH3, rH0, rH1, rW10, rW11, 15, 10, 40, + 0xffff, 0x5bb1, 0x895d, -10306) + R_00_15(rH0, rH1, rH2, rH3, rW12, rW13, 25, 20, 48, + 0x6b90, 0x1122, 0xfd98, 0x7193) + R_00_15(rH2, rH3, rH0, rH1, rW14, rW15, 15, 10, 56, + 0xa679, 0x438e, 0x49b4, 0x0821) + + R_16_31(rH0, rH1, rH2, rH3, rW01, rW06, 27, 23, + 0x0d56, 0x6e0c, 0x1810, 0x6d2d) + R_16_31(rH2, rH3, rH0, rH1, rW11, rW00, 18, 12, + 0x9d02, -32109, 0x124c, 0x2332) + R_16_31(rH0, rH1, rH2, rH3, rW05, rW10, 27, 23, + 0x8ea7, 0x4a33, 0x0245, -18270) + R_16_31(rH2, rH3, rH0, rH1, rW15, rW04, 18, 12, + 0x8eee, -8608, 0xf258, -5095) + R_16_31(rH0, rH1, rH2, rH3, rW09, rW14, 27, 23, + 0x969d, -10697, 0x1cbe, -15288) + R_16_31(rH2, rH3, rH0, rH1, rW03, rW08, 18, 12, + 0x3317, 0x3e99, 0xdbd9, 0x7c15) + R_16_31(rH0, rH1, rH2, rH3, rW13, rW02, 27, 23, + 0xac4b, 0x7772, 0xd8cf, 0x331d) + R_16_31(rH2, rH3, rH0, rH1, rW07, rW12, 18, 12, + 0x6a28, 0x6dd8, 0x219a, 0x3b68) + + R_32_47(rH0, rH1, rH2, rH3, rW05, rW08, 28, 21, + 0x29cb, 0x28e5, 0x4218, -7788) + R_32_47(rH2, rH3, rH0, rH1, rW11, rW14, 16, 9, + 0x473f, 0x06d1, 0x3aae, 0x3036) + R_32_47(rH0, rH1, rH2, rH3, rW01, rW04, 28, 21, + 0xaea1, -15134, 0x640b, -11295) + R_32_47(rH2, rH3, rH0, rH1, rW07, rW10, 16, 9, + 0x8f4c, 0x4887, 0xbc7c, -22499) + R_32_47(rH0, rH1, rH2, rH3, rW13, rW00, 28, 21, + 0x7eb8, -27199, 0x00ea, 0x6050) + R_32_47(rH2, rH3, rH0, rH1, rW03, rW06, 16, 9, + 0xe01a, 0x22fe, 0x4447, 0x69c5) + R_32_47(rH0, rH1, rH2, rH3, rW09, rW12, 28, 21, + 0xb7f3, 0x0253, 0x59b1, 0x4d5b) + R_32_47(rH2, rH3, rH0, rH1, rW15, rW02, 16, 9, + 0x4701, -27017, 0xc7bd, -19859) + + R_48_63(rH0, rH1, rH2, rH3, rW00, rW07, 26, 22, + 0x0988, -1462, 0x4c70, -19401) + R_48_63(rH2, rH3, rH0, rH1, rW14, rW05, 17, 11, + 0xadaf, -5221, 0xfc99, 0x66f7) + R_48_63(rH0, rH1, rH2, rH3, rW12, rW03, 26, 22, + 0x7e80, -16418, 0xba1e, -25587) + R_48_63(rH2, rH3, rH0, rH1, rW10, rW01, 17, 11, + 0x4130, 0x380d, 0xe0c5, 0x738d) + lwz rW00,0(rHP) + R_48_63(rH0, rH1, rH2, rH3, rW08, rW15, 26, 22, + 0xe837, -30770, 0xde8a, 0x69e8) + lwz rW14,4(rHP) + R_48_63(rH2, rH3, rH0, rH1, rW06, rW13, 17, 11, + 0x9e79, 0x260f, 0x256d, -27941) + lwz rW12,8(rHP) + R_48_63(rH0, rH1, rH2, rH3, rW04, rW11, 26, 22, + 0xab75, -20775, 0x4f9e, -28397) + lwz rW10,12(rHP) + R_48_63(rH2, rH3, rH0, rH1, rW02, rW09, 17, 11, + 0x662b, 0x7c56, 0x11b2, 0x0358) + + add rH0,rH0,rW00 + stw rH0,0(rHP) + add rH1,rH1,rW14 + stw rH1,4(rHP) + add rH2,rH2,rW12 + stw rH2,8(rHP) + add rH3,rH3,rW10 + stw rH3,12(rHP) + NEXT_BLOCK + + bdnz ppc_md5_main + + FINALIZE + blr diff --git a/arch/powerpc/crypto/md5-glue.c b/arch/powerpc/crypto/md5-glue.c new file mode 100644 index 000000000000..452fb4dc575f --- /dev/null +++ b/arch/powerpc/crypto/md5-glue.c @@ -0,0 +1,165 @@ +/* + * Glue code for MD5 implementation for PPC assembler + * + * Based on generic implementation. + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <crypto/md5.h> +#include <asm/byteorder.h> + +extern void ppc_md5_transform(u32 *state, const u8 *src, u32 blocks); + +static inline void ppc_md5_clear_context(struct md5_state *sctx) +{ + int count = sizeof(struct md5_state) >> 2; + u32 *ptr = (u32 *)sctx; + + /* make sure we can clear the fast way */ + BUILD_BUG_ON(sizeof(struct md5_state) % 4); + do { *ptr++ = 0; } while (--count); +} + +static int ppc_md5_init(struct shash_desc *desc) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + + sctx->hash[0] = 0x67452301; + sctx->hash[1] = 0xefcdab89; + sctx->hash[2] = 0x98badcfe; + sctx->hash[3] = 0x10325476; + sctx->byte_count = 0; + + return 0; +} + +static int ppc_md5_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + const unsigned int offset = sctx->byte_count & 0x3f; + unsigned int avail = 64 - offset; + const u8 *src = data; + + sctx->byte_count += len; + + if (avail > len) { + memcpy((char *)sctx->block + offset, src, len); + return 0; + } + + if (offset) { + memcpy((char *)sctx->block + offset, src, avail); + ppc_md5_transform(sctx->hash, (const u8 *)sctx->block, 1); + len -= avail; + src += avail; + } + + if (len > 63) { + ppc_md5_transform(sctx->hash, src, len >> 6); + src += len & ~0x3f; + len &= 0x3f; + } + + memcpy((char *)sctx->block, src, len); + return 0; +} + +static int ppc_md5_final(struct shash_desc *desc, u8 *out) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + const unsigned int offset = sctx->byte_count & 0x3f; + const u8 *src = (const u8 *)sctx->block; + u8 *p = (u8 *)src + offset; + int padlen = 55 - offset; + __le64 *pbits = (__le64 *)((char *)sctx->block + 56); + __le32 *dst = (__le32 *)out; + + *p++ = 0x80; + + if (padlen < 0) { + memset(p, 0x00, padlen + sizeof (u64)); + ppc_md5_transform(sctx->hash, src, 1); + p = (char *)sctx->block; + padlen = 56; + } + + memset(p, 0, padlen); + *pbits = cpu_to_le64(sctx->byte_count << 3); + ppc_md5_transform(sctx->hash, src, 1); + + dst[0] = cpu_to_le32(sctx->hash[0]); + dst[1] = cpu_to_le32(sctx->hash[1]); + dst[2] = cpu_to_le32(sctx->hash[2]); + dst[3] = cpu_to_le32(sctx->hash[3]); + + ppc_md5_clear_context(sctx); + return 0; +} + +static int ppc_md5_export(struct shash_desc *desc, void *out) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int ppc_md5_import(struct shash_desc *desc, const void *in) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + +static struct shash_alg alg = { + .digestsize = MD5_DIGEST_SIZE, + .init = ppc_md5_init, + .update = ppc_md5_update, + .final = ppc_md5_final, + .export = ppc_md5_export, + .import = ppc_md5_import, + .descsize = sizeof(struct md5_state), + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name= "md5-ppc", + .cra_priority = 200, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init ppc_md5_mod_init(void) +{ + return crypto_register_shash(&alg); +} + +static void __exit ppc_md5_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(ppc_md5_mod_init); +module_exit(ppc_md5_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, PPC assembler"); + +MODULE_ALIAS_CRYPTO("md5"); +MODULE_ALIAS_CRYPTO("md5-ppc"); diff --git a/arch/powerpc/crypto/sha1-spe-asm.S b/arch/powerpc/crypto/sha1-spe-asm.S new file mode 100644 index 000000000000..fcb6cf002889 --- /dev/null +++ b/arch/powerpc/crypto/sha1-spe-asm.S @@ -0,0 +1,299 @@ +/* + * Fast SHA-1 implementation for SPE instruction set (PPC) + * + * This code makes use of the SPE SIMD instruction set as defined in + * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf + * Implementation is based on optimization guide notes from + * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <asm/ppc_asm.h> +#include <asm/asm-offsets.h> + +#define rHP r3 /* pointer to hash value */ +#define rWP r4 /* pointer to input */ +#define rKP r5 /* pointer to constants */ + +#define rW0 r14 /* 64 bit round words */ +#define rW1 r15 +#define rW2 r16 +#define rW3 r17 +#define rW4 r18 +#define rW5 r19 +#define rW6 r20 +#define rW7 r21 + +#define rH0 r6 /* 32 bit hash values */ +#define rH1 r7 +#define rH2 r8 +#define rH3 r9 +#define rH4 r10 + +#define rT0 r22 /* 64 bit temporary */ +#define rT1 r0 /* 32 bit temporaries */ +#define rT2 r11 +#define rT3 r12 + +#define rK r23 /* 64 bit constant in volatile register */ + +#define LOAD_K01 + +#define LOAD_K11 \ + evlwwsplat rK,0(rKP); + +#define LOAD_K21 \ + evlwwsplat rK,4(rKP); + +#define LOAD_K31 \ + evlwwsplat rK,8(rKP); + +#define LOAD_K41 \ + evlwwsplat rK,12(rKP); + +#define INITIALIZE \ + stwu r1,-128(r1); /* create stack frame */ \ + evstdw r14,8(r1); /* We must save non volatile */ \ + evstdw r15,16(r1); /* registers. Take the chance */ \ + evstdw r16,24(r1); /* and save the SPE part too */ \ + evstdw r17,32(r1); \ + evstdw r18,40(r1); \ + evstdw r19,48(r1); \ + evstdw r20,56(r1); \ + evstdw r21,64(r1); \ + evstdw r22,72(r1); \ + evstdw r23,80(r1); + + +#define FINALIZE \ + evldw r14,8(r1); /* restore SPE registers */ \ + evldw r15,16(r1); \ + evldw r16,24(r1); \ + evldw r17,32(r1); \ + evldw r18,40(r1); \ + evldw r19,48(r1); \ + evldw r20,56(r1); \ + evldw r21,64(r1); \ + evldw r22,72(r1); \ + evldw r23,80(r1); \ + xor r0,r0,r0; \ + stw r0,8(r1); /* Delete sensitive data */ \ + stw r0,16(r1); /* that we might have pushed */ \ + stw r0,24(r1); /* from other context that runs */ \ + stw r0,32(r1); /* the same code. Assume that */ \ + stw r0,40(r1); /* the lower part of the GPRs */ \ + stw r0,48(r1); /* were already overwritten on */ \ + stw r0,56(r1); /* the way down to here */ \ + stw r0,64(r1); \ + stw r0,72(r1); \ + stw r0,80(r1); \ + addi r1,r1,128; /* cleanup stack frame */ + +#ifdef __BIG_ENDIAN__ +#define LOAD_DATA(reg, off) \ + lwz reg,off(rWP); /* load data */ +#define NEXT_BLOCK \ + addi rWP,rWP,64; /* increment per block */ +#else +#define LOAD_DATA(reg, off) \ + lwbrx reg,0,rWP; /* load data */ \ + addi rWP,rWP,4; /* increment per word */ +#define NEXT_BLOCK /* nothing to do */ +#endif + +#define R_00_15(a, b, c, d, e, w0, w1, k, off) \ + LOAD_DATA(w0, off) /* 1: W */ \ + and rT2,b,c; /* 1: F' = B and C */ \ + LOAD_K##k##1 \ + andc rT1,d,b; /* 1: F" = ~B and D */ \ + rotrwi rT0,a,27; /* 1: A' = A rotl 5 */ \ + or rT2,rT2,rT1; /* 1: F = F' or F" */ \ + add e,e,rT0; /* 1: E = E + A' */ \ + rotrwi b,b,2; /* 1: B = B rotl 30 */ \ + add e,e,w0; /* 1: E = E + W */ \ + LOAD_DATA(w1, off+4) /* 2: W */ \ + add e,e,rT2; /* 1: E = E + F */ \ + and rT1,a,b; /* 2: F' = B and C */ \ + add e,e,rK; /* 1: E = E + K */ \ + andc rT2,c,a; /* 2: F" = ~B and D */ \ + add d,d,rK; /* 2: E = E + K */ \ + or rT2,rT2,rT1; /* 2: F = F' or F" */ \ + rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ + add d,d,w1; /* 2: E = E + W */ \ + rotrwi a,a,2; /* 2: B = B rotl 30 */ \ + add d,d,rT0; /* 2: E = E + A' */ \ + evmergelo w1,w1,w0; /* mix W[0]/W[1] */ \ + add d,d,rT2 /* 2: E = E + F */ + +#define R_16_19(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ + and rT2,b,c; /* 1: F' = B and C */ \ + evmergelohi rT0,w7,w6; /* W[-3] */ \ + andc rT1,d,b; /* 1: F" = ~B and D */ \ + evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \ + or rT1,rT1,rT2; /* 1: F = F' or F" */ \ + evxor w0,w0,w4; /* W = W xor W[-8] */ \ + add e,e,rT1; /* 1: E = E + F */ \ + evxor w0,w0,w1; /* W = W xor W[-14] */ \ + rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \ + evrlwi w0,w0,1; /* W = W rotl 1 */ \ + add e,e,rT2; /* 1: E = E + A' */ \ + evaddw rT0,w0,rK; /* WK = W + K */ \ + rotrwi b,b,2; /* 1: B = B rotl 30 */ \ + LOAD_K##k##1 \ + evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \ + add e,e,rT0; /* 1: E = E + WK */ \ + add d,d,rT1; /* 2: E = E + WK */ \ + and rT2,a,b; /* 2: F' = B and C */ \ + andc rT1,c,a; /* 2: F" = ~B and D */ \ + rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ + or rT1,rT1,rT2; /* 2: F = F' or F" */ \ + add d,d,rT0; /* 2: E = E + A' */ \ + rotrwi a,a,2; /* 2: B = B rotl 30 */ \ + add d,d,rT1 /* 2: E = E + F */ + +#define R_20_39(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ + evmergelohi rT0,w7,w6; /* W[-3] */ \ + xor rT2,b,c; /* 1: F' = B xor C */ \ + evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \ + xor rT2,rT2,d; /* 1: F = F' xor D */ \ + evxor w0,w0,w4; /* W = W xor W[-8] */ \ + add e,e,rT2; /* 1: E = E + F */ \ + evxor w0,w0,w1; /* W = W xor W[-14] */ \ + rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \ + evrlwi w0,w0,1; /* W = W rotl 1 */ \ + add e,e,rT2; /* 1: E = E + A' */ \ + evaddw rT0,w0,rK; /* WK = W + K */ \ + rotrwi b,b,2; /* 1: B = B rotl 30 */ \ + LOAD_K##k##1 \ + evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \ + add e,e,rT0; /* 1: E = E + WK */ \ + xor rT2,a,b; /* 2: F' = B xor C */ \ + add d,d,rT1; /* 2: E = E + WK */ \ + xor rT2,rT2,c; /* 2: F = F' xor D */ \ + rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ + add d,d,rT2; /* 2: E = E + F */ \ + rotrwi a,a,2; /* 2: B = B rotl 30 */ \ + add d,d,rT0 /* 2: E = E + A' */ + +#define R_40_59(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ + and rT2,b,c; /* 1: F' = B and C */ \ + evmergelohi rT0,w7,w6; /* W[-3] */ \ + or rT1,b,c; /* 1: F" = B or C */ \ + evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \ + and rT1,d,rT1; /* 1: F" = F" and D */ \ + evxor w0,w0,w4; /* W = W xor W[-8] */ \ + or rT2,rT2,rT1; /* 1: F = F' or F" */ \ + evxor w0,w0,w1; /* W = W xor W[-14] */ \ + add e,e,rT2; /* 1: E = E + F */ \ + evrlwi w0,w0,1; /* W = W rotl 1 */ \ + rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \ + evaddw rT0,w0,rK; /* WK = W + K */ \ + add e,e,rT2; /* 1: E = E + A' */ \ + LOAD_K##k##1 \ + evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \ + rotrwi b,b,2; /* 1: B = B rotl 30 */ \ + add e,e,rT0; /* 1: E = E + WK */ \ + and rT2,a,b; /* 2: F' = B and C */ \ + or rT0,a,b; /* 2: F" = B or C */ \ + add d,d,rT1; /* 2: E = E + WK */ \ + and rT0,c,rT0; /* 2: F" = F" and D */ \ + rotrwi a,a,2; /* 2: B = B rotl 30 */ \ + or rT2,rT2,rT0; /* 2: F = F' or F" */ \ + rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ + add d,d,rT2; /* 2: E = E + F */ \ + add d,d,rT0 /* 2: E = E + A' */ + +#define R_60_79(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ + R_20_39(a, b, c, d, e, w0, w1, w4, w6, w7, k) + +_GLOBAL(ppc_spe_sha1_transform) + INITIALIZE + + lwz rH0,0(rHP) + lwz rH1,4(rHP) + mtctr r5 + lwz rH2,8(rHP) + lis rKP,PPC_SPE_SHA1_K@h + lwz rH3,12(rHP) + ori rKP,rKP,PPC_SPE_SHA1_K@l + lwz rH4,16(rHP) + +ppc_spe_sha1_main: + R_00_15(rH0, rH1, rH2, rH3, rH4, rW1, rW0, 1, 0) + R_00_15(rH3, rH4, rH0, rH1, rH2, rW2, rW1, 0, 8) + R_00_15(rH1, rH2, rH3, rH4, rH0, rW3, rW2, 0, 16) + R_00_15(rH4, rH0, rH1, rH2, rH3, rW4, rW3, 0, 24) + R_00_15(rH2, rH3, rH4, rH0, rH1, rW5, rW4, 0, 32) + R_00_15(rH0, rH1, rH2, rH3, rH4, rW6, rW5, 0, 40) + R_00_15(rH3, rH4, rH0, rH1, rH2, rT3, rW6, 0, 48) + R_00_15(rH1, rH2, rH3, rH4, rH0, rT3, rW7, 0, 56) + + R_16_19(rH4, rH0, rH1, rH2, rH3, rW0, rW1, rW4, rW6, rW7, 0) + R_16_19(rH2, rH3, rH4, rH0, rH1, rW1, rW2, rW5, rW7, rW0, 2) + + R_20_39(rH0, rH1, rH2, rH3, rH4, rW2, rW3, rW6, rW0, rW1, 0) + R_20_39(rH3, rH4, rH0, rH1, rH2, rW3, rW4, rW7, rW1, rW2, 0) + R_20_39(rH1, rH2, rH3, rH4, rH0, rW4, rW5, rW0, rW2, rW3, 0) + R_20_39(rH4, rH0, rH1, rH2, rH3, rW5, rW6, rW1, rW3, rW4, 0) + R_20_39(rH2, rH3, rH4, rH0, rH1, rW6, rW7, rW2, rW4, rW5, 0) + R_20_39(rH0, rH1, rH2, rH3, rH4, rW7, rW0, rW3, rW5, rW6, 0) + R_20_39(rH3, rH4, rH0, rH1, rH2, rW0, rW1, rW4, rW6, rW7, 0) + R_20_39(rH1, rH2, rH3, rH4, rH0, rW1, rW2, rW5, rW7, rW0, 0) + R_20_39(rH4, rH0, rH1, rH2, rH3, rW2, rW3, rW6, rW0, rW1, 0) + R_20_39(rH2, rH3, rH4, rH0, rH1, rW3, rW4, rW7, rW1, rW2, 3) + + R_40_59(rH0, rH1, rH2, rH3, rH4, rW4, rW5, rW0, rW2, rW3, 0) + R_40_59(rH3, rH4, rH0, rH1, rH2, rW5, rW6, rW1, rW3, rW4, 0) + R_40_59(rH1, rH2, rH3, rH4, rH0, rW6, rW7, rW2, rW4, rW5, 0) + R_40_59(rH4, rH0, rH1, rH2, rH3, rW7, rW0, rW3, rW5, rW6, 0) + R_40_59(rH2, rH3, rH4, rH0, rH1, rW0, rW1, rW4, rW6, rW7, 0) + R_40_59(rH0, rH1, rH2, rH3, rH4, rW1, rW2, rW5, rW7, rW0, 0) + R_40_59(rH3, rH4, rH0, rH1, rH2, rW2, rW3, rW6, rW0, rW1, 0) + R_40_59(rH1, rH2, rH3, rH4, rH0, rW3, rW4, rW7, rW1, rW2, 0) + R_40_59(rH4, rH0, rH1, rH2, rH3, rW4, rW5, rW0, rW2, rW3, 0) + R_40_59(rH2, rH3, rH4, rH0, rH1, rW5, rW6, rW1, rW3, rW4, 4) + + R_60_79(rH0, rH1, rH2, rH3, rH4, rW6, rW7, rW2, rW4, rW5, 0) + R_60_79(rH3, rH4, rH0, rH1, rH2, rW7, rW0, rW3, rW5, rW6, 0) + R_60_79(rH1, rH2, rH3, rH4, rH0, rW0, rW1, rW4, rW6, rW7, 0) + R_60_79(rH4, rH0, rH1, rH2, rH3, rW1, rW2, rW5, rW7, rW0, 0) + R_60_79(rH2, rH3, rH4, rH0, rH1, rW2, rW3, rW6, rW0, rW1, 0) + R_60_79(rH0, rH1, rH2, rH3, rH4, rW3, rW4, rW7, rW1, rW2, 0) + R_60_79(rH3, rH4, rH0, rH1, rH2, rW4, rW5, rW0, rW2, rW3, 0) + lwz rT3,0(rHP) + R_60_79(rH1, rH2, rH3, rH4, rH0, rW5, rW6, rW1, rW3, rW4, 0) + lwz rW1,4(rHP) + R_60_79(rH4, rH0, rH1, rH2, rH3, rW6, rW7, rW2, rW4, rW5, 0) + lwz rW2,8(rHP) + R_60_79(rH2, rH3, rH4, rH0, rH1, rW7, rW0, rW3, rW5, rW6, 0) + lwz rW3,12(rHP) + NEXT_BLOCK + lwz rW4,16(rHP) + + add rH0,rH0,rT3 + stw rH0,0(rHP) + add rH1,rH1,rW1 + stw rH1,4(rHP) + add rH2,rH2,rW2 + stw rH2,8(rHP) + add rH3,rH3,rW3 + stw rH3,12(rHP) + add rH4,rH4,rW4 + stw rH4,16(rHP) + + bdnz ppc_spe_sha1_main + + FINALIZE + blr + +.data +.align 4 +PPC_SPE_SHA1_K: + .long 0x5A827999,0x6ED9EBA1,0x8F1BBCDC,0xCA62C1D6 diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c new file mode 100644 index 000000000000..3e1d22212521 --- /dev/null +++ b/arch/powerpc/crypto/sha1-spe-glue.c @@ -0,0 +1,210 @@ +/* + * Glue code for SHA-1 implementation for SPE instructions (PPC) + * + * Based on generic implementation. + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <crypto/sha.h> +#include <asm/byteorder.h> +#include <asm/switch_to.h> +#include <linux/hardirq.h> + +/* + * MAX_BYTES defines the number of bytes that are allowed to be processed + * between preempt_disable() and preempt_enable(). SHA1 takes ~1000 + * operations per 64 bytes. e500 cores can issue two arithmetic instructions + * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). + * Thus 2KB of input data will need an estimated maximum of 18,000 cycles. + * Headroom for cache misses included. Even with the low end model clocked + * at 667 MHz this equals to a critical time window of less than 27us. + * + */ +#define MAX_BYTES 2048 + +extern void ppc_spe_sha1_transform(u32 *state, const u8 *src, u32 blocks); + +static void spe_begin(void) +{ + /* We just start SPE operations and will save SPE registers later. */ + preempt_disable(); + enable_kernel_spe(); +} + +static void spe_end(void) +{ + /* reenable preemption */ + preempt_enable(); +} + +static inline void ppc_sha1_clear_context(struct sha1_state *sctx) +{ + int count = sizeof(struct sha1_state) >> 2; + u32 *ptr = (u32 *)sctx; + + /* make sure we can clear the fast way */ + BUILD_BUG_ON(sizeof(struct sha1_state) % 4); + do { *ptr++ = 0; } while (--count); +} + +static int ppc_spe_sha1_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA1_H0; + sctx->state[1] = SHA1_H1; + sctx->state[2] = SHA1_H2; + sctx->state[3] = SHA1_H3; + sctx->state[4] = SHA1_H4; + sctx->count = 0; + + return 0; +} + +static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + const unsigned int offset = sctx->count & 0x3f; + const unsigned int avail = 64 - offset; + unsigned int bytes; + const u8 *src = data; + + if (avail > len) { + sctx->count += len; + memcpy((char *)sctx->buffer + offset, src, len); + return 0; + } + + sctx->count += len; + + if (offset) { + memcpy((char *)sctx->buffer + offset, src, avail); + + spe_begin(); + ppc_spe_sha1_transform(sctx->state, (const u8 *)sctx->buffer, 1); + spe_end(); + + len -= avail; + src += avail; + } + + while (len > 63) { + bytes = (len > MAX_BYTES) ? MAX_BYTES : len; + bytes = bytes & ~0x3f; + + spe_begin(); + ppc_spe_sha1_transform(sctx->state, src, bytes >> 6); + spe_end(); + + src += bytes; + len -= bytes; + }; + + memcpy((char *)sctx->buffer, src, len); + return 0; +} + +static int ppc_spe_sha1_final(struct shash_desc *desc, u8 *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + const unsigned int offset = sctx->count & 0x3f; + char *p = (char *)sctx->buffer + offset; + int padlen; + __be64 *pbits = (__be64 *)(((char *)&sctx->buffer) + 56); + __be32 *dst = (__be32 *)out; + + padlen = 55 - offset; + *p++ = 0x80; + + spe_begin(); + + if (padlen < 0) { + memset(p, 0x00, padlen + sizeof (u64)); + ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); + p = (char *)sctx->buffer; + padlen = 56; + } + + memset(p, 0, padlen); + *pbits = cpu_to_be64(sctx->count << 3); + ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); + + spe_end(); + + dst[0] = cpu_to_be32(sctx->state[0]); + dst[1] = cpu_to_be32(sctx->state[1]); + dst[2] = cpu_to_be32(sctx->state[2]); + dst[3] = cpu_to_be32(sctx->state[3]); + dst[4] = cpu_to_be32(sctx->state[4]); + + ppc_sha1_clear_context(sctx); + return 0; +} + +static int ppc_spe_sha1_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + +static struct shash_alg alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = ppc_spe_sha1_init, + .update = ppc_spe_sha1_update, + .final = ppc_spe_sha1_final, + .export = ppc_spe_sha1_export, + .import = ppc_spe_sha1_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name= "sha1-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init ppc_spe_sha1_mod_init(void) +{ + return crypto_register_shash(&alg); +} + +static void __exit ppc_spe_sha1_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(ppc_spe_sha1_mod_init); +module_exit(ppc_spe_sha1_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, SPE optimized"); + +MODULE_ALIAS_CRYPTO("sha1"); +MODULE_ALIAS_CRYPTO("sha1-ppc-spe"); diff --git a/arch/powerpc/crypto/sha256-spe-asm.S b/arch/powerpc/crypto/sha256-spe-asm.S new file mode 100644 index 000000000000..2d10e4c08f03 --- /dev/null +++ b/arch/powerpc/crypto/sha256-spe-asm.S @@ -0,0 +1,323 @@ +/* + * Fast SHA-256 implementation for SPE instruction set (PPC) + * + * This code makes use of the SPE SIMD instruction set as defined in + * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf + * Implementation is based on optimization guide notes from + * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <asm/ppc_asm.h> +#include <asm/asm-offsets.h> + +#define rHP r3 /* pointer to hash values in memory */ +#define rKP r24 /* pointer to round constants */ +#define rWP r4 /* pointer to input data */ + +#define rH0 r5 /* 8 32 bit hash values in 8 registers */ +#define rH1 r6 +#define rH2 r7 +#define rH3 r8 +#define rH4 r9 +#define rH5 r10 +#define rH6 r11 +#define rH7 r12 + +#define rW0 r14 /* 64 bit registers. 16 words in 8 registers */ +#define rW1 r15 +#define rW2 r16 +#define rW3 r17 +#define rW4 r18 +#define rW5 r19 +#define rW6 r20 +#define rW7 r21 + +#define rT0 r22 /* 64 bit temporaries */ +#define rT1 r23 +#define rT2 r0 /* 32 bit temporaries */ +#define rT3 r25 + +#define CMP_KN_LOOP +#define CMP_KC_LOOP \ + cmpwi rT1,0; + +#define INITIALIZE \ + stwu r1,-128(r1); /* create stack frame */ \ + evstdw r14,8(r1); /* We must save non volatile */ \ + evstdw r15,16(r1); /* registers. Take the chance */ \ + evstdw r16,24(r1); /* and save the SPE part too */ \ + evstdw r17,32(r1); \ + evstdw r18,40(r1); \ + evstdw r19,48(r1); \ + evstdw r20,56(r1); \ + evstdw r21,64(r1); \ + evstdw r22,72(r1); \ + evstdw r23,80(r1); \ + stw r24,88(r1); /* save normal registers */ \ + stw r25,92(r1); + + +#define FINALIZE \ + evldw r14,8(r1); /* restore SPE registers */ \ + evldw r15,16(r1); \ + evldw r16,24(r1); \ + evldw r17,32(r1); \ + evldw r18,40(r1); \ + evldw r19,48(r1); \ + evldw r20,56(r1); \ + evldw r21,64(r1); \ + evldw r22,72(r1); \ + evldw r23,80(r1); \ + lwz r24,88(r1); /* restore normal registers */ \ + lwz r25,92(r1); \ + xor r0,r0,r0; \ + stw r0,8(r1); /* Delete sensitive data */ \ + stw r0,16(r1); /* that we might have pushed */ \ + stw r0,24(r1); /* from other context that runs */ \ + stw r0,32(r1); /* the same code. Assume that */ \ + stw r0,40(r1); /* the lower part of the GPRs */ \ + stw r0,48(r1); /* was already overwritten on */ \ + stw r0,56(r1); /* the way down to here */ \ + stw r0,64(r1); \ + stw r0,72(r1); \ + stw r0,80(r1); \ + addi r1,r1,128; /* cleanup stack frame */ + +#ifdef __BIG_ENDIAN__ +#define LOAD_DATA(reg, off) \ + lwz reg,off(rWP); /* load data */ +#define NEXT_BLOCK \ + addi rWP,rWP,64; /* increment per block */ +#else +#define LOAD_DATA(reg, off) \ + lwbrx reg,0,rWP; /* load data */ \ + addi rWP,rWP,4; /* increment per word */ +#define NEXT_BLOCK /* nothing to do */ +#endif + +#define R_LOAD_W(a, b, c, d, e, f, g, h, w, off) \ + LOAD_DATA(w, off) /* 1: W */ \ + rotrwi rT0,e,6; /* 1: S1 = e rotr 6 */ \ + rotrwi rT1,e,11; /* 1: S1' = e rotr 11 */ \ + rotrwi rT2,e,25; /* 1: S1" = e rotr 25 */ \ + xor rT0,rT0,rT1; /* 1: S1 = S1 xor S1' */ \ + and rT3,e,f; /* 1: ch = e and f */ \ + xor rT0,rT0,rT2; /* 1: S1 = S1 xor S1" */ \ + andc rT1,g,e; /* 1: ch' = ~e and g */ \ + lwz rT2,off(rKP); /* 1: K */ \ + xor rT3,rT3,rT1; /* 1: ch = ch xor ch' */ \ + add h,h,rT0; /* 1: temp1 = h + S1 */ \ + add rT3,rT3,w; /* 1: temp1' = ch + w */ \ + rotrwi rT0,a,2; /* 1: S0 = a rotr 2 */ \ + add h,h,rT3; /* 1: temp1 = temp1 + temp1' */ \ + rotrwi rT1,a,13; /* 1: S0' = a rotr 13 */ \ + add h,h,rT2; /* 1: temp1 = temp1 + K */ \ + rotrwi rT3,a,22; /* 1: S0" = a rotr 22 */ \ + xor rT0,rT0,rT1; /* 1: S0 = S0 xor S0' */ \ + add d,d,h; /* 1: d = d + temp1 */ \ + xor rT3,rT0,rT3; /* 1: S0 = S0 xor S0" */ \ + evmergelo w,w,w; /* shift W */ \ + or rT2,a,b; /* 1: maj = a or b */ \ + and rT1,a,b; /* 1: maj' = a and b */ \ + and rT2,rT2,c; /* 1: maj = maj and c */ \ + LOAD_DATA(w, off+4) /* 2: W */ \ + or rT2,rT1,rT2; /* 1: maj = maj or maj' */ \ + rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ + add rT3,rT3,rT2; /* 1: temp2 = S0 + maj */ \ + rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ + add h,h,rT3; /* 1: h = temp1 + temp2 */ \ + rotrwi rT2,d,25; /* 2: S1" = e rotr 25 */ \ + xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ + and rT3,d,e; /* 2: ch = e and f */ \ + xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ + andc rT1,f,d; /* 2: ch' = ~e and g */ \ + lwz rT2,off+4(rKP); /* 2: K */ \ + xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ + add g,g,rT0; /* 2: temp1 = h + S1 */ \ + add rT3,rT3,w; /* 2: temp1' = ch + w */ \ + rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ + add g,g,rT3; /* 2: temp1 = temp1 + temp1' */ \ + rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ + add g,g,rT2; /* 2: temp1 = temp1 + K */ \ + rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ + xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ + or rT2,h,a; /* 2: maj = a or b */ \ + xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ + and rT1,h,a; /* 2: maj' = a and b */ \ + and rT2,rT2,b; /* 2: maj = maj and c */ \ + add c,c,g; /* 2: d = d + temp1 */ \ + or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ + add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ + add g,g,rT3 /* 2: h = temp1 + temp2 */ + +#define R_CALC_W(a, b, c, d, e, f, g, h, w0, w1, w4, w5, w7, k, off) \ + rotrwi rT2,e,6; /* 1: S1 = e rotr 6 */ \ + evmergelohi rT0,w0,w1; /* w[-15] */ \ + rotrwi rT3,e,11; /* 1: S1' = e rotr 11 */ \ + evsrwiu rT1,rT0,3; /* s0 = w[-15] >> 3 */ \ + xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ + evrlwi rT0,rT0,25; /* s0' = w[-15] rotr 7 */ \ + rotrwi rT3,e,25; /* 1: S1' = e rotr 25 */ \ + evxor rT1,rT1,rT0; /* s0 = s0 xor s0' */ \ + xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ + evrlwi rT0,rT0,21; /* s0' = w[-15] rotr 18 */ \ + add h,h,rT2; /* 1: temp1 = h + S1 */ \ + evxor rT0,rT0,rT1; /* s0 = s0 xor s0' */ \ + and rT2,e,f; /* 1: ch = e and f */ \ + evaddw w0,w0,rT0; /* w = w[-16] + s0 */ \ + andc rT3,g,e; /* 1: ch' = ~e and g */ \ + evsrwiu rT0,w7,10; /* s1 = w[-2] >> 10 */ \ + xor rT2,rT2,rT3; /* 1: ch = ch xor ch' */ \ + evrlwi rT1,w7,15; /* s1' = w[-2] rotr 17 */ \ + add h,h,rT2; /* 1: temp1 = temp1 + ch */ \ + evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ + rotrwi rT2,a,2; /* 1: S0 = a rotr 2 */ \ + evrlwi rT1,w7,13; /* s1' = w[-2] rotr 19 */ \ + rotrwi rT3,a,13; /* 1: S0' = a rotr 13 */ \ + evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ + xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ + evldw rT1,off(rKP); /* k */ \ + rotrwi rT3,a,22; /* 1: S0' = a rotr 22 */ \ + evaddw w0,w0,rT0; /* w = w + s1 */ \ + xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ + evmergelohi rT0,w4,w5; /* w[-7] */ \ + and rT3,a,b; /* 1: maj = a and b */ \ + evaddw w0,w0,rT0; /* w = w + w[-7] */ \ + CMP_K##k##_LOOP \ + add rT2,rT2,rT3; /* 1: temp2 = S0 + maj */ \ + evaddw rT1,rT1,w0; /* wk = w + k */ \ + xor rT3,a,b; /* 1: maj = a xor b */ \ + evmergehi rT0,rT1,rT1; /* wk1/wk2 */ \ + and rT3,rT3,c; /* 1: maj = maj and c */ \ + add h,h,rT0; /* 1: temp1 = temp1 + wk */ \ + add rT2,rT2,rT3; /* 1: temp2 = temp2 + maj */ \ + add g,g,rT1; /* 2: temp1 = temp1 + wk */ \ + add d,d,h; /* 1: d = d + temp1 */ \ + rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ + add h,h,rT2; /* 1: h = temp1 + temp2 */ \ + rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ + rotrwi rT2,d,25; /* 2: S" = e rotr 25 */ \ + xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ + and rT3,d,e; /* 2: ch = e and f */ \ + xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ + andc rT1,f,d; /* 2: ch' = ~e and g */ \ + add g,g,rT0; /* 2: temp1 = h + S1 */ \ + xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ + rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ + add g,g,rT3; /* 2: temp1 = temp1 + ch */ \ + rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ + rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ + xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ + or rT2,h,a; /* 2: maj = a or b */ \ + and rT1,h,a; /* 2: maj' = a and b */ \ + and rT2,rT2,b; /* 2: maj = maj and c */ \ + xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ + or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ + add c,c,g; /* 2: d = d + temp1 */ \ + add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ + add g,g,rT3 /* 2: h = temp1 + temp2 */ + +_GLOBAL(ppc_spe_sha256_transform) + INITIALIZE + + mtctr r5 + lwz rH0,0(rHP) + lwz rH1,4(rHP) + lwz rH2,8(rHP) + lwz rH3,12(rHP) + lwz rH4,16(rHP) + lwz rH5,20(rHP) + lwz rH6,24(rHP) + lwz rH7,28(rHP) + +ppc_spe_sha256_main: + lis rKP,PPC_SPE_SHA256_K@ha + addi rKP,rKP,PPC_SPE_SHA256_K@l + + R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW0, 0) + R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW1, 8) + R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW2, 16) + R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW3, 24) + R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW4, 32) + R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW5, 40) + R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW6, 48) + R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW7, 56) +ppc_spe_sha256_16_rounds: + addi rKP,rKP,64 + R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, + rW0, rW1, rW4, rW5, rW7, N, 0) + R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, + rW1, rW2, rW5, rW6, rW0, N, 8) + R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, + rW2, rW3, rW6, rW7, rW1, N, 16) + R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, + rW3, rW4, rW7, rW0, rW2, N, 24) + R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, + rW4, rW5, rW0, rW1, rW3, N, 32) + R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, + rW5, rW6, rW1, rW2, rW4, N, 40) + R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, + rW6, rW7, rW2, rW3, rW5, N, 48) + R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, + rW7, rW0, rW3, rW4, rW6, C, 56) + bt gt,ppc_spe_sha256_16_rounds + + lwz rW0,0(rHP) + NEXT_BLOCK + lwz rW1,4(rHP) + lwz rW2,8(rHP) + lwz rW3,12(rHP) + lwz rW4,16(rHP) + lwz rW5,20(rHP) + lwz rW6,24(rHP) + lwz rW7,28(rHP) + + add rH0,rH0,rW0 + stw rH0,0(rHP) + add rH1,rH1,rW1 + stw rH1,4(rHP) + add rH2,rH2,rW2 + stw rH2,8(rHP) + add rH3,rH3,rW3 + stw rH3,12(rHP) + add rH4,rH4,rW4 + stw rH4,16(rHP) + add rH5,rH5,rW5 + stw rH5,20(rHP) + add rH6,rH6,rW6 + stw rH6,24(rHP) + add rH7,rH7,rW7 + stw rH7,28(rHP) + + bdnz ppc_spe_sha256_main + + FINALIZE + blr + +.data +.align 5 +PPC_SPE_SHA256_K: + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c new file mode 100644 index 000000000000..f4a616fe1a82 --- /dev/null +++ b/arch/powerpc/crypto/sha256-spe-glue.c @@ -0,0 +1,275 @@ +/* + * Glue code for SHA-256 implementation for SPE instructions (PPC) + * + * Based on generic implementation. The assembler module takes care + * about the SPE registers so it can run from interrupt context. + * + * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <crypto/sha.h> +#include <asm/byteorder.h> +#include <asm/switch_to.h> +#include <linux/hardirq.h> + +/* + * MAX_BYTES defines the number of bytes that are allowed to be processed + * between preempt_disable() and preempt_enable(). SHA256 takes ~2,000 + * operations per 64 bytes. e500 cores can issue two arithmetic instructions + * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). + * Thus 1KB of input data will need an estimated maximum of 18,000 cycles. + * Headroom for cache misses included. Even with the low end model clocked + * at 667 MHz this equals to a critical time window of less than 27us. + * + */ +#define MAX_BYTES 1024 + +extern void ppc_spe_sha256_transform(u32 *state, const u8 *src, u32 blocks); + +static void spe_begin(void) +{ + /* We just start SPE operations and will save SPE registers later. */ + preempt_disable(); + enable_kernel_spe(); +} + +static void spe_end(void) +{ + /* reenable preemption */ + preempt_enable(); +} + +static inline void ppc_sha256_clear_context(struct sha256_state *sctx) +{ + int count = sizeof(struct sha256_state) >> 2; + u32 *ptr = (u32 *)sctx; + + /* make sure we can clear the fast way */ + BUILD_BUG_ON(sizeof(struct sha256_state) % 4); + do { *ptr++ = 0; } while (--count); +} + +static int ppc_spe_sha256_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; + + return 0; +} + +static int ppc_spe_sha224_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; + + return 0; +} + +static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + const unsigned int offset = sctx->count & 0x3f; + const unsigned int avail = 64 - offset; + unsigned int bytes; + const u8 *src = data; + + if (avail > len) { + sctx->count += len; + memcpy((char *)sctx->buf + offset, src, len); + return 0; + } + + sctx->count += len; + + if (offset) { + memcpy((char *)sctx->buf + offset, src, avail); + + spe_begin(); + ppc_spe_sha256_transform(sctx->state, (const u8 *)sctx->buf, 1); + spe_end(); + + len -= avail; + src += avail; + } + + while (len > 63) { + /* cut input data into smaller blocks */ + bytes = (len > MAX_BYTES) ? MAX_BYTES : len; + bytes = bytes & ~0x3f; + + spe_begin(); + ppc_spe_sha256_transform(sctx->state, src, bytes >> 6); + spe_end(); + + src += bytes; + len -= bytes; + }; + + memcpy((char *)sctx->buf, src, len); + return 0; +} + +static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + const unsigned int offset = sctx->count & 0x3f; + char *p = (char *)sctx->buf + offset; + int padlen; + __be64 *pbits = (__be64 *)(((char *)&sctx->buf) + 56); + __be32 *dst = (__be32 *)out; + + padlen = 55 - offset; + *p++ = 0x80; + + spe_begin(); + + if (padlen < 0) { + memset(p, 0x00, padlen + sizeof (u64)); + ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); + p = (char *)sctx->buf; + padlen = 56; + } + + memset(p, 0, padlen); + *pbits = cpu_to_be64(sctx->count << 3); + ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); + + spe_end(); + + dst[0] = cpu_to_be32(sctx->state[0]); + dst[1] = cpu_to_be32(sctx->state[1]); + dst[2] = cpu_to_be32(sctx->state[2]); + dst[3] = cpu_to_be32(sctx->state[3]); + dst[4] = cpu_to_be32(sctx->state[4]); + dst[5] = cpu_to_be32(sctx->state[5]); + dst[6] = cpu_to_be32(sctx->state[6]); + dst[7] = cpu_to_be32(sctx->state[7]); + + ppc_sha256_clear_context(sctx); + return 0; +} + +static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out) +{ + u32 D[SHA256_DIGEST_SIZE >> 2]; + __be32 *dst = (__be32 *)out; + + ppc_spe_sha256_final(desc, (u8 *)D); + + /* avoid bytewise memcpy */ + dst[0] = D[0]; + dst[1] = D[1]; + dst[2] = D[2]; + dst[3] = D[3]; + dst[4] = D[4]; + dst[5] = D[5]; + dst[6] = D[6]; + + /* clear sensitive data */ + memzero_explicit(D, SHA256_DIGEST_SIZE); + return 0; +} + +static int ppc_spe_sha256_export(struct shash_desc *desc, void *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + +static struct shash_alg algs[2] = { { + .digestsize = SHA256_DIGEST_SIZE, + .init = ppc_spe_sha256_init, + .update = ppc_spe_sha256_update, + .final = ppc_spe_sha256_final, + .export = ppc_spe_sha256_export, + .import = ppc_spe_sha256_import, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name= "sha256-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}, { + .digestsize = SHA224_DIGEST_SIZE, + .init = ppc_spe_sha224_init, + .update = ppc_spe_sha256_update, + .final = ppc_spe_sha224_final, + .export = ppc_spe_sha256_export, + .import = ppc_spe_sha256_import, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name= "sha224-ppc-spe", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +} }; + +static int __init ppc_spe_sha256_mod_init(void) +{ + return crypto_register_shashes(algs, ARRAY_SIZE(algs)); +} + +static void __exit ppc_spe_sha256_mod_fini(void) +{ + crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); +} + +module_init(ppc_spe_sha256_mod_init); +module_exit(ppc_spe_sha256_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, SPE optimized"); + +MODULE_ALIAS_CRYPTO("sha224"); +MODULE_ALIAS_CRYPTO("sha224-ppc-spe"); +MODULE_ALIAS_CRYPTO("sha256"); +MODULE_ALIAS_CRYPTO("sha256-ppc-spe"); diff --git a/arch/powerpc/include/asm/Kbuild b/arch/powerpc/include/asm/Kbuild index 382b28e364dc..4b87205c230c 100644 --- a/arch/powerpc/include/asm/Kbuild +++ b/arch/powerpc/include/asm/Kbuild @@ -1,6 +1,8 @@ - generic-y += clkdev.h +generic-y += div64.h +generic-y += irq_regs.h generic-y += irq_work.h +generic-y += local64.h generic-y += mcs_spinlock.h generic-y += preempt.h generic-y += rwsem.h diff --git a/arch/powerpc/include/asm/asm-compat.h b/arch/powerpc/include/asm/asm-compat.h index 21be8ae8f809..dc85dcb891cf 100644 --- a/arch/powerpc/include/asm/asm-compat.h +++ b/arch/powerpc/include/asm/asm-compat.h @@ -23,6 +23,8 @@ #define PPC_STL stringify_in_c(std) #define PPC_STLU stringify_in_c(stdu) #define PPC_LCMPI stringify_in_c(cmpdi) +#define PPC_LCMPLI stringify_in_c(cmpldi) +#define PPC_LCMP stringify_in_c(cmpd) #define PPC_LONG stringify_in_c(.llong) #define PPC_LONG_ALIGN stringify_in_c(.balign 8) #define PPC_TLNEI stringify_in_c(tdnei) @@ -52,6 +54,8 @@ #define PPC_STL stringify_in_c(stw) #define PPC_STLU stringify_in_c(stwu) #define PPC_LCMPI stringify_in_c(cmpwi) +#define PPC_LCMPLI stringify_in_c(cmplwi) +#define PPC_LCMP stringify_in_c(cmpw) #define PPC_LONG stringify_in_c(.long) #define PPC_LONG_ALIGN stringify_in_c(.balign 4) #define PPC_TLNEI stringify_in_c(twnei) diff --git a/arch/powerpc/include/asm/cache.h b/arch/powerpc/include/asm/cache.h index 34a05a1a990b..0dc42c5082b7 100644 --- a/arch/powerpc/include/asm/cache.h +++ b/arch/powerpc/include/asm/cache.h @@ -76,9 +76,6 @@ extern void _set_L3CR(unsigned long); #define _set_L3CR(val) do { } while(0) #endif -extern void cacheable_memzero(void *p, unsigned int nb); -extern void *cacheable_memcpy(void *, const void *, unsigned int); - #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_CACHE_H */ diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index 5cf5a6d10685..6367b8347dad 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -100,7 +100,7 @@ struct cpu_spec { /* * Processor specific routine to flush tlbs. */ - void (*flush_tlb)(unsigned long inval_selector); + void (*flush_tlb)(unsigned int action); }; @@ -114,6 +114,12 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start, extern const char *powerpc_base_platform; +/* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */ +enum { + TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ + TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */ +}; + #endif /* __ASSEMBLY__ */ /* CPU kernel features */ diff --git a/arch/powerpc/include/asm/dbdma.h b/arch/powerpc/include/asm/dbdma.h index e23f07e73cb3..6c69836b4ec2 100644 --- a/arch/powerpc/include/asm/dbdma.h +++ b/arch/powerpc/include/asm/dbdma.h @@ -42,12 +42,12 @@ struct dbdma_regs { * DBDMA command structure. These fields are all little-endian! */ struct dbdma_cmd { - unsigned short req_count; /* requested byte transfer count */ - unsigned short command; /* command word (has bit-fields) */ - unsigned int phy_addr; /* physical data address */ - unsigned int cmd_dep; /* command-dependent field */ - unsigned short res_count; /* residual count after completion */ - unsigned short xfer_status; /* transfer status */ + __le16 req_count; /* requested byte transfer count */ + __le16 command; /* command word (has bit-fields) */ + __le32 phy_addr; /* physical data address */ + __le32 cmd_dep; /* command-dependent field */ + __le16 res_count; /* residual count after completion */ + __le16 xfer_status; /* transfer status */ }; /* DBDMA command values in command field */ diff --git a/arch/powerpc/include/asm/dcr-native.h b/arch/powerpc/include/asm/dcr-native.h index 7d2e6235726d..4efc11dacb98 100644 --- a/arch/powerpc/include/asm/dcr-native.h +++ b/arch/powerpc/include/asm/dcr-native.h @@ -31,7 +31,7 @@ typedef struct { static inline bool dcr_map_ok_native(dcr_host_native_t host) { - return 1; + return true; } #define dcr_map_native(dev, dcr_n, dcr_c) \ diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h index 38faeded7d59..9f1371bab5fc 100644 --- a/arch/powerpc/include/asm/device.h +++ b/arch/powerpc/include/asm/device.h @@ -8,6 +8,9 @@ struct dma_map_ops; struct device_node; +#ifdef CONFIG_PPC64 +struct pci_dn; +#endif /* * Arch extensions to struct device. @@ -34,6 +37,9 @@ struct dev_archdata { #ifdef CONFIG_SWIOTLB dma_addr_t max_direct_dma_addr; #endif +#ifdef CONFIG_PPC64 + struct pci_dn *pci_data; +#endif #ifdef CONFIG_EEH struct eeh_dev *edev; #endif diff --git a/arch/powerpc/include/asm/div64.h b/arch/powerpc/include/asm/div64.h deleted file mode 100644 index 6cd978cefb28..000000000000 --- a/arch/powerpc/include/asm/div64.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/div64.h> diff --git a/arch/powerpc/include/asm/dma-mapping.h b/arch/powerpc/include/asm/dma-mapping.h index 894d538f3567..9103687b0436 100644 --- a/arch/powerpc/include/asm/dma-mapping.h +++ b/arch/powerpc/include/asm/dma-mapping.h @@ -191,11 +191,11 @@ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) struct dev_archdata *sd = &dev->archdata; if (sd->max_direct_dma_addr && addr + size > sd->max_direct_dma_addr) - return 0; + return false; #endif if (!dev->dma_mask) - return 0; + return false; return addr + size - 1 <= *dev->dma_mask; } diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 55abfd09e47f..a52db28ecc1e 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -29,7 +29,7 @@ struct pci_dev; struct pci_bus; -struct device_node; +struct pci_dn; #ifdef CONFIG_EEH @@ -136,14 +136,14 @@ struct eeh_dev { struct eeh_pe *pe; /* Associated PE */ struct list_head list; /* Form link list in the PE */ struct pci_controller *phb; /* Associated PHB */ - struct device_node *dn; /* Associated device node */ + struct pci_dn *pdn; /* Associated PCI device node */ struct pci_dev *pdev; /* Associated PCI device */ struct pci_bus *bus; /* PCI bus for partial hotplug */ }; -static inline struct device_node *eeh_dev_to_of_node(struct eeh_dev *edev) +static inline struct pci_dn *eeh_dev_to_pdn(struct eeh_dev *edev) { - return edev ? edev->dn : NULL; + return edev ? edev->pdn : NULL; } static inline struct pci_dev *eeh_dev_to_pci_dev(struct eeh_dev *edev) @@ -200,8 +200,7 @@ struct eeh_ops { char *name; int (*init)(void); int (*post_init)(void); - void* (*of_probe)(struct device_node *dn, void *flag); - int (*dev_probe)(struct pci_dev *dev, void *flag); + void* (*probe)(struct pci_dn *pdn, void *data); int (*set_option)(struct eeh_pe *pe, int option); int (*get_pe_addr)(struct eeh_pe *pe); int (*get_state)(struct eeh_pe *pe, int *state); @@ -211,10 +210,10 @@ struct eeh_ops { int (*configure_bridge)(struct eeh_pe *pe); int (*err_inject)(struct eeh_pe *pe, int type, int func, unsigned long addr, unsigned long mask); - int (*read_config)(struct device_node *dn, int where, int size, u32 *val); - int (*write_config)(struct device_node *dn, int where, int size, u32 val); + int (*read_config)(struct pci_dn *pdn, int where, int size, u32 *val); + int (*write_config)(struct pci_dn *pdn, int where, int size, u32 val); int (*next_error)(struct eeh_pe **pe); - int (*restore_config)(struct device_node *dn); + int (*restore_config)(struct pci_dn *pdn); }; extern int eeh_subsystem_flags; @@ -272,7 +271,7 @@ void eeh_pe_restore_bars(struct eeh_pe *pe); const char *eeh_pe_loc_get(struct eeh_pe *pe); struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe); -void *eeh_dev_init(struct device_node *dn, void *data); +void *eeh_dev_init(struct pci_dn *pdn, void *data); void eeh_dev_phb_init_dynamic(struct pci_controller *phb); int eeh_init(void); int __init eeh_ops_register(struct eeh_ops *ops); @@ -280,8 +279,8 @@ int __exit eeh_ops_unregister(const char *name); int eeh_check_failure(const volatile void __iomem *token); int eeh_dev_check_failure(struct eeh_dev *edev); void eeh_addr_cache_build(void); -void eeh_add_device_early(struct device_node *); -void eeh_add_device_tree_early(struct device_node *); +void eeh_add_device_early(struct pci_dn *); +void eeh_add_device_tree_early(struct pci_dn *); void eeh_add_device_late(struct pci_dev *); void eeh_add_device_tree_late(struct pci_bus *); void eeh_add_sysfs_files(struct pci_bus *); @@ -323,7 +322,7 @@ static inline int eeh_init(void) return 0; } -static inline void *eeh_dev_init(struct device_node *dn, void *data) +static inline void *eeh_dev_init(struct pci_dn *pdn, void *data) { return NULL; } @@ -339,9 +338,9 @@ static inline int eeh_check_failure(const volatile void __iomem *token) static inline void eeh_addr_cache_build(void) { } -static inline void eeh_add_device_early(struct device_node *dn) { } +static inline void eeh_add_device_early(struct pci_dn *pdn) { } -static inline void eeh_add_device_tree_early(struct device_node *dn) { } +static inline void eeh_add_device_tree_early(struct pci_dn *pdn) { } static inline void eeh_add_device_late(struct pci_dev *dev) { } diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index 681bc0314b6b..e05808a328db 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -42,7 +42,7 @@ #define FW_FEATURE_SPLPAR ASM_CONST(0x0000000000100000) #define FW_FEATURE_LPAR ASM_CONST(0x0000000000400000) #define FW_FEATURE_PS3_LV1 ASM_CONST(0x0000000000800000) -#define FW_FEATURE_BEAT ASM_CONST(0x0000000001000000) +/* Free ASM_CONST(0x0000000001000000) */ #define FW_FEATURE_CMO ASM_CONST(0x0000000002000000) #define FW_FEATURE_VPHN ASM_CONST(0x0000000004000000) #define FW_FEATURE_XCMO ASM_CONST(0x0000000008000000) @@ -75,8 +75,6 @@ enum { FW_FEATURE_POWERNV_ALWAYS = 0, FW_FEATURE_PS3_POSSIBLE = FW_FEATURE_LPAR | FW_FEATURE_PS3_LV1, FW_FEATURE_PS3_ALWAYS = FW_FEATURE_LPAR | FW_FEATURE_PS3_LV1, - FW_FEATURE_CELLEB_POSSIBLE = FW_FEATURE_LPAR | FW_FEATURE_BEAT, - FW_FEATURE_CELLEB_ALWAYS = 0, FW_FEATURE_NATIVE_POSSIBLE = 0, FW_FEATURE_NATIVE_ALWAYS = 0, FW_FEATURE_POSSIBLE = @@ -89,9 +87,6 @@ enum { #ifdef CONFIG_PPC_PS3 FW_FEATURE_PS3_POSSIBLE | #endif -#ifdef CONFIG_PPC_CELLEB - FW_FEATURE_CELLEB_POSSIBLE | -#endif #ifdef CONFIG_PPC_NATIVE FW_FEATURE_NATIVE_ALWAYS | #endif @@ -106,9 +101,6 @@ enum { #ifdef CONFIG_PPC_PS3 FW_FEATURE_PS3_ALWAYS & #endif -#ifdef CONFIG_PPC_CELLEB - FW_FEATURE_CELLEB_ALWAYS & -#endif #ifdef CONFIG_PPC_NATIVE FW_FEATURE_NATIVE_ALWAYS & #endif diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index f1ea5972f6ec..1e27d6338565 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -29,6 +29,7 @@ #include <linux/bitops.h> #include <asm/machdep.h> #include <asm/types.h> +#include <asm/pci-bridge.h> #define IOMMU_PAGE_SHIFT_4K 12 #define IOMMU_PAGE_SIZE_4K (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K) @@ -78,6 +79,9 @@ struct iommu_table { struct iommu_group *it_group; #endif void (*set_bypass)(struct iommu_table *tbl, bool enable); +#ifdef CONFIG_PPC_POWERNV + void *data; +#endif }; /* Pure 2^n version of get_order */ @@ -169,7 +173,7 @@ extern void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle, struct dma_attrs *attrs); extern void iommu_init_early_pSeries(void); -extern void iommu_init_early_dart(void); +extern void iommu_init_early_dart(struct pci_controller_ops *controller_ops); extern void iommu_init_early_pasemi(void); extern void alloc_dart_table(void); diff --git a/arch/powerpc/include/asm/irq_regs.h b/arch/powerpc/include/asm/irq_regs.h deleted file mode 100644 index ba94b51a0a70..000000000000 --- a/arch/powerpc/include/asm/irq_regs.h +++ /dev/null @@ -1,2 +0,0 @@ -#include <asm-generic/irq_regs.h> - diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h index 2d81e202bdcc..14619a59ec09 100644 --- a/arch/powerpc/include/asm/kvm_book3s_64.h +++ b/arch/powerpc/include/asm/kvm_book3s_64.h @@ -290,11 +290,11 @@ static inline pte_t kvmppc_read_update_linux_pte(pte_t *ptep, int writing, pte_t old_pte, new_pte = __pte(0); while (1) { - old_pte = pte_val(*ptep); + old_pte = *ptep; /* * wait until _PAGE_BUSY is clear then set it atomically */ - if (unlikely(old_pte & _PAGE_BUSY)) { + if (unlikely(pte_val(old_pte) & _PAGE_BUSY)) { cpu_relax(); continue; } @@ -305,16 +305,18 @@ static inline pte_t kvmppc_read_update_linux_pte(pte_t *ptep, int writing, return __pte(0); #endif /* If pte is not present return None */ - if (unlikely(!(old_pte & _PAGE_PRESENT))) + if (unlikely(!(pte_val(old_pte) & _PAGE_PRESENT))) return __pte(0); new_pte = pte_mkyoung(old_pte); if (writing && pte_write(old_pte)) new_pte = pte_mkdirty(new_pte); - if (old_pte == __cmpxchg_u64((unsigned long *)ptep, old_pte, - new_pte)) + if (pte_val(old_pte) == __cmpxchg_u64((unsigned long *)ptep, + pte_val(old_pte), + pte_val(new_pte))) { break; + } } return new_pte; } @@ -335,7 +337,7 @@ static inline bool hpte_read_permission(unsigned long pp, unsigned long key) { if (key) return PP_RWRX <= pp && pp <= PP_RXRX; - return 1; + return true; } static inline bool hpte_write_permission(unsigned long pp, unsigned long key) @@ -373,7 +375,7 @@ static inline bool slot_is_aligned(struct kvm_memory_slot *memslot, unsigned long mask = (pagesize >> PAGE_SHIFT) - 1; if (pagesize <= PAGE_SIZE) - return 1; + return true; return !(memslot->base_gfn & mask) && !(memslot->npages & mask); } diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 8ef05121d3cd..c610961720c7 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -585,7 +585,7 @@ struct kvm_vcpu_arch { pgd_t *pgdir; u8 io_gpr; /* GPR used as IO source/target */ - u8 mmio_is_bigendian; + u8 mmio_host_swabbed; u8 mmio_sign_extend; u8 osi_needed; u8 osi_enabled; diff --git a/arch/powerpc/include/asm/local64.h b/arch/powerpc/include/asm/local64.h deleted file mode 100644 index 36c93b5cc239..000000000000 --- a/arch/powerpc/include/asm/local64.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/local64.h> diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index c8175a3fe560..ef8899432ae7 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -103,9 +103,6 @@ struct machdep_calls { #endif #endif /* CONFIG_PPC64 */ - void (*pci_dma_dev_setup)(struct pci_dev *dev); - void (*pci_dma_bus_setup)(struct pci_bus *bus); - /* Platform set_dma_mask and dma_get_required_mask overrides */ int (*dma_set_mask)(struct device *dev, u64 dma_mask); u64 (*dma_get_required_mask)(struct device *dev); @@ -125,9 +122,8 @@ struct machdep_calls { unsigned int (*get_irq)(void); /* PCI stuff */ - /* Called after scanning the bus, before allocating resources */ + /* Called after allocating resources */ void (*pcibios_fixup)(void); - int (*pci_probe_mode)(struct pci_bus *); void (*pci_irq_fixup)(struct pci_dev *dev); int (*pcibios_root_bridge_prepare)(struct pci_host_bridge *bridge); @@ -237,18 +233,13 @@ struct machdep_calls { /* Called for each PCI bus in the system when it's probed */ void (*pcibios_fixup_bus)(struct pci_bus *); - /* Called when pci_enable_device() is called. Returns 0 to - * allow assignment/enabling of the device. */ - int (*pcibios_enable_device_hook)(struct pci_dev *); - /* Called after scan and before resource survey */ void (*pcibios_fixup_phb)(struct pci_controller *hose); - /* Called during PCI resource reassignment */ - resource_size_t (*pcibios_window_alignment)(struct pci_bus *, unsigned long type); - - /* Reset the secondary bus of bridge */ - void (*pcibios_reset_secondary_bus)(struct pci_dev *dev); +#ifdef CONFIG_PCI_IOV + void (*pcibios_fixup_sriov)(struct pci_dev *pdev); + resource_size_t (*pcibios_iov_resource_alignment)(struct pci_dev *, int resno); +#endif /* CONFIG_PCI_IOV */ /* Called to shutdown machine specific hardware not already controlled * by other drivers. diff --git a/arch/powerpc/include/asm/mmu-hash64.h b/arch/powerpc/include/asm/mmu-hash64.h index 4f13c3ed7acf..1da6a81ce541 100644 --- a/arch/powerpc/include/asm/mmu-hash64.h +++ b/arch/powerpc/include/asm/mmu-hash64.h @@ -112,6 +112,7 @@ #define TLBIEL_INVAL_SET_SHIFT 12 #define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */ +#define POWER8_TLB_SETS 512 /* # sets in POWER8 TLB */ #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/mpc85xx.h b/arch/powerpc/include/asm/mpc85xx.h index 3bef74a9914b..213f3a81593d 100644 --- a/arch/powerpc/include/asm/mpc85xx.h +++ b/arch/powerpc/include/asm/mpc85xx.h @@ -61,6 +61,7 @@ #define SVR_T4240 0x824000 #define SVR_T4120 0x824001 #define SVR_T4160 0x824100 +#define SVR_T4080 0x824102 #define SVR_C291 0x850000 #define SVR_C292 0x850020 #define SVR_C293 0x850030 diff --git a/arch/powerpc/include/asm/mpic.h b/arch/powerpc/include/asm/mpic.h index 754f93d208fa..98697611e7b3 100644 --- a/arch/powerpc/include/asm/mpic.h +++ b/arch/powerpc/include/asm/mpic.h @@ -34,10 +34,6 @@ #define MPIC_GREG_GCONF_BASE_MASK 0x000fffff #define MPIC_GREG_GCONF_MCK 0x08000000 #define MPIC_GREG_GLOBAL_CONF_1 0x00030 -#define MPIC_GREG_GLOBAL_CONF_1_SIE 0x08000000 -#define MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO_MASK 0x70000000 -#define MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO(r) \ - (((r) << 28) & MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO_MASK) #define MPIC_GREG_VENDOR_0 0x00040 #define MPIC_GREG_VENDOR_1 0x00050 #define MPIC_GREG_VENDOR_2 0x00060 @@ -396,14 +392,7 @@ extern struct bus_type mpic_subsys; #define MPIC_REGSET_TSI108 MPIC_REGSET(1) /* Tsi108/109 PIC */ /* Get the version of primary MPIC */ -#ifdef CONFIG_MPIC extern u32 fsl_mpic_primary_get_version(void); -#else -static inline u32 fsl_mpic_primary_get_version(void) -{ - return 0; -} -#endif /* Allocate the controller structure and setup the linux irq descs * for the range if interrupts passed in. No HW initialization is @@ -496,11 +485,5 @@ extern unsigned int mpic_get_coreint_irq(void); /* Fetch Machine Check interrupt from primary mpic */ extern unsigned int mpic_get_mcirq(void); -/* Set the EPIC clock ratio */ -void mpic_set_clk_ratio(struct mpic *mpic, u32 clock_ratio); - -/* Enable/Disable EPIC serial interrupt mode */ -void mpic_set_serial_int(struct mpic *mpic, int enable); - #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_MPIC_H */ diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h new file mode 100644 index 000000000000..ff1ccb375e60 --- /dev/null +++ b/arch/powerpc/include/asm/nmi.h @@ -0,0 +1,4 @@ +#ifndef _ASM_NMI_H +#define _ASM_NMI_H + +#endif /* _ASM_NMI_H */ diff --git a/arch/powerpc/include/asm/nvram.h b/arch/powerpc/include/asm/nvram.h index b0fe0fe4e626..09a518bb7c03 100644 --- a/arch/powerpc/include/asm/nvram.h +++ b/arch/powerpc/include/asm/nvram.h @@ -9,12 +9,43 @@ #ifndef _ASM_POWERPC_NVRAM_H #define _ASM_POWERPC_NVRAM_H - +#include <linux/types.h> #include <linux/errno.h> #include <linux/list.h> #include <uapi/asm/nvram.h> +/* + * Set oops header version to distinguish between old and new format header. + * lnx,oops-log partition max size is 4000, header version > 4000 will + * help in identifying new header. + */ +#define OOPS_HDR_VERSION 5000 + +struct err_log_info { + __be32 error_type; + __be32 seq_num; +}; + +struct nvram_os_partition { + const char *name; + int req_size; /* desired size, in bytes */ + int min_size; /* minimum acceptable size (0 means req_size) */ + long size; /* size of data portion (excluding err_log_info) */ + long index; /* offset of data portion of partition */ + bool os_partition; /* partition initialized by OS, not FW */ +}; + +struct oops_log_info { + __be16 version; + __be16 report_length; + __be64 timestamp; +} __attribute__((packed)); + +extern struct nvram_os_partition oops_log_partition; + #ifdef CONFIG_PPC_PSERIES +extern struct nvram_os_partition rtas_log_partition; + extern int nvram_write_error_log(char * buff, int length, unsigned int err_type, unsigned int err_seq); extern int nvram_read_error_log(char * buff, int length, @@ -50,6 +81,23 @@ extern void pmac_xpram_write(int xpaddr, u8 data); /* Synchronize NVRAM */ extern void nvram_sync(void); +/* Initialize NVRAM OS partition */ +extern int __init nvram_init_os_partition(struct nvram_os_partition *part); + +/* Initialize NVRAM oops partition */ +extern void __init nvram_init_oops_partition(int rtas_partition_exists); + +/* Read a NVRAM partition */ +extern int nvram_read_partition(struct nvram_os_partition *part, char *buff, + int length, unsigned int *err_type, + unsigned int *error_log_cnt); + +/* Write to NVRAM OS partition */ +extern int nvram_write_os_partition(struct nvram_os_partition *part, + char *buff, int length, + unsigned int err_type, + unsigned int error_log_cnt); + /* Determine NVRAM size */ extern ssize_t nvram_get_size(void); diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h new file mode 100644 index 000000000000..0321a909e663 --- /dev/null +++ b/arch/powerpc/include/asm/opal-api.h @@ -0,0 +1,735 @@ +/* + * OPAL API definitions. + * + * Copyright 2011-2015 IBM Corp. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#ifndef __OPAL_API_H +#define __OPAL_API_H + +/****** OPAL APIs ******/ + +/* Return codes */ +#define OPAL_SUCCESS 0 +#define OPAL_PARAMETER -1 +#define OPAL_BUSY -2 +#define OPAL_PARTIAL -3 +#define OPAL_CONSTRAINED -4 +#define OPAL_CLOSED -5 +#define OPAL_HARDWARE -6 +#define OPAL_UNSUPPORTED -7 +#define OPAL_PERMISSION -8 +#define OPAL_NO_MEM -9 +#define OPAL_RESOURCE -10 +#define OPAL_INTERNAL_ERROR -11 +#define OPAL_BUSY_EVENT -12 +#define OPAL_HARDWARE_FROZEN -13 +#define OPAL_WRONG_STATE -14 +#define OPAL_ASYNC_COMPLETION -15 +#define OPAL_EMPTY -16 +#define OPAL_I2C_TIMEOUT -17 +#define OPAL_I2C_INVALID_CMD -18 +#define OPAL_I2C_LBUS_PARITY -19 +#define OPAL_I2C_BKEND_OVERRUN -20 +#define OPAL_I2C_BKEND_ACCESS -21 +#define OPAL_I2C_ARBT_LOST -22 +#define OPAL_I2C_NACK_RCVD -23 +#define OPAL_I2C_STOP_ERR -24 + +/* API Tokens (in r0) */ +#define OPAL_INVALID_CALL -1 +#define OPAL_TEST 0 +#define OPAL_CONSOLE_WRITE 1 +#define OPAL_CONSOLE_READ 2 +#define OPAL_RTC_READ 3 +#define OPAL_RTC_WRITE 4 +#define OPAL_CEC_POWER_DOWN 5 +#define OPAL_CEC_REBOOT 6 +#define OPAL_READ_NVRAM 7 +#define OPAL_WRITE_NVRAM 8 +#define OPAL_HANDLE_INTERRUPT 9 +#define OPAL_POLL_EVENTS 10 +#define OPAL_PCI_SET_HUB_TCE_MEMORY 11 +#define OPAL_PCI_SET_PHB_TCE_MEMORY 12 +#define OPAL_PCI_CONFIG_READ_BYTE 13 +#define OPAL_PCI_CONFIG_READ_HALF_WORD 14 +#define OPAL_PCI_CONFIG_READ_WORD 15 +#define OPAL_PCI_CONFIG_WRITE_BYTE 16 +#define OPAL_PCI_CONFIG_WRITE_HALF_WORD 17 +#define OPAL_PCI_CONFIG_WRITE_WORD 18 +#define OPAL_SET_XIVE 19 +#define OPAL_GET_XIVE 20 +#define OPAL_GET_COMPLETION_TOKEN_STATUS 21 /* obsolete */ +#define OPAL_REGISTER_OPAL_EXCEPTION_HANDLER 22 +#define OPAL_PCI_EEH_FREEZE_STATUS 23 +#define OPAL_PCI_SHPC 24 +#define OPAL_CONSOLE_WRITE_BUFFER_SPACE 25 +#define OPAL_PCI_EEH_FREEZE_CLEAR 26 +#define OPAL_PCI_PHB_MMIO_ENABLE 27 +#define OPAL_PCI_SET_PHB_MEM_WINDOW 28 +#define OPAL_PCI_MAP_PE_MMIO_WINDOW 29 +#define OPAL_PCI_SET_PHB_TABLE_MEMORY 30 +#define OPAL_PCI_SET_PE 31 +#define OPAL_PCI_SET_PELTV 32 +#define OPAL_PCI_SET_MVE 33 +#define OPAL_PCI_SET_MVE_ENABLE 34 +#define OPAL_PCI_GET_XIVE_REISSUE 35 +#define OPAL_PCI_SET_XIVE_REISSUE 36 +#define OPAL_PCI_SET_XIVE_PE 37 +#define OPAL_GET_XIVE_SOURCE 38 +#define OPAL_GET_MSI_32 39 +#define OPAL_GET_MSI_64 40 +#define OPAL_START_CPU 41 +#define OPAL_QUERY_CPU_STATUS 42 +#define OPAL_WRITE_OPPANEL 43 /* unimplemented */ +#define OPAL_PCI_MAP_PE_DMA_WINDOW 44 +#define OPAL_PCI_MAP_PE_DMA_WINDOW_REAL 45 +#define OPAL_PCI_RESET 49 +#define OPAL_PCI_GET_HUB_DIAG_DATA 50 +#define OPAL_PCI_GET_PHB_DIAG_DATA 51 +#define OPAL_PCI_FENCE_PHB 52 +#define OPAL_PCI_REINIT 53 +#define OPAL_PCI_MASK_PE_ERROR 54 +#define OPAL_SET_SLOT_LED_STATUS 55 +#define OPAL_GET_EPOW_STATUS 56 +#define OPAL_SET_SYSTEM_ATTENTION_LED 57 +#define OPAL_RESERVED1 58 +#define OPAL_RESERVED2 59 +#define OPAL_PCI_NEXT_ERROR 60 +#define OPAL_PCI_EEH_FREEZE_STATUS2 61 +#define OPAL_PCI_POLL 62 +#define OPAL_PCI_MSI_EOI 63 +#define OPAL_PCI_GET_PHB_DIAG_DATA2 64 +#define OPAL_XSCOM_READ 65 +#define OPAL_XSCOM_WRITE 66 +#define OPAL_LPC_READ 67 +#define OPAL_LPC_WRITE 68 +#define OPAL_RETURN_CPU 69 +#define OPAL_REINIT_CPUS 70 +#define OPAL_ELOG_READ 71 +#define OPAL_ELOG_WRITE 72 +#define OPAL_ELOG_ACK 73 +#define OPAL_ELOG_RESEND 74 +#define OPAL_ELOG_SIZE 75 +#define OPAL_FLASH_VALIDATE 76 +#define OPAL_FLASH_MANAGE 77 +#define OPAL_FLASH_UPDATE 78 +#define OPAL_RESYNC_TIMEBASE 79 +#define OPAL_CHECK_TOKEN 80 +#define OPAL_DUMP_INIT 81 +#define OPAL_DUMP_INFO 82 +#define OPAL_DUMP_READ 83 +#define OPAL_DUMP_ACK 84 +#define OPAL_GET_MSG 85 +#define OPAL_CHECK_ASYNC_COMPLETION 86 +#define OPAL_SYNC_HOST_REBOOT 87 +#define OPAL_SENSOR_READ 88 +#define OPAL_GET_PARAM 89 +#define OPAL_SET_PARAM 90 +#define OPAL_DUMP_RESEND 91 +#define OPAL_ELOG_SEND 92 /* Deprecated */ +#define OPAL_PCI_SET_PHB_CAPI_MODE 93 +#define OPAL_DUMP_INFO2 94 +#define OPAL_WRITE_OPPANEL_ASYNC 95 +#define OPAL_PCI_ERR_INJECT 96 +#define OPAL_PCI_EEH_FREEZE_SET 97 +#define OPAL_HANDLE_HMI 98 +#define OPAL_CONFIG_CPU_IDLE_STATE 99 +#define OPAL_SLW_SET_REG 100 +#define OPAL_REGISTER_DUMP_REGION 101 +#define OPAL_UNREGISTER_DUMP_REGION 102 +#define OPAL_WRITE_TPO 103 +#define OPAL_READ_TPO 104 +#define OPAL_GET_DPO_STATUS 105 +#define OPAL_OLD_I2C_REQUEST 106 /* Deprecated */ +#define OPAL_IPMI_SEND 107 +#define OPAL_IPMI_RECV 108 +#define OPAL_I2C_REQUEST 109 +#define OPAL_FLASH_READ 110 +#define OPAL_FLASH_WRITE 111 +#define OPAL_FLASH_ERASE 112 +#define OPAL_LAST 112 + +/* Device tree flags */ + +/* Flags set in power-mgmt nodes in device tree if + * respective idle states are supported in the platform. + */ +#define OPAL_PM_NAP_ENABLED 0x00010000 +#define OPAL_PM_SLEEP_ENABLED 0x00020000 +#define OPAL_PM_WINKLE_ENABLED 0x00040000 +#define OPAL_PM_SLEEP_ENABLED_ER1 0x00080000 /* with workaround */ + +#ifndef __ASSEMBLY__ + +/* Other enums */ +enum OpalFreezeState { + OPAL_EEH_STOPPED_NOT_FROZEN = 0, + OPAL_EEH_STOPPED_MMIO_FREEZE = 1, + OPAL_EEH_STOPPED_DMA_FREEZE = 2, + OPAL_EEH_STOPPED_MMIO_DMA_FREEZE = 3, + OPAL_EEH_STOPPED_RESET = 4, + OPAL_EEH_STOPPED_TEMP_UNAVAIL = 5, + OPAL_EEH_STOPPED_PERM_UNAVAIL = 6 +}; + +enum OpalEehFreezeActionToken { + OPAL_EEH_ACTION_CLEAR_FREEZE_MMIO = 1, + OPAL_EEH_ACTION_CLEAR_FREEZE_DMA = 2, + OPAL_EEH_ACTION_CLEAR_FREEZE_ALL = 3, + + OPAL_EEH_ACTION_SET_FREEZE_MMIO = 1, + OPAL_EEH_ACTION_SET_FREEZE_DMA = 2, + OPAL_EEH_ACTION_SET_FREEZE_ALL = 3 +}; + +enum OpalPciStatusToken { + OPAL_EEH_NO_ERROR = 0, + OPAL_EEH_IOC_ERROR = 1, + OPAL_EEH_PHB_ERROR = 2, + OPAL_EEH_PE_ERROR = 3, + OPAL_EEH_PE_MMIO_ERROR = 4, + OPAL_EEH_PE_DMA_ERROR = 5 +}; + +enum OpalPciErrorSeverity { + OPAL_EEH_SEV_NO_ERROR = 0, + OPAL_EEH_SEV_IOC_DEAD = 1, + OPAL_EEH_SEV_PHB_DEAD = 2, + OPAL_EEH_SEV_PHB_FENCED = 3, + OPAL_EEH_SEV_PE_ER = 4, + OPAL_EEH_SEV_INF = 5 +}; + +enum OpalErrinjectType { + OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR = 0, + OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR64 = 1, +}; + +enum OpalErrinjectFunc { + /* IOA bus specific errors */ + OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_ADDR = 0, + OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_DATA = 1, + OPAL_ERR_INJECT_FUNC_IOA_LD_IO_ADDR = 2, + OPAL_ERR_INJECT_FUNC_IOA_LD_IO_DATA = 3, + OPAL_ERR_INJECT_FUNC_IOA_LD_CFG_ADDR = 4, + OPAL_ERR_INJECT_FUNC_IOA_LD_CFG_DATA = 5, + OPAL_ERR_INJECT_FUNC_IOA_ST_MEM_ADDR = 6, + OPAL_ERR_INJECT_FUNC_IOA_ST_MEM_DATA = 7, + OPAL_ERR_INJECT_FUNC_IOA_ST_IO_ADDR = 8, + OPAL_ERR_INJECT_FUNC_IOA_ST_IO_DATA = 9, + OPAL_ERR_INJECT_FUNC_IOA_ST_CFG_ADDR = 10, + OPAL_ERR_INJECT_FUNC_IOA_ST_CFG_DATA = 11, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_ADDR = 12, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_DATA = 13, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_MASTER = 14, + OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_TARGET = 15, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_ADDR = 16, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_DATA = 17, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_MASTER = 18, + OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_TARGET = 19, +}; + +enum OpalMmioWindowType { + OPAL_M32_WINDOW_TYPE = 1, + OPAL_M64_WINDOW_TYPE = 2, + OPAL_IO_WINDOW_TYPE = 3 +}; + +enum OpalExceptionHandler { + OPAL_MACHINE_CHECK_HANDLER = 1, + OPAL_HYPERVISOR_MAINTENANCE_HANDLER = 2, + OPAL_SOFTPATCH_HANDLER = 3 +}; + +enum OpalPendingState { + OPAL_EVENT_OPAL_INTERNAL = 0x1, + OPAL_EVENT_NVRAM = 0x2, + OPAL_EVENT_RTC = 0x4, + OPAL_EVENT_CONSOLE_OUTPUT = 0x8, + OPAL_EVENT_CONSOLE_INPUT = 0x10, + OPAL_EVENT_ERROR_LOG_AVAIL = 0x20, + OPAL_EVENT_ERROR_LOG = 0x40, + OPAL_EVENT_EPOW = 0x80, + OPAL_EVENT_LED_STATUS = 0x100, + OPAL_EVENT_PCI_ERROR = 0x200, + OPAL_EVENT_DUMP_AVAIL = 0x400, + OPAL_EVENT_MSG_PENDING = 0x800, +}; + +enum OpalThreadStatus { + OPAL_THREAD_INACTIVE = 0x0, + OPAL_THREAD_STARTED = 0x1, + OPAL_THREAD_UNAVAILABLE = 0x2 /* opal-v3 */ +}; + +enum OpalPciBusCompare { + OpalPciBusAny = 0, /* Any bus number match */ + OpalPciBus3Bits = 2, /* Match top 3 bits of bus number */ + OpalPciBus4Bits = 3, /* Match top 4 bits of bus number */ + OpalPciBus5Bits = 4, /* Match top 5 bits of bus number */ + OpalPciBus6Bits = 5, /* Match top 6 bits of bus number */ + OpalPciBus7Bits = 6, /* Match top 7 bits of bus number */ + OpalPciBusAll = 7, /* Match bus number exactly */ +}; + +enum OpalDeviceCompare { + OPAL_IGNORE_RID_DEVICE_NUMBER = 0, + OPAL_COMPARE_RID_DEVICE_NUMBER = 1 +}; + +enum OpalFuncCompare { + OPAL_IGNORE_RID_FUNCTION_NUMBER = 0, + OPAL_COMPARE_RID_FUNCTION_NUMBER = 1 +}; + +enum OpalPeAction { + OPAL_UNMAP_PE = 0, + OPAL_MAP_PE = 1 +}; + +enum OpalPeltvAction { + OPAL_REMOVE_PE_FROM_DOMAIN = 0, + OPAL_ADD_PE_TO_DOMAIN = 1 +}; + +enum OpalMveEnableAction { + OPAL_DISABLE_MVE = 0, + OPAL_ENABLE_MVE = 1 +}; + +enum OpalM64Action { + OPAL_DISABLE_M64 = 0, + OPAL_ENABLE_M64_SPLIT = 1, + OPAL_ENABLE_M64_NON_SPLIT = 2 +}; + +enum OpalPciResetScope { + OPAL_RESET_PHB_COMPLETE = 1, + OPAL_RESET_PCI_LINK = 2, + OPAL_RESET_PHB_ERROR = 3, + OPAL_RESET_PCI_HOT = 4, + OPAL_RESET_PCI_FUNDAMENTAL = 5, + OPAL_RESET_PCI_IODA_TABLE = 6 +}; + +enum OpalPciReinitScope { + /* + * Note: we chose values that do not overlap + * OpalPciResetScope as OPAL v2 used the same + * enum for both + */ + OPAL_REINIT_PCI_DEV = 1000 +}; + +enum OpalPciResetState { + OPAL_DEASSERT_RESET = 0, + OPAL_ASSERT_RESET = 1 +}; + +/* + * Address cycle types for LPC accesses. These also correspond + * to the content of the first cell of the "reg" property for + * device nodes on the LPC bus + */ +enum OpalLPCAddressType { + OPAL_LPC_MEM = 0, + OPAL_LPC_IO = 1, + OPAL_LPC_FW = 2, +}; + +enum opal_msg_type { + OPAL_MSG_ASYNC_COMP = 0, /* params[0] = token, params[1] = rc, + * additional params function-specific + */ + OPAL_MSG_MEM_ERR, + OPAL_MSG_EPOW, + OPAL_MSG_SHUTDOWN, /* params[0] = 1 reboot, 0 shutdown */ + OPAL_MSG_HMI_EVT, + OPAL_MSG_DPO, + OPAL_MSG_TYPE_MAX, +}; + +struct opal_msg { + __be32 msg_type; + __be32 reserved; + __be64 params[8]; +}; + +/* System parameter permission */ +enum OpalSysparamPerm { + OPAL_SYSPARAM_READ = 0x1, + OPAL_SYSPARAM_WRITE = 0x2, + OPAL_SYSPARAM_RW = (OPAL_SYSPARAM_READ | OPAL_SYSPARAM_WRITE), +}; + +enum { + OPAL_IPMI_MSG_FORMAT_VERSION_1 = 1, +}; + +struct opal_ipmi_msg { + uint8_t version; + uint8_t netfn; + uint8_t cmd; + uint8_t data[]; +}; + +/* FSP memory errors handling */ +enum OpalMemErr_Version { + OpalMemErr_V1 = 1, +}; + +enum OpalMemErrType { + OPAL_MEM_ERR_TYPE_RESILIENCE = 0, + OPAL_MEM_ERR_TYPE_DYN_DALLOC, +}; + +/* Memory Reilience error type */ +enum OpalMemErr_ResilErrType { + OPAL_MEM_RESILIENCE_CE = 0, + OPAL_MEM_RESILIENCE_UE, + OPAL_MEM_RESILIENCE_UE_SCRUB, +}; + +/* Dynamic Memory Deallocation type */ +enum OpalMemErr_DynErrType { + OPAL_MEM_DYNAMIC_DEALLOC = 0, +}; + +struct OpalMemoryErrorData { + enum OpalMemErr_Version version:8; /* 0x00 */ + enum OpalMemErrType type:8; /* 0x01 */ + __be16 flags; /* 0x02 */ + uint8_t reserved_1[4]; /* 0x04 */ + + union { + /* Memory Resilience corrected/uncorrected error info */ + struct { + enum OpalMemErr_ResilErrType resil_err_type:8; + uint8_t reserved_1[7]; + __be64 physical_address_start; + __be64 physical_address_end; + } resilience; + /* Dynamic memory deallocation error info */ + struct { + enum OpalMemErr_DynErrType dyn_err_type:8; + uint8_t reserved_1[7]; + __be64 physical_address_start; + __be64 physical_address_end; + } dyn_dealloc; + } u; +}; + +/* HMI interrupt event */ +enum OpalHMI_Version { + OpalHMIEvt_V1 = 1, +}; + +enum OpalHMI_Severity { + OpalHMI_SEV_NO_ERROR = 0, + OpalHMI_SEV_WARNING = 1, + OpalHMI_SEV_ERROR_SYNC = 2, + OpalHMI_SEV_FATAL = 3, +}; + +enum OpalHMI_Disposition { + OpalHMI_DISPOSITION_RECOVERED = 0, + OpalHMI_DISPOSITION_NOT_RECOVERED = 1, +}; + +enum OpalHMI_ErrType { + OpalHMI_ERROR_MALFUNC_ALERT = 0, + OpalHMI_ERROR_PROC_RECOV_DONE, + OpalHMI_ERROR_PROC_RECOV_DONE_AGAIN, + OpalHMI_ERROR_PROC_RECOV_MASKED, + OpalHMI_ERROR_TFAC, + OpalHMI_ERROR_TFMR_PARITY, + OpalHMI_ERROR_HA_OVERFLOW_WARN, + OpalHMI_ERROR_XSCOM_FAIL, + OpalHMI_ERROR_XSCOM_DONE, + OpalHMI_ERROR_SCOM_FIR, + OpalHMI_ERROR_DEBUG_TRIG_FIR, + OpalHMI_ERROR_HYP_RESOURCE, + OpalHMI_ERROR_CAPP_RECOVERY, +}; + +struct OpalHMIEvent { + uint8_t version; /* 0x00 */ + uint8_t severity; /* 0x01 */ + uint8_t type; /* 0x02 */ + uint8_t disposition; /* 0x03 */ + uint8_t reserved_1[4]; /* 0x04 */ + + __be64 hmer; + /* TFMR register. Valid only for TFAC and TFMR_PARITY error type. */ + __be64 tfmr; +}; + +enum { + OPAL_P7IOC_DIAG_TYPE_NONE = 0, + OPAL_P7IOC_DIAG_TYPE_RGC = 1, + OPAL_P7IOC_DIAG_TYPE_BI = 2, + OPAL_P7IOC_DIAG_TYPE_CI = 3, + OPAL_P7IOC_DIAG_TYPE_MISC = 4, + OPAL_P7IOC_DIAG_TYPE_I2C = 5, + OPAL_P7IOC_DIAG_TYPE_LAST = 6 +}; + +struct OpalIoP7IOCErrorData { + __be16 type; + + /* GEM */ + __be64 gemXfir; + __be64 gemRfir; + __be64 gemRirqfir; + __be64 gemMask; + __be64 gemRwof; + + /* LEM */ + __be64 lemFir; + __be64 lemErrMask; + __be64 lemAction0; + __be64 lemAction1; + __be64 lemWof; + + union { + struct OpalIoP7IOCRgcErrorData { + __be64 rgcStatus; /* 3E1C10 */ + __be64 rgcLdcp; /* 3E1C18 */ + }rgc; + struct OpalIoP7IOCBiErrorData { + __be64 biLdcp0; /* 3C0100, 3C0118 */ + __be64 biLdcp1; /* 3C0108, 3C0120 */ + __be64 biLdcp2; /* 3C0110, 3C0128 */ + __be64 biFenceStatus; /* 3C0130, 3C0130 */ + + uint8_t biDownbound; /* BI Downbound or Upbound */ + }bi; + struct OpalIoP7IOCCiErrorData { + __be64 ciPortStatus; /* 3Dn008 */ + __be64 ciPortLdcp; /* 3Dn010 */ + + uint8_t ciPort; /* Index of CI port: 0/1 */ + }ci; + }; +}; + +/** + * This structure defines the overlay which will be used to store PHB error + * data upon request. + */ +enum { + OPAL_PHB_ERROR_DATA_VERSION_1 = 1, +}; + +enum { + OPAL_PHB_ERROR_DATA_TYPE_P7IOC = 1, + OPAL_PHB_ERROR_DATA_TYPE_PHB3 = 2 +}; + +enum { + OPAL_P7IOC_NUM_PEST_REGS = 128, + OPAL_PHB3_NUM_PEST_REGS = 256 +}; + +struct OpalIoPhbErrorCommon { + __be32 version; + __be32 ioType; + __be32 len; +}; + +struct OpalIoP7IOCPhbErrorData { + struct OpalIoPhbErrorCommon common; + + __be32 brdgCtl; + + // P7IOC utl regs + __be32 portStatusReg; + __be32 rootCmplxStatus; + __be32 busAgentStatus; + + // P7IOC cfg regs + __be32 deviceStatus; + __be32 slotStatus; + __be32 linkStatus; + __be32 devCmdStatus; + __be32 devSecStatus; + + // cfg AER regs + __be32 rootErrorStatus; + __be32 uncorrErrorStatus; + __be32 corrErrorStatus; + __be32 tlpHdr1; + __be32 tlpHdr2; + __be32 tlpHdr3; + __be32 tlpHdr4; + __be32 sourceId; + + __be32 rsv3; + + // Record data about the call to allocate a buffer. + __be64 errorClass; + __be64 correlator; + + //P7IOC MMIO Error Regs + __be64 p7iocPlssr; // n120 + __be64 p7iocCsr; // n110 + __be64 lemFir; // nC00 + __be64 lemErrorMask; // nC18 + __be64 lemWOF; // nC40 + __be64 phbErrorStatus; // nC80 + __be64 phbFirstErrorStatus; // nC88 + __be64 phbErrorLog0; // nCC0 + __be64 phbErrorLog1; // nCC8 + __be64 mmioErrorStatus; // nD00 + __be64 mmioFirstErrorStatus; // nD08 + __be64 mmioErrorLog0; // nD40 + __be64 mmioErrorLog1; // nD48 + __be64 dma0ErrorStatus; // nD80 + __be64 dma0FirstErrorStatus; // nD88 + __be64 dma0ErrorLog0; // nDC0 + __be64 dma0ErrorLog1; // nDC8 + __be64 dma1ErrorStatus; // nE00 + __be64 dma1FirstErrorStatus; // nE08 + __be64 dma1ErrorLog0; // nE40 + __be64 dma1ErrorLog1; // nE48 + __be64 pestA[OPAL_P7IOC_NUM_PEST_REGS]; + __be64 pestB[OPAL_P7IOC_NUM_PEST_REGS]; +}; + +struct OpalIoPhb3ErrorData { + struct OpalIoPhbErrorCommon common; + + __be32 brdgCtl; + + /* PHB3 UTL regs */ + __be32 portStatusReg; + __be32 rootCmplxStatus; + __be32 busAgentStatus; + + /* PHB3 cfg regs */ + __be32 deviceStatus; + __be32 slotStatus; + __be32 linkStatus; + __be32 devCmdStatus; + __be32 devSecStatus; + + /* cfg AER regs */ + __be32 rootErrorStatus; + __be32 uncorrErrorStatus; + __be32 corrErrorStatus; + __be32 tlpHdr1; + __be32 tlpHdr2; + __be32 tlpHdr3; + __be32 tlpHdr4; + __be32 sourceId; + + __be32 rsv3; + + /* Record data about the call to allocate a buffer */ + __be64 errorClass; + __be64 correlator; + + /* PHB3 MMIO Error Regs */ + __be64 nFir; /* 000 */ + __be64 nFirMask; /* 003 */ + __be64 nFirWOF; /* 008 */ + __be64 phbPlssr; /* 120 */ + __be64 phbCsr; /* 110 */ + __be64 lemFir; /* C00 */ + __be64 lemErrorMask; /* C18 */ + __be64 lemWOF; /* C40 */ + __be64 phbErrorStatus; /* C80 */ + __be64 phbFirstErrorStatus; /* C88 */ + __be64 phbErrorLog0; /* CC0 */ + __be64 phbErrorLog1; /* CC8 */ + __be64 mmioErrorStatus; /* D00 */ + __be64 mmioFirstErrorStatus; /* D08 */ + __be64 mmioErrorLog0; /* D40 */ + __be64 mmioErrorLog1; /* D48 */ + __be64 dma0ErrorStatus; /* D80 */ + __be64 dma0FirstErrorStatus; /* D88 */ + __be64 dma0ErrorLog0; /* DC0 */ + __be64 dma0ErrorLog1; /* DC8 */ + __be64 dma1ErrorStatus; /* E00 */ + __be64 dma1FirstErrorStatus; /* E08 */ + __be64 dma1ErrorLog0; /* E40 */ + __be64 dma1ErrorLog1; /* E48 */ + __be64 pestA[OPAL_PHB3_NUM_PEST_REGS]; + __be64 pestB[OPAL_PHB3_NUM_PEST_REGS]; +}; + +enum { + OPAL_REINIT_CPUS_HILE_BE = (1 << 0), + OPAL_REINIT_CPUS_HILE_LE = (1 << 1), +}; + +typedef struct oppanel_line { + __be64 line; + __be64 line_len; +} oppanel_line_t; + +/* + * SG entries + * + * WARNING: The current implementation requires each entry + * to represent a block that is 4k aligned *and* each block + * size except the last one in the list to be as well. + */ +struct opal_sg_entry { + __be64 data; + __be64 length; +}; + +/* + * Candiate image SG list. + * + * length = VER | length + */ +struct opal_sg_list { + __be64 length; + __be64 next; + struct opal_sg_entry entry[]; +}; + +/* + * Dump region ID range usable by the OS + */ +#define OPAL_DUMP_REGION_HOST_START 0x80 +#define OPAL_DUMP_REGION_LOG_BUF 0x80 +#define OPAL_DUMP_REGION_HOST_END 0xFF + +/* CAPI modes for PHB */ +enum { + OPAL_PHB_CAPI_MODE_PCIE = 0, + OPAL_PHB_CAPI_MODE_CAPI = 1, + OPAL_PHB_CAPI_MODE_SNOOP_OFF = 2, + OPAL_PHB_CAPI_MODE_SNOOP_ON = 3, +}; + +/* OPAL I2C request */ +struct opal_i2c_request { + uint8_t type; +#define OPAL_I2C_RAW_READ 0 +#define OPAL_I2C_RAW_WRITE 1 +#define OPAL_I2C_SM_READ 2 +#define OPAL_I2C_SM_WRITE 3 + uint8_t flags; +#define OPAL_I2C_ADDR_10 0x01 /* Not supported yet */ + uint8_t subaddr_sz; /* Max 4 */ + uint8_t reserved; + __be16 addr; /* 7 or 10 bit address */ + __be16 reserved2; + __be32 subaddr; /* Sub-address if any */ + __be32 size; /* Data size */ + __be64 buffer_ra; /* Buffer real address */ +}; + +#endif /* __ASSEMBLY__ */ + +#endif /* __OPAL_API_H */ diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index 9ee0a30a02ce..042af1abfc4d 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -9,755 +9,17 @@ * 2 of the License, or (at your option) any later version. */ -#ifndef __OPAL_H -#define __OPAL_H +#ifndef _ASM_POWERPC_OPAL_H +#define _ASM_POWERPC_OPAL_H -#ifndef __ASSEMBLY__ -/* - * SG entry - * - * WARNING: The current implementation requires each entry - * to represent a block that is 4k aligned *and* each block - * size except the last one in the list to be as well. - */ -struct opal_sg_entry { - __be64 data; - __be64 length; -}; - -/* SG list */ -struct opal_sg_list { - __be64 length; - __be64 next; - struct opal_sg_entry entry[]; -}; - -/* We calculate number of sg entries based on PAGE_SIZE */ -#define SG_ENTRIES_PER_NODE ((PAGE_SIZE - 16) / sizeof(struct opal_sg_entry)) - -#endif /* __ASSEMBLY__ */ - -/****** OPAL APIs ******/ - -/* Return codes */ -#define OPAL_SUCCESS 0 -#define OPAL_PARAMETER -1 -#define OPAL_BUSY -2 -#define OPAL_PARTIAL -3 -#define OPAL_CONSTRAINED -4 -#define OPAL_CLOSED -5 -#define OPAL_HARDWARE -6 -#define OPAL_UNSUPPORTED -7 -#define OPAL_PERMISSION -8 -#define OPAL_NO_MEM -9 -#define OPAL_RESOURCE -10 -#define OPAL_INTERNAL_ERROR -11 -#define OPAL_BUSY_EVENT -12 -#define OPAL_HARDWARE_FROZEN -13 -#define OPAL_WRONG_STATE -14 -#define OPAL_ASYNC_COMPLETION -15 -#define OPAL_I2C_TIMEOUT -17 -#define OPAL_I2C_INVALID_CMD -18 -#define OPAL_I2C_LBUS_PARITY -19 -#define OPAL_I2C_BKEND_OVERRUN -20 -#define OPAL_I2C_BKEND_ACCESS -21 -#define OPAL_I2C_ARBT_LOST -22 -#define OPAL_I2C_NACK_RCVD -23 -#define OPAL_I2C_STOP_ERR -24 - -/* API Tokens (in r0) */ -#define OPAL_INVALID_CALL -1 -#define OPAL_CONSOLE_WRITE 1 -#define OPAL_CONSOLE_READ 2 -#define OPAL_RTC_READ 3 -#define OPAL_RTC_WRITE 4 -#define OPAL_CEC_POWER_DOWN 5 -#define OPAL_CEC_REBOOT 6 -#define OPAL_READ_NVRAM 7 -#define OPAL_WRITE_NVRAM 8 -#define OPAL_HANDLE_INTERRUPT 9 -#define OPAL_POLL_EVENTS 10 -#define OPAL_PCI_SET_HUB_TCE_MEMORY 11 -#define OPAL_PCI_SET_PHB_TCE_MEMORY 12 -#define OPAL_PCI_CONFIG_READ_BYTE 13 -#define OPAL_PCI_CONFIG_READ_HALF_WORD 14 -#define OPAL_PCI_CONFIG_READ_WORD 15 -#define OPAL_PCI_CONFIG_WRITE_BYTE 16 -#define OPAL_PCI_CONFIG_WRITE_HALF_WORD 17 -#define OPAL_PCI_CONFIG_WRITE_WORD 18 -#define OPAL_SET_XIVE 19 -#define OPAL_GET_XIVE 20 -#define OPAL_GET_COMPLETION_TOKEN_STATUS 21 /* obsolete */ -#define OPAL_REGISTER_OPAL_EXCEPTION_HANDLER 22 -#define OPAL_PCI_EEH_FREEZE_STATUS 23 -#define OPAL_PCI_SHPC 24 -#define OPAL_CONSOLE_WRITE_BUFFER_SPACE 25 -#define OPAL_PCI_EEH_FREEZE_CLEAR 26 -#define OPAL_PCI_PHB_MMIO_ENABLE 27 -#define OPAL_PCI_SET_PHB_MEM_WINDOW 28 -#define OPAL_PCI_MAP_PE_MMIO_WINDOW 29 -#define OPAL_PCI_SET_PHB_TABLE_MEMORY 30 -#define OPAL_PCI_SET_PE 31 -#define OPAL_PCI_SET_PELTV 32 -#define OPAL_PCI_SET_MVE 33 -#define OPAL_PCI_SET_MVE_ENABLE 34 -#define OPAL_PCI_GET_XIVE_REISSUE 35 -#define OPAL_PCI_SET_XIVE_REISSUE 36 -#define OPAL_PCI_SET_XIVE_PE 37 -#define OPAL_GET_XIVE_SOURCE 38 -#define OPAL_GET_MSI_32 39 -#define OPAL_GET_MSI_64 40 -#define OPAL_START_CPU 41 -#define OPAL_QUERY_CPU_STATUS 42 -#define OPAL_WRITE_OPPANEL 43 -#define OPAL_PCI_MAP_PE_DMA_WINDOW 44 -#define OPAL_PCI_MAP_PE_DMA_WINDOW_REAL 45 -#define OPAL_PCI_RESET 49 -#define OPAL_PCI_GET_HUB_DIAG_DATA 50 -#define OPAL_PCI_GET_PHB_DIAG_DATA 51 -#define OPAL_PCI_FENCE_PHB 52 -#define OPAL_PCI_REINIT 53 -#define OPAL_PCI_MASK_PE_ERROR 54 -#define OPAL_SET_SLOT_LED_STATUS 55 -#define OPAL_GET_EPOW_STATUS 56 -#define OPAL_SET_SYSTEM_ATTENTION_LED 57 -#define OPAL_RESERVED1 58 -#define OPAL_RESERVED2 59 -#define OPAL_PCI_NEXT_ERROR 60 -#define OPAL_PCI_EEH_FREEZE_STATUS2 61 -#define OPAL_PCI_POLL 62 -#define OPAL_PCI_MSI_EOI 63 -#define OPAL_PCI_GET_PHB_DIAG_DATA2 64 -#define OPAL_XSCOM_READ 65 -#define OPAL_XSCOM_WRITE 66 -#define OPAL_LPC_READ 67 -#define OPAL_LPC_WRITE 68 -#define OPAL_RETURN_CPU 69 -#define OPAL_REINIT_CPUS 70 -#define OPAL_ELOG_READ 71 -#define OPAL_ELOG_WRITE 72 -#define OPAL_ELOG_ACK 73 -#define OPAL_ELOG_RESEND 74 -#define OPAL_ELOG_SIZE 75 -#define OPAL_FLASH_VALIDATE 76 -#define OPAL_FLASH_MANAGE 77 -#define OPAL_FLASH_UPDATE 78 -#define OPAL_RESYNC_TIMEBASE 79 -#define OPAL_CHECK_TOKEN 80 -#define OPAL_DUMP_INIT 81 -#define OPAL_DUMP_INFO 82 -#define OPAL_DUMP_READ 83 -#define OPAL_DUMP_ACK 84 -#define OPAL_GET_MSG 85 -#define OPAL_CHECK_ASYNC_COMPLETION 86 -#define OPAL_SYNC_HOST_REBOOT 87 -#define OPAL_SENSOR_READ 88 -#define OPAL_GET_PARAM 89 -#define OPAL_SET_PARAM 90 -#define OPAL_DUMP_RESEND 91 -#define OPAL_PCI_SET_PHB_CXL_MODE 93 -#define OPAL_DUMP_INFO2 94 -#define OPAL_PCI_ERR_INJECT 96 -#define OPAL_PCI_EEH_FREEZE_SET 97 -#define OPAL_HANDLE_HMI 98 -#define OPAL_CONFIG_CPU_IDLE_STATE 99 -#define OPAL_SLW_SET_REG 100 -#define OPAL_REGISTER_DUMP_REGION 101 -#define OPAL_UNREGISTER_DUMP_REGION 102 -#define OPAL_WRITE_TPO 103 -#define OPAL_READ_TPO 104 -#define OPAL_IPMI_SEND 107 -#define OPAL_IPMI_RECV 108 -#define OPAL_I2C_REQUEST 109 - -/* Device tree flags */ - -/* Flags set in power-mgmt nodes in device tree if - * respective idle states are supported in the platform. - */ -#define OPAL_PM_NAP_ENABLED 0x00010000 -#define OPAL_PM_SLEEP_ENABLED 0x00020000 -#define OPAL_PM_WINKLE_ENABLED 0x00040000 -#define OPAL_PM_SLEEP_ENABLED_ER1 0x00080000 +#include <asm/opal-api.h> #ifndef __ASSEMBLY__ #include <linux/notifier.h> -/* Other enums */ -enum OpalVendorApiTokens { - OPAL_START_VENDOR_API_RANGE = 1000, OPAL_END_VENDOR_API_RANGE = 1999 -}; - -enum OpalFreezeState { - OPAL_EEH_STOPPED_NOT_FROZEN = 0, - OPAL_EEH_STOPPED_MMIO_FREEZE = 1, - OPAL_EEH_STOPPED_DMA_FREEZE = 2, - OPAL_EEH_STOPPED_MMIO_DMA_FREEZE = 3, - OPAL_EEH_STOPPED_RESET = 4, - OPAL_EEH_STOPPED_TEMP_UNAVAIL = 5, - OPAL_EEH_STOPPED_PERM_UNAVAIL = 6 -}; - -enum OpalEehFreezeActionToken { - OPAL_EEH_ACTION_CLEAR_FREEZE_MMIO = 1, - OPAL_EEH_ACTION_CLEAR_FREEZE_DMA = 2, - OPAL_EEH_ACTION_CLEAR_FREEZE_ALL = 3, - - OPAL_EEH_ACTION_SET_FREEZE_MMIO = 1, - OPAL_EEH_ACTION_SET_FREEZE_DMA = 2, - OPAL_EEH_ACTION_SET_FREEZE_ALL = 3 -}; - -enum OpalPciStatusToken { - OPAL_EEH_NO_ERROR = 0, - OPAL_EEH_IOC_ERROR = 1, - OPAL_EEH_PHB_ERROR = 2, - OPAL_EEH_PE_ERROR = 3, - OPAL_EEH_PE_MMIO_ERROR = 4, - OPAL_EEH_PE_DMA_ERROR = 5 -}; - -enum OpalPciErrorSeverity { - OPAL_EEH_SEV_NO_ERROR = 0, - OPAL_EEH_SEV_IOC_DEAD = 1, - OPAL_EEH_SEV_PHB_DEAD = 2, - OPAL_EEH_SEV_PHB_FENCED = 3, - OPAL_EEH_SEV_PE_ER = 4, - OPAL_EEH_SEV_INF = 5 -}; - -enum OpalErrinjectType { - OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR = 0, - OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR64 = 1, -}; - -enum OpalErrinjectFunc { - /* IOA bus specific errors */ - OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_ADDR = 0, - OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_DATA = 1, - OPAL_ERR_INJECT_FUNC_IOA_LD_IO_ADDR = 2, - OPAL_ERR_INJECT_FUNC_IOA_LD_IO_DATA = 3, - OPAL_ERR_INJECT_FUNC_IOA_LD_CFG_ADDR = 4, - OPAL_ERR_INJECT_FUNC_IOA_LD_CFG_DATA = 5, - OPAL_ERR_INJECT_FUNC_IOA_ST_MEM_ADDR = 6, - OPAL_ERR_INJECT_FUNC_IOA_ST_MEM_DATA = 7, - OPAL_ERR_INJECT_FUNC_IOA_ST_IO_ADDR = 8, - OPAL_ERR_INJECT_FUNC_IOA_ST_IO_DATA = 9, - OPAL_ERR_INJECT_FUNC_IOA_ST_CFG_ADDR = 10, - OPAL_ERR_INJECT_FUNC_IOA_ST_CFG_DATA = 11, - OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_ADDR = 12, - OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_DATA = 13, - OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_MASTER = 14, - OPAL_ERR_INJECT_FUNC_IOA_DMA_RD_TARGET = 15, - OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_ADDR = 16, - OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_DATA = 17, - OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_MASTER = 18, - OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_TARGET = 19, -}; - -enum OpalShpcAction { - OPAL_SHPC_GET_LINK_STATE = 0, - OPAL_SHPC_GET_SLOT_STATE = 1 -}; - -enum OpalShpcLinkState { - OPAL_SHPC_LINK_DOWN = 0, - OPAL_SHPC_LINK_UP = 1 -}; - -enum OpalMmioWindowType { - OPAL_M32_WINDOW_TYPE = 1, - OPAL_M64_WINDOW_TYPE = 2, - OPAL_IO_WINDOW_TYPE = 3 -}; - -enum OpalShpcSlotState { - OPAL_SHPC_DEV_NOT_PRESENT = 0, - OPAL_SHPC_DEV_PRESENT = 1 -}; - -enum OpalExceptionHandler { - OPAL_MACHINE_CHECK_HANDLER = 1, - OPAL_HYPERVISOR_MAINTENANCE_HANDLER = 2, - OPAL_SOFTPATCH_HANDLER = 3 -}; - -enum OpalPendingState { - OPAL_EVENT_OPAL_INTERNAL = 0x1, - OPAL_EVENT_NVRAM = 0x2, - OPAL_EVENT_RTC = 0x4, - OPAL_EVENT_CONSOLE_OUTPUT = 0x8, - OPAL_EVENT_CONSOLE_INPUT = 0x10, - OPAL_EVENT_ERROR_LOG_AVAIL = 0x20, - OPAL_EVENT_ERROR_LOG = 0x40, - OPAL_EVENT_EPOW = 0x80, - OPAL_EVENT_LED_STATUS = 0x100, - OPAL_EVENT_PCI_ERROR = 0x200, - OPAL_EVENT_DUMP_AVAIL = 0x400, - OPAL_EVENT_MSG_PENDING = 0x800, -}; - -enum OpalMessageType { - OPAL_MSG_ASYNC_COMP = 0, /* params[0] = token, params[1] = rc, - * additional params function-specific - */ - OPAL_MSG_MEM_ERR, - OPAL_MSG_EPOW, - OPAL_MSG_SHUTDOWN, /* params[0] = 1 reboot, 0 shutdown */ - OPAL_MSG_HMI_EVT, - OPAL_MSG_TYPE_MAX, -}; - -enum OpalThreadStatus { - OPAL_THREAD_INACTIVE = 0x0, - OPAL_THREAD_STARTED = 0x1, - OPAL_THREAD_UNAVAILABLE = 0x2 /* opal-v3 */ -}; - -enum OpalPciBusCompare { - OpalPciBusAny = 0, /* Any bus number match */ - OpalPciBus3Bits = 2, /* Match top 3 bits of bus number */ - OpalPciBus4Bits = 3, /* Match top 4 bits of bus number */ - OpalPciBus5Bits = 4, /* Match top 5 bits of bus number */ - OpalPciBus6Bits = 5, /* Match top 6 bits of bus number */ - OpalPciBus7Bits = 6, /* Match top 7 bits of bus number */ - OpalPciBusAll = 7, /* Match bus number exactly */ -}; - -enum OpalDeviceCompare { - OPAL_IGNORE_RID_DEVICE_NUMBER = 0, - OPAL_COMPARE_RID_DEVICE_NUMBER = 1 -}; - -enum OpalFuncCompare { - OPAL_IGNORE_RID_FUNCTION_NUMBER = 0, - OPAL_COMPARE_RID_FUNCTION_NUMBER = 1 -}; - -enum OpalPeAction { - OPAL_UNMAP_PE = 0, - OPAL_MAP_PE = 1 -}; - -enum OpalPeltvAction { - OPAL_REMOVE_PE_FROM_DOMAIN = 0, - OPAL_ADD_PE_TO_DOMAIN = 1 -}; - -enum OpalMveEnableAction { - OPAL_DISABLE_MVE = 0, - OPAL_ENABLE_MVE = 1 -}; - -enum OpalM64EnableAction { - OPAL_DISABLE_M64 = 0, - OPAL_ENABLE_M64_SPLIT = 1, - OPAL_ENABLE_M64_NON_SPLIT = 2 -}; - -enum OpalPciResetScope { - OPAL_RESET_PHB_COMPLETE = 1, - OPAL_RESET_PCI_LINK = 2, - OPAL_RESET_PHB_ERROR = 3, - OPAL_RESET_PCI_HOT = 4, - OPAL_RESET_PCI_FUNDAMENTAL = 5, - OPAL_RESET_PCI_IODA_TABLE = 6 -}; - -enum OpalPciReinitScope { - OPAL_REINIT_PCI_DEV = 1000 -}; - -enum OpalPciResetState { - OPAL_DEASSERT_RESET = 0, - OPAL_ASSERT_RESET = 1 -}; - -enum OpalPciMaskAction { - OPAL_UNMASK_ERROR_TYPE = 0, - OPAL_MASK_ERROR_TYPE = 1 -}; - -enum OpalSlotLedType { - OPAL_SLOT_LED_ID_TYPE = 0, - OPAL_SLOT_LED_FAULT_TYPE = 1 -}; - -enum OpalLedAction { - OPAL_TURN_OFF_LED = 0, - OPAL_TURN_ON_LED = 1, - OPAL_QUERY_LED_STATE_AFTER_BUSY = 2 -}; - -enum OpalEpowStatus { - OPAL_EPOW_NONE = 0, - OPAL_EPOW_UPS = 1, - OPAL_EPOW_OVER_AMBIENT_TEMP = 2, - OPAL_EPOW_OVER_INTERNAL_TEMP = 3 -}; - -/* - * Address cycle types for LPC accesses. These also correspond - * to the content of the first cell of the "reg" property for - * device nodes on the LPC bus - */ -enum OpalLPCAddressType { - OPAL_LPC_MEM = 0, - OPAL_LPC_IO = 1, - OPAL_LPC_FW = 2, -}; - -/* System parameter permission */ -enum OpalSysparamPerm { - OPAL_SYSPARAM_READ = 0x1, - OPAL_SYSPARAM_WRITE = 0x2, - OPAL_SYSPARAM_RW = (OPAL_SYSPARAM_READ | OPAL_SYSPARAM_WRITE), -}; - -struct opal_msg { - __be32 msg_type; - __be32 reserved; - __be64 params[8]; -}; - -enum { - OPAL_IPMI_MSG_FORMAT_VERSION_1 = 1, -}; - -struct opal_ipmi_msg { - uint8_t version; - uint8_t netfn; - uint8_t cmd; - uint8_t data[]; -}; - -/* FSP memory errors handling */ -enum OpalMemErr_Version { - OpalMemErr_V1 = 1, -}; - -enum OpalMemErrType { - OPAL_MEM_ERR_TYPE_RESILIENCE = 0, - OPAL_MEM_ERR_TYPE_DYN_DALLOC, - OPAL_MEM_ERR_TYPE_SCRUB, -}; - -/* Memory Reilience error type */ -enum OpalMemErr_ResilErrType { - OPAL_MEM_RESILIENCE_CE = 0, - OPAL_MEM_RESILIENCE_UE, - OPAL_MEM_RESILIENCE_UE_SCRUB, -}; - -/* Dynamic Memory Deallocation type */ -enum OpalMemErr_DynErrType { - OPAL_MEM_DYNAMIC_DEALLOC = 0, -}; - -/* OpalMemoryErrorData->flags */ -#define OPAL_MEM_CORRECTED_ERROR 0x0001 -#define OPAL_MEM_THRESHOLD_EXCEEDED 0x0002 -#define OPAL_MEM_ACK_REQUIRED 0x8000 - -struct OpalMemoryErrorData { - enum OpalMemErr_Version version:8; /* 0x00 */ - enum OpalMemErrType type:8; /* 0x01 */ - __be16 flags; /* 0x02 */ - uint8_t reserved_1[4]; /* 0x04 */ - - union { - /* Memory Resilience corrected/uncorrected error info */ - struct { - enum OpalMemErr_ResilErrType resil_err_type:8; - uint8_t reserved_1[7]; - __be64 physical_address_start; - __be64 physical_address_end; - } resilience; - /* Dynamic memory deallocation error info */ - struct { - enum OpalMemErr_DynErrType dyn_err_type:8; - uint8_t reserved_1[7]; - __be64 physical_address_start; - __be64 physical_address_end; - } dyn_dealloc; - } u; -}; - -/* HMI interrupt event */ -enum OpalHMI_Version { - OpalHMIEvt_V1 = 1, -}; - -enum OpalHMI_Severity { - OpalHMI_SEV_NO_ERROR = 0, - OpalHMI_SEV_WARNING = 1, - OpalHMI_SEV_ERROR_SYNC = 2, - OpalHMI_SEV_FATAL = 3, -}; - -enum OpalHMI_Disposition { - OpalHMI_DISPOSITION_RECOVERED = 0, - OpalHMI_DISPOSITION_NOT_RECOVERED = 1, -}; - -enum OpalHMI_ErrType { - OpalHMI_ERROR_MALFUNC_ALERT = 0, - OpalHMI_ERROR_PROC_RECOV_DONE, - OpalHMI_ERROR_PROC_RECOV_DONE_AGAIN, - OpalHMI_ERROR_PROC_RECOV_MASKED, - OpalHMI_ERROR_TFAC, - OpalHMI_ERROR_TFMR_PARITY, - OpalHMI_ERROR_HA_OVERFLOW_WARN, - OpalHMI_ERROR_XSCOM_FAIL, - OpalHMI_ERROR_XSCOM_DONE, - OpalHMI_ERROR_SCOM_FIR, - OpalHMI_ERROR_DEBUG_TRIG_FIR, - OpalHMI_ERROR_HYP_RESOURCE, -}; - -struct OpalHMIEvent { - uint8_t version; /* 0x00 */ - uint8_t severity; /* 0x01 */ - uint8_t type; /* 0x02 */ - uint8_t disposition; /* 0x03 */ - uint8_t reserved_1[4]; /* 0x04 */ - - __be64 hmer; - /* TFMR register. Valid only for TFAC and TFMR_PARITY error type. */ - __be64 tfmr; -}; - -enum { - OPAL_P7IOC_DIAG_TYPE_NONE = 0, - OPAL_P7IOC_DIAG_TYPE_RGC = 1, - OPAL_P7IOC_DIAG_TYPE_BI = 2, - OPAL_P7IOC_DIAG_TYPE_CI = 3, - OPAL_P7IOC_DIAG_TYPE_MISC = 4, - OPAL_P7IOC_DIAG_TYPE_I2C = 5, - OPAL_P7IOC_DIAG_TYPE_LAST = 6 -}; - -struct OpalIoP7IOCErrorData { - __be16 type; - - /* GEM */ - __be64 gemXfir; - __be64 gemRfir; - __be64 gemRirqfir; - __be64 gemMask; - __be64 gemRwof; - - /* LEM */ - __be64 lemFir; - __be64 lemErrMask; - __be64 lemAction0; - __be64 lemAction1; - __be64 lemWof; - - union { - struct OpalIoP7IOCRgcErrorData { - __be64 rgcStatus; /* 3E1C10 */ - __be64 rgcLdcp; /* 3E1C18 */ - }rgc; - struct OpalIoP7IOCBiErrorData { - __be64 biLdcp0; /* 3C0100, 3C0118 */ - __be64 biLdcp1; /* 3C0108, 3C0120 */ - __be64 biLdcp2; /* 3C0110, 3C0128 */ - __be64 biFenceStatus; /* 3C0130, 3C0130 */ - - u8 biDownbound; /* BI Downbound or Upbound */ - }bi; - struct OpalIoP7IOCCiErrorData { - __be64 ciPortStatus; /* 3Dn008 */ - __be64 ciPortLdcp; /* 3Dn010 */ - - u8 ciPort; /* Index of CI port: 0/1 */ - }ci; - }; -}; - -/** - * This structure defines the overlay which will be used to store PHB error - * data upon request. - */ -enum { - OPAL_PHB_ERROR_DATA_VERSION_1 = 1, -}; - -enum { - OPAL_PHB_ERROR_DATA_TYPE_P7IOC = 1, - OPAL_PHB_ERROR_DATA_TYPE_PHB3 = 2 -}; - -enum { - OPAL_P7IOC_NUM_PEST_REGS = 128, - OPAL_PHB3_NUM_PEST_REGS = 256 -}; - -/* CAPI modes for PHB */ -enum { - OPAL_PHB_CAPI_MODE_PCIE = 0, - OPAL_PHB_CAPI_MODE_CAPI = 1, - OPAL_PHB_CAPI_MODE_SNOOP_OFF = 2, - OPAL_PHB_CAPI_MODE_SNOOP_ON = 3, -}; - -struct OpalIoPhbErrorCommon { - __be32 version; - __be32 ioType; - __be32 len; -}; - -struct OpalIoP7IOCPhbErrorData { - struct OpalIoPhbErrorCommon common; - - __be32 brdgCtl; - - // P7IOC utl regs - __be32 portStatusReg; - __be32 rootCmplxStatus; - __be32 busAgentStatus; - - // P7IOC cfg regs - __be32 deviceStatus; - __be32 slotStatus; - __be32 linkStatus; - __be32 devCmdStatus; - __be32 devSecStatus; - - // cfg AER regs - __be32 rootErrorStatus; - __be32 uncorrErrorStatus; - __be32 corrErrorStatus; - __be32 tlpHdr1; - __be32 tlpHdr2; - __be32 tlpHdr3; - __be32 tlpHdr4; - __be32 sourceId; - - __be32 rsv3; - - // Record data about the call to allocate a buffer. - __be64 errorClass; - __be64 correlator; - - //P7IOC MMIO Error Regs - __be64 p7iocPlssr; // n120 - __be64 p7iocCsr; // n110 - __be64 lemFir; // nC00 - __be64 lemErrorMask; // nC18 - __be64 lemWOF; // nC40 - __be64 phbErrorStatus; // nC80 - __be64 phbFirstErrorStatus; // nC88 - __be64 phbErrorLog0; // nCC0 - __be64 phbErrorLog1; // nCC8 - __be64 mmioErrorStatus; // nD00 - __be64 mmioFirstErrorStatus; // nD08 - __be64 mmioErrorLog0; // nD40 - __be64 mmioErrorLog1; // nD48 - __be64 dma0ErrorStatus; // nD80 - __be64 dma0FirstErrorStatus; // nD88 - __be64 dma0ErrorLog0; // nDC0 - __be64 dma0ErrorLog1; // nDC8 - __be64 dma1ErrorStatus; // nE00 - __be64 dma1FirstErrorStatus; // nE08 - __be64 dma1ErrorLog0; // nE40 - __be64 dma1ErrorLog1; // nE48 - __be64 pestA[OPAL_P7IOC_NUM_PEST_REGS]; - __be64 pestB[OPAL_P7IOC_NUM_PEST_REGS]; -}; - -struct OpalIoPhb3ErrorData { - struct OpalIoPhbErrorCommon common; - - __be32 brdgCtl; - - /* PHB3 UTL regs */ - __be32 portStatusReg; - __be32 rootCmplxStatus; - __be32 busAgentStatus; - - /* PHB3 cfg regs */ - __be32 deviceStatus; - __be32 slotStatus; - __be32 linkStatus; - __be32 devCmdStatus; - __be32 devSecStatus; - - /* cfg AER regs */ - __be32 rootErrorStatus; - __be32 uncorrErrorStatus; - __be32 corrErrorStatus; - __be32 tlpHdr1; - __be32 tlpHdr2; - __be32 tlpHdr3; - __be32 tlpHdr4; - __be32 sourceId; - - __be32 rsv3; - - /* Record data about the call to allocate a buffer */ - __be64 errorClass; - __be64 correlator; - - __be64 nFir; /* 000 */ - __be64 nFirMask; /* 003 */ - __be64 nFirWOF; /* 008 */ - - /* PHB3 MMIO Error Regs */ - __be64 phbPlssr; /* 120 */ - __be64 phbCsr; /* 110 */ - __be64 lemFir; /* C00 */ - __be64 lemErrorMask; /* C18 */ - __be64 lemWOF; /* C40 */ - __be64 phbErrorStatus; /* C80 */ - __be64 phbFirstErrorStatus; /* C88 */ - __be64 phbErrorLog0; /* CC0 */ - __be64 phbErrorLog1; /* CC8 */ - __be64 mmioErrorStatus; /* D00 */ - __be64 mmioFirstErrorStatus; /* D08 */ - __be64 mmioErrorLog0; /* D40 */ - __be64 mmioErrorLog1; /* D48 */ - __be64 dma0ErrorStatus; /* D80 */ - __be64 dma0FirstErrorStatus; /* D88 */ - __be64 dma0ErrorLog0; /* DC0 */ - __be64 dma0ErrorLog1; /* DC8 */ - __be64 dma1ErrorStatus; /* E00 */ - __be64 dma1FirstErrorStatus; /* E08 */ - __be64 dma1ErrorLog0; /* E40 */ - __be64 dma1ErrorLog1; /* E48 */ - __be64 pestA[OPAL_PHB3_NUM_PEST_REGS]; - __be64 pestB[OPAL_PHB3_NUM_PEST_REGS]; -}; - -enum { - OPAL_REINIT_CPUS_HILE_BE = (1 << 0), - OPAL_REINIT_CPUS_HILE_LE = (1 << 1), -}; - -typedef struct oppanel_line { - const char * line; - uint64_t line_len; -} oppanel_line_t; - -/* OPAL I2C request */ -struct opal_i2c_request { - uint8_t type; -#define OPAL_I2C_RAW_READ 0 -#define OPAL_I2C_RAW_WRITE 1 -#define OPAL_I2C_SM_READ 2 -#define OPAL_I2C_SM_WRITE 3 - uint8_t flags; -#define OPAL_I2C_ADDR_10 0x01 /* Not supported yet */ - uint8_t subaddr_sz; /* Max 4 */ - uint8_t reserved; - __be16 addr; /* 7 or 10 bit address */ - __be16 reserved2; - __be32 subaddr; /* Sub-address if any */ - __be32 size; /* Data size */ - __be64 buffer_ra; /* Buffer real address */ -}; +/* We calculate number of sg entries based on PAGE_SIZE */ +#define SG_ENTRIES_PER_NODE ((PAGE_SIZE - 16) / sizeof(struct opal_sg_entry)) /* /sys/firmware/opal */ extern struct kobject *opal_kobj; @@ -932,6 +194,13 @@ int64_t opal_ipmi_recv(uint64_t interface, struct opal_ipmi_msg *msg, int64_t opal_i2c_request(uint64_t async_token, uint32_t bus_id, struct opal_i2c_request *oreq); +int64_t opal_flash_read(uint64_t id, uint64_t offset, uint64_t buf, + uint64_t size, uint64_t token); +int64_t opal_flash_write(uint64_t id, uint64_t offset, uint64_t buf, + uint64_t size, uint64_t token); +int64_t opal_flash_erase(uint64_t id, uint64_t offset, uint64_t size, + uint64_t token); + /* Internal functions */ extern int early_init_dt_scan_opal(unsigned long node, const char *uname, int depth, void *data); @@ -946,8 +215,10 @@ extern void hvc_opal_init_early(void); extern int opal_notifier_register(struct notifier_block *nb); extern int opal_notifier_unregister(struct notifier_block *nb); -extern int opal_message_notifier_register(enum OpalMessageType msg_type, +extern int opal_message_notifier_register(enum opal_msg_type msg_type, struct notifier_block *nb); +extern int opal_message_notifier_unregister(enum opal_msg_type msg_type, + struct notifier_block *nb); extern void opal_notifier_enable(void); extern void opal_notifier_disable(void); extern void opal_notifier_update_evt(uint64_t evt_mask, uint64_t evt_val); @@ -962,7 +233,7 @@ extern int opal_get_sensor_data(u32 sensor_hndl, u32 *sensor_data); struct rtc_time; extern unsigned long opal_get_boot_time(void); extern void opal_nvram_init(void); -extern void opal_flash_init(void); +extern void opal_flash_update_init(void); extern void opal_flash_term_callback(void); extern int opal_elog_init(void); extern void opal_platform_dump_init(void); @@ -983,13 +254,8 @@ struct opal_sg_list *opal_vmalloc_to_sg_list(void *vmalloc_addr, unsigned long vmalloc_size); void opal_free_sg_list(struct opal_sg_list *sg); -/* - * Dump region ID range usable by the OS - */ -#define OPAL_DUMP_REGION_HOST_START 0x80 -#define OPAL_DUMP_REGION_LOG_BUF 0x80 -#define OPAL_DUMP_REGION_HOST_END 0xFF +extern int opal_error_code(int rc); #endif /* __ASSEMBLY__ */ -#endif /* __OPAL_H */ +#endif /* _ASM_POWERPC_OPAL_H */ diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index e5f22c6c4bf9..70bd4381f8e6 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -106,9 +106,9 @@ struct paca_struct { #endif /* CONFIG_PPC_STD_MMU_64 */ #ifdef CONFIG_PPC_BOOK3E - u64 exgen[8] __attribute__((aligned(0x80))); + u64 exgen[8] __aligned(0x40); /* Keep pgd in the same cacheline as the start of extlb */ - pgd_t *pgd __attribute__((aligned(0x80))); /* Current PGD */ + pgd_t *pgd __aligned(0x40); /* Current PGD */ pgd_t *kernel_pgd; /* Kernel PGD */ /* Shared by all threads of a core -- points to tcd of first thread */ diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h index 546d036fe925..1811c44bf34b 100644 --- a/arch/powerpc/include/asm/pci-bridge.h +++ b/arch/powerpc/include/asm/pci-bridge.h @@ -15,6 +15,24 @@ struct device_node; /* + * PCI controller operations + */ +struct pci_controller_ops { + void (*dma_dev_setup)(struct pci_dev *dev); + void (*dma_bus_setup)(struct pci_bus *bus); + + int (*probe_mode)(struct pci_bus *); + + /* Called when pci_enable_device() is called. Returns true to + * allow assignment/enabling of the device. */ + bool (*enable_device_hook)(struct pci_dev *); + + /* Called during PCI resource reassignment */ + resource_size_t (*window_alignment)(struct pci_bus *, unsigned long type); + void (*reset_secondary_bus)(struct pci_dev *dev); +}; + +/* * Structure of a PCI controller (host bridge) */ struct pci_controller { @@ -46,6 +64,7 @@ struct pci_controller { resource_size_t isa_mem_phys; resource_size_t isa_mem_size; + struct pci_controller_ops controller_ops; struct pci_ops *ops; unsigned int __iomem *cfg_addr; void __iomem *cfg_data; @@ -89,6 +108,7 @@ struct pci_controller { #ifdef CONFIG_PPC64 unsigned long buid; + struct pci_dn *pci_data; #endif /* CONFIG_PPC64 */ void *private_data; @@ -154,31 +174,51 @@ static inline int isa_vaddr_is_ioport(void __iomem *address) struct iommu_table; struct pci_dn { + int flags; +#define PCI_DN_FLAG_IOV_VF 0x01 + int busno; /* pci bus number */ int devfn; /* pci device and function number */ + int vendor_id; /* Vendor ID */ + int device_id; /* Device ID */ + int class_code; /* Device class code */ + struct pci_dn *parent; struct pci_controller *phb; /* for pci devices */ struct iommu_table *iommu_table; /* for phb's or bridges */ struct device_node *node; /* back-pointer to the device_node */ int pci_ext_config_space; /* for pci devices */ - struct pci_dev *pcidev; /* back-pointer to the pci device */ #ifdef CONFIG_EEH struct eeh_dev *edev; /* eeh device */ #endif #define IODA_INVALID_PE (-1) #ifdef CONFIG_PPC_POWERNV int pe_number; +#ifdef CONFIG_PCI_IOV + u16 vfs_expanded; /* number of VFs IOV BAR expanded */ + u16 num_vfs; /* number of VFs enabled*/ + int offset; /* PE# for the first VF PE */ +#define M64_PER_IOV 4 + int m64_per_iov; +#define IODA_INVALID_M64 (-1) + int m64_wins[PCI_SRIOV_NUM_BARS][M64_PER_IOV]; +#endif /* CONFIG_PCI_IOV */ #endif + struct list_head child_list; + struct list_head list; }; /* Get the pointer to a device_node's pci_dn */ #define PCI_DN(dn) ((struct pci_dn *) (dn)->data) +extern struct pci_dn *pci_get_pdn_by_devfn(struct pci_bus *bus, + int devfn); extern struct pci_dn *pci_get_pdn(struct pci_dev *pdev); - -extern void * update_dn_pci_info(struct device_node *dn, void *data); +extern struct pci_dn *add_dev_pci_data(struct pci_dev *pdev); +extern void remove_dev_pci_data(struct pci_dev *pdev); +extern void *update_dn_pci_info(struct device_node *dn, void *data); static inline int pci_device_from_OF_node(struct device_node *np, u8 *bus, u8 *devfn) @@ -191,20 +231,12 @@ static inline int pci_device_from_OF_node(struct device_node *np, } #if defined(CONFIG_EEH) -static inline struct eeh_dev *of_node_to_eeh_dev(struct device_node *dn) +static inline struct eeh_dev *pdn_to_eeh_dev(struct pci_dn *pdn) { - /* - * For those OF nodes whose parent isn't PCI bridge, they - * don't have PCI_DN actually. So we have to skip them for - * any EEH operations. - */ - if (!dn || !PCI_DN(dn)) - return NULL; - - return PCI_DN(dn)->edev; + return pdn ? pdn->edev : NULL; } #else -#define of_node_to_eeh_dev(x) (NULL) +#define pdn_to_eeh_dev(x) (NULL) #endif /** Find the bus corresponding to the indicated device node */ diff --git a/arch/powerpc/include/asm/pci.h b/arch/powerpc/include/asm/pci.h index 1b0739bc14b5..4aef8d660999 100644 --- a/arch/powerpc/include/asm/pci.h +++ b/arch/powerpc/include/asm/pci.h @@ -22,7 +22,7 @@ #include <asm-generic/pci-dma-compat.h> -/* Return values for ppc_md.pci_probe_mode function */ +/* Return values for pci_controller_ops.probe_mode function */ #define PCI_PROBE_NONE -1 /* Don't look at this bus at all */ #define PCI_PROBE_NORMAL 0 /* Do normal PCI probing */ #define PCI_PROBE_DEVTREE 1 /* Instantiate from device tree */ diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 4cbe23af400a..5c93f691b495 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -213,6 +213,8 @@ #define PPC_INST_LWZ 0x80000000 #define PPC_INST_STD 0xf8000000 #define PPC_INST_STDU 0xf8000001 +#define PPC_INST_STW 0x90000000 +#define PPC_INST_STWU 0x94000000 #define PPC_INST_MFLR 0x7c0802a6 #define PPC_INST_MTLR 0x7c0803a6 #define PPC_INST_CMPWI 0x2c000000 diff --git a/arch/powerpc/include/asm/ppc-pci.h b/arch/powerpc/include/asm/ppc-pci.h index db1e2b8eff3c..4122a86d6858 100644 --- a/arch/powerpc/include/asm/ppc-pci.h +++ b/arch/powerpc/include/asm/ppc-pci.h @@ -23,8 +23,6 @@ extern void pci_setup_phb_io_dynamic(struct pci_controller *hose, int primary); extern struct list_head hose_list; -extern void find_and_init_phbs(void); - extern struct pci_dev *isa_bridge_pcidev; /* may be NULL if no ISA bus */ /** Bus Unit ID macros; get low and hi 32-bits of the 64-bit BUID */ @@ -33,9 +31,14 @@ extern struct pci_dev *isa_bridge_pcidev; /* may be NULL if no ISA bus */ /* PCI device_node operations */ struct device_node; +struct pci_dn; + typedef void *(*traverse_func)(struct device_node *me, void *data); void *traverse_pci_devices(struct device_node *start, traverse_func pre, void *data); +void *traverse_pci_dn(struct pci_dn *root, + void *(*fn)(struct pci_dn *, void *), + void *data); extern void pci_devs_phb_init(void); extern void pci_devs_phb_init_dynamic(struct pci_controller *phb); @@ -76,7 +79,6 @@ static inline const char *eeh_driver_name(struct pci_dev *pdev) #endif /* CONFIG_EEH */ #else /* CONFIG_PCI */ -static inline void find_and_init_phbs(void) { } static inline void init_pci_config_tokens(void) { } #endif /* !CONFIG_PCI */ diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 7e4612528546..dd0fc18d8103 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -637,105 +637,105 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,945) /* AltiVec Registers (VPRs) */ -#define vr0 0 -#define vr1 1 -#define vr2 2 -#define vr3 3 -#define vr4 4 -#define vr5 5 -#define vr6 6 -#define vr7 7 -#define vr8 8 -#define vr9 9 -#define vr10 10 -#define vr11 11 -#define vr12 12 -#define vr13 13 -#define vr14 14 -#define vr15 15 -#define vr16 16 -#define vr17 17 -#define vr18 18 -#define vr19 19 -#define vr20 20 -#define vr21 21 -#define vr22 22 -#define vr23 23 -#define vr24 24 -#define vr25 25 -#define vr26 26 -#define vr27 27 -#define vr28 28 -#define vr29 29 -#define vr30 30 -#define vr31 31 +#define v0 0 +#define v1 1 +#define v2 2 +#define v3 3 +#define v4 4 +#define v5 5 +#define v6 6 +#define v7 7 +#define v8 8 +#define v9 9 +#define v10 10 +#define v11 11 +#define v12 12 +#define v13 13 +#define v14 14 +#define v15 15 +#define v16 16 +#define v17 17 +#define v18 18 +#define v19 19 +#define v20 20 +#define v21 21 +#define v22 22 +#define v23 23 +#define v24 24 +#define v25 25 +#define v26 26 +#define v27 27 +#define v28 28 +#define v29 29 +#define v30 30 +#define v31 31 /* VSX Registers (VSRs) */ -#define vsr0 0 -#define vsr1 1 -#define vsr2 2 -#define vsr3 3 -#define vsr4 4 -#define vsr5 5 -#define vsr6 6 -#define vsr7 7 -#define vsr8 8 -#define vsr9 9 -#define vsr10 10 -#define vsr11 11 -#define vsr12 12 -#define vsr13 13 -#define vsr14 14 -#define vsr15 15 -#define vsr16 16 -#define vsr17 17 -#define vsr18 18 -#define vsr19 19 -#define vsr20 20 -#define vsr21 21 -#define vsr22 22 -#define vsr23 23 -#define vsr24 24 -#define vsr25 25 -#define vsr26 26 -#define vsr27 27 -#define vsr28 28 -#define vsr29 29 -#define vsr30 30 -#define vsr31 31 -#define vsr32 32 -#define vsr33 33 -#define vsr34 34 -#define vsr35 35 -#define vsr36 36 -#define vsr37 37 -#define vsr38 38 -#define vsr39 39 -#define vsr40 40 -#define vsr41 41 -#define vsr42 42 -#define vsr43 43 -#define vsr44 44 -#define vsr45 45 -#define vsr46 46 -#define vsr47 47 -#define vsr48 48 -#define vsr49 49 -#define vsr50 50 -#define vsr51 51 -#define vsr52 52 -#define vsr53 53 -#define vsr54 54 -#define vsr55 55 -#define vsr56 56 -#define vsr57 57 -#define vsr58 58 -#define vsr59 59 -#define vsr60 60 -#define vsr61 61 -#define vsr62 62 -#define vsr63 63 +#define vs0 0 +#define vs1 1 +#define vs2 2 +#define vs3 3 +#define vs4 4 +#define vs5 5 +#define vs6 6 +#define vs7 7 +#define vs8 8 +#define vs9 9 +#define vs10 10 +#define vs11 11 +#define vs12 12 +#define vs13 13 +#define vs14 14 +#define vs15 15 +#define vs16 16 +#define vs17 17 +#define vs18 18 +#define vs19 19 +#define vs20 20 +#define vs21 21 +#define vs22 22 +#define vs23 23 +#define vs24 24 +#define vs25 25 +#define vs26 26 +#define vs27 27 +#define vs28 28 +#define vs29 29 +#define vs30 30 +#define vs31 31 +#define vs32 32 +#define vs33 33 +#define vs34 34 +#define vs35 35 +#define vs36 36 +#define vs37 37 +#define vs38 38 +#define vs39 39 +#define vs40 40 +#define vs41 41 +#define vs42 42 +#define vs43 43 +#define vs44 44 +#define vs45 45 +#define vs46 46 +#define vs47 47 +#define vs48 48 +#define vs49 49 +#define vs50 50 +#define vs51 51 +#define vs52 52 +#define vs53 53 +#define vs54 54 +#define vs55 55 +#define vs56 56 +#define vs57 57 +#define vs58 58 +#define vs59 59 +#define vs60 60 +#define vs61 61 +#define vs62 62 +#define vs63 63 /* SPE Registers (EVPRs) */ diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index 2e23e92a4372..7a4ede16b283 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -4,6 +4,7 @@ #include <linux/spinlock.h> #include <asm/page.h> +#include <linux/time.h> /* * Definitions for talking to the RTAS on CHRP machines. @@ -273,6 +274,7 @@ inline uint32_t rtas_ext_event_company_id(struct rtas_ext_event_log_v6 *ext_log) #define PSERIES_ELOG_SECT_ID_MANUFACT_INFO (('M' << 8) | 'I') #define PSERIES_ELOG_SECT_ID_CALL_HOME (('C' << 8) | 'H') #define PSERIES_ELOG_SECT_ID_USER_DEF (('U' << 8) | 'D') +#define PSERIES_ELOG_SECT_ID_HOTPLUG (('H' << 8) | 'P') /* Vendor specific Platform Event Log Format, Version 6, section header */ struct pseries_errorlog { @@ -296,6 +298,31 @@ inline uint16_t pseries_errorlog_length(struct pseries_errorlog *sect) return be16_to_cpu(sect->length); } +/* RTAS pseries hotplug errorlog section */ +struct pseries_hp_errorlog { + u8 resource; + u8 action; + u8 id_type; + u8 reserved; + union { + __be32 drc_index; + __be32 drc_count; + char drc_name[1]; + } _drc_u; +}; + +#define PSERIES_HP_ELOG_RESOURCE_CPU 1 +#define PSERIES_HP_ELOG_RESOURCE_MEM 2 +#define PSERIES_HP_ELOG_RESOURCE_SLOT 3 +#define PSERIES_HP_ELOG_RESOURCE_PHB 4 + +#define PSERIES_HP_ELOG_ACTION_ADD 1 +#define PSERIES_HP_ELOG_ACTION_REMOVE 2 + +#define PSERIES_HP_ELOG_ID_DRC_NAME 1 +#define PSERIES_HP_ELOG_ID_DRC_INDEX 2 +#define PSERIES_HP_ELOG_ID_DRC_COUNT 3 + struct pseries_errorlog *get_pseries_errorlog(struct rtas_error_log *log, uint16_t section_id); @@ -327,7 +354,7 @@ extern int rtas_suspend_cpu(struct rtas_suspend_me_data *data); extern int rtas_suspend_last_cpu(struct rtas_suspend_me_data *data); extern int rtas_online_cpus_mask(cpumask_var_t cpus); extern int rtas_offline_cpus_mask(cpumask_var_t cpus); -extern int rtas_ibm_suspend_me(u64 handle, int *vasi_return); +extern int rtas_ibm_suspend_me(u64 handle); struct rtc_time; extern unsigned long rtas_get_boot_time(void); @@ -343,8 +370,12 @@ extern int early_init_dt_scan_rtas(unsigned long node, extern void pSeries_log_error(char *buf, unsigned int err_type, int fatal); #ifdef CONFIG_PPC_PSERIES +extern time64_t last_rtas_event; +extern int clobbering_unread_rtas_event(void); extern int pseries_devicetree_update(s32 scope); extern void post_mobility_fixup(void); +#else +static inline int clobbering_unread_rtas_event(void) { return 0; } #endif #ifdef CONFIG_PPC_RTAS_DAEMON diff --git a/arch/powerpc/include/asm/seccomp.h b/arch/powerpc/include/asm/seccomp.h new file mode 100644 index 000000000000..c1818e35cf02 --- /dev/null +++ b/arch/powerpc/include/asm/seccomp.h @@ -0,0 +1,10 @@ +#ifndef _ASM_POWERPC_SECCOMP_H +#define _ASM_POWERPC_SECCOMP_H + +#include <linux/unistd.h> + +#define __NR_seccomp_sigreturn_32 __NR_sigreturn + +#include <asm-generic/seccomp.h> + +#endif /* _ASM_POWERPC_SECCOMP_H */ diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h index fbdf18cf954c..e9d384cbd021 100644 --- a/arch/powerpc/include/asm/setup.h +++ b/arch/powerpc/include/asm/setup.h @@ -7,7 +7,6 @@ extern void ppc_printk_progress(char *s, unsigned short hex); extern unsigned int rtas_data; -extern int mem_init_done; /* set on boot once kmalloc can be called */ extern unsigned long long memory_limit; extern unsigned long klimit; extern void *zalloc_maybe_bootmem(size_t size, gfp_t mask); diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index d607df5081a7..825663c30945 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -42,7 +42,7 @@ struct smp_ops_t { #ifdef CONFIG_PPC_SMP_MUXED_IPI void (*cause_ipi)(int cpu, unsigned long data); #endif - int (*probe)(void); + void (*probe)(void); int (*kick_cpu)(int nr); void (*setup_cpu)(int nr); void (*bringup_done)(void); @@ -125,7 +125,6 @@ extern irqreturn_t smp_ipi_demux(void); void smp_init_pSeries(void); void smp_init_cell(void); -void smp_init_celleb(void); void smp_setup_cpu_maps(void); extern int __cpu_disable(void); @@ -175,7 +174,7 @@ static inline void set_hard_smp_processor_id(int cpu, int phys) extern int smt_enabled_at_boot; -extern int smp_mpic_probe(void); +extern void smp_mpic_probe(void); extern void smp_mpic_setup_cpu(int cpu); extern int smp_generic_kick_cpu(int nr); extern int smp_generic_cpu_bootable(unsigned int nr); diff --git a/arch/powerpc/include/asm/swab.h b/arch/powerpc/include/asm/swab.h index 96f59de61855..487e09077a3e 100644 --- a/arch/powerpc/include/asm/swab.h +++ b/arch/powerpc/include/asm/swab.h @@ -9,30 +9,4 @@ #include <uapi/asm/swab.h> -static __inline__ __u16 ld_le16(const volatile __u16 *addr) -{ - __u16 val; - - __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (addr), "m" (*addr)); - return val; -} - -static __inline__ void st_le16(volatile __u16 *addr, const __u16 val) -{ - __asm__ __volatile__ ("sthbrx %1,0,%2" : "=m" (*addr) : "r" (val), "r" (addr)); -} - -static __inline__ __u32 ld_le32(const volatile __u32 *addr) -{ - __u32 val; - - __asm__ __volatile__ ("lwbrx %0,0,%1" : "=r" (val) : "r" (addr), "m" (*addr)); - return val; -} - -static __inline__ void st_le32(volatile __u32 *addr, const __u32 val) -{ - __asm__ __volatile__ ("stwbrx %1,0,%2" : "=m" (*addr) : "r" (val), "r" (addr)); -} - #endif /* _ASM_POWERPC_SWAB_H */ diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h index 91062eef582f..f1863a138b4a 100644 --- a/arch/powerpc/include/asm/systbl.h +++ b/arch/powerpc/include/asm/systbl.h @@ -367,3 +367,4 @@ SYSCALL_SPU(getrandom) SYSCALL_SPU(memfd_create) SYSCALL_SPU(bpf) COMPAT_SYS(execveat) +PPC64ONLY(switch_endian) diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index 72489799cf02..7efee4a3240b 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -39,7 +39,6 @@ */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, <0 => BUG */ @@ -55,7 +54,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .flags = 0, \ diff --git a/arch/powerpc/include/asm/ucc_slow.h b/arch/powerpc/include/asm/ucc_slow.h index c44131e68e11..233ef5fe5fde 100644 --- a/arch/powerpc/include/asm/ucc_slow.h +++ b/arch/powerpc/include/asm/ucc_slow.h @@ -251,19 +251,6 @@ void ucc_slow_enable(struct ucc_slow_private * uccs, enum comm_dir mode); */ void ucc_slow_disable(struct ucc_slow_private * uccs, enum comm_dir mode); -/* ucc_slow_poll_transmitter_now - * Immediately forces a poll of the transmitter for data to be sent. - * Typically, the hardware performs a periodic poll for data that the - * transmit routine has set up to be transmitted. In cases where - * this polling cycle is not soon enough, this optional routine can - * be invoked to force a poll right away, instead. Proper use for - * each transmission for which this functionality is desired is to - * call the transmit routine and then this routine right after. - * - * uccs - (In) pointer to the slow UCC structure. - */ -void ucc_slow_poll_transmitter_now(struct ucc_slow_private * uccs); - /* ucc_slow_graceful_stop_tx * Smoothly stops transmission on a specified slow UCC. * diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h index 36b79c31eedd..f4f8b667d75b 100644 --- a/arch/powerpc/include/asm/unistd.h +++ b/arch/powerpc/include/asm/unistd.h @@ -12,7 +12,7 @@ #include <uapi/asm/unistd.h> -#define __NR_syscalls 363 +#define __NR_syscalls 364 #define __NR__exit __NR_exit #define NR_syscalls __NR_syscalls diff --git a/arch/powerpc/include/asm/vga.h b/arch/powerpc/include/asm/vga.h index e5f8dd366212..ab3acd2f2786 100644 --- a/arch/powerpc/include/asm/vga.h +++ b/arch/powerpc/include/asm/vga.h @@ -25,12 +25,12 @@ static inline void scr_writew(u16 val, volatile u16 *addr) { - st_le16(addr, val); + *addr = cpu_to_le16(val); } static inline u16 scr_readw(volatile const u16 *addr) { - return ld_le16(addr); + return le16_to_cpu(*addr); } #define VT_BUF_HAVE_MEMCPYW diff --git a/arch/powerpc/include/asm/xics.h b/arch/powerpc/include/asm/xics.h index 6997f4a271df..0e25bdb190bb 100644 --- a/arch/powerpc/include/asm/xics.h +++ b/arch/powerpc/include/asm/xics.h @@ -146,7 +146,7 @@ extern void xics_update_irq_servers(void); extern void xics_set_cpu_giq(unsigned int gserver, unsigned int join); extern void xics_mask_unknown_vec(unsigned int vec); extern irqreturn_t xics_ipi_dispatch(int cpu); -extern int xics_smp_probe(void); +extern void xics_smp_probe(void); extern void xics_register_ics(struct ics *ics); extern void xics_teardown_cpu(void); extern void xics_kexec_teardown_cpu(int secondary); diff --git a/arch/powerpc/include/uapi/asm/Kbuild b/arch/powerpc/include/uapi/asm/Kbuild index 7a3f795ac218..79c4068be278 100644 --- a/arch/powerpc/include/uapi/asm/Kbuild +++ b/arch/powerpc/include/uapi/asm/Kbuild @@ -25,7 +25,6 @@ header-y += posix_types.h header-y += ps3fb.h header-y += ptrace.h header-y += resource.h -header-y += seccomp.h header-y += sembuf.h header-y += setup.h header-y += shmbuf.h diff --git a/arch/powerpc/include/uapi/asm/ptrace.h b/arch/powerpc/include/uapi/asm/ptrace.h index 77d2ed35b111..8036b385417d 100644 --- a/arch/powerpc/include/uapi/asm/ptrace.h +++ b/arch/powerpc/include/uapi/asm/ptrace.h @@ -136,7 +136,7 @@ struct pt_regs { #endif /* __powerpc64__ */ /* - * Get/set all the altivec registers vr0..vr31, vscr, vrsave, in one go. + * Get/set all the altivec registers v0..v31, vscr, vrsave, in one go. * The transfer totals 34 quadword. Quadwords 0-31 contain the * corresponding vector registers. Quadword 32 contains the vscr as the * last word (offset 12) within that quadword. Quadword 33 contains the diff --git a/arch/powerpc/include/uapi/asm/seccomp.h b/arch/powerpc/include/uapi/asm/seccomp.h deleted file mode 100644 index 00c1d9133cfe..000000000000 --- a/arch/powerpc/include/uapi/asm/seccomp.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef _ASM_POWERPC_SECCOMP_H -#define _ASM_POWERPC_SECCOMP_H - -#include <linux/unistd.h> - -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - -#define __NR_seccomp_read_32 __NR_read -#define __NR_seccomp_write_32 __NR_write -#define __NR_seccomp_exit_32 __NR_exit -#define __NR_seccomp_sigreturn_32 __NR_sigreturn - -#endif /* _ASM_POWERPC_SECCOMP_H */ diff --git a/arch/powerpc/include/uapi/asm/tm.h b/arch/powerpc/include/uapi/asm/tm.h index 5d836b7c1176..5047659815a5 100644 --- a/arch/powerpc/include/uapi/asm/tm.h +++ b/arch/powerpc/include/uapi/asm/tm.h @@ -11,7 +11,7 @@ #define TM_CAUSE_RESCHED 0xde #define TM_CAUSE_TLBI 0xdc #define TM_CAUSE_FAC_UNAV 0xda -#define TM_CAUSE_SYSCALL 0xd8 /* future use */ +#define TM_CAUSE_SYSCALL 0xd8 #define TM_CAUSE_MISC 0xd6 /* future use */ #define TM_CAUSE_SIGNAL 0xd4 #define TM_CAUSE_ALIGNMENT 0xd2 diff --git a/arch/powerpc/include/uapi/asm/unistd.h b/arch/powerpc/include/uapi/asm/unistd.h index ef5b5b1f3123..e4aa173dae62 100644 --- a/arch/powerpc/include/uapi/asm/unistd.h +++ b/arch/powerpc/include/uapi/asm/unistd.h @@ -385,5 +385,6 @@ #define __NR_memfd_create 360 #define __NR_bpf 361 #define __NR_execveat 362 +#define __NR_switch_endian 363 #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */ diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 502cf69b6c89..c1ebbdaac28f 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -33,7 +33,8 @@ obj-y := cputable.o ptrace.o syscalls.o \ signal.o sysfs.o cacheinfo.o time.o \ prom.o traps.o setup-common.o \ udbg.o misc.o io.o dma.o \ - misc_$(CONFIG_WORD_SIZE).o vdso32/ + misc_$(CONFIG_WORD_SIZE).o vdso32/ \ + of_platform.o prom_parse.o obj-$(CONFIG_PPC64) += setup_64.o sys_ppc32.o \ signal_64.o ptrace32.o \ paca.o nvram_64.o firmware.o @@ -47,7 +48,6 @@ obj-$(CONFIG_PPC64) += vdso64/ obj-$(CONFIG_ALTIVEC) += vecemu.o obj-$(CONFIG_PPC_970_NAP) += idle_power4.o obj-$(CONFIG_PPC_P7_NAP) += idle_power7.o -obj-$(CONFIG_PPC_OF) += of_platform.o prom_parse.o procfs-y := proc_powerpc.o obj-$(CONFIG_PROC_FS) += $(procfs-y) rtaspci-$(CONFIG_PPC64)-$(CONFIG_PCI) := rtas_pci.o diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c index ae77b7e59889..c641983bbdd6 100644 --- a/arch/powerpc/kernel/cacheinfo.c +++ b/arch/powerpc/kernel/cacheinfo.c @@ -61,12 +61,22 @@ struct cache_type_info { }; /* These are used to index the cache_type_info array. */ -#define CACHE_TYPE_UNIFIED 0 -#define CACHE_TYPE_INSTRUCTION 1 -#define CACHE_TYPE_DATA 2 +#define CACHE_TYPE_UNIFIED 0 /* cache-size, cache-block-size, etc. */ +#define CACHE_TYPE_UNIFIED_D 1 /* d-cache-size, d-cache-block-size, etc */ +#define CACHE_TYPE_INSTRUCTION 2 +#define CACHE_TYPE_DATA 3 static const struct cache_type_info cache_type_info[] = { { + /* Embedded systems that use cache-size, cache-block-size, + * etc. for the Unified (typically L2) cache. */ + .name = "Unified", + .size_prop = "cache-size", + .line_size_props = { "cache-line-size", + "cache-block-size", }, + .nr_sets_prop = "cache-sets", + }, + { /* PowerPC Processor binding says the [di]-cache-* * must be equal on unified caches, so just use * d-cache properties. */ @@ -293,7 +303,8 @@ static struct cache *cache_find_first_sibling(struct cache *cache) { struct cache *iter; - if (cache->type == CACHE_TYPE_UNIFIED) + if (cache->type == CACHE_TYPE_UNIFIED || + cache->type == CACHE_TYPE_UNIFIED_D) return cache; list_for_each_entry(iter, &cache_list, list) @@ -324,16 +335,29 @@ static bool cache_node_is_unified(const struct device_node *np) return of_get_property(np, "cache-unified", NULL); } -static struct cache *cache_do_one_devnode_unified(struct device_node *node, - int level) +/* + * Unified caches can have two different sets of tags. Most embedded + * use cache-size, etc. for the unified cache size, but open firmware systems + * use d-cache-size, etc. Check on initialization for which type we have, and + * return the appropriate structure type. Assume it's embedded if it isn't + * open firmware. If it's yet a 3rd type, then there will be missing entries + * in /sys/devices/system/cpu/cpu0/cache/index2/, and this code will need + * to be extended further. + */ +static int cache_is_unified_d(const struct device_node *np) { - struct cache *cache; + return of_get_property(np, + cache_type_info[CACHE_TYPE_UNIFIED_D].size_prop, NULL) ? + CACHE_TYPE_UNIFIED_D : CACHE_TYPE_UNIFIED; +} +/* + */ +static struct cache *cache_do_one_devnode_unified(struct device_node *node, int level) +{ pr_debug("creating L%d ucache for %s\n", level, node->full_name); - cache = new_cache(CACHE_TYPE_UNIFIED, level, node); - - return cache; + return new_cache(cache_is_unified_d(node), level, node); } static struct cache *cache_do_one_devnode_split(struct device_node *node, diff --git a/arch/powerpc/kernel/cpu_setup_power.S b/arch/powerpc/kernel/cpu_setup_power.S index 46733535cc0b..9c9b7411b28b 100644 --- a/arch/powerpc/kernel/cpu_setup_power.S +++ b/arch/powerpc/kernel/cpu_setup_power.S @@ -137,15 +137,11 @@ __init_HFSCR: /* * Clear the TLB using the specified IS form of tlbiel instruction * (invalidate by congruence class). P7 has 128 CCs., P8 has 512. - * - * r3 = IS field */ __init_tlb_power7: - li r3,0xc00 /* IS field = 0b11 */ -_GLOBAL(__flush_tlb_power7) li r6,128 mtctr r6 - mr r7,r3 /* IS field */ + li r7,0xc00 /* IS field = 0b11 */ ptesync 2: tlbiel r7 addi r7,r7,0x1000 @@ -154,11 +150,9 @@ _GLOBAL(__flush_tlb_power7) 1: blr __init_tlb_power8: - li r3,0xc00 /* IS field = 0b11 */ -_GLOBAL(__flush_tlb_power8) li r6,512 mtctr r6 - mr r7,r3 /* IS field */ + li r7,0xc00 /* IS field = 0b11 */ ptesync 2: tlbiel r7 addi r7,r7,0x1000 diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index f83046878336..60262fdf35ba 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -71,8 +71,8 @@ extern void __restore_cpu_power7(void); extern void __setup_cpu_power8(unsigned long offset, struct cpu_spec* spec); extern void __restore_cpu_power8(void); extern void __restore_cpu_a2(void); -extern void __flush_tlb_power7(unsigned long inval_selector); -extern void __flush_tlb_power8(unsigned long inval_selector); +extern void __flush_tlb_power7(unsigned int action); +extern void __flush_tlb_power8(unsigned int action); extern long __machine_check_early_realmode_p7(struct pt_regs *regs); extern long __machine_check_early_realmode_p8(struct pt_regs *regs); #endif /* CONFIG_PPC64 */ diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c index 735979764cd4..6e8d764ce47b 100644 --- a/arch/powerpc/kernel/dma-swiotlb.c +++ b/arch/powerpc/kernel/dma-swiotlb.c @@ -116,16 +116,13 @@ void __init swiotlb_detect_4g(void) } } -static int __init swiotlb_late_init(void) +static int __init check_swiotlb_enabled(void) { - if (ppc_swiotlb_enable) { + if (ppc_swiotlb_enable) swiotlb_print_info(); - set_pci_dma_ops(&swiotlb_dma_ops); - ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_swiotlb; - } else { + else swiotlb_free(); - } return 0; } -subsys_initcall(swiotlb_late_init); +subsys_initcall(check_swiotlb_enabled); diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 3b2252e7731b..a4c62eb0ee48 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -164,30 +164,34 @@ __setup("eeh=", eeh_setup); */ static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) { - struct device_node *dn = eeh_dev_to_of_node(edev); + struct pci_dn *pdn = eeh_dev_to_pdn(edev); u32 cfg; int cap, i; int n = 0, l = 0; char buffer[128]; - n += scnprintf(buf+n, len-n, "%s\n", dn->full_name); - pr_warn("EEH: of node=%s\n", dn->full_name); + n += scnprintf(buf+n, len-n, "%04x:%02x:%02x:%01x\n", + edev->phb->global_number, pdn->busno, + PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn)); + pr_warn("EEH: of node=%04x:%02x:%02x:%01x\n", + edev->phb->global_number, pdn->busno, + PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn)); - eeh_ops->read_config(dn, PCI_VENDOR_ID, 4, &cfg); + eeh_ops->read_config(pdn, PCI_VENDOR_ID, 4, &cfg); n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg); pr_warn("EEH: PCI device/vendor: %08x\n", cfg); - eeh_ops->read_config(dn, PCI_COMMAND, 4, &cfg); + eeh_ops->read_config(pdn, PCI_COMMAND, 4, &cfg); n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg); pr_warn("EEH: PCI cmd/status register: %08x\n", cfg); /* Gather bridge-specific registers */ if (edev->mode & EEH_DEV_BRIDGE) { - eeh_ops->read_config(dn, PCI_SEC_STATUS, 2, &cfg); + eeh_ops->read_config(pdn, PCI_SEC_STATUS, 2, &cfg); n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg); pr_warn("EEH: Bridge secondary status: %04x\n", cfg); - eeh_ops->read_config(dn, PCI_BRIDGE_CONTROL, 2, &cfg); + eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &cfg); n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg); pr_warn("EEH: Bridge control: %04x\n", cfg); } @@ -195,11 +199,11 @@ static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) /* Dump out the PCI-X command and status regs */ cap = edev->pcix_cap; if (cap) { - eeh_ops->read_config(dn, cap, 4, &cfg); + eeh_ops->read_config(pdn, cap, 4, &cfg); n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg); pr_warn("EEH: PCI-X cmd: %08x\n", cfg); - eeh_ops->read_config(dn, cap+4, 4, &cfg); + eeh_ops->read_config(pdn, cap+4, 4, &cfg); n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg); pr_warn("EEH: PCI-X status: %08x\n", cfg); } @@ -211,7 +215,7 @@ static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) pr_warn("EEH: PCI-E capabilities and status follow:\n"); for (i=0; i<=8; i++) { - eeh_ops->read_config(dn, cap+4*i, 4, &cfg); + eeh_ops->read_config(pdn, cap+4*i, 4, &cfg); n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); if ((i % 4) == 0) { @@ -238,7 +242,7 @@ static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) pr_warn("EEH: PCI-E AER capability register set follows:\n"); for (i=0; i<=13; i++) { - eeh_ops->read_config(dn, cap+4*i, 4, &cfg); + eeh_ops->read_config(pdn, cap+4*i, 4, &cfg); n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); if ((i % 4) == 0) { @@ -414,11 +418,11 @@ int eeh_dev_check_failure(struct eeh_dev *edev) int ret; int active_flags = (EEH_STATE_MMIO_ACTIVE | EEH_STATE_DMA_ACTIVE); unsigned long flags; - struct device_node *dn; + struct pci_dn *pdn; struct pci_dev *dev; struct eeh_pe *pe, *parent_pe, *phb_pe; int rc = 0; - const char *location; + const char *location = NULL; eeh_stats.total_mmio_ffs++; @@ -429,15 +433,14 @@ int eeh_dev_check_failure(struct eeh_dev *edev) eeh_stats.no_dn++; return 0; } - dn = eeh_dev_to_of_node(edev); dev = eeh_dev_to_pci_dev(edev); pe = eeh_dev_to_pe(edev); /* Access to IO BARs might get this far and still not want checking. */ if (!pe) { eeh_stats.ignored_check++; - pr_debug("EEH: Ignored check for %s %s\n", - eeh_pci_name(dev), dn->full_name); + pr_debug("EEH: Ignored check for %s\n", + eeh_pci_name(dev)); return 0; } @@ -473,10 +476,13 @@ int eeh_dev_check_failure(struct eeh_dev *edev) if (pe->state & EEH_PE_ISOLATED) { pe->check_count++; if (pe->check_count % EEH_MAX_FAILS == 0) { - location = of_get_property(dn, "ibm,loc-code", NULL); + pdn = eeh_dev_to_pdn(edev); + if (pdn->node) + location = of_get_property(pdn->node, "ibm,loc-code", NULL); printk(KERN_ERR "EEH: %d reads ignored for recovering device at " "location=%s driver=%s pci addr=%s\n", - pe->check_count, location, + pe->check_count, + location ? location : "unknown", eeh_driver_name(dev), eeh_pci_name(dev)); printk(KERN_ERR "EEH: Might be infinite loop in %s driver\n", eeh_driver_name(dev)); @@ -667,6 +673,55 @@ int eeh_pci_enable(struct eeh_pe *pe, int function) return rc; } +static void *eeh_disable_and_save_dev_state(void *data, void *userdata) +{ + struct eeh_dev *edev = data; + struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); + struct pci_dev *dev = userdata; + + /* + * The caller should have disabled and saved the + * state for the specified device + */ + if (!pdev || pdev == dev) + return NULL; + + /* Ensure we have D0 power state */ + pci_set_power_state(pdev, PCI_D0); + + /* Save device state */ + pci_save_state(pdev); + + /* + * Disable device to avoid any DMA traffic and + * interrupt from the device + */ + pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); + + return NULL; +} + +static void *eeh_restore_dev_state(void *data, void *userdata) +{ + struct eeh_dev *edev = data; + struct pci_dn *pdn = eeh_dev_to_pdn(edev); + struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); + struct pci_dev *dev = userdata; + + if (!pdev) + return NULL; + + /* Apply customization from firmware */ + if (pdn && eeh_ops->restore_config) + eeh_ops->restore_config(pdn); + + /* The caller should restore state for the specified device */ + if (pdev != dev) + pci_save_state(pdev); + + return NULL; +} + /** * pcibios_set_pcie_slot_reset - Set PCI-E reset state * @dev: pci device struct @@ -689,13 +744,19 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state stat switch (state) { case pcie_deassert_reset: eeh_ops->reset(pe, EEH_RESET_DEACTIVATE); + eeh_unfreeze_pe(pe, false); eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED); + eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev); break; case pcie_hot_reset: + eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); + eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); eeh_ops->reset(pe, EEH_RESET_HOT); break; case pcie_warm_reset: + eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); + eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL); break; @@ -815,15 +876,15 @@ out: */ void eeh_save_bars(struct eeh_dev *edev) { + struct pci_dn *pdn; int i; - struct device_node *dn; - if (!edev) + pdn = eeh_dev_to_pdn(edev); + if (!pdn) return; - dn = eeh_dev_to_of_node(edev); for (i = 0; i < 16; i++) - eeh_ops->read_config(dn, i * 4, 4, &edev->config_space[i]); + eeh_ops->read_config(pdn, i * 4, 4, &edev->config_space[i]); /* * For PCI bridges including root port, we need enable bus @@ -914,7 +975,7 @@ static struct notifier_block eeh_reboot_nb = { int eeh_init(void) { struct pci_controller *hose, *tmp; - struct device_node *phb; + struct pci_dn *pdn; static int cnt = 0; int ret = 0; @@ -949,20 +1010,9 @@ int eeh_init(void) return ret; /* Enable EEH for all adapters */ - if (eeh_has_flag(EEH_PROBE_MODE_DEVTREE)) { - list_for_each_entry_safe(hose, tmp, - &hose_list, list_node) { - phb = hose->dn; - traverse_pci_devices(phb, eeh_ops->of_probe, NULL); - } - } else if (eeh_has_flag(EEH_PROBE_MODE_DEV)) { - list_for_each_entry_safe(hose, tmp, - &hose_list, list_node) - pci_walk_bus(hose->bus, eeh_ops->dev_probe, NULL); - } else { - pr_warn("%s: Invalid probe mode %x", - __func__, eeh_subsystem_flags); - return -EINVAL; + list_for_each_entry_safe(hose, tmp, &hose_list, list_node) { + pdn = hose->pci_data; + traverse_pci_dn(pdn, eeh_ops->probe, NULL); } /* @@ -987,8 +1037,8 @@ int eeh_init(void) core_initcall_sync(eeh_init); /** - * eeh_add_device_early - Enable EEH for the indicated device_node - * @dn: device node for which to set up EEH + * eeh_add_device_early - Enable EEH for the indicated device node + * @pdn: PCI device node for which to set up EEH * * This routine must be used to perform EEH initialization for PCI * devices that were added after system boot (e.g. hotplug, dlpar). @@ -998,44 +1048,41 @@ core_initcall_sync(eeh_init); * on the CEC architecture, type of the device, on earlier boot * command-line arguments & etc. */ -void eeh_add_device_early(struct device_node *dn) +void eeh_add_device_early(struct pci_dn *pdn) { struct pci_controller *phb; + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); - /* - * If we're doing EEH probe based on PCI device, we - * would delay the probe until late stage because - * the PCI device isn't available this moment. - */ - if (!eeh_has_flag(EEH_PROBE_MODE_DEVTREE)) + if (!edev || !eeh_enabled()) return; - if (!of_node_to_eeh_dev(dn)) - return; - phb = of_node_to_eeh_dev(dn)->phb; - /* USB Bus children of PCI devices will not have BUID's */ - if (NULL == phb || 0 == phb->buid) + phb = edev->phb; + if (NULL == phb || + (eeh_has_flag(EEH_PROBE_MODE_DEVTREE) && 0 == phb->buid)) return; - eeh_ops->of_probe(dn, NULL); + eeh_ops->probe(pdn, NULL); } /** * eeh_add_device_tree_early - Enable EEH for the indicated device - * @dn: device node + * @pdn: PCI device node * * This routine must be used to perform EEH initialization for the * indicated PCI device that was added after system boot (e.g. * hotplug, dlpar). */ -void eeh_add_device_tree_early(struct device_node *dn) +void eeh_add_device_tree_early(struct pci_dn *pdn) { - struct device_node *sib; + struct pci_dn *n; - for_each_child_of_node(dn, sib) - eeh_add_device_tree_early(sib); - eeh_add_device_early(dn); + if (!pdn) + return; + + list_for_each_entry(n, &pdn->child_list, list) + eeh_add_device_tree_early(n); + eeh_add_device_early(pdn); } EXPORT_SYMBOL_GPL(eeh_add_device_tree_early); @@ -1048,7 +1095,7 @@ EXPORT_SYMBOL_GPL(eeh_add_device_tree_early); */ void eeh_add_device_late(struct pci_dev *dev) { - struct device_node *dn; + struct pci_dn *pdn; struct eeh_dev *edev; if (!dev || !eeh_enabled()) @@ -1056,8 +1103,8 @@ void eeh_add_device_late(struct pci_dev *dev) pr_debug("EEH: Adding device %s\n", pci_name(dev)); - dn = pci_device_to_OF_node(dev); - edev = of_node_to_eeh_dev(dn); + pdn = pci_get_pdn_by_devfn(dev->bus, dev->devfn); + edev = pdn_to_eeh_dev(pdn); if (edev->pdev == dev) { pr_debug("EEH: Already referenced !\n"); return; @@ -1089,13 +1136,6 @@ void eeh_add_device_late(struct pci_dev *dev) edev->pdev = dev; dev->dev.archdata.edev = edev; - /* - * We have to do the EEH probe here because the PCI device - * hasn't been created yet in the early stage. - */ - if (eeh_has_flag(EEH_PROBE_MODE_DEV)) - eeh_ops->dev_probe(dev, NULL); - eeh_addr_cache_insert_dev(dev); } diff --git a/arch/powerpc/kernel/eeh_cache.c b/arch/powerpc/kernel/eeh_cache.c index 07d8a2423a61..eeabeabea49c 100644 --- a/arch/powerpc/kernel/eeh_cache.c +++ b/arch/powerpc/kernel/eeh_cache.c @@ -171,30 +171,27 @@ eeh_addr_cache_insert(struct pci_dev *dev, unsigned long alo, static void __eeh_addr_cache_insert_dev(struct pci_dev *dev) { - struct device_node *dn; + struct pci_dn *pdn; struct eeh_dev *edev; int i; - dn = pci_device_to_OF_node(dev); - if (!dn) { + pdn = pci_get_pdn_by_devfn(dev->bus, dev->devfn); + if (!pdn) { pr_warn("PCI: no pci dn found for dev=%s\n", pci_name(dev)); return; } - edev = of_node_to_eeh_dev(dn); + edev = pdn_to_eeh_dev(pdn); if (!edev) { - pr_warn("PCI: no EEH dev found for dn=%s\n", - dn->full_name); + pr_warn("PCI: no EEH dev found for %s\n", + pci_name(dev)); return; } /* Skip any devices for which EEH is not enabled. */ if (!edev->pe) { -#ifdef DEBUG - pr_info("PCI: skip building address cache for=%s - %s\n", - pci_name(dev), dn->full_name); -#endif + dev_dbg(&dev->dev, "EEH: Skip building address cache\n"); return; } @@ -282,18 +279,18 @@ void eeh_addr_cache_rmv_dev(struct pci_dev *dev) */ void eeh_addr_cache_build(void) { - struct device_node *dn; + struct pci_dn *pdn; struct eeh_dev *edev; struct pci_dev *dev = NULL; spin_lock_init(&pci_io_addr_cache_root.piar_lock); for_each_pci_dev(dev) { - dn = pci_device_to_OF_node(dev); - if (!dn) + pdn = pci_get_pdn_by_devfn(dev->bus, dev->devfn); + if (!pdn) continue; - edev = of_node_to_eeh_dev(dn); + edev = pdn_to_eeh_dev(pdn); if (!edev) continue; diff --git a/arch/powerpc/kernel/eeh_dev.c b/arch/powerpc/kernel/eeh_dev.c index e5274ee9a75f..aabba94ff9cb 100644 --- a/arch/powerpc/kernel/eeh_dev.c +++ b/arch/powerpc/kernel/eeh_dev.c @@ -43,13 +43,13 @@ /** * eeh_dev_init - Create EEH device according to OF node - * @dn: device node + * @pdn: PCI device node * @data: PHB * * It will create EEH device according to the given OF node. The function * might be called by PCI emunation, DR, PHB hotplug. */ -void *eeh_dev_init(struct device_node *dn, void *data) +void *eeh_dev_init(struct pci_dn *pdn, void *data) { struct pci_controller *phb = data; struct eeh_dev *edev; @@ -63,8 +63,8 @@ void *eeh_dev_init(struct device_node *dn, void *data) } /* Associate EEH device with OF node */ - PCI_DN(dn)->edev = edev; - edev->dn = dn; + pdn->edev = edev; + edev->pdn = pdn; edev->phb = phb; INIT_LIST_HEAD(&edev->list); @@ -80,16 +80,16 @@ void *eeh_dev_init(struct device_node *dn, void *data) */ void eeh_dev_phb_init_dynamic(struct pci_controller *phb) { - struct device_node *dn = phb->dn; + struct pci_dn *root = phb->pci_data; /* EEH PE for PHB */ eeh_phb_pe_create(phb); /* EEH device for PHB */ - eeh_dev_init(dn, phb); + eeh_dev_init(root, phb); /* EEH devices for children OF nodes */ - traverse_pci_devices(dn, eeh_dev_init, phb); + traverse_pci_dn(root, eeh_dev_init, phb); } /** diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index d099540c0f56..24768ff3cb73 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -83,28 +83,6 @@ static inline void eeh_pcid_put(struct pci_dev *pdev) module_put(pdev->driver->driver.owner); } -#if 0 -static void print_device_node_tree(struct pci_dn *pdn, int dent) -{ - int i; - struct device_node *pc; - - if (!pdn) - return; - for (i = 0; i < dent; i++) - printk(" "); - printk("dn=%s mode=%x \tcfg_addr=%x pe_addr=%x \tfull=%s\n", - pdn->node->name, pdn->eeh_mode, pdn->eeh_config_addr, - pdn->eeh_pe_config_addr, pdn->node->full_name); - dent += 3; - pc = pdn->node->child; - while (pc) { - print_device_node_tree(PCI_DN(pc), dent); - pc = pc->sibling; - } -} -#endif - /** * eeh_disable_irq - Disable interrupt for the recovering device * @dev: PCI device diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index 1e4946c36f9e..35f0b62259bb 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -291,27 +291,25 @@ struct eeh_pe *eeh_pe_get(struct eeh_dev *edev) */ static struct eeh_pe *eeh_pe_get_parent(struct eeh_dev *edev) { - struct device_node *dn; struct eeh_dev *parent; + struct pci_dn *pdn = eeh_dev_to_pdn(edev); /* * It might have the case for the indirect parent * EEH device already having associated PE, but * the direct parent EEH device doesn't have yet. */ - dn = edev->dn->parent; - while (dn) { + pdn = pdn ? pdn->parent : NULL; + while (pdn) { /* We're poking out of PCI territory */ - if (!PCI_DN(dn)) return NULL; - - parent = of_node_to_eeh_dev(dn); - /* We're poking out of PCI territory */ - if (!parent) return NULL; + parent = pdn_to_eeh_dev(pdn); + if (!parent) + return NULL; if (parent->pe) return parent->pe; - dn = dn->parent; + pdn = pdn->parent; } return NULL; @@ -330,6 +328,13 @@ int eeh_add_to_parent_pe(struct eeh_dev *edev) { struct eeh_pe *pe, *parent; + /* Check if the PE number is valid */ + if (!eeh_has_flag(EEH_VALID_PE_ZERO) && !edev->pe_config_addr) { + pr_err("%s: Invalid PE#0 for edev 0x%x on PHB#%d\n", + __func__, edev->config_addr, edev->phb->global_number); + return -EINVAL; + } + /* * Search the PE has been existing or not according * to the PE address. If that has been existing, the @@ -338,21 +343,18 @@ int eeh_add_to_parent_pe(struct eeh_dev *edev) */ pe = eeh_pe_get(edev); if (pe && !(pe->type & EEH_PE_INVALID)) { - if (!edev->pe_config_addr) { - pr_err("%s: PE with addr 0x%x already exists\n", - __func__, edev->config_addr); - return -EEXIST; - } - /* Mark the PE as type of PCI bus */ pe->type = EEH_PE_BUS; edev->pe = pe; /* Put the edev to PE */ list_add_tail(&edev->list, &pe->edevs); - pr_debug("EEH: Add %s to Bus PE#%x\n", - edev->dn->full_name, pe->addr); - + pr_debug("EEH: Add %04x:%02x:%02x.%01x to Bus PE#%x\n", + edev->phb->global_number, + edev->config_addr >> 8, + PCI_SLOT(edev->config_addr & 0xFF), + PCI_FUNC(edev->config_addr & 0xFF), + pe->addr); return 0; } else if (pe && (pe->type & EEH_PE_INVALID)) { list_add_tail(&edev->list, &pe->edevs); @@ -368,9 +370,14 @@ int eeh_add_to_parent_pe(struct eeh_dev *edev) parent->type &= ~(EEH_PE_INVALID | EEH_PE_KEEP); parent = parent->parent; } - pr_debug("EEH: Add %s to Device PE#%x, Parent PE#%x\n", - edev->dn->full_name, pe->addr, pe->parent->addr); + pr_debug("EEH: Add %04x:%02x:%02x.%01x to Device " + "PE#%x, Parent PE#%x\n", + edev->phb->global_number, + edev->config_addr >> 8, + PCI_SLOT(edev->config_addr & 0xFF), + PCI_FUNC(edev->config_addr & 0xFF), + pe->addr, pe->parent->addr); return 0; } @@ -409,8 +416,13 @@ int eeh_add_to_parent_pe(struct eeh_dev *edev) list_add_tail(&pe->child, &parent->child_list); list_add_tail(&edev->list, &pe->edevs); edev->pe = pe; - pr_debug("EEH: Add %s to Device PE#%x, Parent PE#%x\n", - edev->dn->full_name, pe->addr, pe->parent->addr); + pr_debug("EEH: Add %04x:%02x:%02x.%01x to " + "Device PE#%x, Parent PE#%x\n", + edev->phb->global_number, + edev->config_addr >> 8, + PCI_SLOT(edev->config_addr & 0xFF), + PCI_FUNC(edev->config_addr & 0xFF), + pe->addr, pe->parent->addr); return 0; } @@ -430,8 +442,11 @@ int eeh_rmv_from_parent_pe(struct eeh_dev *edev) int cnt; if (!edev->pe) { - pr_debug("%s: No PE found for EEH device %s\n", - __func__, edev->dn->full_name); + pr_debug("%s: No PE found for device %04x:%02x:%02x.%01x\n", + __func__, edev->phb->global_number, + edev->config_addr >> 8, + PCI_SLOT(edev->config_addr & 0xFF), + PCI_FUNC(edev->config_addr & 0xFF)); return -EEXIST; } @@ -653,9 +668,9 @@ void eeh_pe_state_clear(struct eeh_pe *pe, int state) * blocked on normal path during the stage. So we need utilize * eeh operations, which is always permitted. */ -static void eeh_bridge_check_link(struct eeh_dev *edev, - struct device_node *dn) +static void eeh_bridge_check_link(struct eeh_dev *edev) { + struct pci_dn *pdn = eeh_dev_to_pdn(edev); int cap; uint32_t val; int timeout = 0; @@ -675,32 +690,32 @@ static void eeh_bridge_check_link(struct eeh_dev *edev, /* Check slot status */ cap = edev->pcie_cap; - eeh_ops->read_config(dn, cap + PCI_EXP_SLTSTA, 2, &val); + eeh_ops->read_config(pdn, cap + PCI_EXP_SLTSTA, 2, &val); if (!(val & PCI_EXP_SLTSTA_PDS)) { pr_debug(" No card in the slot (0x%04x) !\n", val); return; } /* Check power status if we have the capability */ - eeh_ops->read_config(dn, cap + PCI_EXP_SLTCAP, 2, &val); + eeh_ops->read_config(pdn, cap + PCI_EXP_SLTCAP, 2, &val); if (val & PCI_EXP_SLTCAP_PCP) { - eeh_ops->read_config(dn, cap + PCI_EXP_SLTCTL, 2, &val); + eeh_ops->read_config(pdn, cap + PCI_EXP_SLTCTL, 2, &val); if (val & PCI_EXP_SLTCTL_PCC) { pr_debug(" In power-off state, power it on ...\n"); val &= ~(PCI_EXP_SLTCTL_PCC | PCI_EXP_SLTCTL_PIC); val |= (0x0100 & PCI_EXP_SLTCTL_PIC); - eeh_ops->write_config(dn, cap + PCI_EXP_SLTCTL, 2, val); + eeh_ops->write_config(pdn, cap + PCI_EXP_SLTCTL, 2, val); msleep(2 * 1000); } } /* Enable link */ - eeh_ops->read_config(dn, cap + PCI_EXP_LNKCTL, 2, &val); + eeh_ops->read_config(pdn, cap + PCI_EXP_LNKCTL, 2, &val); val &= ~PCI_EXP_LNKCTL_LD; - eeh_ops->write_config(dn, cap + PCI_EXP_LNKCTL, 2, val); + eeh_ops->write_config(pdn, cap + PCI_EXP_LNKCTL, 2, val); /* Check link */ - eeh_ops->read_config(dn, cap + PCI_EXP_LNKCAP, 4, &val); + eeh_ops->read_config(pdn, cap + PCI_EXP_LNKCAP, 4, &val); if (!(val & PCI_EXP_LNKCAP_DLLLARC)) { pr_debug(" No link reporting capability (0x%08x) \n", val); msleep(1000); @@ -713,7 +728,7 @@ static void eeh_bridge_check_link(struct eeh_dev *edev, msleep(20); timeout += 20; - eeh_ops->read_config(dn, cap + PCI_EXP_LNKSTA, 2, &val); + eeh_ops->read_config(pdn, cap + PCI_EXP_LNKSTA, 2, &val); if (val & PCI_EXP_LNKSTA_DLLLA) break; } @@ -728,9 +743,9 @@ static void eeh_bridge_check_link(struct eeh_dev *edev, #define BYTE_SWAP(OFF) (8*((OFF)/4)+3-(OFF)) #define SAVED_BYTE(OFF) (((u8 *)(edev->config_space))[BYTE_SWAP(OFF)]) -static void eeh_restore_bridge_bars(struct eeh_dev *edev, - struct device_node *dn) +static void eeh_restore_bridge_bars(struct eeh_dev *edev) { + struct pci_dn *pdn = eeh_dev_to_pdn(edev); int i; /* @@ -738,49 +753,49 @@ static void eeh_restore_bridge_bars(struct eeh_dev *edev, * Bus numbers and windows: 0x18 - 0x30 */ for (i = 4; i < 13; i++) - eeh_ops->write_config(dn, i*4, 4, edev->config_space[i]); + eeh_ops->write_config(pdn, i*4, 4, edev->config_space[i]); /* Rom: 0x38 */ - eeh_ops->write_config(dn, 14*4, 4, edev->config_space[14]); + eeh_ops->write_config(pdn, 14*4, 4, edev->config_space[14]); /* Cache line & Latency timer: 0xC 0xD */ - eeh_ops->write_config(dn, PCI_CACHE_LINE_SIZE, 1, + eeh_ops->write_config(pdn, PCI_CACHE_LINE_SIZE, 1, SAVED_BYTE(PCI_CACHE_LINE_SIZE)); - eeh_ops->write_config(dn, PCI_LATENCY_TIMER, 1, + eeh_ops->write_config(pdn, PCI_LATENCY_TIMER, 1, SAVED_BYTE(PCI_LATENCY_TIMER)); /* Max latency, min grant, interrupt ping and line: 0x3C */ - eeh_ops->write_config(dn, 15*4, 4, edev->config_space[15]); + eeh_ops->write_config(pdn, 15*4, 4, edev->config_space[15]); /* PCI Command: 0x4 */ - eeh_ops->write_config(dn, PCI_COMMAND, 4, edev->config_space[1]); + eeh_ops->write_config(pdn, PCI_COMMAND, 4, edev->config_space[1]); /* Check the PCIe link is ready */ - eeh_bridge_check_link(edev, dn); + eeh_bridge_check_link(edev); } -static void eeh_restore_device_bars(struct eeh_dev *edev, - struct device_node *dn) +static void eeh_restore_device_bars(struct eeh_dev *edev) { + struct pci_dn *pdn = eeh_dev_to_pdn(edev); int i; u32 cmd; for (i = 4; i < 10; i++) - eeh_ops->write_config(dn, i*4, 4, edev->config_space[i]); + eeh_ops->write_config(pdn, i*4, 4, edev->config_space[i]); /* 12 == Expansion ROM Address */ - eeh_ops->write_config(dn, 12*4, 4, edev->config_space[12]); + eeh_ops->write_config(pdn, 12*4, 4, edev->config_space[12]); - eeh_ops->write_config(dn, PCI_CACHE_LINE_SIZE, 1, + eeh_ops->write_config(pdn, PCI_CACHE_LINE_SIZE, 1, SAVED_BYTE(PCI_CACHE_LINE_SIZE)); - eeh_ops->write_config(dn, PCI_LATENCY_TIMER, 1, + eeh_ops->write_config(pdn, PCI_LATENCY_TIMER, 1, SAVED_BYTE(PCI_LATENCY_TIMER)); /* max latency, min grant, interrupt pin and line */ - eeh_ops->write_config(dn, 15*4, 4, edev->config_space[15]); + eeh_ops->write_config(pdn, 15*4, 4, edev->config_space[15]); /* * Restore PERR & SERR bits, some devices require it, * don't touch the other command bits */ - eeh_ops->read_config(dn, PCI_COMMAND, 4, &cmd); + eeh_ops->read_config(pdn, PCI_COMMAND, 4, &cmd); if (edev->config_space[1] & PCI_COMMAND_PARITY) cmd |= PCI_COMMAND_PARITY; else @@ -789,7 +804,7 @@ static void eeh_restore_device_bars(struct eeh_dev *edev, cmd |= PCI_COMMAND_SERR; else cmd &= ~PCI_COMMAND_SERR; - eeh_ops->write_config(dn, PCI_COMMAND, 4, cmd); + eeh_ops->write_config(pdn, PCI_COMMAND, 4, cmd); } /** @@ -804,16 +819,16 @@ static void eeh_restore_device_bars(struct eeh_dev *edev, static void *eeh_restore_one_device_bars(void *data, void *flag) { struct eeh_dev *edev = (struct eeh_dev *)data; - struct device_node *dn = eeh_dev_to_of_node(edev); + struct pci_dn *pdn = eeh_dev_to_pdn(edev); /* Do special restore for bridges */ if (edev->mode & EEH_DEV_BRIDGE) - eeh_restore_bridge_bars(edev, dn); + eeh_restore_bridge_bars(edev); else - eeh_restore_device_bars(edev, dn); + eeh_restore_device_bars(edev); - if (eeh_ops->restore_config) - eeh_ops->restore_config(dn); + if (eeh_ops->restore_config && pdn) + eeh_ops->restore_config(pdn); return NULL; } diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index d180caf2d6de..8ca9434c40e6 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -34,6 +34,7 @@ #include <asm/ftrace.h> #include <asm/hw_irq.h> #include <asm/context_tracking.h> +#include <asm/tm.h> /* * System calls. @@ -145,6 +146,24 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) andi. r11,r10,_TIF_SYSCALL_DOTRACE bne syscall_dotrace .Lsyscall_dotrace_cont: +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM +BEGIN_FTR_SECTION + b 1f +END_FTR_SECTION_IFCLR(CPU_FTR_TM) + extrdi. r11, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */ + beq+ 1f + + /* Doom the transaction and don't perform the syscall: */ + mfmsr r11 + li r12, 1 + rldimi r11, r12, MSR_TM_LG, 63-MSR_TM_LG + mtmsrd r11, 0 + li r11, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT) + TABORT(R11) + + b .Lsyscall_exit +1: +#endif cmpldi 0,r0,NR_syscalls bge- syscall_enosys @@ -356,6 +375,11 @@ _GLOBAL(ppc64_swapcontext) bl sys_swapcontext b .Lsyscall_exit +_GLOBAL(ppc_switch_endian) + bl save_nvgprs + bl sys_switch_endian + b .Lsyscall_exit + _GLOBAL(ret_from_fork) bl schedule_tail REST_NVGPRS(r1) diff --git a/arch/powerpc/kernel/idle_power7.S b/arch/powerpc/kernel/idle_power7.S index 05adc8bbdef8..eeaa0d5f69d5 100644 --- a/arch/powerpc/kernel/idle_power7.S +++ b/arch/powerpc/kernel/idle_power7.S @@ -94,6 +94,7 @@ _GLOBAL(power7_powersave_common) beq 1f addi r1,r1,INT_FRAME_SIZE ld r0,16(r1) + li r3,0 /* Return 0 (no nap) */ mtlr r0 blr diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c index b6f123ab90ed..2c647b1e62e4 100644 --- a/arch/powerpc/kernel/mce_power.c +++ b/arch/powerpc/kernel/mce_power.c @@ -28,6 +28,55 @@ #include <asm/mce.h> #include <asm/machdep.h> +static void flush_tlb_206(unsigned int num_sets, unsigned int action) +{ + unsigned long rb; + unsigned int i; + + switch (action) { + case TLB_INVAL_SCOPE_GLOBAL: + rb = TLBIEL_INVAL_SET; + break; + case TLB_INVAL_SCOPE_LPID: + rb = TLBIEL_INVAL_SET_LPID; + break; + default: + BUG(); + break; + } + + asm volatile("ptesync" : : : "memory"); + for (i = 0; i < num_sets; i++) { + asm volatile("tlbiel %0" : : "r" (rb)); + rb += 1 << TLBIEL_INVAL_SET_SHIFT; + } + asm volatile("ptesync" : : : "memory"); +} + +/* + * Generic routine to flush TLB on power7. This routine is used as + * flush_tlb hook in cpu_spec for Power7 processor. + * + * action => TLB_INVAL_SCOPE_GLOBAL: Invalidate all TLBs. + * TLB_INVAL_SCOPE_LPID: Invalidate TLB for current LPID. + */ +void __flush_tlb_power7(unsigned int action) +{ + flush_tlb_206(POWER7_TLB_SETS, action); +} + +/* + * Generic routine to flush TLB on power8. This routine is used as + * flush_tlb hook in cpu_spec for power8 processor. + * + * action => TLB_INVAL_SCOPE_GLOBAL: Invalidate all TLBs. + * TLB_INVAL_SCOPE_LPID: Invalidate TLB for current LPID. + */ +void __flush_tlb_power8(unsigned int action) +{ + flush_tlb_206(POWER8_TLB_SETS, action); +} + /* flush SLBs and reload */ static void flush_and_reload_slb(void) { @@ -79,7 +128,7 @@ static long mce_handle_derror(uint64_t dsisr, uint64_t slb_error_bits) } if (dsisr & P7_DSISR_MC_TLB_MULTIHIT_MFTLB) { if (cur_cpu_spec && cur_cpu_spec->flush_tlb) - cur_cpu_spec->flush_tlb(TLBIEL_INVAL_SET); + cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_GLOBAL); /* reset error bits */ dsisr &= ~P7_DSISR_MC_TLB_MULTIHIT_MFTLB; } @@ -110,7 +159,7 @@ static long mce_handle_common_ierror(uint64_t srr1) break; case P7_SRR1_MC_IFETCH_TLB_MULTIHIT: if (cur_cpu_spec && cur_cpu_spec->flush_tlb) { - cur_cpu_spec->flush_tlb(TLBIEL_INVAL_SET); + cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_GLOBAL); handled = 1; } break; diff --git a/arch/powerpc/kernel/nvram_64.c b/arch/powerpc/kernel/nvram_64.c index 34f7c9b7cd96..1e703f8ebad4 100644 --- a/arch/powerpc/kernel/nvram_64.c +++ b/arch/powerpc/kernel/nvram_64.c @@ -26,6 +26,9 @@ #include <linux/init.h> #include <linux/slab.h> #include <linux/spinlock.h> +#include <linux/kmsg_dump.h> +#include <linux/pstore.h> +#include <linux/zlib.h> #include <asm/uaccess.h> #include <asm/nvram.h> #include <asm/rtas.h> @@ -54,6 +57,680 @@ struct nvram_partition { static LIST_HEAD(nvram_partitions); +#ifdef CONFIG_PPC_PSERIES +struct nvram_os_partition rtas_log_partition = { + .name = "ibm,rtas-log", + .req_size = 2079, + .min_size = 1055, + .index = -1, + .os_partition = true +}; +#endif + +struct nvram_os_partition oops_log_partition = { + .name = "lnx,oops-log", + .req_size = 4000, + .min_size = 2000, + .index = -1, + .os_partition = true +}; + +static const char *nvram_os_partitions[] = { +#ifdef CONFIG_PPC_PSERIES + "ibm,rtas-log", +#endif + "lnx,oops-log", + NULL +}; + +static void oops_to_nvram(struct kmsg_dumper *dumper, + enum kmsg_dump_reason reason); + +static struct kmsg_dumper nvram_kmsg_dumper = { + .dump = oops_to_nvram +}; + +/* + * For capturing and compressing an oops or panic report... + + * big_oops_buf[] holds the uncompressed text we're capturing. + * + * oops_buf[] holds the compressed text, preceded by a oops header. + * oops header has u16 holding the version of oops header (to differentiate + * between old and new format header) followed by u16 holding the length of + * the compressed* text (*Or uncompressed, if compression fails.) and u64 + * holding the timestamp. oops_buf[] gets written to NVRAM. + * + * oops_log_info points to the header. oops_data points to the compressed text. + * + * +- oops_buf + * | +- oops_data + * v v + * +-----------+-----------+-----------+------------------------+ + * | version | length | timestamp | text | + * | (2 bytes) | (2 bytes) | (8 bytes) | (oops_data_sz bytes) | + * +-----------+-----------+-----------+------------------------+ + * ^ + * +- oops_log_info + * + * We preallocate these buffers during init to avoid kmalloc during oops/panic. + */ +static size_t big_oops_buf_sz; +static char *big_oops_buf, *oops_buf; +static char *oops_data; +static size_t oops_data_sz; + +/* Compression parameters */ +#define COMPR_LEVEL 6 +#define WINDOW_BITS 12 +#define MEM_LEVEL 4 +static struct z_stream_s stream; + +#ifdef CONFIG_PSTORE +#ifdef CONFIG_PPC_POWERNV +static struct nvram_os_partition skiboot_partition = { + .name = "ibm,skiboot", + .index = -1, + .os_partition = false +}; +#endif + +#ifdef CONFIG_PPC_PSERIES +static struct nvram_os_partition of_config_partition = { + .name = "of-config", + .index = -1, + .os_partition = false +}; +#endif + +static struct nvram_os_partition common_partition = { + .name = "common", + .index = -1, + .os_partition = false +}; + +static enum pstore_type_id nvram_type_ids[] = { + PSTORE_TYPE_DMESG, + PSTORE_TYPE_PPC_COMMON, + -1, + -1, + -1 +}; +static int read_type; +#endif + +/* nvram_write_os_partition + * + * We need to buffer the error logs into nvram to ensure that we have + * the failure information to decode. If we have a severe error there + * is no way to guarantee that the OS or the machine is in a state to + * get back to user land and write the error to disk. For example if + * the SCSI device driver causes a Machine Check by writing to a bad + * IO address, there is no way of guaranteeing that the device driver + * is in any state that is would also be able to write the error data + * captured to disk, thus we buffer it in NVRAM for analysis on the + * next boot. + * + * In NVRAM the partition containing the error log buffer will looks like: + * Header (in bytes): + * +-----------+----------+--------+------------+------------------+ + * | signature | checksum | length | name | data | + * |0 |1 |2 3|4 15|16 length-1| + * +-----------+----------+--------+------------+------------------+ + * + * The 'data' section would look like (in bytes): + * +--------------+------------+-----------------------------------+ + * | event_logged | sequence # | error log | + * |0 3|4 7|8 error_log_size-1| + * +--------------+------------+-----------------------------------+ + * + * event_logged: 0 if event has not been logged to syslog, 1 if it has + * sequence #: The unique sequence # for each event. (until it wraps) + * error log: The error log from event_scan + */ +int nvram_write_os_partition(struct nvram_os_partition *part, + char *buff, int length, + unsigned int err_type, + unsigned int error_log_cnt) +{ + int rc; + loff_t tmp_index; + struct err_log_info info; + + if (part->index == -1) + return -ESPIPE; + + if (length > part->size) + length = part->size; + + info.error_type = cpu_to_be32(err_type); + info.seq_num = cpu_to_be32(error_log_cnt); + + tmp_index = part->index; + + rc = ppc_md.nvram_write((char *)&info, sizeof(struct err_log_info), + &tmp_index); + if (rc <= 0) { + pr_err("%s: Failed nvram_write (%d)\n", __func__, rc); + return rc; + } + + rc = ppc_md.nvram_write(buff, length, &tmp_index); + if (rc <= 0) { + pr_err("%s: Failed nvram_write (%d)\n", __func__, rc); + return rc; + } + + return 0; +} + +/* nvram_read_partition + * + * Reads nvram partition for at most 'length' + */ +int nvram_read_partition(struct nvram_os_partition *part, char *buff, + int length, unsigned int *err_type, + unsigned int *error_log_cnt) +{ + int rc; + loff_t tmp_index; + struct err_log_info info; + + if (part->index == -1) + return -1; + + if (length > part->size) + length = part->size; + + tmp_index = part->index; + + if (part->os_partition) { + rc = ppc_md.nvram_read((char *)&info, + sizeof(struct err_log_info), + &tmp_index); + if (rc <= 0) { + pr_err("%s: Failed nvram_read (%d)\n", __func__, rc); + return rc; + } + } + + rc = ppc_md.nvram_read(buff, length, &tmp_index); + if (rc <= 0) { + pr_err("%s: Failed nvram_read (%d)\n", __func__, rc); + return rc; + } + + if (part->os_partition) { + *error_log_cnt = be32_to_cpu(info.seq_num); + *err_type = be32_to_cpu(info.error_type); + } + + return 0; +} + +/* nvram_init_os_partition + * + * This sets up a partition with an "OS" signature. + * + * The general strategy is the following: + * 1.) If a partition with the indicated name already exists... + * - If it's large enough, use it. + * - Otherwise, recycle it and keep going. + * 2.) Search for a free partition that is large enough. + * 3.) If there's not a free partition large enough, recycle any obsolete + * OS partitions and try again. + * 4.) Will first try getting a chunk that will satisfy the requested size. + * 5.) If a chunk of the requested size cannot be allocated, then try finding + * a chunk that will satisfy the minum needed. + * + * Returns 0 on success, else -1. + */ +int __init nvram_init_os_partition(struct nvram_os_partition *part) +{ + loff_t p; + int size; + + /* Look for ours */ + p = nvram_find_partition(part->name, NVRAM_SIG_OS, &size); + + /* Found one but too small, remove it */ + if (p && size < part->min_size) { + pr_info("nvram: Found too small %s partition," + " removing it...\n", part->name); + nvram_remove_partition(part->name, NVRAM_SIG_OS, NULL); + p = 0; + } + + /* Create one if we didn't find */ + if (!p) { + p = nvram_create_partition(part->name, NVRAM_SIG_OS, + part->req_size, part->min_size); + if (p == -ENOSPC) { + pr_info("nvram: No room to create %s partition, " + "deleting any obsolete OS partitions...\n", + part->name); + nvram_remove_partition(NULL, NVRAM_SIG_OS, + nvram_os_partitions); + p = nvram_create_partition(part->name, NVRAM_SIG_OS, + part->req_size, part->min_size); + } + } + + if (p <= 0) { + pr_err("nvram: Failed to find or create %s" + " partition, err %d\n", part->name, (int)p); + return -1; + } + + part->index = p; + part->size = nvram_get_partition_size(p) - sizeof(struct err_log_info); + + return 0; +} + +/* Derived from logfs_compress() */ +static int nvram_compress(const void *in, void *out, size_t inlen, + size_t outlen) +{ + int err, ret; + + ret = -EIO; + err = zlib_deflateInit2(&stream, COMPR_LEVEL, Z_DEFLATED, WINDOW_BITS, + MEM_LEVEL, Z_DEFAULT_STRATEGY); + if (err != Z_OK) + goto error; + + stream.next_in = in; + stream.avail_in = inlen; + stream.total_in = 0; + stream.next_out = out; + stream.avail_out = outlen; + stream.total_out = 0; + + err = zlib_deflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) + goto error; + + err = zlib_deflateEnd(&stream); + if (err != Z_OK) + goto error; + + if (stream.total_out >= stream.total_in) + goto error; + + ret = stream.total_out; +error: + return ret; +} + +/* Compress the text from big_oops_buf into oops_buf. */ +static int zip_oops(size_t text_len) +{ + struct oops_log_info *oops_hdr = (struct oops_log_info *)oops_buf; + int zipped_len = nvram_compress(big_oops_buf, oops_data, text_len, + oops_data_sz); + if (zipped_len < 0) { + pr_err("nvram: compression failed; returned %d\n", zipped_len); + pr_err("nvram: logging uncompressed oops/panic report\n"); + return -1; + } + oops_hdr->version = cpu_to_be16(OOPS_HDR_VERSION); + oops_hdr->report_length = cpu_to_be16(zipped_len); + oops_hdr->timestamp = cpu_to_be64(ktime_get_real_seconds()); + return 0; +} + +#ifdef CONFIG_PSTORE +static int nvram_pstore_open(struct pstore_info *psi) +{ + /* Reset the iterator to start reading partitions again */ + read_type = -1; + return 0; +} + +/** + * nvram_pstore_write - pstore write callback for nvram + * @type: Type of message logged + * @reason: reason behind dump (oops/panic) + * @id: identifier to indicate the write performed + * @part: pstore writes data to registered buffer in parts, + * part number will indicate the same. + * @count: Indicates oops count + * @compressed: Flag to indicate the log is compressed + * @size: number of bytes written to the registered buffer + * @psi: registered pstore_info structure + * + * Called by pstore_dump() when an oops or panic report is logged in the + * printk buffer. + * Returns 0 on successful write. + */ +static int nvram_pstore_write(enum pstore_type_id type, + enum kmsg_dump_reason reason, + u64 *id, unsigned int part, int count, + bool compressed, size_t size, + struct pstore_info *psi) +{ + int rc; + unsigned int err_type = ERR_TYPE_KERNEL_PANIC; + struct oops_log_info *oops_hdr = (struct oops_log_info *) oops_buf; + + /* part 1 has the recent messages from printk buffer */ + if (part > 1 || (type != PSTORE_TYPE_DMESG)) + return -1; + + if (clobbering_unread_rtas_event()) + return -1; + + oops_hdr->version = cpu_to_be16(OOPS_HDR_VERSION); + oops_hdr->report_length = cpu_to_be16(size); + oops_hdr->timestamp = cpu_to_be64(ktime_get_real_seconds()); + + if (compressed) + err_type = ERR_TYPE_KERNEL_PANIC_GZ; + + rc = nvram_write_os_partition(&oops_log_partition, oops_buf, + (int) (sizeof(*oops_hdr) + size), err_type, count); + + if (rc != 0) + return rc; + + *id = part; + return 0; +} + +/* + * Reads the oops/panic report, rtas, of-config and common partition. + * Returns the length of the data we read from each partition. + * Returns 0 if we've been called before. + */ +static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, + int *count, struct timespec *time, char **buf, + bool *compressed, struct pstore_info *psi) +{ + struct oops_log_info *oops_hdr; + unsigned int err_type, id_no, size = 0; + struct nvram_os_partition *part = NULL; + char *buff = NULL; + int sig = 0; + loff_t p; + + read_type++; + + switch (nvram_type_ids[read_type]) { + case PSTORE_TYPE_DMESG: + part = &oops_log_partition; + *type = PSTORE_TYPE_DMESG; + break; + case PSTORE_TYPE_PPC_COMMON: + sig = NVRAM_SIG_SYS; + part = &common_partition; + *type = PSTORE_TYPE_PPC_COMMON; + *id = PSTORE_TYPE_PPC_COMMON; + time->tv_sec = 0; + time->tv_nsec = 0; + break; +#ifdef CONFIG_PPC_PSERIES + case PSTORE_TYPE_PPC_RTAS: + part = &rtas_log_partition; + *type = PSTORE_TYPE_PPC_RTAS; + time->tv_sec = last_rtas_event; + time->tv_nsec = 0; + break; + case PSTORE_TYPE_PPC_OF: + sig = NVRAM_SIG_OF; + part = &of_config_partition; + *type = PSTORE_TYPE_PPC_OF; + *id = PSTORE_TYPE_PPC_OF; + time->tv_sec = 0; + time->tv_nsec = 0; + break; +#endif +#ifdef CONFIG_PPC_POWERNV + case PSTORE_TYPE_PPC_OPAL: + sig = NVRAM_SIG_FW; + part = &skiboot_partition; + *type = PSTORE_TYPE_PPC_OPAL; + *id = PSTORE_TYPE_PPC_OPAL; + time->tv_sec = 0; + time->tv_nsec = 0; + break; +#endif + default: + return 0; + } + + if (!part->os_partition) { + p = nvram_find_partition(part->name, sig, &size); + if (p <= 0) { + pr_err("nvram: Failed to find partition %s, " + "err %d\n", part->name, (int)p); + return 0; + } + part->index = p; + part->size = size; + } + + buff = kmalloc(part->size, GFP_KERNEL); + + if (!buff) + return -ENOMEM; + + if (nvram_read_partition(part, buff, part->size, &err_type, &id_no)) { + kfree(buff); + return 0; + } + + *count = 0; + + if (part->os_partition) + *id = id_no; + + if (nvram_type_ids[read_type] == PSTORE_TYPE_DMESG) { + size_t length, hdr_size; + + oops_hdr = (struct oops_log_info *)buff; + if (be16_to_cpu(oops_hdr->version) < OOPS_HDR_VERSION) { + /* Old format oops header had 2-byte record size */ + hdr_size = sizeof(u16); + length = be16_to_cpu(oops_hdr->version); + time->tv_sec = 0; + time->tv_nsec = 0; + } else { + hdr_size = sizeof(*oops_hdr); + length = be16_to_cpu(oops_hdr->report_length); + time->tv_sec = be64_to_cpu(oops_hdr->timestamp); + time->tv_nsec = 0; + } + *buf = kmalloc(length, GFP_KERNEL); + if (*buf == NULL) + return -ENOMEM; + memcpy(*buf, buff + hdr_size, length); + kfree(buff); + + if (err_type == ERR_TYPE_KERNEL_PANIC_GZ) + *compressed = true; + else + *compressed = false; + return length; + } + + *buf = buff; + return part->size; +} + +static struct pstore_info nvram_pstore_info = { + .owner = THIS_MODULE, + .name = "nvram", + .open = nvram_pstore_open, + .read = nvram_pstore_read, + .write = nvram_pstore_write, +}; + +static int nvram_pstore_init(void) +{ + int rc = 0; + + if (machine_is(pseries)) { + nvram_type_ids[2] = PSTORE_TYPE_PPC_RTAS; + nvram_type_ids[3] = PSTORE_TYPE_PPC_OF; + } else + nvram_type_ids[2] = PSTORE_TYPE_PPC_OPAL; + + nvram_pstore_info.buf = oops_data; + nvram_pstore_info.bufsize = oops_data_sz; + + spin_lock_init(&nvram_pstore_info.buf_lock); + + rc = pstore_register(&nvram_pstore_info); + if (rc != 0) + pr_err("nvram: pstore_register() failed, defaults to " + "kmsg_dump; returned %d\n", rc); + + return rc; +} +#else +static int nvram_pstore_init(void) +{ + return -1; +} +#endif + +void __init nvram_init_oops_partition(int rtas_partition_exists) +{ + int rc; + + rc = nvram_init_os_partition(&oops_log_partition); + if (rc != 0) { +#ifdef CONFIG_PPC_PSERIES + if (!rtas_partition_exists) { + pr_err("nvram: Failed to initialize oops partition!"); + return; + } + pr_notice("nvram: Using %s partition to log both" + " RTAS errors and oops/panic reports\n", + rtas_log_partition.name); + memcpy(&oops_log_partition, &rtas_log_partition, + sizeof(rtas_log_partition)); +#else + pr_err("nvram: Failed to initialize oops partition!"); + return; +#endif + } + oops_buf = kmalloc(oops_log_partition.size, GFP_KERNEL); + if (!oops_buf) { + pr_err("nvram: No memory for %s partition\n", + oops_log_partition.name); + return; + } + oops_data = oops_buf + sizeof(struct oops_log_info); + oops_data_sz = oops_log_partition.size - sizeof(struct oops_log_info); + + rc = nvram_pstore_init(); + + if (!rc) + return; + + /* + * Figure compression (preceded by elimination of each line's <n> + * severity prefix) will reduce the oops/panic report to at most + * 45% of its original size. + */ + big_oops_buf_sz = (oops_data_sz * 100) / 45; + big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL); + if (big_oops_buf) { + stream.workspace = kmalloc(zlib_deflate_workspacesize( + WINDOW_BITS, MEM_LEVEL), GFP_KERNEL); + if (!stream.workspace) { + pr_err("nvram: No memory for compression workspace; " + "skipping compression of %s partition data\n", + oops_log_partition.name); + kfree(big_oops_buf); + big_oops_buf = NULL; + } + } else { + pr_err("No memory for uncompressed %s data; " + "skipping compression\n", oops_log_partition.name); + stream.workspace = NULL; + } + + rc = kmsg_dump_register(&nvram_kmsg_dumper); + if (rc != 0) { + pr_err("nvram: kmsg_dump_register() failed; returned %d\n", rc); + kfree(oops_buf); + kfree(big_oops_buf); + kfree(stream.workspace); + } +} + +/* + * This is our kmsg_dump callback, called after an oops or panic report + * has been written to the printk buffer. We want to capture as much + * of the printk buffer as possible. First, capture as much as we can + * that we think will compress sufficiently to fit in the lnx,oops-log + * partition. If that's too much, go back and capture uncompressed text. + */ +static void oops_to_nvram(struct kmsg_dumper *dumper, + enum kmsg_dump_reason reason) +{ + struct oops_log_info *oops_hdr = (struct oops_log_info *)oops_buf; + static unsigned int oops_count = 0; + static bool panicking = false; + static DEFINE_SPINLOCK(lock); + unsigned long flags; + size_t text_len; + unsigned int err_type = ERR_TYPE_KERNEL_PANIC_GZ; + int rc = -1; + + switch (reason) { + case KMSG_DUMP_RESTART: + case KMSG_DUMP_HALT: + case KMSG_DUMP_POWEROFF: + /* These are almost always orderly shutdowns. */ + return; + case KMSG_DUMP_OOPS: + break; + case KMSG_DUMP_PANIC: + panicking = true; + break; + case KMSG_DUMP_EMERG: + if (panicking) + /* Panic report already captured. */ + return; + break; + default: + pr_err("%s: ignoring unrecognized KMSG_DUMP_* reason %d\n", + __func__, (int) reason); + return; + } + + if (clobbering_unread_rtas_event()) + return; + + if (!spin_trylock_irqsave(&lock, flags)) + return; + + if (big_oops_buf) { + kmsg_dump_get_buffer(dumper, false, + big_oops_buf, big_oops_buf_sz, &text_len); + rc = zip_oops(text_len); + } + if (rc != 0) { + kmsg_dump_rewind(dumper); + kmsg_dump_get_buffer(dumper, false, + oops_data, oops_data_sz, &text_len); + err_type = ERR_TYPE_KERNEL_PANIC; + oops_hdr->version = cpu_to_be16(OOPS_HDR_VERSION); + oops_hdr->report_length = cpu_to_be16(text_len); + oops_hdr->timestamp = cpu_to_be64(ktime_get_real_seconds()); + } + + (void) nvram_write_os_partition(&oops_log_partition, oops_buf, + (int) (sizeof(*oops_hdr) + text_len), err_type, + ++oops_count); + + spin_unlock_irqrestore(&lock, flags); +} + static loff_t dev_nvram_llseek(struct file *file, loff_t offset, int origin) { int size; diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c index 2f35a72642c6..b60a67d92ebd 100644 --- a/arch/powerpc/kernel/of_platform.c +++ b/arch/powerpc/kernel/of_platform.c @@ -72,7 +72,7 @@ static int of_pci_phb_probe(struct platform_device *dev) /* Register devices with EEH */ if (dev->dev.of_node->child) - eeh_add_device_tree_early(dev->dev.of_node); + eeh_add_device_tree_early(PCI_DN(dev->dev.of_node)); /* Scan the bus */ pcibios_scan_phb(phb); diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 2a525c938158..0d054068a21d 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -76,7 +76,7 @@ struct pci_controller *pcibios_alloc_controller(struct device_node *dev) list_add_tail(&phb->list_node, &hose_list); spin_unlock(&hose_spinlock); phb->dn = dev; - phb->is_dynamic = mem_init_done; + phb->is_dynamic = slab_is_available(); #ifdef CONFIG_PPC64 if (dev) { int nid = of_node_to_nid(dev); @@ -109,8 +109,10 @@ void pcibios_free_controller(struct pci_controller *phb) resource_size_t pcibios_window_alignment(struct pci_bus *bus, unsigned long type) { - if (ppc_md.pcibios_window_alignment) - return ppc_md.pcibios_window_alignment(bus, type); + struct pci_controller *phb = pci_bus_to_host(bus); + + if (phb->controller_ops.window_alignment) + return phb->controller_ops.window_alignment(bus, type); /* * PCI core will figure out the default @@ -122,14 +124,26 @@ resource_size_t pcibios_window_alignment(struct pci_bus *bus, void pcibios_reset_secondary_bus(struct pci_dev *dev) { - if (ppc_md.pcibios_reset_secondary_bus) { - ppc_md.pcibios_reset_secondary_bus(dev); + struct pci_controller *phb = pci_bus_to_host(dev->bus); + + if (phb->controller_ops.reset_secondary_bus) { + phb->controller_ops.reset_secondary_bus(dev); return; } pci_reset_secondary_bus(dev); } +#ifdef CONFIG_PCI_IOV +resource_size_t pcibios_iov_resource_alignment(struct pci_dev *pdev, int resno) +{ + if (ppc_md.pcibios_iov_resource_alignment) + return ppc_md.pcibios_iov_resource_alignment(pdev, resno); + + return pci_iov_resource_size(pdev, resno); +} +#endif /* CONFIG_PCI_IOV */ + static resource_size_t pcibios_io_size(const struct pci_controller *hose) { #ifdef CONFIG_PPC64 @@ -788,6 +802,10 @@ static void pcibios_fixup_resources(struct pci_dev *dev) pci_name(dev)); return; } + + if (dev->is_virtfn) + return; + for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { struct resource *res = dev->resource + i; struct pci_bus_region reg; @@ -942,6 +960,8 @@ static void pcibios_fixup_bridge(struct pci_bus *bus) void pcibios_setup_bus_self(struct pci_bus *bus) { + struct pci_controller *phb; + /* Fix up the bus resources for P2P bridges */ if (bus->self != NULL) pcibios_fixup_bridge(bus); @@ -953,12 +973,14 @@ void pcibios_setup_bus_self(struct pci_bus *bus) ppc_md.pcibios_fixup_bus(bus); /* Setup bus DMA mappings */ - if (ppc_md.pci_dma_bus_setup) - ppc_md.pci_dma_bus_setup(bus); + phb = pci_bus_to_host(bus); + if (phb->controller_ops.dma_bus_setup) + phb->controller_ops.dma_bus_setup(bus); } static void pcibios_setup_device(struct pci_dev *dev) { + struct pci_controller *phb; /* Fixup NUMA node as it may not be setup yet by the generic * code and is needed by the DMA init */ @@ -969,8 +991,9 @@ static void pcibios_setup_device(struct pci_dev *dev) set_dma_offset(&dev->dev, PCI_DRAM_OFFSET); /* Additional platform DMA/iommu setup */ - if (ppc_md.pci_dma_dev_setup) - ppc_md.pci_dma_dev_setup(dev); + phb = pci_bus_to_host(dev->bus); + if (phb->controller_ops.dma_dev_setup) + phb->controller_ops.dma_dev_setup(dev); /* Read default IRQs and fixup if necessary */ pci_read_irq_line(dev); @@ -986,6 +1009,12 @@ int pcibios_add_device(struct pci_dev *dev) */ if (dev->bus->is_added) pcibios_setup_device(dev); + +#ifdef CONFIG_PCI_IOV + if (ppc_md.pcibios_fixup_sriov) + ppc_md.pcibios_fixup_sriov(dev); +#endif /* CONFIG_PCI_IOV */ + return 0; } @@ -1450,8 +1479,10 @@ EXPORT_SYMBOL_GPL(pcibios_finish_adding_to_bus); int pcibios_enable_device(struct pci_dev *dev, int mask) { - if (ppc_md.pcibios_enable_device_hook) - if (ppc_md.pcibios_enable_device_hook(dev)) + struct pci_controller *phb = pci_bus_to_host(dev->bus); + + if (phb->controller_ops.enable_device_hook) + if (!phb->controller_ops.enable_device_hook(dev)) return -EINVAL; return pci_enable_resources(dev, mask); @@ -1624,8 +1655,8 @@ void pcibios_scan_phb(struct pci_controller *hose) /* Get probe mode and perform scan */ mode = PCI_PROBE_NORMAL; - if (node && ppc_md.pci_probe_mode) - mode = ppc_md.pci_probe_mode(bus); + if (node && hose->controller_ops.probe_mode) + mode = hose->controller_ops.probe_mode(bus); pr_debug(" probe mode: %d\n", mode); if (mode == PCI_PROBE_DEVTREE) of_scan_bus(node, bus); diff --git a/arch/powerpc/kernel/pci-hotplug.c b/arch/powerpc/kernel/pci-hotplug.c index 5b789177aa29..7ed85a69a9c2 100644 --- a/arch/powerpc/kernel/pci-hotplug.c +++ b/arch/powerpc/kernel/pci-hotplug.c @@ -73,13 +73,16 @@ void pcibios_add_pci_devices(struct pci_bus * bus) { int slotno, mode, pass, max; struct pci_dev *dev; + struct pci_controller *phb; struct device_node *dn = pci_bus_to_OF_node(bus); - eeh_add_device_tree_early(dn); + eeh_add_device_tree_early(PCI_DN(dn)); + + phb = pci_bus_to_host(bus); mode = PCI_PROBE_NORMAL; - if (ppc_md.pci_probe_mode) - mode = ppc_md.pci_probe_mode(bus); + if (phb->controller_ops.probe_mode) + mode = phb->controller_ops.probe_mode(bus); if (mode == PCI_PROBE_DEVTREE) { /* use ofdt-based probe */ diff --git a/arch/powerpc/kernel/pci_dn.c b/arch/powerpc/kernel/pci_dn.c index 83df3075d3df..b3b4df91b792 100644 --- a/arch/powerpc/kernel/pci_dn.c +++ b/arch/powerpc/kernel/pci_dn.c @@ -32,12 +32,237 @@ #include <asm/ppc-pci.h> #include <asm/firmware.h> +/* + * The function is used to find the firmware data of one + * specific PCI device, which is attached to the indicated + * PCI bus. For VFs, their firmware data is linked to that + * one of PF's bridge. For other devices, their firmware + * data is linked to that of their bridge. + */ +static struct pci_dn *pci_bus_to_pdn(struct pci_bus *bus) +{ + struct pci_bus *pbus; + struct device_node *dn; + struct pci_dn *pdn; + + /* + * We probably have virtual bus which doesn't + * have associated bridge. + */ + pbus = bus; + while (pbus) { + if (pci_is_root_bus(pbus) || pbus->self) + break; + + pbus = pbus->parent; + } + + /* + * Except virtual bus, all PCI buses should + * have device nodes. + */ + dn = pci_bus_to_OF_node(pbus); + pdn = dn ? PCI_DN(dn) : NULL; + + return pdn; +} + +struct pci_dn *pci_get_pdn_by_devfn(struct pci_bus *bus, + int devfn) +{ + struct device_node *dn = NULL; + struct pci_dn *parent, *pdn; + struct pci_dev *pdev = NULL; + + /* Fast path: fetch from PCI device */ + list_for_each_entry(pdev, &bus->devices, bus_list) { + if (pdev->devfn == devfn) { + if (pdev->dev.archdata.pci_data) + return pdev->dev.archdata.pci_data; + + dn = pci_device_to_OF_node(pdev); + break; + } + } + + /* Fast path: fetch from device node */ + pdn = dn ? PCI_DN(dn) : NULL; + if (pdn) + return pdn; + + /* Slow path: fetch from firmware data hierarchy */ + parent = pci_bus_to_pdn(bus); + if (!parent) + return NULL; + + list_for_each_entry(pdn, &parent->child_list, list) { + if (pdn->busno == bus->number && + pdn->devfn == devfn) + return pdn; + } + + return NULL; +} + struct pci_dn *pci_get_pdn(struct pci_dev *pdev) { - struct device_node *dn = pci_device_to_OF_node(pdev); - if (!dn) + struct device_node *dn; + struct pci_dn *parent, *pdn; + + /* Search device directly */ + if (pdev->dev.archdata.pci_data) + return pdev->dev.archdata.pci_data; + + /* Check device node */ + dn = pci_device_to_OF_node(pdev); + pdn = dn ? PCI_DN(dn) : NULL; + if (pdn) + return pdn; + + /* + * VFs don't have device nodes. We hook their + * firmware data to PF's bridge. + */ + parent = pci_bus_to_pdn(pdev->bus); + if (!parent) + return NULL; + + list_for_each_entry(pdn, &parent->child_list, list) { + if (pdn->busno == pdev->bus->number && + pdn->devfn == pdev->devfn) + return pdn; + } + + return NULL; +} + +#ifdef CONFIG_PCI_IOV +static struct pci_dn *add_one_dev_pci_data(struct pci_dn *parent, + struct pci_dev *pdev, + int busno, int devfn) +{ + struct pci_dn *pdn; + + /* Except PHB, we always have the parent */ + if (!parent) + return NULL; + + pdn = kzalloc(sizeof(*pdn), GFP_KERNEL); + if (!pdn) { + dev_warn(&pdev->dev, "%s: Out of memory!\n", __func__); return NULL; - return PCI_DN(dn); + } + + pdn->phb = parent->phb; + pdn->parent = parent; + pdn->busno = busno; + pdn->devfn = devfn; +#ifdef CONFIG_PPC_POWERNV + pdn->pe_number = IODA_INVALID_PE; +#endif + INIT_LIST_HEAD(&pdn->child_list); + INIT_LIST_HEAD(&pdn->list); + list_add_tail(&pdn->list, &parent->child_list); + + /* + * If we already have PCI device instance, lets + * bind them. + */ + if (pdev) + pdev->dev.archdata.pci_data = pdn; + + return pdn; +} +#endif + +struct pci_dn *add_dev_pci_data(struct pci_dev *pdev) +{ +#ifdef CONFIG_PCI_IOV + struct pci_dn *parent, *pdn; + int i; + + /* Only support IOV for now */ + if (!pdev->is_physfn) + return pci_get_pdn(pdev); + + /* Check if VFs have been populated */ + pdn = pci_get_pdn(pdev); + if (!pdn || (pdn->flags & PCI_DN_FLAG_IOV_VF)) + return NULL; + + pdn->flags |= PCI_DN_FLAG_IOV_VF; + parent = pci_bus_to_pdn(pdev->bus); + if (!parent) + return NULL; + + for (i = 0; i < pci_sriov_get_totalvfs(pdev); i++) { + pdn = add_one_dev_pci_data(parent, NULL, + pci_iov_virtfn_bus(pdev, i), + pci_iov_virtfn_devfn(pdev, i)); + if (!pdn) { + dev_warn(&pdev->dev, "%s: Cannot create firmware data for VF#%d\n", + __func__, i); + return NULL; + } + } +#endif /* CONFIG_PCI_IOV */ + + return pci_get_pdn(pdev); +} + +void remove_dev_pci_data(struct pci_dev *pdev) +{ +#ifdef CONFIG_PCI_IOV + struct pci_dn *parent; + struct pci_dn *pdn, *tmp; + int i; + + /* + * VF and VF PE are created/released dynamically, so we need to + * bind/unbind them. Otherwise the VF and VF PE would be mismatched + * when re-enabling SR-IOV. + */ + if (pdev->is_virtfn) { + pdn = pci_get_pdn(pdev); +#ifdef CONFIG_PPC_POWERNV + pdn->pe_number = IODA_INVALID_PE; +#endif + return; + } + + /* Only support IOV PF for now */ + if (!pdev->is_physfn) + return; + + /* Check if VFs have been populated */ + pdn = pci_get_pdn(pdev); + if (!pdn || !(pdn->flags & PCI_DN_FLAG_IOV_VF)) + return; + + pdn->flags &= ~PCI_DN_FLAG_IOV_VF; + parent = pci_bus_to_pdn(pdev->bus); + if (!parent) + return; + + /* + * We might introduce flag to pci_dn in future + * so that we can release VF's firmware data in + * a batch mode. + */ + for (i = 0; i < pci_sriov_get_totalvfs(pdev); i++) { + list_for_each_entry_safe(pdn, tmp, + &parent->child_list, list) { + if (pdn->busno != pci_iov_virtfn_bus(pdev, i) || + pdn->devfn != pci_iov_virtfn_devfn(pdev, i)) + continue; + + if (!list_empty(&pdn->list)) + list_del(&pdn->list); + + kfree(pdn); + } + } +#endif /* CONFIG_PCI_IOV */ } /* @@ -49,6 +274,7 @@ void *update_dn_pci_info(struct device_node *dn, void *data) struct pci_controller *phb = data; const __be32 *type = of_get_property(dn, "ibm,pci-config-space-type", NULL); const __be32 *regs; + struct device_node *parent; struct pci_dn *pdn; pdn = zalloc_maybe_bootmem(sizeof(*pdn), GFP_KERNEL); @@ -69,7 +295,25 @@ void *update_dn_pci_info(struct device_node *dn, void *data) pdn->devfn = (addr >> 8) & 0xff; } + /* vendor/device IDs and class code */ + regs = of_get_property(dn, "vendor-id", NULL); + pdn->vendor_id = regs ? of_read_number(regs, 1) : 0; + regs = of_get_property(dn, "device-id", NULL); + pdn->device_id = regs ? of_read_number(regs, 1) : 0; + regs = of_get_property(dn, "class-code", NULL); + pdn->class_code = regs ? of_read_number(regs, 1) : 0; + + /* Extended config space */ pdn->pci_ext_config_space = (type && of_read_number(type, 1) == 1); + + /* Attach to parent node */ + INIT_LIST_HEAD(&pdn->child_list); + INIT_LIST_HEAD(&pdn->list); + parent = of_get_parent(dn); + pdn->parent = parent ? PCI_DN(parent) : NULL; + if (pdn->parent) + list_add_tail(&pdn->list, &pdn->parent->child_list); + return NULL; } @@ -131,6 +375,46 @@ void *traverse_pci_devices(struct device_node *start, traverse_func pre, return NULL; } +static struct pci_dn *pci_dn_next_one(struct pci_dn *root, + struct pci_dn *pdn) +{ + struct list_head *next = pdn->child_list.next; + + if (next != &pdn->child_list) + return list_entry(next, struct pci_dn, list); + + while (1) { + if (pdn == root) + return NULL; + + next = pdn->list.next; + if (next != &pdn->parent->child_list) + break; + + pdn = pdn->parent; + } + + return list_entry(next, struct pci_dn, list); +} + +void *traverse_pci_dn(struct pci_dn *root, + void *(*fn)(struct pci_dn *, void *), + void *data) +{ + struct pci_dn *pdn = root; + void *ret; + + /* Only scan the child nodes */ + for (pdn = pci_dn_next_one(root, pdn); pdn; + pdn = pci_dn_next_one(root, pdn)) { + ret = fn(pdn, data); + if (ret) + return ret; + } + + return NULL; +} + /** * pci_devs_phb_init_dynamic - setup pci devices under this PHB * phb: pci-to-host bridge (top-level bridge connecting to cpu) @@ -147,8 +431,12 @@ void pci_devs_phb_init_dynamic(struct pci_controller *phb) /* PHB nodes themselves must not match */ update_dn_pci_info(dn, phb); pdn = dn->data; - if (pdn) + if (pdn) { pdn->devfn = pdn->busno = -1; + pdn->vendor_id = pdn->device_id = pdn->class_code = 0; + pdn->phb = phb; + phb->pci_data = pdn; + } /* Update dn->phb ptrs for new phb and children devices */ traverse_pci_devices(dn, update_dn_pci_info, phb); @@ -171,3 +459,16 @@ void __init pci_devs_phb_init(void) list_for_each_entry_safe(phb, tmp, &hose_list, list_node) pci_devs_phb_init_dynamic(phb); } + +static void pci_dev_pdn_setup(struct pci_dev *pdev) +{ + struct pci_dn *pdn; + + if (pdev->dev.archdata.pci_data) + return; + + /* Setup the fast path */ + pdn = pci_get_pdn(pdev); + pdev->dev.archdata.pci_data = pdn; +} +DECLARE_PCI_FIXUP_EARLY(PCI_ANY_ID, PCI_ANY_ID, pci_dev_pdn_setup); diff --git a/arch/powerpc/kernel/pci_of_scan.c b/arch/powerpc/kernel/pci_of_scan.c index e6245e9c7d8d..42e02a2d570b 100644 --- a/arch/powerpc/kernel/pci_of_scan.c +++ b/arch/powerpc/kernel/pci_of_scan.c @@ -207,6 +207,7 @@ void of_scan_pci_bridge(struct pci_dev *dev) { struct device_node *node = dev->dev.of_node; struct pci_bus *bus; + struct pci_controller *phb; const __be32 *busrange, *ranges; int len, i, mode; struct pci_bus_region region; @@ -286,9 +287,11 @@ void of_scan_pci_bridge(struct pci_dev *dev) bus->number); pr_debug(" bus name: %s\n", bus->name); + phb = pci_bus_to_host(bus); + mode = PCI_PROBE_NORMAL; - if (ppc_md.pci_probe_mode) - mode = ppc_md.pci_probe_mode(bus); + if (phb->controller_ops.probe_mode) + mode = phb->controller_ops.probe_mode(bus); pr_debug(" probe mode: %d\n", mode); if (mode == PCI_PROBE_DEVTREE) @@ -305,7 +308,7 @@ static struct pci_dev *of_scan_pci_dev(struct pci_bus *bus, const __be32 *reg; int reglen, devfn; #ifdef CONFIG_EEH - struct eeh_dev *edev = of_node_to_eeh_dev(dn); + struct eeh_dev *edev = pdn_to_eeh_dev(PCI_DN(dn)); #endif pr_debug(" * %s\n", dn->full_name); diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index b4cc7bef6b16..febb50dd5328 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1114,8 +1114,11 @@ static void setup_ksp_vsid(struct task_struct *p, unsigned long sp) */ extern unsigned long dscr_default; /* defined in arch/powerpc/kernel/sysfs.c */ +/* + * Copy architecture-specific thread state + */ int copy_thread(unsigned long clone_flags, unsigned long usp, - unsigned long arg, struct task_struct *p) + unsigned long kthread_arg, struct task_struct *p) { struct pt_regs *childregs, *kregs; extern void ret_from_fork(void); @@ -1127,6 +1130,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, sp -= sizeof(struct pt_regs); childregs = (struct pt_regs *) sp; if (unlikely(p->flags & PF_KTHREAD)) { + /* kernel thread */ struct thread_info *ti = (void *)task_stack_page(p); memset(childregs, 0, sizeof(struct pt_regs)); childregs->gpr[1] = sp + sizeof(struct pt_regs); @@ -1137,11 +1141,12 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, clear_tsk_thread_flag(p, TIF_32BIT); childregs->softe = 1; #endif - childregs->gpr[15] = arg; + childregs->gpr[15] = kthread_arg; p->thread.regs = NULL; /* no user register state */ ti->flags |= _TIF_RESTOREALL; f = ret_from_kernel_thread; } else { + /* user thread */ struct pt_regs *regs = current_pt_regs(); CHECK_FULL_REGS(regs); *childregs = *regs; diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 1a85d8f96739..fd1fe4c37599 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -2898,7 +2898,7 @@ unsigned long __init prom_init(unsigned long r3, unsigned long r4, * Call OF "quiesce" method to shut down pending DMA's from * devices etc... */ - prom_printf("Calling quiesce...\n"); + prom_printf("Quiescing Open Firmware ...\n"); call_prom("quiesce", 0, 0); /* @@ -2910,7 +2910,7 @@ unsigned long __init prom_init(unsigned long r3, unsigned long r4, /* Don't print anything after quiesce under OPAL, it crashes OFW */ if (of_platform != PLATFORM_OPAL) { - prom_printf("returning from prom_init\n"); + prom_printf("Booting Linux via __start() ...\n"); prom_debug("->dt_header_start=0x%x\n", hdr); } diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index 21c45a2d0706..7a488c108410 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -401,7 +401,7 @@ static char *__fetch_rtas_last_error(char *altbuf) buf = altbuf; } else { buf = rtas_err_buf; - if (mem_init_done) + if (slab_is_available()) buf = kmalloc(RTAS_ERROR_LOG_MAX, GFP_ATOMIC); } if (buf) @@ -461,7 +461,7 @@ int rtas_call(int token, int nargs, int nret, int *outputs, ...) if (buff_copy) { log_error(buff_copy, ERR_TYPE_RTAS_LOG, 0); - if (mem_init_done) + if (slab_is_available()) kfree(buff_copy); } return ret; @@ -897,7 +897,7 @@ int rtas_offline_cpus_mask(cpumask_var_t cpus) } EXPORT_SYMBOL(rtas_offline_cpus_mask); -int rtas_ibm_suspend_me(u64 handle, int *vasi_return) +int rtas_ibm_suspend_me(u64 handle) { long state; long rc; @@ -919,13 +919,11 @@ int rtas_ibm_suspend_me(u64 handle, int *vasi_return) printk(KERN_ERR "rtas_ibm_suspend_me: vasi_state returned %ld\n",rc); return rc; } else if (state == H_VASI_ENABLED) { - *vasi_return = RTAS_NOT_SUSPENDABLE; - return 0; + return -EAGAIN; } else if (state != H_VASI_SUSPENDING) { printk(KERN_ERR "rtas_ibm_suspend_me: vasi_state returned state %ld\n", state); - *vasi_return = -1; - return 0; + return -EIO; } if (!alloc_cpumask_var(&offline_mask, GFP_TEMPORARY)) @@ -972,7 +970,7 @@ out: return atomic_read(&data.error); } #else /* CONFIG_PPC_PSERIES */ -int rtas_ibm_suspend_me(u64 handle, int *vasi_return) +int rtas_ibm_suspend_me(u64 handle) { return -ENOSYS; } @@ -1022,7 +1020,6 @@ asmlinkage int ppc_rtas(struct rtas_args __user *uargs) unsigned long flags; char *buff_copy, *errbuf = NULL; int nargs, nret, token; - int rc; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -1054,15 +1051,18 @@ asmlinkage int ppc_rtas(struct rtas_args __user *uargs) if (token == ibm_suspend_me_token) { /* - * rtas_ibm_suspend_me assumes args are in cpu endian, or at least the - * hcall within it requires it. + * rtas_ibm_suspend_me assumes the streamid handle is in cpu + * endian, or at least the hcall within it requires it. */ - int vasi_rc = 0; + int rc = 0; u64 handle = ((u64)be32_to_cpu(args.args[0]) << 32) | be32_to_cpu(args.args[1]); - rc = rtas_ibm_suspend_me(handle, &vasi_rc); - args.rets[0] = cpu_to_be32(vasi_rc); - if (rc) + rc = rtas_ibm_suspend_me(handle); + if (rc == -EAGAIN) + args.rets[0] = cpu_to_be32(RTAS_NOT_SUSPENDABLE); + else if (rc == -EIO) + args.rets[0] = cpu_to_be32(-1); + else if (rc) return rc; goto copy_return; } diff --git a/arch/powerpc/kernel/rtas_pci.c b/arch/powerpc/kernel/rtas_pci.c index ce230da2c015..73f1934582c2 100644 --- a/arch/powerpc/kernel/rtas_pci.c +++ b/arch/powerpc/kernel/rtas_pci.c @@ -113,7 +113,7 @@ static int rtas_pci_read_config(struct pci_bus *bus, ret = rtas_read_config(pdn, where, size, val); if (*val == EEH_IO_ERROR_VALUE(size) && - eeh_dev_check_failure(of_node_to_eeh_dev(dn))) + eeh_dev_check_failure(pdn_to_eeh_dev(pdn))) return PCIBIOS_DEVICE_NOT_FOUND; return ret; @@ -277,50 +277,3 @@ int rtas_setup_phb(struct pci_controller *phb) return 0; } - -void __init find_and_init_phbs(void) -{ - struct device_node *node; - struct pci_controller *phb; - struct device_node *root = of_find_node_by_path("/"); - - for_each_child_of_node(root, node) { - if (node->type == NULL || (strcmp(node->type, "pci") != 0 && - strcmp(node->type, "pciex") != 0)) - continue; - - phb = pcibios_alloc_controller(node); - if (!phb) - continue; - rtas_setup_phb(phb); - pci_process_bridge_OF_ranges(phb, node, 0); - isa_bridge_find_early(phb); - } - - of_node_put(root); - pci_devs_phb_init(); - - /* - * PCI_PROBE_ONLY and PCI_REASSIGN_ALL_BUS can be set via properties - * in chosen. - */ - if (of_chosen) { - const int *prop; - - prop = of_get_property(of_chosen, - "linux,pci-probe-only", NULL); - if (prop) { - if (*prop) - pci_add_flags(PCI_PROBE_ONLY); - else - pci_clear_flags(PCI_PROBE_ONLY); - } - -#ifdef CONFIG_PPC32 /* Will be made generic soon */ - prop = of_get_property(of_chosen, - "linux,pci-assign-all-buses", NULL); - if (prop && *prop) - pci_add_flags(PCI_REASSIGN_ALL_BUS); -#endif /* CONFIG_PPC32 */ - } -} diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 49f553bbb360..c69671c03c3b 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -37,6 +37,7 @@ #include <linux/memblock.h> #include <linux/hugetlb.h> #include <linux/memory.h> +#include <linux/nmi.h> #include <asm/io.h> #include <asm/kdump.h> @@ -779,3 +780,22 @@ unsigned long memory_block_size_bytes(void) struct ppc_pci_io ppc_pci_io; EXPORT_SYMBOL(ppc_pci_io); #endif + +#ifdef CONFIG_HARDLOCKUP_DETECTOR +u64 hw_nmi_get_sample_period(int watchdog_thresh) +{ + return ppc_proc_freq * watchdog_thresh; +} + +/* + * The hardlockup detector breaks PMU event based branches and is likely + * to get false positives in KVM guests, so disable it by default. + */ +static int __init disable_hardlockup_detector(void) +{ + hardlockup_detector_disable(); + + return 0; +} +early_initcall(disable_hardlockup_detector); +#endif diff --git a/arch/powerpc/kernel/syscalls.c b/arch/powerpc/kernel/syscalls.c index b2702e87db0d..5fa92706444b 100644 --- a/arch/powerpc/kernel/syscalls.c +++ b/arch/powerpc/kernel/syscalls.c @@ -121,3 +121,20 @@ long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low, return sys_fadvise64(fd, (u64)offset_high << 32 | offset_low, (u64)len_high << 32 | len_low, advice); } + +long sys_switch_endian(void) +{ + struct thread_info *ti; + + current->thread.regs->msr ^= MSR_LE; + + /* + * Set TIF_RESTOREALL so that r3 isn't clobbered on return to + * userspace. That also has the effect of restoring the non-volatile + * GPRs, so we saved them on the way in here. + */ + ti = current_thread_info(); + ti->flags |= _TIF_RESTOREALL; + + return 0; +} diff --git a/arch/powerpc/kernel/systbl.S b/arch/powerpc/kernel/systbl.S index 7ab5d434e2ee..4d6b1d3a747f 100644 --- a/arch/powerpc/kernel/systbl.S +++ b/arch/powerpc/kernel/systbl.S @@ -22,6 +22,7 @@ #define PPC_SYS(func) .llong DOTSYM(ppc_##func),DOTSYM(ppc_##func) #define OLDSYS(func) .llong DOTSYM(sys_ni_syscall),DOTSYM(sys_ni_syscall) #define SYS32ONLY(func) .llong DOTSYM(sys_ni_syscall),DOTSYM(compat_sys_##func) +#define PPC64ONLY(func) .llong DOTSYM(ppc_##func),DOTSYM(sys_ni_syscall) #define SYSX(f, f3264, f32) .llong DOTSYM(f),DOTSYM(f3264) #else #define SYSCALL(func) .long sys_##func @@ -29,6 +30,7 @@ #define PPC_SYS(func) .long ppc_##func #define OLDSYS(func) .long sys_##func #define SYS32ONLY(func) .long sys_##func +#define PPC64ONLY(func) .long sys_ni_syscall #define SYSX(f, f3264, f32) .long f32 #endif #define SYSCALL_SPU(func) SYSCALL(func) diff --git a/arch/powerpc/kernel/systbl_chk.c b/arch/powerpc/kernel/systbl_chk.c index 238aa63ced8f..2384129f5893 100644 --- a/arch/powerpc/kernel/systbl_chk.c +++ b/arch/powerpc/kernel/systbl_chk.c @@ -21,9 +21,11 @@ #ifdef CONFIG_PPC64 #define OLDSYS(func) -1 #define SYS32ONLY(func) -1 +#define PPC64ONLY(func) __NR_##func #else #define OLDSYS(func) __NR_old##func #define SYS32ONLY(func) __NR_##func +#define PPC64ONLY(func) -1 #endif #define SYSX(f, f3264, f32) -1 diff --git a/arch/powerpc/kernel/tm.S b/arch/powerpc/kernel/tm.S index 2a324f4cb1b9..5754b226da7e 100644 --- a/arch/powerpc/kernel/tm.S +++ b/arch/powerpc/kernel/tm.S @@ -152,9 +152,9 @@ _GLOBAL(tm_reclaim) addi r7, r3, THREAD_TRANSACT_VRSTATE SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */ - mfvscr vr0 + mfvscr v0 li r6, VRSTATE_VSCR - stvx vr0, r7, r6 + stvx v0, r7, r6 dont_backup_vec: mfspr r0, SPRN_VRSAVE std r0, THREAD_TRANSACT_VRSAVE(r3) @@ -359,8 +359,8 @@ _GLOBAL(__tm_recheckpoint) addi r8, r3, THREAD_VRSTATE li r5, VRSTATE_VSCR - lvx vr0, r8, r5 - mtvscr vr0 + lvx v0, r8, r5 + mtvscr v0 REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */ dont_restore_vec: ld r5, THREAD_VRSAVE(r3) diff --git a/arch/powerpc/kernel/udbg.c b/arch/powerpc/kernel/udbg.c index b7aa07279a63..7cc38b5b58bc 100644 --- a/arch/powerpc/kernel/udbg.c +++ b/arch/powerpc/kernel/udbg.c @@ -46,8 +46,6 @@ void __init udbg_early_init(void) #elif defined(CONFIG_PPC_EARLY_DEBUG_MAPLE) /* Maple real mode debug */ udbg_init_maple_realmode(); -#elif defined(CONFIG_PPC_EARLY_DEBUG_BEAT) - udbg_init_debug_beat(); #elif defined(CONFIG_PPC_EARLY_DEBUG_PAS_REALMODE) udbg_init_pas_realmode(); #elif defined(CONFIG_PPC_EARLY_DEBUG_BOOTX) diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S index 74f8050518d6..f5c80d567d8d 100644 --- a/arch/powerpc/kernel/vector.S +++ b/arch/powerpc/kernel/vector.S @@ -24,8 +24,8 @@ _GLOBAL(do_load_up_transact_altivec) stw r4,THREAD_USED_VR(r3) li r10,THREAD_TRANSACT_VRSTATE+VRSTATE_VSCR - lvx vr0,r10,r3 - mtvscr vr0 + lvx v0,r10,r3 + mtvscr v0 addi r10,r3,THREAD_TRANSACT_VRSTATE REST_32VRS(0,r4,r10) @@ -52,8 +52,8 @@ _GLOBAL(vec_enable) */ _GLOBAL(load_vr_state) li r4,VRSTATE_VSCR - lvx vr0,r4,r3 - mtvscr vr0 + lvx v0,r4,r3 + mtvscr v0 REST_32VRS(0,r4,r3) blr @@ -63,9 +63,9 @@ _GLOBAL(load_vr_state) */ _GLOBAL(store_vr_state) SAVE_32VRS(0, r4, r3) - mfvscr vr0 + mfvscr v0 li r4, VRSTATE_VSCR - stvx vr0, r4, r3 + stvx v0, r4, r3 blr /* @@ -104,9 +104,9 @@ _GLOBAL(load_up_altivec) addi r4,r4,THREAD addi r6,r4,THREAD_VRSTATE SAVE_32VRS(0,r5,r6) - mfvscr vr0 + mfvscr v0 li r10,VRSTATE_VSCR - stvx vr0,r10,r6 + stvx v0,r10,r6 /* Disable VMX for last_task_used_altivec */ PPC_LL r5,PT_REGS(r4) toreal(r5) @@ -142,8 +142,8 @@ _GLOBAL(load_up_altivec) li r4,1 li r10,VRSTATE_VSCR stw r4,THREAD_USED_VR(r5) - lvx vr0,r10,r6 - mtvscr vr0 + lvx v0,r10,r6 + mtvscr v0 REST_32VRS(0,r4,r6) #ifndef CONFIG_SMP /* Update last_task_used_altivec to 'current' */ @@ -186,9 +186,9 @@ _GLOBAL(giveup_altivec) addi r7,r3,THREAD_VRSTATE 2: PPC_LCMPI 0,r5,0 SAVE_32VRS(0,r4,r7) - mfvscr vr0 + mfvscr v0 li r4,VRSTATE_VSCR - stvx vr0,r4,r7 + stvx v0,r4,r7 beq 1f PPC_LL r4,_MSR-STACK_FRAME_OVERHEAD(r5) #ifdef CONFIG_VSX diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c index 60081bd75847..93b5f5c9b445 100644 --- a/arch/powerpc/kvm/book3s_hv_ras.c +++ b/arch/powerpc/kvm/book3s_hv_ras.c @@ -84,7 +84,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) } if (dsisr & DSISR_MC_TLB_MULTI) { if (cur_cpu_spec && cur_cpu_spec->flush_tlb) - cur_cpu_spec->flush_tlb(TLBIEL_INVAL_SET_LPID); + cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); dsisr &= ~DSISR_MC_TLB_MULTI; } /* Any other errors we don't understand? */ @@ -102,7 +102,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) break; case SRR1_MC_IFETCH_TLBMULTI: if (cur_cpu_spec && cur_cpu_spec->flush_tlb) - cur_cpu_spec->flush_tlb(TLBIEL_INVAL_SET_LPID); + cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); break; default: handled = 0; diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 24bfe401373e..91bbc845ac66 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -720,7 +720,7 @@ static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu, return; } - if (vcpu->arch.mmio_is_bigendian) { + if (!vcpu->arch.mmio_host_swabbed) { switch (run->mmio.len) { case 8: gpr = *(u64 *)run->mmio.data; break; case 4: gpr = *(u32 *)run->mmio.data; break; @@ -728,10 +728,10 @@ static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu, case 1: gpr = *(u8 *)run->mmio.data; break; } } else { - /* Convert BE data from userland back to LE. */ switch (run->mmio.len) { - case 4: gpr = ld_le32((u32 *)run->mmio.data); break; - case 2: gpr = ld_le16((u16 *)run->mmio.data); break; + case 8: gpr = swab64(*(u64 *)run->mmio.data); break; + case 4: gpr = swab32(*(u32 *)run->mmio.data); break; + case 2: gpr = swab16(*(u16 *)run->mmio.data); break; case 1: gpr = *(u8 *)run->mmio.data; break; } } @@ -780,14 +780,13 @@ int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, int is_default_endian) { int idx, ret; - int is_bigendian; + bool host_swabbed; + /* Pity C doesn't have a logical XOR operator */ if (kvmppc_need_byteswap(vcpu)) { - /* Default endianness is "little endian". */ - is_bigendian = !is_default_endian; + host_swabbed = is_default_endian; } else { - /* Default endianness is "big endian". */ - is_bigendian = is_default_endian; + host_swabbed = !is_default_endian; } if (bytes > sizeof(run->mmio.data)) { @@ -800,7 +799,7 @@ int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, run->mmio.is_write = 0; vcpu->arch.io_gpr = rt; - vcpu->arch.mmio_is_bigendian = is_bigendian; + vcpu->arch.mmio_host_swabbed = host_swabbed; vcpu->mmio_needed = 1; vcpu->mmio_is_write = 0; vcpu->arch.mmio_sign_extend = 0; @@ -840,14 +839,13 @@ int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu, { void *data = run->mmio.data; int idx, ret; - int is_bigendian; + bool host_swabbed; + /* Pity C doesn't have a logical XOR operator */ if (kvmppc_need_byteswap(vcpu)) { - /* Default endianness is "little endian". */ - is_bigendian = !is_default_endian; + host_swabbed = is_default_endian; } else { - /* Default endianness is "big endian". */ - is_bigendian = is_default_endian; + host_swabbed = !is_default_endian; } if (bytes > sizeof(run->mmio.data)) { @@ -862,7 +860,7 @@ int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu, vcpu->mmio_is_write = 1; /* Store the value at the lowest bytes in 'data'. */ - if (is_bigendian) { + if (!host_swabbed) { switch (bytes) { case 8: *(u64 *)data = val; break; case 4: *(u32 *)data = val; break; @@ -870,11 +868,11 @@ int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu, case 1: *(u8 *)data = val; break; } } else { - /* Store LE value into 'data'. */ switch (bytes) { - case 4: st_le32(data, val); break; - case 2: st_le16(data, val); break; - case 1: *(u8 *)data = val; break; + case 8: *(u64 *)data = swab64(val); break; + case 4: *(u32 *)data = swab32(val); break; + case 2: *(u16 *)data = swab16(val); break; + case 1: *(u8 *)data = val; break; } } diff --git a/arch/powerpc/lib/alloc.c b/arch/powerpc/lib/alloc.c index 4a6c2cf890d9..60b0b3fc8fc1 100644 --- a/arch/powerpc/lib/alloc.c +++ b/arch/powerpc/lib/alloc.c @@ -10,7 +10,7 @@ void * __init_refok zalloc_maybe_bootmem(size_t size, gfp_t mask) { void *p; - if (mem_init_done) + if (slab_is_available()) p = kzalloc(size, mask); else { p = memblock_virt_alloc(size, 0); diff --git a/arch/powerpc/lib/copy_32.S b/arch/powerpc/lib/copy_32.S index 55f19f9fd708..6813f80d1eec 100644 --- a/arch/powerpc/lib/copy_32.S +++ b/arch/powerpc/lib/copy_32.S @@ -69,54 +69,6 @@ CACHELINE_BYTES = L1_CACHE_BYTES LG_CACHELINE_BYTES = L1_CACHE_SHIFT CACHELINE_MASK = (L1_CACHE_BYTES-1) -/* - * Use dcbz on the complete cache lines in the destination - * to set them to zero. This requires that the destination - * area is cacheable. -- paulus - */ -_GLOBAL(cacheable_memzero) - mr r5,r4 - li r4,0 - addi r6,r3,-4 - cmplwi 0,r5,4 - blt 7f - stwu r4,4(r6) - beqlr - andi. r0,r6,3 - add r5,r0,r5 - subf r6,r0,r6 - clrlwi r7,r6,32-LG_CACHELINE_BYTES - add r8,r7,r5 - srwi r9,r8,LG_CACHELINE_BYTES - addic. r9,r9,-1 /* total number of complete cachelines */ - ble 2f - xori r0,r7,CACHELINE_MASK & ~3 - srwi. r0,r0,2 - beq 3f - mtctr r0 -4: stwu r4,4(r6) - bdnz 4b -3: mtctr r9 - li r7,4 -10: dcbz r7,r6 - addi r6,r6,CACHELINE_BYTES - bdnz 10b - clrlwi r5,r8,32-LG_CACHELINE_BYTES - addi r5,r5,4 -2: srwi r0,r5,2 - mtctr r0 - bdz 6f -1: stwu r4,4(r6) - bdnz 1b -6: andi. r5,r5,3 -7: cmpwi 0,r5,0 - beqlr - mtctr r5 - addi r6,r6,3 -8: stbu r4,1(r6) - bdnz 8b - blr - _GLOBAL(memset) rlwimi r4,r4,8,16,23 rlwimi r4,r4,16,0,15 @@ -142,85 +94,6 @@ _GLOBAL(memset) bdnz 8b blr -/* - * This version uses dcbz on the complete cache lines in the - * destination area to reduce memory traffic. This requires that - * the destination area is cacheable. - * We only use this version if the source and dest don't overlap. - * -- paulus. - */ -_GLOBAL(cacheable_memcpy) - add r7,r3,r5 /* test if the src & dst overlap */ - add r8,r4,r5 - cmplw 0,r4,r7 - cmplw 1,r3,r8 - crand 0,0,4 /* cr0.lt &= cr1.lt */ - blt memcpy /* if regions overlap */ - - addi r4,r4,-4 - addi r6,r3,-4 - neg r0,r3 - andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */ - beq 58f - - cmplw 0,r5,r0 /* is this more than total to do? */ - blt 63f /* if not much to do */ - andi. r8,r0,3 /* get it word-aligned first */ - subf r5,r0,r5 - mtctr r8 - beq+ 61f -70: lbz r9,4(r4) /* do some bytes */ - stb r9,4(r6) - addi r4,r4,1 - addi r6,r6,1 - bdnz 70b -61: srwi. r0,r0,2 - mtctr r0 - beq 58f -72: lwzu r9,4(r4) /* do some words */ - stwu r9,4(r6) - bdnz 72b - -58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */ - clrlwi r5,r5,32-LG_CACHELINE_BYTES - li r11,4 - mtctr r0 - beq 63f -53: - dcbz r11,r6 - COPY_16_BYTES -#if L1_CACHE_BYTES >= 32 - COPY_16_BYTES -#if L1_CACHE_BYTES >= 64 - COPY_16_BYTES - COPY_16_BYTES -#if L1_CACHE_BYTES >= 128 - COPY_16_BYTES - COPY_16_BYTES - COPY_16_BYTES - COPY_16_BYTES -#endif -#endif -#endif - bdnz 53b - -63: srwi. r0,r5,2 - mtctr r0 - beq 64f -30: lwzu r0,4(r4) - stwu r0,4(r6) - bdnz 30b - -64: andi. r0,r5,3 - mtctr r0 - beq+ 65f -40: lbz r0,4(r4) - stb r0,4(r6) - addi r4,r4,1 - addi r6,r6,1 - bdnz 40b -65: blr - _GLOBAL(memmove) cmplw 0,r3,r4 bgt backwards_memcpy diff --git a/arch/powerpc/lib/copypage_power7.S b/arch/powerpc/lib/copypage_power7.S index d7dafb3777ac..a84d333ecb09 100644 --- a/arch/powerpc/lib/copypage_power7.S +++ b/arch/powerpc/lib/copypage_power7.S @@ -83,23 +83,23 @@ _GLOBAL(copypage_power7) li r12,112 .align 5 -1: lvx vr7,r0,r4 - lvx vr6,r4,r6 - lvx vr5,r4,r7 - lvx vr4,r4,r8 - lvx vr3,r4,r9 - lvx vr2,r4,r10 - lvx vr1,r4,r11 - lvx vr0,r4,r12 +1: lvx v7,r0,r4 + lvx v6,r4,r6 + lvx v5,r4,r7 + lvx v4,r4,r8 + lvx v3,r4,r9 + lvx v2,r4,r10 + lvx v1,r4,r11 + lvx v0,r4,r12 addi r4,r4,128 - stvx vr7,r0,r3 - stvx vr6,r3,r6 - stvx vr5,r3,r7 - stvx vr4,r3,r8 - stvx vr3,r3,r9 - stvx vr2,r3,r10 - stvx vr1,r3,r11 - stvx vr0,r3,r12 + stvx v7,r0,r3 + stvx v6,r3,r6 + stvx v5,r3,r7 + stvx v4,r3,r8 + stvx v3,r3,r9 + stvx v2,r3,r10 + stvx v1,r3,r11 + stvx v0,r3,r12 addi r3,r3,128 bdnz 1b diff --git a/arch/powerpc/lib/copyuser_power7.S b/arch/powerpc/lib/copyuser_power7.S index 92ee840529bc..da0c568d18c4 100644 --- a/arch/powerpc/lib/copyuser_power7.S +++ b/arch/powerpc/lib/copyuser_power7.S @@ -388,29 +388,29 @@ err3; std r0,0(r3) li r11,48 bf cr7*4+3,5f -err3; lvx vr1,r0,r4 +err3; lvx v1,r0,r4 addi r4,r4,16 -err3; stvx vr1,r0,r3 +err3; stvx v1,r0,r3 addi r3,r3,16 5: bf cr7*4+2,6f -err3; lvx vr1,r0,r4 -err3; lvx vr0,r4,r9 +err3; lvx v1,r0,r4 +err3; lvx v0,r4,r9 addi r4,r4,32 -err3; stvx vr1,r0,r3 -err3; stvx vr0,r3,r9 +err3; stvx v1,r0,r3 +err3; stvx v0,r3,r9 addi r3,r3,32 6: bf cr7*4+1,7f -err3; lvx vr3,r0,r4 -err3; lvx vr2,r4,r9 -err3; lvx vr1,r4,r10 -err3; lvx vr0,r4,r11 +err3; lvx v3,r0,r4 +err3; lvx v2,r4,r9 +err3; lvx v1,r4,r10 +err3; lvx v0,r4,r11 addi r4,r4,64 -err3; stvx vr3,r0,r3 -err3; stvx vr2,r3,r9 -err3; stvx vr1,r3,r10 -err3; stvx vr0,r3,r11 +err3; stvx v3,r0,r3 +err3; stvx v2,r3,r9 +err3; stvx v1,r3,r10 +err3; stvx v0,r3,r11 addi r3,r3,64 7: sub r5,r5,r6 @@ -433,23 +433,23 @@ err3; stvx vr0,r3,r11 */ .align 5 8: -err4; lvx vr7,r0,r4 -err4; lvx vr6,r4,r9 -err4; lvx vr5,r4,r10 -err4; lvx vr4,r4,r11 -err4; lvx vr3,r4,r12 -err4; lvx vr2,r4,r14 -err4; lvx vr1,r4,r15 -err4; lvx vr0,r4,r16 +err4; lvx v7,r0,r4 +err4; lvx v6,r4,r9 +err4; lvx v5,r4,r10 +err4; lvx v4,r4,r11 +err4; lvx v3,r4,r12 +err4; lvx v2,r4,r14 +err4; lvx v1,r4,r15 +err4; lvx v0,r4,r16 addi r4,r4,128 -err4; stvx vr7,r0,r3 -err4; stvx vr6,r3,r9 -err4; stvx vr5,r3,r10 -err4; stvx vr4,r3,r11 -err4; stvx vr3,r3,r12 -err4; stvx vr2,r3,r14 -err4; stvx vr1,r3,r15 -err4; stvx vr0,r3,r16 +err4; stvx v7,r0,r3 +err4; stvx v6,r3,r9 +err4; stvx v5,r3,r10 +err4; stvx v4,r3,r11 +err4; stvx v3,r3,r12 +err4; stvx v2,r3,r14 +err4; stvx v1,r3,r15 +err4; stvx v0,r3,r16 addi r3,r3,128 bdnz 8b @@ -463,29 +463,29 @@ err4; stvx vr0,r3,r16 mtocrf 0x01,r6 bf cr7*4+1,9f -err3; lvx vr3,r0,r4 -err3; lvx vr2,r4,r9 -err3; lvx vr1,r4,r10 -err3; lvx vr0,r4,r11 +err3; lvx v3,r0,r4 +err3; lvx v2,r4,r9 +err3; lvx v1,r4,r10 +err3; lvx v0,r4,r11 addi r4,r4,64 -err3; stvx vr3,r0,r3 -err3; stvx vr2,r3,r9 -err3; stvx vr1,r3,r10 -err3; stvx vr0,r3,r11 +err3; stvx v3,r0,r3 +err3; stvx v2,r3,r9 +err3; stvx v1,r3,r10 +err3; stvx v0,r3,r11 addi r3,r3,64 9: bf cr7*4+2,10f -err3; lvx vr1,r0,r4 -err3; lvx vr0,r4,r9 +err3; lvx v1,r0,r4 +err3; lvx v0,r4,r9 addi r4,r4,32 -err3; stvx vr1,r0,r3 -err3; stvx vr0,r3,r9 +err3; stvx v1,r0,r3 +err3; stvx v0,r3,r9 addi r3,r3,32 10: bf cr7*4+3,11f -err3; lvx vr1,r0,r4 +err3; lvx v1,r0,r4 addi r4,r4,16 -err3; stvx vr1,r0,r3 +err3; stvx v1,r0,r3 addi r3,r3,16 /* Up to 15B to go */ @@ -560,42 +560,42 @@ err3; stw r7,4(r3) li r10,32 li r11,48 - LVS(vr16,0,r4) /* Setup permute control vector */ -err3; lvx vr0,0,r4 + LVS(v16,0,r4) /* Setup permute control vector */ +err3; lvx v0,0,r4 addi r4,r4,16 bf cr7*4+3,5f -err3; lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) +err3; lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) addi r4,r4,16 -err3; stvx vr8,r0,r3 +err3; stvx v8,r0,r3 addi r3,r3,16 - vor vr0,vr1,vr1 + vor v0,v1,v1 5: bf cr7*4+2,6f -err3; lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) -err3; lvx vr0,r4,r9 - VPERM(vr9,vr1,vr0,vr16) +err3; lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) +err3; lvx v0,r4,r9 + VPERM(v9,v1,v0,v16) addi r4,r4,32 -err3; stvx vr8,r0,r3 -err3; stvx vr9,r3,r9 +err3; stvx v8,r0,r3 +err3; stvx v9,r3,r9 addi r3,r3,32 6: bf cr7*4+1,7f -err3; lvx vr3,r0,r4 - VPERM(vr8,vr0,vr3,vr16) -err3; lvx vr2,r4,r9 - VPERM(vr9,vr3,vr2,vr16) -err3; lvx vr1,r4,r10 - VPERM(vr10,vr2,vr1,vr16) -err3; lvx vr0,r4,r11 - VPERM(vr11,vr1,vr0,vr16) +err3; lvx v3,r0,r4 + VPERM(v8,v0,v3,v16) +err3; lvx v2,r4,r9 + VPERM(v9,v3,v2,v16) +err3; lvx v1,r4,r10 + VPERM(v10,v2,v1,v16) +err3; lvx v0,r4,r11 + VPERM(v11,v1,v0,v16) addi r4,r4,64 -err3; stvx vr8,r0,r3 -err3; stvx vr9,r3,r9 -err3; stvx vr10,r3,r10 -err3; stvx vr11,r3,r11 +err3; stvx v8,r0,r3 +err3; stvx v9,r3,r9 +err3; stvx v10,r3,r10 +err3; stvx v11,r3,r11 addi r3,r3,64 7: sub r5,r5,r6 @@ -618,31 +618,31 @@ err3; stvx vr11,r3,r11 */ .align 5 8: -err4; lvx vr7,r0,r4 - VPERM(vr8,vr0,vr7,vr16) -err4; lvx vr6,r4,r9 - VPERM(vr9,vr7,vr6,vr16) -err4; lvx vr5,r4,r10 - VPERM(vr10,vr6,vr5,vr16) -err4; lvx vr4,r4,r11 - VPERM(vr11,vr5,vr4,vr16) -err4; lvx vr3,r4,r12 - VPERM(vr12,vr4,vr3,vr16) -err4; lvx vr2,r4,r14 - VPERM(vr13,vr3,vr2,vr16) -err4; lvx vr1,r4,r15 - VPERM(vr14,vr2,vr1,vr16) -err4; lvx vr0,r4,r16 - VPERM(vr15,vr1,vr0,vr16) +err4; lvx v7,r0,r4 + VPERM(v8,v0,v7,v16) +err4; lvx v6,r4,r9 + VPERM(v9,v7,v6,v16) +err4; lvx v5,r4,r10 + VPERM(v10,v6,v5,v16) +err4; lvx v4,r4,r11 + VPERM(v11,v5,v4,v16) +err4; lvx v3,r4,r12 + VPERM(v12,v4,v3,v16) +err4; lvx v2,r4,r14 + VPERM(v13,v3,v2,v16) +err4; lvx v1,r4,r15 + VPERM(v14,v2,v1,v16) +err4; lvx v0,r4,r16 + VPERM(v15,v1,v0,v16) addi r4,r4,128 -err4; stvx vr8,r0,r3 -err4; stvx vr9,r3,r9 -err4; stvx vr10,r3,r10 -err4; stvx vr11,r3,r11 -err4; stvx vr12,r3,r12 -err4; stvx vr13,r3,r14 -err4; stvx vr14,r3,r15 -err4; stvx vr15,r3,r16 +err4; stvx v8,r0,r3 +err4; stvx v9,r3,r9 +err4; stvx v10,r3,r10 +err4; stvx v11,r3,r11 +err4; stvx v12,r3,r12 +err4; stvx v13,r3,r14 +err4; stvx v14,r3,r15 +err4; stvx v15,r3,r16 addi r3,r3,128 bdnz 8b @@ -656,36 +656,36 @@ err4; stvx vr15,r3,r16 mtocrf 0x01,r6 bf cr7*4+1,9f -err3; lvx vr3,r0,r4 - VPERM(vr8,vr0,vr3,vr16) -err3; lvx vr2,r4,r9 - VPERM(vr9,vr3,vr2,vr16) -err3; lvx vr1,r4,r10 - VPERM(vr10,vr2,vr1,vr16) -err3; lvx vr0,r4,r11 - VPERM(vr11,vr1,vr0,vr16) +err3; lvx v3,r0,r4 + VPERM(v8,v0,v3,v16) +err3; lvx v2,r4,r9 + VPERM(v9,v3,v2,v16) +err3; lvx v1,r4,r10 + VPERM(v10,v2,v1,v16) +err3; lvx v0,r4,r11 + VPERM(v11,v1,v0,v16) addi r4,r4,64 -err3; stvx vr8,r0,r3 -err3; stvx vr9,r3,r9 -err3; stvx vr10,r3,r10 -err3; stvx vr11,r3,r11 +err3; stvx v8,r0,r3 +err3; stvx v9,r3,r9 +err3; stvx v10,r3,r10 +err3; stvx v11,r3,r11 addi r3,r3,64 9: bf cr7*4+2,10f -err3; lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) -err3; lvx vr0,r4,r9 - VPERM(vr9,vr1,vr0,vr16) +err3; lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) +err3; lvx v0,r4,r9 + VPERM(v9,v1,v0,v16) addi r4,r4,32 -err3; stvx vr8,r0,r3 -err3; stvx vr9,r3,r9 +err3; stvx v8,r0,r3 +err3; stvx v9,r3,r9 addi r3,r3,32 10: bf cr7*4+3,11f -err3; lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) +err3; lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) addi r4,r4,16 -err3; stvx vr8,r0,r3 +err3; stvx v8,r0,r3 addi r3,r3,16 /* Up to 15B to go */ diff --git a/arch/powerpc/lib/crtsavres.S b/arch/powerpc/lib/crtsavres.S index a5b30c71a8d3..18af0b3d3eb2 100644 --- a/arch/powerpc/lib/crtsavres.S +++ b/arch/powerpc/lib/crtsavres.S @@ -236,78 +236,78 @@ _GLOBAL(_rest32gpr_31_x) _GLOBAL(_savevr_20) li r11,-192 - stvx vr20,r11,r0 + stvx v20,r11,r0 _GLOBAL(_savevr_21) li r11,-176 - stvx vr21,r11,r0 + stvx v21,r11,r0 _GLOBAL(_savevr_22) li r11,-160 - stvx vr22,r11,r0 + stvx v22,r11,r0 _GLOBAL(_savevr_23) li r11,-144 - stvx vr23,r11,r0 + stvx v23,r11,r0 _GLOBAL(_savevr_24) li r11,-128 - stvx vr24,r11,r0 + stvx v24,r11,r0 _GLOBAL(_savevr_25) li r11,-112 - stvx vr25,r11,r0 + stvx v25,r11,r0 _GLOBAL(_savevr_26) li r11,-96 - stvx vr26,r11,r0 + stvx v26,r11,r0 _GLOBAL(_savevr_27) li r11,-80 - stvx vr27,r11,r0 + stvx v27,r11,r0 _GLOBAL(_savevr_28) li r11,-64 - stvx vr28,r11,r0 + stvx v28,r11,r0 _GLOBAL(_savevr_29) li r11,-48 - stvx vr29,r11,r0 + stvx v29,r11,r0 _GLOBAL(_savevr_30) li r11,-32 - stvx vr30,r11,r0 + stvx v30,r11,r0 _GLOBAL(_savevr_31) li r11,-16 - stvx vr31,r11,r0 + stvx v31,r11,r0 blr _GLOBAL(_restvr_20) li r11,-192 - lvx vr20,r11,r0 + lvx v20,r11,r0 _GLOBAL(_restvr_21) li r11,-176 - lvx vr21,r11,r0 + lvx v21,r11,r0 _GLOBAL(_restvr_22) li r11,-160 - lvx vr22,r11,r0 + lvx v22,r11,r0 _GLOBAL(_restvr_23) li r11,-144 - lvx vr23,r11,r0 + lvx v23,r11,r0 _GLOBAL(_restvr_24) li r11,-128 - lvx vr24,r11,r0 + lvx v24,r11,r0 _GLOBAL(_restvr_25) li r11,-112 - lvx vr25,r11,r0 + lvx v25,r11,r0 _GLOBAL(_restvr_26) li r11,-96 - lvx vr26,r11,r0 + lvx v26,r11,r0 _GLOBAL(_restvr_27) li r11,-80 - lvx vr27,r11,r0 + lvx v27,r11,r0 _GLOBAL(_restvr_28) li r11,-64 - lvx vr28,r11,r0 + lvx v28,r11,r0 _GLOBAL(_restvr_29) li r11,-48 - lvx vr29,r11,r0 + lvx v29,r11,r0 _GLOBAL(_restvr_30) li r11,-32 - lvx vr30,r11,r0 + lvx v30,r11,r0 _GLOBAL(_restvr_31) li r11,-16 - lvx vr31,r11,r0 + lvx v31,r11,r0 blr #endif /* CONFIG_ALTIVEC */ @@ -443,101 +443,101 @@ _restgpr0_31: .globl _savevr_20 _savevr_20: li r12,-192 - stvx vr20,r12,r0 + stvx v20,r12,r0 .globl _savevr_21 _savevr_21: li r12,-176 - stvx vr21,r12,r0 + stvx v21,r12,r0 .globl _savevr_22 _savevr_22: li r12,-160 - stvx vr22,r12,r0 + stvx v22,r12,r0 .globl _savevr_23 _savevr_23: li r12,-144 - stvx vr23,r12,r0 + stvx v23,r12,r0 .globl _savevr_24 _savevr_24: li r12,-128 - stvx vr24,r12,r0 + stvx v24,r12,r0 .globl _savevr_25 _savevr_25: li r12,-112 - stvx vr25,r12,r0 + stvx v25,r12,r0 .globl _savevr_26 _savevr_26: li r12,-96 - stvx vr26,r12,r0 + stvx v26,r12,r0 .globl _savevr_27 _savevr_27: li r12,-80 - stvx vr27,r12,r0 + stvx v27,r12,r0 .globl _savevr_28 _savevr_28: li r12,-64 - stvx vr28,r12,r0 + stvx v28,r12,r0 .globl _savevr_29 _savevr_29: li r12,-48 - stvx vr29,r12,r0 + stvx v29,r12,r0 .globl _savevr_30 _savevr_30: li r12,-32 - stvx vr30,r12,r0 + stvx v30,r12,r0 .globl _savevr_31 _savevr_31: li r12,-16 - stvx vr31,r12,r0 + stvx v31,r12,r0 blr .globl _restvr_20 _restvr_20: li r12,-192 - lvx vr20,r12,r0 + lvx v20,r12,r0 .globl _restvr_21 _restvr_21: li r12,-176 - lvx vr21,r12,r0 + lvx v21,r12,r0 .globl _restvr_22 _restvr_22: li r12,-160 - lvx vr22,r12,r0 + lvx v22,r12,r0 .globl _restvr_23 _restvr_23: li r12,-144 - lvx vr23,r12,r0 + lvx v23,r12,r0 .globl _restvr_24 _restvr_24: li r12,-128 - lvx vr24,r12,r0 + lvx v24,r12,r0 .globl _restvr_25 _restvr_25: li r12,-112 - lvx vr25,r12,r0 + lvx v25,r12,r0 .globl _restvr_26 _restvr_26: li r12,-96 - lvx vr26,r12,r0 + lvx v26,r12,r0 .globl _restvr_27 _restvr_27: li r12,-80 - lvx vr27,r12,r0 + lvx v27,r12,r0 .globl _restvr_28 _restvr_28: li r12,-64 - lvx vr28,r12,r0 + lvx v28,r12,r0 .globl _restvr_29 _restvr_29: li r12,-48 - lvx vr29,r12,r0 + lvx v29,r12,r0 .globl _restvr_30 _restvr_30: li r12,-32 - lvx vr30,r12,r0 + lvx v30,r12,r0 .globl _restvr_31 _restvr_31: li r12,-16 - lvx vr31,r12,r0 + lvx v31,r12,r0 blr #endif /* CONFIG_ALTIVEC */ diff --git a/arch/powerpc/lib/ldstfp.S b/arch/powerpc/lib/ldstfp.S index 85aec08ab234..5d0cdbfbe3f2 100644 --- a/arch/powerpc/lib/ldstfp.S +++ b/arch/powerpc/lib/ldstfp.S @@ -184,16 +184,16 @@ _GLOBAL(do_stfd) extab 2b,3b #ifdef CONFIG_ALTIVEC -/* Get the contents of vrN into vr0; N is in r3. */ +/* Get the contents of vrN into v0; N is in r3. */ _GLOBAL(get_vr) mflr r0 rlwinm r3,r3,3,0xf8 bcl 20,31,1f - blr /* vr0 is already in vr0 */ + blr /* v0 is already in v0 */ nop reg = 1 .rept 31 - vor vr0,reg,reg /* assembler doesn't know vmr? */ + vor v0,reg,reg /* assembler doesn't know vmr? */ blr reg = reg + 1 .endr @@ -203,16 +203,16 @@ reg = reg + 1 mtlr r0 bctr -/* Put the contents of vr0 into vrN; N is in r3. */ +/* Put the contents of v0 into vrN; N is in r3. */ _GLOBAL(put_vr) mflr r0 rlwinm r3,r3,3,0xf8 bcl 20,31,1f - blr /* vr0 is already in vr0 */ + blr /* v0 is already in v0 */ nop reg = 1 .rept 31 - vor reg,vr0,vr0 + vor reg,v0,v0 blr reg = reg + 1 .endr @@ -234,13 +234,13 @@ _GLOBAL(do_lvx) MTMSRD(r7) isync beq cr7,1f - stvx vr0,r1,r8 + stvx v0,r1,r8 1: li r9,-EFAULT -2: lvx vr0,0,r4 +2: lvx v0,0,r4 li r9,0 3: beq cr7,4f bl put_vr - lvx vr0,r1,r8 + lvx v0,r1,r8 4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1) mtlr r0 MTMSRD(r6) @@ -262,13 +262,13 @@ _GLOBAL(do_stvx) MTMSRD(r7) isync beq cr7,1f - stvx vr0,r1,r8 + stvx v0,r1,r8 bl get_vr 1: li r9,-EFAULT -2: stvx vr0,0,r4 +2: stvx v0,0,r4 li r9,0 3: beq cr7,4f - lvx vr0,r1,r8 + lvx v0,r1,r8 4: PPC_LL r0,STKFRM+PPC_LR_STKOFF(r1) mtlr r0 MTMSRD(r6) @@ -280,12 +280,12 @@ _GLOBAL(do_stvx) #endif /* CONFIG_ALTIVEC */ #ifdef CONFIG_VSX -/* Get the contents of vsrN into vsr0; N is in r3. */ +/* Get the contents of vsN into vs0; N is in r3. */ _GLOBAL(get_vsr) mflr r0 rlwinm r3,r3,3,0x1f8 bcl 20,31,1f - blr /* vsr0 is already in vsr0 */ + blr /* vs0 is already in vs0 */ nop reg = 1 .rept 63 @@ -299,12 +299,12 @@ reg = reg + 1 mtlr r0 bctr -/* Put the contents of vsr0 into vsrN; N is in r3. */ +/* Put the contents of vs0 into vsN; N is in r3. */ _GLOBAL(put_vsr) mflr r0 rlwinm r3,r3,3,0x1f8 bcl 20,31,1f - blr /* vr0 is already in vr0 */ + blr /* v0 is already in v0 */ nop reg = 1 .rept 63 diff --git a/arch/powerpc/lib/locks.c b/arch/powerpc/lib/locks.c index 170a0346f756..f7deebdf3365 100644 --- a/arch/powerpc/lib/locks.c +++ b/arch/powerpc/lib/locks.c @@ -41,6 +41,7 @@ void __spin_yield(arch_spinlock_t *lock) plpar_hcall_norets(H_CONFER, get_hard_smp_processor_id(holder_cpu), yield_count); } +EXPORT_SYMBOL_GPL(__spin_yield); /* * Waiting for a read lock or a write lock on a rwlock... diff --git a/arch/powerpc/lib/memcpy_power7.S b/arch/powerpc/lib/memcpy_power7.S index 0830587df16e..786234fd4e91 100644 --- a/arch/powerpc/lib/memcpy_power7.S +++ b/arch/powerpc/lib/memcpy_power7.S @@ -321,29 +321,29 @@ _GLOBAL(memcpy_power7) li r11,48 bf cr7*4+3,5f - lvx vr1,r0,r4 + lvx v1,r0,r4 addi r4,r4,16 - stvx vr1,r0,r3 + stvx v1,r0,r3 addi r3,r3,16 5: bf cr7*4+2,6f - lvx vr1,r0,r4 - lvx vr0,r4,r9 + lvx v1,r0,r4 + lvx v0,r4,r9 addi r4,r4,32 - stvx vr1,r0,r3 - stvx vr0,r3,r9 + stvx v1,r0,r3 + stvx v0,r3,r9 addi r3,r3,32 6: bf cr7*4+1,7f - lvx vr3,r0,r4 - lvx vr2,r4,r9 - lvx vr1,r4,r10 - lvx vr0,r4,r11 + lvx v3,r0,r4 + lvx v2,r4,r9 + lvx v1,r4,r10 + lvx v0,r4,r11 addi r4,r4,64 - stvx vr3,r0,r3 - stvx vr2,r3,r9 - stvx vr1,r3,r10 - stvx vr0,r3,r11 + stvx v3,r0,r3 + stvx v2,r3,r9 + stvx v1,r3,r10 + stvx v0,r3,r11 addi r3,r3,64 7: sub r5,r5,r6 @@ -366,23 +366,23 @@ _GLOBAL(memcpy_power7) */ .align 5 8: - lvx vr7,r0,r4 - lvx vr6,r4,r9 - lvx vr5,r4,r10 - lvx vr4,r4,r11 - lvx vr3,r4,r12 - lvx vr2,r4,r14 - lvx vr1,r4,r15 - lvx vr0,r4,r16 + lvx v7,r0,r4 + lvx v6,r4,r9 + lvx v5,r4,r10 + lvx v4,r4,r11 + lvx v3,r4,r12 + lvx v2,r4,r14 + lvx v1,r4,r15 + lvx v0,r4,r16 addi r4,r4,128 - stvx vr7,r0,r3 - stvx vr6,r3,r9 - stvx vr5,r3,r10 - stvx vr4,r3,r11 - stvx vr3,r3,r12 - stvx vr2,r3,r14 - stvx vr1,r3,r15 - stvx vr0,r3,r16 + stvx v7,r0,r3 + stvx v6,r3,r9 + stvx v5,r3,r10 + stvx v4,r3,r11 + stvx v3,r3,r12 + stvx v2,r3,r14 + stvx v1,r3,r15 + stvx v0,r3,r16 addi r3,r3,128 bdnz 8b @@ -396,29 +396,29 @@ _GLOBAL(memcpy_power7) mtocrf 0x01,r6 bf cr7*4+1,9f - lvx vr3,r0,r4 - lvx vr2,r4,r9 - lvx vr1,r4,r10 - lvx vr0,r4,r11 + lvx v3,r0,r4 + lvx v2,r4,r9 + lvx v1,r4,r10 + lvx v0,r4,r11 addi r4,r4,64 - stvx vr3,r0,r3 - stvx vr2,r3,r9 - stvx vr1,r3,r10 - stvx vr0,r3,r11 + stvx v3,r0,r3 + stvx v2,r3,r9 + stvx v1,r3,r10 + stvx v0,r3,r11 addi r3,r3,64 9: bf cr7*4+2,10f - lvx vr1,r0,r4 - lvx vr0,r4,r9 + lvx v1,r0,r4 + lvx v0,r4,r9 addi r4,r4,32 - stvx vr1,r0,r3 - stvx vr0,r3,r9 + stvx v1,r0,r3 + stvx v0,r3,r9 addi r3,r3,32 10: bf cr7*4+3,11f - lvx vr1,r0,r4 + lvx v1,r0,r4 addi r4,r4,16 - stvx vr1,r0,r3 + stvx v1,r0,r3 addi r3,r3,16 /* Up to 15B to go */ @@ -494,42 +494,42 @@ _GLOBAL(memcpy_power7) li r10,32 li r11,48 - LVS(vr16,0,r4) /* Setup permute control vector */ - lvx vr0,0,r4 + LVS(v16,0,r4) /* Setup permute control vector */ + lvx v0,0,r4 addi r4,r4,16 bf cr7*4+3,5f - lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) + lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) addi r4,r4,16 - stvx vr8,r0,r3 + stvx v8,r0,r3 addi r3,r3,16 - vor vr0,vr1,vr1 + vor v0,v1,v1 5: bf cr7*4+2,6f - lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) - lvx vr0,r4,r9 - VPERM(vr9,vr1,vr0,vr16) + lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) + lvx v0,r4,r9 + VPERM(v9,v1,v0,v16) addi r4,r4,32 - stvx vr8,r0,r3 - stvx vr9,r3,r9 + stvx v8,r0,r3 + stvx v9,r3,r9 addi r3,r3,32 6: bf cr7*4+1,7f - lvx vr3,r0,r4 - VPERM(vr8,vr0,vr3,vr16) - lvx vr2,r4,r9 - VPERM(vr9,vr3,vr2,vr16) - lvx vr1,r4,r10 - VPERM(vr10,vr2,vr1,vr16) - lvx vr0,r4,r11 - VPERM(vr11,vr1,vr0,vr16) + lvx v3,r0,r4 + VPERM(v8,v0,v3,v16) + lvx v2,r4,r9 + VPERM(v9,v3,v2,v16) + lvx v1,r4,r10 + VPERM(v10,v2,v1,v16) + lvx v0,r4,r11 + VPERM(v11,v1,v0,v16) addi r4,r4,64 - stvx vr8,r0,r3 - stvx vr9,r3,r9 - stvx vr10,r3,r10 - stvx vr11,r3,r11 + stvx v8,r0,r3 + stvx v9,r3,r9 + stvx v10,r3,r10 + stvx v11,r3,r11 addi r3,r3,64 7: sub r5,r5,r6 @@ -552,31 +552,31 @@ _GLOBAL(memcpy_power7) */ .align 5 8: - lvx vr7,r0,r4 - VPERM(vr8,vr0,vr7,vr16) - lvx vr6,r4,r9 - VPERM(vr9,vr7,vr6,vr16) - lvx vr5,r4,r10 - VPERM(vr10,vr6,vr5,vr16) - lvx vr4,r4,r11 - VPERM(vr11,vr5,vr4,vr16) - lvx vr3,r4,r12 - VPERM(vr12,vr4,vr3,vr16) - lvx vr2,r4,r14 - VPERM(vr13,vr3,vr2,vr16) - lvx vr1,r4,r15 - VPERM(vr14,vr2,vr1,vr16) - lvx vr0,r4,r16 - VPERM(vr15,vr1,vr0,vr16) + lvx v7,r0,r4 + VPERM(v8,v0,v7,v16) + lvx v6,r4,r9 + VPERM(v9,v7,v6,v16) + lvx v5,r4,r10 + VPERM(v10,v6,v5,v16) + lvx v4,r4,r11 + VPERM(v11,v5,v4,v16) + lvx v3,r4,r12 + VPERM(v12,v4,v3,v16) + lvx v2,r4,r14 + VPERM(v13,v3,v2,v16) + lvx v1,r4,r15 + VPERM(v14,v2,v1,v16) + lvx v0,r4,r16 + VPERM(v15,v1,v0,v16) addi r4,r4,128 - stvx vr8,r0,r3 - stvx vr9,r3,r9 - stvx vr10,r3,r10 - stvx vr11,r3,r11 - stvx vr12,r3,r12 - stvx vr13,r3,r14 - stvx vr14,r3,r15 - stvx vr15,r3,r16 + stvx v8,r0,r3 + stvx v9,r3,r9 + stvx v10,r3,r10 + stvx v11,r3,r11 + stvx v12,r3,r12 + stvx v13,r3,r14 + stvx v14,r3,r15 + stvx v15,r3,r16 addi r3,r3,128 bdnz 8b @@ -590,36 +590,36 @@ _GLOBAL(memcpy_power7) mtocrf 0x01,r6 bf cr7*4+1,9f - lvx vr3,r0,r4 - VPERM(vr8,vr0,vr3,vr16) - lvx vr2,r4,r9 - VPERM(vr9,vr3,vr2,vr16) - lvx vr1,r4,r10 - VPERM(vr10,vr2,vr1,vr16) - lvx vr0,r4,r11 - VPERM(vr11,vr1,vr0,vr16) + lvx v3,r0,r4 + VPERM(v8,v0,v3,v16) + lvx v2,r4,r9 + VPERM(v9,v3,v2,v16) + lvx v1,r4,r10 + VPERM(v10,v2,v1,v16) + lvx v0,r4,r11 + VPERM(v11,v1,v0,v16) addi r4,r4,64 - stvx vr8,r0,r3 - stvx vr9,r3,r9 - stvx vr10,r3,r10 - stvx vr11,r3,r11 + stvx v8,r0,r3 + stvx v9,r3,r9 + stvx v10,r3,r10 + stvx v11,r3,r11 addi r3,r3,64 9: bf cr7*4+2,10f - lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) - lvx vr0,r4,r9 - VPERM(vr9,vr1,vr0,vr16) + lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) + lvx v0,r4,r9 + VPERM(v9,v1,v0,v16) addi r4,r4,32 - stvx vr8,r0,r3 - stvx vr9,r3,r9 + stvx v8,r0,r3 + stvx v9,r3,r9 addi r3,r3,32 10: bf cr7*4+3,11f - lvx vr1,r0,r4 - VPERM(vr8,vr0,vr1,vr16) + lvx v1,r0,r4 + VPERM(v8,v0,v1,v16) addi r4,r4,16 - stvx vr8,r0,r3 + stvx v8,r0,r3 addi r3,r3,16 /* Up to 15B to go */ diff --git a/arch/powerpc/lib/ppc_ksyms.c b/arch/powerpc/lib/ppc_ksyms.c index f993959647b5..c7f8e9586316 100644 --- a/arch/powerpc/lib/ppc_ksyms.c +++ b/arch/powerpc/lib/ppc_ksyms.c @@ -8,10 +8,6 @@ EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memmove); EXPORT_SYMBOL(memcmp); EXPORT_SYMBOL(memchr); -#ifdef CONFIG_PPC32 -EXPORT_SYMBOL(cacheable_memcpy); -EXPORT_SYMBOL(cacheable_memzero); -#endif EXPORT_SYMBOL(strcpy); EXPORT_SYMBOL(strncpy); diff --git a/arch/powerpc/lib/rheap.c b/arch/powerpc/lib/rheap.c index a1060a868e69..69abf844c2c3 100644 --- a/arch/powerpc/lib/rheap.c +++ b/arch/powerpc/lib/rheap.c @@ -284,7 +284,7 @@ EXPORT_SYMBOL_GPL(rh_create); */ void rh_destroy(rh_info_t * info) { - if ((info->flags & RHIF_STATIC_BLOCK) == 0 && info->block != NULL) + if ((info->flags & RHIF_STATIC_BLOCK) == 0) kfree(info->block); if ((info->flags & RHIF_STATIC_INFO) == 0) diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index 438dcd3fd0d1..9c8770b5f96f 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile @@ -24,6 +24,7 @@ obj-$(CONFIG_40x) += 40x_mmu.o obj-$(CONFIG_44x) += 44x_mmu.o obj-$(CONFIG_PPC_FSL_BOOK3E) += fsl_booke_mmu.o obj-$(CONFIG_NEED_MULTIPLE_NODES) += numa.o +obj-$(CONFIG_PPC_SPLPAR) += vphn.o obj-$(CONFIG_PPC_MM_SLICES) += slice.o obj-y += hugetlbpage.o ifeq ($(CONFIG_HUGETLB_PAGE),y) diff --git a/arch/powerpc/mm/dma-noncoherent.c b/arch/powerpc/mm/dma-noncoherent.c index d85e86aac7fb..169aba446a74 100644 --- a/arch/powerpc/mm/dma-noncoherent.c +++ b/arch/powerpc/mm/dma-noncoherent.c @@ -228,7 +228,7 @@ __dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t do { SetPageReserved(page); map_page(vaddr, page_to_phys(page), - pgprot_noncached(PAGE_KERNEL)); + pgprot_val(pgprot_noncached(PAGE_KERNEL))); page++; vaddr += PAGE_SIZE; } while (size -= PAGE_SIZE); diff --git a/arch/powerpc/mm/fsl_booke_mmu.c b/arch/powerpc/mm/fsl_booke_mmu.c index b46912fee7cd..9c90e66cffb6 100644 --- a/arch/powerpc/mm/fsl_booke_mmu.c +++ b/arch/powerpc/mm/fsl_booke_mmu.c @@ -181,7 +181,7 @@ static unsigned long map_mem_in_cams_addr(phys_addr_t phys, unsigned long virt, unsigned long cam_sz; cam_sz = calc_cam_sz(ram, virt, phys); - settlbcam(i, virt, phys, cam_sz, PAGE_KERNEL_X, 0); + settlbcam(i, virt, phys, cam_sz, pgprot_val(PAGE_KERNEL_X), 0); ram -= cam_sz; amount_mapped += cam_sz; diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c index 86686514ae13..43dafb9d6a46 100644 --- a/arch/powerpc/mm/hugepage-hash64.c +++ b/arch/powerpc/mm/hugepage-hash64.c @@ -33,7 +33,7 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, * atomically mark the linux large page PMD busy and dirty */ do { - pmd_t pmd = ACCESS_ONCE(*pmdp); + pmd_t pmd = READ_ONCE(*pmdp); old_pmd = pmd_val(pmd); /* If PMD busy, retry the access */ diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 7e408bfc7948..fa9d5c238d22 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -964,7 +964,7 @@ pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift *shift = 0; pgdp = pgdir + pgd_index(ea); - pgd = ACCESS_ONCE(*pgdp); + pgd = READ_ONCE(*pgdp); /* * Always operate on the local stack value. This make sure the * value don't get updated by a parallel THP split/collapse, @@ -1045,7 +1045,7 @@ int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr, if (pte_end < end) end = pte_end; - pte = ACCESS_ONCE(*ptep); + pte = READ_ONCE(*ptep); mask = _PAGE_PRESENT | _PAGE_USER; if (write) mask |= _PAGE_RW; diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index 10471f9bb63f..d747dd7bc90b 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c @@ -132,6 +132,7 @@ void pgtable_cache_add(unsigned shift, void (*ctor)(void *)) align = max_t(unsigned long, align, minalign); name = kasprintf(GFP_KERNEL, "pgtable-2^%d", shift); new = kmem_cache_create(name, table_size, align, 0, ctor); + kfree(name); pgtable_cache[shift - 1] = new; pr_debug("Allocated pgtable cache for order %d\n", shift); } diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index b7285a5870f8..45fda71feb27 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -61,7 +61,6 @@ #define CPU_FTR_NOEXECUTE 0 #endif -int mem_init_done; unsigned long long memory_limit; #ifdef CONFIG_HIGHMEM @@ -377,8 +376,6 @@ void __init mem_init(void) pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n", VMALLOC_START, VMALLOC_END); #endif /* CONFIG_PPC32 */ - - mem_init_done = 1; } void free_initmem(void) diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h index 78c45f392f5b..085b66b10891 100644 --- a/arch/powerpc/mm/mmu_decl.h +++ b/arch/powerpc/mm/mmu_decl.h @@ -96,7 +96,7 @@ extern void _tlbia(void); extern void mapin_ram(void); extern int map_page(unsigned long va, phys_addr_t pa, int flags); extern void setbat(int index, unsigned long virt, phys_addr_t phys, - unsigned int size, int flags); + unsigned int size, pgprot_t prot); extern int __map_without_bats; extern int __allow_ioremap_reserved; diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 0257a7d659ef..5e80621d9324 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -958,6 +958,13 @@ void __init initmem_init(void) memblock_dump_all(); + /* + * Reduce the possible NUMA nodes to the online NUMA nodes, + * since we do not support node hotplug. This ensures that we + * lower the maximum NUMA node ID to what is actually present. + */ + nodes_and(node_possible_map, node_possible_map, node_online_map); + for_each_online_node(nid) { unsigned long start_pfn, end_pfn; @@ -1177,6 +1184,9 @@ u64 memory_hotplug_max(void) /* Virtual Processor Home Node (VPHN) support */ #ifdef CONFIG_PPC_SPLPAR + +#include "vphn.h" + struct topology_update_data { struct topology_update_data *next; unsigned int cpu; @@ -1248,55 +1258,6 @@ static int update_cpu_associativity_changes_mask(void) } /* - * 6 64-bit registers unpacked into 12 32-bit associativity values. To form - * the complete property we have to add the length in the first cell. - */ -#define VPHN_ASSOC_BUFSIZE (6*sizeof(u64)/sizeof(u32) + 1) - -/* - * Convert the associativity domain numbers returned from the hypervisor - * to the sequence they would appear in the ibm,associativity property. - */ -static int vphn_unpack_associativity(const long *packed, __be32 *unpacked) -{ - int i, nr_assoc_doms = 0; - const __be16 *field = (const __be16 *) packed; - -#define VPHN_FIELD_UNUSED (0xffff) -#define VPHN_FIELD_MSB (0x8000) -#define VPHN_FIELD_MASK (~VPHN_FIELD_MSB) - - for (i = 1; i < VPHN_ASSOC_BUFSIZE; i++) { - if (be16_to_cpup(field) == VPHN_FIELD_UNUSED) { - /* All significant fields processed, and remaining - * fields contain the reserved value of all 1's. - * Just store them. - */ - unpacked[i] = *((__be32 *)field); - field += 2; - } else if (be16_to_cpup(field) & VPHN_FIELD_MSB) { - /* Data is in the lower 15 bits of this field */ - unpacked[i] = cpu_to_be32( - be16_to_cpup(field) & VPHN_FIELD_MASK); - field++; - nr_assoc_doms++; - } else { - /* Data is in the lower 15 bits of this field - * concatenated with the next 16 bit field - */ - unpacked[i] = *((__be32 *)field); - field += 2; - nr_assoc_doms++; - } - } - - /* The first cell contains the length of the property */ - unpacked[0] = cpu_to_be32(nr_assoc_doms); - - return nr_assoc_doms; -} - -/* * Retrieve the new associativity information for a virtual processor's * home node. */ @@ -1306,11 +1267,8 @@ static long hcall_vphn(unsigned long cpu, __be32 *associativity) long retbuf[PLPAR_HCALL9_BUFSIZE] = {0}; u64 flags = 1; int hwcpu = get_hard_smp_processor_id(cpu); - int i; rc = plpar_hcall9(H_HOME_NODE_ASSOCIATIVITY, retbuf, flags, hwcpu); - for (i = 0; i < 6; i++) - retbuf[i] = cpu_to_be64(retbuf[i]); vphn_unpack_associativity(retbuf, associativity); return rc; diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index 03b1a3b0fbd5..7692d1bb1bc6 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -54,9 +54,6 @@ extern char etext[], _stext[]; #ifdef HAVE_BATS extern phys_addr_t v_mapped_by_bats(unsigned long va); extern unsigned long p_mapped_by_bats(phys_addr_t pa); -void setbat(int index, unsigned long virt, phys_addr_t phys, - unsigned int size, int flags); - #else /* !HAVE_BATS */ #define v_mapped_by_bats(x) (0UL) #define p_mapped_by_bats(x) (0UL) @@ -110,9 +107,8 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd) __init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte; - extern int mem_init_done; - if (mem_init_done) { + if (slab_is_available()) { pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO); } else { pte = __va(memblock_alloc(PAGE_SIZE, PAGE_SIZE)); @@ -192,7 +188,7 @@ __ioremap_caller(phys_addr_t addr, unsigned long size, unsigned long flags, /* Make sure we have the base flags */ if ((flags & _PAGE_PRESENT) == 0) - flags |= PAGE_KERNEL; + flags |= pgprot_val(PAGE_KERNEL); /* Non-cacheable page cannot be coherent */ if (flags & _PAGE_NO_CACHE) @@ -219,9 +215,9 @@ __ioremap_caller(phys_addr_t addr, unsigned long size, unsigned long flags, * Don't allow anybody to remap normal RAM that we're using. * mem_init() sets high_memory so only do the check after that. */ - if (mem_init_done && (p < virt_to_phys(high_memory)) && + if (slab_is_available() && (p < virt_to_phys(high_memory)) && !(__allow_ioremap_reserved && memblock_is_region_reserved(p, size))) { - printk("__ioremap(): phys addr 0x%llx is RAM lr %pf\n", + printk("__ioremap(): phys addr 0x%llx is RAM lr %ps\n", (unsigned long long)p, __builtin_return_address(0)); return NULL; } @@ -247,7 +243,7 @@ __ioremap_caller(phys_addr_t addr, unsigned long size, unsigned long flags, if ((v = p_mapped_by_tlbcam(p))) goto out; - if (mem_init_done) { + if (slab_is_available()) { struct vm_struct *area; area = get_vm_area_caller(size, VM_IOREMAP, caller); if (area == 0) @@ -266,7 +262,7 @@ __ioremap_caller(phys_addr_t addr, unsigned long size, unsigned long flags, for (i = 0; i < size && err == 0; i += PAGE_SIZE) err = map_page(v+i, p+i, flags); if (err) { - if (mem_init_done) + if (slab_is_available()) vunmap((void *)v); return NULL; } @@ -327,7 +323,7 @@ void __init __mapin_ram_chunk(unsigned long offset, unsigned long top) p = memstart_addr + s; for (; s < top; s += PAGE_SIZE) { ktext = ((char *) v >= _stext && (char *) v < etext); - f = ktext ? PAGE_KERNEL_TEXT : PAGE_KERNEL; + f = ktext ? pgprot_val(PAGE_KERNEL_TEXT) : pgprot_val(PAGE_KERNEL); map_page(v, p, f); #ifdef CONFIG_PPC_STD_MMU_32 if (ktext) diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 6957cc1ca0a7..59daa5eeec25 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -231,7 +231,7 @@ void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size, if ((size == 0) || (paligned == 0)) return NULL; - if (mem_init_done) { + if (slab_is_available()) { struct vm_struct *area; area = __get_vm_area_caller(size, VM_IOREMAP, @@ -315,7 +315,7 @@ void __iounmap(volatile void __iomem *token) { void *addr; - if (!mem_init_done) + if (!slab_is_available()) return; addr = (void *) ((unsigned long __force) @@ -723,7 +723,7 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, assert_spin_locked(&mm->page_table_lock); WARN_ON(!pmd_trans_huge(pmd)); #endif - trace_hugepage_set_pmd(addr, pmd); + trace_hugepage_set_pmd(addr, pmd_val(pmd)); return set_pte_at(mm, addr, pmdp_ptep(pmdp), pmd_pte(pmd)); } diff --git a/arch/powerpc/mm/ppc_mmu_32.c b/arch/powerpc/mm/ppc_mmu_32.c index 5029dc19b517..6b2f3e457171 100644 --- a/arch/powerpc/mm/ppc_mmu_32.c +++ b/arch/powerpc/mm/ppc_mmu_32.c @@ -113,11 +113,12 @@ unsigned long __init mmu_mapin_ram(unsigned long top) * of 2 between 128k and 256M. */ void __init setbat(int index, unsigned long virt, phys_addr_t phys, - unsigned int size, int flags) + unsigned int size, pgprot_t prot) { unsigned int bl; int wimgxpp; struct ppc_bat *bat = BATS[index]; + unsigned long flags = pgprot_val(prot); if ((flags & _PAGE_NO_CACHE) || (cpu_has_feature(CPU_FTR_NEED_COHERENT) == 0)) @@ -224,7 +225,7 @@ void __init MMU_init_hw(void) */ if ( ppc_md.progress ) ppc_md.progress("hash:find piece", 0x322); Hash = __va(memblock_alloc(Hash_size, Hash_size)); - cacheable_memzero(Hash, Hash_size); + memset(Hash, 0, Hash_size); _SDR1 = __pa(Hash) | SDR1_LOW_BITS; Hash_end = (struct hash_pte *) ((unsigned long)Hash + Hash_size); diff --git a/arch/powerpc/mm/tlb_hash64.c b/arch/powerpc/mm/tlb_hash64.c index d2a94b85dbc2..c522969f012d 100644 --- a/arch/powerpc/mm/tlb_hash64.c +++ b/arch/powerpc/mm/tlb_hash64.c @@ -216,7 +216,7 @@ void __flush_hash_table_range(struct mm_struct *mm, unsigned long start, continue; pte = pte_val(*ptep); if (hugepage_shift) - trace_hugepage_invalidate(start, pte_val(pte)); + trace_hugepage_invalidate(start, pte); if (!(pte & _PAGE_HASHPTE)) continue; if (unlikely(hugepage_shift && pmd_trans_huge(*(pmd_t *)pte))) diff --git a/arch/powerpc/mm/vphn.c b/arch/powerpc/mm/vphn.c new file mode 100644 index 000000000000..5f8ef50e5c66 --- /dev/null +++ b/arch/powerpc/mm/vphn.c @@ -0,0 +1,70 @@ +#include <asm/byteorder.h> +#include "vphn.h" + +/* + * The associativity domain numbers are returned from the hypervisor as a + * stream of mixed 16-bit and 32-bit fields. The stream is terminated by the + * special value of "all ones" (aka. 0xffff) and its size may not exceed 48 + * bytes. + * + * --- 16-bit fields --> + * _________________________ + * | 0 | 1 | 2 | 3 | be_packed[0] + * ------+-----+-----+------ + * _________________________ + * | 4 | 5 | 6 | 7 | be_packed[1] + * ------------------------- + * ... + * _________________________ + * | 20 | 21 | 22 | 23 | be_packed[5] + * ------------------------- + * + * Convert to the sequence they would appear in the ibm,associativity property. + */ +int vphn_unpack_associativity(const long *packed, __be32 *unpacked) +{ + __be64 be_packed[VPHN_REGISTER_COUNT]; + int i, nr_assoc_doms = 0; + const __be16 *field = (const __be16 *) be_packed; + u16 last = 0; + bool is_32bit = false; + +#define VPHN_FIELD_UNUSED (0xffff) +#define VPHN_FIELD_MSB (0x8000) +#define VPHN_FIELD_MASK (~VPHN_FIELD_MSB) + + /* Let's fix the values returned by plpar_hcall9() */ + for (i = 0; i < VPHN_REGISTER_COUNT; i++) + be_packed[i] = cpu_to_be64(packed[i]); + + for (i = 1; i < VPHN_ASSOC_BUFSIZE; i++) { + u16 new = be16_to_cpup(field++); + + if (is_32bit) { + /* Let's concatenate the 16 bits of this field to the + * 15 lower bits of the previous field + */ + unpacked[++nr_assoc_doms] = + cpu_to_be32(last << 16 | new); + is_32bit = false; + } else if (new == VPHN_FIELD_UNUSED) + /* This is the list terminator */ + break; + else if (new & VPHN_FIELD_MSB) { + /* Data is in the lower 15 bits of this field */ + unpacked[++nr_assoc_doms] = + cpu_to_be32(new & VPHN_FIELD_MASK); + } else { + /* Data is in the lower 15 bits of this field + * concatenated with the next 16 bit field + */ + last = new; + is_32bit = true; + } + } + + /* The first cell contains the length of the property */ + unpacked[0] = cpu_to_be32(nr_assoc_doms); + + return nr_assoc_doms; +} diff --git a/arch/powerpc/mm/vphn.h b/arch/powerpc/mm/vphn.h new file mode 100644 index 000000000000..fe8b7805b78f --- /dev/null +++ b/arch/powerpc/mm/vphn.h @@ -0,0 +1,16 @@ +#ifndef _ARCH_POWERPC_MM_VPHN_H_ +#define _ARCH_POWERPC_MM_VPHN_H_ + +/* The H_HOME_NODE_ASSOCIATIVITY h_call returns 6 64-bit registers. + */ +#define VPHN_REGISTER_COUNT 6 + +/* + * 6 64-bit registers unpacked into up to 24 be32 associativity values. To + * form the complete property we have to add the length in the first cell. + */ +#define VPHN_ASSOC_BUFSIZE (VPHN_REGISTER_COUNT*sizeof(u64)/sizeof(u16) + 1) + +extern int vphn_unpack_associativity(const long *packed, __be32 *unpacked); + +#endif diff --git a/arch/powerpc/net/Makefile b/arch/powerpc/net/Makefile index 266b3950c3ac..1306a58ac541 100644 --- a/arch/powerpc/net/Makefile +++ b/arch/powerpc/net/Makefile @@ -1,4 +1,4 @@ # # Arch-specific network modules # -obj-$(CONFIG_BPF_JIT) += bpf_jit_64.o bpf_jit_comp.o +obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index c406aa95b2bc..889fd199a821 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -10,12 +10,25 @@ #ifndef _BPF_JIT_H #define _BPF_JIT_H +#ifdef CONFIG_PPC64 +#define BPF_PPC_STACK_R3_OFF 48 #define BPF_PPC_STACK_LOCALS 32 #define BPF_PPC_STACK_BASIC (48+64) #define BPF_PPC_STACK_SAVE (18*8) #define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \ BPF_PPC_STACK_SAVE) #define BPF_PPC_SLOWPATH_FRAME (48+64) +#else +#define BPF_PPC_STACK_R3_OFF 24 +#define BPF_PPC_STACK_LOCALS 16 +#define BPF_PPC_STACK_BASIC (24+32) +#define BPF_PPC_STACK_SAVE (18*4) +#define BPF_PPC_STACKFRAME (BPF_PPC_STACK_BASIC+BPF_PPC_STACK_LOCALS+ \ + BPF_PPC_STACK_SAVE) +#define BPF_PPC_SLOWPATH_FRAME (24+32) +#endif + +#define REG_SZ (BITS_PER_LONG/8) /* * Generated code register usage: @@ -57,7 +70,11 @@ DECLARE_LOAD_FUNC(sk_load_half); DECLARE_LOAD_FUNC(sk_load_byte); DECLARE_LOAD_FUNC(sk_load_byte_msh); +#ifdef CONFIG_PPC64 #define FUNCTION_DESCR_SIZE 24 +#else +#define FUNCTION_DESCR_SIZE 0 +#endif /* * 16-bit immediate helper macros: HA() is for use with sign-extending instrs @@ -86,7 +103,12 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh); #define PPC_LIS(r, i) PPC_ADDIS(r, 0, i) #define PPC_STD(r, base, i) EMIT(PPC_INST_STD | ___PPC_RS(r) | \ ___PPC_RA(base) | ((i) & 0xfffc)) - +#define PPC_STDU(r, base, i) EMIT(PPC_INST_STDU | ___PPC_RS(r) | \ + ___PPC_RA(base) | ((i) & 0xfffc)) +#define PPC_STW(r, base, i) EMIT(PPC_INST_STW | ___PPC_RS(r) | \ + ___PPC_RA(base) | ((i) & 0xfffc)) +#define PPC_STWU(r, base, i) EMIT(PPC_INST_STWU | ___PPC_RS(r) | \ + ___PPC_RA(base) | ((i) & 0xfffc)) #define PPC_LBZ(r, base, i) EMIT(PPC_INST_LBZ | ___PPC_RT(r) | \ ___PPC_RA(base) | IMM_L(i)) @@ -98,6 +120,17 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh); ___PPC_RA(base) | IMM_L(i)) #define PPC_LHBRX(r, base, b) EMIT(PPC_INST_LHBRX | ___PPC_RT(r) | \ ___PPC_RA(base) | ___PPC_RB(b)) + +#ifdef CONFIG_PPC64 +#define PPC_BPF_LL(r, base, i) do { PPC_LD(r, base, i); } while(0) +#define PPC_BPF_STL(r, base, i) do { PPC_STD(r, base, i); } while(0) +#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0) +#else +#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0) +#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0) +#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0) +#endif + /* Convenience helpers for the above with 'far' offsets: */ #define PPC_LBZ_OFFS(r, base, i) do { if ((i) < 32768) PPC_LBZ(r, base, i); \ else { PPC_ADDIS(r, base, IMM_HA(i)); \ @@ -115,6 +148,29 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh); else { PPC_ADDIS(r, base, IMM_HA(i)); \ PPC_LHZ(r, r, IMM_L(i)); } } while(0) +#ifdef CONFIG_PPC64 +#define PPC_LL_OFFS(r, base, i) do { PPC_LD_OFFS(r, base, i); } while(0) +#else +#define PPC_LL_OFFS(r, base, i) do { PPC_LWZ_OFFS(r, base, i); } while(0) +#endif + +#ifdef CONFIG_SMP +#ifdef CONFIG_PPC64 +#define PPC_BPF_LOAD_CPU(r) \ + do { BUILD_BUG_ON(FIELD_SIZEOF(struct paca_struct, paca_index) != 2); \ + PPC_LHZ_OFFS(r, 13, offsetof(struct paca_struct, paca_index)); \ + } while (0) +#else +#define PPC_BPF_LOAD_CPU(r) \ + do { BUILD_BUG_ON(FIELD_SIZEOF(struct thread_info, cpu) != 4); \ + PPC_LHZ_OFFS(r, (1 & ~(THREAD_SIZE - 1)), \ + offsetof(struct thread_info, cpu)); \ + } while(0) +#endif +#else +#define PPC_BPF_LOAD_CPU(r) do { PPC_LI(r, 0); } while(0) +#endif + #define PPC_CMPWI(a, i) EMIT(PPC_INST_CMPWI | ___PPC_RA(a) | IMM_L(i)) #define PPC_CMPDI(a, i) EMIT(PPC_INST_CMPDI | ___PPC_RA(a) | IMM_L(i)) #define PPC_CMPLWI(a, i) EMIT(PPC_INST_CMPLWI | ___PPC_RA(a) | IMM_L(i)) @@ -196,6 +252,12 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh); PPC_ORI(d, d, (uintptr_t)(i) & 0xffff); \ } } while (0); +#ifdef CONFIG_PPC64 +#define PPC_FUNC_ADDR(d,i) do { PPC_LI64(d, i); } while(0) +#else +#define PPC_FUNC_ADDR(d,i) do { PPC_LI32(d, i); } while(0) +#endif + #define PPC_LHBRX_OFFS(r, base, i) \ do { PPC_LI32(r, i); PPC_LHBRX(r, r, base); } while(0) #ifdef __LITTLE_ENDIAN__ diff --git a/arch/powerpc/net/bpf_jit_64.S b/arch/powerpc/net/bpf_jit_asm.S index 8f87d9217122..8ff5a3b5d1c3 100644 --- a/arch/powerpc/net/bpf_jit_64.S +++ b/arch/powerpc/net/bpf_jit_asm.S @@ -34,13 +34,13 @@ */ .globl sk_load_word sk_load_word: - cmpdi r_addr, 0 + PPC_LCMPI r_addr, 0 blt bpf_slow_path_word_neg .globl sk_load_word_positive_offset sk_load_word_positive_offset: /* Are we accessing past headlen? */ subi r_scratch1, r_HL, 4 - cmpd r_scratch1, r_addr + PPC_LCMP r_scratch1, r_addr blt bpf_slow_path_word /* Nope, just hitting the header. cr0 here is eq or gt! */ #ifdef __LITTLE_ENDIAN__ @@ -52,12 +52,12 @@ sk_load_word_positive_offset: .globl sk_load_half sk_load_half: - cmpdi r_addr, 0 + PPC_LCMPI r_addr, 0 blt bpf_slow_path_half_neg .globl sk_load_half_positive_offset sk_load_half_positive_offset: subi r_scratch1, r_HL, 2 - cmpd r_scratch1, r_addr + PPC_LCMP r_scratch1, r_addr blt bpf_slow_path_half #ifdef __LITTLE_ENDIAN__ lhbrx r_A, r_D, r_addr @@ -68,11 +68,11 @@ sk_load_half_positive_offset: .globl sk_load_byte sk_load_byte: - cmpdi r_addr, 0 + PPC_LCMPI r_addr, 0 blt bpf_slow_path_byte_neg .globl sk_load_byte_positive_offset sk_load_byte_positive_offset: - cmpd r_HL, r_addr + PPC_LCMP r_HL, r_addr ble bpf_slow_path_byte lbzx r_A, r_D, r_addr blr @@ -83,11 +83,11 @@ sk_load_byte_positive_offset: */ .globl sk_load_byte_msh sk_load_byte_msh: - cmpdi r_addr, 0 + PPC_LCMPI r_addr, 0 blt bpf_slow_path_byte_msh_neg .globl sk_load_byte_msh_positive_offset sk_load_byte_msh_positive_offset: - cmpd r_HL, r_addr + PPC_LCMP r_HL, r_addr ble bpf_slow_path_byte_msh lbzx r_X, r_D, r_addr rlwinm r_X, r_X, 2, 32-4-2, 31-2 @@ -101,13 +101,13 @@ sk_load_byte_msh_positive_offset: */ #define bpf_slow_path_common(SIZE) \ mflr r0; \ - std r0, 16(r1); \ + PPC_STL r0, PPC_LR_STKOFF(r1); \ /* R3 goes in parameter space of caller's frame */ \ - std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ - std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ - std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ - addi r5, r1, BPF_PPC_STACK_BASIC+(2*8); \ - stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ + PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ + PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ + PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ + addi r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ); \ + PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ /* R3 = r_skb, as passed */ \ mr r4, r_addr; \ li r6, SIZE; \ @@ -115,19 +115,19 @@ sk_load_byte_msh_positive_offset: nop; \ /* R3 = 0 on success */ \ addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ - ld r0, 16(r1); \ - ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ - ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ + PPC_LL r0, PPC_LR_STKOFF(r1); \ + PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ + PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ mtlr r0; \ - cmpdi r3, 0; \ + PPC_LCMPI r3, 0; \ blt bpf_error; /* cr0 = LT */ \ - ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ + PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ /* Great success! */ bpf_slow_path_word: bpf_slow_path_common(4) /* Data value is on stack, and cr0 != LT */ - lwz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) + lwz r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1) blr bpf_slow_path_half: @@ -154,12 +154,12 @@ bpf_slow_path_byte_msh: */ #define sk_negative_common(SIZE) \ mflr r0; \ - std r0, 16(r1); \ + PPC_STL r0, PPC_LR_STKOFF(r1); \ /* R3 goes in parameter space of caller's frame */ \ - std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ - std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ - std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ - stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ + PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ + PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ + PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ + PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ /* R3 = r_skb, as passed */ \ mr r4, r_addr; \ li r5, SIZE; \ @@ -167,19 +167,19 @@ bpf_slow_path_byte_msh: nop; \ /* R3 != 0 on success */ \ addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ - ld r0, 16(r1); \ - ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ - ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ + PPC_LL r0, PPC_LR_STKOFF(r1); \ + PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ + PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ mtlr r0; \ - cmpldi r3, 0; \ + PPC_LCMPLI r3, 0; \ beq bpf_error_slow; /* cr0 = EQ */ \ mr r_addr, r3; \ - ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ + PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ /* Great success! */ bpf_slow_path_word_neg: lis r_scratch1,-32 /* SKF_LL_OFF */ - cmpd r_addr, r_scratch1 /* addr < SKF_* */ + PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ blt bpf_error /* cr0 = LT */ .globl sk_load_word_negative_offset sk_load_word_negative_offset: @@ -189,7 +189,7 @@ sk_load_word_negative_offset: bpf_slow_path_half_neg: lis r_scratch1,-32 /* SKF_LL_OFF */ - cmpd r_addr, r_scratch1 /* addr < SKF_* */ + PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ blt bpf_error /* cr0 = LT */ .globl sk_load_half_negative_offset sk_load_half_negative_offset: @@ -199,7 +199,7 @@ sk_load_half_negative_offset: bpf_slow_path_byte_neg: lis r_scratch1,-32 /* SKF_LL_OFF */ - cmpd r_addr, r_scratch1 /* addr < SKF_* */ + PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ blt bpf_error /* cr0 = LT */ .globl sk_load_byte_negative_offset sk_load_byte_negative_offset: @@ -209,7 +209,7 @@ sk_load_byte_negative_offset: bpf_slow_path_byte_msh_neg: lis r_scratch1,-32 /* SKF_LL_OFF */ - cmpd r_addr, r_scratch1 /* addr < SKF_* */ + PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ blt bpf_error /* cr0 = LT */ .globl sk_load_byte_msh_negative_offset sk_load_byte_msh_negative_offset: @@ -221,7 +221,7 @@ sk_load_byte_msh_negative_offset: bpf_error_slow: /* fabricate a cr0 = lt */ li r_scratch1, -1 - cmpdi r_scratch1, 0 + PPC_LCMPI r_scratch1, 0 bpf_error: /* Entered with cr0 = lt */ li r3, 0 diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index d1916b577f2c..17cea18a09d3 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -1,8 +1,9 @@ -/* bpf_jit_comp.c: BPF JIT compiler for PPC64 +/* bpf_jit_comp.c: BPF JIT compiler * * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation * * Based on the x86 BPF compiler, by Eric Dumazet (eric.dumazet@gmail.com) + * Ported to ppc32 by Denis Kirjanov <kda@linux-powerpc.org> * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -36,11 +37,11 @@ static void bpf_jit_build_prologue(struct bpf_prog *fp, u32 *image, if (ctx->seen & SEEN_DATAREF) { /* If we call any helpers (for loads), save LR */ EMIT(PPC_INST_MFLR | __PPC_RT(R0)); - PPC_STD(0, 1, 16); + PPC_BPF_STL(0, 1, PPC_LR_STKOFF); /* Back up non-volatile regs. */ - PPC_STD(r_D, 1, -(8*(32-r_D))); - PPC_STD(r_HL, 1, -(8*(32-r_HL))); + PPC_BPF_STL(r_D, 1, -(REG_SZ*(32-r_D))); + PPC_BPF_STL(r_HL, 1, -(REG_SZ*(32-r_HL))); } if (ctx->seen & SEEN_MEM) { /* @@ -49,11 +50,10 @@ static void bpf_jit_build_prologue(struct bpf_prog *fp, u32 *image, */ for (i = r_M; i < (r_M+16); i++) { if (ctx->seen & (1 << (i-r_M))) - PPC_STD(i, 1, -(8*(32-i))); + PPC_BPF_STL(i, 1, -(REG_SZ*(32-i))); } } - EMIT(PPC_INST_STDU | __PPC_RS(R1) | __PPC_RA(R1) | - (-BPF_PPC_STACKFRAME & 0xfffc)); + PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME); } if (ctx->seen & SEEN_DATAREF) { @@ -67,7 +67,7 @@ static void bpf_jit_build_prologue(struct bpf_prog *fp, u32 *image, data_len)); PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len)); PPC_SUB(r_HL, r_HL, r_scratch1); - PPC_LD_OFFS(r_D, r_skb, offsetof(struct sk_buff, data)); + PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data)); } if (ctx->seen & SEEN_XREG) { @@ -99,16 +99,16 @@ static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) { PPC_ADDI(1, 1, BPF_PPC_STACKFRAME); if (ctx->seen & SEEN_DATAREF) { - PPC_LD(0, 1, 16); + PPC_BPF_LL(0, 1, PPC_LR_STKOFF); PPC_MTLR(0); - PPC_LD(r_D, 1, -(8*(32-r_D))); - PPC_LD(r_HL, 1, -(8*(32-r_HL))); + PPC_BPF_LL(r_D, 1, -(REG_SZ*(32-r_D))); + PPC_BPF_LL(r_HL, 1, -(REG_SZ*(32-r_HL))); } if (ctx->seen & SEEN_MEM) { /* Restore any saved non-vol registers */ for (i = r_M; i < (r_M+16); i++) { if (ctx->seen & (1 << (i-r_M))) - PPC_LD(i, 1, -(8*(32-i))); + PPC_BPF_LL(i, 1, -(REG_SZ*(32-i))); } } } @@ -355,7 +355,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, ifindex) != 4); BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); - PPC_LD_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff, + PPC_LL_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff, dev)); PPC_CMPDI(r_scratch1, 0); if (ctx->pc_ret0 != -1) { @@ -411,20 +411,8 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, PPC_SRWI(r_A, r_A, 5); break; case BPF_ANC | SKF_AD_CPU: -#ifdef CONFIG_SMP - /* - * PACA ptr is r13: - * raw_smp_processor_id() = local_paca->paca_index - */ - BUILD_BUG_ON(FIELD_SIZEOF(struct paca_struct, - paca_index) != 2); - PPC_LHZ_OFFS(r_A, 13, - offsetof(struct paca_struct, paca_index)); -#else - PPC_LI(r_A, 0); -#endif + PPC_BPF_LOAD_CPU(r_A); break; - /*** Absolute loads from packet header/data ***/ case BPF_LD | BPF_W | BPF_ABS: func = CHOOSE_LOAD_FUNC(K, sk_load_word); @@ -437,7 +425,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, common_load: /* Load from [K]. */ ctx->seen |= SEEN_DATAREF; - PPC_LI64(r_scratch1, func); + PPC_FUNC_ADDR(r_scratch1, func); PPC_MTLR(r_scratch1); PPC_LI32(r_addr, K); PPC_BLRL(); @@ -463,7 +451,7 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, * in the helper functions. */ ctx->seen |= SEEN_DATAREF | SEEN_XREG; - PPC_LI64(r_scratch1, func); + PPC_FUNC_ADDR(r_scratch1, func); PPC_MTLR(r_scratch1); PPC_ADDI(r_addr, r_X, IMM_L(K)); if (K >= 32768) @@ -685,9 +673,11 @@ void bpf_jit_compile(struct bpf_prog *fp) if (image) { bpf_flush_icache(code_base, code_base + (proglen/4)); +#ifdef CONFIG_PPC64 /* Function descriptor nastiness: Address + TOC */ ((u64 *)image)[0] = (u64)code_base; ((u64 *)image)[1] = local_paca->kernel_toc; +#endif fp->bpf_func = (void *)image; fp->jited = true; } diff --git a/arch/powerpc/oprofile/cell/spu_task_sync.c b/arch/powerpc/oprofile/cell/spu_task_sync.c index 1c27831df1ac..ed7b0977072a 100644 --- a/arch/powerpc/oprofile/cell/spu_task_sync.c +++ b/arch/powerpc/oprofile/cell/spu_task_sync.c @@ -22,6 +22,7 @@ #include <linux/kref.h> #include <linux/mm.h> #include <linux/fs.h> +#include <linux/file.h> #include <linux/module.h> #include <linux/notifier.h> #include <linux/numa.h> @@ -322,18 +323,20 @@ get_exec_dcookie_and_offset(struct spu *spu, unsigned int *offsetp, unsigned long app_cookie = 0; unsigned int my_offset = 0; struct vm_area_struct *vma; + struct file *exe_file; struct mm_struct *mm = spu->mm; if (!mm) goto out; - down_read(&mm->mmap_sem); - - if (mm->exe_file) { - app_cookie = fast_get_dcookie(&mm->exe_file->f_path); - pr_debug("got dcookie for %pD\n", mm->exe_file); + exe_file = get_mm_exe_file(mm); + if (exe_file) { + app_cookie = fast_get_dcookie(&exe_file->f_path); + pr_debug("got dcookie for %pD\n", exe_file); + fput(exe_file); } + down_read(&mm->mmap_sem); for (vma = mm->mmap; vma; vma = vma->vm_next) { if (vma->vm_start > spu_ref || vma->vm_end <= spu_ref) continue; diff --git a/arch/powerpc/perf/callchain.c b/arch/powerpc/perf/callchain.c index 2396dda282cd..ead55351b254 100644 --- a/arch/powerpc/perf/callchain.c +++ b/arch/powerpc/perf/callchain.c @@ -243,7 +243,7 @@ static void perf_callchain_user_64(struct perf_callchain_entry *entry, sp = regs->gpr[1]; perf_callchain_store(entry, next_ip); - for (;;) { + while (entry->nr < PERF_MAX_STACK_DEPTH) { fp = (unsigned long __user *) sp; if (!valid_user_sp(sp, 1) || read_user_stack_64(fp, &next_sp)) return; diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 7fd60dcb2cb0..12b638425bb9 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -1837,8 +1837,10 @@ static int power_pmu_event_init(struct perf_event *event) cpuhw->bhrb_filter = ppmu->bhrb_filter_map( event->attr.branch_sample_type); - if(cpuhw->bhrb_filter == -1) + if (cpuhw->bhrb_filter == -1) { + put_cpu_var(cpu_hw_events); return -EOPNOTSUPP; + } } put_cpu_var(cpu_hw_events); diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c index abeb9ec0d117..ec2eb20631d1 100644 --- a/arch/powerpc/perf/hv-24x7.c +++ b/arch/powerpc/perf/hv-24x7.c @@ -142,6 +142,15 @@ static struct attribute_group event_long_desc_group = { static struct kmem_cache *hv_page_cache; +/* + * request_buffer and result_buffer are not required to be 4k aligned, + * but are not allowed to cross any 4k boundary. Aligning them to 4k is + * the simplest way to ensure that. + */ +#define H24x7_DATA_BUFFER_SIZE 4096 +DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); +DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096); + static char *event_name(struct hv_24x7_event_data *ev, int *len) { *len = be16_to_cpu(ev->event_name_len) - 2; @@ -152,6 +161,7 @@ static char *event_desc(struct hv_24x7_event_data *ev, int *len) { unsigned nl = be16_to_cpu(ev->event_name_len); __be16 *desc_len = (__be16 *)(ev->remainder + nl - 2); + *len = be16_to_cpu(*desc_len) - 2; return (char *)ev->remainder + nl; } @@ -162,6 +172,7 @@ static char *event_long_desc(struct hv_24x7_event_data *ev, int *len) __be16 *desc_len_ = (__be16 *)(ev->remainder + nl - 2); unsigned desc_len = be16_to_cpu(*desc_len_); __be16 *long_desc_len = (__be16 *)(ev->remainder + nl + desc_len - 2); + *len = be16_to_cpu(*long_desc_len) - 2; return (char *)ev->remainder + nl + desc_len; } @@ -239,14 +250,12 @@ static unsigned long h_get_24x7_catalog_page_(unsigned long phys_4096, unsigned long index) { pr_devel("h_get_24x7_catalog_page(0x%lx, %lu, %lu)", - phys_4096, - version, - index); + phys_4096, version, index); + WARN_ON(!IS_ALIGNED(phys_4096, 4096)); + return plpar_hcall_norets(H_GET_24X7_CATALOG_PAGE, - phys_4096, - version, - index); + phys_4096, version, index); } static unsigned long h_get_24x7_catalog_page(char page[], @@ -300,6 +309,7 @@ static ssize_t device_show_string(struct device *dev, struct dev_ext_attribute *d; d = container_of(attr, struct dev_ext_attribute, attr); + return sprintf(buf, "%s\n", (char *)d->var); } @@ -314,6 +324,7 @@ static struct attribute *device_str_attr_create_(char *name, char *str) attr->attr.attr.name = name; attr->attr.attr.mode = 0444; attr->attr.show = device_show_string; + return &attr->attr.attr; } @@ -387,7 +398,6 @@ static struct attribute *event_to_attr(unsigned ix, a_ev_name = kasprintf(GFP_KERNEL, "%.*s%s__%d", (int)event_name_len, ev_name, ev_suffix, nonce); - if (!a_ev_name) goto out_val; @@ -637,7 +647,7 @@ static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, #define MAX_4K (SIZE_MAX / 4096) -static void create_events_from_catalog(struct attribute ***events_, +static int create_events_from_catalog(struct attribute ***events_, struct attribute ***event_descs_, struct attribute ***event_long_descs_) { @@ -655,19 +665,25 @@ static void create_events_from_catalog(struct attribute ***events_, void *event_data, *end; struct hv_24x7_event_data *event; struct rb_root ev_uniq = RB_ROOT; + int ret = 0; - if (!page) + if (!page) { + ret = -ENOMEM; goto e_out; + } hret = h_get_24x7_catalog_page(page, 0, 0); - if (hret) + if (hret) { + ret = -EIO; goto e_free; + } catalog_version_num = be64_to_cpu(page_0->version); catalog_page_len = be32_to_cpu(page_0->length); if (MAX_4K < catalog_page_len) { pr_err("invalid page count: %zu\n", catalog_page_len); + ret = -EIO; goto e_free; } @@ -686,6 +702,7 @@ static void create_events_from_catalog(struct attribute ***events_, || (MAX_4K - event_data_offs < event_data_len)) { pr_err("invalid event data offs %zu and/or len %zu\n", event_data_offs, event_data_len); + ret = -EIO; goto e_free; } @@ -694,12 +711,14 @@ static void create_events_from_catalog(struct attribute ***events_, event_data_offs, event_data_offs + event_data_len, catalog_page_len); + ret = -EIO; goto e_free; } if (SIZE_MAX / MAX_EVENTS_PER_EVENT_DATA - 1 < event_entry_count) { pr_err("event_entry_count %zu is invalid\n", event_entry_count); + ret = -EIO; goto e_free; } @@ -712,6 +731,7 @@ static void create_events_from_catalog(struct attribute ***events_, event_data = vmalloc(event_data_bytes); if (!event_data) { pr_err("could not allocate event data\n"); + ret = -ENOMEM; goto e_free; } @@ -731,6 +751,7 @@ static void create_events_from_catalog(struct attribute ***events_, if (hret) { pr_err("failed to get event data in page %zu\n", i + event_data_offs); + ret = -EIO; goto e_event_data; } } @@ -778,18 +799,24 @@ static void create_events_from_catalog(struct attribute ***events_, event_idx_last, event_entry_count, junk_events); events = kmalloc_array(attr_max + 1, sizeof(*events), GFP_KERNEL); - if (!events) + if (!events) { + ret = -ENOMEM; goto e_event_data; + } event_descs = kmalloc_array(event_idx + 1, sizeof(*event_descs), GFP_KERNEL); - if (!event_descs) + if (!event_descs) { + ret = -ENOMEM; goto e_event_attrs; + } event_long_descs = kmalloc_array(event_idx + 1, sizeof(*event_long_descs), GFP_KERNEL); - if (!event_long_descs) + if (!event_long_descs) { + ret = -ENOMEM; goto e_event_descs; + } /* Iterate over the catalog filling in the attribute vector */ for (junk_events = 0, event_attr_ct = 0, desc_ct = 0, long_desc_ct = 0, @@ -843,7 +870,7 @@ static void create_events_from_catalog(struct attribute ***events_, *events_ = events; *event_descs_ = event_descs; *event_long_descs_ = event_long_descs; - return; + return 0; e_event_descs: kfree(event_descs); @@ -857,6 +884,7 @@ e_out: *events_ = NULL; *event_descs_ = NULL; *event_long_descs_ = NULL; + return ret; } static ssize_t catalog_read(struct file *filp, struct kobject *kobj, @@ -872,6 +900,7 @@ static ssize_t catalog_read(struct file *filp, struct kobject *kobj, uint64_t catalog_version_num = 0; void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); struct hv_24x7_catalog_page_0 *page_0 = page; + if (!page) return -ENOMEM; @@ -976,31 +1005,104 @@ static const struct attribute_group *attr_groups[] = { NULL, }; -DEFINE_PER_CPU(char, hv_24x7_reqb[4096]) __aligned(4096); -DEFINE_PER_CPU(char, hv_24x7_resb[4096]) __aligned(4096); +static void log_24x7_hcall(struct hv_24x7_request_buffer *request_buffer, + struct hv_24x7_data_result_buffer *result_buffer, + unsigned long ret) +{ + struct hv_24x7_request *req; + + req = &request_buffer->requests[0]; + pr_notice_ratelimited("hcall failed: [%d %#x %#x %d] => " + "ret 0x%lx (%ld) detail=0x%x failing ix=%x\n", + req->performance_domain, req->data_offset, + req->starting_ix, req->starting_lpar_ix, ret, ret, + result_buffer->detailed_rc, + result_buffer->failing_request_ix); +} + +/* + * Start the process for a new H_GET_24x7_DATA hcall. + */ +static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer, + struct hv_24x7_data_result_buffer *result_buffer) +{ + + memset(request_buffer, 0, 4096); + memset(result_buffer, 0, 4096); + + request_buffer->interface_version = HV_24X7_IF_VERSION_CURRENT; + /* memset above set request_buffer->num_requests to 0 */ +} -static unsigned long single_24x7_request(u8 domain, u32 offset, u16 ix, - u16 lpar, u64 *res, - bool success_expected) +/* + * Commit (i.e perform) the H_GET_24x7_DATA hcall using the data collected + * by 'init_24x7_request()' and 'add_event_to_24x7_request()'. + */ +static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer, + struct hv_24x7_data_result_buffer *result_buffer) { unsigned long ret; /* - * request_buffer and result_buffer are not required to be 4k aligned, - * but are not allowed to cross any 4k boundary. Aligning them to 4k is - * the simplest way to ensure that. + * NOTE: Due to variable number of array elements in request and + * result buffer(s), sizeof() is not reliable. Use the actual + * allocated buffer size, H24x7_DATA_BUFFER_SIZE. */ - struct reqb { - struct hv_24x7_request_buffer buf; - struct hv_24x7_request req; - } __packed *request_buffer; - - struct { - struct hv_24x7_data_result_buffer buf; - struct hv_24x7_result res; - struct hv_24x7_result_element elem; - __be64 result; - } __packed *result_buffer; + ret = plpar_hcall_norets(H_GET_24X7_DATA, + virt_to_phys(request_buffer), H24x7_DATA_BUFFER_SIZE, + virt_to_phys(result_buffer), H24x7_DATA_BUFFER_SIZE); + + if (ret) + log_24x7_hcall(request_buffer, result_buffer, ret); + + return ret; +} + +/* + * Add the given @event to the next slot in the 24x7 request_buffer. + * + * Note that H_GET_24X7_DATA hcall allows reading several counters' + * values in a single HCALL. We expect the caller to add events to the + * request buffer one by one, make the HCALL and process the results. + */ +static int add_event_to_24x7_request(struct perf_event *event, + struct hv_24x7_request_buffer *request_buffer) +{ + u16 idx; + int i; + struct hv_24x7_request *req; + + if (request_buffer->num_requests > 254) { + pr_devel("Too many requests for 24x7 HCALL %d\n", + request_buffer->num_requests); + return -EINVAL; + } + + if (is_physical_domain(event_get_domain(event))) + idx = event_get_core(event); + else + idx = event_get_vcpu(event); + + i = request_buffer->num_requests++; + req = &request_buffer->requests[i]; + + req->performance_domain = event_get_domain(event); + req->data_size = cpu_to_be16(8); + req->data_offset = cpu_to_be32(event_get_offset(event)); + req->starting_lpar_ix = cpu_to_be16(event_get_lpar(event)), + req->max_num_lpars = cpu_to_be16(1); + req->starting_ix = cpu_to_be16(idx); + req->max_ix = cpu_to_be16(1); + + return 0; +} + +static unsigned long single_24x7_request(struct perf_event *event, u64 *count) +{ + unsigned long ret; + struct hv_24x7_request_buffer *request_buffer; + struct hv_24x7_data_result_buffer *result_buffer; + struct hv_24x7_result *resb; BUILD_BUG_ON(sizeof(*request_buffer) > 4096); BUILD_BUG_ON(sizeof(*result_buffer) > 4096); @@ -1008,63 +1110,28 @@ static unsigned long single_24x7_request(u8 domain, u32 offset, u16 ix, request_buffer = (void *)get_cpu_var(hv_24x7_reqb); result_buffer = (void *)get_cpu_var(hv_24x7_resb); - memset(request_buffer, 0, 4096); - memset(result_buffer, 0, 4096); - - *request_buffer = (struct reqb) { - .buf = { - .interface_version = HV_24X7_IF_VERSION_CURRENT, - .num_requests = 1, - }, - .req = { - .performance_domain = domain, - .data_size = cpu_to_be16(8), - .data_offset = cpu_to_be32(offset), - .starting_lpar_ix = cpu_to_be16(lpar), - .max_num_lpars = cpu_to_be16(1), - .starting_ix = cpu_to_be16(ix), - .max_ix = cpu_to_be16(1), - } - }; + init_24x7_request(request_buffer, result_buffer); - ret = plpar_hcall_norets(H_GET_24X7_DATA, - virt_to_phys(request_buffer), sizeof(*request_buffer), - virt_to_phys(result_buffer), sizeof(*result_buffer)); + ret = add_event_to_24x7_request(event, request_buffer); + if (ret) + goto out; + ret = make_24x7_request(request_buffer, result_buffer); if (ret) { - if (success_expected) - pr_err_ratelimited("hcall failed: %d %#x %#x %d => " - "0x%lx (%ld) detail=0x%x failing ix=%x\n", - domain, offset, ix, lpar, ret, ret, - result_buffer->buf.detailed_rc, - result_buffer->buf.failing_request_ix); + log_24x7_hcall(request_buffer, result_buffer, ret); goto out; } - *res = be64_to_cpu(result_buffer->result); + /* process result from hcall */ + resb = &result_buffer->results[0]; + *count = be64_to_cpu(resb->elements[0].element_data[0]); out: + put_cpu_var(hv_24x7_reqb); + put_cpu_var(hv_24x7_resb); return ret; } -static unsigned long event_24x7_request(struct perf_event *event, u64 *res, - bool success_expected) -{ - u16 idx; - unsigned domain = event_get_domain(event); - - if (is_physical_domain(domain)) - idx = event_get_core(event); - else - idx = event_get_vcpu(event); - - return single_24x7_request(event_get_domain(event), - event_get_offset(event), - idx, - event_get_lpar(event), - res, - success_expected); -} static int h_24x7_event_init(struct perf_event *event) { @@ -1133,7 +1200,7 @@ static int h_24x7_event_init(struct perf_event *event) } /* see if the event complains */ - if (event_24x7_request(event, &ct, false)) { + if (single_24x7_request(event, &ct)) { pr_devel("test hcall failed\n"); return -EIO; } @@ -1145,7 +1212,7 @@ static u64 h_24x7_get_value(struct perf_event *event) { unsigned long ret; u64 ct; - ret = event_24x7_request(event, &ct, true); + ret = single_24x7_request(event, &ct); if (ret) /* We checked this in event init, shouldn't fail here... */ return 0; @@ -1153,15 +1220,22 @@ static u64 h_24x7_get_value(struct perf_event *event) return ct; } -static void h_24x7_event_update(struct perf_event *event) +static void update_event_count(struct perf_event *event, u64 now) { s64 prev; - u64 now; - now = h_24x7_get_value(event); + prev = local64_xchg(&event->hw.prev_count, now); local64_add(now - prev, &event->count); } +static void h_24x7_event_read(struct perf_event *event) +{ + u64 now; + + now = h_24x7_get_value(event); + update_event_count(event, now); +} + static void h_24x7_event_start(struct perf_event *event, int flags) { if (flags & PERF_EF_RELOAD) @@ -1170,7 +1244,7 @@ static void h_24x7_event_start(struct perf_event *event, int flags) static void h_24x7_event_stop(struct perf_event *event, int flags) { - h_24x7_event_update(event); + h_24x7_event_read(event); } static int h_24x7_event_add(struct perf_event *event, int flags) @@ -1191,7 +1265,7 @@ static struct pmu h_24x7_pmu = { .del = h_24x7_event_stop, .start = h_24x7_event_start, .stop = h_24x7_event_stop, - .read = h_24x7_event_update, + .read = h_24x7_event_read, }; static int hv_24x7_init(void) @@ -1219,10 +1293,13 @@ static int hv_24x7_init(void) /* sampling not supported */ h_24x7_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT; - create_events_from_catalog(&event_group.attrs, + r = create_events_from_catalog(&event_group.attrs, &event_desc_group.attrs, &event_long_desc_group.attrs); + if (r) + return r; + r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1); if (r) return r; diff --git a/arch/powerpc/perf/hv-24x7.h b/arch/powerpc/perf/hv-24x7.h index 69cd4e690f58..0f9fa21a29f2 100644 --- a/arch/powerpc/perf/hv-24x7.h +++ b/arch/powerpc/perf/hv-24x7.h @@ -50,7 +50,7 @@ struct hv_24x7_request_buffer { __u8 interface_version; __u8 num_requests; __u8 reserved[0xE]; - struct hv_24x7_request requests[]; + struct hv_24x7_request requests[1]; } __packed; struct hv_24x7_result_element { @@ -66,7 +66,7 @@ struct hv_24x7_result_element { __be32 lpar_cfg_instance_id; /* size = @result_element_data_size of cointaining result. */ - __u8 element_data[]; + __u64 element_data[1]; } __packed; struct hv_24x7_result { @@ -87,7 +87,7 @@ struct hv_24x7_result { /* WARNING: only valid for first result element due to variable sizes * of result elements */ /* struct hv_24x7_result_element[@num_elements_returned] */ - struct hv_24x7_result_element elements[]; + struct hv_24x7_result_element elements[1]; } __packed; struct hv_24x7_data_result_buffer { @@ -103,7 +103,7 @@ struct hv_24x7_data_result_buffer { __u8 reserved2[0x8]; /* WARNING: only valid for the first result due to variable sizes of * results */ - struct hv_24x7_result results[]; /* [@num_results] */ + struct hv_24x7_result results[1]; /* [@num_results] */ } __packed; #endif diff --git a/arch/powerpc/platforms/85xx/common.c b/arch/powerpc/platforms/85xx/common.c index 4a9ad871a168..7bfb9b184dd4 100644 --- a/arch/powerpc/platforms/85xx/common.c +++ b/arch/powerpc/platforms/85xx/common.c @@ -40,6 +40,7 @@ static const struct of_device_id mpc85xx_common_ids[] __initconst = { { .compatible = "fsl,qoriq-pcie-v2.4", }, { .compatible = "fsl,qoriq-pcie-v2.3", }, { .compatible = "fsl,qoriq-pcie-v2.2", }, + { .compatible = "fsl,fman", }, {}, }; diff --git a/arch/powerpc/platforms/85xx/corenet_generic.c b/arch/powerpc/platforms/85xx/corenet_generic.c index 1f309ccb096e..9824d2cf79bd 100644 --- a/arch/powerpc/platforms/85xx/corenet_generic.c +++ b/arch/powerpc/platforms/85xx/corenet_generic.c @@ -88,6 +88,15 @@ static const struct of_device_id of_device_ids[] = { .compatible = "simple-bus" }, { + .compatible = "mdio-mux-gpio" + }, + { + .compatible = "fsl,fpga-ngpixis" + }, + { + .compatible = "fsl,fpga-qixis" + }, + { .compatible = "fsl,srio", }, { @@ -108,6 +117,9 @@ static const struct of_device_id of_device_ids[] = { { .compatible = "fsl,qe", }, + { + .compatible = "fsl,fman", + }, /* The following two are for the Freescale hypervisor */ { .name = "hypervisor", diff --git a/arch/powerpc/platforms/85xx/smp.c b/arch/powerpc/platforms/85xx/smp.c index d7c1e69f3070..8631ac5f0e57 100644 --- a/arch/powerpc/platforms/85xx/smp.c +++ b/arch/powerpc/platforms/85xx/smp.c @@ -360,10 +360,10 @@ static void mpc85xx_smp_kexec_down(void *arg) static void map_and_flush(unsigned long paddr) { struct page *page = pfn_to_page(paddr >> PAGE_SHIFT); - unsigned long kaddr = (unsigned long)kmap(page); + unsigned long kaddr = (unsigned long)kmap_atomic(page); flush_dcache_range(kaddr, kaddr + PAGE_SIZE); - kunmap(page); + kunmap_atomic((void *)kaddr); } /** diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig index 391b3f6b54a3..b7f9c408bf24 100644 --- a/arch/powerpc/platforms/Kconfig +++ b/arch/powerpc/platforms/Kconfig @@ -72,11 +72,6 @@ config PPC_SMP_MUXED_IPI cpu. This will enable the generic code to multiplex the 4 messages on to one ipi. -config PPC_UDBG_BEAT - bool "BEAT based debug console" - depends on PPC_CELLEB - default n - config IPIC bool default n diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index 76483e3acd60..7264e91190be 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -2,6 +2,7 @@ config PPC64 bool "64-bit kernel" default n select HAVE_VIRT_CPU_ACCOUNTING + select ZLIB_DEFLATE help This option selects whether a 32-bit or a 64-bit kernel will be built. @@ -15,7 +16,7 @@ choice The most common ones are the desktop and server CPUs (601, 603, 604, 740, 750, 74xx) CPUs from Freescale and IBM, with their embedded 512x/52xx/82xx/83xx/86xx counterparts. - The other embeeded parts, namely 4xx, 8xx, e200 (55xx) and e500 + The other embedded parts, namely 4xx, 8xx, e200 (55xx) and e500 (85xx) each form a family of their own that is not compatible with the others. diff --git a/arch/powerpc/platforms/cell/Kconfig b/arch/powerpc/platforms/cell/Kconfig index 870b6dbd4d18..2f23133ab3d1 100644 --- a/arch/powerpc/platforms/cell/Kconfig +++ b/arch/powerpc/platforms/cell/Kconfig @@ -33,17 +33,6 @@ config PPC_IBM_CELL_BLADE select PPC_UDBG_16550 select UDBG_RTAS_CONSOLE -config PPC_CELLEB - bool "Toshiba's Cell Reference Set 'Celleb' Architecture" - depends on PPC64 && PPC_BOOK3S - select PPC_CELL_NATIVE - select PPC_OF_PLATFORM_PCI - select PCI - select HAS_TXX9_SERIAL - select PPC_UDBG_BEAT - select USB_OHCI_BIG_ENDIAN_MMIO - select USB_EHCI_BIG_ENDIAN_MMIO - config PPC_CELL_QPACE bool "IBM Cell - QPACE" depends on PPC64 && PPC_BOOK3S diff --git a/arch/powerpc/platforms/cell/Makefile b/arch/powerpc/platforms/cell/Makefile index 2d16884f67b9..34699bddfddd 100644 --- a/arch/powerpc/platforms/cell/Makefile +++ b/arch/powerpc/platforms/cell/Makefile @@ -29,18 +29,3 @@ obj-$(CONFIG_AXON_MSI) += axon_msi.o # qpace setup obj-$(CONFIG_PPC_CELL_QPACE) += qpace_setup.o - -# celleb stuff -ifeq ($(CONFIG_PPC_CELLEB),y) -obj-y += celleb_setup.o \ - celleb_pci.o celleb_scc_epci.o \ - celleb_scc_pciex.o \ - celleb_scc_uhc.o \ - spider-pci.o beat.o beat_htab.o \ - beat_hvCall.o beat_interrupt.o \ - beat_iommu.o - -obj-$(CONFIG_PPC_UDBG_BEAT) += beat_udbg.o -obj-$(CONFIG_SERIAL_TXX9) += celleb_scc_sio.o -obj-$(CONFIG_SPU_BASE) += beat_spu_priv1.o -endif diff --git a/arch/powerpc/platforms/cell/beat.c b/arch/powerpc/platforms/cell/beat.c deleted file mode 100644 index affcf566d460..000000000000 --- a/arch/powerpc/platforms/cell/beat.c +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Simple routines for Celleb/Beat - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <linux/export.h> -#include <linux/init.h> -#include <linux/err.h> -#include <linux/rtc.h> -#include <linux/interrupt.h> -#include <linux/irqreturn.h> -#include <linux/reboot.h> - -#include <asm/hvconsole.h> -#include <asm/time.h> -#include <asm/machdep.h> -#include <asm/firmware.h> - -#include "beat_wrapper.h" -#include "beat.h" -#include "beat_interrupt.h" - -static int beat_pm_poweroff_flag; - -void beat_restart(char *cmd) -{ - beat_shutdown_logical_partition(!beat_pm_poweroff_flag); -} - -void beat_power_off(void) -{ - beat_shutdown_logical_partition(0); -} - -u64 beat_halt_code = 0x1000000000000000UL; -EXPORT_SYMBOL(beat_halt_code); - -void beat_halt(void) -{ - beat_shutdown_logical_partition(beat_halt_code); -} - -int beat_set_rtc_time(struct rtc_time *rtc_time) -{ - u64 tim; - tim = mktime(rtc_time->tm_year+1900, - rtc_time->tm_mon+1, rtc_time->tm_mday, - rtc_time->tm_hour, rtc_time->tm_min, rtc_time->tm_sec); - if (beat_rtc_write(tim)) - return -1; - return 0; -} - -void beat_get_rtc_time(struct rtc_time *rtc_time) -{ - u64 tim; - - if (beat_rtc_read(&tim)) - tim = 0; - to_tm(tim, rtc_time); - rtc_time->tm_year -= 1900; - rtc_time->tm_mon -= 1; -} - -#define BEAT_NVRAM_SIZE 4096 - -ssize_t beat_nvram_read(char *buf, size_t count, loff_t *index) -{ - unsigned int i; - unsigned long len; - char *p = buf; - - if (*index >= BEAT_NVRAM_SIZE) - return -ENODEV; - i = *index; - if (i + count > BEAT_NVRAM_SIZE) - count = BEAT_NVRAM_SIZE - i; - - for (; count != 0; count -= len) { - len = count; - if (len > BEAT_NVRW_CNT) - len = BEAT_NVRW_CNT; - if (beat_eeprom_read(i, len, p)) - return -EIO; - - p += len; - i += len; - } - *index = i; - return p - buf; -} - -ssize_t beat_nvram_write(char *buf, size_t count, loff_t *index) -{ - unsigned int i; - unsigned long len; - char *p = buf; - - if (*index >= BEAT_NVRAM_SIZE) - return -ENODEV; - i = *index; - if (i + count > BEAT_NVRAM_SIZE) - count = BEAT_NVRAM_SIZE - i; - - for (; count != 0; count -= len) { - len = count; - if (len > BEAT_NVRW_CNT) - len = BEAT_NVRW_CNT; - if (beat_eeprom_write(i, len, p)) - return -EIO; - - p += len; - i += len; - } - *index = i; - return p - buf; -} - -ssize_t beat_nvram_get_size(void) -{ - return BEAT_NVRAM_SIZE; -} - -int beat_set_xdabr(unsigned long dabr, unsigned long dabrx) -{ - if (beat_set_dabr(dabr, dabrx)) - return -1; - return 0; -} - -int64_t beat_get_term_char(u64 vterm, u64 *len, u64 *t1, u64 *t2) -{ - u64 db[2]; - s64 ret; - - ret = beat_get_characters_from_console(vterm, len, (u8 *)db); - if (ret == 0) { - *t1 = db[0]; - *t2 = db[1]; - } - return ret; -} -EXPORT_SYMBOL(beat_get_term_char); - -int64_t beat_put_term_char(u64 vterm, u64 len, u64 t1, u64 t2) -{ - u64 db[2]; - - db[0] = t1; - db[1] = t2; - return beat_put_characters_to_console(vterm, len, (u8 *)db); -} -EXPORT_SYMBOL(beat_put_term_char); - -void beat_power_save(void) -{ - beat_pause(0); -} - -#ifdef CONFIG_KEXEC -void beat_kexec_cpu_down(int crash, int secondary) -{ - beatic_deinit_IRQ(); -} -#endif - -static irqreturn_t beat_power_event(int virq, void *arg) -{ - printk(KERN_DEBUG "Beat: power button pressed\n"); - beat_pm_poweroff_flag = 1; - ctrl_alt_del(); - return IRQ_HANDLED; -} - -static irqreturn_t beat_reset_event(int virq, void *arg) -{ - printk(KERN_DEBUG "Beat: reset button pressed\n"); - beat_pm_poweroff_flag = 0; - ctrl_alt_del(); - return IRQ_HANDLED; -} - -static struct beat_event_list { - const char *typecode; - irq_handler_t handler; - unsigned int virq; -} beat_event_list[] = { - { "power", beat_power_event, 0 }, - { "reset", beat_reset_event, 0 }, -}; - -static int __init beat_register_event(void) -{ - u64 path[4], data[2]; - int rc, i; - unsigned int virq; - - for (i = 0; i < ARRAY_SIZE(beat_event_list); i++) { - struct beat_event_list *ev = &beat_event_list[i]; - - if (beat_construct_event_receive_port(data) != 0) { - printk(KERN_ERR "Beat: " - "cannot construct event receive port for %s\n", - ev->typecode); - return -EINVAL; - } - - virq = irq_create_mapping(NULL, data[0]); - if (virq == NO_IRQ) { - printk(KERN_ERR "Beat: failed to get virtual IRQ" - " for event receive port for %s\n", - ev->typecode); - beat_destruct_event_receive_port(data[0]); - return -EIO; - } - ev->virq = virq; - - rc = request_irq(virq, ev->handler, 0, - ev->typecode, NULL); - if (rc != 0) { - printk(KERN_ERR "Beat: failed to request virtual IRQ" - " for event receive port for %s\n", - ev->typecode); - beat_destruct_event_receive_port(data[0]); - return rc; - } - - path[0] = 0x1000000065780000ul; /* 1,ex */ - path[1] = 0x627574746f6e0000ul; /* button */ - path[2] = 0; - strncpy((char *)&path[2], ev->typecode, 8); - path[3] = 0; - data[1] = 0; - - beat_create_repository_node(path, data); - } - return 0; -} - -static int __init beat_event_init(void) -{ - if (!firmware_has_feature(FW_FEATURE_BEAT)) - return -EINVAL; - - beat_pm_poweroff_flag = 0; - return beat_register_event(); -} - -device_initcall(beat_event_init); diff --git a/arch/powerpc/platforms/cell/beat.h b/arch/powerpc/platforms/cell/beat.h deleted file mode 100644 index bfcb8e351ae5..000000000000 --- a/arch/powerpc/platforms/cell/beat.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Guest OS Interfaces. - * - * (C) Copyright 2006 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef _CELLEB_BEAT_H -#define _CELLEB_BEAT_H - -int64_t beat_get_term_char(uint64_t, uint64_t *, uint64_t *, uint64_t *); -int64_t beat_put_term_char(uint64_t, uint64_t, uint64_t, uint64_t); -int64_t beat_repository_encode(int, const char *, uint64_t[4]); -void beat_restart(char *); -void beat_power_off(void); -void beat_halt(void); -int beat_set_rtc_time(struct rtc_time *); -void beat_get_rtc_time(struct rtc_time *); -ssize_t beat_nvram_get_size(void); -ssize_t beat_nvram_read(char *, size_t, loff_t *); -ssize_t beat_nvram_write(char *, size_t, loff_t *); -int beat_set_xdabr(unsigned long, unsigned long); -void beat_power_save(void); -void beat_kexec_cpu_down(int, int); - -#endif /* _CELLEB_BEAT_H */ diff --git a/arch/powerpc/platforms/cell/beat_htab.c b/arch/powerpc/platforms/cell/beat_htab.c deleted file mode 100644 index bee9232fe619..000000000000 --- a/arch/powerpc/platforms/cell/beat_htab.c +++ /dev/null @@ -1,445 +0,0 @@ -/* - * "Cell Reference Set" HTAB support. - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This code is based on arch/powerpc/platforms/pseries/lpar.c: - * Copyright (C) 2001 Todd Inglett, IBM Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#undef DEBUG_LOW - -#include <linux/kernel.h> -#include <linux/spinlock.h> - -#include <asm/mmu.h> -#include <asm/page.h> -#include <asm/pgtable.h> -#include <asm/machdep.h> -#include <asm/udbg.h> - -#include "beat_wrapper.h" - -#ifdef DEBUG_LOW -#define DBG_LOW(fmt...) do { udbg_printf(fmt); } while (0) -#else -#define DBG_LOW(fmt...) do { } while (0) -#endif - -static DEFINE_RAW_SPINLOCK(beat_htab_lock); - -static inline unsigned int beat_read_mask(unsigned hpte_group) -{ - unsigned long rmask = 0; - u64 hpte_v[5]; - - beat_read_htab_entries(0, hpte_group + 0, hpte_v); - if (!(hpte_v[0] & HPTE_V_BOLTED)) - rmask |= 0x8000; - if (!(hpte_v[1] & HPTE_V_BOLTED)) - rmask |= 0x4000; - if (!(hpte_v[2] & HPTE_V_BOLTED)) - rmask |= 0x2000; - if (!(hpte_v[3] & HPTE_V_BOLTED)) - rmask |= 0x1000; - beat_read_htab_entries(0, hpte_group + 4, hpte_v); - if (!(hpte_v[0] & HPTE_V_BOLTED)) - rmask |= 0x0800; - if (!(hpte_v[1] & HPTE_V_BOLTED)) - rmask |= 0x0400; - if (!(hpte_v[2] & HPTE_V_BOLTED)) - rmask |= 0x0200; - if (!(hpte_v[3] & HPTE_V_BOLTED)) - rmask |= 0x0100; - hpte_group = ~hpte_group & (htab_hash_mask * HPTES_PER_GROUP); - beat_read_htab_entries(0, hpte_group + 0, hpte_v); - if (!(hpte_v[0] & HPTE_V_BOLTED)) - rmask |= 0x80; - if (!(hpte_v[1] & HPTE_V_BOLTED)) - rmask |= 0x40; - if (!(hpte_v[2] & HPTE_V_BOLTED)) - rmask |= 0x20; - if (!(hpte_v[3] & HPTE_V_BOLTED)) - rmask |= 0x10; - beat_read_htab_entries(0, hpte_group + 4, hpte_v); - if (!(hpte_v[0] & HPTE_V_BOLTED)) - rmask |= 0x08; - if (!(hpte_v[1] & HPTE_V_BOLTED)) - rmask |= 0x04; - if (!(hpte_v[2] & HPTE_V_BOLTED)) - rmask |= 0x02; - if (!(hpte_v[3] & HPTE_V_BOLTED)) - rmask |= 0x01; - return rmask; -} - -static long beat_lpar_hpte_insert(unsigned long hpte_group, - unsigned long vpn, unsigned long pa, - unsigned long rflags, unsigned long vflags, - int psize, int apsize, int ssize) -{ - unsigned long lpar_rc; - u64 hpte_v, hpte_r, slot; - - if (vflags & HPTE_V_SECONDARY) - return -1; - - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW("hpte_insert(group=%lx, va=%016lx, pa=%016lx, " - "rflags=%lx, vflags=%lx, psize=%d)\n", - hpte_group, va, pa, rflags, vflags, psize); - - hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) | - vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; - - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); - - if (rflags & _PAGE_NO_CACHE) - hpte_r &= ~HPTE_R_M; - - raw_spin_lock(&beat_htab_lock); - lpar_rc = beat_read_mask(hpte_group); - if (lpar_rc == 0) { - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW(" full\n"); - raw_spin_unlock(&beat_htab_lock); - return -1; - } - - lpar_rc = beat_insert_htab_entry(0, hpte_group, lpar_rc << 48, - hpte_v, hpte_r, &slot); - raw_spin_unlock(&beat_htab_lock); - - /* - * Since we try and ioremap PHBs we don't own, the pte insert - * will fail. However we must catch the failure in hash_page - * or we will loop forever, so return -2 in this case. - */ - if (unlikely(lpar_rc != 0)) { - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW(" lpar err %lx\n", lpar_rc); - return -2; - } - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW(" -> slot: %lx\n", slot); - - /* We have to pass down the secondary bucket bit here as well */ - return (slot ^ hpte_group) & 15; -} - -static long beat_lpar_hpte_remove(unsigned long hpte_group) -{ - DBG_LOW("hpte_remove(group=%lx)\n", hpte_group); - return -1; -} - -static unsigned long beat_lpar_hpte_getword0(unsigned long slot) -{ - unsigned long dword0; - unsigned long lpar_rc; - u64 dword[5]; - - lpar_rc = beat_read_htab_entries(0, slot & ~3UL, dword); - - dword0 = dword[slot&3]; - - BUG_ON(lpar_rc != 0); - - return dword0; -} - -static void beat_lpar_hptab_clear(void) -{ - unsigned long size_bytes = 1UL << ppc64_pft_size; - unsigned long hpte_count = size_bytes >> 4; - int i; - u64 dummy0, dummy1; - - /* TODO: Use bulk call */ - for (i = 0; i < hpte_count; i++) - beat_write_htab_entry(0, i, 0, 0, -1UL, -1UL, &dummy0, &dummy1); -} - -/* - * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and - * the low 3 bits of flags happen to line up. So no transform is needed. - * We can probably optimize here and assume the high bits of newpp are - * already zero. For now I am paranoid. - */ -static long beat_lpar_hpte_updatepp(unsigned long slot, - unsigned long newpp, - unsigned long vpn, - int psize, int apsize, - int ssize, unsigned long flags) -{ - unsigned long lpar_rc; - u64 dummy0, dummy1; - unsigned long want_v; - - want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M); - - DBG_LOW(" update: " - "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ", - want_v & HPTE_V_AVPN, slot, psize, newpp); - - raw_spin_lock(&beat_htab_lock); - dummy0 = beat_lpar_hpte_getword0(slot); - if ((dummy0 & ~0x7FUL) != (want_v & ~0x7FUL)) { - DBG_LOW("not found !\n"); - raw_spin_unlock(&beat_htab_lock); - return -1; - } - - lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, &dummy0, - &dummy1); - raw_spin_unlock(&beat_htab_lock); - if (lpar_rc != 0 || dummy0 == 0) { - DBG_LOW("not found !\n"); - return -1; - } - - DBG_LOW("ok %lx %lx\n", dummy0, dummy1); - - BUG_ON(lpar_rc != 0); - - return 0; -} - -static long beat_lpar_hpte_find(unsigned long vpn, int psize) -{ - unsigned long hash; - unsigned long i, j; - long slot; - unsigned long want_v, hpte_v; - - hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M); - want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M); - - for (j = 0; j < 2; j++) { - slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; - for (i = 0; i < HPTES_PER_GROUP; i++) { - hpte_v = beat_lpar_hpte_getword0(slot); - - if (HPTE_V_COMPARE(hpte_v, want_v) - && (hpte_v & HPTE_V_VALID) - && (!!(hpte_v & HPTE_V_SECONDARY) == j)) { - /* HPTE matches */ - if (j) - slot = -slot; - return slot; - } - ++slot; - } - hash = ~hash; - } - - return -1; -} - -static void beat_lpar_hpte_updateboltedpp(unsigned long newpp, - unsigned long ea, - int psize, int ssize) -{ - unsigned long vpn; - unsigned long lpar_rc, slot, vsid; - u64 dummy0, dummy1; - - vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M); - vpn = hpt_vpn(ea, vsid, MMU_SEGSIZE_256M); - - raw_spin_lock(&beat_htab_lock); - slot = beat_lpar_hpte_find(vpn, psize); - BUG_ON(slot == -1); - - lpar_rc = beat_write_htab_entry(0, slot, 0, newpp, 0, 7, - &dummy0, &dummy1); - raw_spin_unlock(&beat_htab_lock); - - BUG_ON(lpar_rc != 0); -} - -static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn, - int psize, int apsize, - int ssize, int local) -{ - unsigned long want_v; - unsigned long lpar_rc; - u64 dummy1, dummy2; - unsigned long flags; - - DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", - slot, va, psize, local); - want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M); - - raw_spin_lock_irqsave(&beat_htab_lock, flags); - dummy1 = beat_lpar_hpte_getword0(slot); - - if ((dummy1 & ~0x7FUL) != (want_v & ~0x7FUL)) { - DBG_LOW("not found !\n"); - raw_spin_unlock_irqrestore(&beat_htab_lock, flags); - return; - } - - lpar_rc = beat_write_htab_entry(0, slot, 0, 0, HPTE_V_VALID, 0, - &dummy1, &dummy2); - raw_spin_unlock_irqrestore(&beat_htab_lock, flags); - - BUG_ON(lpar_rc != 0); -} - -void __init hpte_init_beat(void) -{ - ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate; - ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp; - ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; - ppc_md.hpte_insert = beat_lpar_hpte_insert; - ppc_md.hpte_remove = beat_lpar_hpte_remove; - ppc_md.hpte_clear_all = beat_lpar_hptab_clear; -} - -static long beat_lpar_hpte_insert_v3(unsigned long hpte_group, - unsigned long vpn, unsigned long pa, - unsigned long rflags, unsigned long vflags, - int psize, int apsize, int ssize) -{ - unsigned long lpar_rc; - u64 hpte_v, hpte_r, slot; - - if (vflags & HPTE_V_SECONDARY) - return -1; - - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW("hpte_insert(group=%lx, vpn=%016lx, pa=%016lx, " - "rflags=%lx, vflags=%lx, psize=%d)\n", - hpte_group, vpn, pa, rflags, vflags, psize); - - hpte_v = hpte_encode_v(vpn, psize, apsize, MMU_SEGSIZE_256M) | - vflags | HPTE_V_VALID; - hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; - - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW(" hpte_v=%016lx, hpte_r=%016lx\n", hpte_v, hpte_r); - - if (rflags & _PAGE_NO_CACHE) - hpte_r &= ~HPTE_R_M; - - /* insert into not-volted entry */ - lpar_rc = beat_insert_htab_entry3(0, hpte_group, hpte_v, hpte_r, - HPTE_V_BOLTED, 0, &slot); - /* - * Since we try and ioremap PHBs we don't own, the pte insert - * will fail. However we must catch the failure in hash_page - * or we will loop forever, so return -2 in this case. - */ - if (unlikely(lpar_rc != 0)) { - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW(" lpar err %lx\n", lpar_rc); - return -2; - } - if (!(vflags & HPTE_V_BOLTED)) - DBG_LOW(" -> slot: %lx\n", slot); - - /* We have to pass down the secondary bucket bit here as well */ - return (slot ^ hpte_group) & 15; -} - -/* - * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and - * the low 3 bits of flags happen to line up. So no transform is needed. - * We can probably optimize here and assume the high bits of newpp are - * already zero. For now I am paranoid. - */ -static long beat_lpar_hpte_updatepp_v3(unsigned long slot, - unsigned long newpp, - unsigned long vpn, - int psize, int apsize, - int ssize, unsigned long flags) -{ - unsigned long lpar_rc; - unsigned long want_v; - unsigned long pss; - - want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M); - pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize]; - - DBG_LOW(" update: " - "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ", - want_v & HPTE_V_AVPN, slot, psize, newpp); - - lpar_rc = beat_update_htab_permission3(0, slot, want_v, pss, 7, newpp); - - if (lpar_rc == 0xfffffff7) { - DBG_LOW("not found !\n"); - return -1; - } - - DBG_LOW("ok\n"); - - BUG_ON(lpar_rc != 0); - - return 0; -} - -static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn, - int psize, int apsize, - int ssize, int local) -{ - unsigned long want_v; - unsigned long lpar_rc; - unsigned long pss; - - DBG_LOW(" inval : slot=%lx, vpn=%016lx, psize: %d, local: %d\n", - slot, vpn, psize, local); - want_v = hpte_encode_avpn(vpn, psize, MMU_SEGSIZE_256M); - pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc[psize]; - - lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss); - - /* E_busy can be valid output: page may be already replaced */ - BUG_ON(lpar_rc != 0 && lpar_rc != 0xfffffff7); -} - -static int64_t _beat_lpar_hptab_clear_v3(void) -{ - return beat_clear_htab3(0); -} - -static void beat_lpar_hptab_clear_v3(void) -{ - _beat_lpar_hptab_clear_v3(); -} - -void __init hpte_init_beat_v3(void) -{ - if (_beat_lpar_hptab_clear_v3() == 0) { - ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate_v3; - ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp_v3; - ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; - ppc_md.hpte_insert = beat_lpar_hpte_insert_v3; - ppc_md.hpte_remove = beat_lpar_hpte_remove; - ppc_md.hpte_clear_all = beat_lpar_hptab_clear_v3; - } else { - ppc_md.hpte_invalidate = beat_lpar_hpte_invalidate; - ppc_md.hpte_updatepp = beat_lpar_hpte_updatepp; - ppc_md.hpte_updateboltedpp = beat_lpar_hpte_updateboltedpp; - ppc_md.hpte_insert = beat_lpar_hpte_insert; - ppc_md.hpte_remove = beat_lpar_hpte_remove; - ppc_md.hpte_clear_all = beat_lpar_hptab_clear; - } -} diff --git a/arch/powerpc/platforms/cell/beat_hvCall.S b/arch/powerpc/platforms/cell/beat_hvCall.S deleted file mode 100644 index 96c801907126..000000000000 --- a/arch/powerpc/platforms/cell/beat_hvCall.S +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Beat hypervisor call I/F - * - * (C) Copyright 2007 TOSHIBA CORPORATION - * - * This code is based on arch/powerpc/platforms/pseries/hvCall.S. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <asm/ppc_asm.h> - -/* Not implemented on Beat, now */ -#define HCALL_INST_PRECALL -#define HCALL_INST_POSTCALL - - .text - -#define HVSC .long 0x44000022 - -/* Note: takes only 7 input parameters at maximum */ -_GLOBAL(beat_hcall_norets) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - mr r11,r3 - mr r3,r4 - mr r4,r5 - mr r5,r6 - mr r6,r7 - mr r7,r8 - mr r8,r9 - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ - -/* Note: takes 8 input parameters at maximum */ -_GLOBAL(beat_hcall_norets8) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - mr r11,r3 - mr r3,r4 - mr r4,r5 - mr r5,r6 - mr r6,r7 - mr r7,r8 - mr r8,r9 - ld r10,STK_PARAM(R10)(r1) - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ - -/* Note: takes only 6 input parameters, 1 output parameters at maximum */ -_GLOBAL(beat_hcall1) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - std r4,STK_PARAM(R4)(r1) /* save ret buffer */ - - mr r11,r3 - mr r3,r5 - mr r4,r6 - mr r5,r7 - mr r6,r8 - mr r7,r9 - mr r8,r10 - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - ld r12,STK_PARAM(R4)(r1) - std r4, 0(r12) - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ - -/* Note: takes only 6 input parameters, 2 output parameters at maximum */ -_GLOBAL(beat_hcall2) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - std r4,STK_PARAM(R4)(r1) /* save ret buffer */ - - mr r11,r3 - mr r3,r5 - mr r4,r6 - mr r5,r7 - mr r6,r8 - mr r7,r9 - mr r8,r10 - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - ld r12,STK_PARAM(R4)(r1) - std r4, 0(r12) - std r5, 8(r12) - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ - -/* Note: takes only 6 input parameters, 3 output parameters at maximum */ -_GLOBAL(beat_hcall3) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - std r4,STK_PARAM(R4)(r1) /* save ret buffer */ - - mr r11,r3 - mr r3,r5 - mr r4,r6 - mr r5,r7 - mr r6,r8 - mr r7,r9 - mr r8,r10 - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - ld r12,STK_PARAM(R4)(r1) - std r4, 0(r12) - std r5, 8(r12) - std r6, 16(r12) - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ - -/* Note: takes only 6 input parameters, 4 output parameters at maximum */ -_GLOBAL(beat_hcall4) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - std r4,STK_PARAM(R4)(r1) /* save ret buffer */ - - mr r11,r3 - mr r3,r5 - mr r4,r6 - mr r5,r7 - mr r6,r8 - mr r7,r9 - mr r8,r10 - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - ld r12,STK_PARAM(R4)(r1) - std r4, 0(r12) - std r5, 8(r12) - std r6, 16(r12) - std r7, 24(r12) - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ - -/* Note: takes only 6 input parameters, 5 output parameters at maximum */ -_GLOBAL(beat_hcall5) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - std r4,STK_PARAM(R4)(r1) /* save ret buffer */ - - mr r11,r3 - mr r3,r5 - mr r4,r6 - mr r5,r7 - mr r6,r8 - mr r7,r9 - mr r8,r10 - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - ld r12,STK_PARAM(R4)(r1) - std r4, 0(r12) - std r5, 8(r12) - std r6, 16(r12) - std r7, 24(r12) - std r8, 32(r12) - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ - -/* Note: takes only 6 input parameters, 6 output parameters at maximum */ -_GLOBAL(beat_hcall6) - HMT_MEDIUM - - mfcr r0 - stw r0,8(r1) - - HCALL_INST_PRECALL - - std r4,STK_PARAM(R4)(r1) /* save ret buffer */ - - mr r11,r3 - mr r3,r5 - mr r4,r6 - mr r5,r7 - mr r6,r8 - mr r7,r9 - mr r8,r10 - - HVSC /* invoke the hypervisor */ - - HCALL_INST_POSTCALL - - ld r12,STK_PARAM(R4)(r1) - std r4, 0(r12) - std r5, 8(r12) - std r6, 16(r12) - std r7, 24(r12) - std r8, 32(r12) - std r9, 40(r12) - - lwz r0,8(r1) - mtcrf 0xff,r0 - - blr /* return r3 = status */ diff --git a/arch/powerpc/platforms/cell/beat_interrupt.c b/arch/powerpc/platforms/cell/beat_interrupt.c deleted file mode 100644 index 9e5dfbcc00af..000000000000 --- a/arch/powerpc/platforms/cell/beat_interrupt.c +++ /dev/null @@ -1,253 +0,0 @@ -/* - * Celleb/Beat Interrupt controller - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/percpu.h> -#include <linux/types.h> - -#include <asm/machdep.h> - -#include "beat_interrupt.h" -#include "beat_wrapper.h" - -#define MAX_IRQS NR_IRQS -static DEFINE_RAW_SPINLOCK(beatic_irq_mask_lock); -static uint64_t beatic_irq_mask_enable[(MAX_IRQS+255)/64]; -static uint64_t beatic_irq_mask_ack[(MAX_IRQS+255)/64]; - -static struct irq_domain *beatic_host; - -/* - * In this implementation, "virq" == "IRQ plug number", - * "(irq_hw_number_t)hwirq" == "IRQ outlet number". - */ - -/* assumption: locked */ -static inline void beatic_update_irq_mask(unsigned int irq_plug) -{ - int off; - unsigned long masks[4]; - - off = (irq_plug / 256) * 4; - masks[0] = beatic_irq_mask_enable[off + 0] - & beatic_irq_mask_ack[off + 0]; - masks[1] = beatic_irq_mask_enable[off + 1] - & beatic_irq_mask_ack[off + 1]; - masks[2] = beatic_irq_mask_enable[off + 2] - & beatic_irq_mask_ack[off + 2]; - masks[3] = beatic_irq_mask_enable[off + 3] - & beatic_irq_mask_ack[off + 3]; - if (beat_set_interrupt_mask(irq_plug&~255UL, - masks[0], masks[1], masks[2], masks[3]) != 0) - panic("Failed to set mask IRQ!"); -} - -static void beatic_mask_irq(struct irq_data *d) -{ - unsigned long flags; - - raw_spin_lock_irqsave(&beatic_irq_mask_lock, flags); - beatic_irq_mask_enable[d->irq/64] &= ~(1UL << (63 - (d->irq%64))); - beatic_update_irq_mask(d->irq); - raw_spin_unlock_irqrestore(&beatic_irq_mask_lock, flags); -} - -static void beatic_unmask_irq(struct irq_data *d) -{ - unsigned long flags; - - raw_spin_lock_irqsave(&beatic_irq_mask_lock, flags); - beatic_irq_mask_enable[d->irq/64] |= 1UL << (63 - (d->irq%64)); - beatic_update_irq_mask(d->irq); - raw_spin_unlock_irqrestore(&beatic_irq_mask_lock, flags); -} - -static void beatic_ack_irq(struct irq_data *d) -{ - unsigned long flags; - - raw_spin_lock_irqsave(&beatic_irq_mask_lock, flags); - beatic_irq_mask_ack[d->irq/64] &= ~(1UL << (63 - (d->irq%64))); - beatic_update_irq_mask(d->irq); - raw_spin_unlock_irqrestore(&beatic_irq_mask_lock, flags); -} - -static void beatic_end_irq(struct irq_data *d) -{ - s64 err; - unsigned long flags; - - err = beat_downcount_of_interrupt(d->irq); - if (err != 0) { - if ((err & 0xFFFFFFFF) != 0xFFFFFFF5) /* -11: wrong state */ - panic("Failed to downcount IRQ! Error = %16llx", err); - - printk(KERN_ERR "IRQ over-downcounted, plug %d\n", d->irq); - } - raw_spin_lock_irqsave(&beatic_irq_mask_lock, flags); - beatic_irq_mask_ack[d->irq/64] |= 1UL << (63 - (d->irq%64)); - beatic_update_irq_mask(d->irq); - raw_spin_unlock_irqrestore(&beatic_irq_mask_lock, flags); -} - -static struct irq_chip beatic_pic = { - .name = "CELL-BEAT", - .irq_unmask = beatic_unmask_irq, - .irq_mask = beatic_mask_irq, - .irq_eoi = beatic_end_irq, -}; - -/* - * Dispose binding hardware IRQ number (hw) and Virtuql IRQ number (virq), - * update flags. - * - * Note that the number (virq) is already assigned at upper layer. - */ -static void beatic_pic_host_unmap(struct irq_domain *h, unsigned int virq) -{ - beat_destruct_irq_plug(virq); -} - -/* - * Create or update binding hardware IRQ number (hw) and Virtuql - * IRQ number (virq). This is called only once for a given mapping. - * - * Note that the number (virq) is already assigned at upper layer. - */ -static int beatic_pic_host_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - int64_t err; - - err = beat_construct_and_connect_irq_plug(virq, hw); - if (err < 0) - return -EIO; - - irq_set_status_flags(virq, IRQ_LEVEL); - irq_set_chip_and_handler(virq, &beatic_pic, handle_fasteoi_irq); - return 0; -} - -/* - * Translate device-tree interrupt spec to irq_hw_number_t style (ulong), - * to pass away to irq_create_mapping(). - * - * Called from irq_create_of_mapping() only. - * Note: We have only 1 entry to translate. - */ -static int beatic_pic_host_xlate(struct irq_domain *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, - unsigned int *out_flags) -{ - const u64 *intspec2 = (const u64 *)intspec; - - *out_hwirq = *intspec2; - *out_flags |= IRQ_TYPE_LEVEL_LOW; - return 0; -} - -static int beatic_pic_host_match(struct irq_domain *h, struct device_node *np) -{ - /* Match all */ - return 1; -} - -static const struct irq_domain_ops beatic_pic_host_ops = { - .map = beatic_pic_host_map, - .unmap = beatic_pic_host_unmap, - .xlate = beatic_pic_host_xlate, - .match = beatic_pic_host_match, -}; - -/* - * Get an IRQ number - * Note: returns VIRQ - */ -static inline unsigned int beatic_get_irq_plug(void) -{ - int i; - uint64_t pending[4], ub; - - for (i = 0; i < MAX_IRQS; i += 256) { - beat_detect_pending_interrupts(i, pending); - __asm__ ("cntlzd %0,%1":"=r"(ub): - "r"(pending[0] & beatic_irq_mask_enable[i/64+0] - & beatic_irq_mask_ack[i/64+0])); - if (ub != 64) - return i + ub + 0; - __asm__ ("cntlzd %0,%1":"=r"(ub): - "r"(pending[1] & beatic_irq_mask_enable[i/64+1] - & beatic_irq_mask_ack[i/64+1])); - if (ub != 64) - return i + ub + 64; - __asm__ ("cntlzd %0,%1":"=r"(ub): - "r"(pending[2] & beatic_irq_mask_enable[i/64+2] - & beatic_irq_mask_ack[i/64+2])); - if (ub != 64) - return i + ub + 128; - __asm__ ("cntlzd %0,%1":"=r"(ub): - "r"(pending[3] & beatic_irq_mask_enable[i/64+3] - & beatic_irq_mask_ack[i/64+3])); - if (ub != 64) - return i + ub + 192; - } - - return NO_IRQ; -} -unsigned int beatic_get_irq(void) -{ - unsigned int ret; - - ret = beatic_get_irq_plug(); - if (ret != NO_IRQ) - beatic_ack_irq(irq_get_irq_data(ret)); - return ret; -} - -/* - */ -void __init beatic_init_IRQ(void) -{ - int i; - - memset(beatic_irq_mask_enable, 0, sizeof(beatic_irq_mask_enable)); - memset(beatic_irq_mask_ack, 255, sizeof(beatic_irq_mask_ack)); - for (i = 0; i < MAX_IRQS; i += 256) - beat_set_interrupt_mask(i, 0L, 0L, 0L, 0L); - - /* Set out get_irq function */ - ppc_md.get_irq = beatic_get_irq; - - /* Allocate an irq host */ - beatic_host = irq_domain_add_nomap(NULL, ~0, &beatic_pic_host_ops, NULL); - BUG_ON(beatic_host == NULL); - irq_set_default_host(beatic_host); -} - -void beatic_deinit_IRQ(void) -{ - int i; - - for (i = 1; i < nr_irqs; i++) - beat_destruct_irq_plug(i); -} diff --git a/arch/powerpc/platforms/cell/beat_interrupt.h b/arch/powerpc/platforms/cell/beat_interrupt.h deleted file mode 100644 index a7e52f91a078..000000000000 --- a/arch/powerpc/platforms/cell/beat_interrupt.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Celleb/Beat Interrupt controller - * - * (C) Copyright 2006 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef ASM_BEAT_PIC_H -#define ASM_BEAT_PIC_H -#ifdef __KERNEL__ - -extern void beatic_init_IRQ(void); -extern unsigned int beatic_get_irq(void); -extern void beatic_deinit_IRQ(void); - -#endif -#endif /* ASM_BEAT_PIC_H */ diff --git a/arch/powerpc/platforms/cell/beat_iommu.c b/arch/powerpc/platforms/cell/beat_iommu.c deleted file mode 100644 index 3ce685568935..000000000000 --- a/arch/powerpc/platforms/cell/beat_iommu.c +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Support for IOMMU on Celleb platform. - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <linux/kernel.h> -#include <linux/init.h> -#include <linux/dma-mapping.h> -#include <linux/pci.h> -#include <linux/of_platform.h> - -#include <asm/machdep.h> - -#include "beat_wrapper.h" - -#define DMA_FLAGS 0xf800000000000000UL /* r/w permitted, coherency required, - strongest order */ - -static int __init find_dma_window(u64 *io_space_id, u64 *ioid, - u64 *base, u64 *size, u64 *io_page_size) -{ - struct device_node *dn; - const unsigned long *dma_window; - - for_each_node_by_type(dn, "ioif") { - dma_window = of_get_property(dn, "toshiba,dma-window", NULL); - if (dma_window) { - *io_space_id = (dma_window[0] >> 32) & 0xffffffffUL; - *ioid = dma_window[0] & 0x7ffUL; - *base = dma_window[1]; - *size = dma_window[2]; - *io_page_size = 1 << dma_window[3]; - of_node_put(dn); - return 1; - } - } - return 0; -} - -static unsigned long celleb_dma_direct_offset; - -static void __init celleb_init_direct_mapping(void) -{ - u64 lpar_addr, io_addr; - u64 io_space_id, ioid, dma_base, dma_size, io_page_size; - - if (!find_dma_window(&io_space_id, &ioid, &dma_base, &dma_size, - &io_page_size)) { - pr_info("No dma window found !\n"); - return; - } - - for (lpar_addr = 0; lpar_addr < dma_size; lpar_addr += io_page_size) { - io_addr = lpar_addr + dma_base; - (void)beat_put_iopte(io_space_id, io_addr, lpar_addr, - ioid, DMA_FLAGS); - } - - celleb_dma_direct_offset = dma_base; -} - -static void celleb_dma_dev_setup(struct device *dev) -{ - set_dma_ops(dev, &dma_direct_ops); - set_dma_offset(dev, celleb_dma_direct_offset); -} - -static void celleb_pci_dma_dev_setup(struct pci_dev *pdev) -{ - celleb_dma_dev_setup(&pdev->dev); -} - -static int celleb_of_bus_notify(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct device *dev = data; - - /* We are only intereted in device addition */ - if (action != BUS_NOTIFY_ADD_DEVICE) - return 0; - - celleb_dma_dev_setup(dev); - - return 0; -} - -static struct notifier_block celleb_of_bus_notifier = { - .notifier_call = celleb_of_bus_notify -}; - -static int __init celleb_init_iommu(void) -{ - celleb_init_direct_mapping(); - ppc_md.pci_dma_dev_setup = celleb_pci_dma_dev_setup; - bus_register_notifier(&platform_bus_type, &celleb_of_bus_notifier); - - return 0; -} - -machine_arch_initcall(celleb_beat, celleb_init_iommu); diff --git a/arch/powerpc/platforms/cell/beat_spu_priv1.c b/arch/powerpc/platforms/cell/beat_spu_priv1.c deleted file mode 100644 index 13f52589d3a9..000000000000 --- a/arch/powerpc/platforms/cell/beat_spu_priv1.c +++ /dev/null @@ -1,205 +0,0 @@ -/* - * spu hypervisor abstraction for Beat - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <asm/types.h> -#include <asm/spu.h> -#include <asm/spu_priv1.h> - -#include "beat_wrapper.h" - -static inline void _int_mask_set(struct spu *spu, int class, u64 mask) -{ - spu->shadow_int_mask_RW[class] = mask; - beat_set_irq_mask_for_spe(spu->spe_id, class, mask); -} - -static inline u64 _int_mask_get(struct spu *spu, int class) -{ - return spu->shadow_int_mask_RW[class]; -} - -static void int_mask_set(struct spu *spu, int class, u64 mask) -{ - _int_mask_set(spu, class, mask); -} - -static u64 int_mask_get(struct spu *spu, int class) -{ - return _int_mask_get(spu, class); -} - -static void int_mask_and(struct spu *spu, int class, u64 mask) -{ - u64 old_mask; - old_mask = _int_mask_get(spu, class); - _int_mask_set(spu, class, old_mask & mask); -} - -static void int_mask_or(struct spu *spu, int class, u64 mask) -{ - u64 old_mask; - old_mask = _int_mask_get(spu, class); - _int_mask_set(spu, class, old_mask | mask); -} - -static void int_stat_clear(struct spu *spu, int class, u64 stat) -{ - beat_clear_interrupt_status_of_spe(spu->spe_id, class, stat); -} - -static u64 int_stat_get(struct spu *spu, int class) -{ - u64 int_stat; - beat_get_interrupt_status_of_spe(spu->spe_id, class, &int_stat); - return int_stat; -} - -static void cpu_affinity_set(struct spu *spu, int cpu) -{ - return; -} - -static u64 mfc_dar_get(struct spu *spu) -{ - u64 dar; - beat_get_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, mfc_dar_RW), &dar); - return dar; -} - -static u64 mfc_dsisr_get(struct spu *spu) -{ - u64 dsisr; - beat_get_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, mfc_dsisr_RW), &dsisr); - return dsisr; -} - -static void mfc_dsisr_set(struct spu *spu, u64 dsisr) -{ - beat_set_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, mfc_dsisr_RW), dsisr); -} - -static void mfc_sdr_setup(struct spu *spu) -{ - return; -} - -static void mfc_sr1_set(struct spu *spu, u64 sr1) -{ - beat_set_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, mfc_sr1_RW), sr1); -} - -static u64 mfc_sr1_get(struct spu *spu) -{ - u64 sr1; - beat_get_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, mfc_sr1_RW), &sr1); - return sr1; -} - -static void mfc_tclass_id_set(struct spu *spu, u64 tclass_id) -{ - beat_set_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, mfc_tclass_id_RW), tclass_id); -} - -static u64 mfc_tclass_id_get(struct spu *spu) -{ - u64 tclass_id; - beat_get_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, mfc_tclass_id_RW), &tclass_id); - return tclass_id; -} - -static void tlb_invalidate(struct spu *spu) -{ - beat_set_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, tlb_invalidate_entry_W), 0ul); -} - -static void resource_allocation_groupID_set(struct spu *spu, u64 id) -{ - beat_set_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, resource_allocation_groupID_RW), - id); -} - -static u64 resource_allocation_groupID_get(struct spu *spu) -{ - u64 id; - beat_get_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, resource_allocation_groupID_RW), - &id); - return id; -} - -static void resource_allocation_enable_set(struct spu *spu, u64 enable) -{ - beat_set_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, resource_allocation_enable_RW), - enable); -} - -static u64 resource_allocation_enable_get(struct spu *spu) -{ - u64 enable; - beat_get_spe_privileged_state_1_registers( - spu->spe_id, - offsetof(struct spu_priv1, resource_allocation_enable_RW), - &enable); - return enable; -} - -const struct spu_priv1_ops spu_priv1_beat_ops = { - .int_mask_and = int_mask_and, - .int_mask_or = int_mask_or, - .int_mask_set = int_mask_set, - .int_mask_get = int_mask_get, - .int_stat_clear = int_stat_clear, - .int_stat_get = int_stat_get, - .cpu_affinity_set = cpu_affinity_set, - .mfc_dar_get = mfc_dar_get, - .mfc_dsisr_get = mfc_dsisr_get, - .mfc_dsisr_set = mfc_dsisr_set, - .mfc_sdr_setup = mfc_sdr_setup, - .mfc_sr1_set = mfc_sr1_set, - .mfc_sr1_get = mfc_sr1_get, - .mfc_tclass_id_set = mfc_tclass_id_set, - .mfc_tclass_id_get = mfc_tclass_id_get, - .tlb_invalidate = tlb_invalidate, - .resource_allocation_groupID_set = resource_allocation_groupID_set, - .resource_allocation_groupID_get = resource_allocation_groupID_get, - .resource_allocation_enable_set = resource_allocation_enable_set, - .resource_allocation_enable_get = resource_allocation_enable_get, -}; diff --git a/arch/powerpc/platforms/cell/beat_syscall.h b/arch/powerpc/platforms/cell/beat_syscall.h deleted file mode 100644 index 8580dc7e1798..000000000000 --- a/arch/powerpc/platforms/cell/beat_syscall.h +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Beat hypervisor call numbers - * - * (C) Copyright 2004-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef BEAT_BEAT_syscall_H -#define BEAT_BEAT_syscall_H - -#ifdef __ASSEMBLY__ -#define __BEAT_ADD_VENDOR_ID(__x, __v) ((__v)<<60|(__x)) -#else -#define __BEAT_ADD_VENDOR_ID(__x, __v) ((u64)(__v)<<60|(__x)) -#endif -#define HV_allocate_memory __BEAT_ADD_VENDOR_ID(0, 0) -#define HV_construct_virtual_address_space __BEAT_ADD_VENDOR_ID(2, 0) -#define HV_destruct_virtual_address_space __BEAT_ADD_VENDOR_ID(10, 0) -#define HV_get_virtual_address_space_id_of_ppe __BEAT_ADD_VENDOR_ID(4, 0) -#define HV_query_logical_partition_address_region_info \ - __BEAT_ADD_VENDOR_ID(6, 0) -#define HV_release_memory __BEAT_ADD_VENDOR_ID(13, 0) -#define HV_select_virtual_address_space __BEAT_ADD_VENDOR_ID(7, 0) -#define HV_load_range_registers __BEAT_ADD_VENDOR_ID(68, 0) -#define HV_set_ppe_l2cache_rmt_entry __BEAT_ADD_VENDOR_ID(70, 0) -#define HV_set_ppe_tlb_rmt_entry __BEAT_ADD_VENDOR_ID(71, 0) -#define HV_set_spe_tlb_rmt_entry __BEAT_ADD_VENDOR_ID(72, 0) -#define HV_get_io_address_translation_fault_info __BEAT_ADD_VENDOR_ID(14, 0) -#define HV_get_iopte __BEAT_ADD_VENDOR_ID(16, 0) -#define HV_preload_iopt_cache __BEAT_ADD_VENDOR_ID(17, 0) -#define HV_put_iopte __BEAT_ADD_VENDOR_ID(15, 0) -#define HV_connect_event_ports __BEAT_ADD_VENDOR_ID(21, 0) -#define HV_construct_event_receive_port __BEAT_ADD_VENDOR_ID(18, 0) -#define HV_destruct_event_receive_port __BEAT_ADD_VENDOR_ID(19, 0) -#define HV_destruct_event_send_port __BEAT_ADD_VENDOR_ID(22, 0) -#define HV_get_state_of_event_send_port __BEAT_ADD_VENDOR_ID(25, 0) -#define HV_request_to_connect_event_ports __BEAT_ADD_VENDOR_ID(20, 0) -#define HV_send_event_externally __BEAT_ADD_VENDOR_ID(23, 0) -#define HV_send_event_locally __BEAT_ADD_VENDOR_ID(24, 0) -#define HV_construct_and_connect_irq_plug __BEAT_ADD_VENDOR_ID(28, 0) -#define HV_destruct_irq_plug __BEAT_ADD_VENDOR_ID(29, 0) -#define HV_detect_pending_interrupts __BEAT_ADD_VENDOR_ID(26, 0) -#define HV_end_of_interrupt __BEAT_ADD_VENDOR_ID(27, 0) -#define HV_assign_control_signal_notification_port __BEAT_ADD_VENDOR_ID(45, 0) -#define HV_end_of_control_signal_processing __BEAT_ADD_VENDOR_ID(48, 0) -#define HV_get_control_signal __BEAT_ADD_VENDOR_ID(46, 0) -#define HV_set_irq_mask_for_spe __BEAT_ADD_VENDOR_ID(61, 0) -#define HV_shutdown_logical_partition __BEAT_ADD_VENDOR_ID(44, 0) -#define HV_connect_message_ports __BEAT_ADD_VENDOR_ID(35, 0) -#define HV_destruct_message_port __BEAT_ADD_VENDOR_ID(36, 0) -#define HV_receive_message __BEAT_ADD_VENDOR_ID(37, 0) -#define HV_get_message_port_info __BEAT_ADD_VENDOR_ID(34, 0) -#define HV_request_to_connect_message_ports __BEAT_ADD_VENDOR_ID(33, 0) -#define HV_send_message __BEAT_ADD_VENDOR_ID(32, 0) -#define HV_get_logical_ppe_id __BEAT_ADD_VENDOR_ID(69, 0) -#define HV_pause __BEAT_ADD_VENDOR_ID(9, 0) -#define HV_destruct_shared_memory_handle __BEAT_ADD_VENDOR_ID(51, 0) -#define HV_get_shared_memory_info __BEAT_ADD_VENDOR_ID(52, 0) -#define HV_permit_sharing_memory __BEAT_ADD_VENDOR_ID(50, 0) -#define HV_request_to_attach_shared_memory __BEAT_ADD_VENDOR_ID(49, 0) -#define HV_enable_logical_spe_execution __BEAT_ADD_VENDOR_ID(55, 0) -#define HV_construct_logical_spe __BEAT_ADD_VENDOR_ID(53, 0) -#define HV_disable_logical_spe_execution __BEAT_ADD_VENDOR_ID(56, 0) -#define HV_destruct_logical_spe __BEAT_ADD_VENDOR_ID(54, 0) -#define HV_sense_spe_execution_status __BEAT_ADD_VENDOR_ID(58, 0) -#define HV_insert_htab_entry __BEAT_ADD_VENDOR_ID(101, 0) -#define HV_read_htab_entries __BEAT_ADD_VENDOR_ID(95, 0) -#define HV_write_htab_entry __BEAT_ADD_VENDOR_ID(94, 0) -#define HV_assign_io_address_translation_fault_port \ - __BEAT_ADD_VENDOR_ID(100, 0) -#define HV_set_interrupt_mask __BEAT_ADD_VENDOR_ID(73, 0) -#define HV_get_logical_partition_id __BEAT_ADD_VENDOR_ID(74, 0) -#define HV_create_repository_node2 __BEAT_ADD_VENDOR_ID(90, 0) -#define HV_create_repository_node __BEAT_ADD_VENDOR_ID(90, 0) /* alias */ -#define HV_get_repository_node_value2 __BEAT_ADD_VENDOR_ID(91, 0) -#define HV_get_repository_node_value __BEAT_ADD_VENDOR_ID(91, 0) /* alias */ -#define HV_modify_repository_node_value2 __BEAT_ADD_VENDOR_ID(92, 0) -#define HV_modify_repository_node_value __BEAT_ADD_VENDOR_ID(92, 0) /* alias */ -#define HV_remove_repository_node2 __BEAT_ADD_VENDOR_ID(93, 0) -#define HV_remove_repository_node __BEAT_ADD_VENDOR_ID(93, 0) /* alias */ -#define HV_cancel_shared_memory __BEAT_ADD_VENDOR_ID(104, 0) -#define HV_clear_interrupt_status_of_spe __BEAT_ADD_VENDOR_ID(206, 0) -#define HV_construct_spe_irq_outlet __BEAT_ADD_VENDOR_ID(80, 0) -#define HV_destruct_spe_irq_outlet __BEAT_ADD_VENDOR_ID(81, 0) -#define HV_disconnect_ipspc_service __BEAT_ADD_VENDOR_ID(88, 0) -#define HV_execute_ipspc_command __BEAT_ADD_VENDOR_ID(86, 0) -#define HV_get_interrupt_status_of_spe __BEAT_ADD_VENDOR_ID(205, 0) -#define HV_get_spe_privileged_state_1_registers __BEAT_ADD_VENDOR_ID(208, 0) -#define HV_permit_use_of_ipspc_service __BEAT_ADD_VENDOR_ID(85, 0) -#define HV_reinitialize_logical_spe __BEAT_ADD_VENDOR_ID(82, 0) -#define HV_request_ipspc_service __BEAT_ADD_VENDOR_ID(84, 0) -#define HV_stop_ipspc_command __BEAT_ADD_VENDOR_ID(87, 0) -#define HV_set_spe_privileged_state_1_registers __BEAT_ADD_VENDOR_ID(204, 0) -#define HV_get_status_of_ipspc_service __BEAT_ADD_VENDOR_ID(203, 0) -#define HV_put_characters_to_console __BEAT_ADD_VENDOR_ID(0x101, 1) -#define HV_get_characters_from_console __BEAT_ADD_VENDOR_ID(0x102, 1) -#define HV_get_base_clock __BEAT_ADD_VENDOR_ID(0x111, 1) -#define HV_set_base_clock __BEAT_ADD_VENDOR_ID(0x112, 1) -#define HV_get_frame_cycle __BEAT_ADD_VENDOR_ID(0x114, 1) -#define HV_disable_console __BEAT_ADD_VENDOR_ID(0x115, 1) -#define HV_disable_all_console __BEAT_ADD_VENDOR_ID(0x116, 1) -#define HV_oneshot_timer __BEAT_ADD_VENDOR_ID(0x117, 1) -#define HV_set_dabr __BEAT_ADD_VENDOR_ID(0x118, 1) -#define HV_get_dabr __BEAT_ADD_VENDOR_ID(0x119, 1) -#define HV_start_hv_stats __BEAT_ADD_VENDOR_ID(0x21c, 1) -#define HV_stop_hv_stats __BEAT_ADD_VENDOR_ID(0x21d, 1) -#define HV_get_hv_stats __BEAT_ADD_VENDOR_ID(0x21e, 1) -#define HV_get_hv_error_stats __BEAT_ADD_VENDOR_ID(0x221, 1) -#define HV_get_stats __BEAT_ADD_VENDOR_ID(0x224, 1) -#define HV_get_heap_stats __BEAT_ADD_VENDOR_ID(0x225, 1) -#define HV_get_memory_stats __BEAT_ADD_VENDOR_ID(0x227, 1) -#define HV_get_memory_detail __BEAT_ADD_VENDOR_ID(0x228, 1) -#define HV_set_priority_of_irq_outlet __BEAT_ADD_VENDOR_ID(0x122, 1) -#define HV_get_physical_spe_by_reservation_id __BEAT_ADD_VENDOR_ID(0x128, 1) -#define HV_get_spe_context __BEAT_ADD_VENDOR_ID(0x129, 1) -#define HV_set_spe_context __BEAT_ADD_VENDOR_ID(0x12a, 1) -#define HV_downcount_of_interrupt __BEAT_ADD_VENDOR_ID(0x12e, 1) -#define HV_peek_spe_context __BEAT_ADD_VENDOR_ID(0x12f, 1) -#define HV_read_bpa_register __BEAT_ADD_VENDOR_ID(0x131, 1) -#define HV_write_bpa_register __BEAT_ADD_VENDOR_ID(0x132, 1) -#define HV_map_context_table_of_spe __BEAT_ADD_VENDOR_ID(0x137, 1) -#define HV_get_slb_for_logical_spe __BEAT_ADD_VENDOR_ID(0x138, 1) -#define HV_set_slb_for_logical_spe __BEAT_ADD_VENDOR_ID(0x139, 1) -#define HV_init_pm __BEAT_ADD_VENDOR_ID(0x150, 1) -#define HV_set_pm_signal __BEAT_ADD_VENDOR_ID(0x151, 1) -#define HV_get_pm_signal __BEAT_ADD_VENDOR_ID(0x152, 1) -#define HV_set_pm_config __BEAT_ADD_VENDOR_ID(0x153, 1) -#define HV_get_pm_config __BEAT_ADD_VENDOR_ID(0x154, 1) -#define HV_get_inner_trace_data __BEAT_ADD_VENDOR_ID(0x155, 1) -#define HV_set_ext_trace_buffer __BEAT_ADD_VENDOR_ID(0x156, 1) -#define HV_get_ext_trace_buffer __BEAT_ADD_VENDOR_ID(0x157, 1) -#define HV_set_pm_interrupt __BEAT_ADD_VENDOR_ID(0x158, 1) -#define HV_get_pm_interrupt __BEAT_ADD_VENDOR_ID(0x159, 1) -#define HV_kick_pm __BEAT_ADD_VENDOR_ID(0x160, 1) -#define HV_construct_pm_context __BEAT_ADD_VENDOR_ID(0x164, 1) -#define HV_destruct_pm_context __BEAT_ADD_VENDOR_ID(0x165, 1) -#define HV_be_slow __BEAT_ADD_VENDOR_ID(0x170, 1) -#define HV_assign_ipspc_server_connection_status_notification_port \ - __BEAT_ADD_VENDOR_ID(0x173, 1) -#define HV_get_raid_of_physical_spe __BEAT_ADD_VENDOR_ID(0x174, 1) -#define HV_set_physical_spe_to_rag __BEAT_ADD_VENDOR_ID(0x175, 1) -#define HV_release_physical_spe_from_rag __BEAT_ADD_VENDOR_ID(0x176, 1) -#define HV_rtc_read __BEAT_ADD_VENDOR_ID(0x190, 1) -#define HV_rtc_write __BEAT_ADD_VENDOR_ID(0x191, 1) -#define HV_eeprom_read __BEAT_ADD_VENDOR_ID(0x192, 1) -#define HV_eeprom_write __BEAT_ADD_VENDOR_ID(0x193, 1) -#define HV_insert_htab_entry3 __BEAT_ADD_VENDOR_ID(0x104, 1) -#define HV_invalidate_htab_entry3 __BEAT_ADD_VENDOR_ID(0x105, 1) -#define HV_update_htab_permission3 __BEAT_ADD_VENDOR_ID(0x106, 1) -#define HV_clear_htab3 __BEAT_ADD_VENDOR_ID(0x107, 1) -#endif diff --git a/arch/powerpc/platforms/cell/beat_udbg.c b/arch/powerpc/platforms/cell/beat_udbg.c deleted file mode 100644 index 350735bc8888..000000000000 --- a/arch/powerpc/platforms/cell/beat_udbg.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * udbg function for Beat - * - * (C) Copyright 2006 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <linux/kernel.h> -#include <linux/console.h> - -#include <asm/machdep.h> -#include <asm/prom.h> -#include <asm/udbg.h> - -#include "beat.h" - -#define celleb_vtermno 0 - -static void udbg_putc_beat(char c) -{ - unsigned long rc; - - if (c == '\n') - udbg_putc_beat('\r'); - - rc = beat_put_term_char(celleb_vtermno, 1, (uint64_t)c << 56, 0); -} - -/* Buffered chars getc */ -static u64 inbuflen; -static u64 inbuf[2]; /* must be 2 u64s */ - -static int udbg_getc_poll_beat(void) -{ - /* The interface is tricky because it may return up to 16 chars. - * We save them statically for future calls to udbg_getc(). - */ - char ch, *buf = (char *)inbuf; - int i; - long rc; - if (inbuflen == 0) { - /* get some more chars. */ - inbuflen = 0; - rc = beat_get_term_char(celleb_vtermno, &inbuflen, - inbuf+0, inbuf+1); - if (rc != 0) - inbuflen = 0; /* otherwise inbuflen is garbage */ - } - if (inbuflen <= 0 || inbuflen > 16) { - /* Catch error case as well as other oddities (corruption) */ - inbuflen = 0; - return -1; - } - ch = buf[0]; - for (i = 1; i < inbuflen; i++) /* shuffle them down. */ - buf[i-1] = buf[i]; - inbuflen--; - return ch; -} - -static int udbg_getc_beat(void) -{ - int ch; - for (;;) { - ch = udbg_getc_poll_beat(); - if (ch == -1) { - /* This shouldn't be needed...but... */ - volatile unsigned long delay; - for (delay = 0; delay < 2000000; delay++) - ; - } else { - return ch; - } - } -} - -/* call this from early_init() for a working debug console on - * vterm capable LPAR machines - */ -void __init udbg_init_debug_beat(void) -{ - udbg_putc = udbg_putc_beat; - udbg_getc = udbg_getc_beat; - udbg_getc_poll = udbg_getc_poll_beat; -} diff --git a/arch/powerpc/platforms/cell/beat_wrapper.h b/arch/powerpc/platforms/cell/beat_wrapper.h deleted file mode 100644 index c1109969f242..000000000000 --- a/arch/powerpc/platforms/cell/beat_wrapper.h +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Beat hypervisor call I/F - * - * (C) Copyright 2007 TOSHIBA CORPORATION - * - * This code is based on arch/powerpc/platforms/pseries/plpar_wrapper.h. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ -#ifndef BEAT_HCALL -#include <linux/string.h> -#include "beat_syscall.h" - -/* defined in hvCall.S */ -extern s64 beat_hcall_norets(u64 opcode, ...); -extern s64 beat_hcall_norets8(u64 opcode, u64 arg1, u64 arg2, u64 arg3, - u64 arg4, u64 arg5, u64 arg6, u64 arg7, u64 arg8); -extern s64 beat_hcall1(u64 opcode, u64 retbuf[1], ...); -extern s64 beat_hcall2(u64 opcode, u64 retbuf[2], ...); -extern s64 beat_hcall3(u64 opcode, u64 retbuf[3], ...); -extern s64 beat_hcall4(u64 opcode, u64 retbuf[4], ...); -extern s64 beat_hcall5(u64 opcode, u64 retbuf[5], ...); -extern s64 beat_hcall6(u64 opcode, u64 retbuf[6], ...); - -static inline s64 beat_downcount_of_interrupt(u64 plug_id) -{ - return beat_hcall_norets(HV_downcount_of_interrupt, plug_id); -} - -static inline s64 beat_set_interrupt_mask(u64 index, - u64 val0, u64 val1, u64 val2, u64 val3) -{ - return beat_hcall_norets(HV_set_interrupt_mask, index, - val0, val1, val2, val3); -} - -static inline s64 beat_destruct_irq_plug(u64 plug_id) -{ - return beat_hcall_norets(HV_destruct_irq_plug, plug_id); -} - -static inline s64 beat_construct_and_connect_irq_plug(u64 plug_id, - u64 outlet_id) -{ - return beat_hcall_norets(HV_construct_and_connect_irq_plug, plug_id, - outlet_id); -} - -static inline s64 beat_detect_pending_interrupts(u64 index, u64 *retbuf) -{ - return beat_hcall4(HV_detect_pending_interrupts, retbuf, index); -} - -static inline s64 beat_pause(u64 style) -{ - return beat_hcall_norets(HV_pause, style); -} - -static inline s64 beat_read_htab_entries(u64 htab_id, u64 index, u64 *retbuf) -{ - return beat_hcall5(HV_read_htab_entries, retbuf, htab_id, index); -} - -static inline s64 beat_insert_htab_entry(u64 htab_id, u64 group, - u64 bitmask, u64 hpte_v, u64 hpte_r, u64 *slot) -{ - u64 dummy[3]; - s64 ret; - - ret = beat_hcall3(HV_insert_htab_entry, dummy, htab_id, group, - bitmask, hpte_v, hpte_r); - *slot = dummy[0]; - return ret; -} - -static inline s64 beat_write_htab_entry(u64 htab_id, u64 slot, - u64 hpte_v, u64 hpte_r, u64 mask_v, u64 mask_r, - u64 *ret_v, u64 *ret_r) -{ - u64 dummy[2]; - s64 ret; - - ret = beat_hcall2(HV_write_htab_entry, dummy, htab_id, slot, - hpte_v, hpte_r, mask_v, mask_r); - *ret_v = dummy[0]; - *ret_r = dummy[1]; - return ret; -} - -static inline s64 beat_insert_htab_entry3(u64 htab_id, u64 group, - u64 hpte_v, u64 hpte_r, u64 mask_v, u64 value_v, u64 *slot) -{ - u64 dummy[1]; - s64 ret; - - ret = beat_hcall1(HV_insert_htab_entry3, dummy, htab_id, group, - hpte_v, hpte_r, mask_v, value_v); - *slot = dummy[0]; - return ret; -} - -static inline s64 beat_invalidate_htab_entry3(u64 htab_id, u64 group, - u64 va, u64 pss) -{ - return beat_hcall_norets(HV_invalidate_htab_entry3, - htab_id, group, va, pss); -} - -static inline s64 beat_update_htab_permission3(u64 htab_id, u64 group, - u64 va, u64 pss, u64 ptel_mask, u64 ptel_value) -{ - return beat_hcall_norets(HV_update_htab_permission3, - htab_id, group, va, pss, ptel_mask, ptel_value); -} - -static inline s64 beat_clear_htab3(u64 htab_id) -{ - return beat_hcall_norets(HV_clear_htab3, htab_id); -} - -static inline void beat_shutdown_logical_partition(u64 code) -{ - (void)beat_hcall_norets(HV_shutdown_logical_partition, code); -} - -static inline s64 beat_rtc_write(u64 time_from_epoch) -{ - return beat_hcall_norets(HV_rtc_write, time_from_epoch); -} - -static inline s64 beat_rtc_read(u64 *time_from_epoch) -{ - u64 dummy[1]; - s64 ret; - - ret = beat_hcall1(HV_rtc_read, dummy); - *time_from_epoch = dummy[0]; - return ret; -} - -#define BEAT_NVRW_CNT (sizeof(u64) * 6) - -static inline s64 beat_eeprom_write(u64 index, u64 length, u8 *buffer) -{ - u64 b[6]; - - if (length > BEAT_NVRW_CNT) - return -1; - memcpy(b, buffer, sizeof(b)); - return beat_hcall_norets8(HV_eeprom_write, index, length, - b[0], b[1], b[2], b[3], b[4], b[5]); -} - -static inline s64 beat_eeprom_read(u64 index, u64 length, u8 *buffer) -{ - u64 b[6]; - s64 ret; - - if (length > BEAT_NVRW_CNT) - return -1; - ret = beat_hcall6(HV_eeprom_read, b, index, length); - memcpy(buffer, b, length); - return ret; -} - -static inline s64 beat_set_dabr(u64 value, u64 style) -{ - return beat_hcall_norets(HV_set_dabr, value, style); -} - -static inline s64 beat_get_characters_from_console(u64 termno, u64 *len, - u8 *buffer) -{ - u64 dummy[3]; - s64 ret; - - ret = beat_hcall3(HV_get_characters_from_console, dummy, termno, len); - *len = dummy[0]; - memcpy(buffer, dummy + 1, *len); - return ret; -} - -static inline s64 beat_put_characters_to_console(u64 termno, u64 len, - u8 *buffer) -{ - u64 b[2]; - - memcpy(b, buffer, len); - return beat_hcall_norets(HV_put_characters_to_console, termno, len, - b[0], b[1]); -} - -static inline s64 beat_get_spe_privileged_state_1_registers( - u64 id, u64 offsetof, u64 *value) -{ - u64 dummy[1]; - s64 ret; - - ret = beat_hcall1(HV_get_spe_privileged_state_1_registers, dummy, id, - offsetof); - *value = dummy[0]; - return ret; -} - -static inline s64 beat_set_irq_mask_for_spe(u64 id, u64 class, u64 mask) -{ - return beat_hcall_norets(HV_set_irq_mask_for_spe, id, class, mask); -} - -static inline s64 beat_clear_interrupt_status_of_spe(u64 id, u64 class, - u64 mask) -{ - return beat_hcall_norets(HV_clear_interrupt_status_of_spe, - id, class, mask); -} - -static inline s64 beat_set_spe_privileged_state_1_registers( - u64 id, u64 offsetof, u64 value) -{ - return beat_hcall_norets(HV_set_spe_privileged_state_1_registers, - id, offsetof, value); -} - -static inline s64 beat_get_interrupt_status_of_spe(u64 id, u64 class, u64 *val) -{ - u64 dummy[1]; - s64 ret; - - ret = beat_hcall1(HV_get_interrupt_status_of_spe, dummy, id, class); - *val = dummy[0]; - return ret; -} - -static inline s64 beat_put_iopte(u64 ioas_id, u64 io_addr, u64 real_addr, - u64 ioid, u64 flags) -{ - return beat_hcall_norets(HV_put_iopte, ioas_id, io_addr, real_addr, - ioid, flags); -} - -static inline s64 beat_construct_event_receive_port(u64 *port) -{ - u64 dummy[1]; - s64 ret; - - ret = beat_hcall1(HV_construct_event_receive_port, dummy); - *port = dummy[0]; - return ret; -} - -static inline s64 beat_destruct_event_receive_port(u64 port) -{ - s64 ret; - - ret = beat_hcall_norets(HV_destruct_event_receive_port, port); - return ret; -} - -static inline s64 beat_create_repository_node(u64 path[4], u64 data[2]) -{ - s64 ret; - - ret = beat_hcall_norets(HV_create_repository_node2, - path[0], path[1], path[2], path[3], data[0], data[1]); - return ret; -} - -static inline s64 beat_get_repository_node_value(u64 lpid, u64 path[4], - u64 data[2]) -{ - s64 ret; - - ret = beat_hcall2(HV_get_repository_node_value2, data, - lpid, path[0], path[1], path[2], path[3]); - return ret; -} - -#endif diff --git a/arch/powerpc/platforms/cell/cell.h b/arch/powerpc/platforms/cell/cell.h new file mode 100644 index 000000000000..ef143dfee068 --- /dev/null +++ b/arch/powerpc/platforms/cell/cell.h @@ -0,0 +1,24 @@ +/* + * Cell Platform common data structures + * + * Copyright 2015, Daniel Axtens, IBM Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef CELL_H +#define CELL_H + +#include <asm/pci-bridge.h> + +extern struct pci_controller_ops cell_pci_controller_ops; + +#endif diff --git a/arch/powerpc/platforms/cell/celleb_pci.c b/arch/powerpc/platforms/cell/celleb_pci.c deleted file mode 100644 index 3ce70ded2d6a..000000000000 --- a/arch/powerpc/platforms/cell/celleb_pci.c +++ /dev/null @@ -1,500 +0,0 @@ -/* - * Support for PCI on Celleb platform. - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This code is based on arch/powerpc/kernel/rtas_pci.c: - * Copyright (C) 2001 Dave Engebretsen, IBM Corporation - * Copyright (C) 2003 Anton Blanchard <anton@au.ibm.com>, IBM - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/threads.h> -#include <linux/pci.h> -#include <linux/string.h> -#include <linux/init.h> -#include <linux/memblock.h> -#include <linux/pci_regs.h> -#include <linux/of.h> -#include <linux/of_device.h> -#include <linux/slab.h> - -#include <asm/io.h> -#include <asm/irq.h> -#include <asm/prom.h> -#include <asm/pci-bridge.h> -#include <asm/ppc-pci.h> - -#include "celleb_pci.h" - -#define MAX_PCI_DEVICES 32 -#define MAX_PCI_FUNCTIONS 8 -#define MAX_PCI_BASE_ADDRS 3 /* use 64 bit address */ - -/* definition for fake pci configuration area for GbE, .... ,and etc. */ - -struct celleb_pci_resource { - struct resource r[MAX_PCI_BASE_ADDRS]; -}; - -struct celleb_pci_private { - unsigned char *fake_config[MAX_PCI_DEVICES][MAX_PCI_FUNCTIONS]; - struct celleb_pci_resource *res[MAX_PCI_DEVICES][MAX_PCI_FUNCTIONS]; -}; - -static inline u8 celleb_fake_config_readb(void *addr) -{ - u8 *p = addr; - return *p; -} - -static inline u16 celleb_fake_config_readw(void *addr) -{ - __le16 *p = addr; - return le16_to_cpu(*p); -} - -static inline u32 celleb_fake_config_readl(void *addr) -{ - __le32 *p = addr; - return le32_to_cpu(*p); -} - -static inline void celleb_fake_config_writeb(u32 val, void *addr) -{ - u8 *p = addr; - *p = val; -} - -static inline void celleb_fake_config_writew(u32 val, void *addr) -{ - __le16 val16; - __le16 *p = addr; - val16 = cpu_to_le16(val); - *p = val16; -} - -static inline void celleb_fake_config_writel(u32 val, void *addr) -{ - __le32 val32; - __le32 *p = addr; - val32 = cpu_to_le32(val); - *p = val32; -} - -static unsigned char *get_fake_config_start(struct pci_controller *hose, - int devno, int fn) -{ - struct celleb_pci_private *private = hose->private_data; - - if (private == NULL) - return NULL; - - return private->fake_config[devno][fn]; -} - -static struct celleb_pci_resource *get_resource_start( - struct pci_controller *hose, - int devno, int fn) -{ - struct celleb_pci_private *private = hose->private_data; - - if (private == NULL) - return NULL; - - return private->res[devno][fn]; -} - - -static void celleb_config_read_fake(unsigned char *config, int where, - int size, u32 *val) -{ - char *p = config + where; - - switch (size) { - case 1: - *val = celleb_fake_config_readb(p); - break; - case 2: - *val = celleb_fake_config_readw(p); - break; - case 4: - *val = celleb_fake_config_readl(p); - break; - } -} - -static void celleb_config_write_fake(unsigned char *config, int where, - int size, u32 val) -{ - char *p = config + where; - - switch (size) { - case 1: - celleb_fake_config_writeb(val, p); - break; - case 2: - celleb_fake_config_writew(val, p); - break; - case 4: - celleb_fake_config_writel(val, p); - break; - } -} - -static int celleb_fake_pci_read_config(struct pci_bus *bus, - unsigned int devfn, int where, int size, u32 *val) -{ - char *config; - struct pci_controller *hose = pci_bus_to_host(bus); - unsigned int devno = devfn >> 3; - unsigned int fn = devfn & 0x7; - - /* allignment check */ - BUG_ON(where % size); - - pr_debug(" fake read: bus=0x%x, ", bus->number); - config = get_fake_config_start(hose, devno, fn); - - pr_debug("devno=0x%x, where=0x%x, size=0x%x, ", devno, where, size); - if (!config) { - pr_debug("failed\n"); - return PCIBIOS_DEVICE_NOT_FOUND; - } - - celleb_config_read_fake(config, where, size, val); - pr_debug("val=0x%x\n", *val); - - return PCIBIOS_SUCCESSFUL; -} - - -static int celleb_fake_pci_write_config(struct pci_bus *bus, - unsigned int devfn, int where, int size, u32 val) -{ - char *config; - struct pci_controller *hose = pci_bus_to_host(bus); - struct celleb_pci_resource *res; - unsigned int devno = devfn >> 3; - unsigned int fn = devfn & 0x7; - - /* allignment check */ - BUG_ON(where % size); - - config = get_fake_config_start(hose, devno, fn); - - if (!config) - return PCIBIOS_DEVICE_NOT_FOUND; - - if (val == ~0) { - int i = (where - PCI_BASE_ADDRESS_0) >> 3; - - switch (where) { - case PCI_BASE_ADDRESS_0: - case PCI_BASE_ADDRESS_2: - if (size != 4) - return PCIBIOS_DEVICE_NOT_FOUND; - res = get_resource_start(hose, devno, fn); - if (!res) - return PCIBIOS_DEVICE_NOT_FOUND; - celleb_config_write_fake(config, where, size, - (res->r[i].end - res->r[i].start)); - return PCIBIOS_SUCCESSFUL; - case PCI_BASE_ADDRESS_1: - case PCI_BASE_ADDRESS_3: - case PCI_BASE_ADDRESS_4: - case PCI_BASE_ADDRESS_5: - break; - default: - break; - } - } - - celleb_config_write_fake(config, where, size, val); - pr_debug(" fake write: where=%x, size=%d, val=%x\n", - where, size, val); - - return PCIBIOS_SUCCESSFUL; -} - -static struct pci_ops celleb_fake_pci_ops = { - .read = celleb_fake_pci_read_config, - .write = celleb_fake_pci_write_config, -}; - -static inline void celleb_setup_pci_base_addrs(struct pci_controller *hose, - unsigned int devno, unsigned int fn, - unsigned int num_base_addr) -{ - u32 val; - unsigned char *config; - struct celleb_pci_resource *res; - - config = get_fake_config_start(hose, devno, fn); - res = get_resource_start(hose, devno, fn); - - if (!config || !res) - return; - - switch (num_base_addr) { - case 3: - val = (res->r[2].start & 0xfffffff0) - | PCI_BASE_ADDRESS_MEM_TYPE_64; - celleb_config_write_fake(config, PCI_BASE_ADDRESS_4, 4, val); - val = res->r[2].start >> 32; - celleb_config_write_fake(config, PCI_BASE_ADDRESS_5, 4, val); - /* FALLTHROUGH */ - case 2: - val = (res->r[1].start & 0xfffffff0) - | PCI_BASE_ADDRESS_MEM_TYPE_64; - celleb_config_write_fake(config, PCI_BASE_ADDRESS_2, 4, val); - val = res->r[1].start >> 32; - celleb_config_write_fake(config, PCI_BASE_ADDRESS_3, 4, val); - /* FALLTHROUGH */ - case 1: - val = (res->r[0].start & 0xfffffff0) - | PCI_BASE_ADDRESS_MEM_TYPE_64; - celleb_config_write_fake(config, PCI_BASE_ADDRESS_0, 4, val); - val = res->r[0].start >> 32; - celleb_config_write_fake(config, PCI_BASE_ADDRESS_1, 4, val); - break; - } - - val = PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; - celleb_config_write_fake(config, PCI_COMMAND, 2, val); -} - -static int __init celleb_setup_fake_pci_device(struct device_node *node, - struct pci_controller *hose) -{ - unsigned int rlen; - int num_base_addr = 0; - u32 val; - const u32 *wi0, *wi1, *wi2, *wi3, *wi4; - unsigned int devno, fn; - struct celleb_pci_private *private = hose->private_data; - unsigned char **config = NULL; - struct celleb_pci_resource **res = NULL; - const char *name; - const unsigned long *li; - int size, result; - - if (private == NULL) { - printk(KERN_ERR "PCI: " - "memory space for pci controller is not assigned\n"); - goto error; - } - - name = of_get_property(node, "model", &rlen); - if (!name) { - printk(KERN_ERR "PCI: model property not found.\n"); - goto error; - } - - wi4 = of_get_property(node, "reg", &rlen); - if (wi4 == NULL) - goto error; - - devno = ((wi4[0] >> 8) & 0xff) >> 3; - fn = (wi4[0] >> 8) & 0x7; - - pr_debug("PCI: celleb_setup_fake_pci() %s devno=%x fn=%x\n", name, - devno, fn); - - size = 256; - config = &private->fake_config[devno][fn]; - *config = zalloc_maybe_bootmem(size, GFP_KERNEL); - if (*config == NULL) { - printk(KERN_ERR "PCI: " - "not enough memory for fake configuration space\n"); - goto error; - } - pr_debug("PCI: fake config area assigned 0x%016lx\n", - (unsigned long)*config); - - size = sizeof(struct celleb_pci_resource); - res = &private->res[devno][fn]; - *res = zalloc_maybe_bootmem(size, GFP_KERNEL); - if (*res == NULL) { - printk(KERN_ERR - "PCI: not enough memory for resource data space\n"); - goto error; - } - pr_debug("PCI: res assigned 0x%016lx\n", (unsigned long)*res); - - wi0 = of_get_property(node, "device-id", NULL); - wi1 = of_get_property(node, "vendor-id", NULL); - wi2 = of_get_property(node, "class-code", NULL); - wi3 = of_get_property(node, "revision-id", NULL); - if (!wi0 || !wi1 || !wi2 || !wi3) { - printk(KERN_ERR "PCI: Missing device tree properties.\n"); - goto error; - } - - celleb_config_write_fake(*config, PCI_DEVICE_ID, 2, wi0[0] & 0xffff); - celleb_config_write_fake(*config, PCI_VENDOR_ID, 2, wi1[0] & 0xffff); - pr_debug("class-code = 0x%08x\n", wi2[0]); - - celleb_config_write_fake(*config, PCI_CLASS_PROG, 1, wi2[0] & 0xff); - celleb_config_write_fake(*config, PCI_CLASS_DEVICE, 2, - (wi2[0] >> 8) & 0xffff); - celleb_config_write_fake(*config, PCI_REVISION_ID, 1, wi3[0]); - - while (num_base_addr < MAX_PCI_BASE_ADDRS) { - result = of_address_to_resource(node, - num_base_addr, &(*res)->r[num_base_addr]); - if (result) - break; - num_base_addr++; - } - - celleb_setup_pci_base_addrs(hose, devno, fn, num_base_addr); - - li = of_get_property(node, "interrupts", &rlen); - if (!li) { - printk(KERN_ERR "PCI: interrupts not found.\n"); - goto error; - } - val = li[0]; - celleb_config_write_fake(*config, PCI_INTERRUPT_PIN, 1, 1); - celleb_config_write_fake(*config, PCI_INTERRUPT_LINE, 1, val); - -#ifdef DEBUG - pr_debug("PCI: %s irq=%ld\n", name, li[0]); - for (i = 0; i < 6; i++) { - celleb_config_read_fake(*config, - PCI_BASE_ADDRESS_0 + 0x4 * i, 4, - &val); - pr_debug("PCI: %s fn=%d base_address_%d=0x%x\n", - name, fn, i, val); - } -#endif - - celleb_config_write_fake(*config, PCI_HEADER_TYPE, 1, - PCI_HEADER_TYPE_NORMAL); - - return 0; - -error: - if (mem_init_done) { - if (config && *config) - kfree(*config); - if (res && *res) - kfree(*res); - - } else { - if (config && *config) { - size = 256; - memblock_free(__pa(*config), size); - } - if (res && *res) { - size = sizeof(struct celleb_pci_resource); - memblock_free(__pa(*res), size); - } - } - - return 1; -} - -static int __init phb_set_bus_ranges(struct device_node *dev, - struct pci_controller *phb) -{ - const int *bus_range; - unsigned int len; - - bus_range = of_get_property(dev, "bus-range", &len); - if (bus_range == NULL || len < 2 * sizeof(int)) - return 1; - - phb->first_busno = bus_range[0]; - phb->last_busno = bus_range[1]; - - return 0; -} - -static void __init celleb_alloc_private_mem(struct pci_controller *hose) -{ - hose->private_data = - zalloc_maybe_bootmem(sizeof(struct celleb_pci_private), - GFP_KERNEL); -} - -static int __init celleb_setup_fake_pci(struct device_node *dev, - struct pci_controller *phb) -{ - struct device_node *node; - - phb->ops = &celleb_fake_pci_ops; - celleb_alloc_private_mem(phb); - - for (node = of_get_next_child(dev, NULL); - node != NULL; node = of_get_next_child(dev, node)) - celleb_setup_fake_pci_device(node, phb); - - return 0; -} - -static struct celleb_phb_spec celleb_fake_pci_spec __initdata = { - .setup = celleb_setup_fake_pci, -}; - -static const struct of_device_id celleb_phb_match[] __initconst = { - { - .name = "pci-pseudo", - .data = &celleb_fake_pci_spec, - }, { - .name = "epci", - .data = &celleb_epci_spec, - }, { - .name = "pcie", - .data = &celleb_pciex_spec, - }, { - }, -}; - -int __init celleb_setup_phb(struct pci_controller *phb) -{ - struct device_node *dev = phb->dn; - const struct of_device_id *match; - const struct celleb_phb_spec *phb_spec; - int rc; - - match = of_match_node(celleb_phb_match, dev); - if (!match) - return 1; - - phb_set_bus_ranges(dev, phb); - phb->buid = 1; - - phb_spec = match->data; - rc = (*phb_spec->setup)(dev, phb); - if (rc) - return 1; - - if (phb_spec->ops) - iowa_register_bus(phb, phb_spec->ops, - phb_spec->iowa_init, - phb_spec->iowa_data); - return 0; -} - -int celleb_pci_probe_mode(struct pci_bus *bus) -{ - return PCI_PROBE_DEVTREE; -} diff --git a/arch/powerpc/platforms/cell/celleb_pci.h b/arch/powerpc/platforms/cell/celleb_pci.h deleted file mode 100644 index a801fcc5f389..000000000000 --- a/arch/powerpc/platforms/cell/celleb_pci.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * pci prototypes for Celleb platform - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef _CELLEB_PCI_H -#define _CELLEB_PCI_H - -#include <linux/pci.h> - -#include <asm/pci-bridge.h> -#include <asm/prom.h> -#include <asm/ppc-pci.h> -#include <asm/io-workarounds.h> - -struct iowa_bus; - -struct celleb_phb_spec { - int (*setup)(struct device_node *, struct pci_controller *); - struct ppc_pci_io *ops; - int (*iowa_init)(struct iowa_bus *, void *); - void *iowa_data; -}; - -extern int celleb_setup_phb(struct pci_controller *); -extern int celleb_pci_probe_mode(struct pci_bus *); - -extern struct celleb_phb_spec celleb_epci_spec; -extern struct celleb_phb_spec celleb_pciex_spec; - -#endif /* _CELLEB_PCI_H */ diff --git a/arch/powerpc/platforms/cell/celleb_scc.h b/arch/powerpc/platforms/cell/celleb_scc.h deleted file mode 100644 index b596a711c348..000000000000 --- a/arch/powerpc/platforms/cell/celleb_scc.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - * SCC (Super Companion Chip) definitions - * - * (C) Copyright 2004-2006 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#ifndef _CELLEB_SCC_H -#define _CELLEB_SCC_H - -#define PCI_VENDOR_ID_TOSHIBA_2 0x102f -#define PCI_DEVICE_ID_TOSHIBA_SCC_PCIEXC_BRIDGE 0x01b0 -#define PCI_DEVICE_ID_TOSHIBA_SCC_EPCI_BRIDGE 0x01b1 -#define PCI_DEVICE_ID_TOSHIBA_SCC_BRIDGE 0x01b2 -#define PCI_DEVICE_ID_TOSHIBA_SCC_GBE 0x01b3 -#define PCI_DEVICE_ID_TOSHIBA_SCC_ATA 0x01b4 -#define PCI_DEVICE_ID_TOSHIBA_SCC_USB2 0x01b5 -#define PCI_DEVICE_ID_TOSHIBA_SCC_USB 0x01b6 -#define PCI_DEVICE_ID_TOSHIBA_SCC_ENCDEC 0x01b7 - -#define SCC_EPCI_REG 0x0000d000 - -/* EPCI registers */ -#define SCC_EPCI_CNF10_REG 0x010 -#define SCC_EPCI_CNF14_REG 0x014 -#define SCC_EPCI_CNF18_REG 0x018 -#define SCC_EPCI_PVBAT 0x100 -#define SCC_EPCI_VPMBAT 0x104 -#define SCC_EPCI_VPIBAT 0x108 -#define SCC_EPCI_VCSR 0x110 -#define SCC_EPCI_VIENAB 0x114 -#define SCC_EPCI_VISTAT 0x118 -#define SCC_EPCI_VRDCOUNT 0x124 -#define SCC_EPCI_BAM0 0x12c -#define SCC_EPCI_BAM1 0x134 -#define SCC_EPCI_BAM2 0x13c -#define SCC_EPCI_IADR 0x164 -#define SCC_EPCI_CLKRST 0x800 -#define SCC_EPCI_INTSET 0x804 -#define SCC_EPCI_STATUS 0x808 -#define SCC_EPCI_ABTSET 0x80c -#define SCC_EPCI_WATRP 0x810 -#define SCC_EPCI_DUMYRADR 0x814 -#define SCC_EPCI_SWRESP 0x818 -#define SCC_EPCI_CNTOPT 0x81c -#define SCC_EPCI_ECMODE 0xf00 -#define SCC_EPCI_IOM_AC_NUM 5 -#define SCC_EPCI_IOM_ACTE(n) (0xf10 + (n) * 4) -#define SCC_EPCI_IOT_AC_NUM 4 -#define SCC_EPCI_IOT_ACTE(n) (0xf30 + (n) * 4) -#define SCC_EPCI_MAEA 0xf50 -#define SCC_EPCI_MAEC 0xf54 -#define SCC_EPCI_CKCTRL 0xff0 - -/* bits for SCC_EPCI_VCSR */ -#define SCC_EPCI_VCSR_FRE 0x00020000 -#define SCC_EPCI_VCSR_FWE 0x00010000 -#define SCC_EPCI_VCSR_DR 0x00000400 -#define SCC_EPCI_VCSR_SR 0x00000008 -#define SCC_EPCI_VCSR_AT 0x00000004 - -/* bits for SCC_EPCI_VIENAB/SCC_EPCI_VISTAT */ -#define SCC_EPCI_VISTAT_PMPE 0x00000008 -#define SCC_EPCI_VISTAT_PMFE 0x00000004 -#define SCC_EPCI_VISTAT_PRA 0x00000002 -#define SCC_EPCI_VISTAT_PRD 0x00000001 -#define SCC_EPCI_VISTAT_ALL 0x0000000f - -#define SCC_EPCI_VIENAB_PMPEE 0x00000008 -#define SCC_EPCI_VIENAB_PMFEE 0x00000004 -#define SCC_EPCI_VIENAB_PRA 0x00000002 -#define SCC_EPCI_VIENAB_PRD 0x00000001 -#define SCC_EPCI_VIENAB_ALL 0x0000000f - -/* bits for SCC_EPCI_CLKRST */ -#define SCC_EPCI_CLKRST_CKS_MASK 0x00030000 -#define SCC_EPCI_CLKRST_CKS_2 0x00000000 -#define SCC_EPCI_CLKRST_CKS_4 0x00010000 -#define SCC_EPCI_CLKRST_CKS_8 0x00020000 -#define SCC_EPCI_CLKRST_PCICRST 0x00000400 -#define SCC_EPCI_CLKRST_BC 0x00000200 -#define SCC_EPCI_CLKRST_PCIRST 0x00000100 -#define SCC_EPCI_CLKRST_PCKEN 0x00000001 - -/* bits for SCC_EPCI_INTSET/SCC_EPCI_STATUS */ -#define SCC_EPCI_INT_2M 0x01000000 -#define SCC_EPCI_INT_RERR 0x00200000 -#define SCC_EPCI_INT_SERR 0x00100000 -#define SCC_EPCI_INT_PRTER 0x00080000 -#define SCC_EPCI_INT_SER 0x00040000 -#define SCC_EPCI_INT_PER 0x00020000 -#define SCC_EPCI_INT_PAI 0x00010000 -#define SCC_EPCI_INT_1M 0x00000100 -#define SCC_EPCI_INT_PME 0x00000010 -#define SCC_EPCI_INT_INTD 0x00000008 -#define SCC_EPCI_INT_INTC 0x00000004 -#define SCC_EPCI_INT_INTB 0x00000002 -#define SCC_EPCI_INT_INTA 0x00000001 -#define SCC_EPCI_INT_DEVINT 0x0000000f -#define SCC_EPCI_INT_ALL 0x003f001f -#define SCC_EPCI_INT_ALLERR 0x003f0000 - -/* bits for SCC_EPCI_CKCTRL */ -#define SCC_EPCI_CKCTRL_CRST0 0x00010000 -#define SCC_EPCI_CKCTRL_CRST1 0x00020000 -#define SCC_EPCI_CKCTRL_OCLKEN 0x00000100 -#define SCC_EPCI_CKCTRL_LCLKEN 0x00000001 - -#define SCC_EPCI_IDSEL_AD_TO_SLOT(ad) ((ad) - 10) -#define SCC_EPCI_MAX_DEVNU SCC_EPCI_IDSEL_AD_TO_SLOT(32) - -/* bits for SCC_EPCI_CNTOPT */ -#define SCC_EPCI_CNTOPT_O2PMB 0x00000002 - -/* SCC PCIEXC SMMIO registers */ -#define PEXCADRS 0x000 -#define PEXCWDATA 0x004 -#define PEXCRDATA 0x008 -#define PEXDADRS 0x010 -#define PEXDCMND 0x014 -#define PEXDWDATA 0x018 -#define PEXDRDATA 0x01c -#define PEXREQID 0x020 -#define PEXTIDMAP 0x024 -#define PEXINTMASK 0x028 -#define PEXINTSTS 0x02c -#define PEXAERRMASK 0x030 -#define PEXAERRSTS 0x034 -#define PEXPRERRMASK 0x040 -#define PEXPRERRSTS 0x044 -#define PEXPRERRID01 0x048 -#define PEXPRERRID23 0x04c -#define PEXVDMASK 0x050 -#define PEXVDSTS 0x054 -#define PEXRCVCPLIDA 0x060 -#define PEXLENERRIDA 0x068 -#define PEXPHYPLLST 0x070 -#define PEXDMRDEN0 0x100 -#define PEXDMRDADR0 0x104 -#define PEXDMRDENX 0x110 -#define PEXDMRDADRX 0x114 -#define PEXECMODE 0xf00 -#define PEXMAEA(n) (0xf50 + (8 * n)) -#define PEXMAEC(n) (0xf54 + (8 * n)) -#define PEXCCRCTRL 0xff0 - -/* SCC PCIEXC bits and shifts for PEXCADRS */ -#define PEXCADRS_BYTE_EN_SHIFT 20 -#define PEXCADRS_CMD_SHIFT 16 -#define PEXCADRS_CMD_READ (0xa << PEXCADRS_CMD_SHIFT) -#define PEXCADRS_CMD_WRITE (0xb << PEXCADRS_CMD_SHIFT) - -/* SCC PCIEXC shifts for PEXDADRS */ -#define PEXDADRS_BUSNO_SHIFT 20 -#define PEXDADRS_DEVNO_SHIFT 15 -#define PEXDADRS_FUNCNO_SHIFT 12 - -/* SCC PCIEXC bits and shifts for PEXDCMND */ -#define PEXDCMND_BYTE_EN_SHIFT 4 -#define PEXDCMND_IO_READ 0x2 -#define PEXDCMND_IO_WRITE 0x3 -#define PEXDCMND_CONFIG_READ 0xa -#define PEXDCMND_CONFIG_WRITE 0xb - -/* SCC PCIEXC bits for PEXPHYPLLST */ -#define PEXPHYPLLST_PEXPHYAPLLST 0x00000001 - -/* SCC PCIEXC bits for PEXECMODE */ -#define PEXECMODE_ALL_THROUGH 0x00000000 -#define PEXECMODE_ALL_8BIT 0x00550155 -#define PEXECMODE_ALL_16BIT 0x00aa02aa - -/* SCC PCIEXC bits for PEXCCRCTRL */ -#define PEXCCRCTRL_PEXIPCOREEN 0x00040000 -#define PEXCCRCTRL_PEXIPCONTEN 0x00020000 -#define PEXCCRCTRL_PEXPHYPLLEN 0x00010000 -#define PEXCCRCTRL_PCIEXCAOCKEN 0x00000100 - -/* SCC PCIEXC port configuration registers */ -#define PEXTCERRCHK 0x21c -#define PEXTAMAPB0 0x220 -#define PEXTAMAPL0 0x224 -#define PEXTAMAPB(n) (PEXTAMAPB0 + 8 * (n)) -#define PEXTAMAPL(n) (PEXTAMAPL0 + 8 * (n)) -#define PEXCHVC0P 0x500 -#define PEXCHVC0NP 0x504 -#define PEXCHVC0C 0x508 -#define PEXCDVC0P 0x50c -#define PEXCDVC0NP 0x510 -#define PEXCDVC0C 0x514 -#define PEXCHVCXP 0x518 -#define PEXCHVCXNP 0x51c -#define PEXCHVCXC 0x520 -#define PEXCDVCXP 0x524 -#define PEXCDVCXNP 0x528 -#define PEXCDVCXC 0x52c -#define PEXCTTRG 0x530 -#define PEXTSCTRL 0x700 -#define PEXTSSTS 0x704 -#define PEXSKPCTRL 0x708 - -/* UHC registers */ -#define SCC_UHC_CKRCTRL 0xff0 -#define SCC_UHC_ECMODE 0xf00 - -/* bits for SCC_UHC_CKRCTRL */ -#define SCC_UHC_F48MCKLEN 0x00000001 -#define SCC_UHC_P_SUSPEND 0x00000002 -#define SCC_UHC_PHY_SUSPEND_SEL 0x00000004 -#define SCC_UHC_HCLKEN 0x00000100 -#define SCC_UHC_USBEN 0x00010000 -#define SCC_UHC_USBCEN 0x00020000 -#define SCC_UHC_PHYEN 0x00040000 - -/* bits for SCC_UHC_ECMODE */ -#define SCC_UHC_ECMODE_BY_BYTE 0x00000555 -#define SCC_UHC_ECMODE_BY_WORD 0x00000aaa - -#endif /* _CELLEB_SCC_H */ diff --git a/arch/powerpc/platforms/cell/celleb_scc_epci.c b/arch/powerpc/platforms/cell/celleb_scc_epci.c deleted file mode 100644 index 9438bbed402f..000000000000 --- a/arch/powerpc/platforms/cell/celleb_scc_epci.c +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Support for SCC external PCI - * - * (C) Copyright 2004-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/threads.h> -#include <linux/pci.h> -#include <linux/init.h> -#include <linux/pci_regs.h> - -#include <asm/io.h> -#include <asm/irq.h> -#include <asm/prom.h> -#include <asm/pci-bridge.h> -#include <asm/ppc-pci.h> - -#include "celleb_scc.h" -#include "celleb_pci.h" - -#define MAX_PCI_DEVICES 32 -#define MAX_PCI_FUNCTIONS 8 - -#define iob() __asm__ __volatile__("eieio; sync":::"memory") - -static inline PCI_IO_ADDR celleb_epci_get_epci_base( - struct pci_controller *hose) -{ - /* - * Note: - * Celleb epci uses cfg_addr as a base address for - * epci control registers. - */ - - return hose->cfg_addr; -} - -static inline PCI_IO_ADDR celleb_epci_get_epci_cfg( - struct pci_controller *hose) -{ - /* - * Note: - * Celleb epci uses cfg_data as a base address for - * configuration area for epci devices. - */ - - return hose->cfg_data; -} - -static inline void clear_and_disable_master_abort_interrupt( - struct pci_controller *hose) -{ - PCI_IO_ADDR epci_base; - PCI_IO_ADDR reg; - epci_base = celleb_epci_get_epci_base(hose); - reg = epci_base + PCI_COMMAND; - out_be32(reg, in_be32(reg) | (PCI_STATUS_REC_MASTER_ABORT << 16)); -} - -static int celleb_epci_check_abort(struct pci_controller *hose, - PCI_IO_ADDR addr) -{ - PCI_IO_ADDR reg; - PCI_IO_ADDR epci_base; - u32 val; - - iob(); - epci_base = celleb_epci_get_epci_base(hose); - - reg = epci_base + PCI_COMMAND; - val = in_be32(reg); - - if (val & (PCI_STATUS_REC_MASTER_ABORT << 16)) { - out_be32(reg, - (val & 0xffff) | (PCI_STATUS_REC_MASTER_ABORT << 16)); - - /* clear PCI Controller error, FRE, PMFE */ - reg = epci_base + SCC_EPCI_STATUS; - out_be32(reg, SCC_EPCI_INT_PAI); - - reg = epci_base + SCC_EPCI_VCSR; - val = in_be32(reg) & 0xffff; - val |= SCC_EPCI_VCSR_FRE; - out_be32(reg, val); - - reg = epci_base + SCC_EPCI_VISTAT; - out_be32(reg, SCC_EPCI_VISTAT_PMFE); - return PCIBIOS_DEVICE_NOT_FOUND; - } - - return PCIBIOS_SUCCESSFUL; -} - -static PCI_IO_ADDR celleb_epci_make_config_addr(struct pci_bus *bus, - struct pci_controller *hose, unsigned int devfn, int where) -{ - PCI_IO_ADDR addr; - - if (bus != hose->bus) - addr = celleb_epci_get_epci_cfg(hose) + - (((bus->number & 0xff) << 16) - | ((devfn & 0xff) << 8) - | (where & 0xff) - | 0x01000000); - else - addr = celleb_epci_get_epci_cfg(hose) + - (((devfn & 0xff) << 8) | (where & 0xff)); - - pr_debug("EPCI: config_addr = 0x%p\n", addr); - - return addr; -} - -static int celleb_epci_read_config(struct pci_bus *bus, - unsigned int devfn, int where, int size, u32 *val) -{ - PCI_IO_ADDR epci_base; - PCI_IO_ADDR addr; - struct pci_controller *hose = pci_bus_to_host(bus); - - /* allignment check */ - BUG_ON(where % size); - - if (!celleb_epci_get_epci_cfg(hose)) - return PCIBIOS_DEVICE_NOT_FOUND; - - if (bus->number == hose->first_busno && devfn == 0) { - /* EPCI controller self */ - - epci_base = celleb_epci_get_epci_base(hose); - addr = epci_base + where; - - switch (size) { - case 1: - *val = in_8(addr); - break; - case 2: - *val = in_be16(addr); - break; - case 4: - *val = in_be32(addr); - break; - default: - return PCIBIOS_DEVICE_NOT_FOUND; - } - - } else { - - clear_and_disable_master_abort_interrupt(hose); - addr = celleb_epci_make_config_addr(bus, hose, devfn, where); - - switch (size) { - case 1: - *val = in_8(addr); - break; - case 2: - *val = in_le16(addr); - break; - case 4: - *val = in_le32(addr); - break; - default: - return PCIBIOS_DEVICE_NOT_FOUND; - } - } - - pr_debug("EPCI: " - "addr=0x%p, devfn=0x%x, where=0x%x, size=0x%x, val=0x%x\n", - addr, devfn, where, size, *val); - - return celleb_epci_check_abort(hose, NULL); -} - -static int celleb_epci_write_config(struct pci_bus *bus, - unsigned int devfn, int where, int size, u32 val) -{ - PCI_IO_ADDR epci_base; - PCI_IO_ADDR addr; - struct pci_controller *hose = pci_bus_to_host(bus); - - /* allignment check */ - BUG_ON(where % size); - - if (!celleb_epci_get_epci_cfg(hose)) - return PCIBIOS_DEVICE_NOT_FOUND; - - if (bus->number == hose->first_busno && devfn == 0) { - /* EPCI controller self */ - - epci_base = celleb_epci_get_epci_base(hose); - addr = epci_base + where; - - switch (size) { - case 1: - out_8(addr, val); - break; - case 2: - out_be16(addr, val); - break; - case 4: - out_be32(addr, val); - break; - default: - return PCIBIOS_DEVICE_NOT_FOUND; - } - - } else { - - clear_and_disable_master_abort_interrupt(hose); - addr = celleb_epci_make_config_addr(bus, hose, devfn, where); - - switch (size) { - case 1: - out_8(addr, val); - break; - case 2: - out_le16(addr, val); - break; - case 4: - out_le32(addr, val); - break; - default: - return PCIBIOS_DEVICE_NOT_FOUND; - } - } - - return celleb_epci_check_abort(hose, addr); -} - -struct pci_ops celleb_epci_ops = { - .read = celleb_epci_read_config, - .write = celleb_epci_write_config, -}; - -/* to be moved in FW */ -static int __init celleb_epci_init(struct pci_controller *hose) -{ - u32 val; - PCI_IO_ADDR reg; - PCI_IO_ADDR epci_base; - int hwres = 0; - - epci_base = celleb_epci_get_epci_base(hose); - - /* PCI core reset(Internal bus and PCI clock) */ - reg = epci_base + SCC_EPCI_CKCTRL; - val = in_be32(reg); - if (val == 0x00030101) - hwres = 1; - else { - val &= ~(SCC_EPCI_CKCTRL_CRST0 | SCC_EPCI_CKCTRL_CRST1); - out_be32(reg, val); - - /* set PCI core clock */ - val = in_be32(reg); - val |= (SCC_EPCI_CKCTRL_OCLKEN | SCC_EPCI_CKCTRL_LCLKEN); - out_be32(reg, val); - - /* release PCI core reset (internal bus) */ - val = in_be32(reg); - val |= SCC_EPCI_CKCTRL_CRST0; - out_be32(reg, val); - - /* set PCI clock select */ - reg = epci_base + SCC_EPCI_CLKRST; - val = in_be32(reg); - val &= ~SCC_EPCI_CLKRST_CKS_MASK; - val |= SCC_EPCI_CLKRST_CKS_2; - out_be32(reg, val); - - /* set arbiter */ - reg = epci_base + SCC_EPCI_ABTSET; - out_be32(reg, 0x0f1f001f); /* temporary value */ - - /* buffer on */ - reg = epci_base + SCC_EPCI_CLKRST; - val = in_be32(reg); - val |= SCC_EPCI_CLKRST_BC; - out_be32(reg, val); - - /* PCI clock enable */ - val = in_be32(reg); - val |= SCC_EPCI_CLKRST_PCKEN; - out_be32(reg, val); - - /* release PCI core reset (all) */ - reg = epci_base + SCC_EPCI_CKCTRL; - val = in_be32(reg); - val |= (SCC_EPCI_CKCTRL_CRST0 | SCC_EPCI_CKCTRL_CRST1); - out_be32(reg, val); - - /* set base translation registers. (already set by Beat) */ - - /* set base address masks. (already set by Beat) */ - } - - /* release interrupt masks and clear all interrupts */ - reg = epci_base + SCC_EPCI_INTSET; - out_be32(reg, 0x013f011f); /* all interrupts enable */ - reg = epci_base + SCC_EPCI_VIENAB; - val = SCC_EPCI_VIENAB_PMPEE | SCC_EPCI_VIENAB_PMFEE; - out_be32(reg, val); - reg = epci_base + SCC_EPCI_STATUS; - out_be32(reg, 0xffffffff); - reg = epci_base + SCC_EPCI_VISTAT; - out_be32(reg, 0xffffffff); - - /* disable PCI->IB address translation */ - reg = epci_base + SCC_EPCI_VCSR; - val = in_be32(reg); - val &= ~(SCC_EPCI_VCSR_DR | SCC_EPCI_VCSR_AT); - out_be32(reg, val); - - /* set base addresses. (no need to set?) */ - - /* memory space, bus master enable */ - reg = epci_base + PCI_COMMAND; - val = PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; - out_be32(reg, val); - - /* endian mode setup */ - reg = epci_base + SCC_EPCI_ECMODE; - val = 0x00550155; - out_be32(reg, val); - - /* set control option */ - reg = epci_base + SCC_EPCI_CNTOPT; - val = in_be32(reg); - val |= SCC_EPCI_CNTOPT_O2PMB; - out_be32(reg, val); - - /* XXX: temporay: set registers for address conversion setup */ - reg = epci_base + SCC_EPCI_CNF10_REG; - out_be32(reg, 0x80000008); - reg = epci_base + SCC_EPCI_CNF14_REG; - out_be32(reg, 0x40000008); - - reg = epci_base + SCC_EPCI_BAM0; - out_be32(reg, 0x80000000); - reg = epci_base + SCC_EPCI_BAM1; - out_be32(reg, 0xe0000000); - - reg = epci_base + SCC_EPCI_PVBAT; - out_be32(reg, 0x80000000); - - if (!hwres) { - /* release external PCI reset */ - reg = epci_base + SCC_EPCI_CLKRST; - val = in_be32(reg); - val |= SCC_EPCI_CLKRST_PCIRST; - out_be32(reg, val); - } - - return 0; -} - -static int __init celleb_setup_epci(struct device_node *node, - struct pci_controller *hose) -{ - struct resource r; - - pr_debug("PCI: celleb_setup_epci()\n"); - - /* - * Note: - * Celleb epci uses cfg_addr and cfg_data member of - * pci_controller structure in irregular way. - * - * cfg_addr is used to map for control registers of - * celleb epci. - * - * cfg_data is used for configuration area of devices - * on Celleb epci buses. - */ - - if (of_address_to_resource(node, 0, &r)) - goto error; - hose->cfg_addr = ioremap(r.start, resource_size(&r)); - if (!hose->cfg_addr) - goto error; - pr_debug("EPCI: cfg_addr map 0x%016llx->0x%016lx + 0x%016llx\n", - r.start, (unsigned long)hose->cfg_addr, resource_size(&r)); - - if (of_address_to_resource(node, 2, &r)) - goto error; - hose->cfg_data = ioremap(r.start, resource_size(&r)); - if (!hose->cfg_data) - goto error; - pr_debug("EPCI: cfg_data map 0x%016llx->0x%016lx + 0x%016llx\n", - r.start, (unsigned long)hose->cfg_data, resource_size(&r)); - - hose->ops = &celleb_epci_ops; - celleb_epci_init(hose); - - return 0; - -error: - if (hose->cfg_addr) - iounmap(hose->cfg_addr); - - if (hose->cfg_data) - iounmap(hose->cfg_data); - return 1; -} - -struct celleb_phb_spec celleb_epci_spec __initdata = { - .setup = celleb_setup_epci, - .ops = &spiderpci_ops, - .iowa_init = &spiderpci_iowa_init, - .iowa_data = (void *)0, -}; diff --git a/arch/powerpc/platforms/cell/celleb_scc_pciex.c b/arch/powerpc/platforms/cell/celleb_scc_pciex.c deleted file mode 100644 index 94170e4f2ce7..000000000000 --- a/arch/powerpc/platforms/cell/celleb_scc_pciex.c +++ /dev/null @@ -1,538 +0,0 @@ -/* - * Support for Celleb PCI-Express. - * - * (C) Copyright 2007-2008 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/pci.h> -#include <linux/string.h> -#include <linux/slab.h> -#include <linux/init.h> -#include <linux/delay.h> -#include <linux/interrupt.h> - -#include <asm/io.h> -#include <asm/irq.h> -#include <asm/iommu.h> -#include <asm/byteorder.h> - -#include "celleb_scc.h" -#include "celleb_pci.h" - -#define PEX_IN(base, off) in_be32((void __iomem *)(base) + (off)) -#define PEX_OUT(base, off, data) out_be32((void __iomem *)(base) + (off), (data)) - -static void scc_pciex_io_flush(struct iowa_bus *bus) -{ - (void)PEX_IN(bus->phb->cfg_addr, PEXDMRDEN0); -} - -/* - * Memory space access to device on PCIEX - */ -#define PCIEX_MMIO_READ(name, ret) \ -static ret scc_pciex_##name(const PCI_IO_ADDR addr) \ -{ \ - ret val = __do_##name(addr); \ - scc_pciex_io_flush(iowa_mem_find_bus(addr)); \ - return val; \ -} - -#define PCIEX_MMIO_READ_STR(name) \ -static void scc_pciex_##name(const PCI_IO_ADDR addr, void *buf, \ - unsigned long count) \ -{ \ - __do_##name(addr, buf, count); \ - scc_pciex_io_flush(iowa_mem_find_bus(addr)); \ -} - -PCIEX_MMIO_READ(readb, u8) -PCIEX_MMIO_READ(readw, u16) -PCIEX_MMIO_READ(readl, u32) -PCIEX_MMIO_READ(readq, u64) -PCIEX_MMIO_READ(readw_be, u16) -PCIEX_MMIO_READ(readl_be, u32) -PCIEX_MMIO_READ(readq_be, u64) -PCIEX_MMIO_READ_STR(readsb) -PCIEX_MMIO_READ_STR(readsw) -PCIEX_MMIO_READ_STR(readsl) - -static void scc_pciex_memcpy_fromio(void *dest, const PCI_IO_ADDR src, - unsigned long n) -{ - __do_memcpy_fromio(dest, src, n); - scc_pciex_io_flush(iowa_mem_find_bus(src)); -} - -/* - * I/O port access to devices on PCIEX. - */ - -static inline unsigned long get_bus_address(struct pci_controller *phb, - unsigned long port) -{ - return port - ((unsigned long)(phb->io_base_virt) - _IO_BASE); -} - -static u32 scc_pciex_read_port(struct pci_controller *phb, - unsigned long port, int size) -{ - unsigned int byte_enable; - unsigned int cmd, shift; - unsigned long addr; - u32 data, ret; - - BUG_ON(((port & 0x3ul) + size) > 4); - - addr = get_bus_address(phb, port); - shift = addr & 0x3ul; - byte_enable = ((1 << size) - 1) << shift; - cmd = PEXDCMND_IO_READ | (byte_enable << PEXDCMND_BYTE_EN_SHIFT); - PEX_OUT(phb->cfg_addr, PEXDADRS, (addr & ~0x3ul)); - PEX_OUT(phb->cfg_addr, PEXDCMND, cmd); - data = PEX_IN(phb->cfg_addr, PEXDRDATA); - ret = (data >> (shift * 8)) & (0xFFFFFFFF >> ((4 - size) * 8)); - - pr_debug("PCIEX:PIO READ:port=0x%lx, addr=0x%lx, size=%d, be=%x," - " cmd=%x, data=%x, ret=%x\n", port, addr, size, byte_enable, - cmd, data, ret); - - return ret; -} - -static void scc_pciex_write_port(struct pci_controller *phb, - unsigned long port, int size, u32 val) -{ - unsigned int byte_enable; - unsigned int cmd, shift; - unsigned long addr; - u32 data; - - BUG_ON(((port & 0x3ul) + size) > 4); - - addr = get_bus_address(phb, port); - shift = addr & 0x3ul; - byte_enable = ((1 << size) - 1) << shift; - cmd = PEXDCMND_IO_WRITE | (byte_enable << PEXDCMND_BYTE_EN_SHIFT); - data = (val & (0xFFFFFFFF >> (4 - size) * 8)) << (shift * 8); - PEX_OUT(phb->cfg_addr, PEXDADRS, (addr & ~0x3ul)); - PEX_OUT(phb->cfg_addr, PEXDCMND, cmd); - PEX_OUT(phb->cfg_addr, PEXDWDATA, data); - - pr_debug("PCIEX:PIO WRITE:port=0x%lx, addr=%lx, size=%d, val=%x," - " be=%x, cmd=%x, data=%x\n", port, addr, size, val, - byte_enable, cmd, data); -} - -static u8 __scc_pciex_inb(struct pci_controller *phb, unsigned long port) -{ - return (u8)scc_pciex_read_port(phb, port, 1); -} - -static u16 __scc_pciex_inw(struct pci_controller *phb, unsigned long port) -{ - u32 data; - if ((port & 0x3ul) < 3) - data = scc_pciex_read_port(phb, port, 2); - else { - u32 d1 = scc_pciex_read_port(phb, port, 1); - u32 d2 = scc_pciex_read_port(phb, port + 1, 1); - data = d1 | (d2 << 8); - } - return (u16)data; -} - -static u32 __scc_pciex_inl(struct pci_controller *phb, unsigned long port) -{ - unsigned int mod = port & 0x3ul; - u32 data; - if (mod == 0) - data = scc_pciex_read_port(phb, port, 4); - else { - u32 d1 = scc_pciex_read_port(phb, port, 4 - mod); - u32 d2 = scc_pciex_read_port(phb, port + 1, mod); - data = d1 | (d2 << (mod * 8)); - } - return data; -} - -static void __scc_pciex_outb(struct pci_controller *phb, - u8 val, unsigned long port) -{ - scc_pciex_write_port(phb, port, 1, (u32)val); -} - -static void __scc_pciex_outw(struct pci_controller *phb, - u16 val, unsigned long port) -{ - if ((port & 0x3ul) < 3) - scc_pciex_write_port(phb, port, 2, (u32)val); - else { - u32 d1 = val & 0x000000FF; - u32 d2 = (val & 0x0000FF00) >> 8; - scc_pciex_write_port(phb, port, 1, d1); - scc_pciex_write_port(phb, port + 1, 1, d2); - } -} - -static void __scc_pciex_outl(struct pci_controller *phb, - u32 val, unsigned long port) -{ - unsigned int mod = port & 0x3ul; - if (mod == 0) - scc_pciex_write_port(phb, port, 4, val); - else { - u32 d1 = val & (0xFFFFFFFFul >> (mod * 8)); - u32 d2 = val >> ((4 - mod) * 8); - scc_pciex_write_port(phb, port, 4 - mod, d1); - scc_pciex_write_port(phb, port + 1, mod, d2); - } -} - -#define PCIEX_PIO_FUNC(size, name) \ -static u##size scc_pciex_in##name(unsigned long port) \ -{ \ - struct iowa_bus *bus = iowa_pio_find_bus(port); \ - u##size data = __scc_pciex_in##name(bus->phb, port); \ - scc_pciex_io_flush(bus); \ - return data; \ -} \ -static void scc_pciex_ins##name(unsigned long p, void *b, unsigned long c) \ -{ \ - struct iowa_bus *bus = iowa_pio_find_bus(p); \ - __le##size *dst = b; \ - for (; c != 0; c--, dst++) \ - *dst = cpu_to_le##size(__scc_pciex_in##name(bus->phb, p)); \ - scc_pciex_io_flush(bus); \ -} \ -static void scc_pciex_out##name(u##size val, unsigned long port) \ -{ \ - struct iowa_bus *bus = iowa_pio_find_bus(port); \ - __scc_pciex_out##name(bus->phb, val, port); \ -} \ -static void scc_pciex_outs##name(unsigned long p, const void *b, \ - unsigned long c) \ -{ \ - struct iowa_bus *bus = iowa_pio_find_bus(p); \ - const __le##size *src = b; \ - for (; c != 0; c--, src++) \ - __scc_pciex_out##name(bus->phb, le##size##_to_cpu(*src), p); \ -} -#define __le8 u8 -#define cpu_to_le8(x) (x) -#define le8_to_cpu(x) (x) -PCIEX_PIO_FUNC(8, b) -PCIEX_PIO_FUNC(16, w) -PCIEX_PIO_FUNC(32, l) - -static struct ppc_pci_io scc_pciex_ops = { - .readb = scc_pciex_readb, - .readw = scc_pciex_readw, - .readl = scc_pciex_readl, - .readq = scc_pciex_readq, - .readw_be = scc_pciex_readw_be, - .readl_be = scc_pciex_readl_be, - .readq_be = scc_pciex_readq_be, - .readsb = scc_pciex_readsb, - .readsw = scc_pciex_readsw, - .readsl = scc_pciex_readsl, - .memcpy_fromio = scc_pciex_memcpy_fromio, - .inb = scc_pciex_inb, - .inw = scc_pciex_inw, - .inl = scc_pciex_inl, - .outb = scc_pciex_outb, - .outw = scc_pciex_outw, - .outl = scc_pciex_outl, - .insb = scc_pciex_insb, - .insw = scc_pciex_insw, - .insl = scc_pciex_insl, - .outsb = scc_pciex_outsb, - .outsw = scc_pciex_outsw, - .outsl = scc_pciex_outsl, -}; - -static int __init scc_pciex_iowa_init(struct iowa_bus *bus, void *data) -{ - dma_addr_t dummy_page_da; - void *dummy_page_va; - - dummy_page_va = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!dummy_page_va) { - pr_err("PCIEX:Alloc dummy_page_va failed\n"); - return -1; - } - - dummy_page_da = dma_map_single(bus->phb->parent, dummy_page_va, - PAGE_SIZE, DMA_FROM_DEVICE); - if (dma_mapping_error(bus->phb->parent, dummy_page_da)) { - pr_err("PCIEX:Map dummy page failed.\n"); - kfree(dummy_page_va); - return -1; - } - - PEX_OUT(bus->phb->cfg_addr, PEXDMRDADR0, dummy_page_da); - - return 0; -} - -/* - * config space access - */ -#define MK_PEXDADRS(bus_no, dev_no, func_no, addr) \ - ((uint32_t)(((addr) & ~0x3UL) | \ - ((bus_no) << PEXDADRS_BUSNO_SHIFT) | \ - ((dev_no) << PEXDADRS_DEVNO_SHIFT) | \ - ((func_no) << PEXDADRS_FUNCNO_SHIFT))) - -#define MK_PEXDCMND_BYTE_EN(addr, size) \ - ((((0x1 << (size))-1) << ((addr) & 0x3)) << PEXDCMND_BYTE_EN_SHIFT) -#define MK_PEXDCMND(cmd, addr, size) ((cmd) | MK_PEXDCMND_BYTE_EN(addr, size)) - -static uint32_t config_read_pciex_dev(unsigned int __iomem *base, - uint64_t bus_no, uint64_t dev_no, uint64_t func_no, - uint64_t off, uint64_t size) -{ - uint32_t ret; - uint32_t addr, cmd; - - addr = MK_PEXDADRS(bus_no, dev_no, func_no, off); - cmd = MK_PEXDCMND(PEXDCMND_CONFIG_READ, off, size); - PEX_OUT(base, PEXDADRS, addr); - PEX_OUT(base, PEXDCMND, cmd); - ret = (PEX_IN(base, PEXDRDATA) - >> ((off & (4-size)) * 8)) & ((0x1 << (size * 8)) - 1); - return ret; -} - -static void config_write_pciex_dev(unsigned int __iomem *base, uint64_t bus_no, - uint64_t dev_no, uint64_t func_no, uint64_t off, uint64_t size, - uint32_t data) -{ - uint32_t addr, cmd; - - addr = MK_PEXDADRS(bus_no, dev_no, func_no, off); - cmd = MK_PEXDCMND(PEXDCMND_CONFIG_WRITE, off, size); - PEX_OUT(base, PEXDADRS, addr); - PEX_OUT(base, PEXDCMND, cmd); - PEX_OUT(base, PEXDWDATA, - (data & ((0x1 << (size * 8)) - 1)) << ((off & (4-size)) * 8)); -} - -#define MK_PEXCADRS_BYTE_EN(off, len) \ - ((((0x1 << (len)) - 1) << ((off) & 0x3)) << PEXCADRS_BYTE_EN_SHIFT) -#define MK_PEXCADRS(cmd, addr, size) \ - ((cmd) | MK_PEXCADRS_BYTE_EN(addr, size) | ((addr) & ~0x3)) -static uint32_t config_read_pciex_rc(unsigned int __iomem *base, - uint32_t where, uint32_t size) -{ - PEX_OUT(base, PEXCADRS, MK_PEXCADRS(PEXCADRS_CMD_READ, where, size)); - return (PEX_IN(base, PEXCRDATA) - >> ((where & (4 - size)) * 8)) & ((0x1 << (size * 8)) - 1); -} - -static void config_write_pciex_rc(unsigned int __iomem *base, uint32_t where, - uint32_t size, uint32_t val) -{ - uint32_t data; - - data = (val & ((0x1 << (size * 8)) - 1)) << ((where & (4 - size)) * 8); - PEX_OUT(base, PEXCADRS, MK_PEXCADRS(PEXCADRS_CMD_WRITE, where, size)); - PEX_OUT(base, PEXCWDATA, data); -} - -/* Interfaces */ -/* Note: Work-around - * On SCC PCIEXC, one device is seen on all 32 dev_no. - * As SCC PCIEXC can have only one device on the bus, we look only one dev_no. - * (dev_no = 1) - */ -static int scc_pciex_read_config(struct pci_bus *bus, unsigned int devfn, - int where, int size, unsigned int *val) -{ - struct pci_controller *phb = pci_bus_to_host(bus); - - if (bus->number == phb->first_busno && PCI_SLOT(devfn) != 1) { - *val = ~0; - return PCIBIOS_DEVICE_NOT_FOUND; - } - - if (bus->number == 0 && PCI_SLOT(devfn) == 0) - *val = config_read_pciex_rc(phb->cfg_addr, where, size); - else - *val = config_read_pciex_dev(phb->cfg_addr, bus->number, - PCI_SLOT(devfn), PCI_FUNC(devfn), where, size); - - return PCIBIOS_SUCCESSFUL; -} - -static int scc_pciex_write_config(struct pci_bus *bus, unsigned int devfn, - int where, int size, unsigned int val) -{ - struct pci_controller *phb = pci_bus_to_host(bus); - - if (bus->number == phb->first_busno && PCI_SLOT(devfn) != 1) - return PCIBIOS_DEVICE_NOT_FOUND; - - if (bus->number == 0 && PCI_SLOT(devfn) == 0) - config_write_pciex_rc(phb->cfg_addr, where, size, val); - else - config_write_pciex_dev(phb->cfg_addr, bus->number, - PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val); - return PCIBIOS_SUCCESSFUL; -} - -static struct pci_ops scc_pciex_pci_ops = { - .read = scc_pciex_read_config, - .write = scc_pciex_write_config, -}; - -static void pciex_clear_intr_all(unsigned int __iomem *base) -{ - PEX_OUT(base, PEXAERRSTS, 0xffffffff); - PEX_OUT(base, PEXPRERRSTS, 0xffffffff); - PEX_OUT(base, PEXINTSTS, 0xffffffff); -} - -#if 0 -static void pciex_disable_intr_all(unsigned int *base) -{ - PEX_OUT(base, PEXINTMASK, 0x0); - PEX_OUT(base, PEXAERRMASK, 0x0); - PEX_OUT(base, PEXPRERRMASK, 0x0); - PEX_OUT(base, PEXVDMASK, 0x0); -} -#endif - -static void pciex_enable_intr_all(unsigned int __iomem *base) -{ - PEX_OUT(base, PEXINTMASK, 0x0000e7f1); - PEX_OUT(base, PEXAERRMASK, 0x03ff01ff); - PEX_OUT(base, PEXPRERRMASK, 0x0001010f); - PEX_OUT(base, PEXVDMASK, 0x00000001); -} - -static void pciex_check_status(unsigned int __iomem *base) -{ - uint32_t err = 0; - uint32_t intsts, aerr, prerr, rcvcp, lenerr; - uint32_t maea, maec; - - intsts = PEX_IN(base, PEXINTSTS); - aerr = PEX_IN(base, PEXAERRSTS); - prerr = PEX_IN(base, PEXPRERRSTS); - rcvcp = PEX_IN(base, PEXRCVCPLIDA); - lenerr = PEX_IN(base, PEXLENERRIDA); - - if (intsts || aerr || prerr || rcvcp || lenerr) - err = 1; - - pr_info("PCEXC interrupt!!\n"); - pr_info("PEXINTSTS :0x%08x\n", intsts); - pr_info("PEXAERRSTS :0x%08x\n", aerr); - pr_info("PEXPRERRSTS :0x%08x\n", prerr); - pr_info("PEXRCVCPLIDA :0x%08x\n", rcvcp); - pr_info("PEXLENERRIDA :0x%08x\n", lenerr); - - /* print detail of Protection Error */ - if (intsts & 0x00004000) { - uint32_t i, n; - for (i = 0; i < 4; i++) { - n = 1 << i; - if (prerr & n) { - maea = PEX_IN(base, PEXMAEA(i)); - maec = PEX_IN(base, PEXMAEC(i)); - pr_info("PEXMAEC%d :0x%08x\n", i, maec); - pr_info("PEXMAEA%d :0x%08x\n", i, maea); - } - } - } - - if (err) - pciex_clear_intr_all(base); -} - -static irqreturn_t pciex_handle_internal_irq(int irq, void *dev_id) -{ - struct pci_controller *phb = dev_id; - - pr_debug("PCIEX:pciex_handle_internal_irq(irq=%d)\n", irq); - - BUG_ON(phb->cfg_addr == NULL); - - pciex_check_status(phb->cfg_addr); - - return IRQ_HANDLED; -} - -static __init int celleb_setup_pciex(struct device_node *node, - struct pci_controller *phb) -{ - struct resource r; - int virq; - - /* SMMIO registers; used inside this file */ - if (of_address_to_resource(node, 0, &r)) { - pr_err("PCIEXC:Failed to get config resource.\n"); - return 1; - } - phb->cfg_addr = ioremap(r.start, resource_size(&r)); - if (!phb->cfg_addr) { - pr_err("PCIEXC:Failed to remap SMMIO region.\n"); - return 1; - } - - /* Not use cfg_data, cmd and data regs are near address reg */ - phb->cfg_data = NULL; - - /* set pci_ops */ - phb->ops = &scc_pciex_pci_ops; - - /* internal interrupt handler */ - virq = irq_of_parse_and_map(node, 1); - if (!virq) { - pr_err("PCIEXC:Failed to map irq\n"); - goto error; - } - if (request_irq(virq, pciex_handle_internal_irq, - 0, "pciex", (void *)phb)) { - pr_err("PCIEXC:Failed to request irq\n"); - goto error; - } - - /* enable all interrupts */ - pciex_clear_intr_all(phb->cfg_addr); - pciex_enable_intr_all(phb->cfg_addr); - /* MSI: TBD */ - - return 0; - -error: - phb->cfg_data = NULL; - if (phb->cfg_addr) - iounmap(phb->cfg_addr); - phb->cfg_addr = NULL; - return 1; -} - -struct celleb_phb_spec celleb_pciex_spec __initdata = { - .setup = celleb_setup_pciex, - .ops = &scc_pciex_ops, - .iowa_init = &scc_pciex_iowa_init, -}; diff --git a/arch/powerpc/platforms/cell/celleb_scc_sio.c b/arch/powerpc/platforms/cell/celleb_scc_sio.c deleted file mode 100644 index c8eb57193826..000000000000 --- a/arch/powerpc/platforms/cell/celleb_scc_sio.c +++ /dev/null @@ -1,99 +0,0 @@ -/* - * setup serial port in SCC - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <linux/tty.h> -#include <linux/serial.h> -#include <linux/serial_core.h> -#include <linux/console.h> - -#include <asm/io.h> -#include <asm/prom.h> - -/* sio irq0=0xb00010022 irq0=0xb00010023 irq2=0xb00010024 - mmio=0xfff000-0x1000,0xff2000-0x1000 */ -static int txx9_serial_bitmap __initdata; - -static struct { - uint32_t offset; - uint32_t index; -} txx9_scc_tab[3] __initdata = { - { 0x300, 0 }, /* 0xFFF300 */ - { 0x400, 0 }, /* 0xFFF400 */ - { 0x800, 1 } /* 0xFF2800 */ -}; - -static int __init txx9_serial_init(void) -{ - extern int early_serial_txx9_setup(struct uart_port *port); - struct device_node *node; - int i; - struct uart_port req; - struct of_phandle_args irq; - struct resource res; - - for_each_compatible_node(node, "serial", "toshiba,sio-scc") { - for (i = 0; i < ARRAY_SIZE(txx9_scc_tab); i++) { - if (!(txx9_serial_bitmap & (1<<i))) - continue; - - if (of_irq_parse_one(node, i, &irq)) - continue; - if (of_address_to_resource(node, - txx9_scc_tab[i].index, &res)) - continue; - - memset(&req, 0, sizeof(req)); - req.line = i; - req.iotype = UPIO_MEM; - req.mapbase = res.start + txx9_scc_tab[i].offset; -#ifdef CONFIG_SERIAL_TXX9_CONSOLE - req.membase = ioremap(req.mapbase, 0x24); -#endif - req.irq = irq_create_of_mapping(&irq); - req.flags |= UPF_IOREMAP | UPF_BUGGY_UART - /*HAVE_CTS_LINE*/; - req.uartclk = 83300000; - early_serial_txx9_setup(&req); - } - } - - return 0; -} - -static int __init txx9_serial_config(char *ptr) -{ - int i; - - for (;;) { - switch (get_option(&ptr, &i)) { - default: - return 0; - case 2: - txx9_serial_bitmap |= 1 << i; - break; - case 1: - txx9_serial_bitmap |= 1 << i; - return 0; - } - } -} -__setup("txx9_serial=", txx9_serial_config); - -console_initcall(txx9_serial_init); diff --git a/arch/powerpc/platforms/cell/celleb_scc_uhc.c b/arch/powerpc/platforms/cell/celleb_scc_uhc.c deleted file mode 100644 index d63b720bfe3a..000000000000 --- a/arch/powerpc/platforms/cell/celleb_scc_uhc.c +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SCC (Super Companion Chip) UHC setup - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#include <linux/kernel.h> -#include <linux/pci.h> - -#include <asm/delay.h> -#include <asm/io.h> -#include <asm/machdep.h> - -#include "celleb_scc.h" - -#define UHC_RESET_WAIT_MAX 10000 - -static inline int uhc_clkctrl_ready(u32 val) -{ - const u32 mask = SCC_UHC_USBCEN | SCC_UHC_USBCEN; - return((val & mask) == mask); -} - -/* - * UHC(usb host controller) enable function. - * affect to both of OHCI and EHCI core module. - */ -static void enable_scc_uhc(struct pci_dev *dev) -{ - void __iomem *uhc_base; - u32 __iomem *uhc_clkctrl; - u32 __iomem *uhc_ecmode; - u32 val = 0; - int i; - - if (!machine_is(celleb_beat) && - !machine_is(celleb_native)) - return; - - uhc_base = ioremap(pci_resource_start(dev, 0), - pci_resource_len(dev, 0)); - if (!uhc_base) { - printk(KERN_ERR "failed to map UHC register base.\n"); - return; - } - uhc_clkctrl = uhc_base + SCC_UHC_CKRCTRL; - uhc_ecmode = uhc_base + SCC_UHC_ECMODE; - - /* setup for normal mode */ - val |= SCC_UHC_F48MCKLEN; - out_be32(uhc_clkctrl, val); - val |= SCC_UHC_PHY_SUSPEND_SEL; - out_be32(uhc_clkctrl, val); - udelay(10); - val |= SCC_UHC_PHYEN; - out_be32(uhc_clkctrl, val); - udelay(50); - - /* disable reset */ - val |= SCC_UHC_HCLKEN; - out_be32(uhc_clkctrl, val); - val |= (SCC_UHC_USBCEN | SCC_UHC_USBEN); - out_be32(uhc_clkctrl, val); - i = 0; - while (!uhc_clkctrl_ready(in_be32(uhc_clkctrl))) { - udelay(10); - if (i++ > UHC_RESET_WAIT_MAX) { - printk(KERN_ERR "Failed to disable UHC reset %x\n", - in_be32(uhc_clkctrl)); - break; - } - } - - /* Endian Conversion Mode for Master ALL area */ - out_be32(uhc_ecmode, SCC_UHC_ECMODE_BY_BYTE); - - iounmap(uhc_base); -} - -DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_TOSHIBA_2, - PCI_DEVICE_ID_TOSHIBA_SCC_USB, enable_scc_uhc); diff --git a/arch/powerpc/platforms/cell/celleb_setup.c b/arch/powerpc/platforms/cell/celleb_setup.c deleted file mode 100644 index 90be8ec51686..000000000000 --- a/arch/powerpc/platforms/cell/celleb_setup.c +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Celleb setup code - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - * - * This code is based on arch/powerpc/platforms/cell/setup.c: - * Copyright (C) 1995 Linus Torvalds - * Adapted from 'alpha' version by Gary Thomas - * Modified by Cort Dougan (cort@cs.nmt.edu) - * Modified by PPC64 Team, IBM Corp - * Modified by Cell Team, IBM Deutschland Entwicklung GmbH - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#undef DEBUG - -#include <linux/cpu.h> -#include <linux/sched.h> -#include <linux/kernel.h> -#include <linux/export.h> -#include <linux/mm.h> -#include <linux/stddef.h> -#include <linux/unistd.h> -#include <linux/reboot.h> -#include <linux/init.h> -#include <linux/delay.h> -#include <linux/irq.h> -#include <linux/seq_file.h> -#include <linux/root_dev.h> -#include <linux/console.h> -#include <linux/of_platform.h> - -#include <asm/mmu.h> -#include <asm/processor.h> -#include <asm/io.h> -#include <asm/prom.h> -#include <asm/machdep.h> -#include <asm/cputable.h> -#include <asm/irq.h> -#include <asm/time.h> -#include <asm/spu_priv1.h> -#include <asm/firmware.h> -#include <asm/rtas.h> -#include <asm/cell-regs.h> - -#include "beat_interrupt.h" -#include "beat_wrapper.h" -#include "beat.h" -#include "celleb_pci.h" -#include "interrupt.h" -#include "pervasive.h" -#include "ras.h" - -static char celleb_machine_type[128] = "Celleb"; - -static void celleb_show_cpuinfo(struct seq_file *m) -{ - struct device_node *root; - const char *model = ""; - - root = of_find_node_by_path("/"); - if (root) - model = of_get_property(root, "model", NULL); - /* using "CHRP" is to trick anaconda into installing FCx into Celleb */ - seq_printf(m, "machine\t\t: %s %s\n", celleb_machine_type, model); - of_node_put(root); -} - -static int __init celleb_machine_type_hack(char *ptr) -{ - strlcpy(celleb_machine_type, ptr, sizeof(celleb_machine_type)); - return 0; -} - -__setup("celleb_machine_type_hack=", celleb_machine_type_hack); - -static void celleb_progress(char *s, unsigned short hex) -{ - printk("*** %04x : %s\n", hex, s ? s : ""); -} - -static void __init celleb_setup_arch_common(void) -{ - /* init to some ~sane value until calibrate_delay() runs */ - loops_per_jiffy = 50000000; - -#ifdef CONFIG_DUMMY_CONSOLE - conswitchp = &dummy_con; -#endif -} - -static const struct of_device_id celleb_bus_ids[] __initconst = { - { .type = "scc", }, - { .type = "ioif", }, /* old style */ - {}, -}; - -static int __init celleb_publish_devices(void) -{ - /* Publish OF platform devices for southbridge IOs */ - of_platform_bus_probe(NULL, celleb_bus_ids, NULL); - - return 0; -} -machine_device_initcall(celleb_beat, celleb_publish_devices); -machine_device_initcall(celleb_native, celleb_publish_devices); - - -/* - * functions for Celleb-Beat - */ -static void __init celleb_setup_arch_beat(void) -{ -#ifdef CONFIG_SPU_BASE - spu_priv1_ops = &spu_priv1_beat_ops; - spu_management_ops = &spu_management_of_ops; -#endif - - celleb_setup_arch_common(); -} - -static int __init celleb_probe_beat(void) -{ - unsigned long root = of_get_flat_dt_root(); - - if (!of_flat_dt_is_compatible(root, "Beat")) - return 0; - - powerpc_firmware_features |= FW_FEATURE_CELLEB_ALWAYS - | FW_FEATURE_BEAT | FW_FEATURE_LPAR; - hpte_init_beat_v3(); - pm_power_off = beat_power_off; - - return 1; -} - - -/* - * functions for Celleb-native - */ -static void __init celleb_init_IRQ_native(void) -{ - iic_init_IRQ(); - spider_init_IRQ(); -} - -static void __init celleb_setup_arch_native(void) -{ -#ifdef CONFIG_SPU_BASE - spu_priv1_ops = &spu_priv1_mmio_ops; - spu_management_ops = &spu_management_of_ops; -#endif - - cbe_regs_init(); - -#ifdef CONFIG_CBE_RAS - cbe_ras_init(); -#endif - -#ifdef CONFIG_SMP - smp_init_cell(); -#endif - - cbe_pervasive_init(); - - /* XXX: nvram initialization should be added */ - - celleb_setup_arch_common(); -} - -static int __init celleb_probe_native(void) -{ - unsigned long root = of_get_flat_dt_root(); - - if (of_flat_dt_is_compatible(root, "Beat") || - !of_flat_dt_is_compatible(root, "TOSHIBA,Celleb")) - return 0; - - powerpc_firmware_features |= FW_FEATURE_CELLEB_ALWAYS; - hpte_init_native(); - pm_power_off = rtas_power_off; - - return 1; -} - - -/* - * machine definitions - */ -define_machine(celleb_beat) { - .name = "Cell Reference Set (Beat)", - .probe = celleb_probe_beat, - .setup_arch = celleb_setup_arch_beat, - .show_cpuinfo = celleb_show_cpuinfo, - .restart = beat_restart, - .halt = beat_halt, - .get_rtc_time = beat_get_rtc_time, - .set_rtc_time = beat_set_rtc_time, - .calibrate_decr = generic_calibrate_decr, - .progress = celleb_progress, - .power_save = beat_power_save, - .nvram_size = beat_nvram_get_size, - .nvram_read = beat_nvram_read, - .nvram_write = beat_nvram_write, - .set_dabr = beat_set_xdabr, - .init_IRQ = beatic_init_IRQ, - .get_irq = beatic_get_irq, - .pci_probe_mode = celleb_pci_probe_mode, - .pci_setup_phb = celleb_setup_phb, -#ifdef CONFIG_KEXEC - .kexec_cpu_down = beat_kexec_cpu_down, -#endif -}; - -define_machine(celleb_native) { - .name = "Cell Reference Set (native)", - .probe = celleb_probe_native, - .setup_arch = celleb_setup_arch_native, - .show_cpuinfo = celleb_show_cpuinfo, - .restart = rtas_restart, - .halt = rtas_halt, - .get_boot_time = rtas_get_boot_time, - .get_rtc_time = rtas_get_rtc_time, - .set_rtc_time = rtas_set_rtc_time, - .calibrate_decr = generic_calibrate_decr, - .progress = celleb_progress, - .pci_probe_mode = celleb_pci_probe_mode, - .pci_setup_phb = celleb_setup_phb, - .init_IRQ = celleb_init_IRQ_native, -}; diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c index 4c11421847be..3af8324c122e 100644 --- a/arch/powerpc/platforms/cell/interrupt.c +++ b/arch/powerpc/platforms/cell/interrupt.c @@ -163,7 +163,7 @@ static unsigned int iic_get_irq(void) void iic_setup_cpu(void) { - out_be64(this_cpu_ptr(&cpu_iic.regs->prio), 0xff); + out_be64(&this_cpu_ptr(&cpu_iic)->regs->prio, 0xff); } u8 iic_get_target_id(int cpu) diff --git a/arch/powerpc/platforms/cell/iommu.c b/arch/powerpc/platforms/cell/iommu.c index c7c8720aa39f..21b502398bf3 100644 --- a/arch/powerpc/platforms/cell/iommu.c +++ b/arch/powerpc/platforms/cell/iommu.c @@ -39,6 +39,7 @@ #include <asm/firmware.h> #include <asm/cell-regs.h> +#include "cell.h" #include "interrupt.h" /* Define CELL_IOMMU_REAL_UNMAP to actually unmap non-used pages @@ -197,7 +198,7 @@ static int tce_build_cell(struct iommu_table *tbl, long index, long npages, io_pte = (unsigned long *)tbl->it_base + (index - tbl->it_offset); - for (i = 0; i < npages; i++, uaddr += tbl->it_page_shift) + for (i = 0; i < npages; i++, uaddr += (1 << tbl->it_page_shift)) io_pte[i] = base_pte | (__pa(uaddr) & CBE_IOPTE_RPN_Mask); mb(); @@ -857,7 +858,7 @@ static int __init cell_iommu_init_disabled(void) cell_dma_direct_offset += base; if (cell_dma_direct_offset != 0) - ppc_md.pci_dma_dev_setup = cell_pci_dma_dev_setup; + cell_pci_controller_ops.dma_dev_setup = cell_pci_dma_dev_setup; printk("iommu: disabled, direct DMA offset is 0x%lx\n", cell_dma_direct_offset); @@ -1197,8 +1198,8 @@ static int __init cell_iommu_init(void) if (cell_iommu_init_disabled() == 0) goto bail; - /* Setup various ppc_md. callbacks */ - ppc_md.pci_dma_dev_setup = cell_pci_dma_dev_setup; + /* Setup various callbacks */ + cell_pci_controller_ops.dma_dev_setup = cell_pci_dma_dev_setup; ppc_md.dma_get_required_mask = cell_dma_get_required_mask; ppc_md.tce_build = tce_build_cell; ppc_md.tce_free = tce_free_cell; @@ -1234,5 +1235,3 @@ static int __init cell_iommu_init(void) return 0; } machine_arch_initcall(cell, cell_iommu_init); -machine_arch_initcall(celleb_native, cell_iommu_init); - diff --git a/arch/powerpc/platforms/cell/setup.c b/arch/powerpc/platforms/cell/setup.c index d62aa982d530..36cff28d0293 100644 --- a/arch/powerpc/platforms/cell/setup.c +++ b/arch/powerpc/platforms/cell/setup.c @@ -54,6 +54,7 @@ #include <asm/cell-regs.h> #include <asm/io-workarounds.h> +#include "cell.h" #include "interrupt.h" #include "pervasive.h" #include "ras.h" @@ -126,6 +127,8 @@ static int cell_setup_phb(struct pci_controller *phb) if (rc) return rc; + phb->controller_ops = cell_pci_controller_ops; + np = phb->dn; model = of_get_property(np, "model", NULL); if (model == NULL || strcmp(np->name, "pci")) @@ -279,3 +282,5 @@ define_machine(cell) { .init_IRQ = cell_init_irq, .pci_setup_phb = cell_setup_phb, }; + +struct pci_controller_ops cell_pci_controller_ops; diff --git a/arch/powerpc/platforms/cell/smp.c b/arch/powerpc/platforms/cell/smp.c index b64e7ead752f..895560f4be69 100644 --- a/arch/powerpc/platforms/cell/smp.c +++ b/arch/powerpc/platforms/cell/smp.c @@ -102,13 +102,6 @@ static inline int smp_startup_cpu(unsigned int lcpu) return 1; } -static int __init smp_iic_probe(void) -{ - iic_request_IPIs(); - - return num_possible_cpus(); -} - static void smp_cell_setup_cpu(int cpu) { if (cpu != boot_cpuid) @@ -139,7 +132,7 @@ static int smp_cell_kick_cpu(int nr) static struct smp_ops_t bpa_iic_smp_ops = { .message_pass = iic_message_pass, - .probe = smp_iic_probe, + .probe = iic_request_IPIs, .kick_cpu = smp_cell_kick_cpu, .setup_cpu = smp_cell_setup_cpu, .cpu_bootable = smp_generic_cpu_bootable, diff --git a/arch/powerpc/platforms/cell/spu_callbacks.c b/arch/powerpc/platforms/cell/spu_callbacks.c index b0ec78e8ad68..a494028b2cdf 100644 --- a/arch/powerpc/platforms/cell/spu_callbacks.c +++ b/arch/powerpc/platforms/cell/spu_callbacks.c @@ -39,6 +39,7 @@ static void *spu_syscall_table[] = { #define PPC_SYS(func) sys_ni_syscall, #define OLDSYS(func) sys_ni_syscall, #define SYS32ONLY(func) sys_ni_syscall, +#define PPC64ONLY(func) sys_ni_syscall, #define SYSX(f, f3264, f32) sys_ni_syscall, #define SYSCALL_SPU(func) sys_##func, diff --git a/arch/powerpc/platforms/chrp/setup.c b/arch/powerpc/platforms/chrp/setup.c index 860a59eb8ea2..15ebc4e8a151 100644 --- a/arch/powerpc/platforms/chrp/setup.c +++ b/arch/powerpc/platforms/chrp/setup.c @@ -253,7 +253,7 @@ static void briq_restart(char *cmd) * But unfortunately, the firmware does not connect /chosen/{stdin,stdout} * the the built-in serial node. Instead, a /failsafe node is created. */ -static void chrp_init_early(void) +static __init void chrp_init_early(void) { struct device_node *node; const char *property; diff --git a/arch/powerpc/platforms/maple/maple.h b/arch/powerpc/platforms/maple/maple.h index c6911ddc479f..eecfa182b06e 100644 --- a/arch/powerpc/platforms/maple/maple.h +++ b/arch/powerpc/platforms/maple/maple.h @@ -10,3 +10,5 @@ extern void maple_calibrate_decr(void); extern void maple_pci_init(void); extern void maple_pci_irq_fixup(struct pci_dev *dev); extern int maple_pci_get_legacy_ide_irq(struct pci_dev *dev, int channel); + +extern struct pci_controller_ops maple_pci_controller_ops; diff --git a/arch/powerpc/platforms/maple/pci.c b/arch/powerpc/platforms/maple/pci.c index d3a13067ec42..a923230e575b 100644 --- a/arch/powerpc/platforms/maple/pci.c +++ b/arch/powerpc/platforms/maple/pci.c @@ -510,6 +510,7 @@ static int __init maple_add_bridge(struct device_node *dev) return -ENOMEM; hose->first_busno = bus_range ? bus_range[0] : 0; hose->last_busno = bus_range ? bus_range[1] : 0xff; + hose->controller_ops = maple_pci_controller_ops; disp_name = NULL; if (of_device_is_compatible(dev, "u3-agp")) { @@ -660,3 +661,6 @@ static void quirk_ipr_msi(struct pci_dev *dev) } DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_OBSIDIAN, quirk_ipr_msi); + +struct pci_controller_ops maple_pci_controller_ops = { +}; diff --git a/arch/powerpc/platforms/maple/setup.c b/arch/powerpc/platforms/maple/setup.c index 56b85cd61aaf..a837188544c8 100644 --- a/arch/powerpc/platforms/maple/setup.c +++ b/arch/powerpc/platforms/maple/setup.c @@ -203,7 +203,7 @@ static void __init maple_init_early(void) { DBG(" -> maple_init_early\n"); - iommu_init_early_dart(); + iommu_init_early_dart(&maple_pci_controller_ops); DBG(" <- maple_init_early\n"); } diff --git a/arch/powerpc/platforms/pasemi/iommu.c b/arch/powerpc/platforms/pasemi/iommu.c index 2e576f2ae442..b8f567b2ea19 100644 --- a/arch/powerpc/platforms/pasemi/iommu.c +++ b/arch/powerpc/platforms/pasemi/iommu.c @@ -27,6 +27,8 @@ #include <asm/machdep.h> #include <asm/firmware.h> +#include "pasemi.h" + #define IOBMAP_PAGE_SHIFT 12 #define IOBMAP_PAGE_SIZE (1 << IOBMAP_PAGE_SHIFT) #define IOBMAP_PAGE_MASK (IOBMAP_PAGE_SIZE - 1) @@ -248,8 +250,8 @@ void __init iommu_init_early_pasemi(void) iob_init(NULL); - ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pasemi; - ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pasemi; + pasemi_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pasemi; + pasemi_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pasemi; ppc_md.tce_build = iobmap_build; ppc_md.tce_free = iobmap_free; set_pci_dma_ops(&dma_iommu_ops); diff --git a/arch/powerpc/platforms/pasemi/pasemi.h b/arch/powerpc/platforms/pasemi/pasemi.h index ea65bf0eb897..11f230a48227 100644 --- a/arch/powerpc/platforms/pasemi/pasemi.h +++ b/arch/powerpc/platforms/pasemi/pasemi.h @@ -30,5 +30,6 @@ static inline void restore_astate(int cpu) } #endif +extern struct pci_controller_ops pasemi_pci_controller_ops; #endif /* _PASEMI_PASEMI_H */ diff --git a/arch/powerpc/platforms/pasemi/pci.c b/arch/powerpc/platforms/pasemi/pci.c index aa862713258c..f3a68a0fef23 100644 --- a/arch/powerpc/platforms/pasemi/pci.c +++ b/arch/powerpc/platforms/pasemi/pci.c @@ -31,6 +31,8 @@ #include <asm/ppc-pci.h> +#include "pasemi.h" + #define PA_PXP_CFA(bus, devfn, off) (((bus) << 20) | ((devfn) << 12) | (off)) static inline int pa_pxp_offset_valid(u8 bus, u8 devfn, int offset) @@ -199,6 +201,7 @@ static int __init pas_add_bridge(struct device_node *dev) hose->first_busno = 0; hose->last_busno = 0xff; + hose->controller_ops = pasemi_pci_controller_ops; setup_pa_pxp(hose); @@ -239,3 +242,5 @@ void __iomem *pasemi_pci_getcfgaddr(struct pci_dev *dev, int offset) return (void __iomem *)pa_pxp_cfg_addr(hose, dev->bus->number, dev->devfn, offset); } + +struct pci_controller_ops pasemi_pci_controller_ops; diff --git a/arch/powerpc/platforms/powermac/bootx_init.c b/arch/powerpc/platforms/powermac/bootx_init.c index 3e91ef538114..76f5013c35e5 100644 --- a/arch/powerpc/platforms/powermac/bootx_init.c +++ b/arch/powerpc/platforms/powermac/bootx_init.c @@ -246,7 +246,7 @@ static void __init bootx_scan_dt_build_strings(unsigned long base, DBG(" detected display ! adding properties names !\n"); bootx_dt_add_string("linux,boot-display", mem_end); bootx_dt_add_string("linux,opened", mem_end); - strncpy(bootx_disp_path, namep, 255); + strlcpy(bootx_disp_path, namep, sizeof(bootx_disp_path)); } /* get and store all property names */ diff --git a/arch/powerpc/platforms/powermac/pci.c b/arch/powerpc/platforms/powermac/pci.c index f4071a67ad00..59ab16fa600f 100644 --- a/arch/powerpc/platforms/powermac/pci.c +++ b/arch/powerpc/platforms/powermac/pci.c @@ -27,6 +27,8 @@ #include <asm/grackle.h> #include <asm/ppc-pci.h> +#include "pmac.h" + #undef DEBUG #ifdef DEBUG @@ -798,6 +800,7 @@ static int __init pmac_add_bridge(struct device_node *dev) return -ENOMEM; hose->first_busno = bus_range ? bus_range[0] : 0; hose->last_busno = bus_range ? bus_range[1] : 0xff; + hose->controller_ops = pmac_pci_controller_ops; disp_name = NULL; @@ -942,7 +945,7 @@ void __init pmac_pci_init(void) } #ifdef CONFIG_PPC32 -int pmac_pci_enable_device_hook(struct pci_dev *dev) +static bool pmac_pci_enable_device_hook(struct pci_dev *dev) { struct device_node* node; int updatecfg = 0; @@ -958,11 +961,11 @@ int pmac_pci_enable_device_hook(struct pci_dev *dev) && !node) { printk(KERN_INFO "Apple USB OHCI %s disabled by firmware\n", pci_name(dev)); - return -EINVAL; + return false; } if (!node) - return 0; + return true; uninorth_child = node->parent && of_device_is_compatible(node->parent, "uni-north"); @@ -1003,7 +1006,7 @@ int pmac_pci_enable_device_hook(struct pci_dev *dev) L1_CACHE_BYTES >> 2); } - return 0; + return true; } void pmac_pci_fixup_ohci(struct pci_dev *dev) @@ -1223,3 +1226,30 @@ static void fixup_u4_pcie(struct pci_dev* dev) pci_write_config_dword(dev, PCI_PREF_MEMORY_BASE, 0); } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_U4_PCIE, fixup_u4_pcie); + +#ifdef CONFIG_PPC64 +static int pmac_pci_probe_mode(struct pci_bus *bus) +{ + struct device_node *node = pci_bus_to_OF_node(bus); + + /* We need to use normal PCI probing for the AGP bus, + * since the device for the AGP bridge isn't in the tree. + * Same for the PCIe host on U4 and the HT host bridge. + */ + if (bus->self == NULL && (of_device_is_compatible(node, "u3-agp") || + of_device_is_compatible(node, "u4-pcie") || + of_device_is_compatible(node, "u3-ht"))) + return PCI_PROBE_NORMAL; + return PCI_PROBE_DEVTREE; +} +#endif /* CONFIG_PPC64 */ + +struct pci_controller_ops pmac_pci_controller_ops = { +#ifdef CONFIG_PPC64 + .probe_mode = pmac_pci_probe_mode, +#endif +#ifdef CONFIG_PPC32 + .enable_device_hook = pmac_pci_enable_device_hook, +#endif +}; + diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c index 4c24bf60d39d..59cfc9d63c2d 100644 --- a/arch/powerpc/platforms/powermac/pic.c +++ b/arch/powerpc/platforms/powermac/pic.c @@ -321,6 +321,9 @@ static void __init pmac_pic_probe_oldstyle(void) max_irqs = max_real_irqs = 64; /* We might have a second cascaded heathrow */ + + /* Compensate for of_node_put() in of_find_node_by_name() */ + of_node_get(master); slave = of_find_node_by_name(master, "mac-io"); /* Check ordering of master & slave */ diff --git a/arch/powerpc/platforms/powermac/pmac.h b/arch/powerpc/platforms/powermac/pmac.h index 8327cce2bdb0..e7f8163d6769 100644 --- a/arch/powerpc/platforms/powermac/pmac.h +++ b/arch/powerpc/platforms/powermac/pmac.h @@ -25,7 +25,6 @@ extern void pmac_pci_init(void); extern void pmac_nvram_update(void); extern unsigned char pmac_nvram_read_byte(int addr); extern void pmac_nvram_write_byte(int addr, unsigned char val); -extern int pmac_pci_enable_device_hook(struct pci_dev *dev); extern void pmac_pcibios_after_init(void); extern int of_show_percpuinfo(struct seq_file *m, int i); @@ -39,4 +38,6 @@ extern void low_cpu_die(void) __attribute__((noreturn)); extern int pmac_nvram_init(void); extern void pmac_pic_init(void); +extern struct pci_controller_ops pmac_pci_controller_ops; + #endif /* __PMAC_H__ */ diff --git a/arch/powerpc/platforms/powermac/setup.c b/arch/powerpc/platforms/powermac/setup.c index 713d36d45d1d..8dd78f4e1af4 100644 --- a/arch/powerpc/platforms/powermac/setup.c +++ b/arch/powerpc/platforms/powermac/setup.c @@ -473,7 +473,7 @@ static void __init pmac_init_early(void) udbg_adb_init(!!strstr(boot_command_line, "btextdbg")); #ifdef CONFIG_PPC64 - iommu_init_early_dart(); + iommu_init_early_dart(&pmac_pci_controller_ops); #endif /* SMP Init has to be done early as we need to patch up @@ -637,24 +637,6 @@ static int __init pmac_probe(void) return 1; } -#ifdef CONFIG_PPC64 -/* Move that to pci.c */ -static int pmac_pci_probe_mode(struct pci_bus *bus) -{ - struct device_node *node = pci_bus_to_OF_node(bus); - - /* We need to use normal PCI probing for the AGP bus, - * since the device for the AGP bridge isn't in the tree. - * Same for the PCIe host on U4 and the HT host bridge. - */ - if (bus->self == NULL && (of_device_is_compatible(node, "u3-agp") || - of_device_is_compatible(node, "u4-pcie") || - of_device_is_compatible(node, "u3-ht"))) - return PCI_PROBE_NORMAL; - return PCI_PROBE_DEVTREE; -} -#endif /* CONFIG_PPC64 */ - define_machine(powermac) { .name = "PowerMac", .probe = pmac_probe, @@ -674,12 +656,10 @@ define_machine(powermac) { .feature_call = pmac_do_feature_call, .progress = udbg_progress, #ifdef CONFIG_PPC64 - .pci_probe_mode = pmac_pci_probe_mode, .power_save = power4_idle, .enable_pmcs = power4_enable_pmcs, #endif /* CONFIG_PPC64 */ #ifdef CONFIG_PPC32 - .pcibios_enable_device_hook = pmac_pci_enable_device_hook, .pcibios_after_init = pmac_pcibios_after_init, .phys_mem_access_prot = pci_phys_mem_access_prot, #endif diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index af094ae03dbb..28a147ca32ba 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -268,14 +268,14 @@ static void __init psurge_quad_init(void) mdelay(33); } -static int __init smp_psurge_probe(void) +static void __init smp_psurge_probe(void) { int i, ncpus; struct device_node *dn; /* We don't do SMP on the PPC601 -- paulus */ if (PVR_VER(mfspr(SPRN_PVR)) == 1) - return 1; + return; /* * The powersurge cpu board can be used in the generation @@ -289,7 +289,7 @@ static int __init smp_psurge_probe(void) */ dn = of_find_node_by_name(NULL, "hammerhead"); if (dn == NULL) - return 1; + return; of_node_put(dn); hhead_base = ioremap(HAMMERHEAD_BASE, 0x800); @@ -310,13 +310,13 @@ static int __init smp_psurge_probe(void) /* not a dual-cpu card */ iounmap(hhead_base); psurge_type = PSURGE_NONE; - return 1; + return; } ncpus = 2; } if (psurge_secondary_ipi_init()) - return 1; + return; psurge_start = ioremap(PSURGE_START, 4); psurge_pri_intr = ioremap(PSURGE_PRI_INTR, 4); @@ -332,8 +332,6 @@ static int __init smp_psurge_probe(void) set_cpu_present(i, true); if (ppc_md.progress) ppc_md.progress("smp_psurge_probe - done", 0x352); - - return ncpus; } static int __init smp_psurge_kick_cpu(int nr) @@ -766,7 +764,7 @@ static void __init smp_core99_setup(int ncpus) powersave_nap = 0; } -static int __init smp_core99_probe(void) +static void __init smp_core99_probe(void) { struct device_node *cpus; int ncpus = 0; @@ -781,7 +779,7 @@ static int __init smp_core99_probe(void) /* Nothing more to do if less than 2 of them */ if (ncpus <= 1) - return 1; + return; /* We need to perform some early initialisations before we can start * setting up SMP as we are running before initcalls @@ -797,8 +795,6 @@ static int __init smp_core99_probe(void) /* Collect l2cr and l3cr values from CPU 0 */ core99_init_caches(0); - - return ncpus; } static int smp_core99_kick_cpu(int nr) diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig index 45a8ed0585cd..4b044d8cb49a 100644 --- a/arch/powerpc/platforms/powernv/Kconfig +++ b/arch/powerpc/platforms/powernv/Kconfig @@ -19,10 +19,3 @@ config PPC_POWERNV select CPU_FREQ_GOV_CONSERVATIVE select PPC_DOORBELL default y - -config PPC_POWERNV_RTAS - depends on PPC_POWERNV - bool "Support for RTAS based PowerNV platforms such as BML" - default y - select PPC_ICS_RTAS - select PPC_RTAS diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile index 6f3c5d33c3af..33e44f37212f 100644 --- a/arch/powerpc/platforms/powernv/Makefile +++ b/arch/powerpc/platforms/powernv/Makefile @@ -5,7 +5,7 @@ obj-y += opal-msglog.o opal-hmi.o opal-power.o obj-$(CONFIG_SMP) += smp.o subcore.o subcore-asm.o obj-$(CONFIG_PCI) += pci.o pci-p5ioc2.o pci-ioda.o -obj-$(CONFIG_EEH) += eeh-ioda.o eeh-powernv.o +obj-$(CONFIG_EEH) += eeh-powernv.o obj-$(CONFIG_PPC_SCOM) += opal-xscom.o obj-$(CONFIG_MEMORY_FAILURE) += opal-memory-errors.o obj-$(CONFIG_TRACEPOINTS) += opal-tracepoints.o diff --git a/arch/powerpc/platforms/powernv/eeh-ioda.c b/arch/powerpc/platforms/powernv/eeh-ioda.c deleted file mode 100644 index 2809c9895288..000000000000 --- a/arch/powerpc/platforms/powernv/eeh-ioda.c +++ /dev/null @@ -1,1149 +0,0 @@ -/* - * The file intends to implement the functions needed by EEH, which is - * built on IODA compliant chip. Actually, lots of functions related - * to EEH would be built based on the OPAL APIs. - * - * Copyright Benjamin Herrenschmidt & Gavin Shan, IBM Corporation 2013. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#include <linux/debugfs.h> -#include <linux/delay.h> -#include <linux/io.h> -#include <linux/irq.h> -#include <linux/kernel.h> -#include <linux/msi.h> -#include <linux/notifier.h> -#include <linux/pci.h> -#include <linux/string.h> - -#include <asm/eeh.h> -#include <asm/eeh_event.h> -#include <asm/io.h> -#include <asm/iommu.h> -#include <asm/msi_bitmap.h> -#include <asm/opal.h> -#include <asm/pci-bridge.h> -#include <asm/ppc-pci.h> -#include <asm/tce.h> - -#include "powernv.h" -#include "pci.h" - -static int ioda_eeh_nb_init = 0; - -static int ioda_eeh_event(struct notifier_block *nb, - unsigned long events, void *change) -{ - uint64_t changed_evts = (uint64_t)change; - - /* - * We simply send special EEH event if EEH has - * been enabled, or clear pending events in - * case that we enable EEH soon - */ - if (!(changed_evts & OPAL_EVENT_PCI_ERROR) || - !(events & OPAL_EVENT_PCI_ERROR)) - return 0; - - if (eeh_enabled()) - eeh_send_failure_event(NULL); - else - opal_notifier_update_evt(OPAL_EVENT_PCI_ERROR, 0x0ul); - - return 0; -} - -static struct notifier_block ioda_eeh_nb = { - .notifier_call = ioda_eeh_event, - .next = NULL, - .priority = 0 -}; - -#ifdef CONFIG_DEBUG_FS -static ssize_t ioda_eeh_ei_write(struct file *filp, - const char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct pci_controller *hose = filp->private_data; - struct pnv_phb *phb = hose->private_data; - struct eeh_dev *edev; - struct eeh_pe *pe; - int pe_no, type, func; - unsigned long addr, mask; - char buf[50]; - int ret; - - if (!phb->eeh_ops || !phb->eeh_ops->err_inject) - return -ENXIO; - - ret = simple_write_to_buffer(buf, sizeof(buf), ppos, user_buf, count); - if (!ret) - return -EFAULT; - - /* Retrieve parameters */ - ret = sscanf(buf, "%x:%x:%x:%lx:%lx", - &pe_no, &type, &func, &addr, &mask); - if (ret != 5) - return -EINVAL; - - /* Retrieve PE */ - edev = kzalloc(sizeof(*edev), GFP_KERNEL); - if (!edev) - return -ENOMEM; - edev->phb = hose; - edev->pe_config_addr = pe_no; - pe = eeh_pe_get(edev); - kfree(edev); - if (!pe) - return -ENODEV; - - /* Do error injection */ - ret = phb->eeh_ops->err_inject(pe, type, func, addr, mask); - return ret < 0 ? ret : count; -} - -static const struct file_operations ioda_eeh_ei_fops = { - .open = simple_open, - .llseek = no_llseek, - .write = ioda_eeh_ei_write, -}; - -static int ioda_eeh_dbgfs_set(void *data, int offset, u64 val) -{ - struct pci_controller *hose = data; - struct pnv_phb *phb = hose->private_data; - - out_be64(phb->regs + offset, val); - return 0; -} - -static int ioda_eeh_dbgfs_get(void *data, int offset, u64 *val) -{ - struct pci_controller *hose = data; - struct pnv_phb *phb = hose->private_data; - - *val = in_be64(phb->regs + offset); - return 0; -} - -static int ioda_eeh_outb_dbgfs_set(void *data, u64 val) -{ - return ioda_eeh_dbgfs_set(data, 0xD10, val); -} - -static int ioda_eeh_outb_dbgfs_get(void *data, u64 *val) -{ - return ioda_eeh_dbgfs_get(data, 0xD10, val); -} - -static int ioda_eeh_inbA_dbgfs_set(void *data, u64 val) -{ - return ioda_eeh_dbgfs_set(data, 0xD90, val); -} - -static int ioda_eeh_inbA_dbgfs_get(void *data, u64 *val) -{ - return ioda_eeh_dbgfs_get(data, 0xD90, val); -} - -static int ioda_eeh_inbB_dbgfs_set(void *data, u64 val) -{ - return ioda_eeh_dbgfs_set(data, 0xE10, val); -} - -static int ioda_eeh_inbB_dbgfs_get(void *data, u64 *val) -{ - return ioda_eeh_dbgfs_get(data, 0xE10, val); -} - -DEFINE_SIMPLE_ATTRIBUTE(ioda_eeh_outb_dbgfs_ops, ioda_eeh_outb_dbgfs_get, - ioda_eeh_outb_dbgfs_set, "0x%llx\n"); -DEFINE_SIMPLE_ATTRIBUTE(ioda_eeh_inbA_dbgfs_ops, ioda_eeh_inbA_dbgfs_get, - ioda_eeh_inbA_dbgfs_set, "0x%llx\n"); -DEFINE_SIMPLE_ATTRIBUTE(ioda_eeh_inbB_dbgfs_ops, ioda_eeh_inbB_dbgfs_get, - ioda_eeh_inbB_dbgfs_set, "0x%llx\n"); -#endif /* CONFIG_DEBUG_FS */ - - -/** - * ioda_eeh_post_init - Chip dependent post initialization - * @hose: PCI controller - * - * The function will be called after eeh PEs and devices - * have been built. That means the EEH is ready to supply - * service with I/O cache. - */ -static int ioda_eeh_post_init(struct pci_controller *hose) -{ - struct pnv_phb *phb = hose->private_data; - int ret; - - /* Register OPAL event notifier */ - if (!ioda_eeh_nb_init) { - ret = opal_notifier_register(&ioda_eeh_nb); - if (ret) { - pr_err("%s: Can't register OPAL event notifier (%d)\n", - __func__, ret); - return ret; - } - - ioda_eeh_nb_init = 1; - } - -#ifdef CONFIG_DEBUG_FS - if (!phb->has_dbgfs && phb->dbgfs) { - phb->has_dbgfs = 1; - - debugfs_create_file("err_injct", 0200, - phb->dbgfs, hose, - &ioda_eeh_ei_fops); - - debugfs_create_file("err_injct_outbound", 0600, - phb->dbgfs, hose, - &ioda_eeh_outb_dbgfs_ops); - debugfs_create_file("err_injct_inboundA", 0600, - phb->dbgfs, hose, - &ioda_eeh_inbA_dbgfs_ops); - debugfs_create_file("err_injct_inboundB", 0600, - phb->dbgfs, hose, - &ioda_eeh_inbB_dbgfs_ops); - } -#endif - - /* If EEH is enabled, we're going to rely on that. - * Otherwise, we restore to conventional mechanism - * to clear frozen PE during PCI config access. - */ - if (eeh_enabled()) - phb->flags |= PNV_PHB_FLAG_EEH; - else - phb->flags &= ~PNV_PHB_FLAG_EEH; - - return 0; -} - -/** - * ioda_eeh_set_option - Set EEH operation or I/O setting - * @pe: EEH PE - * @option: options - * - * Enable or disable EEH option for the indicated PE. The - * function also can be used to enable I/O or DMA for the - * PE. - */ -static int ioda_eeh_set_option(struct eeh_pe *pe, int option) -{ - struct pci_controller *hose = pe->phb; - struct pnv_phb *phb = hose->private_data; - bool freeze_pe = false; - int enable, ret = 0; - s64 rc; - - /* Check on PE number */ - if (pe->addr < 0 || pe->addr >= phb->ioda.total_pe) { - pr_err("%s: PE address %x out of range [0, %x] " - "on PHB#%x\n", - __func__, pe->addr, phb->ioda.total_pe, - hose->global_number); - return -EINVAL; - } - - switch (option) { - case EEH_OPT_DISABLE: - return -EPERM; - case EEH_OPT_ENABLE: - return 0; - case EEH_OPT_THAW_MMIO: - enable = OPAL_EEH_ACTION_CLEAR_FREEZE_MMIO; - break; - case EEH_OPT_THAW_DMA: - enable = OPAL_EEH_ACTION_CLEAR_FREEZE_DMA; - break; - case EEH_OPT_FREEZE_PE: - freeze_pe = true; - enable = OPAL_EEH_ACTION_SET_FREEZE_ALL; - break; - default: - pr_warn("%s: Invalid option %d\n", - __func__, option); - return -EINVAL; - } - - /* If PHB supports compound PE, to handle it */ - if (freeze_pe) { - if (phb->freeze_pe) { - phb->freeze_pe(phb, pe->addr); - } else { - rc = opal_pci_eeh_freeze_set(phb->opal_id, - pe->addr, - enable); - if (rc != OPAL_SUCCESS) { - pr_warn("%s: Failure %lld freezing " - "PHB#%x-PE#%x\n", - __func__, rc, - phb->hose->global_number, pe->addr); - ret = -EIO; - } - } - } else { - if (phb->unfreeze_pe) { - ret = phb->unfreeze_pe(phb, pe->addr, enable); - } else { - rc = opal_pci_eeh_freeze_clear(phb->opal_id, - pe->addr, - enable); - if (rc != OPAL_SUCCESS) { - pr_warn("%s: Failure %lld enable %d " - "for PHB#%x-PE#%x\n", - __func__, rc, option, - phb->hose->global_number, pe->addr); - ret = -EIO; - } - } - } - - return ret; -} - -static void ioda_eeh_phb_diag(struct eeh_pe *pe) -{ - struct pnv_phb *phb = pe->phb->private_data; - long rc; - - rc = opal_pci_get_phb_diag_data2(phb->opal_id, pe->data, - PNV_PCI_DIAG_BUF_SIZE); - if (rc != OPAL_SUCCESS) - pr_warn("%s: Failed to get diag-data for PHB#%x (%ld)\n", - __func__, pe->phb->global_number, rc); -} - -static int ioda_eeh_get_phb_state(struct eeh_pe *pe) -{ - struct pnv_phb *phb = pe->phb->private_data; - u8 fstate; - __be16 pcierr; - s64 rc; - int result = 0; - - rc = opal_pci_eeh_freeze_status(phb->opal_id, - pe->addr, - &fstate, - &pcierr, - NULL); - if (rc != OPAL_SUCCESS) { - pr_warn("%s: Failure %lld getting PHB#%x state\n", - __func__, rc, phb->hose->global_number); - return EEH_STATE_NOT_SUPPORT; - } - - /* - * Check PHB state. If the PHB is frozen for the - * first time, to dump the PHB diag-data. - */ - if (be16_to_cpu(pcierr) != OPAL_EEH_PHB_ERROR) { - result = (EEH_STATE_MMIO_ACTIVE | - EEH_STATE_DMA_ACTIVE | - EEH_STATE_MMIO_ENABLED | - EEH_STATE_DMA_ENABLED); - } else if (!(pe->state & EEH_PE_ISOLATED)) { - eeh_pe_state_mark(pe, EEH_PE_ISOLATED); - ioda_eeh_phb_diag(pe); - - if (eeh_has_flag(EEH_EARLY_DUMP_LOG)) - pnv_pci_dump_phb_diag_data(pe->phb, pe->data); - } - - return result; -} - -static int ioda_eeh_get_pe_state(struct eeh_pe *pe) -{ - struct pnv_phb *phb = pe->phb->private_data; - u8 fstate; - __be16 pcierr; - s64 rc; - int result; - - /* - * We don't clobber hardware frozen state until PE - * reset is completed. In order to keep EEH core - * moving forward, we have to return operational - * state during PE reset. - */ - if (pe->state & EEH_PE_RESET) { - result = (EEH_STATE_MMIO_ACTIVE | - EEH_STATE_DMA_ACTIVE | - EEH_STATE_MMIO_ENABLED | - EEH_STATE_DMA_ENABLED); - return result; - } - - /* - * Fetch PE state from hardware. If the PHB - * supports compound PE, let it handle that. - */ - if (phb->get_pe_state) { - fstate = phb->get_pe_state(phb, pe->addr); - } else { - rc = opal_pci_eeh_freeze_status(phb->opal_id, - pe->addr, - &fstate, - &pcierr, - NULL); - if (rc != OPAL_SUCCESS) { - pr_warn("%s: Failure %lld getting PHB#%x-PE%x state\n", - __func__, rc, phb->hose->global_number, pe->addr); - return EEH_STATE_NOT_SUPPORT; - } - } - - /* Figure out state */ - switch (fstate) { - case OPAL_EEH_STOPPED_NOT_FROZEN: - result = (EEH_STATE_MMIO_ACTIVE | - EEH_STATE_DMA_ACTIVE | - EEH_STATE_MMIO_ENABLED | - EEH_STATE_DMA_ENABLED); - break; - case OPAL_EEH_STOPPED_MMIO_FREEZE: - result = (EEH_STATE_DMA_ACTIVE | - EEH_STATE_DMA_ENABLED); - break; - case OPAL_EEH_STOPPED_DMA_FREEZE: - result = (EEH_STATE_MMIO_ACTIVE | - EEH_STATE_MMIO_ENABLED); - break; - case OPAL_EEH_STOPPED_MMIO_DMA_FREEZE: - result = 0; - break; - case OPAL_EEH_STOPPED_RESET: - result = EEH_STATE_RESET_ACTIVE; - break; - case OPAL_EEH_STOPPED_TEMP_UNAVAIL: - result = EEH_STATE_UNAVAILABLE; - break; - case OPAL_EEH_STOPPED_PERM_UNAVAIL: - result = EEH_STATE_NOT_SUPPORT; - break; - default: - result = EEH_STATE_NOT_SUPPORT; - pr_warn("%s: Invalid PHB#%x-PE#%x state %x\n", - __func__, phb->hose->global_number, - pe->addr, fstate); - } - - /* - * If PHB supports compound PE, to freeze all - * slave PEs for consistency. - * - * If the PE is switching to frozen state for the - * first time, to dump the PHB diag-data. - */ - if (!(result & EEH_STATE_NOT_SUPPORT) && - !(result & EEH_STATE_UNAVAILABLE) && - !(result & EEH_STATE_MMIO_ACTIVE) && - !(result & EEH_STATE_DMA_ACTIVE) && - !(pe->state & EEH_PE_ISOLATED)) { - if (phb->freeze_pe) - phb->freeze_pe(phb, pe->addr); - - eeh_pe_state_mark(pe, EEH_PE_ISOLATED); - ioda_eeh_phb_diag(pe); - - if (eeh_has_flag(EEH_EARLY_DUMP_LOG)) - pnv_pci_dump_phb_diag_data(pe->phb, pe->data); - } - - return result; -} - -/** - * ioda_eeh_get_state - Retrieve the state of PE - * @pe: EEH PE - * - * The PE's state should be retrieved from the PEEV, PEST - * IODA tables. Since the OPAL has exported the function - * to do it, it'd better to use that. - */ -static int ioda_eeh_get_state(struct eeh_pe *pe) -{ - struct pnv_phb *phb = pe->phb->private_data; - - /* Sanity check on PE number. PHB PE should have 0 */ - if (pe->addr < 0 || - pe->addr >= phb->ioda.total_pe) { - pr_warn("%s: PHB#%x-PE#%x out of range [0, %x]\n", - __func__, phb->hose->global_number, - pe->addr, phb->ioda.total_pe); - return EEH_STATE_NOT_SUPPORT; - } - - if (pe->type & EEH_PE_PHB) - return ioda_eeh_get_phb_state(pe); - - return ioda_eeh_get_pe_state(pe); -} - -static s64 ioda_eeh_phb_poll(struct pnv_phb *phb) -{ - s64 rc = OPAL_HARDWARE; - - while (1) { - rc = opal_pci_poll(phb->opal_id); - if (rc <= 0) - break; - - if (system_state < SYSTEM_RUNNING) - udelay(1000 * rc); - else - msleep(rc); - } - - return rc; -} - -int ioda_eeh_phb_reset(struct pci_controller *hose, int option) -{ - struct pnv_phb *phb = hose->private_data; - s64 rc = OPAL_HARDWARE; - - pr_debug("%s: Reset PHB#%x, option=%d\n", - __func__, hose->global_number, option); - - /* Issue PHB complete reset request */ - if (option == EEH_RESET_FUNDAMENTAL || - option == EEH_RESET_HOT) - rc = opal_pci_reset(phb->opal_id, - OPAL_RESET_PHB_COMPLETE, - OPAL_ASSERT_RESET); - else if (option == EEH_RESET_DEACTIVATE) - rc = opal_pci_reset(phb->opal_id, - OPAL_RESET_PHB_COMPLETE, - OPAL_DEASSERT_RESET); - if (rc < 0) - goto out; - - /* - * Poll state of the PHB until the request is done - * successfully. The PHB reset is usually PHB complete - * reset followed by hot reset on root bus. So we also - * need the PCI bus settlement delay. - */ - rc = ioda_eeh_phb_poll(phb); - if (option == EEH_RESET_DEACTIVATE) { - if (system_state < SYSTEM_RUNNING) - udelay(1000 * EEH_PE_RST_SETTLE_TIME); - else - msleep(EEH_PE_RST_SETTLE_TIME); - } -out: - if (rc != OPAL_SUCCESS) - return -EIO; - - return 0; -} - -static int ioda_eeh_root_reset(struct pci_controller *hose, int option) -{ - struct pnv_phb *phb = hose->private_data; - s64 rc = OPAL_SUCCESS; - - pr_debug("%s: Reset PHB#%x, option=%d\n", - __func__, hose->global_number, option); - - /* - * During the reset deassert time, we needn't care - * the reset scope because the firmware does nothing - * for fundamental or hot reset during deassert phase. - */ - if (option == EEH_RESET_FUNDAMENTAL) - rc = opal_pci_reset(phb->opal_id, - OPAL_RESET_PCI_FUNDAMENTAL, - OPAL_ASSERT_RESET); - else if (option == EEH_RESET_HOT) - rc = opal_pci_reset(phb->opal_id, - OPAL_RESET_PCI_HOT, - OPAL_ASSERT_RESET); - else if (option == EEH_RESET_DEACTIVATE) - rc = opal_pci_reset(phb->opal_id, - OPAL_RESET_PCI_HOT, - OPAL_DEASSERT_RESET); - if (rc < 0) - goto out; - - /* Poll state of the PHB until the request is done */ - rc = ioda_eeh_phb_poll(phb); - if (option == EEH_RESET_DEACTIVATE) - msleep(EEH_PE_RST_SETTLE_TIME); -out: - if (rc != OPAL_SUCCESS) - return -EIO; - - return 0; -} - -static int ioda_eeh_bridge_reset(struct pci_dev *dev, int option) - -{ - struct device_node *dn = pci_device_to_OF_node(dev); - struct eeh_dev *edev = of_node_to_eeh_dev(dn); - int aer = edev ? edev->aer_cap : 0; - u32 ctrl; - - pr_debug("%s: Reset PCI bus %04x:%02x with option %d\n", - __func__, pci_domain_nr(dev->bus), - dev->bus->number, option); - - switch (option) { - case EEH_RESET_FUNDAMENTAL: - case EEH_RESET_HOT: - /* Don't report linkDown event */ - if (aer) { - eeh_ops->read_config(dn, aer + PCI_ERR_UNCOR_MASK, - 4, &ctrl); - ctrl |= PCI_ERR_UNC_SURPDN; - eeh_ops->write_config(dn, aer + PCI_ERR_UNCOR_MASK, - 4, ctrl); - } - - eeh_ops->read_config(dn, PCI_BRIDGE_CONTROL, 2, &ctrl); - ctrl |= PCI_BRIDGE_CTL_BUS_RESET; - eeh_ops->write_config(dn, PCI_BRIDGE_CONTROL, 2, ctrl); - msleep(EEH_PE_RST_HOLD_TIME); - - break; - case EEH_RESET_DEACTIVATE: - eeh_ops->read_config(dn, PCI_BRIDGE_CONTROL, 2, &ctrl); - ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; - eeh_ops->write_config(dn, PCI_BRIDGE_CONTROL, 2, ctrl); - msleep(EEH_PE_RST_SETTLE_TIME); - - /* Continue reporting linkDown event */ - if (aer) { - eeh_ops->read_config(dn, aer + PCI_ERR_UNCOR_MASK, - 4, &ctrl); - ctrl &= ~PCI_ERR_UNC_SURPDN; - eeh_ops->write_config(dn, aer + PCI_ERR_UNCOR_MASK, - 4, ctrl); - } - - break; - } - - return 0; -} - -void pnv_pci_reset_secondary_bus(struct pci_dev *dev) -{ - struct pci_controller *hose; - - if (pci_is_root_bus(dev->bus)) { - hose = pci_bus_to_host(dev->bus); - ioda_eeh_root_reset(hose, EEH_RESET_HOT); - ioda_eeh_root_reset(hose, EEH_RESET_DEACTIVATE); - } else { - ioda_eeh_bridge_reset(dev, EEH_RESET_HOT); - ioda_eeh_bridge_reset(dev, EEH_RESET_DEACTIVATE); - } -} - -/** - * ioda_eeh_reset - Reset the indicated PE - * @pe: EEH PE - * @option: reset option - * - * Do reset on the indicated PE. For PCI bus sensitive PE, - * we need to reset the parent p2p bridge. The PHB has to - * be reinitialized if the p2p bridge is root bridge. For - * PCI device sensitive PE, we will try to reset the device - * through FLR. For now, we don't have OPAL APIs to do HARD - * reset yet, so all reset would be SOFT (HOT) reset. - */ -static int ioda_eeh_reset(struct eeh_pe *pe, int option) -{ - struct pci_controller *hose = pe->phb; - struct pci_bus *bus; - int ret; - - /* - * For PHB reset, we always have complete reset. For those PEs whose - * primary bus derived from root complex (root bus) or root port - * (usually bus#1), we apply hot or fundamental reset on the root port. - * For other PEs, we always have hot reset on the PE primary bus. - * - * Here, we have different design to pHyp, which always clear the - * frozen state during PE reset. However, the good idea here from - * benh is to keep frozen state before we get PE reset done completely - * (until BAR restore). With the frozen state, HW drops illegal IO - * or MMIO access, which can incur recrusive frozen PE during PE - * reset. The side effect is that EEH core has to clear the frozen - * state explicitly after BAR restore. - */ - if (pe->type & EEH_PE_PHB) { - ret = ioda_eeh_phb_reset(hose, option); - } else { - struct pnv_phb *phb; - s64 rc; - - /* - * The frozen PE might be caused by PAPR error injection - * registers, which are expected to be cleared after hitting - * frozen PE as stated in the hardware spec. Unfortunately, - * that's not true on P7IOC. So we have to clear it manually - * to avoid recursive EEH errors during recovery. - */ - phb = hose->private_data; - if (phb->model == PNV_PHB_MODEL_P7IOC && - (option == EEH_RESET_HOT || - option == EEH_RESET_FUNDAMENTAL)) { - rc = opal_pci_reset(phb->opal_id, - OPAL_RESET_PHB_ERROR, - OPAL_ASSERT_RESET); - if (rc != OPAL_SUCCESS) { - pr_warn("%s: Failure %lld clearing " - "error injection registers\n", - __func__, rc); - return -EIO; - } - } - - bus = eeh_pe_bus_get(pe); - if (pci_is_root_bus(bus) || - pci_is_root_bus(bus->parent)) - ret = ioda_eeh_root_reset(hose, option); - else - ret = ioda_eeh_bridge_reset(bus->self, option); - } - - return ret; -} - -/** - * ioda_eeh_get_log - Retrieve error log - * @pe: frozen PE - * @severity: permanent or temporary error - * @drv_log: device driver log - * @len: length of device driver log - * - * Retrieve error log, which contains log from device driver - * and firmware. - */ -static int ioda_eeh_get_log(struct eeh_pe *pe, int severity, - char *drv_log, unsigned long len) -{ - if (!eeh_has_flag(EEH_EARLY_DUMP_LOG)) - pnv_pci_dump_phb_diag_data(pe->phb, pe->data); - - return 0; -} - -/** - * ioda_eeh_configure_bridge - Configure the PCI bridges for the indicated PE - * @pe: EEH PE - * - * For particular PE, it might have included PCI bridges. In order - * to make the PE work properly, those PCI bridges should be configured - * correctly. However, we need do nothing on P7IOC since the reset - * function will do everything that should be covered by the function. - */ -static int ioda_eeh_configure_bridge(struct eeh_pe *pe) -{ - return 0; -} - -static int ioda_eeh_err_inject(struct eeh_pe *pe, int type, int func, - unsigned long addr, unsigned long mask) -{ - struct pci_controller *hose = pe->phb; - struct pnv_phb *phb = hose->private_data; - s64 ret; - - /* Sanity check on error type */ - if (type != OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR && - type != OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR64) { - pr_warn("%s: Invalid error type %d\n", - __func__, type); - return -ERANGE; - } - - if (func < OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_ADDR || - func > OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_TARGET) { - pr_warn("%s: Invalid error function %d\n", - __func__, func); - return -ERANGE; - } - - /* Firmware supports error injection ? */ - if (!opal_check_token(OPAL_PCI_ERR_INJECT)) { - pr_warn("%s: Firmware doesn't support error injection\n", - __func__); - return -ENXIO; - } - - /* Do error injection */ - ret = opal_pci_err_inject(phb->opal_id, pe->addr, - type, func, addr, mask); - if (ret != OPAL_SUCCESS) { - pr_warn("%s: Failure %lld injecting error " - "%d-%d to PHB#%x-PE#%x\n", - __func__, ret, type, func, - hose->global_number, pe->addr); - return -EIO; - } - - return 0; -} - -static void ioda_eeh_hub_diag_common(struct OpalIoP7IOCErrorData *data) -{ - /* GEM */ - if (data->gemXfir || data->gemRfir || - data->gemRirqfir || data->gemMask || data->gemRwof) - pr_info(" GEM: %016llx %016llx %016llx %016llx %016llx\n", - be64_to_cpu(data->gemXfir), - be64_to_cpu(data->gemRfir), - be64_to_cpu(data->gemRirqfir), - be64_to_cpu(data->gemMask), - be64_to_cpu(data->gemRwof)); - - /* LEM */ - if (data->lemFir || data->lemErrMask || - data->lemAction0 || data->lemAction1 || data->lemWof) - pr_info(" LEM: %016llx %016llx %016llx %016llx %016llx\n", - be64_to_cpu(data->lemFir), - be64_to_cpu(data->lemErrMask), - be64_to_cpu(data->lemAction0), - be64_to_cpu(data->lemAction1), - be64_to_cpu(data->lemWof)); -} - -static void ioda_eeh_hub_diag(struct pci_controller *hose) -{ - struct pnv_phb *phb = hose->private_data; - struct OpalIoP7IOCErrorData *data = &phb->diag.hub_diag; - long rc; - - rc = opal_pci_get_hub_diag_data(phb->hub_id, data, sizeof(*data)); - if (rc != OPAL_SUCCESS) { - pr_warn("%s: Failed to get HUB#%llx diag-data (%ld)\n", - __func__, phb->hub_id, rc); - return; - } - - switch (data->type) { - case OPAL_P7IOC_DIAG_TYPE_RGC: - pr_info("P7IOC diag-data for RGC\n\n"); - ioda_eeh_hub_diag_common(data); - if (data->rgc.rgcStatus || data->rgc.rgcLdcp) - pr_info(" RGC: %016llx %016llx\n", - be64_to_cpu(data->rgc.rgcStatus), - be64_to_cpu(data->rgc.rgcLdcp)); - break; - case OPAL_P7IOC_DIAG_TYPE_BI: - pr_info("P7IOC diag-data for BI %s\n\n", - data->bi.biDownbound ? "Downbound" : "Upbound"); - ioda_eeh_hub_diag_common(data); - if (data->bi.biLdcp0 || data->bi.biLdcp1 || - data->bi.biLdcp2 || data->bi.biFenceStatus) - pr_info(" BI: %016llx %016llx %016llx %016llx\n", - be64_to_cpu(data->bi.biLdcp0), - be64_to_cpu(data->bi.biLdcp1), - be64_to_cpu(data->bi.biLdcp2), - be64_to_cpu(data->bi.biFenceStatus)); - break; - case OPAL_P7IOC_DIAG_TYPE_CI: - pr_info("P7IOC diag-data for CI Port %d\n\n", - data->ci.ciPort); - ioda_eeh_hub_diag_common(data); - if (data->ci.ciPortStatus || data->ci.ciPortLdcp) - pr_info(" CI: %016llx %016llx\n", - be64_to_cpu(data->ci.ciPortStatus), - be64_to_cpu(data->ci.ciPortLdcp)); - break; - case OPAL_P7IOC_DIAG_TYPE_MISC: - pr_info("P7IOC diag-data for MISC\n\n"); - ioda_eeh_hub_diag_common(data); - break; - case OPAL_P7IOC_DIAG_TYPE_I2C: - pr_info("P7IOC diag-data for I2C\n\n"); - ioda_eeh_hub_diag_common(data); - break; - default: - pr_warn("%s: Invalid type of HUB#%llx diag-data (%d)\n", - __func__, phb->hub_id, data->type); - } -} - -static int ioda_eeh_get_pe(struct pci_controller *hose, - u16 pe_no, struct eeh_pe **pe) -{ - struct pnv_phb *phb = hose->private_data; - struct pnv_ioda_pe *pnv_pe; - struct eeh_pe *dev_pe; - struct eeh_dev edev; - - /* - * If PHB supports compound PE, to fetch - * the master PE because slave PE is invisible - * to EEH core. - */ - pnv_pe = &phb->ioda.pe_array[pe_no]; - if (pnv_pe->flags & PNV_IODA_PE_SLAVE) { - pnv_pe = pnv_pe->master; - WARN_ON(!pnv_pe || - !(pnv_pe->flags & PNV_IODA_PE_MASTER)); - pe_no = pnv_pe->pe_number; - } - - /* Find the PE according to PE# */ - memset(&edev, 0, sizeof(struct eeh_dev)); - edev.phb = hose; - edev.pe_config_addr = pe_no; - dev_pe = eeh_pe_get(&edev); - if (!dev_pe) - return -EEXIST; - - /* Freeze the (compound) PE */ - *pe = dev_pe; - if (!(dev_pe->state & EEH_PE_ISOLATED)) - phb->freeze_pe(phb, pe_no); - - /* - * At this point, we're sure the (compound) PE should - * have been frozen. However, we still need poke until - * hitting the frozen PE on top level. - */ - dev_pe = dev_pe->parent; - while (dev_pe && !(dev_pe->type & EEH_PE_PHB)) { - int ret; - int active_flags = (EEH_STATE_MMIO_ACTIVE | - EEH_STATE_DMA_ACTIVE); - - ret = eeh_ops->get_state(dev_pe, NULL); - if (ret <= 0 || (ret & active_flags) == active_flags) { - dev_pe = dev_pe->parent; - continue; - } - - /* Frozen parent PE */ - *pe = dev_pe; - if (!(dev_pe->state & EEH_PE_ISOLATED)) - phb->freeze_pe(phb, dev_pe->addr); - - /* Next one */ - dev_pe = dev_pe->parent; - } - - return 0; -} - -/** - * ioda_eeh_next_error - Retrieve next error for EEH core to handle - * @pe: The affected PE - * - * The function is expected to be called by EEH core while it gets - * special EEH event (without binding PE). The function calls to - * OPAL APIs for next error to handle. The informational error is - * handled internally by platform. However, the dead IOC, dead PHB, - * fenced PHB and frozen PE should be handled by EEH core eventually. - */ -static int ioda_eeh_next_error(struct eeh_pe **pe) -{ - struct pci_controller *hose; - struct pnv_phb *phb; - struct eeh_pe *phb_pe, *parent_pe; - __be64 frozen_pe_no; - __be16 err_type, severity; - int active_flags = (EEH_STATE_MMIO_ACTIVE | EEH_STATE_DMA_ACTIVE); - long rc; - int state, ret = EEH_NEXT_ERR_NONE; - - /* - * While running here, it's safe to purge the event queue. - * And we should keep the cached OPAL notifier event sychronized - * between the kernel and firmware. - */ - eeh_remove_event(NULL, false); - opal_notifier_update_evt(OPAL_EVENT_PCI_ERROR, 0x0ul); - - list_for_each_entry(hose, &hose_list, list_node) { - /* - * If the subordinate PCI buses of the PHB has been - * removed or is exactly under error recovery, we - * needn't take care of it any more. - */ - phb = hose->private_data; - phb_pe = eeh_phb_pe_get(hose); - if (!phb_pe || (phb_pe->state & EEH_PE_ISOLATED)) - continue; - - rc = opal_pci_next_error(phb->opal_id, - &frozen_pe_no, &err_type, &severity); - - /* If OPAL API returns error, we needn't proceed */ - if (rc != OPAL_SUCCESS) { - pr_devel("%s: Invalid return value on " - "PHB#%x (0x%lx) from opal_pci_next_error", - __func__, hose->global_number, rc); - continue; - } - - /* If the PHB doesn't have error, stop processing */ - if (be16_to_cpu(err_type) == OPAL_EEH_NO_ERROR || - be16_to_cpu(severity) == OPAL_EEH_SEV_NO_ERROR) { - pr_devel("%s: No error found on PHB#%x\n", - __func__, hose->global_number); - continue; - } - - /* - * Processing the error. We're expecting the error with - * highest priority reported upon multiple errors on the - * specific PHB. - */ - pr_devel("%s: Error (%d, %d, %llu) on PHB#%x\n", - __func__, be16_to_cpu(err_type), be16_to_cpu(severity), - be64_to_cpu(frozen_pe_no), hose->global_number); - switch (be16_to_cpu(err_type)) { - case OPAL_EEH_IOC_ERROR: - if (be16_to_cpu(severity) == OPAL_EEH_SEV_IOC_DEAD) { - pr_err("EEH: dead IOC detected\n"); - ret = EEH_NEXT_ERR_DEAD_IOC; - } else if (be16_to_cpu(severity) == OPAL_EEH_SEV_INF) { - pr_info("EEH: IOC informative error " - "detected\n"); - ioda_eeh_hub_diag(hose); - ret = EEH_NEXT_ERR_NONE; - } - - break; - case OPAL_EEH_PHB_ERROR: - if (be16_to_cpu(severity) == OPAL_EEH_SEV_PHB_DEAD) { - *pe = phb_pe; - pr_err("EEH: dead PHB#%x detected, " - "location: %s\n", - hose->global_number, - eeh_pe_loc_get(phb_pe)); - ret = EEH_NEXT_ERR_DEAD_PHB; - } else if (be16_to_cpu(severity) == - OPAL_EEH_SEV_PHB_FENCED) { - *pe = phb_pe; - pr_err("EEH: Fenced PHB#%x detected, " - "location: %s\n", - hose->global_number, - eeh_pe_loc_get(phb_pe)); - ret = EEH_NEXT_ERR_FENCED_PHB; - } else if (be16_to_cpu(severity) == OPAL_EEH_SEV_INF) { - pr_info("EEH: PHB#%x informative error " - "detected, location: %s\n", - hose->global_number, - eeh_pe_loc_get(phb_pe)); - ioda_eeh_phb_diag(phb_pe); - pnv_pci_dump_phb_diag_data(hose, phb_pe->data); - ret = EEH_NEXT_ERR_NONE; - } - - break; - case OPAL_EEH_PE_ERROR: - /* - * If we can't find the corresponding PE, we - * just try to unfreeze. - */ - if (ioda_eeh_get_pe(hose, - be64_to_cpu(frozen_pe_no), pe)) { - /* Try best to clear it */ - pr_info("EEH: Clear non-existing PHB#%x-PE#%llx\n", - hose->global_number, frozen_pe_no); - pr_info("EEH: PHB location: %s\n", - eeh_pe_loc_get(phb_pe)); - opal_pci_eeh_freeze_clear(phb->opal_id, frozen_pe_no, - OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); - ret = EEH_NEXT_ERR_NONE; - } else if ((*pe)->state & EEH_PE_ISOLATED || - eeh_pe_passed(*pe)) { - ret = EEH_NEXT_ERR_NONE; - } else { - pr_err("EEH: Frozen PE#%x on PHB#%x detected\n", - (*pe)->addr, (*pe)->phb->global_number); - pr_err("EEH: PE location: %s, PHB location: %s\n", - eeh_pe_loc_get(*pe), eeh_pe_loc_get(phb_pe)); - ret = EEH_NEXT_ERR_FROZEN_PE; - } - - break; - default: - pr_warn("%s: Unexpected error type %d\n", - __func__, be16_to_cpu(err_type)); - } - - /* - * EEH core will try recover from fenced PHB or - * frozen PE. In the time for frozen PE, EEH core - * enable IO path for that before collecting logs, - * but it ruins the site. So we have to dump the - * log in advance here. - */ - if ((ret == EEH_NEXT_ERR_FROZEN_PE || - ret == EEH_NEXT_ERR_FENCED_PHB) && - !((*pe)->state & EEH_PE_ISOLATED)) { - eeh_pe_state_mark(*pe, EEH_PE_ISOLATED); - ioda_eeh_phb_diag(*pe); - - if (eeh_has_flag(EEH_EARLY_DUMP_LOG)) - pnv_pci_dump_phb_diag_data((*pe)->phb, - (*pe)->data); - } - - /* - * We probably have the frozen parent PE out there and - * we need have to handle frozen parent PE firstly. - */ - if (ret == EEH_NEXT_ERR_FROZEN_PE) { - parent_pe = (*pe)->parent; - while (parent_pe) { - /* Hit the ceiling ? */ - if (parent_pe->type & EEH_PE_PHB) - break; - - /* Frozen parent PE ? */ - state = ioda_eeh_get_state(parent_pe); - if (state > 0 && - (state & active_flags) != active_flags) - *pe = parent_pe; - - /* Next parent level */ - parent_pe = parent_pe->parent; - } - - /* We possibly migrate to another PE */ - eeh_pe_state_mark(*pe, EEH_PE_ISOLATED); - } - - /* - * If we have no errors on the specific PHB or only - * informative error there, we continue poking it. - * Otherwise, we need actions to be taken by upper - * layer. - */ - if (ret > EEH_NEXT_ERR_INF) - break; - } - - return ret; -} - -struct pnv_eeh_ops ioda_eeh_ops = { - .post_init = ioda_eeh_post_init, - .set_option = ioda_eeh_set_option, - .get_state = ioda_eeh_get_state, - .reset = ioda_eeh_reset, - .get_log = ioda_eeh_get_log, - .configure_bridge = ioda_eeh_configure_bridge, - .err_inject = ioda_eeh_err_inject, - .next_error = ioda_eeh_next_error -}; diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c index e261869adc86..ce738ab3d5a9 100644 --- a/arch/powerpc/platforms/powernv/eeh-powernv.c +++ b/arch/powerpc/platforms/powernv/eeh-powernv.c @@ -12,6 +12,7 @@ */ #include <linux/atomic.h> +#include <linux/debugfs.h> #include <linux/delay.h> #include <linux/export.h> #include <linux/init.h> @@ -38,12 +39,14 @@ #include "powernv.h" #include "pci.h" +static bool pnv_eeh_nb_init = false; + /** - * powernv_eeh_init - EEH platform dependent initialization + * pnv_eeh_init - EEH platform dependent initialization * * EEH platform dependent initialization on powernv */ -static int powernv_eeh_init(void) +static int pnv_eeh_init(void) { struct pci_controller *hose; struct pnv_phb *phb; @@ -85,37 +88,280 @@ static int powernv_eeh_init(void) return 0; } +static int pnv_eeh_event(struct notifier_block *nb, + unsigned long events, void *change) +{ + uint64_t changed_evts = (uint64_t)change; + + /* + * We simply send special EEH event if EEH has + * been enabled, or clear pending events in + * case that we enable EEH soon + */ + if (!(changed_evts & OPAL_EVENT_PCI_ERROR) || + !(events & OPAL_EVENT_PCI_ERROR)) + return 0; + + if (eeh_enabled()) + eeh_send_failure_event(NULL); + else + opal_notifier_update_evt(OPAL_EVENT_PCI_ERROR, 0x0ul); + + return 0; +} + +static struct notifier_block pnv_eeh_nb = { + .notifier_call = pnv_eeh_event, + .next = NULL, + .priority = 0 +}; + +#ifdef CONFIG_DEBUG_FS +static ssize_t pnv_eeh_ei_write(struct file *filp, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct pci_controller *hose = filp->private_data; + struct eeh_dev *edev; + struct eeh_pe *pe; + int pe_no, type, func; + unsigned long addr, mask; + char buf[50]; + int ret; + + if (!eeh_ops || !eeh_ops->err_inject) + return -ENXIO; + + /* Copy over argument buffer */ + ret = simple_write_to_buffer(buf, sizeof(buf), ppos, user_buf, count); + if (!ret) + return -EFAULT; + + /* Retrieve parameters */ + ret = sscanf(buf, "%x:%x:%x:%lx:%lx", + &pe_no, &type, &func, &addr, &mask); + if (ret != 5) + return -EINVAL; + + /* Retrieve PE */ + edev = kzalloc(sizeof(*edev), GFP_KERNEL); + if (!edev) + return -ENOMEM; + edev->phb = hose; + edev->pe_config_addr = pe_no; + pe = eeh_pe_get(edev); + kfree(edev); + if (!pe) + return -ENODEV; + + /* Do error injection */ + ret = eeh_ops->err_inject(pe, type, func, addr, mask); + return ret < 0 ? ret : count; +} + +static const struct file_operations pnv_eeh_ei_fops = { + .open = simple_open, + .llseek = no_llseek, + .write = pnv_eeh_ei_write, +}; + +static int pnv_eeh_dbgfs_set(void *data, int offset, u64 val) +{ + struct pci_controller *hose = data; + struct pnv_phb *phb = hose->private_data; + + out_be64(phb->regs + offset, val); + return 0; +} + +static int pnv_eeh_dbgfs_get(void *data, int offset, u64 *val) +{ + struct pci_controller *hose = data; + struct pnv_phb *phb = hose->private_data; + + *val = in_be64(phb->regs + offset); + return 0; +} + +static int pnv_eeh_outb_dbgfs_set(void *data, u64 val) +{ + return pnv_eeh_dbgfs_set(data, 0xD10, val); +} + +static int pnv_eeh_outb_dbgfs_get(void *data, u64 *val) +{ + return pnv_eeh_dbgfs_get(data, 0xD10, val); +} + +static int pnv_eeh_inbA_dbgfs_set(void *data, u64 val) +{ + return pnv_eeh_dbgfs_set(data, 0xD90, val); +} + +static int pnv_eeh_inbA_dbgfs_get(void *data, u64 *val) +{ + return pnv_eeh_dbgfs_get(data, 0xD90, val); +} + +static int pnv_eeh_inbB_dbgfs_set(void *data, u64 val) +{ + return pnv_eeh_dbgfs_set(data, 0xE10, val); +} + +static int pnv_eeh_inbB_dbgfs_get(void *data, u64 *val) +{ + return pnv_eeh_dbgfs_get(data, 0xE10, val); +} + +DEFINE_SIMPLE_ATTRIBUTE(pnv_eeh_outb_dbgfs_ops, pnv_eeh_outb_dbgfs_get, + pnv_eeh_outb_dbgfs_set, "0x%llx\n"); +DEFINE_SIMPLE_ATTRIBUTE(pnv_eeh_inbA_dbgfs_ops, pnv_eeh_inbA_dbgfs_get, + pnv_eeh_inbA_dbgfs_set, "0x%llx\n"); +DEFINE_SIMPLE_ATTRIBUTE(pnv_eeh_inbB_dbgfs_ops, pnv_eeh_inbB_dbgfs_get, + pnv_eeh_inbB_dbgfs_set, "0x%llx\n"); +#endif /* CONFIG_DEBUG_FS */ + /** - * powernv_eeh_post_init - EEH platform dependent post initialization + * pnv_eeh_post_init - EEH platform dependent post initialization * * EEH platform dependent post initialization on powernv. When * the function is called, the EEH PEs and devices should have * been built. If the I/O cache staff has been built, EEH is * ready to supply service. */ -static int powernv_eeh_post_init(void) +static int pnv_eeh_post_init(void) { struct pci_controller *hose; struct pnv_phb *phb; int ret = 0; + /* Register OPAL event notifier */ + if (!pnv_eeh_nb_init) { + ret = opal_notifier_register(&pnv_eeh_nb); + if (ret) { + pr_warn("%s: Can't register OPAL event notifier (%d)\n", + __func__, ret); + return ret; + } + + pnv_eeh_nb_init = true; + } + list_for_each_entry(hose, &hose_list, list_node) { phb = hose->private_data; - if (phb->eeh_ops && phb->eeh_ops->post_init) { - ret = phb->eeh_ops->post_init(hose); - if (ret) - break; - } + /* + * If EEH is enabled, we're going to rely on that. + * Otherwise, we restore to conventional mechanism + * to clear frozen PE during PCI config access. + */ + if (eeh_enabled()) + phb->flags |= PNV_PHB_FLAG_EEH; + else + phb->flags &= ~PNV_PHB_FLAG_EEH; + + /* Create debugfs entries */ +#ifdef CONFIG_DEBUG_FS + if (phb->has_dbgfs || !phb->dbgfs) + continue; + + phb->has_dbgfs = 1; + debugfs_create_file("err_injct", 0200, + phb->dbgfs, hose, + &pnv_eeh_ei_fops); + + debugfs_create_file("err_injct_outbound", 0600, + phb->dbgfs, hose, + &pnv_eeh_outb_dbgfs_ops); + debugfs_create_file("err_injct_inboundA", 0600, + phb->dbgfs, hose, + &pnv_eeh_inbA_dbgfs_ops); + debugfs_create_file("err_injct_inboundB", 0600, + phb->dbgfs, hose, + &pnv_eeh_inbB_dbgfs_ops); +#endif /* CONFIG_DEBUG_FS */ } + return ret; } +static int pnv_eeh_cap_start(struct pci_dn *pdn) +{ + u32 status; + + if (!pdn) + return 0; + + pnv_pci_cfg_read(pdn, PCI_STATUS, 2, &status); + if (!(status & PCI_STATUS_CAP_LIST)) + return 0; + + return PCI_CAPABILITY_LIST; +} + +static int pnv_eeh_find_cap(struct pci_dn *pdn, int cap) +{ + int pos = pnv_eeh_cap_start(pdn); + int cnt = 48; /* Maximal number of capabilities */ + u32 id; + + if (!pos) + return 0; + + while (cnt--) { + pnv_pci_cfg_read(pdn, pos, 1, &pos); + if (pos < 0x40) + break; + + pos &= ~3; + pnv_pci_cfg_read(pdn, pos + PCI_CAP_LIST_ID, 1, &id); + if (id == 0xff) + break; + + /* Found */ + if (id == cap) + return pos; + + /* Next one */ + pos += PCI_CAP_LIST_NEXT; + } + + return 0; +} + +static int pnv_eeh_find_ecap(struct pci_dn *pdn, int cap) +{ + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); + u32 header; + int pos = 256, ttl = (4096 - 256) / 8; + + if (!edev || !edev->pcie_cap) + return 0; + if (pnv_pci_cfg_read(pdn, pos, 4, &header) != PCIBIOS_SUCCESSFUL) + return 0; + else if (!header) + return 0; + + while (ttl-- > 0) { + if (PCI_EXT_CAP_ID(header) == cap && pos) + return pos; + + pos = PCI_EXT_CAP_NEXT(header); + if (pos < 256) + break; + + if (pnv_pci_cfg_read(pdn, pos, 4, &header) != PCIBIOS_SUCCESSFUL) + break; + } + + return 0; +} + /** - * powernv_eeh_dev_probe - Do probe on PCI device - * @dev: PCI device - * @flag: unused + * pnv_eeh_probe - Do probe on PCI device + * @pdn: PCI device node + * @data: unused * * When EEH module is installed during system boot, all PCI devices * are checked one by one to see if it supports EEH. The function @@ -129,12 +375,12 @@ static int powernv_eeh_post_init(void) * was possiblly triggered by EEH core, the binding between EEH device * and the PCI device isn't built yet. */ -static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag) +static void *pnv_eeh_probe(struct pci_dn *pdn, void *data) { - struct pci_controller *hose = pci_bus_to_host(dev->bus); + struct pci_controller *hose = pdn->phb; struct pnv_phb *phb = hose->private_data; - struct device_node *dn = pci_device_to_OF_node(dev); - struct eeh_dev *edev = of_node_to_eeh_dev(dn); + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); + uint32_t pcie_flags; int ret; /* @@ -143,40 +389,42 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag) * the root bridge. So it's not reasonable to continue * the probing. */ - if (!dn || !edev || edev->pe) - return 0; + if (!edev || edev->pe) + return NULL; /* Skip for PCI-ISA bridge */ - if ((dev->class >> 8) == PCI_CLASS_BRIDGE_ISA) - return 0; + if ((pdn->class_code >> 8) == PCI_CLASS_BRIDGE_ISA) + return NULL; /* Initialize eeh device */ - edev->class_code = dev->class; + edev->class_code = pdn->class_code; edev->mode &= 0xFFFFFF00; - if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + edev->pcix_cap = pnv_eeh_find_cap(pdn, PCI_CAP_ID_PCIX); + edev->pcie_cap = pnv_eeh_find_cap(pdn, PCI_CAP_ID_EXP); + edev->aer_cap = pnv_eeh_find_ecap(pdn, PCI_EXT_CAP_ID_ERR); + if ((edev->class_code >> 8) == PCI_CLASS_BRIDGE_PCI) { edev->mode |= EEH_DEV_BRIDGE; - edev->pcix_cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); - if (pci_is_pcie(dev)) { - edev->pcie_cap = pci_pcie_cap(dev); - - if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) - edev->mode |= EEH_DEV_ROOT_PORT; - else if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) - edev->mode |= EEH_DEV_DS_PORT; - - edev->aer_cap = pci_find_ext_capability(dev, - PCI_EXT_CAP_ID_ERR); + if (edev->pcie_cap) { + pnv_pci_cfg_read(pdn, edev->pcie_cap + PCI_EXP_FLAGS, + 2, &pcie_flags); + pcie_flags = (pcie_flags & PCI_EXP_FLAGS_TYPE) >> 4; + if (pcie_flags == PCI_EXP_TYPE_ROOT_PORT) + edev->mode |= EEH_DEV_ROOT_PORT; + else if (pcie_flags == PCI_EXP_TYPE_DOWNSTREAM) + edev->mode |= EEH_DEV_DS_PORT; + } } - edev->config_addr = ((dev->bus->number << 8) | dev->devfn); - edev->pe_config_addr = phb->bdfn_to_pe(phb, dev->bus, dev->devfn & 0xff); + edev->config_addr = (pdn->busno << 8) | (pdn->devfn); + edev->pe_config_addr = phb->ioda.pe_rmap[edev->config_addr]; /* Create PE */ ret = eeh_add_to_parent_pe(edev); if (ret) { - pr_warn("%s: Can't add PCI dev %s to parent PE (%d)\n", - __func__, pci_name(dev), ret); - return ret; + pr_warn("%s: Can't add PCI dev %04x:%02x:%02x.%01x to parent PE (%d)\n", + __func__, hose->global_number, pdn->busno, + PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn), ret); + return NULL; } /* @@ -195,8 +443,10 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag) * Broadcom Austin 4-ports NICs (14e4:1657) * Broadcom Shiner 2-ports 10G NICs (14e4:168e) */ - if ((dev->vendor == PCI_VENDOR_ID_BROADCOM && dev->device == 0x1657) || - (dev->vendor == PCI_VENDOR_ID_BROADCOM && dev->device == 0x168e)) + if ((pdn->vendor_id == PCI_VENDOR_ID_BROADCOM && + pdn->device_id == 0x1657) || + (pdn->vendor_id == PCI_VENDOR_ID_BROADCOM && + pdn->device_id == 0x168e)) edev->pe->state |= EEH_PE_CFG_RESTRICTED; /* @@ -206,7 +456,8 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag) * to PE reset. */ if (!edev->pe->bus) - edev->pe->bus = dev->bus; + edev->pe->bus = pci_find_bus(hose->global_number, + pdn->busno); /* * Enable EEH explicitly so that we will do EEH check @@ -217,11 +468,11 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag) /* Save memory bars */ eeh_save_bars(edev); - return 0; + return NULL; } /** - * powernv_eeh_set_option - Initialize EEH or MMIO/DMA reenable + * pnv_eeh_set_option - Initialize EEH or MMIO/DMA reenable * @pe: EEH PE * @option: operation to be issued * @@ -229,36 +480,236 @@ static int powernv_eeh_dev_probe(struct pci_dev *dev, void *flag) * Currently, following options are support according to PAPR: * Enable EEH, Disable EEH, Enable MMIO and Enable DMA */ -static int powernv_eeh_set_option(struct eeh_pe *pe, int option) +static int pnv_eeh_set_option(struct eeh_pe *pe, int option) { struct pci_controller *hose = pe->phb; struct pnv_phb *phb = hose->private_data; - int ret = -EEXIST; + bool freeze_pe = false; + int opt, ret = 0; + s64 rc; + + /* Sanity check on option */ + switch (option) { + case EEH_OPT_DISABLE: + return -EPERM; + case EEH_OPT_ENABLE: + return 0; + case EEH_OPT_THAW_MMIO: + opt = OPAL_EEH_ACTION_CLEAR_FREEZE_MMIO; + break; + case EEH_OPT_THAW_DMA: + opt = OPAL_EEH_ACTION_CLEAR_FREEZE_DMA; + break; + case EEH_OPT_FREEZE_PE: + freeze_pe = true; + opt = OPAL_EEH_ACTION_SET_FREEZE_ALL; + break; + default: + pr_warn("%s: Invalid option %d\n", __func__, option); + return -EINVAL; + } - /* - * What we need do is pass it down for hardware - * implementation to handle it. - */ - if (phb->eeh_ops && phb->eeh_ops->set_option) - ret = phb->eeh_ops->set_option(pe, option); + /* If PHB supports compound PE, to handle it */ + if (freeze_pe) { + if (phb->freeze_pe) { + phb->freeze_pe(phb, pe->addr); + } else { + rc = opal_pci_eeh_freeze_set(phb->opal_id, + pe->addr, opt); + if (rc != OPAL_SUCCESS) { + pr_warn("%s: Failure %lld freezing " + "PHB#%x-PE#%x\n", + __func__, rc, + phb->hose->global_number, pe->addr); + ret = -EIO; + } + } + } else { + if (phb->unfreeze_pe) { + ret = phb->unfreeze_pe(phb, pe->addr, opt); + } else { + rc = opal_pci_eeh_freeze_clear(phb->opal_id, + pe->addr, opt); + if (rc != OPAL_SUCCESS) { + pr_warn("%s: Failure %lld enable %d " + "for PHB#%x-PE#%x\n", + __func__, rc, option, + phb->hose->global_number, pe->addr); + ret = -EIO; + } + } + } return ret; } /** - * powernv_eeh_get_pe_addr - Retrieve PE address + * pnv_eeh_get_pe_addr - Retrieve PE address * @pe: EEH PE * * Retrieve the PE address according to the given tranditional * PCI BDF (Bus/Device/Function) address. */ -static int powernv_eeh_get_pe_addr(struct eeh_pe *pe) +static int pnv_eeh_get_pe_addr(struct eeh_pe *pe) { return pe->addr; } +static void pnv_eeh_get_phb_diag(struct eeh_pe *pe) +{ + struct pnv_phb *phb = pe->phb->private_data; + s64 rc; + + rc = opal_pci_get_phb_diag_data2(phb->opal_id, pe->data, + PNV_PCI_DIAG_BUF_SIZE); + if (rc != OPAL_SUCCESS) + pr_warn("%s: Failure %lld getting PHB#%x diag-data\n", + __func__, rc, pe->phb->global_number); +} + +static int pnv_eeh_get_phb_state(struct eeh_pe *pe) +{ + struct pnv_phb *phb = pe->phb->private_data; + u8 fstate; + __be16 pcierr; + s64 rc; + int result = 0; + + rc = opal_pci_eeh_freeze_status(phb->opal_id, + pe->addr, + &fstate, + &pcierr, + NULL); + if (rc != OPAL_SUCCESS) { + pr_warn("%s: Failure %lld getting PHB#%x state\n", + __func__, rc, phb->hose->global_number); + return EEH_STATE_NOT_SUPPORT; + } + + /* + * Check PHB state. If the PHB is frozen for the + * first time, to dump the PHB diag-data. + */ + if (be16_to_cpu(pcierr) != OPAL_EEH_PHB_ERROR) { + result = (EEH_STATE_MMIO_ACTIVE | + EEH_STATE_DMA_ACTIVE | + EEH_STATE_MMIO_ENABLED | + EEH_STATE_DMA_ENABLED); + } else if (!(pe->state & EEH_PE_ISOLATED)) { + eeh_pe_state_mark(pe, EEH_PE_ISOLATED); + pnv_eeh_get_phb_diag(pe); + + if (eeh_has_flag(EEH_EARLY_DUMP_LOG)) + pnv_pci_dump_phb_diag_data(pe->phb, pe->data); + } + + return result; +} + +static int pnv_eeh_get_pe_state(struct eeh_pe *pe) +{ + struct pnv_phb *phb = pe->phb->private_data; + u8 fstate; + __be16 pcierr; + s64 rc; + int result; + + /* + * We don't clobber hardware frozen state until PE + * reset is completed. In order to keep EEH core + * moving forward, we have to return operational + * state during PE reset. + */ + if (pe->state & EEH_PE_RESET) { + result = (EEH_STATE_MMIO_ACTIVE | + EEH_STATE_DMA_ACTIVE | + EEH_STATE_MMIO_ENABLED | + EEH_STATE_DMA_ENABLED); + return result; + } + + /* + * Fetch PE state from hardware. If the PHB + * supports compound PE, let it handle that. + */ + if (phb->get_pe_state) { + fstate = phb->get_pe_state(phb, pe->addr); + } else { + rc = opal_pci_eeh_freeze_status(phb->opal_id, + pe->addr, + &fstate, + &pcierr, + NULL); + if (rc != OPAL_SUCCESS) { + pr_warn("%s: Failure %lld getting PHB#%x-PE%x state\n", + __func__, rc, phb->hose->global_number, + pe->addr); + return EEH_STATE_NOT_SUPPORT; + } + } + + /* Figure out state */ + switch (fstate) { + case OPAL_EEH_STOPPED_NOT_FROZEN: + result = (EEH_STATE_MMIO_ACTIVE | + EEH_STATE_DMA_ACTIVE | + EEH_STATE_MMIO_ENABLED | + EEH_STATE_DMA_ENABLED); + break; + case OPAL_EEH_STOPPED_MMIO_FREEZE: + result = (EEH_STATE_DMA_ACTIVE | + EEH_STATE_DMA_ENABLED); + break; + case OPAL_EEH_STOPPED_DMA_FREEZE: + result = (EEH_STATE_MMIO_ACTIVE | + EEH_STATE_MMIO_ENABLED); + break; + case OPAL_EEH_STOPPED_MMIO_DMA_FREEZE: + result = 0; + break; + case OPAL_EEH_STOPPED_RESET: + result = EEH_STATE_RESET_ACTIVE; + break; + case OPAL_EEH_STOPPED_TEMP_UNAVAIL: + result = EEH_STATE_UNAVAILABLE; + break; + case OPAL_EEH_STOPPED_PERM_UNAVAIL: + result = EEH_STATE_NOT_SUPPORT; + break; + default: + result = EEH_STATE_NOT_SUPPORT; + pr_warn("%s: Invalid PHB#%x-PE#%x state %x\n", + __func__, phb->hose->global_number, + pe->addr, fstate); + } + + /* + * If PHB supports compound PE, to freeze all + * slave PEs for consistency. + * + * If the PE is switching to frozen state for the + * first time, to dump the PHB diag-data. + */ + if (!(result & EEH_STATE_NOT_SUPPORT) && + !(result & EEH_STATE_UNAVAILABLE) && + !(result & EEH_STATE_MMIO_ACTIVE) && + !(result & EEH_STATE_DMA_ACTIVE) && + !(pe->state & EEH_PE_ISOLATED)) { + if (phb->freeze_pe) + phb->freeze_pe(phb, pe->addr); + + eeh_pe_state_mark(pe, EEH_PE_ISOLATED); + pnv_eeh_get_phb_diag(pe); + + if (eeh_has_flag(EEH_EARLY_DUMP_LOG)) + pnv_pci_dump_phb_diag_data(pe->phb, pe->data); + } + + return result; +} + /** - * powernv_eeh_get_state - Retrieve PE state + * pnv_eeh_get_state - Retrieve PE state * @pe: EEH PE * @delay: delay while PE state is temporarily unavailable * @@ -267,64 +718,279 @@ static int powernv_eeh_get_pe_addr(struct eeh_pe *pe) * we prefer passing down to hardware implementation to handle * it. */ -static int powernv_eeh_get_state(struct eeh_pe *pe, int *delay) +static int pnv_eeh_get_state(struct eeh_pe *pe, int *delay) +{ + int ret; + + if (pe->type & EEH_PE_PHB) + ret = pnv_eeh_get_phb_state(pe); + else + ret = pnv_eeh_get_pe_state(pe); + + if (!delay) + return ret; + + /* + * If the PE state is temporarily unavailable, + * to inform the EEH core delay for default + * period (1 second) + */ + *delay = 0; + if (ret & EEH_STATE_UNAVAILABLE) + *delay = 1000; + + return ret; +} + +static s64 pnv_eeh_phb_poll(struct pnv_phb *phb) +{ + s64 rc = OPAL_HARDWARE; + + while (1) { + rc = opal_pci_poll(phb->opal_id); + if (rc <= 0) + break; + + if (system_state < SYSTEM_RUNNING) + udelay(1000 * rc); + else + msleep(rc); + } + + return rc; +} + +int pnv_eeh_phb_reset(struct pci_controller *hose, int option) { - struct pci_controller *hose = pe->phb; struct pnv_phb *phb = hose->private_data; - int ret = EEH_STATE_NOT_SUPPORT; + s64 rc = OPAL_HARDWARE; + + pr_debug("%s: Reset PHB#%x, option=%d\n", + __func__, hose->global_number, option); + + /* Issue PHB complete reset request */ + if (option == EEH_RESET_FUNDAMENTAL || + option == EEH_RESET_HOT) + rc = opal_pci_reset(phb->opal_id, + OPAL_RESET_PHB_COMPLETE, + OPAL_ASSERT_RESET); + else if (option == EEH_RESET_DEACTIVATE) + rc = opal_pci_reset(phb->opal_id, + OPAL_RESET_PHB_COMPLETE, + OPAL_DEASSERT_RESET); + if (rc < 0) + goto out; - if (phb->eeh_ops && phb->eeh_ops->get_state) { - ret = phb->eeh_ops->get_state(pe); + /* + * Poll state of the PHB until the request is done + * successfully. The PHB reset is usually PHB complete + * reset followed by hot reset on root bus. So we also + * need the PCI bus settlement delay. + */ + rc = pnv_eeh_phb_poll(phb); + if (option == EEH_RESET_DEACTIVATE) { + if (system_state < SYSTEM_RUNNING) + udelay(1000 * EEH_PE_RST_SETTLE_TIME); + else + msleep(EEH_PE_RST_SETTLE_TIME); + } +out: + if (rc != OPAL_SUCCESS) + return -EIO; - /* - * If the PE state is temporarily unavailable, - * to inform the EEH core delay for default - * period (1 second) - */ - if (delay) { - *delay = 0; - if (ret & EEH_STATE_UNAVAILABLE) - *delay = 1000; + return 0; +} + +static int pnv_eeh_root_reset(struct pci_controller *hose, int option) +{ + struct pnv_phb *phb = hose->private_data; + s64 rc = OPAL_HARDWARE; + + pr_debug("%s: Reset PHB#%x, option=%d\n", + __func__, hose->global_number, option); + + /* + * During the reset deassert time, we needn't care + * the reset scope because the firmware does nothing + * for fundamental or hot reset during deassert phase. + */ + if (option == EEH_RESET_FUNDAMENTAL) + rc = opal_pci_reset(phb->opal_id, + OPAL_RESET_PCI_FUNDAMENTAL, + OPAL_ASSERT_RESET); + else if (option == EEH_RESET_HOT) + rc = opal_pci_reset(phb->opal_id, + OPAL_RESET_PCI_HOT, + OPAL_ASSERT_RESET); + else if (option == EEH_RESET_DEACTIVATE) + rc = opal_pci_reset(phb->opal_id, + OPAL_RESET_PCI_HOT, + OPAL_DEASSERT_RESET); + if (rc < 0) + goto out; + + /* Poll state of the PHB until the request is done */ + rc = pnv_eeh_phb_poll(phb); + if (option == EEH_RESET_DEACTIVATE) + msleep(EEH_PE_RST_SETTLE_TIME); +out: + if (rc != OPAL_SUCCESS) + return -EIO; + + return 0; +} + +static int pnv_eeh_bridge_reset(struct pci_dev *dev, int option) +{ + struct pci_dn *pdn = pci_get_pdn_by_devfn(dev->bus, dev->devfn); + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); + int aer = edev ? edev->aer_cap : 0; + u32 ctrl; + + pr_debug("%s: Reset PCI bus %04x:%02x with option %d\n", + __func__, pci_domain_nr(dev->bus), + dev->bus->number, option); + + switch (option) { + case EEH_RESET_FUNDAMENTAL: + case EEH_RESET_HOT: + /* Don't report linkDown event */ + if (aer) { + eeh_ops->read_config(pdn, aer + PCI_ERR_UNCOR_MASK, + 4, &ctrl); + ctrl |= PCI_ERR_UNC_SURPDN; + eeh_ops->write_config(pdn, aer + PCI_ERR_UNCOR_MASK, + 4, ctrl); } + + eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &ctrl); + ctrl |= PCI_BRIDGE_CTL_BUS_RESET; + eeh_ops->write_config(pdn, PCI_BRIDGE_CONTROL, 2, ctrl); + + msleep(EEH_PE_RST_HOLD_TIME); + break; + case EEH_RESET_DEACTIVATE: + eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &ctrl); + ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; + eeh_ops->write_config(pdn, PCI_BRIDGE_CONTROL, 2, ctrl); + + msleep(EEH_PE_RST_SETTLE_TIME); + + /* Continue reporting linkDown event */ + if (aer) { + eeh_ops->read_config(pdn, aer + PCI_ERR_UNCOR_MASK, + 4, &ctrl); + ctrl &= ~PCI_ERR_UNC_SURPDN; + eeh_ops->write_config(pdn, aer + PCI_ERR_UNCOR_MASK, + 4, ctrl); + } + + break; } - return ret; + return 0; +} + +void pnv_pci_reset_secondary_bus(struct pci_dev *dev) +{ + struct pci_controller *hose; + + if (pci_is_root_bus(dev->bus)) { + hose = pci_bus_to_host(dev->bus); + pnv_eeh_root_reset(hose, EEH_RESET_HOT); + pnv_eeh_root_reset(hose, EEH_RESET_DEACTIVATE); + } else { + pnv_eeh_bridge_reset(dev, EEH_RESET_HOT); + pnv_eeh_bridge_reset(dev, EEH_RESET_DEACTIVATE); + } } /** - * powernv_eeh_reset - Reset the specified PE + * pnv_eeh_reset - Reset the specified PE * @pe: EEH PE * @option: reset option * - * Reset the specified PE + * Do reset on the indicated PE. For PCI bus sensitive PE, + * we need to reset the parent p2p bridge. The PHB has to + * be reinitialized if the p2p bridge is root bridge. For + * PCI device sensitive PE, we will try to reset the device + * through FLR. For now, we don't have OPAL APIs to do HARD + * reset yet, so all reset would be SOFT (HOT) reset. */ -static int powernv_eeh_reset(struct eeh_pe *pe, int option) +static int pnv_eeh_reset(struct eeh_pe *pe, int option) { struct pci_controller *hose = pe->phb; - struct pnv_phb *phb = hose->private_data; - int ret = -EEXIST; + struct pci_bus *bus; + int ret; + + /* + * For PHB reset, we always have complete reset. For those PEs whose + * primary bus derived from root complex (root bus) or root port + * (usually bus#1), we apply hot or fundamental reset on the root port. + * For other PEs, we always have hot reset on the PE primary bus. + * + * Here, we have different design to pHyp, which always clear the + * frozen state during PE reset. However, the good idea here from + * benh is to keep frozen state before we get PE reset done completely + * (until BAR restore). With the frozen state, HW drops illegal IO + * or MMIO access, which can incur recrusive frozen PE during PE + * reset. The side effect is that EEH core has to clear the frozen + * state explicitly after BAR restore. + */ + if (pe->type & EEH_PE_PHB) { + ret = pnv_eeh_phb_reset(hose, option); + } else { + struct pnv_phb *phb; + s64 rc; - if (phb->eeh_ops && phb->eeh_ops->reset) - ret = phb->eeh_ops->reset(pe, option); + /* + * The frozen PE might be caused by PAPR error injection + * registers, which are expected to be cleared after hitting + * frozen PE as stated in the hardware spec. Unfortunately, + * that's not true on P7IOC. So we have to clear it manually + * to avoid recursive EEH errors during recovery. + */ + phb = hose->private_data; + if (phb->model == PNV_PHB_MODEL_P7IOC && + (option == EEH_RESET_HOT || + option == EEH_RESET_FUNDAMENTAL)) { + rc = opal_pci_reset(phb->opal_id, + OPAL_RESET_PHB_ERROR, + OPAL_ASSERT_RESET); + if (rc != OPAL_SUCCESS) { + pr_warn("%s: Failure %lld clearing " + "error injection registers\n", + __func__, rc); + return -EIO; + } + } + + bus = eeh_pe_bus_get(pe); + if (pci_is_root_bus(bus) || + pci_is_root_bus(bus->parent)) + ret = pnv_eeh_root_reset(hose, option); + else + ret = pnv_eeh_bridge_reset(bus->self, option); + } return ret; } /** - * powernv_eeh_wait_state - Wait for PE state + * pnv_eeh_wait_state - Wait for PE state * @pe: EEH PE * @max_wait: maximal period in microsecond * * Wait for the state of associated PE. It might take some time * to retrieve the PE's state. */ -static int powernv_eeh_wait_state(struct eeh_pe *pe, int max_wait) +static int pnv_eeh_wait_state(struct eeh_pe *pe, int max_wait) { int ret; int mwait; while (1) { - ret = powernv_eeh_get_state(pe, &mwait); + ret = pnv_eeh_get_state(pe, &mwait); /* * If the PE's state is temporarily unavailable, @@ -348,7 +1014,7 @@ static int powernv_eeh_wait_state(struct eeh_pe *pe, int max_wait) } /** - * powernv_eeh_get_log - Retrieve error log + * pnv_eeh_get_log - Retrieve error log * @pe: EEH PE * @severity: temporary or permanent error log * @drv_log: driver log to be combined with retrieved error log @@ -356,41 +1022,30 @@ static int powernv_eeh_wait_state(struct eeh_pe *pe, int max_wait) * * Retrieve the temporary or permanent error from the PE. */ -static int powernv_eeh_get_log(struct eeh_pe *pe, int severity, - char *drv_log, unsigned long len) +static int pnv_eeh_get_log(struct eeh_pe *pe, int severity, + char *drv_log, unsigned long len) { - struct pci_controller *hose = pe->phb; - struct pnv_phb *phb = hose->private_data; - int ret = -EEXIST; + if (!eeh_has_flag(EEH_EARLY_DUMP_LOG)) + pnv_pci_dump_phb_diag_data(pe->phb, pe->data); - if (phb->eeh_ops && phb->eeh_ops->get_log) - ret = phb->eeh_ops->get_log(pe, severity, drv_log, len); - - return ret; + return 0; } /** - * powernv_eeh_configure_bridge - Configure PCI bridges in the indicated PE + * pnv_eeh_configure_bridge - Configure PCI bridges in the indicated PE * @pe: EEH PE * * The function will be called to reconfigure the bridges included * in the specified PE so that the mulfunctional PE would be recovered * again. */ -static int powernv_eeh_configure_bridge(struct eeh_pe *pe) +static int pnv_eeh_configure_bridge(struct eeh_pe *pe) { - struct pci_controller *hose = pe->phb; - struct pnv_phb *phb = hose->private_data; - int ret = 0; - - if (phb->eeh_ops && phb->eeh_ops->configure_bridge) - ret = phb->eeh_ops->configure_bridge(pe); - - return ret; + return 0; } /** - * powernv_pe_err_inject - Inject specified error to the indicated PE + * pnv_pe_err_inject - Inject specified error to the indicated PE * @pe: the indicated PE * @type: error type * @func: specific error type @@ -401,22 +1056,52 @@ static int powernv_eeh_configure_bridge(struct eeh_pe *pe) * determined by @type and @func, to the indicated PE for * testing purpose. */ -static int powernv_eeh_err_inject(struct eeh_pe *pe, int type, int func, - unsigned long addr, unsigned long mask) +static int pnv_eeh_err_inject(struct eeh_pe *pe, int type, int func, + unsigned long addr, unsigned long mask) { struct pci_controller *hose = pe->phb; struct pnv_phb *phb = hose->private_data; - int ret = -EEXIST; + s64 rc; + + /* Sanity check on error type */ + if (type != OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR && + type != OPAL_ERR_INJECT_TYPE_IOA_BUS_ERR64) { + pr_warn("%s: Invalid error type %d\n", + __func__, type); + return -ERANGE; + } - if (phb->eeh_ops && phb->eeh_ops->err_inject) - ret = phb->eeh_ops->err_inject(pe, type, func, addr, mask); + if (func < OPAL_ERR_INJECT_FUNC_IOA_LD_MEM_ADDR || + func > OPAL_ERR_INJECT_FUNC_IOA_DMA_WR_TARGET) { + pr_warn("%s: Invalid error function %d\n", + __func__, func); + return -ERANGE; + } - return ret; + /* Firmware supports error injection ? */ + if (!opal_check_token(OPAL_PCI_ERR_INJECT)) { + pr_warn("%s: Firmware doesn't support error injection\n", + __func__); + return -ENXIO; + } + + /* Do error injection */ + rc = opal_pci_err_inject(phb->opal_id, pe->addr, + type, func, addr, mask); + if (rc != OPAL_SUCCESS) { + pr_warn("%s: Failure %lld injecting error " + "%d-%d to PHB#%x-PE#%x\n", + __func__, rc, type, func, + hose->global_number, pe->addr); + return -EIO; + } + + return 0; } -static inline bool powernv_eeh_cfg_blocked(struct device_node *dn) +static inline bool pnv_eeh_cfg_blocked(struct pci_dn *pdn) { - struct eeh_dev *edev = of_node_to_eeh_dev(dn); + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); if (!edev || !edev->pe) return false; @@ -427,51 +1112,377 @@ static inline bool powernv_eeh_cfg_blocked(struct device_node *dn) return false; } -static int powernv_eeh_read_config(struct device_node *dn, - int where, int size, u32 *val) +static int pnv_eeh_read_config(struct pci_dn *pdn, + int where, int size, u32 *val) { - if (powernv_eeh_cfg_blocked(dn)) { + if (!pdn) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (pnv_eeh_cfg_blocked(pdn)) { *val = 0xFFFFFFFF; return PCIBIOS_SET_FAILED; } - return pnv_pci_cfg_read(dn, where, size, val); + return pnv_pci_cfg_read(pdn, where, size, val); } -static int powernv_eeh_write_config(struct device_node *dn, - int where, int size, u32 val) +static int pnv_eeh_write_config(struct pci_dn *pdn, + int where, int size, u32 val) { - if (powernv_eeh_cfg_blocked(dn)) + if (!pdn) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (pnv_eeh_cfg_blocked(pdn)) return PCIBIOS_SET_FAILED; - return pnv_pci_cfg_write(dn, where, size, val); + return pnv_pci_cfg_write(pdn, where, size, val); +} + +static void pnv_eeh_dump_hub_diag_common(struct OpalIoP7IOCErrorData *data) +{ + /* GEM */ + if (data->gemXfir || data->gemRfir || + data->gemRirqfir || data->gemMask || data->gemRwof) + pr_info(" GEM: %016llx %016llx %016llx %016llx %016llx\n", + be64_to_cpu(data->gemXfir), + be64_to_cpu(data->gemRfir), + be64_to_cpu(data->gemRirqfir), + be64_to_cpu(data->gemMask), + be64_to_cpu(data->gemRwof)); + + /* LEM */ + if (data->lemFir || data->lemErrMask || + data->lemAction0 || data->lemAction1 || data->lemWof) + pr_info(" LEM: %016llx %016llx %016llx %016llx %016llx\n", + be64_to_cpu(data->lemFir), + be64_to_cpu(data->lemErrMask), + be64_to_cpu(data->lemAction0), + be64_to_cpu(data->lemAction1), + be64_to_cpu(data->lemWof)); +} + +static void pnv_eeh_get_and_dump_hub_diag(struct pci_controller *hose) +{ + struct pnv_phb *phb = hose->private_data; + struct OpalIoP7IOCErrorData *data = &phb->diag.hub_diag; + long rc; + + rc = opal_pci_get_hub_diag_data(phb->hub_id, data, sizeof(*data)); + if (rc != OPAL_SUCCESS) { + pr_warn("%s: Failed to get HUB#%llx diag-data (%ld)\n", + __func__, phb->hub_id, rc); + return; + } + + switch (data->type) { + case OPAL_P7IOC_DIAG_TYPE_RGC: + pr_info("P7IOC diag-data for RGC\n\n"); + pnv_eeh_dump_hub_diag_common(data); + if (data->rgc.rgcStatus || data->rgc.rgcLdcp) + pr_info(" RGC: %016llx %016llx\n", + be64_to_cpu(data->rgc.rgcStatus), + be64_to_cpu(data->rgc.rgcLdcp)); + break; + case OPAL_P7IOC_DIAG_TYPE_BI: + pr_info("P7IOC diag-data for BI %s\n\n", + data->bi.biDownbound ? "Downbound" : "Upbound"); + pnv_eeh_dump_hub_diag_common(data); + if (data->bi.biLdcp0 || data->bi.biLdcp1 || + data->bi.biLdcp2 || data->bi.biFenceStatus) + pr_info(" BI: %016llx %016llx %016llx %016llx\n", + be64_to_cpu(data->bi.biLdcp0), + be64_to_cpu(data->bi.biLdcp1), + be64_to_cpu(data->bi.biLdcp2), + be64_to_cpu(data->bi.biFenceStatus)); + break; + case OPAL_P7IOC_DIAG_TYPE_CI: + pr_info("P7IOC diag-data for CI Port %d\n\n", + data->ci.ciPort); + pnv_eeh_dump_hub_diag_common(data); + if (data->ci.ciPortStatus || data->ci.ciPortLdcp) + pr_info(" CI: %016llx %016llx\n", + be64_to_cpu(data->ci.ciPortStatus), + be64_to_cpu(data->ci.ciPortLdcp)); + break; + case OPAL_P7IOC_DIAG_TYPE_MISC: + pr_info("P7IOC diag-data for MISC\n\n"); + pnv_eeh_dump_hub_diag_common(data); + break; + case OPAL_P7IOC_DIAG_TYPE_I2C: + pr_info("P7IOC diag-data for I2C\n\n"); + pnv_eeh_dump_hub_diag_common(data); + break; + default: + pr_warn("%s: Invalid type of HUB#%llx diag-data (%d)\n", + __func__, phb->hub_id, data->type); + } +} + +static int pnv_eeh_get_pe(struct pci_controller *hose, + u16 pe_no, struct eeh_pe **pe) +{ + struct pnv_phb *phb = hose->private_data; + struct pnv_ioda_pe *pnv_pe; + struct eeh_pe *dev_pe; + struct eeh_dev edev; + + /* + * If PHB supports compound PE, to fetch + * the master PE because slave PE is invisible + * to EEH core. + */ + pnv_pe = &phb->ioda.pe_array[pe_no]; + if (pnv_pe->flags & PNV_IODA_PE_SLAVE) { + pnv_pe = pnv_pe->master; + WARN_ON(!pnv_pe || + !(pnv_pe->flags & PNV_IODA_PE_MASTER)); + pe_no = pnv_pe->pe_number; + } + + /* Find the PE according to PE# */ + memset(&edev, 0, sizeof(struct eeh_dev)); + edev.phb = hose; + edev.pe_config_addr = pe_no; + dev_pe = eeh_pe_get(&edev); + if (!dev_pe) + return -EEXIST; + + /* Freeze the (compound) PE */ + *pe = dev_pe; + if (!(dev_pe->state & EEH_PE_ISOLATED)) + phb->freeze_pe(phb, pe_no); + + /* + * At this point, we're sure the (compound) PE should + * have been frozen. However, we still need poke until + * hitting the frozen PE on top level. + */ + dev_pe = dev_pe->parent; + while (dev_pe && !(dev_pe->type & EEH_PE_PHB)) { + int ret; + int active_flags = (EEH_STATE_MMIO_ACTIVE | + EEH_STATE_DMA_ACTIVE); + + ret = eeh_ops->get_state(dev_pe, NULL); + if (ret <= 0 || (ret & active_flags) == active_flags) { + dev_pe = dev_pe->parent; + continue; + } + + /* Frozen parent PE */ + *pe = dev_pe; + if (!(dev_pe->state & EEH_PE_ISOLATED)) + phb->freeze_pe(phb, dev_pe->addr); + + /* Next one */ + dev_pe = dev_pe->parent; + } + + return 0; } /** - * powernv_eeh_next_error - Retrieve next EEH error to handle + * pnv_eeh_next_error - Retrieve next EEH error to handle * @pe: Affected PE * - * Using OPAL API, to retrieve next EEH error for EEH core to handle + * The function is expected to be called by EEH core while it gets + * special EEH event (without binding PE). The function calls to + * OPAL APIs for next error to handle. The informational error is + * handled internally by platform. However, the dead IOC, dead PHB, + * fenced PHB and frozen PE should be handled by EEH core eventually. */ -static int powernv_eeh_next_error(struct eeh_pe **pe) +static int pnv_eeh_next_error(struct eeh_pe **pe) { struct pci_controller *hose; - struct pnv_phb *phb = NULL; + struct pnv_phb *phb; + struct eeh_pe *phb_pe, *parent_pe; + __be64 frozen_pe_no; + __be16 err_type, severity; + int active_flags = (EEH_STATE_MMIO_ACTIVE | EEH_STATE_DMA_ACTIVE); + long rc; + int state, ret = EEH_NEXT_ERR_NONE; + + /* + * While running here, it's safe to purge the event queue. + * And we should keep the cached OPAL notifier event sychronized + * between the kernel and firmware. + */ + eeh_remove_event(NULL, false); + opal_notifier_update_evt(OPAL_EVENT_PCI_ERROR, 0x0ul); list_for_each_entry(hose, &hose_list, list_node) { + /* + * If the subordinate PCI buses of the PHB has been + * removed or is exactly under error recovery, we + * needn't take care of it any more. + */ phb = hose->private_data; - break; - } + phb_pe = eeh_phb_pe_get(hose); + if (!phb_pe || (phb_pe->state & EEH_PE_ISOLATED)) + continue; + + rc = opal_pci_next_error(phb->opal_id, + &frozen_pe_no, &err_type, &severity); + if (rc != OPAL_SUCCESS) { + pr_devel("%s: Invalid return value on " + "PHB#%x (0x%lx) from opal_pci_next_error", + __func__, hose->global_number, rc); + continue; + } + + /* If the PHB doesn't have error, stop processing */ + if (be16_to_cpu(err_type) == OPAL_EEH_NO_ERROR || + be16_to_cpu(severity) == OPAL_EEH_SEV_NO_ERROR) { + pr_devel("%s: No error found on PHB#%x\n", + __func__, hose->global_number); + continue; + } + + /* + * Processing the error. We're expecting the error with + * highest priority reported upon multiple errors on the + * specific PHB. + */ + pr_devel("%s: Error (%d, %d, %llu) on PHB#%x\n", + __func__, be16_to_cpu(err_type), + be16_to_cpu(severity), be64_to_cpu(frozen_pe_no), + hose->global_number); + switch (be16_to_cpu(err_type)) { + case OPAL_EEH_IOC_ERROR: + if (be16_to_cpu(severity) == OPAL_EEH_SEV_IOC_DEAD) { + pr_err("EEH: dead IOC detected\n"); + ret = EEH_NEXT_ERR_DEAD_IOC; + } else if (be16_to_cpu(severity) == OPAL_EEH_SEV_INF) { + pr_info("EEH: IOC informative error " + "detected\n"); + pnv_eeh_get_and_dump_hub_diag(hose); + ret = EEH_NEXT_ERR_NONE; + } + + break; + case OPAL_EEH_PHB_ERROR: + if (be16_to_cpu(severity) == OPAL_EEH_SEV_PHB_DEAD) { + *pe = phb_pe; + pr_err("EEH: dead PHB#%x detected, " + "location: %s\n", + hose->global_number, + eeh_pe_loc_get(phb_pe)); + ret = EEH_NEXT_ERR_DEAD_PHB; + } else if (be16_to_cpu(severity) == + OPAL_EEH_SEV_PHB_FENCED) { + *pe = phb_pe; + pr_err("EEH: Fenced PHB#%x detected, " + "location: %s\n", + hose->global_number, + eeh_pe_loc_get(phb_pe)); + ret = EEH_NEXT_ERR_FENCED_PHB; + } else if (be16_to_cpu(severity) == OPAL_EEH_SEV_INF) { + pr_info("EEH: PHB#%x informative error " + "detected, location: %s\n", + hose->global_number, + eeh_pe_loc_get(phb_pe)); + pnv_eeh_get_phb_diag(phb_pe); + pnv_pci_dump_phb_diag_data(hose, phb_pe->data); + ret = EEH_NEXT_ERR_NONE; + } + + break; + case OPAL_EEH_PE_ERROR: + /* + * If we can't find the corresponding PE, we + * just try to unfreeze. + */ + if (pnv_eeh_get_pe(hose, + be64_to_cpu(frozen_pe_no), pe)) { + /* Try best to clear it */ + pr_info("EEH: Clear non-existing PHB#%x-PE#%llx\n", + hose->global_number, frozen_pe_no); + pr_info("EEH: PHB location: %s\n", + eeh_pe_loc_get(phb_pe)); + opal_pci_eeh_freeze_clear(phb->opal_id, + frozen_pe_no, + OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); + ret = EEH_NEXT_ERR_NONE; + } else if ((*pe)->state & EEH_PE_ISOLATED || + eeh_pe_passed(*pe)) { + ret = EEH_NEXT_ERR_NONE; + } else { + pr_err("EEH: Frozen PE#%x " + "on PHB#%x detected\n", + (*pe)->addr, + (*pe)->phb->global_number); + pr_err("EEH: PE location: %s, " + "PHB location: %s\n", + eeh_pe_loc_get(*pe), + eeh_pe_loc_get(phb_pe)); + ret = EEH_NEXT_ERR_FROZEN_PE; + } + + break; + default: + pr_warn("%s: Unexpected error type %d\n", + __func__, be16_to_cpu(err_type)); + } - if (phb && phb->eeh_ops->next_error) - return phb->eeh_ops->next_error(pe); + /* + * EEH core will try recover from fenced PHB or + * frozen PE. In the time for frozen PE, EEH core + * enable IO path for that before collecting logs, + * but it ruins the site. So we have to dump the + * log in advance here. + */ + if ((ret == EEH_NEXT_ERR_FROZEN_PE || + ret == EEH_NEXT_ERR_FENCED_PHB) && + !((*pe)->state & EEH_PE_ISOLATED)) { + eeh_pe_state_mark(*pe, EEH_PE_ISOLATED); + pnv_eeh_get_phb_diag(*pe); + + if (eeh_has_flag(EEH_EARLY_DUMP_LOG)) + pnv_pci_dump_phb_diag_data((*pe)->phb, + (*pe)->data); + } - return -EEXIST; + /* + * We probably have the frozen parent PE out there and + * we need have to handle frozen parent PE firstly. + */ + if (ret == EEH_NEXT_ERR_FROZEN_PE) { + parent_pe = (*pe)->parent; + while (parent_pe) { + /* Hit the ceiling ? */ + if (parent_pe->type & EEH_PE_PHB) + break; + + /* Frozen parent PE ? */ + state = eeh_ops->get_state(parent_pe, NULL); + if (state > 0 && + (state & active_flags) != active_flags) + *pe = parent_pe; + + /* Next parent level */ + parent_pe = parent_pe->parent; + } + + /* We possibly migrate to another PE */ + eeh_pe_state_mark(*pe, EEH_PE_ISOLATED); + } + + /* + * If we have no errors on the specific PHB or only + * informative error there, we continue poking it. + * Otherwise, we need actions to be taken by upper + * layer. + */ + if (ret > EEH_NEXT_ERR_INF) + break; + } + + return ret; } -static int powernv_eeh_restore_config(struct device_node *dn) +static int pnv_eeh_restore_config(struct pci_dn *pdn) { - struct eeh_dev *edev = of_node_to_eeh_dev(dn); + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); struct pnv_phb *phb; s64 ret; @@ -490,24 +1501,23 @@ static int powernv_eeh_restore_config(struct device_node *dn) return 0; } -static struct eeh_ops powernv_eeh_ops = { +static struct eeh_ops pnv_eeh_ops = { .name = "powernv", - .init = powernv_eeh_init, - .post_init = powernv_eeh_post_init, - .of_probe = NULL, - .dev_probe = powernv_eeh_dev_probe, - .set_option = powernv_eeh_set_option, - .get_pe_addr = powernv_eeh_get_pe_addr, - .get_state = powernv_eeh_get_state, - .reset = powernv_eeh_reset, - .wait_state = powernv_eeh_wait_state, - .get_log = powernv_eeh_get_log, - .configure_bridge = powernv_eeh_configure_bridge, - .err_inject = powernv_eeh_err_inject, - .read_config = powernv_eeh_read_config, - .write_config = powernv_eeh_write_config, - .next_error = powernv_eeh_next_error, - .restore_config = powernv_eeh_restore_config + .init = pnv_eeh_init, + .post_init = pnv_eeh_post_init, + .probe = pnv_eeh_probe, + .set_option = pnv_eeh_set_option, + .get_pe_addr = pnv_eeh_get_pe_addr, + .get_state = pnv_eeh_get_state, + .reset = pnv_eeh_reset, + .wait_state = pnv_eeh_wait_state, + .get_log = pnv_eeh_get_log, + .configure_bridge = pnv_eeh_configure_bridge, + .err_inject = pnv_eeh_err_inject, + .read_config = pnv_eeh_read_config, + .write_config = pnv_eeh_write_config, + .next_error = pnv_eeh_next_error, + .restore_config = pnv_eeh_restore_config }; /** @@ -521,7 +1531,7 @@ static int __init eeh_powernv_init(void) int ret = -EINVAL; eeh_set_pe_aux_size(PNV_PCI_DIAG_BUF_SIZE); - ret = eeh_ops_register(&powernv_eeh_ops); + ret = eeh_ops_register(&pnv_eeh_ops); if (!ret) pr_info("EEH: PowerNV platform initialized\n"); else diff --git a/arch/powerpc/platforms/powernv/opal-dump.c b/arch/powerpc/platforms/powernv/opal-dump.c index 23260f7dfa7a..5aa9c1ce4de3 100644 --- a/arch/powerpc/platforms/powernv/opal-dump.c +++ b/arch/powerpc/platforms/powernv/opal-dump.c @@ -452,5 +452,6 @@ void __init opal_platform_dump_init(void) return; } - opal_dump_resend_notification(); + if (opal_check_token(OPAL_DUMP_RESEND)) + opal_dump_resend_notification(); } diff --git a/arch/powerpc/platforms/powernv/opal-elog.c b/arch/powerpc/platforms/powernv/opal-elog.c index 518fe95dbf24..38ce757e5e2a 100644 --- a/arch/powerpc/platforms/powernv/opal-elog.c +++ b/arch/powerpc/platforms/powernv/opal-elog.c @@ -313,7 +313,8 @@ int __init opal_elog_init(void) } /* We are now ready to pull error logs from opal. */ - opal_resend_pending_logs(); + if (opal_check_token(OPAL_ELOG_RESEND)) + opal_resend_pending_logs(); return 0; } diff --git a/arch/powerpc/platforms/powernv/opal-flash.c b/arch/powerpc/platforms/powernv/opal-flash.c index 5c21d9c07f45..4ec6219287fc 100644 --- a/arch/powerpc/platforms/powernv/opal-flash.c +++ b/arch/powerpc/platforms/powernv/opal-flash.c @@ -120,7 +120,11 @@ static struct image_header_t image_header; static struct image_data_t image_data; static struct validate_flash_t validate_flash_data; static struct manage_flash_t manage_flash_data; -static struct update_flash_t update_flash_data; + +/* Initialize update_flash_data status to No Operation */ +static struct update_flash_t update_flash_data = { + .status = FLASH_NO_OP, +}; static DEFINE_MUTEX(image_data_mutex); @@ -542,7 +546,7 @@ static struct attribute_group image_op_attr_group = { .attrs = image_op_attrs, }; -void __init opal_flash_init(void) +void __init opal_flash_update_init(void) { int ret; diff --git a/arch/powerpc/platforms/powernv/opal-nvram.c b/arch/powerpc/platforms/powernv/opal-nvram.c index f9896fd5d04a..9db4398ded5d 100644 --- a/arch/powerpc/platforms/powernv/opal-nvram.c +++ b/arch/powerpc/platforms/powernv/opal-nvram.c @@ -16,6 +16,7 @@ #include <linux/of.h> #include <asm/opal.h> +#include <asm/nvram.h> #include <asm/machdep.h> static unsigned int nvram_size; @@ -62,6 +63,15 @@ static ssize_t opal_nvram_write(char *buf, size_t count, loff_t *index) return count; } +static int __init opal_nvram_init_log_partitions(void) +{ + /* Scan nvram for partitions */ + nvram_scan_partitions(); + nvram_init_oops_partition(0); + return 0; +} +machine_arch_initcall(powernv, opal_nvram_init_log_partitions); + void __init opal_nvram_init(void) { struct device_node *np; diff --git a/arch/powerpc/platforms/powernv/opal-power.c b/arch/powerpc/platforms/powernv/opal-power.c index 48bf5b080bcf..ac46c2c24f99 100644 --- a/arch/powerpc/platforms/powernv/opal-power.c +++ b/arch/powerpc/platforms/powernv/opal-power.c @@ -29,8 +29,9 @@ static int opal_power_control_event(struct notifier_block *nb, switch (type) { case SOFT_REBOOT: - /* Fall through. The service processor is responsible for - * bringing the machine back up */ + pr_info("OPAL: reboot requested\n"); + orderly_reboot(); + break; case SOFT_OFF: pr_info("OPAL: poweroff requested\n"); orderly_poweroff(true); diff --git a/arch/powerpc/platforms/powernv/opal-sensor.c b/arch/powerpc/platforms/powernv/opal-sensor.c index 4ab67ef7abc9..655250499d18 100644 --- a/arch/powerpc/platforms/powernv/opal-sensor.c +++ b/arch/powerpc/platforms/powernv/opal-sensor.c @@ -46,18 +46,28 @@ int opal_get_sensor_data(u32 sensor_hndl, u32 *sensor_data) mutex_lock(&opal_sensor_mutex); ret = opal_sensor_read(sensor_hndl, token, &data); - if (ret != OPAL_ASYNC_COMPLETION) - goto out_token; + switch (ret) { + case OPAL_ASYNC_COMPLETION: + ret = opal_async_wait_response(token, &msg); + if (ret) { + pr_err("%s: Failed to wait for the async response, %d\n", + __func__, ret); + goto out_token; + } - ret = opal_async_wait_response(token, &msg); - if (ret) { - pr_err("%s: Failed to wait for the async response, %d\n", - __func__, ret); - goto out_token; - } + ret = opal_error_code(be64_to_cpu(msg.params[1])); + *sensor_data = be32_to_cpu(data); + break; + + case OPAL_SUCCESS: + ret = 0; + *sensor_data = be32_to_cpu(data); + break; - *sensor_data = be32_to_cpu(data); - ret = be64_to_cpu(msg.params[1]); + default: + ret = opal_error_code(ret); + break; + } out_token: mutex_unlock(&opal_sensor_mutex); diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S index fcbe899fe299..a7ade94cdf87 100644 --- a/arch/powerpc/platforms/powernv/opal-wrappers.S +++ b/arch/powerpc/platforms/powernv/opal-wrappers.S @@ -286,9 +286,12 @@ OPAL_CALL(opal_handle_hmi, OPAL_HANDLE_HMI); OPAL_CALL(opal_slw_set_reg, OPAL_SLW_SET_REG); OPAL_CALL(opal_register_dump_region, OPAL_REGISTER_DUMP_REGION); OPAL_CALL(opal_unregister_dump_region, OPAL_UNREGISTER_DUMP_REGION); -OPAL_CALL(opal_pci_set_phb_cxl_mode, OPAL_PCI_SET_PHB_CXL_MODE); +OPAL_CALL(opal_pci_set_phb_cxl_mode, OPAL_PCI_SET_PHB_CAPI_MODE); OPAL_CALL(opal_tpo_write, OPAL_WRITE_TPO); OPAL_CALL(opal_tpo_read, OPAL_READ_TPO); OPAL_CALL(opal_ipmi_send, OPAL_IPMI_SEND); OPAL_CALL(opal_ipmi_recv, OPAL_IPMI_RECV); OPAL_CALL(opal_i2c_request, OPAL_I2C_REQUEST); +OPAL_CALL(opal_flash_read, OPAL_FLASH_READ); +OPAL_CALL(opal_flash_write, OPAL_FLASH_WRITE); +OPAL_CALL(opal_flash_erase, OPAL_FLASH_ERASE); diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index 18fd4e71c9c1..2241565b0739 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -23,6 +23,8 @@ #include <linux/kobject.h> #include <linux/delay.h> #include <linux/memblock.h> +#include <linux/kthread.h> +#include <linux/freezer.h> #include <asm/machdep.h> #include <asm/opal.h> @@ -58,6 +60,7 @@ static struct atomic_notifier_head opal_msg_notifier_head[OPAL_MSG_TYPE_MAX]; static DEFINE_SPINLOCK(opal_notifier_lock); static uint64_t last_notified_mask = 0x0ul; static atomic_t opal_notifier_hold = ATOMIC_INIT(0); +static uint32_t opal_heartbeat; static void opal_reinit_cores(void) { @@ -302,23 +305,26 @@ void opal_notifier_disable(void) * Opal message notifier based on message type. Allow subscribers to get * notified for specific messgae type. */ -int opal_message_notifier_register(enum OpalMessageType msg_type, +int opal_message_notifier_register(enum opal_msg_type msg_type, struct notifier_block *nb) { - if (!nb) { - pr_warning("%s: Invalid argument (%p)\n", - __func__, nb); - return -EINVAL; - } - if (msg_type > OPAL_MSG_TYPE_MAX) { - pr_warning("%s: Invalid message type argument (%d)\n", + if (!nb || msg_type >= OPAL_MSG_TYPE_MAX) { + pr_warning("%s: Invalid arguments, msg_type:%d\n", __func__, msg_type); return -EINVAL; } + return atomic_notifier_chain_register( &opal_msg_notifier_head[msg_type], nb); } +int opal_message_notifier_unregister(enum opal_msg_type msg_type, + struct notifier_block *nb) +{ + return atomic_notifier_chain_unregister( + &opal_msg_notifier_head[msg_type], nb); +} + static void opal_message_do_notify(uint32_t msg_type, void *msg) { /* notify subscribers */ @@ -351,7 +357,7 @@ static void opal_handle_message(void) type = be32_to_cpu(msg.msg_type); /* Sanity check */ - if (type > OPAL_MSG_TYPE_MAX) { + if (type >= OPAL_MSG_TYPE_MAX) { pr_warning("%s: Unknown message type: %u\n", __func__, type); return; } @@ -665,6 +671,9 @@ static void __init opal_dump_region_init(void) uint64_t size; int rc; + if (!opal_check_token(OPAL_REGISTER_DUMP_REGION)) + return; + /* Register kernel log buffer */ addr = log_buf_addr_get(); if (addr == NULL) @@ -684,6 +693,15 @@ static void __init opal_dump_region_init(void) "rc = %d\n", rc); } +static void opal_flash_init(struct device_node *opal_node) +{ + struct device_node *np; + + for_each_child_of_node(opal_node, np) + if (of_device_is_compatible(np, "ibm,opal-flash")) + of_platform_device_create(np, NULL, NULL); +} + static void opal_ipmi_init(struct device_node *opal_node) { struct device_node *np; @@ -741,6 +759,29 @@ static void __init opal_irq_init(struct device_node *dn) } } +static int kopald(void *unused) +{ + set_freezable(); + do { + try_to_freeze(); + opal_poll_events(NULL); + msleep_interruptible(opal_heartbeat); + } while (!kthread_should_stop()); + + return 0; +} + +static void opal_init_heartbeat(void) +{ + /* Old firwmware, we assume the HVC heartbeat is sufficient */ + if (of_property_read_u32(opal_node, "ibm,heartbeat-ms", + &opal_heartbeat) != 0) + opal_heartbeat = 0; + + if (opal_heartbeat) + kthread_run(kopald, NULL, "kopald"); +} + static int __init opal_init(void) { struct device_node *np, *consoles; @@ -769,6 +810,9 @@ static int __init opal_init(void) /* Create i2c platform devices */ opal_i2c_create_devs(); + /* Setup a heatbeat thread if requested by OPAL */ + opal_init_heartbeat(); + /* Find all OPAL interrupts and request them */ opal_irq_init(opal_node); @@ -782,7 +826,7 @@ static int __init opal_init(void) /* Setup error log interface */ rc = opal_elog_init(); /* Setup code update interface */ - opal_flash_init(); + opal_flash_update_init(); /* Setup platform dump extract interface */ opal_platform_dump_init(); /* Setup system parameters interface */ @@ -791,8 +835,11 @@ static int __init opal_init(void) opal_msglog_init(); } + /* Initialize OPAL IPMI backend */ opal_ipmi_init(opal_node); + opal_flash_init(opal_node); + return 0; } machine_subsys_initcall(powernv, opal_init); @@ -823,13 +870,17 @@ void opal_shutdown(void) } /* Unregister memory dump region */ - opal_unregister_dump_region(OPAL_DUMP_REGION_LOG_BUF); + if (opal_check_token(OPAL_UNREGISTER_DUMP_REGION)) + opal_unregister_dump_region(OPAL_DUMP_REGION_LOG_BUF); } /* Export this so that test modules can use it */ EXPORT_SYMBOL_GPL(opal_invalid_call); EXPORT_SYMBOL_GPL(opal_ipmi_send); EXPORT_SYMBOL_GPL(opal_ipmi_recv); +EXPORT_SYMBOL_GPL(opal_flash_read); +EXPORT_SYMBOL_GPL(opal_flash_write); +EXPORT_SYMBOL_GPL(opal_flash_erase); /* Convert a region of vmalloc memory to an opal sg list */ struct opal_sg_list *opal_vmalloc_to_sg_list(void *vmalloc_addr, @@ -894,6 +945,25 @@ void opal_free_sg_list(struct opal_sg_list *sg) } } +int opal_error_code(int rc) +{ + switch (rc) { + case OPAL_SUCCESS: return 0; + + case OPAL_PARAMETER: return -EINVAL; + case OPAL_ASYNC_COMPLETION: return -EINPROGRESS; + case OPAL_BUSY_EVENT: return -EBUSY; + case OPAL_NO_MEM: return -ENOMEM; + + case OPAL_UNSUPPORTED: return -EIO; + case OPAL_HARDWARE: return -EIO; + case OPAL_INTERNAL_ERROR: return -EIO; + default: + pr_err("%s: unexpected OPAL error %d\n", __func__, rc); + return -EIO; + } +} + EXPORT_SYMBOL_GPL(opal_poll_events); EXPORT_SYMBOL_GPL(opal_rtc_read); EXPORT_SYMBOL_GPL(opal_rtc_write); diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 6c9ff2b95119..920c252d1f49 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -44,6 +44,9 @@ #include "powernv.h" #include "pci.h" +/* 256M DMA window, 4K TCE pages, 8 bytes TCE */ +#define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8) + static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level, const char *fmt, ...) { @@ -56,11 +59,18 @@ static void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level, vaf.fmt = fmt; vaf.va = &args; - if (pe->pdev) + if (pe->flags & PNV_IODA_PE_DEV) strlcpy(pfix, dev_name(&pe->pdev->dev), sizeof(pfix)); - else + else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) sprintf(pfix, "%04x:%02x ", pci_domain_nr(pe->pbus), pe->pbus->number); +#ifdef CONFIG_PCI_IOV + else if (pe->flags & PNV_IODA_PE_VF) + sprintf(pfix, "%04x:%02x:%2x.%d", + pci_domain_nr(pe->parent_dev->bus), + (pe->rid & 0xff00) >> 8, + PCI_SLOT(pe->rid), PCI_FUNC(pe->rid)); +#endif /* CONFIG_PCI_IOV*/ printk("%spci %s: [PE# %.3d] %pV", level, pfix, pe->pe_number, &vaf); @@ -591,7 +601,7 @@ static int pnv_ioda_set_peltv(struct pnv_phb *phb, bool is_add) { struct pnv_ioda_pe *slave; - struct pci_dev *pdev; + struct pci_dev *pdev = NULL; int ret; /* @@ -630,8 +640,12 @@ static int pnv_ioda_set_peltv(struct pnv_phb *phb, if (pe->flags & (PNV_IODA_PE_BUS_ALL | PNV_IODA_PE_BUS)) pdev = pe->pbus->self; - else + else if (pe->flags & PNV_IODA_PE_DEV) pdev = pe->pdev->bus->self; +#ifdef CONFIG_PCI_IOV + else if (pe->flags & PNV_IODA_PE_VF) + pdev = pe->parent_dev->bus->self; +#endif /* CONFIG_PCI_IOV */ while (pdev) { struct pci_dn *pdn = pci_get_pdn(pdev); struct pnv_ioda_pe *parent; @@ -649,6 +663,87 @@ static int pnv_ioda_set_peltv(struct pnv_phb *phb, return 0; } +#ifdef CONFIG_PCI_IOV +static int pnv_ioda_deconfigure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe) +{ + struct pci_dev *parent; + uint8_t bcomp, dcomp, fcomp; + int64_t rc; + long rid_end, rid; + + /* Currently, we just deconfigure VF PE. Bus PE will always there.*/ + if (pe->pbus) { + int count; + + dcomp = OPAL_IGNORE_RID_DEVICE_NUMBER; + fcomp = OPAL_IGNORE_RID_FUNCTION_NUMBER; + parent = pe->pbus->self; + if (pe->flags & PNV_IODA_PE_BUS_ALL) + count = pe->pbus->busn_res.end - pe->pbus->busn_res.start + 1; + else + count = 1; + + switch(count) { + case 1: bcomp = OpalPciBusAll; break; + case 2: bcomp = OpalPciBus7Bits; break; + case 4: bcomp = OpalPciBus6Bits; break; + case 8: bcomp = OpalPciBus5Bits; break; + case 16: bcomp = OpalPciBus4Bits; break; + case 32: bcomp = OpalPciBus3Bits; break; + default: + dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n", + count); + /* Do an exact match only */ + bcomp = OpalPciBusAll; + } + rid_end = pe->rid + (count << 8); + } else { + if (pe->flags & PNV_IODA_PE_VF) + parent = pe->parent_dev; + else + parent = pe->pdev->bus->self; + bcomp = OpalPciBusAll; + dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER; + fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER; + rid_end = pe->rid + 1; + } + + /* Clear the reverse map */ + for (rid = pe->rid; rid < rid_end; rid++) + phb->ioda.pe_rmap[rid] = 0; + + /* Release from all parents PELT-V */ + while (parent) { + struct pci_dn *pdn = pci_get_pdn(parent); + if (pdn && pdn->pe_number != IODA_INVALID_PE) { + rc = opal_pci_set_peltv(phb->opal_id, pdn->pe_number, + pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN); + /* XXX What to do in case of error ? */ + } + parent = parent->bus->self; + } + + opal_pci_eeh_freeze_set(phb->opal_id, pe->pe_number, + OPAL_EEH_ACTION_CLEAR_FREEZE_ALL); + + /* Disassociate PE in PELT */ + rc = opal_pci_set_peltv(phb->opal_id, pe->pe_number, + pe->pe_number, OPAL_REMOVE_PE_FROM_DOMAIN); + if (rc) + pe_warn(pe, "OPAL error %ld remove self from PELTV\n", rc); + rc = opal_pci_set_pe(phb->opal_id, pe->pe_number, pe->rid, + bcomp, dcomp, fcomp, OPAL_UNMAP_PE); + if (rc) + pe_err(pe, "OPAL error %ld trying to setup PELT table\n", rc); + + pe->pbus = NULL; + pe->pdev = NULL; + pe->parent_dev = NULL; + + return 0; +} +#endif /* CONFIG_PCI_IOV */ + static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe) { struct pci_dev *parent; @@ -675,15 +770,19 @@ static int pnv_ioda_configure_pe(struct pnv_phb *phb, struct pnv_ioda_pe *pe) case 16: bcomp = OpalPciBus4Bits; break; case 32: bcomp = OpalPciBus3Bits; break; default: - pr_err("%s: Number of subordinate busses %d" - " unsupported\n", - pci_name(pe->pbus->self), count); + dev_err(&pe->pbus->dev, "Number of subordinate buses %d unsupported\n", + count); /* Do an exact match only */ bcomp = OpalPciBusAll; } rid_end = pe->rid + (count << 8); } else { - parent = pe->pdev->bus->self; +#ifdef CONFIG_PCI_IOV + if (pe->flags & PNV_IODA_PE_VF) + parent = pe->parent_dev; + else +#endif /* CONFIG_PCI_IOV */ + parent = pe->pdev->bus->self; bcomp = OpalPciBusAll; dcomp = OPAL_COMPARE_RID_DEVICE_NUMBER; fcomp = OPAL_COMPARE_RID_FUNCTION_NUMBER; @@ -774,6 +873,78 @@ static unsigned int pnv_ioda_dma_weight(struct pci_dev *dev) return 10; } +#ifdef CONFIG_PCI_IOV +static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset) +{ + struct pci_dn *pdn = pci_get_pdn(dev); + int i; + struct resource *res, res2; + resource_size_t size; + u16 num_vfs; + + if (!dev->is_physfn) + return -EINVAL; + + /* + * "offset" is in VFs. The M64 windows are sized so that when they + * are segmented, each segment is the same size as the IOV BAR. + * Each segment is in a separate PE, and the high order bits of the + * address are the PE number. Therefore, each VF's BAR is in a + * separate PE, and changing the IOV BAR start address changes the + * range of PEs the VFs are in. + */ + num_vfs = pdn->num_vfs; + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { + res = &dev->resource[i + PCI_IOV_RESOURCES]; + if (!res->flags || !res->parent) + continue; + + if (!pnv_pci_is_mem_pref_64(res->flags)) + continue; + + /* + * The actual IOV BAR range is determined by the start address + * and the actual size for num_vfs VFs BAR. This check is to + * make sure that after shifting, the range will not overlap + * with another device. + */ + size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); + res2.flags = res->flags; + res2.start = res->start + (size * offset); + res2.end = res2.start + (size * num_vfs) - 1; + + if (res2.end > res->end) { + dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n", + i, &res2, res, num_vfs, offset); + return -EBUSY; + } + } + + /* + * After doing so, there would be a "hole" in the /proc/iomem when + * offset is a positive value. It looks like the device return some + * mmio back to the system, which actually no one could use it. + */ + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { + res = &dev->resource[i + PCI_IOV_RESOURCES]; + if (!res->flags || !res->parent) + continue; + + if (!pnv_pci_is_mem_pref_64(res->flags)) + continue; + + size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); + res2 = *res; + res->start += size * offset; + + dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (enabling %d VFs shifted by %d)\n", + i, &res2, res, num_vfs, offset); + pci_update_resource(dev, i + PCI_IOV_RESOURCES); + } + return 0; +} +#endif /* CONFIG_PCI_IOV */ + #if 0 static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev) { @@ -857,7 +1028,6 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe) pci_name(dev)); continue; } - pdn->pcidev = dev; pdn->pe_number = pe->pe_number; pe->dma_weight += pnv_ioda_dma_weight(dev); if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate) @@ -916,6 +1086,10 @@ static void pnv_ioda_setup_bus_PE(struct pci_bus *bus, int all) return; } + pe->tce32_table = kzalloc_node(sizeof(struct iommu_table), + GFP_KERNEL, hose->node); + pe->tce32_table->data = pe; + /* Associate it with all child devices */ pnv_ioda_setup_same_PE(bus, pe); @@ -974,6 +1148,441 @@ static void pnv_pci_ioda_setup_PEs(void) } } +#ifdef CONFIG_PCI_IOV +static int pnv_pci_vf_release_m64(struct pci_dev *pdev) +{ + struct pci_bus *bus; + struct pci_controller *hose; + struct pnv_phb *phb; + struct pci_dn *pdn; + int i, j; + + bus = pdev->bus; + hose = pci_bus_to_host(bus); + phb = hose->private_data; + pdn = pci_get_pdn(pdev); + + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) + for (j = 0; j < M64_PER_IOV; j++) { + if (pdn->m64_wins[i][j] == IODA_INVALID_M64) + continue; + opal_pci_phb_mmio_enable(phb->opal_id, + OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 0); + clear_bit(pdn->m64_wins[i][j], &phb->ioda.m64_bar_alloc); + pdn->m64_wins[i][j] = IODA_INVALID_M64; + } + + return 0; +} + +static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs) +{ + struct pci_bus *bus; + struct pci_controller *hose; + struct pnv_phb *phb; + struct pci_dn *pdn; + unsigned int win; + struct resource *res; + int i, j; + int64_t rc; + int total_vfs; + resource_size_t size, start; + int pe_num; + int vf_groups; + int vf_per_group; + + bus = pdev->bus; + hose = pci_bus_to_host(bus); + phb = hose->private_data; + pdn = pci_get_pdn(pdev); + total_vfs = pci_sriov_get_totalvfs(pdev); + + /* Initialize the m64_wins to IODA_INVALID_M64 */ + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) + for (j = 0; j < M64_PER_IOV; j++) + pdn->m64_wins[i][j] = IODA_INVALID_M64; + + if (pdn->m64_per_iov == M64_PER_IOV) { + vf_groups = (num_vfs <= M64_PER_IOV) ? num_vfs: M64_PER_IOV; + vf_per_group = (num_vfs <= M64_PER_IOV)? 1: + roundup_pow_of_two(num_vfs) / pdn->m64_per_iov; + } else { + vf_groups = 1; + vf_per_group = 1; + } + + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { + res = &pdev->resource[i + PCI_IOV_RESOURCES]; + if (!res->flags || !res->parent) + continue; + + if (!pnv_pci_is_mem_pref_64(res->flags)) + continue; + + for (j = 0; j < vf_groups; j++) { + do { + win = find_next_zero_bit(&phb->ioda.m64_bar_alloc, + phb->ioda.m64_bar_idx + 1, 0); + + if (win >= phb->ioda.m64_bar_idx + 1) + goto m64_failed; + } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc)); + + pdn->m64_wins[i][j] = win; + + if (pdn->m64_per_iov == M64_PER_IOV) { + size = pci_iov_resource_size(pdev, + PCI_IOV_RESOURCES + i); + size = size * vf_per_group; + start = res->start + size * j; + } else { + size = resource_size(res); + start = res->start; + } + + /* Map the M64 here */ + if (pdn->m64_per_iov == M64_PER_IOV) { + pe_num = pdn->offset + j; + rc = opal_pci_map_pe_mmio_window(phb->opal_id, + pe_num, OPAL_M64_WINDOW_TYPE, + pdn->m64_wins[i][j], 0); + } + + rc = opal_pci_set_phb_mem_window(phb->opal_id, + OPAL_M64_WINDOW_TYPE, + pdn->m64_wins[i][j], + start, + 0, /* unused */ + size); + + + if (rc != OPAL_SUCCESS) { + dev_err(&pdev->dev, "Failed to map M64 window #%d: %lld\n", + win, rc); + goto m64_failed; + } + + if (pdn->m64_per_iov == M64_PER_IOV) + rc = opal_pci_phb_mmio_enable(phb->opal_id, + OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 2); + else + rc = opal_pci_phb_mmio_enable(phb->opal_id, + OPAL_M64_WINDOW_TYPE, pdn->m64_wins[i][j], 1); + + if (rc != OPAL_SUCCESS) { + dev_err(&pdev->dev, "Failed to enable M64 window #%d: %llx\n", + win, rc); + goto m64_failed; + } + } + } + return 0; + +m64_failed: + pnv_pci_vf_release_m64(pdev); + return -EBUSY; +} + +static void pnv_pci_ioda2_release_dma_pe(struct pci_dev *dev, struct pnv_ioda_pe *pe) +{ + struct pci_bus *bus; + struct pci_controller *hose; + struct pnv_phb *phb; + struct iommu_table *tbl; + unsigned long addr; + int64_t rc; + + bus = dev->bus; + hose = pci_bus_to_host(bus); + phb = hose->private_data; + tbl = pe->tce32_table; + addr = tbl->it_base; + + opal_pci_map_pe_dma_window(phb->opal_id, pe->pe_number, + pe->pe_number << 1, 1, __pa(addr), + 0, 0x1000); + + rc = opal_pci_map_pe_dma_window_real(pe->phb->opal_id, + pe->pe_number, + (pe->pe_number << 1) + 1, + pe->tce_bypass_base, + 0); + if (rc) + pe_warn(pe, "OPAL error %ld release DMA window\n", rc); + + iommu_free_table(tbl, of_node_full_name(dev->dev.of_node)); + free_pages(addr, get_order(TCE32_TABLE_SIZE)); + pe->tce32_table = NULL; +} + +static void pnv_ioda_release_vf_PE(struct pci_dev *pdev, u16 num_vfs) +{ + struct pci_bus *bus; + struct pci_controller *hose; + struct pnv_phb *phb; + struct pnv_ioda_pe *pe, *pe_n; + struct pci_dn *pdn; + u16 vf_index; + int64_t rc; + + bus = pdev->bus; + hose = pci_bus_to_host(bus); + phb = hose->private_data; + pdn = pci_get_pdn(pdev); + + if (!pdev->is_physfn) + return; + + if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) { + int vf_group; + int vf_per_group; + int vf_index1; + + vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov; + + for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) + for (vf_index = vf_group * vf_per_group; + vf_index < (vf_group + 1) * vf_per_group && + vf_index < num_vfs; + vf_index++) + for (vf_index1 = vf_group * vf_per_group; + vf_index1 < (vf_group + 1) * vf_per_group && + vf_index1 < num_vfs; + vf_index1++){ + + rc = opal_pci_set_peltv(phb->opal_id, + pdn->offset + vf_index, + pdn->offset + vf_index1, + OPAL_REMOVE_PE_FROM_DOMAIN); + + if (rc) + dev_warn(&pdev->dev, "%s: Failed to unlink same group PE#%d(%lld)\n", + __func__, + pdn->offset + vf_index1, rc); + } + } + + list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) { + if (pe->parent_dev != pdev) + continue; + + pnv_pci_ioda2_release_dma_pe(pdev, pe); + + /* Remove from list */ + mutex_lock(&phb->ioda.pe_list_mutex); + list_del(&pe->list); + mutex_unlock(&phb->ioda.pe_list_mutex); + + pnv_ioda_deconfigure_pe(phb, pe); + + pnv_ioda_free_pe(phb, pe->pe_number); + } +} + +void pnv_pci_sriov_disable(struct pci_dev *pdev) +{ + struct pci_bus *bus; + struct pci_controller *hose; + struct pnv_phb *phb; + struct pci_dn *pdn; + struct pci_sriov *iov; + u16 num_vfs; + + bus = pdev->bus; + hose = pci_bus_to_host(bus); + phb = hose->private_data; + pdn = pci_get_pdn(pdev); + iov = pdev->sriov; + num_vfs = pdn->num_vfs; + + /* Release VF PEs */ + pnv_ioda_release_vf_PE(pdev, num_vfs); + + if (phb->type == PNV_PHB_IODA2) { + if (pdn->m64_per_iov == 1) + pnv_pci_vf_resource_shift(pdev, -pdn->offset); + + /* Release M64 windows */ + pnv_pci_vf_release_m64(pdev); + + /* Release PE numbers */ + bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs); + pdn->offset = 0; + } +} + +static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb, + struct pnv_ioda_pe *pe); +static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs) +{ + struct pci_bus *bus; + struct pci_controller *hose; + struct pnv_phb *phb; + struct pnv_ioda_pe *pe; + int pe_num; + u16 vf_index; + struct pci_dn *pdn; + int64_t rc; + + bus = pdev->bus; + hose = pci_bus_to_host(bus); + phb = hose->private_data; + pdn = pci_get_pdn(pdev); + + if (!pdev->is_physfn) + return; + + /* Reserve PE for each VF */ + for (vf_index = 0; vf_index < num_vfs; vf_index++) { + pe_num = pdn->offset + vf_index; + + pe = &phb->ioda.pe_array[pe_num]; + pe->pe_number = pe_num; + pe->phb = phb; + pe->flags = PNV_IODA_PE_VF; + pe->pbus = NULL; + pe->parent_dev = pdev; + pe->tce32_seg = -1; + pe->mve_number = -1; + pe->rid = (pci_iov_virtfn_bus(pdev, vf_index) << 8) | + pci_iov_virtfn_devfn(pdev, vf_index); + + pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%d\n", + hose->global_number, pdev->bus->number, + PCI_SLOT(pci_iov_virtfn_devfn(pdev, vf_index)), + PCI_FUNC(pci_iov_virtfn_devfn(pdev, vf_index)), pe_num); + + if (pnv_ioda_configure_pe(phb, pe)) { + /* XXX What do we do here ? */ + if (pe_num) + pnv_ioda_free_pe(phb, pe_num); + pe->pdev = NULL; + continue; + } + + pe->tce32_table = kzalloc_node(sizeof(struct iommu_table), + GFP_KERNEL, hose->node); + pe->tce32_table->data = pe; + + /* Put PE to the list */ + mutex_lock(&phb->ioda.pe_list_mutex); + list_add_tail(&pe->list, &phb->ioda.pe_list); + mutex_unlock(&phb->ioda.pe_list_mutex); + + pnv_pci_ioda2_setup_dma_pe(phb, pe); + } + + if (pdn->m64_per_iov == M64_PER_IOV && num_vfs > M64_PER_IOV) { + int vf_group; + int vf_per_group; + int vf_index1; + + vf_per_group = roundup_pow_of_two(num_vfs) / pdn->m64_per_iov; + + for (vf_group = 0; vf_group < M64_PER_IOV; vf_group++) { + for (vf_index = vf_group * vf_per_group; + vf_index < (vf_group + 1) * vf_per_group && + vf_index < num_vfs; + vf_index++) { + for (vf_index1 = vf_group * vf_per_group; + vf_index1 < (vf_group + 1) * vf_per_group && + vf_index1 < num_vfs; + vf_index1++) { + + rc = opal_pci_set_peltv(phb->opal_id, + pdn->offset + vf_index, + pdn->offset + vf_index1, + OPAL_ADD_PE_TO_DOMAIN); + + if (rc) + dev_warn(&pdev->dev, "%s: Failed to link same group PE#%d(%lld)\n", + __func__, + pdn->offset + vf_index1, rc); + } + } + } + } +} + +int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +{ + struct pci_bus *bus; + struct pci_controller *hose; + struct pnv_phb *phb; + struct pci_dn *pdn; + int ret; + + bus = pdev->bus; + hose = pci_bus_to_host(bus); + phb = hose->private_data; + pdn = pci_get_pdn(pdev); + + if (phb->type == PNV_PHB_IODA2) { + /* Calculate available PE for required VFs */ + mutex_lock(&phb->ioda.pe_alloc_mutex); + pdn->offset = bitmap_find_next_zero_area( + phb->ioda.pe_alloc, phb->ioda.total_pe, + 0, num_vfs, 0); + if (pdn->offset >= phb->ioda.total_pe) { + mutex_unlock(&phb->ioda.pe_alloc_mutex); + dev_info(&pdev->dev, "Failed to enable VF%d\n", num_vfs); + pdn->offset = 0; + return -EBUSY; + } + bitmap_set(phb->ioda.pe_alloc, pdn->offset, num_vfs); + pdn->num_vfs = num_vfs; + mutex_unlock(&phb->ioda.pe_alloc_mutex); + + /* Assign M64 window accordingly */ + ret = pnv_pci_vf_assign_m64(pdev, num_vfs); + if (ret) { + dev_info(&pdev->dev, "Not enough M64 window resources\n"); + goto m64_failed; + } + + /* + * When using one M64 BAR to map one IOV BAR, we need to shift + * the IOV BAR according to the PE# allocated to the VFs. + * Otherwise, the PE# for the VF will conflict with others. + */ + if (pdn->m64_per_iov == 1) { + ret = pnv_pci_vf_resource_shift(pdev, pdn->offset); + if (ret) + goto m64_failed; + } + } + + /* Setup VF PEs */ + pnv_ioda_setup_vf_PE(pdev, num_vfs); + + return 0; + +m64_failed: + bitmap_clear(phb->ioda.pe_alloc, pdn->offset, num_vfs); + pdn->offset = 0; + + return ret; +} + +int pcibios_sriov_disable(struct pci_dev *pdev) +{ + pnv_pci_sriov_disable(pdev); + + /* Release PCI data */ + remove_dev_pci_data(pdev); + return 0; +} + +int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +{ + /* Allocate PCI data */ + add_dev_pci_data(pdev); + + pnv_pci_sriov_enable(pdev, num_vfs); + return 0; +} +#endif /* CONFIG_PCI_IOV */ + static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev) { struct pci_dn *pdn = pci_get_pdn(pdev); @@ -989,7 +1598,7 @@ static void pnv_pci_ioda_dma_dev_setup(struct pnv_phb *phb, struct pci_dev *pdev pe = &phb->ioda.pe_array[pdn->pe_number]; WARN_ON(get_dma_ops(&pdev->dev) != &dma_iommu_ops); - set_iommu_table_base_and_group(&pdev->dev, &pe->tce32_table); + set_iommu_table_base_and_group(&pdev->dev, pe->tce32_table); } static int pnv_pci_ioda_dma_set_mask(struct pnv_phb *phb, @@ -1016,7 +1625,7 @@ static int pnv_pci_ioda_dma_set_mask(struct pnv_phb *phb, } else { dev_info(&pdev->dev, "Using 32-bit DMA via iommu\n"); set_dma_ops(&pdev->dev, &dma_iommu_ops); - set_iommu_table_base(&pdev->dev, &pe->tce32_table); + set_iommu_table_base(&pdev->dev, pe->tce32_table); } *pdev->dev.dma_mask = dma_mask; return 0; @@ -1053,9 +1662,9 @@ static void pnv_ioda_setup_bus_dma(struct pnv_ioda_pe *pe, list_for_each_entry(dev, &bus->devices, bus_list) { if (add_to_iommu_group) set_iommu_table_base_and_group(&dev->dev, - &pe->tce32_table); + pe->tce32_table); else - set_iommu_table_base(&dev->dev, &pe->tce32_table); + set_iommu_table_base(&dev->dev, pe->tce32_table); if (dev->subordinate) pnv_ioda_setup_bus_dma(pe, dev->subordinate, @@ -1145,8 +1754,7 @@ static void pnv_pci_ioda2_tce_invalidate(struct pnv_ioda_pe *pe, void pnv_pci_ioda_tce_invalidate(struct iommu_table *tbl, __be64 *startp, __be64 *endp, bool rm) { - struct pnv_ioda_pe *pe = container_of(tbl, struct pnv_ioda_pe, - tce32_table); + struct pnv_ioda_pe *pe = tbl->data; struct pnv_phb *phb = pe->phb; if (phb->type == PNV_PHB_IODA1) @@ -1167,9 +1775,6 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb, int64_t rc; void *addr; - /* 256M DMA window, 4K TCE pages, 8 bytes TCE */ -#define TCE32_TABLE_SIZE ((0x10000000 / 0x1000) * 8) - /* XXX FIXME: Handle 64-bit only DMA devices */ /* XXX FIXME: Provide 64-bit DMA facilities & non-4K TCE tables etc.. */ /* XXX FIXME: Allocate multi-level tables on PHB3 */ @@ -1212,7 +1817,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb, } /* Setup linux iommu table */ - tbl = &pe->tce32_table; + tbl = pe->tce32_table; pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs, base << 28, IOMMU_PAGE_SHIFT_4K); @@ -1232,12 +1837,19 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb, TCE_PCI_SWINV_PAIR); } iommu_init_table(tbl, phb->hose->node); - iommu_register_group(tbl, phb->hose->global_number, pe->pe_number); - if (pe->pdev) + if (pe->flags & PNV_IODA_PE_DEV) { + iommu_register_group(tbl, phb->hose->global_number, + pe->pe_number); set_iommu_table_base_and_group(&pe->pdev->dev, tbl); - else + } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) { + iommu_register_group(tbl, phb->hose->global_number, + pe->pe_number); pnv_ioda_setup_bus_dma(pe, pe->pbus, true); + } else if (pe->flags & PNV_IODA_PE_VF) { + iommu_register_group(tbl, phb->hose->global_number, + pe->pe_number); + } return; fail: @@ -1250,8 +1862,7 @@ static void pnv_pci_ioda_setup_dma_pe(struct pnv_phb *phb, static void pnv_pci_ioda2_set_bypass(struct iommu_table *tbl, bool enable) { - struct pnv_ioda_pe *pe = container_of(tbl, struct pnv_ioda_pe, - tce32_table); + struct pnv_ioda_pe *pe = tbl->data; uint16_t window_id = (pe->pe_number << 1 ) + 1; int64_t rc; @@ -1296,10 +1907,10 @@ static void pnv_pci_ioda2_setup_bypass_pe(struct pnv_phb *phb, pe->tce_bypass_base = 1ull << 59; /* Install set_bypass callback for VFIO */ - pe->tce32_table.set_bypass = pnv_pci_ioda2_set_bypass; + pe->tce32_table->set_bypass = pnv_pci_ioda2_set_bypass; /* Enable bypass by default */ - pnv_pci_ioda2_set_bypass(&pe->tce32_table, true); + pnv_pci_ioda2_set_bypass(pe->tce32_table, true); } static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb, @@ -1347,7 +1958,7 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb, } /* Setup linux iommu table */ - tbl = &pe->tce32_table; + tbl = pe->tce32_table; pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, 0, IOMMU_PAGE_SHIFT_4K); @@ -1365,12 +1976,19 @@ static void pnv_pci_ioda2_setup_dma_pe(struct pnv_phb *phb, tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE); } iommu_init_table(tbl, phb->hose->node); - iommu_register_group(tbl, phb->hose->global_number, pe->pe_number); - if (pe->pdev) + if (pe->flags & PNV_IODA_PE_DEV) { + iommu_register_group(tbl, phb->hose->global_number, + pe->pe_number); set_iommu_table_base_and_group(&pe->pdev->dev, tbl); - else + } else if (pe->flags & (PNV_IODA_PE_BUS | PNV_IODA_PE_BUS_ALL)) { + iommu_register_group(tbl, phb->hose->global_number, + pe->pe_number); pnv_ioda_setup_bus_dma(pe, pe->pbus, true); + } else if (pe->flags & PNV_IODA_PE_VF) { + iommu_register_group(tbl, phb->hose->global_number, + pe->pe_number); + } /* Also create a bypass window */ if (!pnv_iommu_bypass_disabled) @@ -1731,6 +2349,73 @@ static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) static void pnv_pci_init_ioda_msis(struct pnv_phb *phb) { } #endif /* CONFIG_PCI_MSI */ +#ifdef CONFIG_PCI_IOV +static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev) +{ + struct pci_controller *hose; + struct pnv_phb *phb; + struct resource *res; + int i; + resource_size_t size; + struct pci_dn *pdn; + int mul, total_vfs; + + if (!pdev->is_physfn || pdev->is_added) + return; + + hose = pci_bus_to_host(pdev->bus); + phb = hose->private_data; + + pdn = pci_get_pdn(pdev); + pdn->vfs_expanded = 0; + + total_vfs = pci_sriov_get_totalvfs(pdev); + pdn->m64_per_iov = 1; + mul = phb->ioda.total_pe; + + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { + res = &pdev->resource[i + PCI_IOV_RESOURCES]; + if (!res->flags || res->parent) + continue; + if (!pnv_pci_is_mem_pref_64(res->flags)) { + dev_warn(&pdev->dev, " non M64 VF BAR%d: %pR\n", + i, res); + continue; + } + + size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES); + + /* bigger than 64M */ + if (size > (1 << 26)) { + dev_info(&pdev->dev, "PowerNV: VF BAR%d: %pR IOV size is bigger than 64M, roundup power2\n", + i, res); + pdn->m64_per_iov = M64_PER_IOV; + mul = roundup_pow_of_two(total_vfs); + break; + } + } + + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { + res = &pdev->resource[i + PCI_IOV_RESOURCES]; + if (!res->flags || res->parent) + continue; + if (!pnv_pci_is_mem_pref_64(res->flags)) { + dev_warn(&pdev->dev, "Skipping expanding VF BAR%d: %pR\n", + i, res); + continue; + } + + dev_dbg(&pdev->dev, " Fixing VF BAR%d: %pR to\n", i, res); + size = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES); + res->end = res->start + size * mul - 1; + dev_dbg(&pdev->dev, " %pR\n", res); + dev_info(&pdev->dev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)", + i, res, mul); + } + pdn->vfs_expanded = mul; +} +#endif /* CONFIG_PCI_IOV */ + /* * This function is supposed to be called on basis of PE from top * to bottom style. So the the I/O or MMIO segment assigned to @@ -1777,7 +2462,8 @@ static void pnv_ioda_setup_pe_seg(struct pci_controller *hose, region.start += phb->ioda.io_segsize; index++; } - } else if (res->flags & IORESOURCE_MEM) { + } else if ((res->flags & IORESOURCE_MEM) && + !pnv_pci_is_mem_pref_64(res->flags)) { region.start = res->start - hose->mem_offset[0] - phb->ioda.m32_pci_base; @@ -1907,10 +2593,29 @@ static resource_size_t pnv_pci_window_alignment(struct pci_bus *bus, return phb->ioda.io_segsize; } +#ifdef CONFIG_PCI_IOV +static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev, + int resno) +{ + struct pci_dn *pdn = pci_get_pdn(pdev); + resource_size_t align, iov_align; + + iov_align = resource_size(&pdev->resource[resno]); + if (iov_align) + return iov_align; + + align = pci_iov_resource_size(pdev, resno); + if (pdn->vfs_expanded) + return pdn->vfs_expanded * align; + + return align; +} +#endif /* CONFIG_PCI_IOV */ + /* Prevent enabling devices for which we couldn't properly * assign a PE */ -static int pnv_pci_enable_device_hook(struct pci_dev *dev) +static bool pnv_pci_enable_device_hook(struct pci_dev *dev) { struct pci_controller *hose = pci_bus_to_host(dev->bus); struct pnv_phb *phb = hose->private_data; @@ -1922,13 +2627,13 @@ static int pnv_pci_enable_device_hook(struct pci_dev *dev) * PEs isn't ready. */ if (!phb->initialized) - return 0; + return true; pdn = pci_get_pdn(dev); if (!pdn || pdn->pe_number == IODA_INVALID_PE) - return -EINVAL; + return false; - return 0; + return true; } static u32 pnv_ioda_bdfn_to_pe(struct pnv_phb *phb, struct pci_bus *bus, @@ -1988,9 +2693,11 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, hose->last_busno = 0xff; } hose->private_data = phb; + hose->controller_ops = pnv_pci_controller_ops; phb->hub_id = hub_id; phb->opal_id = phb_id; phb->type = ioda_type; + mutex_init(&phb->ioda.pe_alloc_mutex); /* Detect specific models for error handling */ if (of_device_is_compatible(np, "ibm,p7ioc-pciex")) @@ -2050,6 +2757,7 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, INIT_LIST_HEAD(&phb->ioda.pe_dma_list); INIT_LIST_HEAD(&phb->ioda.pe_list); + mutex_init(&phb->ioda.pe_list_mutex); /* Calculate how many 32-bit TCE segments we have */ phb->ioda.tce32_count = phb->ioda.m32_pci_base >> 28; @@ -2078,9 +2786,6 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, phb->get_pe_state = pnv_ioda_get_pe_state; phb->freeze_pe = pnv_ioda_freeze_pe; phb->unfreeze_pe = pnv_ioda_unfreeze_pe; -#ifdef CONFIG_EEH - phb->eeh_ops = &ioda_eeh_ops; -#endif /* Setup RID -> PE mapping function */ phb->bdfn_to_pe = pnv_ioda_bdfn_to_pe; @@ -2104,9 +2809,15 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, * the child P2P bridges) can form individual PE. */ ppc_md.pcibios_fixup = pnv_pci_ioda_fixup; - ppc_md.pcibios_enable_device_hook = pnv_pci_enable_device_hook; - ppc_md.pcibios_window_alignment = pnv_pci_window_alignment; - ppc_md.pcibios_reset_secondary_bus = pnv_pci_reset_secondary_bus; + pnv_pci_controller_ops.enable_device_hook = pnv_pci_enable_device_hook; + pnv_pci_controller_ops.window_alignment = pnv_pci_window_alignment; + pnv_pci_controller_ops.reset_secondary_bus = pnv_pci_reset_secondary_bus; + +#ifdef CONFIG_PCI_IOV + ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources; + ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment; +#endif + pci_add_flags(PCI_REASSIGN_ALL_RSRC); /* Reset IODA tables to a clean state */ @@ -2121,8 +2832,8 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, */ if (is_kdump_kernel()) { pr_info(" Issue PHB reset ...\n"); - ioda_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL); - ioda_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE); + pnv_eeh_phb_reset(hose, EEH_RESET_FUNDAMENTAL); + pnv_eeh_phb_reset(hose, EEH_RESET_DEACTIVATE); } /* Remove M64 resource if we can't configure it successfully */ diff --git a/arch/powerpc/platforms/powernv/pci-p5ioc2.c b/arch/powerpc/platforms/powernv/pci-p5ioc2.c index 6ef6d4d8e7e2..4729ca793813 100644 --- a/arch/powerpc/platforms/powernv/pci-p5ioc2.c +++ b/arch/powerpc/platforms/powernv/pci-p5ioc2.c @@ -133,6 +133,7 @@ static void __init pnv_pci_init_p5ioc2_phb(struct device_node *np, u64 hub_id, phb->hose->first_busno = 0; phb->hose->last_busno = 0xff; phb->hose->private_data = phb; + phb->hose->controller_ops = pnv_pci_controller_ops; phb->hub_id = hub_id; phb->opal_id = phb_id; phb->type = PNV_PHB_P5IOC2; diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index 54323d6b5166..bca2aeb6e4b6 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -366,9 +366,9 @@ static void pnv_pci_handle_eeh_config(struct pnv_phb *phb, u32 pe_no) spin_unlock_irqrestore(&phb->lock, flags); } -static void pnv_pci_config_check_eeh(struct pnv_phb *phb, - struct device_node *dn) +static void pnv_pci_config_check_eeh(struct pci_dn *pdn) { + struct pnv_phb *phb = pdn->phb->private_data; u8 fstate; __be16 pcierr; int pe_no; @@ -379,7 +379,7 @@ static void pnv_pci_config_check_eeh(struct pnv_phb *phb, * setup that yet. So all ER errors should be mapped to * reserved PE. */ - pe_no = PCI_DN(dn)->pe_number; + pe_no = pdn->pe_number; if (pe_no == IODA_INVALID_PE) { if (phb->type == PNV_PHB_P5IOC2) pe_no = 0; @@ -407,8 +407,7 @@ static void pnv_pci_config_check_eeh(struct pnv_phb *phb, } cfg_dbg(" -> EEH check, bdfn=%04x PE#%d fstate=%x\n", - (PCI_DN(dn)->busno << 8) | (PCI_DN(dn)->devfn), - pe_no, fstate); + (pdn->busno << 8) | (pdn->devfn), pe_no, fstate); /* Clear the frozen state if applicable */ if (fstate == OPAL_EEH_STOPPED_MMIO_FREEZE || @@ -425,10 +424,9 @@ static void pnv_pci_config_check_eeh(struct pnv_phb *phb, } } -int pnv_pci_cfg_read(struct device_node *dn, +int pnv_pci_cfg_read(struct pci_dn *pdn, int where, int size, u32 *val) { - struct pci_dn *pdn = PCI_DN(dn); struct pnv_phb *phb = pdn->phb->private_data; u32 bdfn = (pdn->busno << 8) | pdn->devfn; s64 rc; @@ -462,10 +460,9 @@ int pnv_pci_cfg_read(struct device_node *dn, return PCIBIOS_SUCCESSFUL; } -int pnv_pci_cfg_write(struct device_node *dn, +int pnv_pci_cfg_write(struct pci_dn *pdn, int where, int size, u32 val) { - struct pci_dn *pdn = PCI_DN(dn); struct pnv_phb *phb = pdn->phb->private_data; u32 bdfn = (pdn->busno << 8) | pdn->devfn; @@ -489,18 +486,17 @@ int pnv_pci_cfg_write(struct device_node *dn, } #if CONFIG_EEH -static bool pnv_pci_cfg_check(struct pci_controller *hose, - struct device_node *dn) +static bool pnv_pci_cfg_check(struct pci_dn *pdn) { struct eeh_dev *edev = NULL; - struct pnv_phb *phb = hose->private_data; + struct pnv_phb *phb = pdn->phb->private_data; /* EEH not enabled ? */ if (!(phb->flags & PNV_PHB_FLAG_EEH)) return true; /* PE reset or device removed ? */ - edev = of_node_to_eeh_dev(dn); + edev = pdn->edev; if (edev) { if (edev->pe && (edev->pe->state & EEH_PE_CFG_BLOCKED)) @@ -513,8 +509,7 @@ static bool pnv_pci_cfg_check(struct pci_controller *hose, return true; } #else -static inline pnv_pci_cfg_check(struct pci_controller *hose, - struct device_node *dn) +static inline pnv_pci_cfg_check(struct pci_dn *pdn) { return true; } @@ -524,32 +519,26 @@ static int pnv_pci_read_config(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val) { - struct device_node *dn, *busdn = pci_bus_to_OF_node(bus); struct pci_dn *pdn; struct pnv_phb *phb; - bool found = false; int ret; *val = 0xFFFFFFFF; - for (dn = busdn->child; dn; dn = dn->sibling) { - pdn = PCI_DN(dn); - if (pdn && pdn->devfn == devfn) { - phb = pdn->phb->private_data; - found = true; - break; - } - } + pdn = pci_get_pdn_by_devfn(bus, devfn); + if (!pdn) + return PCIBIOS_DEVICE_NOT_FOUND; - if (!found || !pnv_pci_cfg_check(pdn->phb, dn)) + if (!pnv_pci_cfg_check(pdn)) return PCIBIOS_DEVICE_NOT_FOUND; - ret = pnv_pci_cfg_read(dn, where, size, val); - if (phb->flags & PNV_PHB_FLAG_EEH) { + ret = pnv_pci_cfg_read(pdn, where, size, val); + phb = pdn->phb->private_data; + if (phb->flags & PNV_PHB_FLAG_EEH && pdn->edev) { if (*val == EEH_IO_ERROR_VALUE(size) && - eeh_dev_check_failure(of_node_to_eeh_dev(dn))) + eeh_dev_check_failure(pdn->edev)) return PCIBIOS_DEVICE_NOT_FOUND; } else { - pnv_pci_config_check_eeh(phb, dn); + pnv_pci_config_check_eeh(pdn); } return ret; @@ -559,27 +548,21 @@ static int pnv_pci_write_config(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val) { - struct device_node *dn, *busdn = pci_bus_to_OF_node(bus); struct pci_dn *pdn; struct pnv_phb *phb; - bool found = false; int ret; - for (dn = busdn->child; dn; dn = dn->sibling) { - pdn = PCI_DN(dn); - if (pdn && pdn->devfn == devfn) { - phb = pdn->phb->private_data; - found = true; - break; - } - } + pdn = pci_get_pdn_by_devfn(bus, devfn); + if (!pdn) + return PCIBIOS_DEVICE_NOT_FOUND; - if (!found || !pnv_pci_cfg_check(pdn->phb, dn)) + if (!pnv_pci_cfg_check(pdn)) return PCIBIOS_DEVICE_NOT_FOUND; - ret = pnv_pci_cfg_write(dn, where, size, val); + ret = pnv_pci_cfg_write(pdn, where, size, val); + phb = pdn->phb->private_data; if (!(phb->flags & PNV_PHB_FLAG_EEH)) - pnv_pci_config_check_eeh(phb, dn); + pnv_pci_config_check_eeh(pdn); return ret; } @@ -679,66 +662,31 @@ void pnv_pci_setup_iommu_table(struct iommu_table *tbl, tbl->it_type = TCE_PCI; } -static struct iommu_table *pnv_pci_setup_bml_iommu(struct pci_controller *hose) -{ - struct iommu_table *tbl; - const __be64 *basep, *swinvp; - const __be32 *sizep; - - basep = of_get_property(hose->dn, "linux,tce-base", NULL); - sizep = of_get_property(hose->dn, "linux,tce-size", NULL); - if (basep == NULL || sizep == NULL) { - pr_err("PCI: %s has missing tce entries !\n", - hose->dn->full_name); - return NULL; - } - tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, hose->node); - if (WARN_ON(!tbl)) - return NULL; - pnv_pci_setup_iommu_table(tbl, __va(be64_to_cpup(basep)), - be32_to_cpup(sizep), 0, IOMMU_PAGE_SHIFT_4K); - iommu_init_table(tbl, hose->node); - iommu_register_group(tbl, pci_domain_nr(hose->bus), 0); - - /* Deal with SW invalidated TCEs when needed (BML way) */ - swinvp = of_get_property(hose->dn, "linux,tce-sw-invalidate-info", - NULL); - if (swinvp) { - tbl->it_busno = be64_to_cpu(swinvp[1]); - tbl->it_index = (unsigned long)ioremap(be64_to_cpup(swinvp), 8); - tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE; - } - return tbl; -} - -static void pnv_pci_dma_fallback_setup(struct pci_controller *hose, - struct pci_dev *pdev) -{ - struct device_node *np = pci_bus_to_OF_node(hose->bus); - struct pci_dn *pdn; - - if (np == NULL) - return; - pdn = PCI_DN(np); - if (!pdn->iommu_table) - pdn->iommu_table = pnv_pci_setup_bml_iommu(hose); - if (!pdn->iommu_table) - return; - set_iommu_table_base_and_group(&pdev->dev, pdn->iommu_table); -} - static void pnv_pci_dma_dev_setup(struct pci_dev *pdev) { struct pci_controller *hose = pci_bus_to_host(pdev->bus); struct pnv_phb *phb = hose->private_data; +#ifdef CONFIG_PCI_IOV + struct pnv_ioda_pe *pe; + struct pci_dn *pdn; + + /* Fix the VF pdn PE number */ + if (pdev->is_virtfn) { + pdn = pci_get_pdn(pdev); + WARN_ON(pdn->pe_number != IODA_INVALID_PE); + list_for_each_entry(pe, &phb->ioda.pe_list, list) { + if (pe->rid == ((pdev->bus->number << 8) | + (pdev->devfn & 0xff))) { + pdn->pe_number = pe->pe_number; + pe->pdev = pdev; + break; + } + } + } +#endif /* CONFIG_PCI_IOV */ - /* If we have no phb structure, try to setup a fallback based on - * the device-tree (RTAS PCI for example) - */ if (phb && phb->dma_dev_setup) phb->dma_dev_setup(phb, pdev); - else - pnv_pci_dma_fallback_setup(hose, pdev); } int pnv_pci_dma_set_mask(struct pci_dev *pdev, u64 dma_mask) @@ -784,44 +732,36 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_IBM, 0x3b9, pnv_p7ioc_rc_quirk); void __init pnv_pci_init(void) { struct device_node *np; + bool found_ioda = false; pci_add_flags(PCI_CAN_SKIP_ISA_ALIGN); - /* OPAL absent, try POPAL first then RTAS detection of PHBs */ - if (!firmware_has_feature(FW_FEATURE_OPAL)) { -#ifdef CONFIG_PPC_POWERNV_RTAS - init_pci_config_tokens(); - find_and_init_phbs(); -#endif /* CONFIG_PPC_POWERNV_RTAS */ - } - /* OPAL is here, do our normal stuff */ - else { - int found_ioda = 0; + /* If we don't have OPAL, eg. in sim, just skip PCI probe */ + if (!firmware_has_feature(FW_FEATURE_OPAL)) + return; - /* Look for IODA IO-Hubs. We don't support mixing IODA - * and p5ioc2 due to the need to change some global - * probing flags - */ - for_each_compatible_node(np, NULL, "ibm,ioda-hub") { - pnv_pci_init_ioda_hub(np); - found_ioda = 1; - } + /* Look for IODA IO-Hubs. We don't support mixing IODA + * and p5ioc2 due to the need to change some global + * probing flags + */ + for_each_compatible_node(np, NULL, "ibm,ioda-hub") { + pnv_pci_init_ioda_hub(np); + found_ioda = true; + } - /* Look for p5ioc2 IO-Hubs */ - if (!found_ioda) - for_each_compatible_node(np, NULL, "ibm,p5ioc2") - pnv_pci_init_p5ioc2_hub(np); + /* Look for p5ioc2 IO-Hubs */ + if (!found_ioda) + for_each_compatible_node(np, NULL, "ibm,p5ioc2") + pnv_pci_init_p5ioc2_hub(np); - /* Look for ioda2 built-in PHB3's */ - for_each_compatible_node(np, NULL, "ibm,ioda2-phb") - pnv_pci_init_ioda2_phb(np); - } + /* Look for ioda2 built-in PHB3's */ + for_each_compatible_node(np, NULL, "ibm,ioda2-phb") + pnv_pci_init_ioda2_phb(np); /* Setup the linkage between OF nodes and PHBs */ pci_devs_phb_init(); /* Configure IOMMU DMA hooks */ - ppc_md.pci_dma_dev_setup = pnv_pci_dma_dev_setup; ppc_md.tce_build = pnv_tce_build_vm; ppc_md.tce_free = pnv_tce_free_vm; ppc_md.tce_build_rm = pnv_tce_build_rm; @@ -837,3 +777,7 @@ void __init pnv_pci_init(void) } machine_subsys_initcall_sync(powernv, tce_iommu_bus_notifier_init); + +struct pci_controller_ops pnv_pci_controller_ops = { + .dma_dev_setup = pnv_pci_dma_dev_setup, +}; diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index 6c02ff8dd69f..070ee888fc95 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -23,6 +23,7 @@ enum pnv_phb_model { #define PNV_IODA_PE_BUS_ALL (1 << 2) /* PE has subordinate buses */ #define PNV_IODA_PE_MASTER (1 << 3) /* Master PE in compound case */ #define PNV_IODA_PE_SLAVE (1 << 4) /* Slave PE in compound case */ +#define PNV_IODA_PE_VF (1 << 5) /* PE for one VF */ /* Data associated with a PE, including IOMMU tracking etc.. */ struct pnv_phb; @@ -34,6 +35,9 @@ struct pnv_ioda_pe { * entire bus (& children). In the former case, pdev * is populated, in the later case, pbus is. */ +#ifdef CONFIG_PCI_IOV + struct pci_dev *parent_dev; +#endif struct pci_dev *pdev; struct pci_bus *pbus; @@ -53,7 +57,7 @@ struct pnv_ioda_pe { /* "Base" iommu table, ie, 4K TCEs, 32-bit DMA */ int tce32_seg; int tce32_segcount; - struct iommu_table tce32_table; + struct iommu_table *tce32_table; phys_addr_t tce_inval_reg_phys; /* 64-bit TCE bypass region */ @@ -75,22 +79,6 @@ struct pnv_ioda_pe { struct list_head list; }; -/* IOC dependent EEH operations */ -#ifdef CONFIG_EEH -struct pnv_eeh_ops { - int (*post_init)(struct pci_controller *hose); - int (*set_option)(struct eeh_pe *pe, int option); - int (*get_state)(struct eeh_pe *pe); - int (*reset)(struct eeh_pe *pe, int option); - int (*get_log)(struct eeh_pe *pe, int severity, - char *drv_log, unsigned long len); - int (*configure_bridge)(struct eeh_pe *pe); - int (*err_inject)(struct eeh_pe *pe, int type, int func, - unsigned long addr, unsigned long mask); - int (*next_error)(struct eeh_pe **pe); -}; -#endif /* CONFIG_EEH */ - #define PNV_PHB_FLAG_EEH (1 << 0) struct pnv_phb { @@ -104,10 +92,6 @@ struct pnv_phb { int initialized; spinlock_t lock; -#ifdef CONFIG_EEH - struct pnv_eeh_ops *eeh_ops; -#endif - #ifdef CONFIG_DEBUG_FS int has_dbgfs; struct dentry *dbgfs; @@ -165,6 +149,8 @@ struct pnv_phb { /* PE allocation bitmap */ unsigned long *pe_alloc; + /* PE allocation mutex */ + struct mutex pe_alloc_mutex; /* M32 & IO segment maps */ unsigned int *m32_segmap; @@ -179,6 +165,7 @@ struct pnv_phb { * on the sequence of creation */ struct list_head pe_list; + struct mutex pe_list_mutex; /* Reverse map of PEs, will have to extend if * we are to support more than 256 PEs, indexed @@ -213,15 +200,12 @@ struct pnv_phb { }; extern struct pci_ops pnv_pci_ops; -#ifdef CONFIG_EEH -extern struct pnv_eeh_ops ioda_eeh_ops; -#endif void pnv_pci_dump_phb_diag_data(struct pci_controller *hose, unsigned char *log_buff); -int pnv_pci_cfg_read(struct device_node *dn, +int pnv_pci_cfg_read(struct pci_dn *pdn, int where, int size, u32 *val); -int pnv_pci_cfg_write(struct device_node *dn, +int pnv_pci_cfg_write(struct pci_dn *pdn, int where, int size, u32 val); extern void pnv_pci_setup_iommu_table(struct iommu_table *tbl, void *tce_mem, u64 tce_size, @@ -232,6 +216,6 @@ extern void pnv_pci_init_ioda2_phb(struct device_node *np); extern void pnv_pci_ioda_tce_invalidate(struct iommu_table *tbl, __be64 *startp, __be64 *endp, bool rm); extern void pnv_pci_reset_secondary_bus(struct pci_dev *dev); -extern int ioda_eeh_phb_reset(struct pci_controller *hose, int option); +extern int pnv_eeh_phb_reset(struct pci_controller *hose, int option); #endif /* __POWERNV_PCI_H */ diff --git a/arch/powerpc/platforms/powernv/powernv.h b/arch/powerpc/platforms/powernv/powernv.h index 604c48e7879a..826d2c9bea56 100644 --- a/arch/powerpc/platforms/powernv/powernv.h +++ b/arch/powerpc/platforms/powernv/powernv.h @@ -29,6 +29,8 @@ static inline u64 pnv_pci_dma_get_required_mask(struct pci_dev *pdev) } #endif +extern struct pci_controller_ops pnv_pci_controller_ops; + extern u32 pnv_get_supported_cpuidle_states(void); extern void pnv_lpc_init(void); diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index d2de7d5d7574..16fdcb23f4c3 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -32,7 +32,6 @@ #include <asm/machdep.h> #include <asm/firmware.h> #include <asm/xics.h> -#include <asm/rtas.h> #include <asm/opal.h> #include <asm/kexec.h> #include <asm/smp.h> @@ -278,20 +277,6 @@ static void __init pnv_setup_machdep_opal(void) ppc_md.handle_hmi_exception = opal_handle_hmi_exception; } -#ifdef CONFIG_PPC_POWERNV_RTAS -static void __init pnv_setup_machdep_rtas(void) -{ - if (rtas_token("get-time-of-day") != RTAS_UNKNOWN_SERVICE) { - ppc_md.get_boot_time = rtas_get_boot_time; - ppc_md.get_rtc_time = rtas_get_rtc_time; - ppc_md.set_rtc_time = rtas_set_rtc_time; - } - ppc_md.restart = rtas_restart; - pm_power_off = rtas_power_off; - ppc_md.halt = rtas_halt; -} -#endif /* CONFIG_PPC_POWERNV_RTAS */ - static u32 supported_cpuidle_states; int pnv_save_sprs_for_winkle(void) @@ -409,37 +394,39 @@ static int __init pnv_init_idle_states(void) { struct device_node *power_mgt; int dt_idle_states; - const __be32 *idle_state_flags; - u32 len_flags, flags; + u32 *flags; int i; supported_cpuidle_states = 0; if (cpuidle_disable != IDLE_NO_OVERRIDE) - return 0; + goto out; if (!firmware_has_feature(FW_FEATURE_OPALv3)) - return 0; + goto out; power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); if (!power_mgt) { pr_warn("opal: PowerMgmt Node not found\n"); - return 0; + goto out; + } + dt_idle_states = of_property_count_u32_elems(power_mgt, + "ibm,cpu-idle-state-flags"); + if (dt_idle_states < 0) { + pr_warn("cpuidle-powernv: no idle states found in the DT\n"); + goto out; } - idle_state_flags = of_get_property(power_mgt, - "ibm,cpu-idle-state-flags", &len_flags); - if (!idle_state_flags) { - pr_warn("DT-PowerMgmt: missing ibm,cpu-idle-state-flags\n"); - return 0; + flags = kzalloc(sizeof(*flags) * dt_idle_states, GFP_KERNEL); + if (of_property_read_u32_array(power_mgt, + "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); + goto out_free; } - dt_idle_states = len_flags / sizeof(u32); + for (i = 0; i < dt_idle_states; i++) + supported_cpuidle_states |= flags[i]; - for (i = 0; i < dt_idle_states; i++) { - flags = be32_to_cpu(idle_state_flags[i]); - supported_cpuidle_states |= flags; - } if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { patch_instruction( (unsigned int *)pnv_fastsleep_workaround_at_entry, @@ -449,6 +436,9 @@ static int __init pnv_init_idle_states(void) PPC_INST_NOP); } pnv_alloc_idle_core_states(); +out_free: + kfree(flags); +out: return 0; } @@ -465,10 +455,6 @@ static int __init pnv_probe(void) if (firmware_has_feature(FW_FEATURE_OPAL)) pnv_setup_machdep_opal(); -#ifdef CONFIG_PPC_POWERNV_RTAS - else if (rtas.base) - pnv_setup_machdep_rtas(); -#endif /* CONFIG_PPC_POWERNV_RTAS */ pr_debug("PowerNV detected !\n"); diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index 38a45088f633..8f70ba681a78 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -25,7 +25,6 @@ #include <asm/machdep.h> #include <asm/cputable.h> #include <asm/firmware.h> -#include <asm/rtas.h> #include <asm/vdso_datapage.h> #include <asm/cputhreads.h> #include <asm/xics.h> @@ -251,18 +250,6 @@ void __init pnv_smp_init(void) { smp_ops = &pnv_smp_ops; - /* XXX We don't yet have a proper entry point from HAL, for - * now we rely on kexec-style entry from BML - */ - -#ifdef CONFIG_PPC_RTAS - /* Non-lpar has additional take/give timebase */ - if (rtas_token("freeze-time-base") != RTAS_UNKNOWN_SERVICE) { - smp_ops->give_timebase = rtas_give_timebase; - smp_ops->take_timebase = rtas_take_timebase; - } -#endif /* CONFIG_PPC_RTAS */ - #ifdef CONFIG_HOTPLUG_CPU ppc_md.cpu_die = pnv_smp_cpu_kill_self; #endif diff --git a/arch/powerpc/platforms/ps3/smp.c b/arch/powerpc/platforms/ps3/smp.c index b358bec6c8cb..3c7707af3384 100644 --- a/arch/powerpc/platforms/ps3/smp.c +++ b/arch/powerpc/platforms/ps3/smp.c @@ -57,7 +57,7 @@ static void ps3_smp_message_pass(int cpu, int msg) " (%d)\n", __func__, __LINE__, cpu, msg, result); } -static int __init ps3_smp_probe(void) +static void __init ps3_smp_probe(void) { int cpu; @@ -100,8 +100,6 @@ static int __init ps3_smp_probe(void) DBG(" <- %s:%d: (%d)\n", __func__, __LINE__, cpu); } - - return 2; } void ps3_smp_cleanup_cpu(int cpu) diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index a758a9c3bbba..54c87d5d349d 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -16,7 +16,6 @@ config PPC_PSERIES select PPC_UDBG_16550 select PPC_NATIVE select PPC_PCI_CHOICE if EXPERT - select ZLIB_DEFLATE select PPC_DOORBELL select HAVE_CONTEXT_TRACKING select HOTPLUG_CPU if SMP diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c index c22bb1b4beb8..b4b11096ea8b 100644 --- a/arch/powerpc/platforms/pseries/dlpar.c +++ b/arch/powerpc/platforms/pseries/dlpar.c @@ -10,6 +10,8 @@ * 2 as published by the Free Software Foundation. */ +#define pr_fmt(fmt) "dlpar: " fmt + #include <linux/kernel.h> #include <linux/notifier.h> #include <linux/spinlock.h> @@ -535,13 +537,125 @@ static ssize_t dlpar_cpu_release(const char *buf, size_t count) return count; } +#endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */ + +static int handle_dlpar_errorlog(struct pseries_hp_errorlog *hp_elog) +{ + int rc; + + /* pseries error logs are in BE format, convert to cpu type */ + switch (hp_elog->id_type) { + case PSERIES_HP_ELOG_ID_DRC_COUNT: + hp_elog->_drc_u.drc_count = + be32_to_cpu(hp_elog->_drc_u.drc_count); + break; + case PSERIES_HP_ELOG_ID_DRC_INDEX: + hp_elog->_drc_u.drc_index = + be32_to_cpu(hp_elog->_drc_u.drc_index); + } + + switch (hp_elog->resource) { + case PSERIES_HP_ELOG_RESOURCE_MEM: + rc = dlpar_memory(hp_elog); + break; + default: + pr_warn_ratelimited("Invalid resource (%d) specified\n", + hp_elog->resource); + rc = -EINVAL; + } + + return rc; +} + +static ssize_t dlpar_store(struct class *class, struct class_attribute *attr, + const char *buf, size_t count) +{ + struct pseries_hp_errorlog *hp_elog; + const char *arg; + int rc; + + hp_elog = kzalloc(sizeof(*hp_elog), GFP_KERNEL); + if (!hp_elog) { + rc = -ENOMEM; + goto dlpar_store_out; + } + + /* Parse out the request from the user, this will be in the form + * <resource> <action> <id_type> <id> + */ + arg = buf; + if (!strncmp(arg, "memory", 6)) { + hp_elog->resource = PSERIES_HP_ELOG_RESOURCE_MEM; + arg += strlen("memory "); + } else { + pr_err("Invalid resource specified: \"%s\"\n", buf); + rc = -EINVAL; + goto dlpar_store_out; + } + + if (!strncmp(arg, "add", 3)) { + hp_elog->action = PSERIES_HP_ELOG_ACTION_ADD; + arg += strlen("add "); + } else if (!strncmp(arg, "remove", 6)) { + hp_elog->action = PSERIES_HP_ELOG_ACTION_REMOVE; + arg += strlen("remove "); + } else { + pr_err("Invalid action specified: \"%s\"\n", buf); + rc = -EINVAL; + goto dlpar_store_out; + } + + if (!strncmp(arg, "index", 5)) { + u32 index; + + hp_elog->id_type = PSERIES_HP_ELOG_ID_DRC_INDEX; + arg += strlen("index "); + if (kstrtou32(arg, 0, &index)) { + rc = -EINVAL; + pr_err("Invalid drc_index specified: \"%s\"\n", buf); + goto dlpar_store_out; + } + + hp_elog->_drc_u.drc_index = cpu_to_be32(index); + } else if (!strncmp(arg, "count", 5)) { + u32 count; + + hp_elog->id_type = PSERIES_HP_ELOG_ID_DRC_COUNT; + arg += strlen("count "); + if (kstrtou32(arg, 0, &count)) { + rc = -EINVAL; + pr_err("Invalid count specified: \"%s\"\n", buf); + goto dlpar_store_out; + } + + hp_elog->_drc_u.drc_count = cpu_to_be32(count); + } else { + pr_err("Invalid id_type specified: \"%s\"\n", buf); + rc = -EINVAL; + goto dlpar_store_out; + } + + rc = handle_dlpar_errorlog(hp_elog); + +dlpar_store_out: + kfree(hp_elog); + return rc ? rc : count; +} + +static CLASS_ATTR(dlpar, S_IWUSR, NULL, dlpar_store); + static int __init pseries_dlpar_init(void) { + int rc; + +#ifdef CONFIG_ARCH_CPU_PROBE_RELEASE ppc_md.cpu_probe = dlpar_cpu_probe; ppc_md.cpu_release = dlpar_cpu_release; +#endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */ - return 0; + rc = sysfs_create_file(kernel_kobj, &class_attr_dlpar.attr); + + return rc; } machine_device_initcall(pseries, pseries_dlpar_init); -#endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */ diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c index a6c7e19f5eb3..2039397cc75d 100644 --- a/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -118,9 +118,8 @@ static int pseries_eeh_init(void) return 0; } -static int pseries_eeh_cap_start(struct device_node *dn) +static int pseries_eeh_cap_start(struct pci_dn *pdn) { - struct pci_dn *pdn = PCI_DN(dn); u32 status; if (!pdn) @@ -134,10 +133,9 @@ static int pseries_eeh_cap_start(struct device_node *dn) } -static int pseries_eeh_find_cap(struct device_node *dn, int cap) +static int pseries_eeh_find_cap(struct pci_dn *pdn, int cap) { - struct pci_dn *pdn = PCI_DN(dn); - int pos = pseries_eeh_cap_start(dn); + int pos = pseries_eeh_cap_start(pdn); int cnt = 48; /* Maximal number of capabilities */ u32 id; @@ -160,10 +158,9 @@ static int pseries_eeh_find_cap(struct device_node *dn, int cap) return 0; } -static int pseries_eeh_find_ecap(struct device_node *dn, int cap) +static int pseries_eeh_find_ecap(struct pci_dn *pdn, int cap) { - struct pci_dn *pdn = PCI_DN(dn); - struct eeh_dev *edev = of_node_to_eeh_dev(dn); + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); u32 header; int pos = 256; int ttl = (4096 - 256) / 8; @@ -191,53 +188,44 @@ static int pseries_eeh_find_ecap(struct device_node *dn, int cap) } /** - * pseries_eeh_of_probe - EEH probe on the given device - * @dn: OF node - * @flag: Unused + * pseries_eeh_probe - EEH probe on the given device + * @pdn: PCI device node + * @data: Unused * * When EEH module is installed during system boot, all PCI devices * are checked one by one to see if it supports EEH. The function * is introduced for the purpose. */ -static void *pseries_eeh_of_probe(struct device_node *dn, void *flag) +static void *pseries_eeh_probe(struct pci_dn *pdn, void *data) { struct eeh_dev *edev; struct eeh_pe pe; - struct pci_dn *pdn = PCI_DN(dn); - const __be32 *classp, *vendorp, *devicep; - u32 class_code; - const __be32 *regs; u32 pcie_flags; int enable = 0; int ret; /* Retrieve OF node and eeh device */ - edev = of_node_to_eeh_dev(dn); - if (edev->pe || !of_device_is_available(dn)) + edev = pdn_to_eeh_dev(pdn); + if (!edev || edev->pe) return NULL; - /* Retrieve class/vendor/device IDs */ - classp = of_get_property(dn, "class-code", NULL); - vendorp = of_get_property(dn, "vendor-id", NULL); - devicep = of_get_property(dn, "device-id", NULL); - - /* Skip for bad OF node or PCI-ISA bridge */ - if (!classp || !vendorp || !devicep) - return NULL; - if (dn->type && !strcmp(dn->type, "isa")) + /* Check class/vendor/device IDs */ + if (!pdn->vendor_id || !pdn->device_id || !pdn->class_code) return NULL; - class_code = of_read_number(classp, 1); + /* Skip for PCI-ISA bridge */ + if ((pdn->class_code >> 8) == PCI_CLASS_BRIDGE_ISA) + return NULL; /* * Update class code and mode of eeh device. We need * correctly reflects that current device is root port * or PCIe switch downstream port. */ - edev->class_code = class_code; - edev->pcix_cap = pseries_eeh_find_cap(dn, PCI_CAP_ID_PCIX); - edev->pcie_cap = pseries_eeh_find_cap(dn, PCI_CAP_ID_EXP); - edev->aer_cap = pseries_eeh_find_ecap(dn, PCI_EXT_CAP_ID_ERR); + edev->class_code = pdn->class_code; + edev->pcix_cap = pseries_eeh_find_cap(pdn, PCI_CAP_ID_PCIX); + edev->pcie_cap = pseries_eeh_find_cap(pdn, PCI_CAP_ID_EXP); + edev->aer_cap = pseries_eeh_find_ecap(pdn, PCI_EXT_CAP_ID_ERR); edev->mode &= 0xFFFFFF00; if ((edev->class_code >> 8) == PCI_CLASS_BRIDGE_PCI) { edev->mode |= EEH_DEV_BRIDGE; @@ -252,24 +240,16 @@ static void *pseries_eeh_of_probe(struct device_node *dn, void *flag) } } - /* Retrieve the device address */ - regs = of_get_property(dn, "reg", NULL); - if (!regs) { - pr_warn("%s: OF node property %s::reg not found\n", - __func__, dn->full_name); - return NULL; - } - /* Initialize the fake PE */ memset(&pe, 0, sizeof(struct eeh_pe)); pe.phb = edev->phb; - pe.config_addr = of_read_number(regs, 1); + pe.config_addr = (pdn->busno << 16) | (pdn->devfn << 8); /* Enable EEH on the device */ ret = eeh_ops->set_option(&pe, EEH_OPT_ENABLE); if (!ret) { - edev->config_addr = of_read_number(regs, 1); /* Retrieve PE address */ + edev->config_addr = (pdn->busno << 16) | (pdn->devfn << 8); edev->pe_config_addr = eeh_ops->get_pe_addr(&pe); pe.addr = edev->pe_config_addr; @@ -285,16 +265,17 @@ static void *pseries_eeh_of_probe(struct device_node *dn, void *flag) eeh_add_flag(EEH_ENABLED); eeh_add_to_parent_pe(edev); - pr_debug("%s: EEH enabled on %s PHB#%d-PE#%x, config addr#%x\n", - __func__, dn->full_name, pe.phb->global_number, - pe.addr, pe.config_addr); - } else if (dn->parent && of_node_to_eeh_dev(dn->parent) && - (of_node_to_eeh_dev(dn->parent))->pe) { + pr_debug("%s: EEH enabled on %02x:%02x.%01x PHB#%d-PE#%x\n", + __func__, pdn->busno, PCI_SLOT(pdn->devfn), + PCI_FUNC(pdn->devfn), pe.phb->global_number, + pe.addr); + } else if (pdn->parent && pdn_to_eeh_dev(pdn->parent) && + (pdn_to_eeh_dev(pdn->parent))->pe) { /* This device doesn't support EEH, but it may have an * EEH parent, in which case we mark it as supported. */ - edev->config_addr = of_node_to_eeh_dev(dn->parent)->config_addr; - edev->pe_config_addr = of_node_to_eeh_dev(dn->parent)->pe_config_addr; + edev->config_addr = pdn_to_eeh_dev(pdn->parent)->config_addr; + edev->pe_config_addr = pdn_to_eeh_dev(pdn->parent)->pe_config_addr; eeh_add_to_parent_pe(edev); } } @@ -670,45 +651,36 @@ static int pseries_eeh_configure_bridge(struct eeh_pe *pe) /** * pseries_eeh_read_config - Read PCI config space - * @dn: device node + * @pdn: PCI device node * @where: PCI address * @size: size to read * @val: return value * * Read config space from the speicifed device */ -static int pseries_eeh_read_config(struct device_node *dn, int where, int size, u32 *val) +static int pseries_eeh_read_config(struct pci_dn *pdn, int where, int size, u32 *val) { - struct pci_dn *pdn; - - pdn = PCI_DN(dn); - return rtas_read_config(pdn, where, size, val); } /** * pseries_eeh_write_config - Write PCI config space - * @dn: device node + * @pdn: PCI device node * @where: PCI address * @size: size to write * @val: value to be written * * Write config space to the specified device */ -static int pseries_eeh_write_config(struct device_node *dn, int where, int size, u32 val) +static int pseries_eeh_write_config(struct pci_dn *pdn, int where, int size, u32 val) { - struct pci_dn *pdn; - - pdn = PCI_DN(dn); - return rtas_write_config(pdn, where, size, val); } static struct eeh_ops pseries_eeh_ops = { .name = "pseries", .init = pseries_eeh_init, - .of_probe = pseries_eeh_of_probe, - .dev_probe = NULL, + .probe = pseries_eeh_probe, .set_option = pseries_eeh_set_option, .get_pe_addr = pseries_eeh_get_pe_addr, .get_state = pseries_eeh_get_state, diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c index fa41f0da5b6f..0ced387e1463 100644 --- a/arch/powerpc/platforms/pseries/hotplug-memory.c +++ b/arch/powerpc/platforms/pseries/hotplug-memory.c @@ -9,11 +9,14 @@ * 2 of the License, or (at your option) any later version. */ +#define pr_fmt(fmt) "pseries-hotplug-mem: " fmt + #include <linux/of.h> #include <linux/of_address.h> #include <linux/memblock.h> #include <linux/memory.h> #include <linux/memory_hotplug.h> +#include <linux/slab.h> #include <asm/firmware.h> #include <asm/machdep.h> @@ -21,6 +24,8 @@ #include <asm/sparsemem.h> #include "pseries.h" +static bool rtas_hp_event; + unsigned long pseries_memory_block_size(void) { struct device_node *np; @@ -64,6 +69,67 @@ unsigned long pseries_memory_block_size(void) return memblock_size; } +static void dlpar_free_drconf_property(struct property *prop) +{ + kfree(prop->name); + kfree(prop->value); + kfree(prop); +} + +static struct property *dlpar_clone_drconf_property(struct device_node *dn) +{ + struct property *prop, *new_prop; + struct of_drconf_cell *lmbs; + u32 num_lmbs, *p; + int i; + + prop = of_find_property(dn, "ibm,dynamic-memory", NULL); + if (!prop) + return NULL; + + new_prop = kzalloc(sizeof(*new_prop), GFP_KERNEL); + if (!new_prop) + return NULL; + + new_prop->name = kstrdup(prop->name, GFP_KERNEL); + new_prop->value = kmalloc(prop->length, GFP_KERNEL); + if (!new_prop->name || !new_prop->value) { + dlpar_free_drconf_property(new_prop); + return NULL; + } + + memcpy(new_prop->value, prop->value, prop->length); + new_prop->length = prop->length; + + /* Convert the property to cpu endian-ness */ + p = new_prop->value; + *p = be32_to_cpu(*p); + + num_lmbs = *p++; + lmbs = (struct of_drconf_cell *)p; + + for (i = 0; i < num_lmbs; i++) { + lmbs[i].base_addr = be64_to_cpu(lmbs[i].base_addr); + lmbs[i].drc_index = be32_to_cpu(lmbs[i].drc_index); + lmbs[i].flags = be32_to_cpu(lmbs[i].flags); + } + + return new_prop; +} + +static struct memory_block *lmb_to_memblock(struct of_drconf_cell *lmb) +{ + unsigned long section_nr; + struct mem_section *mem_sect; + struct memory_block *mem_block; + + section_nr = pfn_to_section_nr(PFN_DOWN(lmb->base_addr)); + mem_sect = __nr_to_section(section_nr); + + mem_block = find_memory_block(mem_sect); + return mem_block; +} + #ifdef CONFIG_MEMORY_HOTREMOVE static int pseries_remove_memblock(unsigned long base, unsigned int memblock_size) { @@ -122,6 +188,173 @@ static int pseries_remove_mem_node(struct device_node *np) pseries_remove_memblock(base, lmb_size); return 0; } + +static bool lmb_is_removable(struct of_drconf_cell *lmb) +{ + int i, scns_per_block; + int rc = 1; + unsigned long pfn, block_sz; + u64 phys_addr; + + if (!(lmb->flags & DRCONF_MEM_ASSIGNED)) + return false; + + block_sz = memory_block_size_bytes(); + scns_per_block = block_sz / MIN_MEMORY_BLOCK_SIZE; + phys_addr = lmb->base_addr; + + for (i = 0; i < scns_per_block; i++) { + pfn = PFN_DOWN(phys_addr); + if (!pfn_present(pfn)) + continue; + + rc &= is_mem_section_removable(pfn, PAGES_PER_SECTION); + phys_addr += MIN_MEMORY_BLOCK_SIZE; + } + + return rc ? true : false; +} + +static int dlpar_add_lmb(struct of_drconf_cell *); + +static int dlpar_remove_lmb(struct of_drconf_cell *lmb) +{ + struct memory_block *mem_block; + unsigned long block_sz; + int nid, rc; + + if (!lmb_is_removable(lmb)) + return -EINVAL; + + mem_block = lmb_to_memblock(lmb); + if (!mem_block) + return -EINVAL; + + rc = device_offline(&mem_block->dev); + put_device(&mem_block->dev); + if (rc) + return rc; + + block_sz = pseries_memory_block_size(); + nid = memory_add_physaddr_to_nid(lmb->base_addr); + + remove_memory(nid, lmb->base_addr, block_sz); + + /* Update memory regions for memory remove */ + memblock_remove(lmb->base_addr, block_sz); + + dlpar_release_drc(lmb->drc_index); + + lmb->flags &= ~DRCONF_MEM_ASSIGNED; + return 0; +} + +static int dlpar_memory_remove_by_count(u32 lmbs_to_remove, + struct property *prop) +{ + struct of_drconf_cell *lmbs; + int lmbs_removed = 0; + int lmbs_available = 0; + u32 num_lmbs, *p; + int i, rc; + + pr_info("Attempting to hot-remove %d LMB(s)\n", lmbs_to_remove); + + if (lmbs_to_remove == 0) + return -EINVAL; + + p = prop->value; + num_lmbs = *p++; + lmbs = (struct of_drconf_cell *)p; + + /* Validate that there are enough LMBs to satisfy the request */ + for (i = 0; i < num_lmbs; i++) { + if (lmbs[i].flags & DRCONF_MEM_ASSIGNED) + lmbs_available++; + } + + if (lmbs_available < lmbs_to_remove) + return -EINVAL; + + for (i = 0; i < num_lmbs && lmbs_removed < lmbs_to_remove; i++) { + rc = dlpar_remove_lmb(&lmbs[i]); + if (rc) + continue; + + lmbs_removed++; + + /* Mark this lmb so we can add it later if all of the + * requested LMBs cannot be removed. + */ + lmbs[i].reserved = 1; + } + + if (lmbs_removed != lmbs_to_remove) { + pr_err("Memory hot-remove failed, adding LMB's back\n"); + + for (i = 0; i < num_lmbs; i++) { + if (!lmbs[i].reserved) + continue; + + rc = dlpar_add_lmb(&lmbs[i]); + if (rc) + pr_err("Failed to add LMB back, drc index %x\n", + lmbs[i].drc_index); + + lmbs[i].reserved = 0; + } + + rc = -EINVAL; + } else { + for (i = 0; i < num_lmbs; i++) { + if (!lmbs[i].reserved) + continue; + + pr_info("Memory at %llx was hot-removed\n", + lmbs[i].base_addr); + + lmbs[i].reserved = 0; + } + rc = 0; + } + + return rc; +} + +static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop) +{ + struct of_drconf_cell *lmbs; + u32 num_lmbs, *p; + int lmb_found; + int i, rc; + + pr_info("Attempting to hot-remove LMB, drc index %x\n", drc_index); + + p = prop->value; + num_lmbs = *p++; + lmbs = (struct of_drconf_cell *)p; + + lmb_found = 0; + for (i = 0; i < num_lmbs; i++) { + if (lmbs[i].drc_index == drc_index) { + lmb_found = 1; + rc = dlpar_remove_lmb(&lmbs[i]); + break; + } + } + + if (!lmb_found) + rc = -EINVAL; + + if (rc) + pr_info("Failed to hot-remove memory at %llx\n", + lmbs[i].base_addr); + else + pr_info("Memory at %llx was hot-removed\n", lmbs[i].base_addr); + + return rc; +} + #else static inline int pseries_remove_memblock(unsigned long base, unsigned int memblock_size) @@ -132,8 +365,261 @@ static inline int pseries_remove_mem_node(struct device_node *np) { return 0; } +static inline int dlpar_memory_remove(struct pseries_hp_errorlog *hp_elog) +{ + return -EOPNOTSUPP; +} +static int dlpar_remove_lmb(struct of_drconf_cell *lmb) +{ + return -EOPNOTSUPP; +} +static int dlpar_memory_remove_by_count(u32 lmbs_to_remove, + struct property *prop) +{ + return -EOPNOTSUPP; +} +static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop) +{ + return -EOPNOTSUPP; +} + #endif /* CONFIG_MEMORY_HOTREMOVE */ +static int dlpar_add_lmb(struct of_drconf_cell *lmb) +{ + struct memory_block *mem_block; + unsigned long block_sz; + int nid, rc; + + if (lmb->flags & DRCONF_MEM_ASSIGNED) + return -EINVAL; + + block_sz = memory_block_size_bytes(); + + rc = dlpar_acquire_drc(lmb->drc_index); + if (rc) + return rc; + + /* Find the node id for this address */ + nid = memory_add_physaddr_to_nid(lmb->base_addr); + + /* Add the memory */ + rc = add_memory(nid, lmb->base_addr, block_sz); + if (rc) { + dlpar_release_drc(lmb->drc_index); + return rc; + } + + /* Register this block of memory */ + rc = memblock_add(lmb->base_addr, block_sz); + if (rc) { + remove_memory(nid, lmb->base_addr, block_sz); + dlpar_release_drc(lmb->drc_index); + return rc; + } + + mem_block = lmb_to_memblock(lmb); + if (!mem_block) { + remove_memory(nid, lmb->base_addr, block_sz); + dlpar_release_drc(lmb->drc_index); + return -EINVAL; + } + + rc = device_online(&mem_block->dev); + put_device(&mem_block->dev); + if (rc) { + remove_memory(nid, lmb->base_addr, block_sz); + dlpar_release_drc(lmb->drc_index); + return rc; + } + + lmb->flags |= DRCONF_MEM_ASSIGNED; + return 0; +} + +static int dlpar_memory_add_by_count(u32 lmbs_to_add, struct property *prop) +{ + struct of_drconf_cell *lmbs; + u32 num_lmbs, *p; + int lmbs_available = 0; + int lmbs_added = 0; + int i, rc; + + pr_info("Attempting to hot-add %d LMB(s)\n", lmbs_to_add); + + if (lmbs_to_add == 0) + return -EINVAL; + + p = prop->value; + num_lmbs = *p++; + lmbs = (struct of_drconf_cell *)p; + + /* Validate that there are enough LMBs to satisfy the request */ + for (i = 0; i < num_lmbs; i++) { + if (!(lmbs[i].flags & DRCONF_MEM_ASSIGNED)) + lmbs_available++; + } + + if (lmbs_available < lmbs_to_add) + return -EINVAL; + + for (i = 0; i < num_lmbs && lmbs_to_add != lmbs_added; i++) { + rc = dlpar_add_lmb(&lmbs[i]); + if (rc) + continue; + + lmbs_added++; + + /* Mark this lmb so we can remove it later if all of the + * requested LMBs cannot be added. + */ + lmbs[i].reserved = 1; + } + + if (lmbs_added != lmbs_to_add) { + pr_err("Memory hot-add failed, removing any added LMBs\n"); + + for (i = 0; i < num_lmbs; i++) { + if (!lmbs[i].reserved) + continue; + + rc = dlpar_remove_lmb(&lmbs[i]); + if (rc) + pr_err("Failed to remove LMB, drc index %x\n", + be32_to_cpu(lmbs[i].drc_index)); + } + rc = -EINVAL; + } else { + for (i = 0; i < num_lmbs; i++) { + if (!lmbs[i].reserved) + continue; + + pr_info("Memory at %llx (drc index %x) was hot-added\n", + lmbs[i].base_addr, lmbs[i].drc_index); + lmbs[i].reserved = 0; + } + } + + return rc; +} + +static int dlpar_memory_add_by_index(u32 drc_index, struct property *prop) +{ + struct of_drconf_cell *lmbs; + u32 num_lmbs, *p; + int i, lmb_found; + int rc; + + pr_info("Attempting to hot-add LMB, drc index %x\n", drc_index); + + p = prop->value; + num_lmbs = *p++; + lmbs = (struct of_drconf_cell *)p; + + lmb_found = 0; + for (i = 0; i < num_lmbs; i++) { + if (lmbs[i].drc_index == drc_index) { + lmb_found = 1; + rc = dlpar_add_lmb(&lmbs[i]); + break; + } + } + + if (!lmb_found) + rc = -EINVAL; + + if (rc) + pr_info("Failed to hot-add memory, drc index %x\n", drc_index); + else + pr_info("Memory at %llx (drc index %x) was hot-added\n", + lmbs[i].base_addr, drc_index); + + return rc; +} + +static void dlpar_update_drconf_property(struct device_node *dn, + struct property *prop) +{ + struct of_drconf_cell *lmbs; + u32 num_lmbs, *p; + int i; + + /* Convert the property back to BE */ + p = prop->value; + num_lmbs = *p; + *p = cpu_to_be32(*p); + p++; + + lmbs = (struct of_drconf_cell *)p; + for (i = 0; i < num_lmbs; i++) { + lmbs[i].base_addr = cpu_to_be64(lmbs[i].base_addr); + lmbs[i].drc_index = cpu_to_be32(lmbs[i].drc_index); + lmbs[i].flags = cpu_to_be32(lmbs[i].flags); + } + + rtas_hp_event = true; + of_update_property(dn, prop); + rtas_hp_event = false; +} + +int dlpar_memory(struct pseries_hp_errorlog *hp_elog) +{ + struct device_node *dn; + struct property *prop; + u32 count, drc_index; + int rc; + + count = hp_elog->_drc_u.drc_count; + drc_index = hp_elog->_drc_u.drc_index; + + lock_device_hotplug(); + + dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); + if (!dn) { + rc = -EINVAL; + goto dlpar_memory_out; + } + + prop = dlpar_clone_drconf_property(dn); + if (!prop) { + rc = -EINVAL; + goto dlpar_memory_out; + } + + switch (hp_elog->action) { + case PSERIES_HP_ELOG_ACTION_ADD: + if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT) + rc = dlpar_memory_add_by_count(count, prop); + else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) + rc = dlpar_memory_add_by_index(drc_index, prop); + else + rc = -EINVAL; + break; + case PSERIES_HP_ELOG_ACTION_REMOVE: + if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT) + rc = dlpar_memory_remove_by_count(count, prop); + else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) + rc = dlpar_memory_remove_by_index(drc_index, prop); + else + rc = -EINVAL; + break; + default: + pr_err("Invalid action (%d) specified\n", hp_elog->action); + rc = -EINVAL; + break; + } + + if (rc) + dlpar_free_drconf_property(prop); + else + dlpar_update_drconf_property(dn, prop); + +dlpar_memory_out: + of_node_put(dn); + unlock_device_hotplug(); + return rc; +} + static int pseries_add_mem_node(struct device_node *np) { const char *type; @@ -174,6 +660,9 @@ static int pseries_update_drconf_memory(struct of_reconfig_data *pr) __be32 *p; int i, rc = -EINVAL; + if (rtas_hp_event) + return 0; + memblock_size = pseries_memory_block_size(); if (!memblock_size) return -EINVAL; diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 7803a19adb31..61d5a17f45c0 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -49,6 +49,7 @@ #include <asm/mmzone.h> #include <asm/plpar_wrappers.h> +#include "pseries.h" static void tce_invalidate_pSeries_sw(struct iommu_table *tbl, __be64 *startp, __be64 *endp) @@ -1307,16 +1308,16 @@ void iommu_init_early_pSeries(void) ppc_md.tce_free = tce_free_pSeriesLP; } ppc_md.tce_get = tce_get_pSeriesLP; - ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeriesLP; - ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeriesLP; + pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeriesLP; + pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeriesLP; ppc_md.dma_set_mask = dma_set_mask_pSeriesLP; ppc_md.dma_get_required_mask = dma_get_required_mask_pSeriesLP; } else { ppc_md.tce_build = tce_build_pSeries; ppc_md.tce_free = tce_free_pSeries; ppc_md.tce_get = tce_get_pseries; - ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_pSeries; - ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_pSeries; + pseries_pci_controller_ops.dma_bus_setup = pci_dma_bus_setup_pSeries; + pseries_pci_controller_ops.dma_dev_setup = pci_dma_dev_setup_pSeries; } diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c index 8f35d525cede..ceb18d349459 100644 --- a/arch/powerpc/platforms/pseries/mobility.c +++ b/arch/powerpc/platforms/pseries/mobility.c @@ -320,28 +320,34 @@ static ssize_t migrate_store(struct class *class, struct class_attribute *attr, { u64 streamid; int rc; - int vasi_rc = 0; rc = kstrtou64(buf, 0, &streamid); if (rc) return rc; do { - rc = rtas_ibm_suspend_me(streamid, &vasi_rc); - if (!rc && vasi_rc == RTAS_NOT_SUSPENDABLE) + rc = rtas_ibm_suspend_me(streamid); + if (rc == -EAGAIN) ssleep(1); - } while (!rc && vasi_rc == RTAS_NOT_SUSPENDABLE); + } while (rc == -EAGAIN); if (rc) return rc; - if (vasi_rc) - return vasi_rc; post_mobility_fixup(); return count; } +/* + * Used by drmgr to determine the kernel behavior of the migration interface. + * + * Version 1: Performs all PAPR requirements for migration including + * firmware activation and device tree update. + */ +#define MIGRATION_API_VERSION 1 + static CLASS_ATTR(migration, S_IWUSR, NULL, migrate_store); +static CLASS_ATTR_STRING(api_version, S_IRUGO, __stringify(MIGRATION_API_VERSION)); static int __init mobility_sysfs_init(void) { @@ -352,7 +358,13 @@ static int __init mobility_sysfs_init(void) return -ENOMEM; rc = sysfs_create_file(mobility_kobj, &class_attr_migration.attr); + if (rc) + pr_err("mobility: unable to create migration sysfs file (%d)\n", rc); - return rc; + rc = sysfs_create_file(mobility_kobj, &class_attr_api_version.attr.attr); + if (rc) + pr_err("mobility: unable to create api_version sysfs file (%d)\n", rc); + + return 0; } machine_device_initcall(pseries, mobility_sysfs_init); diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index 691a154c286d..c8d24f9a6948 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -195,6 +195,7 @@ static struct device_node *find_pe_total_msi(struct pci_dev *dev, int *total) static struct device_node *find_pe_dn(struct pci_dev *dev, int *total) { struct device_node *dn; + struct pci_dn *pdn; struct eeh_dev *edev; /* Found our PE and assume 8 at that point. */ @@ -204,10 +205,11 @@ static struct device_node *find_pe_dn(struct pci_dev *dev, int *total) return NULL; /* Get the top level device in the PE */ - edev = of_node_to_eeh_dev(dn); + edev = pdn_to_eeh_dev(PCI_DN(dn)); if (edev->pe) edev = list_first_entry(&edev->pe->edevs, struct eeh_dev, list); - dn = eeh_dev_to_of_node(edev); + pdn = eeh_dev_to_pdn(edev); + dn = pdn ? pdn->node : NULL; if (!dn) return NULL; diff --git a/arch/powerpc/platforms/pseries/nvram.c b/arch/powerpc/platforms/pseries/nvram.c index 054a0ed5c7ee..9f8184175c86 100644 --- a/arch/powerpc/platforms/pseries/nvram.c +++ b/arch/powerpc/platforms/pseries/nvram.c @@ -20,7 +20,6 @@ #include <linux/kmsg_dump.h> #include <linux/pstore.h> #include <linux/ctype.h> -#include <linux/zlib.h> #include <asm/uaccess.h> #include <asm/nvram.h> #include <asm/rtas.h> @@ -30,129 +29,17 @@ /* Max bytes to read/write in one go */ #define NVRW_CNT 0x20 -/* - * Set oops header version to distinguish between old and new format header. - * lnx,oops-log partition max size is 4000, header version > 4000 will - * help in identifying new header. - */ -#define OOPS_HDR_VERSION 5000 - static unsigned int nvram_size; static int nvram_fetch, nvram_store; static char nvram_buf[NVRW_CNT]; /* assume this is in the first 4GB */ static DEFINE_SPINLOCK(nvram_lock); -struct err_log_info { - __be32 error_type; - __be32 seq_num; -}; - -struct nvram_os_partition { - const char *name; - int req_size; /* desired size, in bytes */ - int min_size; /* minimum acceptable size (0 means req_size) */ - long size; /* size of data portion (excluding err_log_info) */ - long index; /* offset of data portion of partition */ - bool os_partition; /* partition initialized by OS, not FW */ -}; - -static struct nvram_os_partition rtas_log_partition = { - .name = "ibm,rtas-log", - .req_size = 2079, - .min_size = 1055, - .index = -1, - .os_partition = true -}; - -static struct nvram_os_partition oops_log_partition = { - .name = "lnx,oops-log", - .req_size = 4000, - .min_size = 2000, - .index = -1, - .os_partition = true -}; - -static const char *pseries_nvram_os_partitions[] = { - "ibm,rtas-log", - "lnx,oops-log", - NULL -}; - -struct oops_log_info { - __be16 version; - __be16 report_length; - __be64 timestamp; -} __attribute__((packed)); - -static void oops_to_nvram(struct kmsg_dumper *dumper, - enum kmsg_dump_reason reason); - -static struct kmsg_dumper nvram_kmsg_dumper = { - .dump = oops_to_nvram -}; - /* See clobbering_unread_rtas_event() */ #define NVRAM_RTAS_READ_TIMEOUT 5 /* seconds */ -static unsigned long last_unread_rtas_event; /* timestamp */ - -/* - * For capturing and compressing an oops or panic report... - - * big_oops_buf[] holds the uncompressed text we're capturing. - * - * oops_buf[] holds the compressed text, preceded by a oops header. - * oops header has u16 holding the version of oops header (to differentiate - * between old and new format header) followed by u16 holding the length of - * the compressed* text (*Or uncompressed, if compression fails.) and u64 - * holding the timestamp. oops_buf[] gets written to NVRAM. - * - * oops_log_info points to the header. oops_data points to the compressed text. - * - * +- oops_buf - * | +- oops_data - * v v - * +-----------+-----------+-----------+------------------------+ - * | version | length | timestamp | text | - * | (2 bytes) | (2 bytes) | (8 bytes) | (oops_data_sz bytes) | - * +-----------+-----------+-----------+------------------------+ - * ^ - * +- oops_log_info - * - * We preallocate these buffers during init to avoid kmalloc during oops/panic. - */ -static size_t big_oops_buf_sz; -static char *big_oops_buf, *oops_buf; -static char *oops_data; -static size_t oops_data_sz; - -/* Compression parameters */ -#define COMPR_LEVEL 6 -#define WINDOW_BITS 12 -#define MEM_LEVEL 4 -static struct z_stream_s stream; +static time64_t last_unread_rtas_event; /* timestamp */ #ifdef CONFIG_PSTORE -static struct nvram_os_partition of_config_partition = { - .name = "of-config", - .index = -1, - .os_partition = false -}; - -static struct nvram_os_partition common_partition = { - .name = "common", - .index = -1, - .os_partition = false -}; - -static enum pstore_type_id nvram_type_ids[] = { - PSTORE_TYPE_DMESG, - PSTORE_TYPE_PPC_RTAS, - PSTORE_TYPE_PPC_OF, - PSTORE_TYPE_PPC_COMMON, - -1 -}; -static int read_type; -static unsigned long last_rtas_event; +time64_t last_rtas_event; #endif static ssize_t pSeries_nvram_read(char *buf, size_t count, loff_t *index) @@ -246,132 +133,26 @@ static ssize_t pSeries_nvram_get_size(void) return nvram_size ? nvram_size : -ENODEV; } - -/* nvram_write_os_partition, nvram_write_error_log +/* nvram_write_error_log * * We need to buffer the error logs into nvram to ensure that we have - * the failure information to decode. If we have a severe error there - * is no way to guarantee that the OS or the machine is in a state to - * get back to user land and write the error to disk. For example if - * the SCSI device driver causes a Machine Check by writing to a bad - * IO address, there is no way of guaranteeing that the device driver - * is in any state that is would also be able to write the error data - * captured to disk, thus we buffer it in NVRAM for analysis on the - * next boot. - * - * In NVRAM the partition containing the error log buffer will looks like: - * Header (in bytes): - * +-----------+----------+--------+------------+------------------+ - * | signature | checksum | length | name | data | - * |0 |1 |2 3|4 15|16 length-1| - * +-----------+----------+--------+------------+------------------+ - * - * The 'data' section would look like (in bytes): - * +--------------+------------+-----------------------------------+ - * | event_logged | sequence # | error log | - * |0 3|4 7|8 error_log_size-1| - * +--------------+------------+-----------------------------------+ - * - * event_logged: 0 if event has not been logged to syslog, 1 if it has - * sequence #: The unique sequence # for each event. (until it wraps) - * error log: The error log from event_scan + * the failure information to decode. */ -static int nvram_write_os_partition(struct nvram_os_partition *part, - char *buff, int length, - unsigned int err_type, - unsigned int error_log_cnt) -{ - int rc; - loff_t tmp_index; - struct err_log_info info; - - if (part->index == -1) { - return -ESPIPE; - } - - if (length > part->size) { - length = part->size; - } - - info.error_type = cpu_to_be32(err_type); - info.seq_num = cpu_to_be32(error_log_cnt); - - tmp_index = part->index; - - rc = ppc_md.nvram_write((char *)&info, sizeof(struct err_log_info), &tmp_index); - if (rc <= 0) { - pr_err("%s: Failed nvram_write (%d)\n", __func__, rc); - return rc; - } - - rc = ppc_md.nvram_write(buff, length, &tmp_index); - if (rc <= 0) { - pr_err("%s: Failed nvram_write (%d)\n", __func__, rc); - return rc; - } - - return 0; -} - int nvram_write_error_log(char * buff, int length, unsigned int err_type, unsigned int error_log_cnt) { int rc = nvram_write_os_partition(&rtas_log_partition, buff, length, err_type, error_log_cnt); if (!rc) { - last_unread_rtas_event = get_seconds(); + last_unread_rtas_event = ktime_get_real_seconds(); #ifdef CONFIG_PSTORE - last_rtas_event = get_seconds(); + last_rtas_event = ktime_get_real_seconds(); #endif } return rc; } -/* nvram_read_partition - * - * Reads nvram partition for at most 'length' - */ -static int nvram_read_partition(struct nvram_os_partition *part, char *buff, - int length, unsigned int *err_type, - unsigned int *error_log_cnt) -{ - int rc; - loff_t tmp_index; - struct err_log_info info; - - if (part->index == -1) - return -1; - - if (length > part->size) - length = part->size; - - tmp_index = part->index; - - if (part->os_partition) { - rc = ppc_md.nvram_read((char *)&info, - sizeof(struct err_log_info), - &tmp_index); - if (rc <= 0) { - pr_err("%s: Failed nvram_read (%d)\n", __func__, rc); - return rc; - } - } - - rc = ppc_md.nvram_read(buff, length, &tmp_index); - if (rc <= 0) { - pr_err("%s: Failed nvram_read (%d)\n", __func__, rc); - return rc; - } - - if (part->os_partition) { - *error_log_cnt = be32_to_cpu(info.seq_num); - *err_type = be32_to_cpu(info.error_type); - } - - return 0; -} - /* nvram_read_error_log * * Reads nvram for error log for at most 'length' @@ -407,67 +188,6 @@ int nvram_clear_error_log(void) return 0; } -/* pseries_nvram_init_os_partition - * - * This sets up a partition with an "OS" signature. - * - * The general strategy is the following: - * 1.) If a partition with the indicated name already exists... - * - If it's large enough, use it. - * - Otherwise, recycle it and keep going. - * 2.) Search for a free partition that is large enough. - * 3.) If there's not a free partition large enough, recycle any obsolete - * OS partitions and try again. - * 4.) Will first try getting a chunk that will satisfy the requested size. - * 5.) If a chunk of the requested size cannot be allocated, then try finding - * a chunk that will satisfy the minum needed. - * - * Returns 0 on success, else -1. - */ -static int __init pseries_nvram_init_os_partition(struct nvram_os_partition - *part) -{ - loff_t p; - int size; - - /* Look for ours */ - p = nvram_find_partition(part->name, NVRAM_SIG_OS, &size); - - /* Found one but too small, remove it */ - if (p && size < part->min_size) { - pr_info("nvram: Found too small %s partition," - " removing it...\n", part->name); - nvram_remove_partition(part->name, NVRAM_SIG_OS, NULL); - p = 0; - } - - /* Create one if we didn't find */ - if (!p) { - p = nvram_create_partition(part->name, NVRAM_SIG_OS, - part->req_size, part->min_size); - if (p == -ENOSPC) { - pr_info("nvram: No room to create %s partition, " - "deleting any obsolete OS partitions...\n", - part->name); - nvram_remove_partition(NULL, NVRAM_SIG_OS, - pseries_nvram_os_partitions); - p = nvram_create_partition(part->name, NVRAM_SIG_OS, - part->req_size, part->min_size); - } - } - - if (p <= 0) { - pr_err("nvram: Failed to find or create %s" - " partition, err %d\n", part->name, (int)p); - return -1; - } - - part->index = p; - part->size = nvram_get_partition_size(p) - sizeof(struct err_log_info); - - return 0; -} - /* * Are we using the ibm,rtas-log for oops/panic reports? And if so, * would logging this oops/panic overwrite an RTAS event that rtas_errd @@ -476,321 +196,14 @@ static int __init pseries_nvram_init_os_partition(struct nvram_os_partition * We assume that if rtas_errd hasn't read the RTAS event in * NVRAM_RTAS_READ_TIMEOUT seconds, it's probably not going to. */ -static int clobbering_unread_rtas_event(void) +int clobbering_unread_rtas_event(void) { return (oops_log_partition.index == rtas_log_partition.index && last_unread_rtas_event - && get_seconds() - last_unread_rtas_event <= + && ktime_get_real_seconds() - last_unread_rtas_event <= NVRAM_RTAS_READ_TIMEOUT); } -/* Derived from logfs_compress() */ -static int nvram_compress(const void *in, void *out, size_t inlen, - size_t outlen) -{ - int err, ret; - - ret = -EIO; - err = zlib_deflateInit2(&stream, COMPR_LEVEL, Z_DEFLATED, WINDOW_BITS, - MEM_LEVEL, Z_DEFAULT_STRATEGY); - if (err != Z_OK) - goto error; - - stream.next_in = in; - stream.avail_in = inlen; - stream.total_in = 0; - stream.next_out = out; - stream.avail_out = outlen; - stream.total_out = 0; - - err = zlib_deflate(&stream, Z_FINISH); - if (err != Z_STREAM_END) - goto error; - - err = zlib_deflateEnd(&stream); - if (err != Z_OK) - goto error; - - if (stream.total_out >= stream.total_in) - goto error; - - ret = stream.total_out; -error: - return ret; -} - -/* Compress the text from big_oops_buf into oops_buf. */ -static int zip_oops(size_t text_len) -{ - struct oops_log_info *oops_hdr = (struct oops_log_info *)oops_buf; - int zipped_len = nvram_compress(big_oops_buf, oops_data, text_len, - oops_data_sz); - if (zipped_len < 0) { - pr_err("nvram: compression failed; returned %d\n", zipped_len); - pr_err("nvram: logging uncompressed oops/panic report\n"); - return -1; - } - oops_hdr->version = cpu_to_be16(OOPS_HDR_VERSION); - oops_hdr->report_length = cpu_to_be16(zipped_len); - oops_hdr->timestamp = cpu_to_be64(get_seconds()); - return 0; -} - -#ifdef CONFIG_PSTORE -static int nvram_pstore_open(struct pstore_info *psi) -{ - /* Reset the iterator to start reading partitions again */ - read_type = -1; - return 0; -} - -/** - * nvram_pstore_write - pstore write callback for nvram - * @type: Type of message logged - * @reason: reason behind dump (oops/panic) - * @id: identifier to indicate the write performed - * @part: pstore writes data to registered buffer in parts, - * part number will indicate the same. - * @count: Indicates oops count - * @compressed: Flag to indicate the log is compressed - * @size: number of bytes written to the registered buffer - * @psi: registered pstore_info structure - * - * Called by pstore_dump() when an oops or panic report is logged in the - * printk buffer. - * Returns 0 on successful write. - */ -static int nvram_pstore_write(enum pstore_type_id type, - enum kmsg_dump_reason reason, - u64 *id, unsigned int part, int count, - bool compressed, size_t size, - struct pstore_info *psi) -{ - int rc; - unsigned int err_type = ERR_TYPE_KERNEL_PANIC; - struct oops_log_info *oops_hdr = (struct oops_log_info *) oops_buf; - - /* part 1 has the recent messages from printk buffer */ - if (part > 1 || type != PSTORE_TYPE_DMESG || - clobbering_unread_rtas_event()) - return -1; - - oops_hdr->version = cpu_to_be16(OOPS_HDR_VERSION); - oops_hdr->report_length = cpu_to_be16(size); - oops_hdr->timestamp = cpu_to_be64(get_seconds()); - - if (compressed) - err_type = ERR_TYPE_KERNEL_PANIC_GZ; - - rc = nvram_write_os_partition(&oops_log_partition, oops_buf, - (int) (sizeof(*oops_hdr) + size), err_type, count); - - if (rc != 0) - return rc; - - *id = part; - return 0; -} - -/* - * Reads the oops/panic report, rtas, of-config and common partition. - * Returns the length of the data we read from each partition. - * Returns 0 if we've been called before. - */ -static ssize_t nvram_pstore_read(u64 *id, enum pstore_type_id *type, - int *count, struct timespec *time, char **buf, - bool *compressed, struct pstore_info *psi) -{ - struct oops_log_info *oops_hdr; - unsigned int err_type, id_no, size = 0; - struct nvram_os_partition *part = NULL; - char *buff = NULL; - int sig = 0; - loff_t p; - - read_type++; - - switch (nvram_type_ids[read_type]) { - case PSTORE_TYPE_DMESG: - part = &oops_log_partition; - *type = PSTORE_TYPE_DMESG; - break; - case PSTORE_TYPE_PPC_RTAS: - part = &rtas_log_partition; - *type = PSTORE_TYPE_PPC_RTAS; - time->tv_sec = last_rtas_event; - time->tv_nsec = 0; - break; - case PSTORE_TYPE_PPC_OF: - sig = NVRAM_SIG_OF; - part = &of_config_partition; - *type = PSTORE_TYPE_PPC_OF; - *id = PSTORE_TYPE_PPC_OF; - time->tv_sec = 0; - time->tv_nsec = 0; - break; - case PSTORE_TYPE_PPC_COMMON: - sig = NVRAM_SIG_SYS; - part = &common_partition; - *type = PSTORE_TYPE_PPC_COMMON; - *id = PSTORE_TYPE_PPC_COMMON; - time->tv_sec = 0; - time->tv_nsec = 0; - break; - default: - return 0; - } - - if (!part->os_partition) { - p = nvram_find_partition(part->name, sig, &size); - if (p <= 0) { - pr_err("nvram: Failed to find partition %s, " - "err %d\n", part->name, (int)p); - return 0; - } - part->index = p; - part->size = size; - } - - buff = kmalloc(part->size, GFP_KERNEL); - - if (!buff) - return -ENOMEM; - - if (nvram_read_partition(part, buff, part->size, &err_type, &id_no)) { - kfree(buff); - return 0; - } - - *count = 0; - - if (part->os_partition) - *id = id_no; - - if (nvram_type_ids[read_type] == PSTORE_TYPE_DMESG) { - size_t length, hdr_size; - - oops_hdr = (struct oops_log_info *)buff; - if (be16_to_cpu(oops_hdr->version) < OOPS_HDR_VERSION) { - /* Old format oops header had 2-byte record size */ - hdr_size = sizeof(u16); - length = be16_to_cpu(oops_hdr->version); - time->tv_sec = 0; - time->tv_nsec = 0; - } else { - hdr_size = sizeof(*oops_hdr); - length = be16_to_cpu(oops_hdr->report_length); - time->tv_sec = be64_to_cpu(oops_hdr->timestamp); - time->tv_nsec = 0; - } - *buf = kmalloc(length, GFP_KERNEL); - if (*buf == NULL) - return -ENOMEM; - memcpy(*buf, buff + hdr_size, length); - kfree(buff); - - if (err_type == ERR_TYPE_KERNEL_PANIC_GZ) - *compressed = true; - else - *compressed = false; - return length; - } - - *buf = buff; - return part->size; -} - -static struct pstore_info nvram_pstore_info = { - .owner = THIS_MODULE, - .name = "nvram", - .open = nvram_pstore_open, - .read = nvram_pstore_read, - .write = nvram_pstore_write, -}; - -static int nvram_pstore_init(void) -{ - int rc = 0; - - nvram_pstore_info.buf = oops_data; - nvram_pstore_info.bufsize = oops_data_sz; - - spin_lock_init(&nvram_pstore_info.buf_lock); - - rc = pstore_register(&nvram_pstore_info); - if (rc != 0) - pr_err("nvram: pstore_register() failed, defaults to " - "kmsg_dump; returned %d\n", rc); - - return rc; -} -#else -static int nvram_pstore_init(void) -{ - return -1; -} -#endif - -static void __init nvram_init_oops_partition(int rtas_partition_exists) -{ - int rc; - - rc = pseries_nvram_init_os_partition(&oops_log_partition); - if (rc != 0) { - if (!rtas_partition_exists) - return; - pr_notice("nvram: Using %s partition to log both" - " RTAS errors and oops/panic reports\n", - rtas_log_partition.name); - memcpy(&oops_log_partition, &rtas_log_partition, - sizeof(rtas_log_partition)); - } - oops_buf = kmalloc(oops_log_partition.size, GFP_KERNEL); - if (!oops_buf) { - pr_err("nvram: No memory for %s partition\n", - oops_log_partition.name); - return; - } - oops_data = oops_buf + sizeof(struct oops_log_info); - oops_data_sz = oops_log_partition.size - sizeof(struct oops_log_info); - - rc = nvram_pstore_init(); - - if (!rc) - return; - - /* - * Figure compression (preceded by elimination of each line's <n> - * severity prefix) will reduce the oops/panic report to at most - * 45% of its original size. - */ - big_oops_buf_sz = (oops_data_sz * 100) / 45; - big_oops_buf = kmalloc(big_oops_buf_sz, GFP_KERNEL); - if (big_oops_buf) { - stream.workspace = kmalloc(zlib_deflate_workspacesize( - WINDOW_BITS, MEM_LEVEL), GFP_KERNEL); - if (!stream.workspace) { - pr_err("nvram: No memory for compression workspace; " - "skipping compression of %s partition data\n", - oops_log_partition.name); - kfree(big_oops_buf); - big_oops_buf = NULL; - } - } else { - pr_err("No memory for uncompressed %s data; " - "skipping compression\n", oops_log_partition.name); - stream.workspace = NULL; - } - - rc = kmsg_dump_register(&nvram_kmsg_dumper); - if (rc != 0) { - pr_err("nvram: kmsg_dump_register() failed; returned %d\n", rc); - kfree(oops_buf); - kfree(big_oops_buf); - kfree(stream.workspace); - } -} - static int __init pseries_nvram_init_log_partitions(void) { int rc; @@ -798,7 +211,7 @@ static int __init pseries_nvram_init_log_partitions(void) /* Scan nvram for partitions */ nvram_scan_partitions(); - rc = pseries_nvram_init_os_partition(&rtas_log_partition); + rc = nvram_init_os_partition(&rtas_log_partition); nvram_init_oops_partition(rc == 0); return 0; } @@ -834,72 +247,3 @@ int __init pSeries_nvram_init(void) return 0; } - -/* - * This is our kmsg_dump callback, called after an oops or panic report - * has been written to the printk buffer. We want to capture as much - * of the printk buffer as possible. First, capture as much as we can - * that we think will compress sufficiently to fit in the lnx,oops-log - * partition. If that's too much, go back and capture uncompressed text. - */ -static void oops_to_nvram(struct kmsg_dumper *dumper, - enum kmsg_dump_reason reason) -{ - struct oops_log_info *oops_hdr = (struct oops_log_info *)oops_buf; - static unsigned int oops_count = 0; - static bool panicking = false; - static DEFINE_SPINLOCK(lock); - unsigned long flags; - size_t text_len; - unsigned int err_type = ERR_TYPE_KERNEL_PANIC_GZ; - int rc = -1; - - switch (reason) { - case KMSG_DUMP_RESTART: - case KMSG_DUMP_HALT: - case KMSG_DUMP_POWEROFF: - /* These are almost always orderly shutdowns. */ - return; - case KMSG_DUMP_OOPS: - break; - case KMSG_DUMP_PANIC: - panicking = true; - break; - case KMSG_DUMP_EMERG: - if (panicking) - /* Panic report already captured. */ - return; - break; - default: - pr_err("%s: ignoring unrecognized KMSG_DUMP_* reason %d\n", - __func__, (int) reason); - return; - } - - if (clobbering_unread_rtas_event()) - return; - - if (!spin_trylock_irqsave(&lock, flags)) - return; - - if (big_oops_buf) { - kmsg_dump_get_buffer(dumper, false, - big_oops_buf, big_oops_buf_sz, &text_len); - rc = zip_oops(text_len); - } - if (rc != 0) { - kmsg_dump_rewind(dumper); - kmsg_dump_get_buffer(dumper, false, - oops_data, oops_data_sz, &text_len); - err_type = ERR_TYPE_KERNEL_PANIC; - oops_hdr->version = cpu_to_be16(OOPS_HDR_VERSION); - oops_hdr->report_length = cpu_to_be16(text_len); - oops_hdr->timestamp = cpu_to_be64(get_seconds()); - } - - (void) nvram_write_os_partition(&oops_log_partition, oops_buf, - (int) (sizeof(*oops_hdr) + text_len), err_type, - ++oops_count); - - spin_unlock_irqrestore(&lock, flags); -} diff --git a/arch/powerpc/platforms/pseries/pci_dlpar.c b/arch/powerpc/platforms/pseries/pci_dlpar.c index 89e23811199c..5d4a3df59d0c 100644 --- a/arch/powerpc/platforms/pseries/pci_dlpar.c +++ b/arch/powerpc/platforms/pseries/pci_dlpar.c @@ -32,6 +32,8 @@ #include <asm/firmware.h> #include <asm/eeh.h> +#include "pseries.h" + static struct pci_bus * find_bus_among_children(struct pci_bus *bus, struct device_node *dn) @@ -75,6 +77,7 @@ struct pci_controller *init_phb_dynamic(struct device_node *dn) return NULL; rtas_setup_phb(phb); pci_process_bridge_OF_ranges(phb, dn, 0); + phb->controller_ops = pseries_pci_controller_ops; pci_devs_phb_init_dynamic(phb); @@ -82,7 +85,7 @@ struct pci_controller *init_phb_dynamic(struct device_node *dn) eeh_dev_phb_init_dynamic(phb); if (dn->child) - eeh_add_device_tree_early(dn); + eeh_add_device_tree_early(PCI_DN(dn)); pcibios_scan_phb(phb); pcibios_finish_adding_to_bus(phb->bus); diff --git a/arch/powerpc/platforms/pseries/pseries.h b/arch/powerpc/platforms/pseries/pseries.h index 1796c5438cc6..8411c27293e4 100644 --- a/arch/powerpc/platforms/pseries/pseries.h +++ b/arch/powerpc/platforms/pseries/pseries.h @@ -11,6 +11,7 @@ #define _PSERIES_PSERIES_H #include <linux/interrupt.h> +#include <asm/rtas.h> struct device_node; @@ -60,11 +61,24 @@ extern struct device_node *dlpar_configure_connector(__be32, struct device_node *); extern int dlpar_attach_node(struct device_node *); extern int dlpar_detach_node(struct device_node *); +extern int dlpar_acquire_drc(u32 drc_index); +extern int dlpar_release_drc(u32 drc_index); + +#ifdef CONFIG_MEMORY_HOTPLUG +int dlpar_memory(struct pseries_hp_errorlog *hp_elog); +#else +static inline int dlpar_memory(struct pseries_hp_errorlog *hp_elog) +{ + return -EOPNOTSUPP; +} +#endif /* PCI root bridge prepare function override for pseries */ struct pci_host_bridge; int pseries_root_bridge_prepare(struct pci_host_bridge *bridge); +extern struct pci_controller_ops pseries_pci_controller_ops; + unsigned long pseries_memory_block_size(void); #endif /* _PSERIES_PSERIES_H */ diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index e445b6701f50..df6a7041922b 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -265,7 +265,7 @@ static int pci_dn_reconfig_notifier(struct notifier_block *nb, unsigned long act update_dn_pci_info(np, pci->phb); /* Create EEH device for the OF node */ - eeh_dev_init(np, pci->phb); + eeh_dev_init(PCI_DN(np), pci->phb); } break; default: @@ -461,6 +461,47 @@ static long pseries_little_endian_exceptions(void) } #endif +static void __init find_and_init_phbs(void) +{ + struct device_node *node; + struct pci_controller *phb; + struct device_node *root = of_find_node_by_path("/"); + + for_each_child_of_node(root, node) { + if (node->type == NULL || (strcmp(node->type, "pci") != 0 && + strcmp(node->type, "pciex") != 0)) + continue; + + phb = pcibios_alloc_controller(node); + if (!phb) + continue; + rtas_setup_phb(phb); + pci_process_bridge_OF_ranges(phb, node, 0); + isa_bridge_find_early(phb); + phb->controller_ops = pseries_pci_controller_ops; + } + + of_node_put(root); + pci_devs_phb_init(); + + /* + * PCI_PROBE_ONLY and PCI_REASSIGN_ALL_BUS can be set via properties + * in chosen. + */ + if (of_chosen) { + const int *prop; + + prop = of_get_property(of_chosen, + "linux,pci-probe-only", NULL); + if (prop) { + if (*prop) + pci_add_flags(PCI_PROBE_ONLY); + else + pci_clear_flags(PCI_PROBE_ONLY); + } + } +} + static void __init pSeries_setup_arch(void) { set_arch_panic_timeout(10, ARCH_PANIC_TIMEOUT); @@ -793,6 +834,10 @@ static int pSeries_pci_probe_mode(struct pci_bus *bus) void pSeries_final_fixup(void) { } #endif +struct pci_controller_ops pseries_pci_controller_ops = { + .probe_mode = pSeries_pci_probe_mode, +}; + define_machine(pseries) { .name = "pSeries", .probe = pSeries_probe, @@ -801,7 +846,6 @@ define_machine(pseries) { .show_cpuinfo = pSeries_show_cpuinfo, .log_error = pSeries_log_error, .pcibios_fixup = pSeries_final_fixup, - .pci_probe_mode = pSeries_pci_probe_mode, .restart = rtas_restart, .halt = rtas_halt, .panic = rtas_os_term, diff --git a/arch/powerpc/platforms/pseries/smp.c b/arch/powerpc/platforms/pseries/smp.c index a3555b10c1a5..6932ea803e33 100644 --- a/arch/powerpc/platforms/pseries/smp.c +++ b/arch/powerpc/platforms/pseries/smp.c @@ -197,16 +197,14 @@ static void pSeries_cause_ipi_mux(int cpu, unsigned long data) xics_cause_ipi(cpu, data); } -static __init int pSeries_smp_probe(void) +static __init void pSeries_smp_probe(void) { - int ret = xics_smp_probe(); + xics_smp_probe(); if (cpu_has_feature(CPU_FTR_DBELL)) { xics_cause_ipi = smp_ops->cause_ipi; smp_ops->cause_ipi = pSeries_cause_ipi_mux; } - - return ret; } static struct smp_ops_t pSeries_mpic_smp_ops = { diff --git a/arch/powerpc/relocs_check.pl b/arch/powerpc/relocs_check.pl deleted file mode 100755 index 3f46e8b9c56d..000000000000 --- a/arch/powerpc/relocs_check.pl +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/perl - -# Copyright © 2009 IBM Corporation - -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version -# 2 of the License, or (at your option) any later version. - -# This script checks the relocations of a vmlinux for "suspicious" -# relocations. - -use strict; -use warnings; - -if ($#ARGV != 1) { - die "$0 [path to objdump] [path to vmlinux]\n"; -} - -# Have Kbuild supply the path to objdump so we handle cross compilation. -my $objdump = shift; -my $vmlinux = shift; -my $bad_relocs_count = 0; -my $bad_relocs = ""; -my $old_binutils = 0; - -open(FD, "$objdump -R $vmlinux|") or die; -while (<FD>) { - study $_; - - # Only look at relocation lines. - next if (!/\s+R_/); - - # These relocations are okay - # On PPC64: - # R_PPC64_RELATIVE, R_PPC64_NONE, R_PPC64_ADDR64 - # On PPC: - # R_PPC_RELATIVE, R_PPC_ADDR16_HI, - # R_PPC_ADDR16_HA,R_PPC_ADDR16_LO, - # R_PPC_NONE - - next if (/\bR_PPC64_RELATIVE\b/ or /\bR_PPC64_NONE\b/ or - /\bR_PPC64_ADDR64\s+mach_/); - next if (/\bR_PPC_ADDR16_LO\b/ or /\bR_PPC_ADDR16_HI\b/ or - /\bR_PPC_ADDR16_HA\b/ or /\bR_PPC_RELATIVE\b/ or - /\bR_PPC_NONE\b/); - - # If we see this type of relocation it's an idication that - # we /may/ be using an old version of binutils. - if (/R_PPC64_UADDR64/) { - $old_binutils++; - } - - $bad_relocs_count++; - $bad_relocs .= $_; -} - -if ($bad_relocs_count) { - print "WARNING: $bad_relocs_count bad relocations\n"; - print $bad_relocs; -} - -if ($old_binutils) { - print "WARNING: You need at least binutils >= 2.19 to build a ". - "CONFIG_RELOCATABLE kernel\n"; -} diff --git a/arch/powerpc/relocs_check.sh b/arch/powerpc/relocs_check.sh new file mode 100755 index 000000000000..2e4ebd0e25b3 --- /dev/null +++ b/arch/powerpc/relocs_check.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +# Copyright © 2015 IBM Corporation + +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version +# 2 of the License, or (at your option) any later version. + +# This script checks the relocations of a vmlinux for "suspicious" +# relocations. + +# based on relocs_check.pl +# Copyright © 2009 IBM Corporation + +if [ $# -lt 2 ]; then + echo "$0 [path to objdump] [path to vmlinux]" 1>&2 + exit 1 +fi + +# Have Kbuild supply the path to objdump so we handle cross compilation. +objdump="$1" +vmlinux="$2" + +bad_relocs=$( +"$objdump" -R "$vmlinux" | + # Only look at relocation lines. + grep -E '\<R_' | + # These relocations are okay + # On PPC64: + # R_PPC64_RELATIVE, R_PPC64_NONE + # R_PPC64_ADDR64 mach_<name> + # On PPC: + # R_PPC_RELATIVE, R_PPC_ADDR16_HI, + # R_PPC_ADDR16_HA,R_PPC_ADDR16_LO, + # R_PPC_NONE + grep -F -w -v 'R_PPC64_RELATIVE +R_PPC64_NONE +R_PPC_ADDR16_LO +R_PPC_ADDR16_HI +R_PPC_ADDR16_HA +R_PPC_RELATIVE +R_PPC_NONE' | + grep -E -v '\<R_PPC64_ADDR64[[:space:]]+mach_' +) + +if [ -z "$bad_relocs" ]; then + exit 0 +fi + +num_bad=$(echo "$bad_relocs" | wc -l) +echo "WARNING: $num_bad bad relocations" +echo "$bad_relocs" + +# If we see this type of relocation it's an idication that +# we /may/ be using an old version of binutils. +if echo "$bad_relocs" | grep -q -F -w R_PPC64_UADDR64; then + echo "WARNING: You need at least binutils >= 2.19 to build a CONFIG_RELOCATABLE kernel" +fi diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c index 9e5353ff6d1b..d00a5663e312 100644 --- a/arch/powerpc/sysdev/dart_iommu.c +++ b/arch/powerpc/sysdev/dart_iommu.c @@ -369,7 +369,7 @@ static int dart_dma_set_mask(struct device *dev, u64 dma_mask) return 0; } -void __init iommu_init_early_dart(void) +void __init iommu_init_early_dart(struct pci_controller_ops *controller_ops) { struct device_node *dn; @@ -395,8 +395,8 @@ void __init iommu_init_early_dart(void) if (dart_is_u4) ppc_md.dma_set_mask = dart_dma_set_mask; - ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_dart; - ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_dart; + controller_ops->dma_dev_setup = pci_dma_dev_setup_dart; + controller_ops->dma_bus_setup = pci_dma_bus_setup_dart; /* Setup pci_dma ops */ set_pci_dma_ops(&dma_iommu_ops); @@ -404,8 +404,8 @@ void __init iommu_init_early_dart(void) bail: /* If init failed, use direct iommu and null setup functions */ - ppc_md.pci_dma_dev_setup = NULL; - ppc_md.pci_dma_bus_setup = NULL; + controller_ops->dma_dev_setup = NULL; + controller_ops->dma_bus_setup = NULL; /* Setup pci_dma ops */ set_pci_dma_ops(&dma_direct_ops); diff --git a/arch/powerpc/sysdev/dcr.c b/arch/powerpc/sysdev/dcr.c index 2d8a101b6b9e..121e26fffd50 100644 --- a/arch/powerpc/sysdev/dcr.c +++ b/arch/powerpc/sysdev/dcr.c @@ -54,7 +54,7 @@ bool dcr_map_ok_generic(dcr_host_t host) else if (host.type == DCR_HOST_MMIO) return dcr_map_ok_mmio(host.host.mmio); else - return 0; + return false; } EXPORT_SYMBOL_GPL(dcr_map_ok_generic); diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index 4bbb4b8dfd09..f086c6f22dc9 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -162,7 +162,17 @@ static void fsl_compose_msi_msg(struct pci_dev *pdev, int hwirq, msg->address_lo = lower_32_bits(address); msg->address_hi = upper_32_bits(address); - msg->data = hwirq; + /* + * MPIC version 2.0 has erratum PIC1. It causes + * that neither MSI nor MSI-X can work fine. + * This is a workaround to allow MSI-X to function + * properly. It only works for MSI-X, we prevent + * MSI on buggy chips in fsl_setup_msi_irqs(). + */ + if (msi_data->feature & MSI_HW_ERRATA_ENDIAN) + msg->data = __swab32(hwirq); + else + msg->data = hwirq; pr_debug("%s: allocated srs: %d, ibs: %d\n", __func__, (hwirq >> msi_data->srs_shift) & MSI_SRS_MASK, @@ -180,8 +190,16 @@ static int fsl_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) struct msi_msg msg; struct fsl_msi *msi_data; - if (type == PCI_CAP_ID_MSIX) - pr_debug("fslmsi: MSI-X untested, trying anyway.\n"); + if (type == PCI_CAP_ID_MSI) { + /* + * MPIC version 2.0 has erratum PIC1. For now MSI + * could not work. So check to prevent MSI from + * being used on the board with this erratum. + */ + list_for_each_entry(msi_data, &msi_head, list) + if (msi_data->feature & MSI_HW_ERRATA_ENDIAN) + return -EINVAL; + } /* * If the PCI node has an fsl,msi property, then we need to use it @@ -446,6 +464,11 @@ static int fsl_of_msi_probe(struct platform_device *dev) msi->feature = features->fsl_pic_ip; + /* For erratum PIC1 on MPIC version 2.0*/ + if ((features->fsl_pic_ip & FSL_PIC_IP_MASK) == FSL_PIC_IP_MPIC + && (fsl_mpic_primary_get_version() == 0x0200)) + msi->feature |= MSI_HW_ERRATA_ENDIAN; + /* * Remember the phandle, so that we can match with any PCI nodes * that have an "fsl,msi" property. diff --git a/arch/powerpc/sysdev/fsl_msi.h b/arch/powerpc/sysdev/fsl_msi.h index 420cfcbdac01..a67359d993e5 100644 --- a/arch/powerpc/sysdev/fsl_msi.h +++ b/arch/powerpc/sysdev/fsl_msi.h @@ -27,6 +27,8 @@ #define FSL_PIC_IP_IPIC 0x00000002 #define FSL_PIC_IP_VMPIC 0x00000003 +#define MSI_HW_ERRATA_ENDIAN 0x00000010 + struct fsl_msi_cascade_data; struct fsl_msi { diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index 4b74c276e427..9a8fcf0d79d7 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -111,6 +111,18 @@ static struct pci_ops fsl_indirect_pcie_ops = #define MAX_PHYS_ADDR_BITS 40 static u64 pci64_dma_offset = 1ull << MAX_PHYS_ADDR_BITS; +#ifdef CONFIG_SWIOTLB +static void setup_swiotlb_ops(struct pci_controller *hose) +{ + if (ppc_swiotlb_enable) { + hose->controller_ops.dma_dev_setup = pci_dma_dev_setup_swiotlb; + set_pci_dma_ops(&swiotlb_dma_ops); + } +} +#else +static inline void setup_swiotlb_ops(struct pci_controller *hose) {} +#endif + static int fsl_pci_dma_set_mask(struct device *dev, u64 dma_mask) { if (!dev->dma_mask || !dma_supported(dev, dma_mask)) @@ -548,6 +560,9 @@ int fsl_add_bridge(struct platform_device *pdev, int is_primary) /* Setup PEX window registers */ setup_pci_atmu(hose); + /* Set up controller operations */ + setup_swiotlb_ops(hose); + return 0; no_bridge: diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index bbfbbf2025fd..b2b8447a227a 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -655,7 +655,6 @@ static inline struct mpic * mpic_from_irq_data(struct irq_data *d) static inline void mpic_eoi(struct mpic *mpic) { mpic_cpu_write(MPIC_INFO(CPU_EOI), 0); - (void)mpic_cpu_read(MPIC_INFO(CPU_WHOAMI)); } /* @@ -1676,31 +1675,6 @@ void __init mpic_init(struct mpic *mpic) mpic_err_int_init(mpic, MPIC_FSL_ERR_INT); } -void __init mpic_set_clk_ratio(struct mpic *mpic, u32 clock_ratio) -{ - u32 v; - - v = mpic_read(mpic->gregs, MPIC_GREG_GLOBAL_CONF_1); - v &= ~MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO_MASK; - v |= MPIC_GREG_GLOBAL_CONF_1_CLK_RATIO(clock_ratio); - mpic_write(mpic->gregs, MPIC_GREG_GLOBAL_CONF_1, v); -} - -void __init mpic_set_serial_int(struct mpic *mpic, int enable) -{ - unsigned long flags; - u32 v; - - raw_spin_lock_irqsave(&mpic_lock, flags); - v = mpic_read(mpic->gregs, MPIC_GREG_GLOBAL_CONF_1); - if (enable) - v |= MPIC_GREG_GLOBAL_CONF_1_SIE; - else - v &= ~MPIC_GREG_GLOBAL_CONF_1_SIE; - mpic_write(mpic->gregs, MPIC_GREG_GLOBAL_CONF_1, v); - raw_spin_unlock_irqrestore(&mpic_lock, flags); -} - void mpic_irq_set_priority(unsigned int irq, unsigned int pri) { struct mpic *mpic = mpic_find(irq); @@ -1923,7 +1897,7 @@ void smp_mpic_message_pass(int cpu, int msg) msg * MPIC_INFO(CPU_IPI_DISPATCH_STRIDE), physmask); } -int __init smp_mpic_probe(void) +void __init smp_mpic_probe(void) { int nr_cpus; @@ -1935,8 +1909,6 @@ int __init smp_mpic_probe(void) if (nr_cpus > 1) mpic_request_ipis(); - - return nr_cpus; } void smp_mpic_setup_cpu(int cpu) diff --git a/arch/powerpc/sysdev/qe_lib/qe_io.c b/arch/powerpc/sysdev/qe_lib/qe_io.c index d09994164daf..7ea0174f6d3d 100644 --- a/arch/powerpc/sysdev/qe_lib/qe_io.c +++ b/arch/powerpc/sysdev/qe_lib/qe_io.c @@ -190,28 +190,3 @@ int par_io_of_config(struct device_node *np) return 0; } EXPORT_SYMBOL(par_io_of_config); - -#ifdef DEBUG -static void dump_par_io(void) -{ - unsigned int i; - - printk(KERN_INFO "%s: par_io=%p\n", __func__, par_io); - for (i = 0; i < num_par_io_ports; i++) { - printk(KERN_INFO " cpodr[%u]=%08x\n", i, - in_be32(&par_io[i].cpodr)); - printk(KERN_INFO " cpdata[%u]=%08x\n", i, - in_be32(&par_io[i].cpdata)); - printk(KERN_INFO " cpdir1[%u]=%08x\n", i, - in_be32(&par_io[i].cpdir1)); - printk(KERN_INFO " cpdir2[%u]=%08x\n", i, - in_be32(&par_io[i].cpdir2)); - printk(KERN_INFO " cppar1[%u]=%08x\n", i, - in_be32(&par_io[i].cppar1)); - printk(KERN_INFO " cppar2[%u]=%08x\n", i, - in_be32(&par_io[i].cppar2)); - } - -} -EXPORT_SYMBOL(dump_par_io); -#endif /* DEBUG */ diff --git a/arch/powerpc/sysdev/qe_lib/ucc_slow.c b/arch/powerpc/sysdev/qe_lib/ucc_slow.c index befaf1123f7f..5f91628209eb 100644 --- a/arch/powerpc/sysdev/qe_lib/ucc_slow.c +++ b/arch/powerpc/sysdev/qe_lib/ucc_slow.c @@ -43,11 +43,6 @@ u32 ucc_slow_get_qe_cr_subblock(int uccs_num) } EXPORT_SYMBOL(ucc_slow_get_qe_cr_subblock); -void ucc_slow_poll_transmitter_now(struct ucc_slow_private * uccs) -{ - out_be16(&uccs->us_regs->utodr, UCC_SLOW_TOD); -} - void ucc_slow_graceful_stop_tx(struct ucc_slow_private * uccs) { struct ucc_slow_info *us_info = uccs->us_info; diff --git a/arch/powerpc/sysdev/xics/xics-common.c b/arch/powerpc/sysdev/xics/xics-common.c index 125743b58c70..878a54036a25 100644 --- a/arch/powerpc/sysdev/xics/xics-common.c +++ b/arch/powerpc/sysdev/xics/xics-common.c @@ -140,15 +140,13 @@ static void xics_request_ipi(void) IRQF_PERCPU | IRQF_NO_THREAD, "IPI", NULL)); } -int __init xics_smp_probe(void) +void __init xics_smp_probe(void) { /* Setup cause_ipi callback based on which ICP is used */ smp_ops->cause_ipi = icp_ops->cause_ipi; /* Register all the IPIs */ xics_request_ipi(); - - return num_possible_cpus(); } #endif /* CONFIG_SMP */ diff --git a/arch/s390/Kbuild b/arch/s390/Kbuild index 647c3eccc3d0..2938934c6518 100644 --- a/arch/s390/Kbuild +++ b/arch/s390/Kbuild @@ -4,6 +4,5 @@ obj-$(CONFIG_KVM) += kvm/ obj-$(CONFIG_CRYPTO_HW) += crypto/ obj-$(CONFIG_S390_HYPFS_FS) += hypfs/ obj-$(CONFIG_APPLDATA_BASE) += appldata/ -obj-$(CONFIG_MATHEMU) += math-emu/ obj-y += net/ obj-$(CONFIG_PCI) += pci/ diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 6321fd8bf813..de2726a487b0 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -35,7 +35,7 @@ config GENERIC_BUG_RELATIVE_POINTERS def_bool y config ARCH_DMA_ADDR_T_64BIT - def_bool 64BIT + def_bool y config GENERIC_LOCKBREAK def_bool y if SMP && PREEMPT @@ -59,7 +59,7 @@ config PCI_QUIRKS def_bool n config ARCH_SUPPORTS_UPROBES - def_bool 64BIT + def_bool y config S390 def_bool y @@ -111,19 +111,19 @@ config S390 select GENERIC_TIME_VSYSCALL select HAVE_ALIGNED_STRUCT_PAGE if SLUB select HAVE_ARCH_AUDITSYSCALL - select HAVE_ARCH_JUMP_LABEL if !MARCH_G5 + select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK - select HAVE_ARCH_TRANSPARENT_HUGEPAGE if 64BIT - select HAVE_BPF_JIT if 64BIT && PACK_STACK + select HAVE_ARCH_TRANSPARENT_HUGEPAGE + select HAVE_BPF_JIT if PACK_STACK select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL select HAVE_DEBUG_KMEMLEAK - select HAVE_DYNAMIC_FTRACE if 64BIT - select HAVE_DYNAMIC_FTRACE_WITH_REGS if 64BIT + select HAVE_DYNAMIC_FTRACE + select HAVE_DYNAMIC_FTRACE_WITH_REGS select HAVE_FTRACE_MCOUNT_RECORD - select HAVE_FUNCTION_GRAPH_TRACER if 64BIT - select HAVE_FUNCTION_TRACER if 64BIT + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_TRACER select HAVE_FUTEX_CMPXCHG if FUTEX select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP @@ -133,7 +133,7 @@ config S390 select HAVE_KERNEL_XZ select HAVE_KPROBES select HAVE_KRETPROBES - select HAVE_KVM if 64BIT + select HAVE_KVM select HAVE_LIVEPATCH select HAVE_MEMBLOCK select HAVE_MEMBLOCK_NODE_MAP @@ -143,7 +143,6 @@ config S390 select HAVE_PERF_EVENTS select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_SYSCALL_TRACEPOINTS - select HAVE_UID16 if 32BIT select HAVE_VIRT_CPU_ACCOUNTING select MODULES_USE_ELF_RELA select NO_BOOTMEM @@ -199,18 +198,11 @@ config HAVE_MARCH_Z13_FEATURES choice prompt "Processor type" - default MARCH_G5 - -config MARCH_G5 - bool "System/390 model G5 and G6" - depends on !64BIT - help - Select this to build a 31 bit kernel that works - on all ESA/390 and z/Architecture machines. + default MARCH_Z900 config MARCH_Z900 bool "IBM zSeries model z800 and z900" - select HAVE_MARCH_Z900_FEATURES if 64BIT + select HAVE_MARCH_Z900_FEATURES help Select this to enable optimizations for model z800/z900 (2064 and 2066 series). This will enable some optimizations that are not @@ -218,7 +210,7 @@ config MARCH_Z900 config MARCH_Z990 bool "IBM zSeries model z890 and z990" - select HAVE_MARCH_Z990_FEATURES if 64BIT + select HAVE_MARCH_Z990_FEATURES help Select this to enable optimizations for model z890/z990 (2084 and 2086 series). The kernel will be slightly faster but will not work @@ -226,7 +218,7 @@ config MARCH_Z990 config MARCH_Z9_109 bool "IBM System z9" - select HAVE_MARCH_Z9_109_FEATURES if 64BIT + select HAVE_MARCH_Z9_109_FEATURES help Select this to enable optimizations for IBM System z9 (2094 and 2096 series). The kernel will be slightly faster but will not work @@ -234,7 +226,7 @@ config MARCH_Z9_109 config MARCH_Z10 bool "IBM System z10" - select HAVE_MARCH_Z10_FEATURES if 64BIT + select HAVE_MARCH_Z10_FEATURES help Select this to enable optimizations for IBM System z10 (2097 and 2098 series). The kernel will be slightly faster but will not work @@ -242,7 +234,7 @@ config MARCH_Z10 config MARCH_Z196 bool "IBM zEnterprise 114 and 196" - select HAVE_MARCH_Z196_FEATURES if 64BIT + select HAVE_MARCH_Z196_FEATURES help Select this to enable optimizations for IBM zEnterprise 114 and 196 (2818 and 2817 series). The kernel will be slightly faster but will @@ -250,7 +242,7 @@ config MARCH_Z196 config MARCH_ZEC12 bool "IBM zBC12 and zEC12" - select HAVE_MARCH_ZEC12_FEATURES if 64BIT + select HAVE_MARCH_ZEC12_FEATURES help Select this to enable optimizations for IBM zBC12 and zEC12 (2828 and 2827 series). The kernel will be slightly faster but will not work on @@ -258,7 +250,7 @@ config MARCH_ZEC12 config MARCH_Z13 bool "IBM z13" - select HAVE_MARCH_Z13_FEATURES if 64BIT + select HAVE_MARCH_Z13_FEATURES help Select this to enable optimizations for IBM z13 (2964 series). The kernel will be slightly faster but will not work on older @@ -266,9 +258,6 @@ config MARCH_Z13 endchoice -config MARCH_G5_TUNE - def_bool TUNE_G5 || MARCH_G5 && TUNE_DEFAULT - config MARCH_Z900_TUNE def_bool TUNE_Z900 || MARCH_Z900 && TUNE_DEFAULT @@ -307,9 +296,6 @@ config TUNE_DEFAULT Tune the generated code for the target processor for which the kernel will be compiled. -config TUNE_G5 - bool "System/390 model G5 and G6" - config TUNE_Z900 bool "IBM zSeries model z800 and z900" @@ -335,21 +321,14 @@ endchoice config 64BIT def_bool y - prompt "64 bit kernel" - help - Select this option if you have an IBM z/Architecture machine - and want to use the 64 bit addressing mode. - -config 32BIT - def_bool y if !64BIT config COMPAT def_bool y prompt "Kernel support for 31 bit emulation" - depends on 64BIT select COMPAT_BINFMT_ELF if BINFMT_ELF select ARCH_WANT_OLD_COMPAT_IPC select COMPAT_OLD_SIGACTION + depends on MULTIUSER help Select this option if you want to enable your system kernel to handle system-calls from ELF binaries for 31 bit ESA. This option @@ -385,8 +364,7 @@ config NR_CPUS int "Maximum number of CPUs (2-512)" range 2 512 depends on SMP - default "32" if !64BIT - default "64" if 64BIT + default "64" help This allows you to specify the maximum number of CPUs which this kernel will support. The maximum supported value is 512 and the @@ -427,15 +405,6 @@ config SCHED_TOPOLOGY source kernel/Kconfig.preempt -config MATHEMU - def_bool y - prompt "IEEE FPU emulation" - depends on MARCH_G5 - help - This option is required for IEEE compliant floating point arithmetic - on older ESA/390 machines. Say Y unless you know your machine doesn't - need this. - source kernel/Kconfig.hz endmenu @@ -446,7 +415,6 @@ config ARCH_SPARSEMEM_ENABLE def_bool y select SPARSEMEM_VMEMMAP_ENABLE select SPARSEMEM_VMEMMAP - select SPARSEMEM_STATIC if !64BIT config ARCH_SPARSEMEM_DEFAULT def_bool y @@ -462,7 +430,6 @@ config ARCH_ENABLE_MEMORY_HOTREMOVE config ARCH_ENABLE_SPLIT_PMD_PTLOCK def_bool y - depends on 64BIT config FORCE_MAX_ZONEORDER int @@ -537,7 +504,6 @@ config QDIO menuconfig PCI bool "PCI support" - depends on 64BIT select HAVE_DMA_ATTRS select PCI_MSI help @@ -607,7 +573,6 @@ config CHSC_SCH config SCM_BUS def_bool y - depends on 64BIT prompt "SCM bus driver" help Bus driver for Storage Class Memory. @@ -629,7 +594,7 @@ menu "Dump support" config CRASH_DUMP bool "kernel crash dumps" - depends on 64BIT && SMP + depends on SMP select KEXEC help Generate crash dump after being started by kexec. @@ -668,7 +633,7 @@ endmenu menu "Power Management" config ARCH_HIBERNATION_POSSIBLE - def_bool y if 64BIT + def_bool y source "kernel/power/Kconfig" @@ -819,7 +784,6 @@ source "arch/s390/kvm/Kconfig" config S390_GUEST def_bool y prompt "s390 support for virtio devices" - depends on 64BIT select TTY select VIRTUALIZATION select VIRTIO diff --git a/arch/s390/Makefile b/arch/s390/Makefile index acb6859c6a95..667b1bca5681 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -13,15 +13,6 @@ # Copyright (C) 1994 by Linus Torvalds # -ifndef CONFIG_64BIT -LD_BFD := elf32-s390 -LDFLAGS := -m elf_s390 -KBUILD_CFLAGS += -m31 -KBUILD_AFLAGS += -m31 -UTS_MACHINE := s390 -STACK_SIZE := 8192 -CHECKFLAGS += -D__s390__ -msize-long -else LD_BFD := elf64-s390 LDFLAGS := -m elf64_s390 KBUILD_AFLAGS_MODULE += -fPIC @@ -31,11 +22,9 @@ KBUILD_AFLAGS += -m64 UTS_MACHINE := s390x STACK_SIZE := 16384 CHECKFLAGS += -D__s390__ -D__s390x__ -endif export LD_BFD -mflags-$(CONFIG_MARCH_G5) := -march=g5 mflags-$(CONFIG_MARCH_Z900) := -march=z900 mflags-$(CONFIG_MARCH_Z990) := -march=z990 mflags-$(CONFIG_MARCH_Z9_109) := -march=z9-109 @@ -47,7 +36,6 @@ mflags-$(CONFIG_MARCH_Z13) := -march=z13 aflags-y += $(mflags-y) cflags-y += $(mflags-y) -cflags-$(CONFIG_MARCH_G5_TUNE) += -mtune=g5 cflags-$(CONFIG_MARCH_Z900_TUNE) += -mtune=z900 cflags-$(CONFIG_MARCH_Z990_TUNE) += -mtune=z990 cflags-$(CONFIG_MARCH_Z9_109_TUNE) += -mtune=z9-109 @@ -104,7 +92,7 @@ KBUILD_AFLAGS += $(aflags-y) OBJCOPYFLAGS := -O binary head-y := arch/s390/kernel/head.o -head-y += arch/s390/kernel/$(if $(CONFIG_64BIT),head64.o,head31.o) +head-y += arch/s390/kernel/head64.o # See arch/s390/Kbuild for content of core part of the kernel core-y += arch/s390/ @@ -129,9 +117,7 @@ zfcpdump: $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ vdso_install: -ifeq ($(CONFIG_64BIT),y) $(Q)$(MAKE) $(build)=arch/$(ARCH)/kernel/vdso64 $@ -endif $(Q)$(MAKE) $(build)=arch/$(ARCH)/kernel/vdso32 $@ archclean: diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index f90d1fc6d603..d4788111c161 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -4,13 +4,11 @@ # create a compressed vmlinux image from the original vmlinux # -BITS := $(if $(CONFIG_64BIT),64,31) - targets := vmlinux.lds vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 targets += vmlinux.bin.xz vmlinux.bin.lzma vmlinux.bin.lzo vmlinux.bin.lz4 -targets += misc.o piggy.o sizes.h head$(BITS).o +targets += misc.o piggy.o sizes.h head.o -KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2 +KBUILD_CFLAGS := -m64 -D__KERNEL__ $(LINUX_INCLUDE) -O2 KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING KBUILD_CFLAGS += $(cflags-y) -fno-delete-null-pointer-checks KBUILD_CFLAGS += $(call cc-option,-mpacked-stack) @@ -19,7 +17,7 @@ KBUILD_CFLAGS += $(call cc-option,-ffreestanding) GCOV_PROFILE := n OBJECTS := $(addprefix $(objtree)/arch/s390/kernel/, head.o sclp.o ebcdic.o) -OBJECTS += $(obj)/head$(BITS).o $(obj)/misc.o $(obj)/piggy.o +OBJECTS += $(obj)/head.o $(obj)/misc.o $(obj)/piggy.o LDFLAGS_vmlinux := --oformat $(LD_BFD) -e startup -T $(obj)/vmlinux: $(obj)/vmlinux.lds $(OBJECTS) @@ -34,8 +32,8 @@ quiet_cmd_sizes = GEN $@ $(obj)/sizes.h: vmlinux $(call if_changed,sizes) -AFLAGS_head$(BITS).o += -I$(obj) -$(obj)/head$(BITS).o: $(obj)/sizes.h +AFLAGS_head.o += -I$(obj) +$(obj)/head.o: $(obj)/sizes.h CFLAGS_misc.o += -I$(obj) $(obj)/misc.o: $(obj)/sizes.h diff --git a/arch/s390/boot/compressed/head64.S b/arch/s390/boot/compressed/head.S index f86a4eef28a9..f86a4eef28a9 100644 --- a/arch/s390/boot/compressed/head64.S +++ b/arch/s390/boot/compressed/head.S diff --git a/arch/s390/boot/compressed/head31.S b/arch/s390/boot/compressed/head31.S deleted file mode 100644 index e8c9e18b8039..000000000000 --- a/arch/s390/boot/compressed/head31.S +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Startup glue code to uncompress the kernel - * - * Copyright IBM Corp. 2010 - * - * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> - */ - -#include <linux/init.h> -#include <linux/linkage.h> -#include <asm/asm-offsets.h> -#include <asm/thread_info.h> -#include <asm/page.h> -#include "sizes.h" - -__HEAD -ENTRY(startup_continue) - basr %r13,0 # get base -.LPG1: - # setup stack - l %r15,.Lstack-.LPG1(%r13) - ahi %r15,-96 - l %r1,.Ldecompress-.LPG1(%r13) - basr %r14,%r1 - # setup registers for memory mover & branch to target - lr %r4,%r2 - l %r2,.Loffset-.LPG1(%r13) - la %r4,0(%r2,%r4) - l %r3,.Lmvsize-.LPG1(%r13) - lr %r5,%r3 - # move the memory mover someplace safe - la %r1,0x200 - mvc 0(mover_end-mover,%r1),mover-.LPG1(%r13) - # decompress image is started at 0x11000 - lr %r6,%r2 - br %r1 -mover: - mvcle %r2,%r4,0 - jo mover - br %r6 -mover_end: - - .align 8 -.Lstack: - .long 0x8000 + (1<<(PAGE_SHIFT+THREAD_ORDER)) -.Ldecompress: - .long decompress_kernel -.Loffset: - .long 0x11000 -.Lmvsize: - .long SZ__bss_start diff --git a/arch/s390/boot/compressed/vmlinux.lds.S b/arch/s390/boot/compressed/vmlinux.lds.S index 8e1fb8239287..747735f83426 100644 --- a/arch/s390/boot/compressed/vmlinux.lds.S +++ b/arch/s390/boot/compressed/vmlinux.lds.S @@ -1,12 +1,7 @@ #include <asm-generic/vmlinux.lds.h> -#ifdef CONFIG_64BIT OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") OUTPUT_ARCH(s390:64-bit) -#else -OUTPUT_FORMAT("elf32-s390", "elf32-s390", "elf32-s390") -OUTPUT_ARCH(s390:31-bit) -#endif ENTRY(startup) diff --git a/arch/s390/crypto/crypt_s390.h b/arch/s390/crypto/crypt_s390.h index 6c5cc6da7111..ba3b2aefddf5 100644 --- a/arch/s390/crypto/crypt_s390.h +++ b/arch/s390/crypto/crypt_s390.h @@ -369,14 +369,10 @@ static inline int crypt_s390_func_available(int func, if (facility_mask & CRYPT_S390_MSA && !test_facility(17)) return 0; - - if (facility_mask & CRYPT_S390_MSA3 && - (!test_facility(2) || !test_facility(76))) + if (facility_mask & CRYPT_S390_MSA3 && !test_facility(76)) return 0; - if (facility_mask & CRYPT_S390_MSA4 && - (!test_facility(2) || !test_facility(77))) + if (facility_mask & CRYPT_S390_MSA4 && !test_facility(77)) return 0; - switch (func & CRYPT_S390_OP_MASK) { case CRYPT_S390_KM: ret = crypt_s390_km(KM_QUERY, &status, NULL, NULL, 0); diff --git a/arch/s390/hypfs/hypfs_diag0c.c b/arch/s390/hypfs/hypfs_diag0c.c index d4c0d3717543..24c747a0fcc3 100644 --- a/arch/s390/hypfs/hypfs_diag0c.c +++ b/arch/s390/hypfs/hypfs_diag0c.c @@ -19,13 +19,9 @@ static void diag0c(struct hypfs_diag0c_entry *entry) { asm volatile ( -#ifdef CONFIG_64BIT " sam31\n" " diag %0,%0,0x0c\n" " sam64\n" -#else - " diag %0,%0,0x0c\n" -#endif : /* no output register */ : "a" (entry) : "memory"); diff --git a/arch/s390/hypfs/inode.c b/arch/s390/hypfs/inode.c index df7d8cbee377..3f5c799b7fb5 100644 --- a/arch/s390/hypfs/inode.c +++ b/arch/s390/hypfs/inode.c @@ -437,8 +437,6 @@ struct dentry *hypfs_create_str(struct dentry *dir, static const struct file_operations hypfs_file_ops = { .open = hypfs_open, .release = hypfs_release, - .read = new_sync_read, - .write = new_sync_write, .read_iter = hypfs_read_iter, .write_iter = hypfs_write_iter, .llseek = no_llseek, diff --git a/arch/s390/include/asm/appldata.h b/arch/s390/include/asm/appldata.h index 32a705987156..16887c5fd989 100644 --- a/arch/s390/include/asm/appldata.h +++ b/arch/s390/include/asm/appldata.h @@ -9,28 +9,6 @@ #include <asm/io.h> -#ifndef CONFIG_64BIT - -#define APPLDATA_START_INTERVAL_REC 0x00 /* Function codes for */ -#define APPLDATA_STOP_REC 0x01 /* DIAG 0xDC */ -#define APPLDATA_GEN_EVENT_REC 0x02 -#define APPLDATA_START_CONFIG_REC 0x03 - -/* - * Parameter list for DIAGNOSE X'DC' - */ -struct appldata_parameter_list { - u16 diag; /* The DIAGNOSE code X'00DC' */ - u8 function; /* The function code for the DIAGNOSE */ - u8 parlist_length; /* Length of the parameter list */ - u32 product_id_addr; /* Address of the 16-byte product ID */ - u16 reserved; - u16 buffer_length; /* Length of the application data buffer */ - u32 buffer_addr; /* Address of the application data buffer */ -} __attribute__ ((packed)); - -#else /* CONFIG_64BIT */ - #define APPLDATA_START_INTERVAL_REC 0x80 #define APPLDATA_STOP_REC 0x81 #define APPLDATA_GEN_EVENT_REC 0x82 @@ -51,8 +29,6 @@ struct appldata_parameter_list { u64 buffer_addr; } __attribute__ ((packed)); -#endif /* CONFIG_64BIT */ - struct appldata_product_id { char prod_nr[7]; /* product number */ u16 prod_fn; /* product function */ diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h index fa934fe080c1..adbe3802e377 100644 --- a/arch/s390/include/asm/atomic.h +++ b/arch/s390/include/asm/atomic.h @@ -160,8 +160,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u) #define ATOMIC64_INIT(i) { (i) } -#ifdef CONFIG_64BIT - #define __ATOMIC64_NO_BARRIER "\n" #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES @@ -274,99 +272,6 @@ static inline long long atomic64_cmpxchg(atomic64_t *v, #undef __ATOMIC64_LOOP -#else /* CONFIG_64BIT */ - -typedef struct { - long long counter; -} atomic64_t; - -static inline long long atomic64_read(const atomic64_t *v) -{ - register_pair rp; - - asm volatile( - " lm %0,%N0,%1" - : "=&d" (rp) : "Q" (v->counter) ); - return rp.pair; -} - -static inline void atomic64_set(atomic64_t *v, long long i) -{ - register_pair rp = {.pair = i}; - - asm volatile( - " stm %1,%N1,%0" - : "=Q" (v->counter) : "d" (rp) ); -} - -static inline long long atomic64_xchg(atomic64_t *v, long long new) -{ - register_pair rp_new = {.pair = new}; - register_pair rp_old; - - asm volatile( - " lm %0,%N0,%1\n" - "0: cds %0,%2,%1\n" - " jl 0b\n" - : "=&d" (rp_old), "+Q" (v->counter) - : "d" (rp_new) - : "cc"); - return rp_old.pair; -} - -static inline long long atomic64_cmpxchg(atomic64_t *v, - long long old, long long new) -{ - register_pair rp_old = {.pair = old}; - register_pair rp_new = {.pair = new}; - - asm volatile( - " cds %0,%2,%1" - : "+&d" (rp_old), "+Q" (v->counter) - : "d" (rp_new) - : "cc"); - return rp_old.pair; -} - - -static inline long long atomic64_add_return(long long i, atomic64_t *v) -{ - long long old, new; - - do { - old = atomic64_read(v); - new = old + i; - } while (atomic64_cmpxchg(v, old, new) != old); - return new; -} - -static inline void atomic64_set_mask(unsigned long long mask, atomic64_t *v) -{ - long long old, new; - - do { - old = atomic64_read(v); - new = old | mask; - } while (atomic64_cmpxchg(v, old, new) != old); -} - -static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v) -{ - long long old, new; - - do { - old = atomic64_read(v); - new = old & mask; - } while (atomic64_cmpxchg(v, old, new) != old); -} - -static inline void atomic64_add(long long i, atomic64_t *v) -{ - atomic64_add_return(i, v); -} - -#endif /* CONFIG_64BIT */ - static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u) { long long c, old; diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h index 520542477678..9b68e98a724f 100644 --- a/arch/s390/include/asm/bitops.h +++ b/arch/s390/include/asm/bitops.h @@ -51,32 +51,6 @@ #define __BITOPS_NO_BARRIER "\n" -#ifndef CONFIG_64BIT - -#define __BITOPS_OR "or" -#define __BITOPS_AND "nr" -#define __BITOPS_XOR "xr" -#define __BITOPS_BARRIER "\n" - -#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier) \ -({ \ - unsigned long __old, __new; \ - \ - typecheck(unsigned long *, (__addr)); \ - asm volatile( \ - " l %0,%2\n" \ - "0: lr %1,%0\n" \ - __op_string " %1,%3\n" \ - " cs %0,%1,%2\n" \ - " jl 0b" \ - : "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\ - : "d" (__val) \ - : "cc", "memory"); \ - __old; \ -}) - -#else /* CONFIG_64BIT */ - #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES #define __BITOPS_OR "laog" @@ -125,8 +99,6 @@ #endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */ -#endif /* CONFIG_64BIT */ - #define __BITOPS_WORDS(bits) (((bits) + BITS_PER_LONG - 1) / BITS_PER_LONG) static inline unsigned long * diff --git a/arch/s390/include/asm/cmpxchg.h b/arch/s390/include/asm/cmpxchg.h index 6259895fcd97..4eadec466b8c 100644 --- a/arch/s390/include/asm/cmpxchg.h +++ b/arch/s390/include/asm/cmpxchg.h @@ -80,15 +80,10 @@ extern void __cmpxchg_double_called_with_bad_pointer(void); ({ \ __typeof__(p1) __p1 = (p1); \ __typeof__(p2) __p2 = (p2); \ - int __ret; \ BUILD_BUG_ON(sizeof(*(p1)) != sizeof(long)); \ BUILD_BUG_ON(sizeof(*(p2)) != sizeof(long)); \ VM_BUG_ON((unsigned long)((__p1) + 1) != (unsigned long)(__p2));\ - if (sizeof(long) == 4) \ - __ret = __cmpxchg_double_4(__p1, __p2, o1, o2, n1, n2); \ - else \ - __ret = __cmpxchg_double_8(__p1, __p2, o1, o2, n1, n2); \ - __ret; \ + __cmpxchg_double_8(__p1, __p2, o1, o2, n1, n2); \ }) #define system_has_cmpxchg_double() 1 diff --git a/arch/s390/include/asm/cputime.h b/arch/s390/include/asm/cputime.h index b91e960e4045..221b454c734a 100644 --- a/arch/s390/include/asm/cputime.h +++ b/arch/s390/include/asm/cputime.h @@ -22,15 +22,7 @@ typedef unsigned long long __nocast cputime64_t; static inline unsigned long __div(unsigned long long n, unsigned long base) { -#ifndef CONFIG_64BIT - register_pair rp; - - rp.pair = n >> 1; - asm ("dr %0,%1" : "+d" (rp) : "d" (base >> 1)); - return rp.subreg.odd; -#else /* CONFIG_64BIT */ return n / base; -#endif /* CONFIG_64BIT */ } #define cputime_one_jiffy jiffies_to_cputime(1) @@ -101,17 +93,8 @@ static inline void cputime_to_timespec(const cputime_t cputime, struct timespec *value) { unsigned long long __cputime = (__force unsigned long long) cputime; -#ifndef CONFIG_64BIT - register_pair rp; - - rp.pair = __cputime >> 1; - asm ("dr %0,%1" : "+d" (rp) : "d" (CPUTIME_PER_SEC / 2)); - value->tv_nsec = rp.subreg.even * NSEC_PER_USEC / CPUTIME_PER_USEC; - value->tv_sec = rp.subreg.odd; -#else value->tv_nsec = (__cputime % CPUTIME_PER_SEC) * NSEC_PER_USEC / CPUTIME_PER_USEC; value->tv_sec = __cputime / CPUTIME_PER_SEC; -#endif } /* @@ -129,17 +112,8 @@ static inline void cputime_to_timeval(const cputime_t cputime, struct timeval *value) { unsigned long long __cputime = (__force unsigned long long) cputime; -#ifndef CONFIG_64BIT - register_pair rp; - - rp.pair = __cputime >> 1; - asm ("dr %0,%1" : "+d" (rp) : "d" (CPUTIME_PER_USEC / 2)); - value->tv_usec = rp.subreg.even / CPUTIME_PER_USEC; - value->tv_sec = rp.subreg.odd; -#else value->tv_usec = (__cputime % CPUTIME_PER_SEC) / CPUTIME_PER_USEC; value->tv_sec = __cputime / CPUTIME_PER_SEC; -#endif } /* diff --git a/arch/s390/include/asm/ctl_reg.h b/arch/s390/include/asm/ctl_reg.h index 31ab9f346d7e..cfad7fca01d6 100644 --- a/arch/s390/include/asm/ctl_reg.h +++ b/arch/s390/include/asm/ctl_reg.h @@ -9,20 +9,12 @@ #include <linux/bug.h> -#ifdef CONFIG_64BIT -# define __CTL_LOAD "lctlg" -# define __CTL_STORE "stctg" -#else -# define __CTL_LOAD "lctl" -# define __CTL_STORE "stctl" -#endif - #define __ctl_load(array, low, high) { \ typedef struct { char _[sizeof(array)]; } addrtype; \ \ BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ asm volatile( \ - __CTL_LOAD " %1,%2,%0\n" \ + " lctlg %1,%2,%0\n" \ : : "Q" (*(addrtype *)(&array)), "i" (low), "i" (high));\ } @@ -31,7 +23,7 @@ \ BUILD_BUG_ON(sizeof(addrtype) != (high - low + 1) * sizeof(long));\ asm volatile( \ - __CTL_STORE " %1,%2,%0\n" \ + " stctg %1,%2,%0\n" \ : "=Q" (*(addrtype *)(&array)) \ : "i" (low), "i" (high)); \ } @@ -60,9 +52,7 @@ void smp_ctl_clear_bit(int cr, int bit); union ctlreg0 { unsigned long val; struct { -#ifdef CONFIG_64BIT unsigned long : 32; -#endif unsigned long : 3; unsigned long lap : 1; /* Low-address-protection control */ unsigned long : 4; diff --git a/arch/s390/include/asm/elf.h b/arch/s390/include/asm/elf.h index a5c4978462c1..3ad48f22de78 100644 --- a/arch/s390/include/asm/elf.h +++ b/arch/s390/include/asm/elf.h @@ -107,11 +107,7 @@ /* * These are used to set parameters in the core dumps. */ -#ifndef CONFIG_64BIT -#define ELF_CLASS ELFCLASS32 -#else /* CONFIG_64BIT */ #define ELF_CLASS ELFCLASS64 -#endif /* CONFIG_64BIT */ #define ELF_DATA ELFDATA2MSB #define ELF_ARCH EM_S390 diff --git a/arch/s390/include/asm/idals.h b/arch/s390/include/asm/idals.h index ea5a6e45fd93..a7b2d7504049 100644 --- a/arch/s390/include/asm/idals.h +++ b/arch/s390/include/asm/idals.h @@ -19,11 +19,7 @@ #include <asm/cio.h> #include <asm/uaccess.h> -#ifdef CONFIG_64BIT #define IDA_SIZE_LOG 12 /* 11 for 2k , 12 for 4k */ -#else -#define IDA_SIZE_LOG 11 /* 11 for 2k , 12 for 4k */ -#endif #define IDA_BLOCK_SIZE (1L<<IDA_SIZE_LOG) /* @@ -32,11 +28,7 @@ static inline int idal_is_needed(void *vaddr, unsigned int length) { -#ifdef CONFIG_64BIT return ((__pa(vaddr) + length - 1) >> 31) != 0; -#else - return 0; -#endif } @@ -77,7 +69,6 @@ static inline unsigned long *idal_create_words(unsigned long *idaws, static inline int set_normalized_cda(struct ccw1 * ccw, void *vaddr) { -#ifdef CONFIG_64BIT unsigned int nridaws; unsigned long *idal; @@ -93,7 +84,6 @@ set_normalized_cda(struct ccw1 * ccw, void *vaddr) ccw->flags |= CCW_FLAG_IDA; vaddr = idal; } -#endif ccw->cda = (__u32)(unsigned long) vaddr; return 0; } @@ -104,12 +94,10 @@ set_normalized_cda(struct ccw1 * ccw, void *vaddr) static inline void clear_normalized_cda(struct ccw1 * ccw) { -#ifdef CONFIG_64BIT if (ccw->flags & CCW_FLAG_IDA) { kfree((void *)(unsigned long) ccw->cda); ccw->flags &= ~CCW_FLAG_IDA; } -#endif ccw->cda = 0; } @@ -181,12 +169,8 @@ idal_buffer_free(struct idal_buffer *ib) static inline int __idal_buffer_is_needed(struct idal_buffer *ib) { -#ifdef CONFIG_64BIT return ib->size > (4096ul << ib->page_order) || idal_is_needed(ib->data[0], ib->size); -#else - return ib->size > (4096ul << ib->page_order); -#endif } /* diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index 343ea7c987aa..ff95d15a2384 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -57,7 +57,6 @@ enum interruption_class { IRQIO_TAP, IRQIO_VMR, IRQIO_LCS, - IRQIO_CLW, IRQIO_CTC, IRQIO_APB, IRQIO_ADM, diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index 2b77e235b5fb..69972b7957ee 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -8,14 +8,6 @@ #define JUMP_LABEL_NOP_SIZE 6 #define JUMP_LABEL_NOP_OFFSET 2 -#ifdef CONFIG_64BIT -#define ASM_PTR ".quad" -#define ASM_ALIGN ".balign 8" -#else -#define ASM_PTR ".long" -#define ASM_ALIGN ".balign 4" -#endif - /* * We use a brcl 0,2 instruction for jump labels at compile time so it * can be easily distinguished from a hotpatch generated instruction. @@ -24,8 +16,8 @@ static __always_inline bool arch_static_branch(struct static_key *key) { asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n" ".pushsection __jump_table, \"aw\"\n" - ASM_ALIGN "\n" - ASM_PTR " 0b, %l[label], %0\n" + ".balign 8\n" + ".quad 0b, %l[label], %0\n" ".popsection\n" : : "X" (key) : : label); return false; diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 34fbcac61133..663f23e37460 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -13,163 +13,6 @@ #include <asm/cpu.h> #include <asm/types.h> -#ifdef CONFIG_32BIT - -#define LC_ORDER 0 -#define LC_PAGES 1 - -struct save_area { - u32 ext_save; - u64 timer; - u64 clk_cmp; - u8 pad1[24]; - u8 psw[8]; - u32 pref_reg; - u8 pad2[20]; - u32 acc_regs[16]; - u64 fp_regs[4]; - u32 gp_regs[16]; - u32 ctrl_regs[16]; -} __packed; - -struct save_area_ext { - struct save_area sa; - __vector128 vx_regs[32]; -}; - -struct _lowcore { - psw_t restart_psw; /* 0x0000 */ - psw_t restart_old_psw; /* 0x0008 */ - __u8 pad_0x0010[0x0014-0x0010]; /* 0x0010 */ - __u32 ipl_parmblock_ptr; /* 0x0014 */ - psw_t external_old_psw; /* 0x0018 */ - psw_t svc_old_psw; /* 0x0020 */ - psw_t program_old_psw; /* 0x0028 */ - psw_t mcck_old_psw; /* 0x0030 */ - psw_t io_old_psw; /* 0x0038 */ - __u8 pad_0x0040[0x0058-0x0040]; /* 0x0040 */ - psw_t external_new_psw; /* 0x0058 */ - psw_t svc_new_psw; /* 0x0060 */ - psw_t program_new_psw; /* 0x0068 */ - psw_t mcck_new_psw; /* 0x0070 */ - psw_t io_new_psw; /* 0x0078 */ - __u32 ext_params; /* 0x0080 */ - __u16 ext_cpu_addr; /* 0x0084 */ - __u16 ext_int_code; /* 0x0086 */ - __u16 svc_ilc; /* 0x0088 */ - __u16 svc_code; /* 0x008a */ - __u16 pgm_ilc; /* 0x008c */ - __u16 pgm_code; /* 0x008e */ - __u32 trans_exc_code; /* 0x0090 */ - __u16 mon_class_num; /* 0x0094 */ - __u8 per_code; /* 0x0096 */ - __u8 per_atmid; /* 0x0097 */ - __u32 per_address; /* 0x0098 */ - __u32 monitor_code; /* 0x009c */ - __u8 exc_access_id; /* 0x00a0 */ - __u8 per_access_id; /* 0x00a1 */ - __u8 op_access_id; /* 0x00a2 */ - __u8 ar_mode_id; /* 0x00a3 */ - __u8 pad_0x00a4[0x00b8-0x00a4]; /* 0x00a4 */ - __u16 subchannel_id; /* 0x00b8 */ - __u16 subchannel_nr; /* 0x00ba */ - __u32 io_int_parm; /* 0x00bc */ - __u32 io_int_word; /* 0x00c0 */ - __u8 pad_0x00c4[0x00c8-0x00c4]; /* 0x00c4 */ - __u32 stfl_fac_list; /* 0x00c8 */ - __u8 pad_0x00cc[0x00d4-0x00cc]; /* 0x00cc */ - __u32 extended_save_area_addr; /* 0x00d4 */ - __u32 cpu_timer_save_area[2]; /* 0x00d8 */ - __u32 clock_comp_save_area[2]; /* 0x00e0 */ - __u32 mcck_interruption_code[2]; /* 0x00e8 */ - __u8 pad_0x00f0[0x00f4-0x00f0]; /* 0x00f0 */ - __u32 external_damage_code; /* 0x00f4 */ - __u32 failing_storage_address; /* 0x00f8 */ - __u8 pad_0x00fc[0x0100-0x00fc]; /* 0x00fc */ - psw_t psw_save_area; /* 0x0100 */ - __u32 prefixreg_save_area; /* 0x0108 */ - __u8 pad_0x010c[0x0120-0x010c]; /* 0x010c */ - - /* CPU register save area: defined by architecture */ - __u32 access_regs_save_area[16]; /* 0x0120 */ - __u32 floating_pt_save_area[8]; /* 0x0160 */ - __u32 gpregs_save_area[16]; /* 0x0180 */ - __u32 cregs_save_area[16]; /* 0x01c0 */ - - /* Save areas. */ - __u32 save_area_sync[8]; /* 0x0200 */ - __u32 save_area_async[8]; /* 0x0220 */ - __u32 save_area_restart[1]; /* 0x0240 */ - - /* CPU flags. */ - __u32 cpu_flags; /* 0x0244 */ - - /* Return psws. */ - psw_t return_psw; /* 0x0248 */ - psw_t return_mcck_psw; /* 0x0250 */ - - /* CPU time accounting values */ - __u64 sync_enter_timer; /* 0x0258 */ - __u64 async_enter_timer; /* 0x0260 */ - __u64 mcck_enter_timer; /* 0x0268 */ - __u64 exit_timer; /* 0x0270 */ - __u64 user_timer; /* 0x0278 */ - __u64 system_timer; /* 0x0280 */ - __u64 steal_timer; /* 0x0288 */ - __u64 last_update_timer; /* 0x0290 */ - __u64 last_update_clock; /* 0x0298 */ - __u64 int_clock; /* 0x02a0 */ - __u64 mcck_clock; /* 0x02a8 */ - __u64 clock_comparator; /* 0x02b0 */ - - /* Current process. */ - __u32 current_task; /* 0x02b8 */ - __u32 thread_info; /* 0x02bc */ - __u32 kernel_stack; /* 0x02c0 */ - - /* Interrupt, panic and restart stack. */ - __u32 async_stack; /* 0x02c4 */ - __u32 panic_stack; /* 0x02c8 */ - __u32 restart_stack; /* 0x02cc */ - - /* Restart function and parameter. */ - __u32 restart_fn; /* 0x02d0 */ - __u32 restart_data; /* 0x02d4 */ - __u32 restart_source; /* 0x02d8 */ - - /* Address space pointer. */ - __u32 kernel_asce; /* 0x02dc */ - __u32 user_asce; /* 0x02e0 */ - __u32 current_pid; /* 0x02e4 */ - - /* SMP info area */ - __u32 cpu_nr; /* 0x02e8 */ - __u32 softirq_pending; /* 0x02ec */ - __u32 percpu_offset; /* 0x02f0 */ - __u32 machine_flags; /* 0x02f4 */ - __u8 pad_0x02f8[0x02fc-0x02f8]; /* 0x02f8 */ - __u32 spinlock_lockval; /* 0x02fc */ - - __u8 pad_0x0300[0x0e00-0x0300]; /* 0x0300 */ - - /* - * 0xe00 contains the address of the IPL Parameter Information - * block. Dump tools need IPIB for IPL after dump. - * Note: do not change the position of any fields in 0x0e00-0x0f00 - */ - __u32 ipib; /* 0x0e00 */ - __u32 ipib_checksum; /* 0x0e04 */ - __u32 vmcore_info; /* 0x0e08 */ - __u8 pad_0x0e0c[0x0e18-0x0e0c]; /* 0x0e0c */ - __u32 os_info; /* 0x0e18 */ - __u8 pad_0x0e1c[0x0f00-0x0e1c]; /* 0x0e1c */ - - /* Extended facility list */ - __u64 stfle_fac_list[32]; /* 0x0f00 */ -} __packed; - -#else /* CONFIG_32BIT */ - #define LC_ORDER 1 #define LC_PAGES 2 @@ -354,8 +197,6 @@ struct _lowcore { __u8 vector_save_area[1024]; /* 0x1c00 */ } __packed; -#endif /* CONFIG_32BIT */ - #define S390_lowcore (*((struct _lowcore *) 0)) extern struct _lowcore *lowcore_ptr[]; diff --git a/arch/s390/include/asm/mman.h b/arch/s390/include/asm/mman.h index 9977e08df5bd..b55a59e1d134 100644 --- a/arch/s390/include/asm/mman.h +++ b/arch/s390/include/asm/mman.h @@ -8,7 +8,7 @@ #include <uapi/asm/mman.h> -#if !defined(__ASSEMBLY__) && defined(CONFIG_64BIT) +#ifndef __ASSEMBLY__ int s390_mmap_check(unsigned long addr, unsigned long len, unsigned long flags); #define arch_mmap_check(addr, len, flags) s390_mmap_check(addr, len, flags) #endif diff --git a/arch/s390/include/asm/mmu_context.h b/arch/s390/include/asm/mmu_context.h index 8fb3802f8fad..d25d9ff10ba8 100644 --- a/arch/s390/include/asm/mmu_context.h +++ b/arch/s390/include/asm/mmu_context.h @@ -19,9 +19,7 @@ static inline int init_new_context(struct task_struct *tsk, atomic_set(&mm->context.attach_count, 0); mm->context.flush_mm = 0; mm->context.asce_bits = _ASCE_TABLE_LENGTH | _ASCE_USER_BITS; -#ifdef CONFIG_64BIT mm->context.asce_bits |= _ASCE_TYPE_REGION3; -#endif mm->context.has_pgste = 0; mm->context.use_skey = 0; mm->context.asce_limit = STACK_TOP_MAX; @@ -110,10 +108,8 @@ static inline void activate_mm(struct mm_struct *prev, static inline void arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) { -#ifdef CONFIG_64BIT if (oldmm->context.asce_limit < mm->context.asce_limit) crst_table_downgrade(mm, oldmm->context.asce_limit); -#endif } static inline void arch_exit_mmap(struct mm_struct *mm) diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index 933355e0d091..6d6556ca24aa 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -10,8 +10,6 @@ */ #define __my_cpu_offset S390_lowcore.percpu_offset -#ifdef CONFIG_64BIT - /* * For 64 bit module code, the module may be more than 4G above the * per cpu area, use weak definitions to force the compiler to @@ -183,8 +181,6 @@ #define this_cpu_cmpxchg_double_4 arch_this_cpu_cmpxchg_double #define this_cpu_cmpxchg_double_8 arch_this_cpu_cmpxchg_double -#endif /* CONFIG_64BIT */ - #include <asm-generic/percpu.h> #endif /* __ARCH_S390_PERCPU__ */ diff --git a/arch/s390/include/asm/perf_event.h b/arch/s390/include/asm/perf_event.h index 159a8ec6da9a..4cb19fe76dd9 100644 --- a/arch/s390/include/asm/perf_event.h +++ b/arch/s390/include/asm/perf_event.h @@ -9,8 +9,6 @@ #ifndef _ASM_S390_PERF_EVENT_H #define _ASM_S390_PERF_EVENT_H -#ifdef CONFIG_64BIT - #include <linux/perf_event.h> #include <linux/device.h> #include <asm/cpu_mf.h> @@ -92,5 +90,4 @@ struct sf_raw_sample { int perf_reserve_sampling(void); void perf_release_sampling(void); -#endif /* CONFIG_64BIT */ #endif /* _ASM_S390_PERF_EVENT_H */ diff --git a/arch/s390/include/asm/pgalloc.h b/arch/s390/include/asm/pgalloc.h index 3009c2ba46d2..51e7fb634ebc 100644 --- a/arch/s390/include/asm/pgalloc.h +++ b/arch/s390/include/asm/pgalloc.h @@ -33,11 +33,7 @@ static inline void clear_table(unsigned long *s, unsigned long val, size_t n) *s = val; n = (n / 256) - 1; asm volatile( -#ifdef CONFIG_64BIT " mvc 8(248,%0),0(%0)\n" -#else - " mvc 4(252,%0),0(%0)\n" -#endif "0: mvc 256(256,%0),0(%0)\n" " la %0,256(%0)\n" " brct %1,0b\n" @@ -50,24 +46,6 @@ static inline void crst_table_init(unsigned long *crst, unsigned long entry) clear_table(crst, entry, sizeof(unsigned long)*2048); } -#ifndef CONFIG_64BIT - -static inline unsigned long pgd_entry_type(struct mm_struct *mm) -{ - return _SEGMENT_ENTRY_EMPTY; -} - -#define pud_alloc_one(mm,address) ({ BUG(); ((pud_t *)2); }) -#define pud_free(mm, x) do { } while (0) - -#define pmd_alloc_one(mm,address) ({ BUG(); ((pmd_t *)2); }) -#define pmd_free(mm, x) do { } while (0) - -#define pgd_populate(mm, pgd, pud) BUG() -#define pud_populate(mm, pud, pmd) BUG() - -#else /* CONFIG_64BIT */ - static inline unsigned long pgd_entry_type(struct mm_struct *mm) { if (mm->context.asce_limit <= (1UL << 31)) @@ -119,8 +97,6 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) pud_val(*pud) = _REGION3_ENTRY | __pa(pmd); } -#endif /* CONFIG_64BIT */ - static inline pgd_t *pgd_alloc(struct mm_struct *mm) { spin_lock_init(&mm->context.list_lock); diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index e08ec38f8c6e..989cfae9e202 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -66,15 +66,9 @@ extern unsigned long zero_page_mask; * table can map * PGDIR_SHIFT determines what a third-level page table entry can map */ -#ifndef CONFIG_64BIT -# define PMD_SHIFT 20 -# define PUD_SHIFT 20 -# define PGDIR_SHIFT 20 -#else /* CONFIG_64BIT */ -# define PMD_SHIFT 20 -# define PUD_SHIFT 31 -# define PGDIR_SHIFT 42 -#endif /* CONFIG_64BIT */ +#define PMD_SHIFT 20 +#define PUD_SHIFT 31 +#define PGDIR_SHIFT 42 #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) @@ -90,15 +84,8 @@ extern unsigned long zero_page_mask; * that leads to 1024 pte per pgd */ #define PTRS_PER_PTE 256 -#ifndef CONFIG_64BIT -#define __PAGETABLE_PUD_FOLDED -#define PTRS_PER_PMD 1 -#define __PAGETABLE_PMD_FOLDED -#define PTRS_PER_PUD 1 -#else /* CONFIG_64BIT */ #define PTRS_PER_PMD 2048 #define PTRS_PER_PUD 2048 -#endif /* CONFIG_64BIT */ #define PTRS_PER_PGD 2048 #define FIRST_USER_ADDRESS 0UL @@ -127,23 +114,19 @@ extern struct page *vmemmap; #define VMEM_MAX_PHYS ((unsigned long) vmemmap) -#ifdef CONFIG_64BIT extern unsigned long MODULES_VADDR; extern unsigned long MODULES_END; #define MODULES_VADDR MODULES_VADDR #define MODULES_END MODULES_END #define MODULES_LEN (1UL << 31) -#endif static inline int is_module_addr(void *addr) { -#ifdef CONFIG_64BIT BUILD_BUG_ON(MODULES_LEN > (1UL << 31)); if (addr < (void *)MODULES_VADDR) return 0; if (addr > (void *)MODULES_END) return 0; -#endif return 1; } @@ -284,56 +267,6 @@ static inline int is_module_addr(void *addr) * pte_swap is true for the bit pattern .10...xxxx10, (pte & 0x603) == 0x402 */ -#ifndef CONFIG_64BIT - -/* Bits in the segment table address-space-control-element */ -#define _ASCE_SPACE_SWITCH 0x80000000UL /* space switch event */ -#define _ASCE_ORIGIN_MASK 0x7ffff000UL /* segment table origin */ -#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */ -#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */ -#define _ASCE_TABLE_LENGTH 0x7f /* 128 x 64 entries = 8k */ - -/* Bits in the segment table entry */ -#define _SEGMENT_ENTRY_BITS 0x7fffffffUL /* Valid segment table bits */ -#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL /* page table origin */ -#define _SEGMENT_ENTRY_PROTECT 0x200 /* page protection bit */ -#define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */ -#define _SEGMENT_ENTRY_COMMON 0x10 /* common segment bit */ -#define _SEGMENT_ENTRY_PTL 0x0f /* page table length */ - -#define _SEGMENT_ENTRY_DIRTY 0 /* No sw dirty bit for 31-bit */ -#define _SEGMENT_ENTRY_YOUNG 0 /* No sw young bit for 31-bit */ -#define _SEGMENT_ENTRY_READ 0 /* No sw read bit for 31-bit */ -#define _SEGMENT_ENTRY_WRITE 0 /* No sw write bit for 31-bit */ -#define _SEGMENT_ENTRY_LARGE 0 /* No large pages for 31-bit */ -#define _SEGMENT_ENTRY_BITS_LARGE 0 -#define _SEGMENT_ENTRY_ORIGIN_LARGE 0 - -#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL) -#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID) - -/* - * Segment table entry encoding (I = invalid, R = read-only bit): - * ..R...I..... - * prot-none ..1...1..... - * read-only ..1...0..... - * read-write ..0...0..... - * empty ..0...1..... - */ - -/* Page status table bits for virtualization */ -#define PGSTE_ACC_BITS 0xf0000000UL -#define PGSTE_FP_BIT 0x08000000UL -#define PGSTE_PCL_BIT 0x00800000UL -#define PGSTE_HR_BIT 0x00400000UL -#define PGSTE_HC_BIT 0x00200000UL -#define PGSTE_GR_BIT 0x00040000UL -#define PGSTE_GC_BIT 0x00020000UL -#define PGSTE_UC_BIT 0x00008000UL /* user dirty (migration) */ -#define PGSTE_IN_BIT 0x00004000UL /* IPTE notify bit */ - -#else /* CONFIG_64BIT */ - /* Bits in the segment/region table address-space-control-element */ #define _ASCE_ORIGIN ~0xfffUL/* segment table origin */ #define _ASCE_PRIVATE_SPACE 0x100 /* private space control */ @@ -417,8 +350,6 @@ static inline int is_module_addr(void *addr) #define PGSTE_UC_BIT 0x0000800000000000UL /* user dirty (migration) */ #define PGSTE_IN_BIT 0x0000400000000000UL /* IPTE notify bit */ -#endif /* CONFIG_64BIT */ - /* Guest Page State used for virtualization */ #define _PGSTE_GPS_ZERO 0x0000000080000000UL #define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL @@ -509,19 +440,6 @@ static inline int mm_use_skey(struct mm_struct *mm) /* * pgd/pmd/pte query functions */ -#ifndef CONFIG_64BIT - -static inline int pgd_present(pgd_t pgd) { return 1; } -static inline int pgd_none(pgd_t pgd) { return 0; } -static inline int pgd_bad(pgd_t pgd) { return 0; } - -static inline int pud_present(pud_t pud) { return 1; } -static inline int pud_none(pud_t pud) { return 0; } -static inline int pud_large(pud_t pud) { return 0; } -static inline int pud_bad(pud_t pud) { return 0; } - -#else /* CONFIG_64BIT */ - static inline int pgd_present(pgd_t pgd) { if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) @@ -583,8 +501,6 @@ static inline int pud_bad(pud_t pud) return (pud_val(pud) & mask) != 0; } -#endif /* CONFIG_64BIT */ - static inline int pmd_present(pmd_t pmd) { return pmd_val(pmd) != _SEGMENT_ENTRY_INVALID; @@ -916,18 +832,14 @@ static inline int pte_unused(pte_t pte) static inline void pgd_clear(pgd_t *pgd) { -#ifdef CONFIG_64BIT if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) pgd_val(*pgd) = _REGION2_ENTRY_EMPTY; -#endif } static inline void pud_clear(pud_t *pud) { -#ifdef CONFIG_64BIT if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) pud_val(*pud) = _REGION3_ENTRY_EMPTY; -#endif } static inline void pmd_clear(pmd_t *pmdp) @@ -1026,10 +938,6 @@ static inline void __ptep_ipte(unsigned long address, pte_t *ptep) { unsigned long pto = (unsigned long) ptep; -#ifndef CONFIG_64BIT - /* pto in ESA mode must point to the start of the segment table */ - pto &= 0x7ffffc00; -#endif /* Invalidation + global TLB flush for the pte */ asm volatile( " ipte %2,%3" @@ -1040,10 +948,6 @@ static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep) { unsigned long pto = (unsigned long) ptep; -#ifndef CONFIG_64BIT - /* pto in ESA mode must point to the start of the segment table */ - pto &= 0x7ffffc00; -#endif /* Invalidation + local TLB flush for the pte */ asm volatile( " .insn rrf,0xb2210000,%2,%3,0,1" @@ -1054,10 +958,6 @@ static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep) { unsigned long pto = (unsigned long) ptep; -#ifndef CONFIG_64BIT - /* pto in ESA mode must point to the start of the segment table */ - pto &= 0x7ffffc00; -#endif /* Invalidate a range of ptes + global TLB flush of the ptes */ do { asm volatile( @@ -1376,17 +1276,6 @@ static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) #define pgd_offset_k(address) pgd_offset(&init_mm, address) -#ifndef CONFIG_64BIT - -#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) -#define pud_deref(pmd) ({ BUG(); 0UL; }) -#define pgd_deref(pmd) ({ BUG(); 0UL; }) - -#define pud_offset(pgd, address) ((pud_t *) pgd) -#define pmd_offset(pud, address) ((pmd_t *) pud + pmd_index(address)) - -#else /* CONFIG_64BIT */ - #define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) #define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN) #define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) @@ -1407,8 +1296,6 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) return pmd + pmd_index(address); } -#endif /* CONFIG_64BIT */ - #define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot)) #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) #define pte_page(x) pfn_to_page(pte_pfn(x)) @@ -1729,11 +1616,9 @@ static inline int has_transparent_hugepage(void) * 0000000000111111111122222222223333333333444444444455 5555 5 55566 66 * 0123456789012345678901234567890123456789012345678901 2345 6 78901 23 */ -#ifndef CONFIG_64BIT -#define __SWP_OFFSET_MASK (~0UL >> 12) -#else + #define __SWP_OFFSET_MASK (~0UL >> 11) -#endif + static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) { pte_t pte; diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index e7cbbdcdee13..dedb6218544b 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -19,7 +19,6 @@ #define _CIF_ASCE (1<<CIF_ASCE) #define _CIF_NOHZ_DELAY (1<<CIF_NOHZ_DELAY) - #ifndef __ASSEMBLY__ #include <linux/linkage.h> @@ -66,13 +65,6 @@ extern void execve_tail(void); /* * User space process size: 2GB for 31 bit, 4TB or 8PT for 64 bit. */ -#ifndef CONFIG_64BIT - -#define TASK_SIZE (1UL << 31) -#define TASK_MAX_SIZE (1UL << 31) -#define TASK_UNMAPPED_BASE (1UL << 30) - -#else /* CONFIG_64BIT */ #define TASK_SIZE_OF(tsk) ((tsk)->mm->context.asce_limit) #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_31BIT) ? \ @@ -80,15 +72,8 @@ extern void execve_tail(void); #define TASK_SIZE TASK_SIZE_OF(current) #define TASK_MAX_SIZE (1UL << 53) -#endif /* CONFIG_64BIT */ - -#ifndef CONFIG_64BIT -#define STACK_TOP (1UL << 31) -#define STACK_TOP_MAX (1UL << 31) -#else /* CONFIG_64BIT */ #define STACK_TOP (1UL << (test_thread_flag(TIF_31BIT) ? 31:42)) #define STACK_TOP_MAX (1UL << 42) -#endif /* CONFIG_64BIT */ #define HAVE_ARCH_PICK_MMAP_LAYOUT @@ -115,10 +100,8 @@ struct thread_struct { /* cpu runtime instrumentation */ struct runtime_instr_cb *ri_cb; int ri_signum; -#ifdef CONFIG_64BIT unsigned char trap_tdb[256]; /* Transaction abort diagnose block */ __vector128 *vxrs; /* Vector register save area */ -#endif }; /* Flag to disable transactions. */ @@ -181,11 +164,7 @@ struct task_struct; struct mm_struct; struct seq_file; -#ifdef CONFIG_64BIT -extern void show_cacheinfo(struct seq_file *m); -#else -static inline void show_cacheinfo(struct seq_file *m) { } -#endif +void show_cacheinfo(struct seq_file *m); /* Free all resources held by a thread. */ extern void release_thread(struct task_struct *); @@ -229,11 +208,7 @@ static inline void psw_set_key(unsigned int key) */ static inline void __load_psw(psw_t psw) { -#ifndef CONFIG_64BIT - asm volatile("lpsw %0" : : "Q" (psw) : "cc"); -#else asm volatile("lpswe %0" : : "Q" (psw) : "cc"); -#endif } /* @@ -247,22 +222,12 @@ static inline void __load_psw_mask (unsigned long mask) psw.mask = mask; -#ifndef CONFIG_64BIT - asm volatile( - " basr %0,0\n" - "0: ahi %0,1f-0b\n" - " st %0,%O1+4(%R1)\n" - " lpsw %1\n" - "1:" - : "=&d" (addr), "=Q" (psw) : "Q" (psw) : "memory", "cc"); -#else /* CONFIG_64BIT */ asm volatile( " larl %0,1f\n" " stg %0,%O1+8(%R1)\n" " lpswe %1\n" "1:" : "=&d" (addr), "=Q" (psw) : "Q" (psw) : "memory", "cc"); -#endif /* CONFIG_64BIT */ } /* @@ -270,20 +235,12 @@ static inline void __load_psw_mask (unsigned long mask) */ static inline unsigned long __rewind_psw(psw_t psw, unsigned long ilc) { -#ifndef CONFIG_64BIT - if (psw.addr & PSW_ADDR_AMODE) - /* 31 bit mode */ - return (psw.addr - ilc) | PSW_ADDR_AMODE; - /* 24 bit mode */ - return (psw.addr - ilc) & ((1UL << 24) - 1); -#else unsigned long mask; mask = (psw.mask & PSW_MASK_EA) ? -1UL : (psw.mask & PSW_MASK_BA) ? (1UL << 31) - 1 : (1UL << 24) - 1; return (psw.addr - ilc) & mask; -#endif } /* @@ -305,26 +262,6 @@ static inline void __noreturn disabled_wait(unsigned long code) * Store status and then load disabled wait psw, * the processor is dead afterwards */ -#ifndef CONFIG_64BIT - asm volatile( - " stctl 0,0,0(%2)\n" - " ni 0(%2),0xef\n" /* switch off protection */ - " lctl 0,0,0(%2)\n" - " stpt 0xd8\n" /* store timer */ - " stckc 0xe0\n" /* store clock comparator */ - " stpx 0x108\n" /* store prefix register */ - " stam 0,15,0x120\n" /* store access registers */ - " std 0,0x160\n" /* store f0 */ - " std 2,0x168\n" /* store f2 */ - " std 4,0x170\n" /* store f4 */ - " std 6,0x178\n" /* store f6 */ - " stm 0,15,0x180\n" /* store general registers */ - " stctl 0,15,0x1c0\n" /* store control registers */ - " oi 0x1c0,0x10\n" /* fake protection bit */ - " lpsw 0(%1)" - : "=m" (ctl_buf) - : "a" (&dw_psw), "a" (&ctl_buf), "m" (dw_psw) : "cc"); -#else /* CONFIG_64BIT */ asm volatile( " stctg 0,0,0(%2)\n" " ni 4(%2),0xef\n" /* switch off protection */ @@ -357,7 +294,6 @@ static inline void __noreturn disabled_wait(unsigned long code) " lpswe 0(%1)" : "=m" (ctl_buf) : "a" (&dw_psw), "a" (&ctl_buf), "m" (dw_psw) : "cc", "0", "1"); -#endif /* CONFIG_64BIT */ while (1); } diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index be317feff7ac..6feda2599282 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -40,12 +40,8 @@ struct psw_bits { unsigned long long ri : 1; /* Runtime Instrumentation */ unsigned long long : 6; unsigned long long eaba : 2; /* Addressing Mode */ -#ifdef CONFIG_64BIT unsigned long long : 31; unsigned long long ia : 64;/* Instruction Address */ -#else - unsigned long long ia : 31;/* Instruction Address */ -#endif }; enum { diff --git a/arch/s390/include/asm/qdio.h b/arch/s390/include/asm/qdio.h index 06f3034605a1..998b61cd0e56 100644 --- a/arch/s390/include/asm/qdio.h +++ b/arch/s390/include/asm/qdio.h @@ -211,11 +211,6 @@ struct qdio_buffer_element { u8 scount; u8 sflags; u32 length; -#ifdef CONFIG_32BIT - /* private: */ - void *res2; - /* public: */ -#endif void *addr; } __attribute__ ((packed, aligned(16))); @@ -232,11 +227,6 @@ struct qdio_buffer { * @sbal: absolute SBAL address */ struct sl_element { -#ifdef CONFIG_32BIT - /* private: */ - unsigned long reserved; - /* public: */ -#endif unsigned long sbal; } __attribute__ ((packed)); diff --git a/arch/s390/include/asm/runtime_instr.h b/arch/s390/include/asm/runtime_instr.h index 830da737ff85..402ad6df4897 100644 --- a/arch/s390/include/asm/runtime_instr.h +++ b/arch/s390/include/asm/runtime_instr.h @@ -72,27 +72,19 @@ static inline void store_runtime_instr_cb(struct runtime_instr_cb *cb) static inline void save_ri_cb(struct runtime_instr_cb *cb_prev) { -#ifdef CONFIG_64BIT if (cb_prev) store_runtime_instr_cb(cb_prev); -#endif } static inline void restore_ri_cb(struct runtime_instr_cb *cb_next, struct runtime_instr_cb *cb_prev) { -#ifdef CONFIG_64BIT if (cb_next) load_runtime_instr_cb(cb_next); else if (cb_prev) load_runtime_instr_cb(&runtime_instr_empty_cb); -#endif } -#ifdef CONFIG_64BIT -extern void exit_thread_runtime_instr(void); -#else -static inline void exit_thread_runtime_instr(void) { } -#endif +void exit_thread_runtime_instr(void); #endif /* _RUNTIME_INSTR_H */ diff --git a/arch/s390/include/asm/rwsem.h b/arch/s390/include/asm/rwsem.h index 487f9b64efb9..4b43ee7e6776 100644 --- a/arch/s390/include/asm/rwsem.h +++ b/arch/s390/include/asm/rwsem.h @@ -39,17 +39,10 @@ #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead" #endif -#ifndef CONFIG_64BIT -#define RWSEM_UNLOCKED_VALUE 0x00000000 -#define RWSEM_ACTIVE_BIAS 0x00000001 -#define RWSEM_ACTIVE_MASK 0x0000ffff -#define RWSEM_WAITING_BIAS (-0x00010000) -#else /* CONFIG_64BIT */ #define RWSEM_UNLOCKED_VALUE 0x0000000000000000L #define RWSEM_ACTIVE_BIAS 0x0000000000000001L #define RWSEM_ACTIVE_MASK 0x00000000ffffffffL #define RWSEM_WAITING_BIAS (-0x0000000100000000L) -#endif /* CONFIG_64BIT */ #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) @@ -61,19 +54,11 @@ static inline void __down_read(struct rw_semaphore *sem) signed long old, new; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: lr %1,%0\n" - " ahi %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: lgr %1,%0\n" " aghi %1,%4\n" " csg %0,%1,%2\n" " jl 0b" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "i" (RWSEM_ACTIVE_READ_BIAS) : "cc", "memory"); @@ -89,15 +74,6 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) signed long old, new; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: ltr %1,%0\n" - " jm 1f\n" - " ahi %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b\n" - "1:" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: ltgr %1,%0\n" " jm 1f\n" @@ -105,7 +81,6 @@ static inline int __down_read_trylock(struct rw_semaphore *sem) " csg %0,%1,%2\n" " jl 0b\n" "1:" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "i" (RWSEM_ACTIVE_READ_BIAS) : "cc", "memory"); @@ -121,19 +96,11 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) tmp = RWSEM_ACTIVE_WRITE_BIAS; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: lr %1,%0\n" - " a %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: lgr %1,%0\n" " ag %1,%4\n" " csg %0,%1,%2\n" " jl 0b" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "m" (tmp) : "cc", "memory"); @@ -154,19 +121,11 @@ static inline int __down_write_trylock(struct rw_semaphore *sem) signed long old; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%1\n" - "0: ltr %0,%0\n" - " jnz 1f\n" - " cs %0,%3,%1\n" - " jl 0b\n" -#else /* CONFIG_64BIT */ " lg %0,%1\n" "0: ltgr %0,%0\n" " jnz 1f\n" " csg %0,%3,%1\n" " jl 0b\n" -#endif /* CONFIG_64BIT */ "1:" : "=&d" (old), "=Q" (sem->count) : "Q" (sem->count), "d" (RWSEM_ACTIVE_WRITE_BIAS) @@ -182,19 +141,11 @@ static inline void __up_read(struct rw_semaphore *sem) signed long old, new; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: lr %1,%0\n" - " ahi %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: lgr %1,%0\n" " aghi %1,%4\n" " csg %0,%1,%2\n" " jl 0b" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "i" (-RWSEM_ACTIVE_READ_BIAS) : "cc", "memory"); @@ -212,19 +163,11 @@ static inline void __up_write(struct rw_semaphore *sem) tmp = -RWSEM_ACTIVE_WRITE_BIAS; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: lr %1,%0\n" - " a %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: lgr %1,%0\n" " ag %1,%4\n" " csg %0,%1,%2\n" " jl 0b" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "m" (tmp) : "cc", "memory"); @@ -242,19 +185,11 @@ static inline void __downgrade_write(struct rw_semaphore *sem) tmp = -RWSEM_WAITING_BIAS; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: lr %1,%0\n" - " a %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: lgr %1,%0\n" " ag %1,%4\n" " csg %0,%1,%2\n" " jl 0b" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "m" (tmp) : "cc", "memory"); @@ -270,19 +205,11 @@ static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem) signed long old, new; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: lr %1,%0\n" - " ar %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: lgr %1,%0\n" " agr %1,%4\n" " csg %0,%1,%2\n" " jl 0b" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "d" (delta) : "cc", "memory"); @@ -296,19 +223,11 @@ static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) signed long old, new; asm volatile( -#ifndef CONFIG_64BIT - " l %0,%2\n" - "0: lr %1,%0\n" - " ar %1,%4\n" - " cs %0,%1,%2\n" - " jl 0b" -#else /* CONFIG_64BIT */ " lg %0,%2\n" "0: lgr %1,%0\n" " agr %1,%4\n" " csg %0,%1,%2\n" " jl 0b" -#endif /* CONFIG_64BIT */ : "=&d" (old), "=&d" (new), "=Q" (sem->count) : "Q" (sem->count), "d" (delta) : "cc", "memory"); diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h index b8d1e54b4733..b8ffc1bd0a9f 100644 --- a/arch/s390/include/asm/setup.h +++ b/arch/s390/include/asm/setup.h @@ -15,19 +15,11 @@ #include <asm/lowcore.h> #include <asm/types.h> -#ifndef CONFIG_64BIT -#define IPL_DEVICE (*(unsigned long *) (0x10404)) -#define INITRD_START (*(unsigned long *) (0x1040C)) -#define INITRD_SIZE (*(unsigned long *) (0x10414)) -#define OLDMEM_BASE (*(unsigned long *) (0x1041C)) -#define OLDMEM_SIZE (*(unsigned long *) (0x10424)) -#else /* CONFIG_64BIT */ #define IPL_DEVICE (*(unsigned long *) (0x10400)) #define INITRD_START (*(unsigned long *) (0x10408)) #define INITRD_SIZE (*(unsigned long *) (0x10410)) #define OLDMEM_BASE (*(unsigned long *) (0x10418)) #define OLDMEM_SIZE (*(unsigned long *) (0x10420)) -#endif /* CONFIG_64BIT */ #define COMMAND_LINE ((char *) (0x10480)) extern int memory_end_set; @@ -68,26 +60,8 @@ extern void detect_memory_memblock(void); #define MACHINE_HAS_PFMF MACHINE_HAS_EDAT1 #define MACHINE_HAS_HPAGE MACHINE_HAS_EDAT1 -#ifndef CONFIG_64BIT -#define MACHINE_HAS_IEEE (S390_lowcore.machine_flags & MACHINE_FLAG_IEEE) -#define MACHINE_HAS_CSP (S390_lowcore.machine_flags & MACHINE_FLAG_CSP) -#define MACHINE_HAS_IDTE (0) -#define MACHINE_HAS_DIAG44 (1) -#define MACHINE_HAS_MVPG (S390_lowcore.machine_flags & MACHINE_FLAG_MVPG) -#define MACHINE_HAS_EDAT1 (0) -#define MACHINE_HAS_EDAT2 (0) -#define MACHINE_HAS_LPP (0) -#define MACHINE_HAS_TOPOLOGY (0) -#define MACHINE_HAS_TE (0) -#define MACHINE_HAS_TLB_LC (0) -#define MACHINE_HAS_VX (0) -#define MACHINE_HAS_CAD (0) -#else /* CONFIG_64BIT */ -#define MACHINE_HAS_IEEE (1) -#define MACHINE_HAS_CSP (1) #define MACHINE_HAS_IDTE (S390_lowcore.machine_flags & MACHINE_FLAG_IDTE) #define MACHINE_HAS_DIAG44 (S390_lowcore.machine_flags & MACHINE_FLAG_DIAG44) -#define MACHINE_HAS_MVPG (1) #define MACHINE_HAS_EDAT1 (S390_lowcore.machine_flags & MACHINE_FLAG_EDAT1) #define MACHINE_HAS_EDAT2 (S390_lowcore.machine_flags & MACHINE_FLAG_EDAT2) #define MACHINE_HAS_LPP (S390_lowcore.machine_flags & MACHINE_FLAG_LPP) @@ -96,7 +70,6 @@ extern void detect_memory_memblock(void); #define MACHINE_HAS_TLB_LC (S390_lowcore.machine_flags & MACHINE_FLAG_TLB_LC) #define MACHINE_HAS_VX (S390_lowcore.machine_flags & MACHINE_FLAG_VX) #define MACHINE_HAS_CAD (S390_lowcore.machine_flags & MACHINE_FLAG_CAD) -#endif /* CONFIG_64BIT */ /* * Console mode. Override with conmode= @@ -135,19 +108,11 @@ extern void (*_machine_power_off)(void); #else /* __ASSEMBLY__ */ -#ifndef CONFIG_64BIT -#define IPL_DEVICE 0x10404 -#define INITRD_START 0x1040C -#define INITRD_SIZE 0x10414 -#define OLDMEM_BASE 0x1041C -#define OLDMEM_SIZE 0x10424 -#else /* CONFIG_64BIT */ #define IPL_DEVICE 0x10400 #define INITRD_START 0x10408 #define INITRD_SIZE 0x10410 #define OLDMEM_BASE 0x10418 #define OLDMEM_SIZE 0x10420 -#endif /* CONFIG_64BIT */ #define COMMAND_LINE 0x10480 #endif /* __ASSEMBLY__ */ diff --git a/arch/s390/include/asm/sfp-util.h b/arch/s390/include/asm/sfp-util.h index 5959bfb3b693..c8b7cf9d6279 100644 --- a/arch/s390/include/asm/sfp-util.h +++ b/arch/s390/include/asm/sfp-util.h @@ -51,7 +51,6 @@ wl = __wl; \ }) -#ifdef CONFIG_64BIT #define udiv_qrnnd(q, r, n1, n0, d) \ do { unsigned long __n; \ unsigned int __r, __d; \ @@ -60,15 +59,6 @@ (q) = __n / __d; \ (r) = __n % __d; \ } while (0) -#else -#define udiv_qrnnd(q, r, n1, n0, d) \ - do { unsigned int __r; \ - (q) = __udiv_qrnnd (&__r, (n1), (n0), (d)); \ - (r) = __r; \ - } while (0) -extern unsigned long __udiv_qrnnd (unsigned int *, unsigned int, - unsigned int , unsigned int); -#endif #define UDIV_NEEDS_NORMALIZATION 0 diff --git a/arch/s390/include/asm/sparsemem.h b/arch/s390/include/asm/sparsemem.h index a60d085ddb4d..487428b6d099 100644 --- a/arch/s390/include/asm/sparsemem.h +++ b/arch/s390/include/asm/sparsemem.h @@ -1,16 +1,7 @@ #ifndef _ASM_S390_SPARSEMEM_H #define _ASM_S390_SPARSEMEM_H -#ifdef CONFIG_64BIT - #define SECTION_SIZE_BITS 28 #define MAX_PHYSMEM_BITS 46 -#else - -#define SECTION_SIZE_BITS 25 -#define MAX_PHYSMEM_BITS 31 - -#endif /* CONFIG_64BIT */ - #endif /* _ASM_S390_SPARSEMEM_H */ diff --git a/arch/s390/include/asm/switch_to.h b/arch/s390/include/asm/switch_to.h index 2542a7e4c8b4..d62e7a69605f 100644 --- a/arch/s390/include/asm/switch_to.h +++ b/arch/s390/include/asm/switch_to.h @@ -18,9 +18,6 @@ static inline int test_fp_ctl(u32 fpc) u32 orig_fpc; int rc; - if (!MACHINE_HAS_IEEE) - return 0; - asm volatile( " efpc %1\n" " sfpc %2\n" @@ -35,9 +32,6 @@ static inline int test_fp_ctl(u32 fpc) static inline void save_fp_ctl(u32 *fpc) { - if (!MACHINE_HAS_IEEE) - return; - asm volatile( " stfpc %0\n" : "+Q" (*fpc)); @@ -47,9 +41,6 @@ static inline int restore_fp_ctl(u32 *fpc) { int rc; - if (!MACHINE_HAS_IEEE) - return 0; - asm volatile( " lfpc %1\n" "0: la %0,0\n" @@ -65,8 +56,6 @@ static inline void save_fp_regs(freg_t *fprs) asm volatile("std 2,%0" : "=Q" (fprs[2])); asm volatile("std 4,%0" : "=Q" (fprs[4])); asm volatile("std 6,%0" : "=Q" (fprs[6])); - if (!MACHINE_HAS_IEEE) - return; asm volatile("std 1,%0" : "=Q" (fprs[1])); asm volatile("std 3,%0" : "=Q" (fprs[3])); asm volatile("std 5,%0" : "=Q" (fprs[5])); @@ -87,8 +76,6 @@ static inline void restore_fp_regs(freg_t *fprs) asm volatile("ld 2,%0" : : "Q" (fprs[2])); asm volatile("ld 4,%0" : : "Q" (fprs[4])); asm volatile("ld 6,%0" : : "Q" (fprs[6])); - if (!MACHINE_HAS_IEEE) - return; asm volatile("ld 1,%0" : : "Q" (fprs[1])); asm volatile("ld 3,%0" : : "Q" (fprs[3])); asm volatile("ld 5,%0" : : "Q" (fprs[5])); @@ -140,22 +127,18 @@ static inline void restore_vx_regs(__vector128 *vxrs) static inline void save_fp_vx_regs(struct task_struct *task) { -#ifdef CONFIG_64BIT if (task->thread.vxrs) save_vx_regs(task->thread.vxrs); else -#endif - save_fp_regs(task->thread.fp_regs.fprs); + save_fp_regs(task->thread.fp_regs.fprs); } static inline void restore_fp_vx_regs(struct task_struct *task) { -#ifdef CONFIG_64BIT if (task->thread.vxrs) restore_vx_regs(task->thread.vxrs); else -#endif - restore_fp_regs(task->thread.fp_regs.fprs); + restore_fp_regs(task->thread.fp_regs.fprs); } static inline void save_access_regs(unsigned int *acrs) diff --git a/arch/s390/include/asm/syscall.h b/arch/s390/include/asm/syscall.h index 5bc12598ae9e..6ba0bf928909 100644 --- a/arch/s390/include/asm/syscall.h +++ b/arch/s390/include/asm/syscall.h @@ -95,6 +95,6 @@ static inline int syscall_get_arch(void) if (test_tsk_thread_flag(current, TIF_31BIT)) return AUDIT_ARCH_S390; #endif - return sizeof(long) == 8 ? AUDIT_ARCH_S390X : AUDIT_ARCH_S390; + return AUDIT_ARCH_S390X; } #endif /* _ASM_SYSCALL_H */ diff --git a/arch/s390/include/asm/thread_info.h b/arch/s390/include/asm/thread_info.h index ef1df718642d..4c27ec764c36 100644 --- a/arch/s390/include/asm/thread_info.h +++ b/arch/s390/include/asm/thread_info.h @@ -10,13 +10,8 @@ /* * Size of kernel stack for each process */ -#ifndef CONFIG_64BIT -#define THREAD_ORDER 1 -#define ASYNC_ORDER 1 -#else /* CONFIG_64BIT */ #define THREAD_ORDER 2 #define ASYNC_ORDER 2 -#endif /* CONFIG_64BIT */ #define THREAD_SIZE (PAGE_SIZE << THREAD_ORDER) #define ASYNC_SIZE (PAGE_SIZE << ASYNC_ORDER) @@ -34,7 +29,6 @@ */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long sys_call_table; /* System call table address */ unsigned int cpu; /* current CPU */ @@ -51,7 +45,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ @@ -66,6 +59,8 @@ static inline struct thread_info *current_thread_info(void) return (struct thread_info *) S390_lowcore.thread_info; } +void arch_release_task_struct(struct task_struct *tsk); + #define THREAD_SIZE_ORDER THREAD_ORDER #endif @@ -99,10 +94,6 @@ static inline struct thread_info *current_thread_info(void) #define _TIF_31BIT (1<<TIF_31BIT) #define _TIF_SINGLE_STEP (1<<TIF_SINGLE_STEP) -#ifdef CONFIG_64BIT #define is_32bit_task() (test_thread_flag(TIF_31BIT)) -#else -#define is_32bit_task() (1) -#endif #endif /* _ASM_THREAD_INFO_H */ diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h index 06d8741ad6f4..7a92e69c50bc 100644 --- a/arch/s390/include/asm/tlb.h +++ b/arch/s390/include/asm/tlb.h @@ -118,12 +118,10 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, unsigned long address) { -#ifdef CONFIG_64BIT if (tlb->mm->context.asce_limit <= (1UL << 31)) return; pgtable_pmd_page_dtor(virt_to_page(pmd)); tlb_remove_table(tlb, pmd); -#endif } /* @@ -136,11 +134,9 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, unsigned long address) { -#ifdef CONFIG_64BIT if (tlb->mm->context.asce_limit <= (1UL << 42)) return; tlb_remove_table(tlb, pud); -#endif } #define tlb_start_vma(tlb, vma) do { } while (0) diff --git a/arch/s390/include/asm/tlbflush.h b/arch/s390/include/asm/tlbflush.h index 16c9c88658c8..ca148f7c3eaa 100644 --- a/arch/s390/include/asm/tlbflush.h +++ b/arch/s390/include/asm/tlbflush.h @@ -49,13 +49,6 @@ static inline void __tlb_flush_global(void) register unsigned long reg4 asm("4"); long dummy; -#ifndef CONFIG_64BIT - if (!MACHINE_HAS_CSP) { - smp_ptlb_all(); - return; - } -#endif /* CONFIG_64BIT */ - dummy = 0; reg2 = reg3 = 0; reg4 = ((unsigned long) &dummy) + 1; diff --git a/arch/s390/include/asm/types.h b/arch/s390/include/asm/types.h index dccef3ca91fa..6740f4f9781f 100644 --- a/arch/s390/include/asm/types.h +++ b/arch/s390/include/asm/types.h @@ -8,21 +8,4 @@ #include <uapi/asm/types.h> -/* - * These aren't exported outside the kernel to avoid name space clashes - */ - -#ifndef __ASSEMBLY__ - -#ifndef CONFIG_64BIT -typedef union { - unsigned long long pair; - struct { - unsigned long even; - unsigned long odd; - } subreg; -} register_pair; - -#endif /* ! CONFIG_64BIT */ -#endif /* __ASSEMBLY__ */ #endif /* _S390_TYPES_H */ diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index cd4c68e0398d..d64a7a62164f 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -372,5 +372,6 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo } int copy_to_user_real(void __user *dest, void *src, unsigned long count); +void s390_kernel_write(void *dst, const void *src, size_t size); #endif /* __S390_UACCESS_H */ diff --git a/arch/s390/include/asm/unistd.h b/arch/s390/include/asm/unistd.h index 651886353551..91f56b1d8156 100644 --- a/arch/s390/include/asm/unistd.h +++ b/arch/s390/include/asm/unistd.h @@ -9,11 +9,7 @@ #include <uapi/asm/unistd.h> -#ifndef CONFIG_64BIT -#define __IGNORE_select -#else #define __IGNORE_time -#endif /* Ignore NUMA system calls. Not wired up on s390. */ #define __IGNORE_mbind @@ -43,10 +39,6 @@ #define __ARCH_WANT_SYS_OLDUMOUNT #define __ARCH_WANT_SYS_SIGPENDING #define __ARCH_WANT_SYS_SIGPROCMASK -# ifndef CONFIG_64BIT -# define __ARCH_WANT_STAT64 -# define __ARCH_WANT_SYS_TIME -# endif # ifdef CONFIG_COMPAT # define __ARCH_WANT_COMPAT_SYS_TIME # endif diff --git a/arch/s390/include/asm/vdso.h b/arch/s390/include/asm/vdso.h index a62526d09201..787acd4f9668 100644 --- a/arch/s390/include/asm/vdso.h +++ b/arch/s390/include/asm/vdso.h @@ -42,10 +42,8 @@ struct vdso_per_cpu_data { extern struct vdso_data *vdso_data; -#ifdef CONFIG_64BIT int vdso_alloc_per_cpu(struct _lowcore *lowcore); void vdso_free_per_cpu(struct _lowcore *lowcore); -#endif #endif /* __ASSEMBLY__ */ #endif /* __S390_VDSO_H__ */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 31fab2676fe9..ffb87617a36c 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -26,25 +26,21 @@ CFLAGS_dumpstack.o += -fno-optimize-sibling-calls # CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"' -CFLAGS_sysinfo.o += -Iinclude/math-emu -Iarch/s390/math-emu -w +CFLAGS_sysinfo.o += -w obj-y := traps.o time.o process.o base.o early.o setup.o idle.o vtime.o obj-y += processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o nmi.o obj-y += debug.o irq.o ipl.o dis.o diag.o sclp.o vdso.o obj-y += sysinfo.o jump_label.o lgr.o os_info.o machine_kexec.o pgm_check.o -obj-y += dumpstack.o +obj-y += runtime_instr.o cache.o dumpstack.o +obj-y += entry.o reipl.o relocate_kernel.o -obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o) -obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o) -obj-y += $(if $(CONFIG_64BIT),relocate_kernel64.o,relocate_kernel.o) - -extra-y += head.o vmlinux.lds -extra-y += $(if $(CONFIG_64BIT),head64.o,head31.o) +extra-y += head.o head64.o vmlinux.lds obj-$(CONFIG_MODULES) += s390_ksyms.o module.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_SCHED_BOOK) += topology.o -obj-$(CONFIG_HIBERNATION) += suspend.o swsusp_asm64.o +obj-$(CONFIG_HIBERNATION) += suspend.o swsusp.o obj-$(CONFIG_AUDIT) += audit.o compat-obj-$(CONFIG_AUDIT) += compat_audit.o obj-$(CONFIG_COMPAT) += compat_linux.o compat_signal.o @@ -56,13 +52,9 @@ obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_UPROBES) += uprobes.o -ifdef CONFIG_64BIT -obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_cpum_cf.o perf_cpum_sf.o \ - perf_cpum_cf_events.o -obj-y += runtime_instr.o cache.o -endif +obj-$(CONFIG_PERF_EVENTS) += perf_event.o perf_cpum_cf.o perf_cpum_sf.o +obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf_events.o # vdso -obj-$(CONFIG_64BIT) += vdso64/ -obj-$(CONFIG_32BIT) += vdso32/ +obj-y += vdso64/ obj-$(CONFIG_COMPAT) += vdso32/ diff --git a/arch/s390/kernel/asm-offsets.c b/arch/s390/kernel/asm-offsets.c index 8dc4db10d160..c7d1b9d09011 100644 --- a/arch/s390/kernel/asm-offsets.c +++ b/arch/s390/kernel/asm-offsets.c @@ -34,7 +34,6 @@ int main(void) DEFINE(__THREAD_per_paid, offsetof(struct task_struct, thread.per_event.paid)); BLANK(); DEFINE(__TI_task, offsetof(struct thread_info, task)); - DEFINE(__TI_domain, offsetof(struct thread_info, exec_domain)); DEFINE(__TI_flags, offsetof(struct thread_info, flags)); DEFINE(__TI_sysc_table, offsetof(struct thread_info, sys_call_table)); DEFINE(__TI_cpu, offsetof(struct thread_info, cpu)); @@ -166,9 +165,6 @@ int main(void) DEFINE(__LC_FPREGS_SAVE_AREA, offsetof(struct _lowcore, floating_pt_save_area)); DEFINE(__LC_GPREGS_SAVE_AREA, offsetof(struct _lowcore, gpregs_save_area)); DEFINE(__LC_CREGS_SAVE_AREA, offsetof(struct _lowcore, cregs_save_area)); -#ifdef CONFIG_32BIT - DEFINE(SAVE_AREA_BASE, offsetof(struct _lowcore, extended_save_area_addr)); -#else /* CONFIG_32BIT */ DEFINE(__LC_DATA_EXC_CODE, offsetof(struct _lowcore, data_exc_code)); DEFINE(__LC_MCCK_FAIL_STOR_ADDR, offsetof(struct _lowcore, failing_storage_address)); DEFINE(__LC_VX_SAVE_AREA_ADDR, offsetof(struct _lowcore, vector_save_area_addr)); @@ -184,6 +180,5 @@ int main(void) DEFINE(__GMAP_ASCE, offsetof(struct gmap, asce)); DEFINE(__SIE_PROG0C, offsetof(struct kvm_s390_sie_block, prog0c)); DEFINE(__SIE_PROG20, offsetof(struct kvm_s390_sie_block, prog20)); -#endif /* CONFIG_32BIT */ return 0; } diff --git a/arch/s390/kernel/base.S b/arch/s390/kernel/base.S index f74a53d339b0..daed3fde42ec 100644 --- a/arch/s390/kernel/base.S +++ b/arch/s390/kernel/base.S @@ -11,8 +11,6 @@ #include <asm/ptrace.h> #include <asm/sigp.h> -#ifdef CONFIG_64BIT - ENTRY(s390_base_mcck_handler) basr %r13,0 0: lg %r15,__LC_PANIC_STACK # load panic stack @@ -131,77 +129,3 @@ ENTRY(diag308_reset) .Lfpctl: .long 0 .previous - -#else /* CONFIG_64BIT */ - -ENTRY(s390_base_mcck_handler) - basr %r13,0 -0: l %r15,__LC_PANIC_STACK # load panic stack - ahi %r15,-STACK_FRAME_OVERHEAD - l %r1,2f-0b(%r13) - l %r1,0(%r1) - ltr %r1,%r1 - jz 1f - basr %r14,%r1 -1: lm %r0,%r15,__LC_GPREGS_SAVE_AREA - lpsw __LC_MCK_OLD_PSW - -2: .long s390_base_mcck_handler_fn - - .section .bss - .align 4 - .globl s390_base_mcck_handler_fn -s390_base_mcck_handler_fn: - .long 0 - .previous - -ENTRY(s390_base_ext_handler) - stm %r0,%r15,__LC_SAVE_AREA_ASYNC - basr %r13,0 -0: ahi %r15,-STACK_FRAME_OVERHEAD - l %r1,2f-0b(%r13) - l %r1,0(%r1) - ltr %r1,%r1 - jz 1f - basr %r14,%r1 -1: lm %r0,%r15,__LC_SAVE_AREA_ASYNC - ni __LC_EXT_OLD_PSW+1,0xfd # clear wait state bit - lpsw __LC_EXT_OLD_PSW - -2: .long s390_base_ext_handler_fn - - .section .bss - .align 4 - .globl s390_base_ext_handler_fn -s390_base_ext_handler_fn: - .long 0 - .previous - -ENTRY(s390_base_pgm_handler) - stm %r0,%r15,__LC_SAVE_AREA_SYNC - basr %r13,0 -0: ahi %r15,-STACK_FRAME_OVERHEAD - l %r1,2f-0b(%r13) - l %r1,0(%r1) - ltr %r1,%r1 - jz 1f - basr %r14,%r1 - lm %r0,%r15,__LC_SAVE_AREA_SYNC - lpsw __LC_PGM_OLD_PSW - -1: lpsw disabled_wait_psw-0b(%r13) - -2: .long s390_base_pgm_handler_fn - -disabled_wait_psw: - .align 8 - .long 0x000a0000,0x00000000 + s390_base_pgm_handler - - .section .bss - .align 4 - .globl s390_base_pgm_handler_fn -s390_base_pgm_handler_fn: - .long 0 - .previous - -#endif /* CONFIG_64BIT */ diff --git a/arch/s390/kernel/cache.c b/arch/s390/kernel/cache.c index 0969d113b3d6..bff5e3b6d822 100644 --- a/arch/s390/kernel/cache.c +++ b/arch/s390/kernel/cache.c @@ -70,6 +70,8 @@ void show_cacheinfo(struct seq_file *m) struct cacheinfo *cache; int idx; + if (!test_facility(34)) + return; get_online_cpus(); this_cpu_ci = get_cpu_cacheinfo(cpumask_any(cpu_online_mask)); for (idx = 0; idx < this_cpu_ci->num_leaves; idx++) { @@ -159,6 +161,8 @@ int populate_cache_leaves(unsigned int cpu) union cache_topology ct; enum cache_type ctype; + if (!test_facility(34)) + return -EOPNOTSUPP; ct.raw = ecag(EXTRACT_TOPOLOGY, 0, 0); for (idx = 0, level = 0; level < this_cpu_ci->num_levels && idx < this_cpu_ci->num_leaves; idx++, level++) { diff --git a/arch/s390/kernel/compat_signal.c b/arch/s390/kernel/compat_signal.c index bc1df12dd4f8..fe8d6924efaa 100644 --- a/arch/s390/kernel/compat_signal.c +++ b/arch/s390/kernel/compat_signal.c @@ -370,16 +370,6 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size) return (void __user *)((sp - frame_size) & -8ul); } -static inline int map_signal(int sig) -{ - if (current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32) - return current_thread_info()->exec_domain->signal_invmap[sig]; - else - return sig; -} - static int setup_frame32(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs) { @@ -449,7 +439,7 @@ static int setup_frame32(struct ksignal *ksig, sigset_t *set, (regs->psw.mask & ~PSW_MASK_ASC); regs->psw.addr = (__force __u64) ksig->ka.sa.sa_handler; - regs->gprs[2] = map_signal(sig); + regs->gprs[2] = sig; regs->gprs[3] = (__force __u64) &frame->sc; /* We forgot to include these in the sigcontext. @@ -532,7 +522,7 @@ static int setup_rt_frame32(struct ksignal *ksig, sigset_t *set, (regs->psw.mask & ~PSW_MASK_ASC); regs->psw.addr = (__u64 __force) ksig->ka.sa.sa_handler; - regs->gprs[2] = map_signal(ksig->sig); + regs->gprs[2] = ksig->sig; regs->gprs[3] = (__force __u64) &frame->info; regs->gprs[4] = (__force __u64) &frame->uc; regs->gprs[5] = task_thread_info(current)->last_break; diff --git a/arch/s390/kernel/cpcmd.c b/arch/s390/kernel/cpcmd.c index d7b0c4d27880..199ec92ef4fe 100644 --- a/arch/s390/kernel/cpcmd.c +++ b/arch/s390/kernel/cpcmd.c @@ -27,13 +27,9 @@ static int diag8_noresponse(int cmdlen) register unsigned long reg3 asm ("3") = cmdlen; asm volatile( -#ifndef CONFIG_64BIT - " diag %1,%0,0x8\n" -#else /* CONFIG_64BIT */ " sam31\n" " diag %1,%0,0x8\n" " sam64\n" -#endif /* CONFIG_64BIT */ : "+d" (reg3) : "d" (reg2) : "cc"); return reg3; } @@ -46,17 +42,11 @@ static int diag8_response(int cmdlen, char *response, int *rlen) register unsigned long reg5 asm ("5") = *rlen; asm volatile( -#ifndef CONFIG_64BIT - " diag %2,%0,0x8\n" - " brc 8,1f\n" - " ar %1,%4\n" -#else /* CONFIG_64BIT */ " sam31\n" " diag %2,%0,0x8\n" " sam64\n" " brc 8,1f\n" " agr %1,%4\n" -#endif /* CONFIG_64BIT */ "1:\n" : "+d" (reg4), "+d" (reg5) : "d" (reg2), "d" (reg3), "d" (*rlen) : "cc"); diff --git a/arch/s390/kernel/diag.c b/arch/s390/kernel/diag.c index 8237fc07ac79..2f69243bf700 100644 --- a/arch/s390/kernel/diag.c +++ b/arch/s390/kernel/diag.c @@ -18,13 +18,9 @@ int diag14(unsigned long rx, unsigned long ry1, unsigned long subcode) int rc = 0; asm volatile( -#ifdef CONFIG_64BIT " sam31\n" " diag %2,2,0x14\n" " sam64\n" -#else - " diag %2,2,0x14\n" -#endif " ipm %0\n" " srl %0,28\n" : "=d" (rc), "+d" (_ry2) @@ -52,7 +48,6 @@ int diag210(struct diag210 *addr) spin_lock_irqsave(&diag210_lock, flags); diag210_tmp = *addr; -#ifdef CONFIG_64BIT asm volatile( " lhi %0,-1\n" " sam31\n" @@ -62,16 +57,6 @@ int diag210(struct diag210 *addr) "1: sam64\n" EX_TABLE(0b, 1b) : "=&d" (ccode) : "a" (&diag210_tmp) : "cc", "memory"); -#else - asm volatile( - " lhi %0,-1\n" - " diag %1,0,0x210\n" - "0: ipm %0\n" - " srl %0,28\n" - "1:\n" - EX_TABLE(0b, 1b) - : "=&d" (ccode) : "a" (&diag210_tmp) : "cc", "memory"); -#endif *addr = diag210_tmp; spin_unlock_irqrestore(&diag210_lock, flags); diff --git a/arch/s390/kernel/dis.c b/arch/s390/kernel/dis.c index 533430307da8..8140d10c6785 100644 --- a/arch/s390/kernel/dis.c +++ b/arch/s390/kernel/dis.c @@ -32,12 +32,6 @@ #include <asm/debug.h> #include <asm/irq.h> -#ifndef CONFIG_64BIT -#define ONELONG "%08lx: " -#else /* CONFIG_64BIT */ -#define ONELONG "%016lx: " -#endif /* CONFIG_64BIT */ - enum { UNUSED, /* Indicates the end of the operand list */ R_8, /* GPR starting at position 8 */ @@ -536,12 +530,10 @@ static char *long_insn_name[] = { }; static struct s390_insn opcode[] = { -#ifdef CONFIG_64BIT { "bprp", 0xc5, INSTR_MII_UPI }, { "bpp", 0xc7, INSTR_SMI_U0RDP }, { "trtr", 0xd0, INSTR_SS_L0RDRD }, { "lmd", 0xef, INSTR_SS_RRRDRD3 }, -#endif { "spm", 0x04, INSTR_RR_R0 }, { "balr", 0x05, INSTR_RR_RR }, { "bctr", 0x06, INSTR_RR_RR }, @@ -725,11 +717,9 @@ static struct s390_insn opcode[] = { }; static struct s390_insn opcode_01[] = { -#ifdef CONFIG_64BIT { "ptff", 0x04, INSTR_E }, { "pfpo", 0x0a, INSTR_E }, { "sam64", 0x0e, INSTR_E }, -#endif { "pr", 0x01, INSTR_E }, { "upt", 0x02, INSTR_E }, { "sckpf", 0x07, INSTR_E }, @@ -741,7 +731,6 @@ static struct s390_insn opcode_01[] = { }; static struct s390_insn opcode_a5[] = { -#ifdef CONFIG_64BIT { "iihh", 0x00, INSTR_RI_RU }, { "iihl", 0x01, INSTR_RI_RU }, { "iilh", 0x02, INSTR_RI_RU }, @@ -758,12 +747,10 @@ static struct s390_insn opcode_a5[] = { { "llihl", 0x0d, INSTR_RI_RU }, { "llilh", 0x0e, INSTR_RI_RU }, { "llill", 0x0f, INSTR_RI_RU }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_a7[] = { -#ifdef CONFIG_64BIT { "tmhh", 0x02, INSTR_RI_RU }, { "tmhl", 0x03, INSTR_RI_RU }, { "brctg", 0x07, INSTR_RI_RP }, @@ -771,7 +758,6 @@ static struct s390_insn opcode_a7[] = { { "aghi", 0x0b, INSTR_RI_RI }, { "mghi", 0x0d, INSTR_RI_RI }, { "cghi", 0x0f, INSTR_RI_RI }, -#endif { "tmlh", 0x00, INSTR_RI_RU }, { "tmll", 0x01, INSTR_RI_RU }, { "brc", 0x04, INSTR_RI_UP }, @@ -785,18 +771,15 @@ static struct s390_insn opcode_a7[] = { }; static struct s390_insn opcode_aa[] = { -#ifdef CONFIG_64BIT { { 0, LONG_INSN_RINEXT }, 0x00, INSTR_RI_RI }, { "rion", 0x01, INSTR_RI_RI }, { "tric", 0x02, INSTR_RI_RI }, { "rioff", 0x03, INSTR_RI_RI }, { { 0, LONG_INSN_RIEMIT }, 0x04, INSTR_RI_RI }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_b2[] = { -#ifdef CONFIG_64BIT { "stckf", 0x7c, INSTR_S_RD }, { "lpp", 0x80, INSTR_S_RD }, { "lcctl", 0x84, INSTR_S_RD }, @@ -819,7 +802,6 @@ static struct s390_insn opcode_b2[] = { { "tend", 0xf8, INSTR_S_00 }, { "niai", 0xfa, INSTR_IE_UU }, { { 0, LONG_INSN_TABORT }, 0xfc, INSTR_S_RD }, -#endif { "stidp", 0x02, INSTR_S_RD }, { "sck", 0x04, INSTR_S_RD }, { "stck", 0x05, INSTR_S_RD }, @@ -908,7 +890,6 @@ static struct s390_insn opcode_b2[] = { }; static struct s390_insn opcode_b3[] = { -#ifdef CONFIG_64BIT { "maylr", 0x38, INSTR_RRF_F0FF }, { "mylr", 0x39, INSTR_RRF_F0FF }, { "mayr", 0x3a, INSTR_RRF_F0FF }, @@ -996,7 +977,6 @@ static struct s390_insn opcode_b3[] = { { "qaxtr", 0xfd, INSTR_RRF_FUFF }, { "iextr", 0xfe, INSTR_RRF_F0FR }, { "rrxtr", 0xff, INSTR_RRF_FFRU }, -#endif { "lpebr", 0x00, INSTR_RRE_FF }, { "lnebr", 0x01, INSTR_RRE_FF }, { "ltebr", 0x02, INSTR_RRE_FF }, @@ -1091,7 +1071,6 @@ static struct s390_insn opcode_b3[] = { }; static struct s390_insn opcode_b9[] = { -#ifdef CONFIG_64BIT { "lpgr", 0x00, INSTR_RRE_RR }, { "lngr", 0x01, INSTR_RRE_RR }, { "ltgr", 0x02, INSTR_RRE_RR }, @@ -1204,7 +1183,6 @@ static struct s390_insn opcode_b9[] = { { "srk", 0xf9, INSTR_RRF_R0RR2 }, { "alrk", 0xfa, INSTR_RRF_R0RR2 }, { "slrk", 0xfb, INSTR_RRF_R0RR2 }, -#endif { "kmac", 0x1e, INSTR_RRE_RR }, { "lrvr", 0x1f, INSTR_RRE_RR }, { "km", 0x2e, INSTR_RRE_RR }, @@ -1224,7 +1202,6 @@ static struct s390_insn opcode_b9[] = { }; static struct s390_insn opcode_c0[] = { -#ifdef CONFIG_64BIT { "lgfi", 0x01, INSTR_RIL_RI }, { "xihf", 0x06, INSTR_RIL_RU }, { "xilf", 0x07, INSTR_RIL_RU }, @@ -1236,7 +1213,6 @@ static struct s390_insn opcode_c0[] = { { "oilf", 0x0d, INSTR_RIL_RU }, { "llihf", 0x0e, INSTR_RIL_RU }, { "llilf", 0x0f, INSTR_RIL_RU }, -#endif { "larl", 0x00, INSTR_RIL_RP }, { "brcl", 0x04, INSTR_RIL_UP }, { "brasl", 0x05, INSTR_RIL_RP }, @@ -1244,7 +1220,6 @@ static struct s390_insn opcode_c0[] = { }; static struct s390_insn opcode_c2[] = { -#ifdef CONFIG_64BIT { "msgfi", 0x00, INSTR_RIL_RI }, { "msfi", 0x01, INSTR_RIL_RI }, { "slgfi", 0x04, INSTR_RIL_RU }, @@ -1257,12 +1232,10 @@ static struct s390_insn opcode_c2[] = { { "cfi", 0x0d, INSTR_RIL_RI }, { "clgfi", 0x0e, INSTR_RIL_RU }, { "clfi", 0x0f, INSTR_RIL_RU }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_c4[] = { -#ifdef CONFIG_64BIT { "llhrl", 0x02, INSTR_RIL_RP }, { "lghrl", 0x04, INSTR_RIL_RP }, { "lhrl", 0x05, INSTR_RIL_RP }, @@ -1274,12 +1247,10 @@ static struct s390_insn opcode_c4[] = { { "lrl", 0x0d, INSTR_RIL_RP }, { { 0, LONG_INSN_LLGFRL }, 0x0e, INSTR_RIL_RP }, { "strl", 0x0f, INSTR_RIL_RP }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_c6[] = { -#ifdef CONFIG_64BIT { "exrl", 0x00, INSTR_RIL_RP }, { "pfdrl", 0x02, INSTR_RIL_UP }, { "cghrl", 0x04, INSTR_RIL_RP }, @@ -1292,35 +1263,29 @@ static struct s390_insn opcode_c6[] = { { "crl", 0x0d, INSTR_RIL_RP }, { { 0, LONG_INSN_CLGFRL }, 0x0e, INSTR_RIL_RP }, { "clrl", 0x0f, INSTR_RIL_RP }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_c8[] = { -#ifdef CONFIG_64BIT { "mvcos", 0x00, INSTR_SSF_RRDRD }, { "ectg", 0x01, INSTR_SSF_RRDRD }, { "csst", 0x02, INSTR_SSF_RRDRD }, { "lpd", 0x04, INSTR_SSF_RRDRD2 }, { "lpdg", 0x05, INSTR_SSF_RRDRD2 }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_cc[] = { -#ifdef CONFIG_64BIT { "brcth", 0x06, INSTR_RIL_RP }, { "aih", 0x08, INSTR_RIL_RI }, { "alsih", 0x0a, INSTR_RIL_RI }, { { 0, LONG_INSN_ALSIHN }, 0x0b, INSTR_RIL_RI }, { "cih", 0x0d, INSTR_RIL_RI }, { "clih", 0x0f, INSTR_RIL_RI }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_e3[] = { -#ifdef CONFIG_64BIT { "ltg", 0x02, INSTR_RXY_RRRD }, { "lrag", 0x03, INSTR_RXY_RRRD }, { "lg", 0x04, INSTR_RXY_RRRD }, @@ -1414,7 +1379,6 @@ static struct s390_insn opcode_e3[] = { { "clhf", 0xcf, INSTR_RXY_RRRD }, { { 0, LONG_INSN_MPCIFC }, 0xd0, INSTR_RXY_RRRD }, { { 0, LONG_INSN_STPCIFC }, 0xd4, INSTR_RXY_RRRD }, -#endif { "lrv", 0x1e, INSTR_RXY_RRRD }, { "lrvh", 0x1f, INSTR_RXY_RRRD }, { "strv", 0x3e, INSTR_RXY_RRRD }, @@ -1426,7 +1390,6 @@ static struct s390_insn opcode_e3[] = { }; static struct s390_insn opcode_e5[] = { -#ifdef CONFIG_64BIT { "strag", 0x02, INSTR_SSE_RDRD }, { "mvhhi", 0x44, INSTR_SIL_RDI }, { "mvghi", 0x48, INSTR_SIL_RDI }, @@ -1439,7 +1402,6 @@ static struct s390_insn opcode_e5[] = { { { 0, LONG_INSN_CLFHSI }, 0x5d, INSTR_SIL_RDU }, { { 0, LONG_INSN_TBEGIN }, 0x60, INSTR_SIL_RDU }, { { 0, LONG_INSN_TBEGINC }, 0x61, INSTR_SIL_RDU }, -#endif { "lasp", 0x00, INSTR_SSE_RDRD }, { "tprot", 0x01, INSTR_SSE_RDRD }, { "mvcsk", 0x0e, INSTR_SSE_RDRD }, @@ -1448,7 +1410,6 @@ static struct s390_insn opcode_e5[] = { }; static struct s390_insn opcode_e7[] = { -#ifdef CONFIG_64BIT { "lcbb", 0x27, INSTR_RXE_RRRDM }, { "vgef", 0x13, INSTR_VRV_VVRDM }, { "vgeg", 0x12, INSTR_VRV_VVRDM }, @@ -1588,11 +1549,9 @@ static struct s390_insn opcode_e7[] = { { "vfsq", 0xce, INSTR_VRR_VV000MM }, { "vfs", 0xe2, INSTR_VRR_VVV00MM }, { "vftci", 0x4a, INSTR_VRI_VVIMM }, -#endif }; static struct s390_insn opcode_eb[] = { -#ifdef CONFIG_64BIT { "lmg", 0x04, INSTR_RSY_RRRD }, { "srag", 0x0a, INSTR_RSY_RRRD }, { "slag", 0x0b, INSTR_RSY_RRRD }, @@ -1659,7 +1618,6 @@ static struct s390_insn opcode_eb[] = { { "stric", 0x61, INSTR_RSY_RDRM }, { "mric", 0x62, INSTR_RSY_RDRM }, { { 0, LONG_INSN_STCCTM }, 0x17, INSTR_RSY_RMRD }, -#endif { "rll", 0x1d, INSTR_RSY_RRRD }, { "mvclu", 0x8e, INSTR_RSY_RRRD }, { "tp", 0xc0, INSTR_RSL_R0RD }, @@ -1667,7 +1625,6 @@ static struct s390_insn opcode_eb[] = { }; static struct s390_insn opcode_ec[] = { -#ifdef CONFIG_64BIT { "brxhg", 0x44, INSTR_RIE_RRP }, { "brxlg", 0x45, INSTR_RIE_RRP }, { { 0, LONG_INSN_RISBLG }, 0x51, INSTR_RIE_RRUUU }, @@ -1701,12 +1658,10 @@ static struct s390_insn opcode_ec[] = { { "clgib", 0xfd, INSTR_RIS_RURDU }, { "cib", 0xfe, INSTR_RIS_RURDI }, { "clib", 0xff, INSTR_RIS_RURDU }, -#endif { "", 0, INSTR_INVALID } }; static struct s390_insn opcode_ed[] = { -#ifdef CONFIG_64BIT { "mayl", 0x38, INSTR_RXF_FRRDF }, { "myl", 0x39, INSTR_RXF_FRRDF }, { "may", 0x3a, INSTR_RXF_FRRDF }, @@ -1731,7 +1686,6 @@ static struct s390_insn opcode_ed[] = { { "czxt", 0xa9, INSTR_RSL_LRDFU }, { "cdzt", 0xaa, INSTR_RSL_LRDFU }, { "cxzt", 0xab, INSTR_RSL_LRDFU }, -#endif { "ldeb", 0x04, INSTR_RXE_FRRD }, { "lxdb", 0x05, INSTR_RXE_FRRD }, { "lxeb", 0x06, INSTR_RXE_FRRD }, @@ -2051,7 +2005,7 @@ void show_code(struct pt_regs *regs) else *ptr++ = ' '; addr = regs->psw.addr + start - 32; - ptr += sprintf(ptr, ONELONG, addr); + ptr += sprintf(ptr, "%016lx: ", addr); if (start + opsize >= end) break; for (i = 0; i < opsize; i++) diff --git a/arch/s390/kernel/dumpstack.c b/arch/s390/kernel/dumpstack.c index a99852e96a77..dc8e20473484 100644 --- a/arch/s390/kernel/dumpstack.c +++ b/arch/s390/kernel/dumpstack.c @@ -18,16 +18,6 @@ #include <asm/dis.h> #include <asm/ipl.h> -#ifndef CONFIG_64BIT -#define LONG "%08lx " -#define FOURLONG "%08lx %08lx %08lx %08lx\n" -static int kstack_depth_to_print = 12; -#else /* CONFIG_64BIT */ -#define LONG "%016lx " -#define FOURLONG "%016lx %016lx %016lx %016lx\n" -static int kstack_depth_to_print = 20; -#endif /* CONFIG_64BIT */ - /* * For show_trace we have tree different stack to consider: * - the panic stack which is used if the kernel stack has overflown @@ -115,12 +105,12 @@ void show_stack(struct task_struct *task, unsigned long *sp) else stack = sp; - for (i = 0; i < kstack_depth_to_print; i++) { + for (i = 0; i < 20; i++) { if (((addr_t) stack & (THREAD_SIZE-1)) == 0) break; if ((i * sizeof(long) % 32) == 0) printk("%s ", i == 0 ? "" : "\n"); - printk(LONG, *stack++); + printk("%016lx ", *stack++); } printk("\n"); show_trace(task, sp); @@ -128,10 +118,8 @@ void show_stack(struct task_struct *task, unsigned long *sp) static void show_last_breaking_event(struct pt_regs *regs) { -#ifdef CONFIG_64BIT printk("Last Breaking-Event-Address:\n"); printk(" [<%016lx>] %pSR\n", regs->args[0], (void *)regs->args[0]); -#endif } static inline int mask_bits(struct pt_regs *regs, unsigned long bits) @@ -155,16 +143,14 @@ void show_registers(struct pt_regs *regs) mask_bits(regs, PSW_MASK_MCHECK), mask_bits(regs, PSW_MASK_WAIT), mask_bits(regs, PSW_MASK_PSTATE), mask_bits(regs, PSW_MASK_ASC), mask_bits(regs, PSW_MASK_CC), mask_bits(regs, PSW_MASK_PM)); -#ifdef CONFIG_64BIT printk(" EA:%x", mask_bits(regs, PSW_MASK_EA | PSW_MASK_BA)); -#endif - printk("\n%s GPRS: " FOURLONG, mode, + printk("\n%s GPRS: %016lx %016lx %016lx %016lx\n", mode, regs->gprs[0], regs->gprs[1], regs->gprs[2], regs->gprs[3]); - printk(" " FOURLONG, + printk(" %016lx %016lx %016lx %016lx\n", regs->gprs[4], regs->gprs[5], regs->gprs[6], regs->gprs[7]); - printk(" " FOURLONG, + printk(" %016lx %016lx %016lx %016lx\n", regs->gprs[8], regs->gprs[9], regs->gprs[10], regs->gprs[11]); - printk(" " FOURLONG, + printk(" %016lx %016lx %016lx %016lx\n", regs->gprs[12], regs->gprs[13], regs->gprs[14], regs->gprs[15]); show_code(regs); } diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c index 4427ab7ac23a..549a73a4b543 100644 --- a/arch/s390/kernel/early.c +++ b/arch/s390/kernel/early.c @@ -64,7 +64,6 @@ asm( " .align 4\n" " .type savesys_ipl_nss, @function\n" "savesys_ipl_nss:\n" -#ifdef CONFIG_64BIT " stmg 6,15,48(15)\n" " lgr 14,3\n" " sam31\n" @@ -72,13 +71,6 @@ asm( " sam64\n" " lgr 2,14\n" " lmg 6,15,48(15)\n" -#else - " stm 6,15,24(15)\n" - " lr 14,3\n" - " diag 2,14,0x8\n" - " lr 2,14\n" - " lm 6,15,24(15)\n" -#endif " br 14\n" " .size savesys_ipl_nss, .-savesys_ipl_nss\n" " .previous\n"); @@ -240,7 +232,6 @@ static noinline __init void detect_machine_type(void) static __init void setup_topology(void) { -#ifdef CONFIG_64BIT int max_mnest; if (!test_facility(11)) @@ -251,7 +242,6 @@ static __init void setup_topology(void) break; } topology_max_mnest = max_mnest; -#endif } static void early_pgm_check_handler(void) @@ -290,58 +280,6 @@ static noinline __init void setup_facility_list(void) ARRAY_SIZE(S390_lowcore.stfle_fac_list)); } -static __init void detect_mvpg(void) -{ -#ifndef CONFIG_64BIT - int rc; - - asm volatile( - " la 0,0\n" - " mvpg %2,%2\n" - "0: la %0,0\n" - "1:\n" - EX_TABLE(0b,1b) - : "=d" (rc) : "0" (-EOPNOTSUPP), "a" (0) : "memory", "cc", "0"); - if (!rc) - S390_lowcore.machine_flags |= MACHINE_FLAG_MVPG; -#endif -} - -static __init void detect_ieee(void) -{ -#ifndef CONFIG_64BIT - int rc, tmp; - - asm volatile( - " efpc %1,0\n" - "0: la %0,0\n" - "1:\n" - EX_TABLE(0b,1b) - : "=d" (rc), "=d" (tmp): "0" (-EOPNOTSUPP) : "cc"); - if (!rc) - S390_lowcore.machine_flags |= MACHINE_FLAG_IEEE; -#endif -} - -static __init void detect_csp(void) -{ -#ifndef CONFIG_64BIT - int rc; - - asm volatile( - " la 0,0\n" - " la 1,0\n" - " la 2,4\n" - " csp 0,2\n" - "0: la %0,0\n" - "1:\n" - EX_TABLE(0b,1b) - : "=d" (rc) : "0" (-EOPNOTSUPP) : "cc", "0", "1", "2"); - if (!rc) - S390_lowcore.machine_flags |= MACHINE_FLAG_CSP; -#endif -} - static __init void detect_diag9c(void) { unsigned int cpu_address; @@ -360,7 +298,6 @@ static __init void detect_diag9c(void) static __init void detect_diag44(void) { -#ifdef CONFIG_64BIT int rc; asm volatile( @@ -371,12 +308,10 @@ static __init void detect_diag44(void) : "=d" (rc) : "0" (-EOPNOTSUPP) : "cc"); if (!rc) S390_lowcore.machine_flags |= MACHINE_FLAG_DIAG44; -#endif } static __init void detect_machine_facilities(void) { -#ifdef CONFIG_64BIT if (test_facility(8)) { S390_lowcore.machine_flags |= MACHINE_FLAG_EDAT1; __ctl_set_bit(0, 23); @@ -393,7 +328,6 @@ static __init void detect_machine_facilities(void) S390_lowcore.machine_flags |= MACHINE_FLAG_TLB_LC; if (test_facility(129)) S390_lowcore.machine_flags |= MACHINE_FLAG_VX; -#endif } static int __init cad_setup(char *str) @@ -501,9 +435,6 @@ void __init startup_init(void) ipl_update_parameters(); setup_boot_command_line(); create_kernel_nss(); - detect_mvpg(); - detect_ieee(); - detect_csp(); detect_diag9c(); detect_diag44(); detect_machine_facilities(); diff --git a/arch/s390/kernel/entry.S b/arch/s390/kernel/entry.S index 398329b2b518..99b44acbfcc7 100644 --- a/arch/s390/kernel/entry.S +++ b/arch/s390/kernel/entry.S @@ -22,27 +22,28 @@ #include <asm/irq.h> __PT_R0 = __PT_GPRS -__PT_R1 = __PT_GPRS + 4 -__PT_R2 = __PT_GPRS + 8 -__PT_R3 = __PT_GPRS + 12 -__PT_R4 = __PT_GPRS + 16 -__PT_R5 = __PT_GPRS + 20 -__PT_R6 = __PT_GPRS + 24 -__PT_R7 = __PT_GPRS + 28 -__PT_R8 = __PT_GPRS + 32 -__PT_R9 = __PT_GPRS + 36 -__PT_R10 = __PT_GPRS + 40 -__PT_R11 = __PT_GPRS + 44 -__PT_R12 = __PT_GPRS + 48 -__PT_R13 = __PT_GPRS + 524 -__PT_R14 = __PT_GPRS + 56 -__PT_R15 = __PT_GPRS + 60 +__PT_R1 = __PT_GPRS + 8 +__PT_R2 = __PT_GPRS + 16 +__PT_R3 = __PT_GPRS + 24 +__PT_R4 = __PT_GPRS + 32 +__PT_R5 = __PT_GPRS + 40 +__PT_R6 = __PT_GPRS + 48 +__PT_R7 = __PT_GPRS + 56 +__PT_R8 = __PT_GPRS + 64 +__PT_R9 = __PT_GPRS + 72 +__PT_R10 = __PT_GPRS + 80 +__PT_R11 = __PT_GPRS + 88 +__PT_R12 = __PT_GPRS + 96 +__PT_R13 = __PT_GPRS + 104 +__PT_R14 = __PT_GPRS + 112 +__PT_R15 = __PT_GPRS + 120 STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER STACK_SIZE = 1 << STACK_SHIFT -STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE +STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE -_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED) +_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ + _TIF_UPROBE) _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ _TIF_SYSCALL_TRACEPOINT) _CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE) @@ -53,16 +54,14 @@ _PIF_WORK = (_PIF_PER_TRAP) .macro TRACE_IRQS_ON #ifdef CONFIG_TRACE_IRQFLAGS basr %r2,%r0 - l %r1,BASED(.Lc_hardirqs_on) - basr %r14,%r1 # call trace_hardirqs_on_caller + brasl %r14,trace_hardirqs_on_caller #endif .endm .macro TRACE_IRQS_OFF #ifdef CONFIG_TRACE_IRQFLAGS basr %r2,%r0 - l %r1,BASED(.Lc_hardirqs_off) - basr %r14,%r1 # call trace_hardirqs_off_caller + brasl %r14,trace_hardirqs_off_caller #endif .endm @@ -70,73 +69,104 @@ _PIF_WORK = (_PIF_PER_TRAP) #ifdef CONFIG_LOCKDEP tm __PT_PSW+1(%r11),0x01 # returning to user ? jz .+10 - l %r1,BASED(.Lc_lockdep_sys_exit) - basr %r14,%r1 # call lockdep_sys_exit + brasl %r14,lockdep_sys_exit +#endif + .endm + + .macro LPP newpp +#if IS_ENABLED(CONFIG_KVM) + tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_LPP + jz .+8 + .insn s,0xb2800000,\newpp +#endif + .endm + + .macro HANDLE_SIE_INTERCEPT scratch,reason +#if IS_ENABLED(CONFIG_KVM) + tmhh %r8,0x0001 # interrupting from user ? + jnz .+62 + lgr \scratch,%r9 + slg \scratch,BASED(.Lsie_critical) + clg \scratch,BASED(.Lsie_critical_length) + .if \reason==1 + # Some program interrupts are suppressing (e.g. protection). + # We must also check the instruction after SIE in that case. + # do_protection_exception will rewind to .Lrewind_pad + jh .+42 + .else + jhe .+42 + .endif + lg %r14,__SF_EMPTY(%r15) # get control block pointer + LPP __SF_EMPTY+16(%r15) # set host id + ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE + lctlg %c1,%c1,__LC_USER_ASCE # load primary asce + larl %r9,sie_exit # skip forward to sie_exit + mvi __SF_EMPTY+31(%r15),\reason # set exit reason #endif .endm .macro CHECK_STACK stacksize,savearea #ifdef CONFIG_CHECK_STACK tml %r15,\stacksize - CONFIG_STACK_GUARD - la %r14,\savearea + lghi %r14,\savearea jz stack_overflow #endif .endm .macro SWITCH_ASYNC savearea,stack,shift - tmh %r8,0x0001 # interrupting from user ? + tmhh %r8,0x0001 # interrupting from user ? jnz 1f - lr %r14,%r9 - sl %r14,BASED(.Lc_critical_start) - cl %r14,BASED(.Lc_critical_length) + lgr %r14,%r9 + slg %r14,BASED(.Lcritical_start) + clg %r14,BASED(.Lcritical_length) jhe 0f - la %r11,\savearea # inside critical section, do cleanup - bras %r14,cleanup_critical - tmh %r8,0x0001 # retest problem state after cleanup + lghi %r11,\savearea # inside critical section, do cleanup + brasl %r14,cleanup_critical + tmhh %r8,0x0001 # retest problem state after cleanup jnz 1f -0: l %r14,\stack # are we already on the target stack? - slr %r14,%r15 - sra %r14,\shift +0: lg %r14,\stack # are we already on the target stack? + slgr %r14,%r15 + srag %r14,%r14,\shift jnz 1f CHECK_STACK 1<<\shift,\savearea - ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) + aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j 2f -1: l %r15,\stack # load target stack +1: lg %r15,\stack # load target stack 2: la %r11,STACK_FRAME_OVERHEAD(%r15) .endm - .macro ADD64 high,low,timer - al \high,\timer - al \low,4+\timer - brc 12,.+8 - ahi \high,1 - .endm - - .macro SUB64 high,low,timer - sl \high,\timer - sl \low,4+\timer - brc 3,.+8 - ahi \high,-1 + .macro UPDATE_VTIME scratch,enter_timer + lg \scratch,__LC_EXIT_TIMER + slg \scratch,\enter_timer + alg \scratch,__LC_USER_TIMER + stg \scratch,__LC_USER_TIMER + lg \scratch,__LC_LAST_UPDATE_TIMER + slg \scratch,__LC_EXIT_TIMER + alg \scratch,__LC_SYSTEM_TIMER + stg \scratch,__LC_SYSTEM_TIMER + mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer .endm - .macro UPDATE_VTIME high,low,enter_timer - lm \high,\low,__LC_EXIT_TIMER - SUB64 \high,\low,\enter_timer - ADD64 \high,\low,__LC_USER_TIMER - stm \high,\low,__LC_USER_TIMER - lm \high,\low,__LC_LAST_UPDATE_TIMER - SUB64 \high,\low,__LC_EXIT_TIMER - ADD64 \high,\low,__LC_SYSTEM_TIMER - stm \high,\low,__LC_SYSTEM_TIMER - mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer + .macro LAST_BREAK scratch + srag \scratch,%r10,23 + jz .+10 + stg %r10,__TI_last_break(%r12) .endm .macro REENABLE_IRQS - st %r8,__LC_RETURN_PSW + stg %r8,__LC_RETURN_PSW ni __LC_RETURN_PSW,0xbf ssm __LC_RETURN_PSW .endm + .macro STCK savearea +#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES + .insn s,0xb27c0000,\savearea # store clock fast +#else + .insn s,0xb2050000,\savearea # store clock +#endif + .endm + .section .kprobes.text, "ax" /* @@ -147,19 +177,19 @@ _PIF_WORK = (_PIF_PER_TRAP) * gpr2 = prev */ ENTRY(__switch_to) - stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task - st %r15,__THREAD_ksp(%r2) # store kernel stack of prev - l %r4,__THREAD_info(%r2) # get thread_info of prev - l %r5,__THREAD_info(%r3) # get thread_info of next - lr %r15,%r5 - ahi %r15,STACK_INIT # end of kernel stack of next - st %r3,__LC_CURRENT # store task struct of next - st %r5,__LC_THREAD_INFO # store thread info of next - st %r15,__LC_KERNEL_STACK # store end of kernel stack + stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task + stg %r15,__THREAD_ksp(%r2) # store kernel stack of prev + lg %r4,__THREAD_info(%r2) # get thread_info of prev + lg %r5,__THREAD_info(%r3) # get thread_info of next + lgr %r15,%r5 + aghi %r15,STACK_INIT # end of kernel stack of next + stg %r3,__LC_CURRENT # store task struct of next + stg %r5,__LC_THREAD_INFO # store thread info of next + stg %r15,__LC_KERNEL_STACK # store end of kernel stack lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 - mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next - l %r15,__THREAD_ksp(%r3) # load kernel stack of next - lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task + mvc __LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next + lg %r15,__THREAD_ksp(%r3) # load kernel stack of next + lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task br %r14 .L__critical_start: @@ -170,75 +200,83 @@ ENTRY(__switch_to) ENTRY(system_call) stpt __LC_SYNC_ENTER_TIMER -.Lsysc_stm: - stm %r8,%r15,__LC_SAVE_AREA_SYNC - l %r12,__LC_THREAD_INFO - l %r13,__LC_SVC_NEW_PSW+4 - lhi %r14,_PIF_SYSCALL +.Lsysc_stmg: + stmg %r8,%r15,__LC_SAVE_AREA_SYNC + lg %r10,__LC_LAST_BREAK + lg %r12,__LC_THREAD_INFO + lghi %r14,_PIF_SYSCALL .Lsysc_per: - l %r15,__LC_KERNEL_STACK + lg %r15,__LC_KERNEL_STACK la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs .Lsysc_vtime: - UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER - stm %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC - mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW + UPDATE_VTIME %r13,__LC_SYNC_ENTER_TIMER + LAST_BREAK %r13 + stmg %r0,%r7,__PT_R0(%r11) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC + mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC - st %r14,__PT_FLAGS(%r11) + stg %r14,__PT_FLAGS(%r11) .Lsysc_do_svc: - l %r10,__TI_sysc_table(%r12) # 31 bit system call table - lh %r8,__PT_INT_CODE+2(%r11) - sla %r8,2 # shift and test for svc0 + lg %r10,__TI_sysc_table(%r12) # address of system call table + llgh %r8,__PT_INT_CODE+2(%r11) + slag %r8,%r8,2 # shift and test for svc 0 jnz .Lsysc_nr_ok # svc 0: system call number in %r1 - cl %r1,BASED(.Lnr_syscalls) + llgfr %r1,%r1 # clear high word in r1 + cghi %r1,NR_syscalls jnl .Lsysc_nr_ok sth %r1,__PT_INT_CODE+2(%r11) - lr %r8,%r1 - sla %r8,2 + slag %r8,%r1,2 .Lsysc_nr_ok: - xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) - st %r2,__PT_ORIG_GPR2(%r11) - st %r7,STACK_FRAME_OVERHEAD(%r15) - l %r9,0(%r8,%r10) # get system call addr. - tm __TI_flags+3(%r12),_TIF_TRACE + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + stg %r2,__PT_ORIG_GPR2(%r11) + stg %r7,STACK_FRAME_OVERHEAD(%r15) + lgf %r9,0(%r8,%r10) # get system call add. + tm __TI_flags+7(%r12),_TIF_TRACE jnz .Lsysc_tracesys basr %r14,%r9 # call sys_xxxx - st %r2,__PT_R2(%r11) # store return value + stg %r2,__PT_R2(%r11) # store return value .Lsysc_return: LOCKDEP_SYS_EXIT .Lsysc_tif: tm __PT_PSW+1(%r11),0x01 # returning to user ? jno .Lsysc_restore - tm __PT_FLAGS+3(%r11),_PIF_WORK + tm __PT_FLAGS+7(%r11),_PIF_WORK jnz .Lsysc_work - tm __TI_flags+3(%r12),_TIF_WORK - jnz .Lsysc_work # check for thread work - tm __LC_CPU_FLAGS+3,_CIF_WORK + tm __TI_flags+7(%r12),_TIF_WORK + jnz .Lsysc_work # check for work + tm __LC_CPU_FLAGS+7,_CIF_WORK jnz .Lsysc_work .Lsysc_restore: - mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) + lg %r14,__LC_VDSO_PER_CPU + lmg %r0,%r10,__PT_R0(%r11) + mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) stpt __LC_EXIT_TIMER - lm %r0,%r15,__PT_R0(%r11) - lpsw __LC_RETURN_PSW + mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER + lmg %r11,%r15,__PT_R11(%r11) + lpswe __LC_RETURN_PSW .Lsysc_done: # # One of the work bits is on. Find out which one. # .Lsysc_work: - tm __LC_CPU_FLAGS+3,_CIF_MCCK_PENDING + tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING jo .Lsysc_mcck_pending - tm __TI_flags+3(%r12),_TIF_NEED_RESCHED + tm __TI_flags+7(%r12),_TIF_NEED_RESCHED jo .Lsysc_reschedule - tm __PT_FLAGS+3(%r11),_PIF_PER_TRAP +#ifdef CONFIG_UPROBES + tm __TI_flags+7(%r12),_TIF_UPROBE + jo .Lsysc_uprobe_notify +#endif + tm __PT_FLAGS+7(%r11),_PIF_PER_TRAP jo .Lsysc_singlestep - tm __TI_flags+3(%r12),_TIF_SIGPENDING + tm __TI_flags+7(%r12),_TIF_SIGPENDING jo .Lsysc_sigpending - tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME + tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME jo .Lsysc_notify_resume - tm __LC_CPU_FLAGS+3,_CIF_ASCE + tm __LC_CPU_FLAGS+7,_CIF_ASCE jo .Lsysc_uaccess j .Lsysc_return # beware of critical section cleanup @@ -246,109 +284,109 @@ ENTRY(system_call) # _TIF_NEED_RESCHED is set, call schedule # .Lsysc_reschedule: - l %r1,BASED(.Lc_schedule) - la %r14,BASED(.Lsysc_return) - br %r1 # call schedule + larl %r14,.Lsysc_return + jg schedule # # _CIF_MCCK_PENDING is set, call handler # .Lsysc_mcck_pending: - l %r1,BASED(.Lc_handle_mcck) - la %r14,BASED(.Lsysc_return) - br %r1 # TIF bit will be cleared by handler + larl %r14,.Lsysc_return + jg s390_handle_mcck # TIF bit will be cleared by handler # # _CIF_ASCE is set, load user space asce # .Lsysc_uaccess: - ni __LC_CPU_FLAGS+3,255-_CIF_ASCE - lctl %c1,%c1,__LC_USER_ASCE # load primary asce + ni __LC_CPU_FLAGS+7,255-_CIF_ASCE + lctlg %c1,%c1,__LC_USER_ASCE # load primary asce j .Lsysc_return # # _TIF_SIGPENDING is set, call do_signal # .Lsysc_sigpending: - lr %r2,%r11 # pass pointer to pt_regs - l %r1,BASED(.Lc_do_signal) - basr %r14,%r1 # call do_signal - tm __PT_FLAGS+3(%r11),_PIF_SYSCALL + lgr %r2,%r11 # pass pointer to pt_regs + brasl %r14,do_signal + tm __PT_FLAGS+7(%r11),_PIF_SYSCALL jno .Lsysc_return - lm %r2,%r7,__PT_R2(%r11) # load svc arguments - l %r10,__TI_sysc_table(%r12) # 31 bit system call table - xr %r8,%r8 # svc 0 returns -ENOSYS - clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2) + lmg %r2,%r7,__PT_R2(%r11) # load svc arguments + lg %r10,__TI_sysc_table(%r12) # address of system call table + lghi %r8,0 # svc 0 returns -ENOSYS + llgh %r1,__PT_INT_CODE+2(%r11) # load new svc number + cghi %r1,NR_syscalls jnl .Lsysc_nr_ok # invalid svc number -> do svc 0 - lh %r8,__PT_INT_CODE+2(%r11) # load new svc number - sla %r8,2 + slag %r8,%r1,2 j .Lsysc_nr_ok # restart svc # # _TIF_NOTIFY_RESUME is set, call do_notify_resume # .Lsysc_notify_resume: - lr %r2,%r11 # pass pointer to pt_regs - l %r1,BASED(.Lc_do_notify_resume) - la %r14,BASED(.Lsysc_return) - br %r1 # call do_notify_resume + lgr %r2,%r11 # pass pointer to pt_regs + larl %r14,.Lsysc_return + jg do_notify_resume + +# +# _TIF_UPROBE is set, call uprobe_notify_resume +# +#ifdef CONFIG_UPROBES +.Lsysc_uprobe_notify: + lgr %r2,%r11 # pass pointer to pt_regs + larl %r14,.Lsysc_return + jg uprobe_notify_resume +#endif # # _PIF_PER_TRAP is set, call do_per_trap # .Lsysc_singlestep: - ni __PT_FLAGS+3(%r11),255-_PIF_PER_TRAP - lr %r2,%r11 # pass pointer to pt_regs - l %r1,BASED(.Lc_do_per_trap) - la %r14,BASED(.Lsysc_return) - br %r1 # call do_per_trap + ni __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP + lgr %r2,%r11 # pass pointer to pt_regs + larl %r14,.Lsysc_return + jg do_per_trap # # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before # and after the system call # .Lsysc_tracesys: - l %r1,BASED(.Lc_trace_enter) - lr %r2,%r11 # pass pointer to pt_regs + lgr %r2,%r11 # pass pointer to pt_regs la %r3,0 - xr %r0,%r0 - icm %r0,3,__PT_INT_CODE+2(%r11) - st %r0,__PT_R2(%r11) - basr %r14,%r1 # call do_syscall_trace_enter - cl %r2,BASED(.Lnr_syscalls) - jnl .Lsysc_tracenogo - lr %r8,%r2 - sll %r8,2 - l %r9,0(%r8,%r10) + llgh %r0,__PT_INT_CODE+2(%r11) + stg %r0,__PT_R2(%r11) + brasl %r14,do_syscall_trace_enter + lghi %r0,NR_syscalls + clgr %r0,%r2 + jnh .Lsysc_tracenogo + sllg %r8,%r2,2 + lgf %r9,0(%r8,%r10) .Lsysc_tracego: - lm %r3,%r7,__PT_R3(%r11) - st %r7,STACK_FRAME_OVERHEAD(%r15) - l %r2,__PT_ORIG_GPR2(%r11) + lmg %r3,%r7,__PT_R3(%r11) + stg %r7,STACK_FRAME_OVERHEAD(%r15) + lg %r2,__PT_ORIG_GPR2(%r11) basr %r14,%r9 # call sys_xxx - st %r2,__PT_R2(%r11) # store return value + stg %r2,__PT_R2(%r11) # store return value .Lsysc_tracenogo: - tm __TI_flags+3(%r12),_TIF_TRACE + tm __TI_flags+7(%r12),_TIF_TRACE jz .Lsysc_return - l %r1,BASED(.Lc_trace_exit) - lr %r2,%r11 # pass pointer to pt_regs - la %r14,BASED(.Lsysc_return) - br %r1 # call do_syscall_trace_exit + lgr %r2,%r11 # pass pointer to pt_regs + larl %r14,.Lsysc_return + jg do_syscall_trace_exit # # a new process exits the kernel with ret_from_fork # ENTRY(ret_from_fork) la %r11,STACK_FRAME_OVERHEAD(%r15) - l %r12,__LC_THREAD_INFO - l %r13,__LC_SVC_NEW_PSW+4 - l %r1,BASED(.Lc_schedule_tail) - basr %r14,%r1 # call schedule_tail + lg %r12,__LC_THREAD_INFO + brasl %r14,schedule_tail TRACE_IRQS_ON ssm __LC_SVC_NEW_PSW # reenable interrupts tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? jne .Lsysc_tracenogo # it's a kernel thread - lm %r9,%r10,__PT_R9(%r11) # load gprs + lmg %r9,%r10,__PT_R9(%r11) # load gprs ENTRY(kernel_thread_starter) la %r2,0(%r10) basr %r14,%r9 @@ -360,46 +398,54 @@ ENTRY(kernel_thread_starter) ENTRY(pgm_check_handler) stpt __LC_SYNC_ENTER_TIMER - stm %r8,%r15,__LC_SAVE_AREA_SYNC - l %r12,__LC_THREAD_INFO - l %r13,__LC_SVC_NEW_PSW+4 - lm %r8,%r9,__LC_PGM_OLD_PSW - tmh %r8,0x0001 # test problem state bit + stmg %r8,%r15,__LC_SAVE_AREA_SYNC + lg %r10,__LC_LAST_BREAK + lg %r12,__LC_THREAD_INFO + larl %r13,system_call + lmg %r8,%r9,__LC_PGM_OLD_PSW + HANDLE_SIE_INTERCEPT %r14,1 + tmhh %r8,0x0001 # test problem state bit jnz 1f # -> fault in user space - tmh %r8,0x4000 # PER bit set in old PSW ? + tmhh %r8,0x4000 # PER bit set in old PSW ? jnz 0f # -> enabled, can't be a double fault tm __LC_PGM_ILC+3,0x80 # check for per exception jnz .Lpgm_svcper # -> single stepped svc 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC - ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) + aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j 2f -1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER - l %r15,__LC_KERNEL_STACK +1: UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER + LAST_BREAK %r14 + lg %r15,__LC_KERNEL_STACK + lg %r14,__TI_task(%r12) + lghi %r13,__LC_PGM_TDB + tm __LC_PGM_ILC+2,0x02 # check for transaction abort + jz 2f + mvc __THREAD_trap_tdb(256,%r14),0(%r13) 2: la %r11,STACK_FRAME_OVERHEAD(%r15) - stm %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC - stm %r8,%r9,__PT_PSW(%r11) + stmg %r0,%r7,__PT_R0(%r11) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC + stmg %r8,%r9,__PT_PSW(%r11) mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC - mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE - xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) + mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) + stg %r10,__PT_ARGS(%r11) tm __LC_PGM_ILC+3,0x80 # check for per exception jz 0f - l %r1,__TI_task(%r12) - tmh %r8,0x0001 # kernel per event ? + tmhh %r8,0x0001 # kernel per event ? jz .Lpgm_kprobe - oi __PT_FLAGS+3(%r11),_PIF_PER_TRAP - mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS - mvc __THREAD_per_cause(2,%r1),__LC_PER_CODE - mvc __THREAD_per_paid(1,%r1),__LC_PER_ACCESS_ID + oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP + mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS + mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE + mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID 0: REENABLE_IRQS - xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) - l %r1,BASED(.Lc_jump_table) - la %r10,0x7f - n %r10,__PT_INT_CODE(%r11) - je .Lsysc_return + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + larl %r1,pgm_check_table + llgh %r10,__PT_INT_CODE+2(%r11) + nill %r10,0x007f sll %r10,2 - l %r1,0(%r10,%r1) # load address of handler routine - lr %r2,%r11 # pass pointer to pt_regs + je .Lsysc_return + lgf %r1,0(%r10,%r1) # load address of handler routine + lgr %r2,%r11 # pass pointer to pt_regs basr %r14,%r1 # branch to interrupt-handler j .Lsysc_return @@ -408,54 +454,55 @@ ENTRY(pgm_check_handler) # .Lpgm_kprobe: REENABLE_IRQS - xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) - l %r1,BASED(.Lc_do_per_trap) - lr %r2,%r11 # pass pointer to pt_regs - basr %r14,%r1 # call do_per_trap + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + lgr %r2,%r11 # pass pointer to pt_regs + brasl %r14,do_per_trap j .Lsysc_return # # single stepped system call # .Lpgm_svcper: - mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW - mvc __LC_RETURN_PSW+4(4),BASED(.Lc_sysc_per) - lhi %r14,_PIF_SYSCALL | _PIF_PER_TRAP - lpsw __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs + mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW + larl %r14,.Lsysc_per + stg %r14,__LC_RETURN_PSW+8 + lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP + lpswe __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs /* * IO interrupt handler routine */ - ENTRY(io_int_handler) - stck __LC_INT_CLOCK + STCK __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER - stm %r8,%r15,__LC_SAVE_AREA_ASYNC - l %r12,__LC_THREAD_INFO - l %r13,__LC_SVC_NEW_PSW+4 - lm %r8,%r9,__LC_IO_OLD_PSW - tmh %r8,0x0001 # interrupting from user ? + stmg %r8,%r15,__LC_SAVE_AREA_ASYNC + lg %r10,__LC_LAST_BREAK + lg %r12,__LC_THREAD_INFO + larl %r13,system_call + lmg %r8,%r9,__LC_IO_OLD_PSW + HANDLE_SIE_INTERCEPT %r14,2 + SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT + tmhh %r8,0x0001 # interrupting from user? jz .Lio_skip - UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER + UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER + LAST_BREAK %r14 .Lio_skip: - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT - stm %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC - stm %r8,%r9,__PT_PSW(%r11) + stmg %r0,%r7,__PT_R0(%r11) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC + stmg %r8,%r9,__PT_PSW(%r11) mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID - xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) TRACE_IRQS_OFF - xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) .Lio_loop: - l %r1,BASED(.Lc_do_IRQ) - lr %r2,%r11 # pass pointer to pt_regs - lhi %r3,IO_INTERRUPT + lgr %r2,%r11 # pass pointer to pt_regs + lghi %r3,IO_INTERRUPT tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? jz .Lio_call - lhi %r3,THIN_INTERRUPT + lghi %r3,THIN_INTERRUPT .Lio_call: - basr %r14,%r1 # call do_IRQ - tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR + brasl %r14,do_IRQ + tm __LC_MACHINE_FLAGS+6,0x10 # MACHINE_FLAG_LPAR jz .Lio_return tpi 0 jz .Lio_return @@ -465,21 +512,26 @@ ENTRY(io_int_handler) LOCKDEP_SYS_EXIT TRACE_IRQS_ON .Lio_tif: - tm __TI_flags+3(%r12),_TIF_WORK + tm __TI_flags+7(%r12),_TIF_WORK jnz .Lio_work # there is work to do (signals etc.) - tm __LC_CPU_FLAGS+3,_CIF_WORK + tm __LC_CPU_FLAGS+7,_CIF_WORK jnz .Lio_work .Lio_restore: - mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) + lg %r14,__LC_VDSO_PER_CPU + lmg %r0,%r10,__PT_R0(%r11) + mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) stpt __LC_EXIT_TIMER - lm %r0,%r15,__PT_R0(%r11) - lpsw __LC_RETURN_PSW + mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER + lmg %r11,%r15,__PT_R11(%r11) + lpswe __LC_RETURN_PSW .Lio_done: # # There is work todo, find out in which context we have been interrupted: # 1) if we return to user space we can do all _TIF_WORK work -# 2) if we return to kernel code and preemptive scheduling is enabled check +# 2) if we return to kernel code and kvm is enabled check if we need to +# modify the psw to leave SIE +# 3) if we return to kernel code and preemptive scheduling is enabled check # the preemption counter and if it is zero call preempt_schedule_irq # Before any work can be done, a switch to the kernel stack is required. # @@ -489,21 +541,20 @@ ENTRY(io_int_handler) #ifdef CONFIG_PREEMPT # check for preemptive scheduling icm %r0,15,__TI_precount(%r12) - jnz .Lio_restore # preemption disabled - tm __TI_flags+3(%r12),_TIF_NEED_RESCHED + jnz .Lio_restore # preemption is disabled + tm __TI_flags+7(%r12),_TIF_NEED_RESCHED jno .Lio_restore # switch to kernel stack - l %r1,__PT_R15(%r11) - ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) + lg %r1,__PT_R15(%r11) + aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) - xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) + xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) la %r11,STACK_FRAME_OVERHEAD(%r1) - lr %r15,%r1 + lgr %r15,%r1 # TRACE_IRQS_ON already done at .Lio_return, call # TRACE_IRQS_OFF to keep things symmetrical TRACE_IRQS_OFF - l %r1,BASED(.Lc_preempt_irq) - basr %r14,%r1 # call preempt_schedule_irq + brasl %r14,preempt_schedule_irq j .Lio_return #else j .Lio_restore @@ -513,25 +564,25 @@ ENTRY(io_int_handler) # Need to do work before returning to userspace, switch to kernel stack # .Lio_work_user: - l %r1,__LC_KERNEL_STACK + lg %r1,__LC_KERNEL_STACK mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) - xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) + xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) la %r11,STACK_FRAME_OVERHEAD(%r1) - lr %r15,%r1 + lgr %r15,%r1 # # One of the work bits is on. Find out which one. # .Lio_work_tif: - tm __LC_CPU_FLAGS+3(%r12),_CIF_MCCK_PENDING + tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING jo .Lio_mcck_pending - tm __TI_flags+3(%r12),_TIF_NEED_RESCHED + tm __TI_flags+7(%r12),_TIF_NEED_RESCHED jo .Lio_reschedule - tm __TI_flags+3(%r12),_TIF_SIGPENDING + tm __TI_flags+7(%r12),_TIF_SIGPENDING jo .Lio_sigpending - tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME + tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME jo .Lio_notify_resume - tm __LC_CPU_FLAGS+3,_CIF_ASCE + tm __LC_CPU_FLAGS+7,_CIF_ASCE jo .Lio_uaccess j .Lio_return # beware of critical section cleanup @@ -540,8 +591,7 @@ ENTRY(io_int_handler) # .Lio_mcck_pending: # TRACE_IRQS_ON already done at .Lio_return - l %r1,BASED(.Lc_handle_mcck) - basr %r14,%r1 # TIF bit will be cleared by handler + brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler TRACE_IRQS_OFF j .Lio_return @@ -549,8 +599,8 @@ ENTRY(io_int_handler) # _CIF_ASCE is set, load user space asce # .Lio_uaccess: - ni __LC_CPU_FLAGS+3,255-_CIF_ASCE - lctl %c1,%c1,__LC_USER_ASCE # load primary asce + ni __LC_CPU_FLAGS+7,255-_CIF_ASCE + lctlg %c1,%c1,__LC_USER_ASCE # load primary asce j .Lio_return # @@ -558,35 +608,32 @@ ENTRY(io_int_handler) # .Lio_reschedule: # TRACE_IRQS_ON already done at .Lio_return - l %r1,BASED(.Lc_schedule) ssm __LC_SVC_NEW_PSW # reenable interrupts - basr %r14,%r1 # call scheduler + brasl %r14,schedule # call scheduler ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts TRACE_IRQS_OFF j .Lio_return # -# _TIF_SIGPENDING is set, call do_signal +# _TIF_SIGPENDING or is set, call do_signal # .Lio_sigpending: # TRACE_IRQS_ON already done at .Lio_return - l %r1,BASED(.Lc_do_signal) ssm __LC_SVC_NEW_PSW # reenable interrupts - lr %r2,%r11 # pass pointer to pt_regs - basr %r14,%r1 # call do_signal + lgr %r2,%r11 # pass pointer to pt_regs + brasl %r14,do_signal ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts TRACE_IRQS_OFF j .Lio_return # -# _TIF_SIGPENDING is set, call do_signal +# _TIF_NOTIFY_RESUME or is set, call do_notify_resume # .Lio_notify_resume: # TRACE_IRQS_ON already done at .Lio_return - l %r1,BASED(.Lc_do_notify_resume) ssm __LC_SVC_NEW_PSW # reenable interrupts - lr %r2,%r11 # pass pointer to pt_regs - basr %r14,%r1 # call do_notify_resume + lgr %r2,%r11 # pass pointer to pt_regs + brasl %r14,do_notify_resume ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts TRACE_IRQS_OFF j .Lio_return @@ -594,45 +641,47 @@ ENTRY(io_int_handler) /* * External interrupt handler routine */ - ENTRY(ext_int_handler) - stck __LC_INT_CLOCK + STCK __LC_INT_CLOCK stpt __LC_ASYNC_ENTER_TIMER - stm %r8,%r15,__LC_SAVE_AREA_ASYNC - l %r12,__LC_THREAD_INFO - l %r13,__LC_SVC_NEW_PSW+4 - lm %r8,%r9,__LC_EXT_OLD_PSW - tmh %r8,0x0001 # interrupting from user ? + stmg %r8,%r15,__LC_SAVE_AREA_ASYNC + lg %r10,__LC_LAST_BREAK + lg %r12,__LC_THREAD_INFO + larl %r13,system_call + lmg %r8,%r9,__LC_EXT_OLD_PSW + HANDLE_SIE_INTERCEPT %r14,3 + SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT + tmhh %r8,0x0001 # interrupting from user ? jz .Lext_skip - UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER + UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER + LAST_BREAK %r14 .Lext_skip: - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT - stm %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC - stm %r8,%r9,__PT_PSW(%r11) + stmg %r0,%r7,__PT_R0(%r11) + mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC + stmg %r8,%r9,__PT_PSW(%r11) + lghi %r1,__LC_EXT_PARAMS2 mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS - xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) + mvc __PT_INT_PARM_LONG(8,%r11),0(%r1) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) TRACE_IRQS_OFF - l %r1,BASED(.Lc_do_IRQ) - lr %r2,%r11 # pass pointer to pt_regs - lhi %r3,EXT_INTERRUPT - basr %r14,%r1 # call do_IRQ + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + lgr %r2,%r11 # pass pointer to pt_regs + lghi %r3,EXT_INTERRUPT + brasl %r14,do_IRQ j .Lio_return /* * Load idle PSW. The second "half" of this function is in .Lcleanup_idle. */ ENTRY(psw_idle) - st %r3,__SF_EMPTY(%r15) - basr %r1,0 - la %r1,.Lpsw_idle_lpsw+4-.(%r1) - st %r1,__SF_EMPTY+4(%r15) - oi __SF_EMPTY+4(%r15),0x80 - stck __CLOCK_IDLE_ENTER(%r2) + stg %r3,__SF_EMPTY(%r15) + larl %r1,.Lpsw_idle_lpsw+4 + stg %r1,__SF_EMPTY+8(%r15) + STCK __CLOCK_IDLE_ENTER(%r2) stpt __TIMER_IDLE_ENTER(%r2) .Lpsw_idle_lpsw: - lpsw __SF_EMPTY(%r15) + lpswe __SF_EMPTY(%r15) br %r14 .Lpsw_idle_end: @@ -641,17 +690,19 @@ ENTRY(psw_idle) /* * Machine check handler routines */ - ENTRY(mcck_int_handler) - stck __LC_MCCK_CLOCK - spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer - lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs - l %r12,__LC_THREAD_INFO - l %r13,__LC_SVC_NEW_PSW+4 - lm %r8,%r9,__LC_MCK_OLD_PSW + STCK __LC_MCCK_CLOCK + la %r1,4095 # revalidate r1 + spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer + lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs + lg %r10,__LC_LAST_BREAK + lg %r12,__LC_THREAD_INFO + larl %r13,system_call + lmg %r8,%r9,__LC_MCK_OLD_PSW + HANDLE_SIE_INTERCEPT %r14,4 tm __LC_MCCK_CODE,0x80 # system damage? jo .Lmcck_panic # yes -> rest of mcck code invalid - la %r14,__LC_CPU_TIMER_SAVE_AREA + lghi %r14,__LC_CPU_TIMER_SAVE_AREA mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid? jo 3f @@ -669,76 +720,76 @@ ENTRY(mcck_int_handler) mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid? jno .Lmcck_panic # no -> skip cleanup critical + SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_PANIC_STACK,PAGE_SHIFT tm %r8,0x0001 # interrupting from user ? jz .Lmcck_skip - UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER + UPDATE_VTIME %r14,__LC_MCCK_ENTER_TIMER + LAST_BREAK %r14 .Lmcck_skip: - SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT - stm %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32 - stm %r8,%r9,__PT_PSW(%r11) - xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) - xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) - l %r1,BASED(.Lc_do_machine_check) - lr %r2,%r11 # pass pointer to pt_regs - basr %r14,%r1 # call s390_do_machine_check + lghi %r14,__LC_GPREGS_SAVE_AREA+64 + stmg %r0,%r7,__PT_R0(%r11) + mvc __PT_R8(64,%r11),0(%r14) + stmg %r8,%r9,__PT_PSW(%r11) + xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + lgr %r2,%r11 # pass pointer to pt_regs + brasl %r14,s390_do_machine_check tm __PT_PSW+1(%r11),0x01 # returning to user ? jno .Lmcck_return - l %r1,__LC_KERNEL_STACK # switch to kernel stack + lg %r1,__LC_KERNEL_STACK # switch to kernel stack mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) - xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) - la %r11,STACK_FRAME_OVERHEAD(%r15) - lr %r15,%r1 + xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) + la %r11,STACK_FRAME_OVERHEAD(%r1) + lgr %r15,%r1 ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off - tm __LC_CPU_FLAGS+3,_CIF_MCCK_PENDING + tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING jno .Lmcck_return TRACE_IRQS_OFF - l %r1,BASED(.Lc_handle_mcck) - basr %r14,%r1 # call s390_handle_mcck + brasl %r14,s390_handle_mcck TRACE_IRQS_ON .Lmcck_return: - mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW + lg %r14,__LC_VDSO_PER_CPU + lmg %r0,%r10,__PT_R0(%r11) + mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? jno 0f - lm %r0,%r15,__PT_R0(%r11) stpt __LC_EXIT_TIMER - lpsw __LC_RETURN_MCCK_PSW -0: lm %r0,%r15,__PT_R0(%r11) - lpsw __LC_RETURN_MCCK_PSW + mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER +0: lmg %r11,%r15,__PT_R11(%r11) + lpswe __LC_RETURN_MCCK_PSW .Lmcck_panic: - l %r14,__LC_PANIC_STACK - slr %r14,%r15 - sra %r14,PAGE_SHIFT + lg %r14,__LC_PANIC_STACK + slgr %r14,%r15 + srag %r14,%r14,PAGE_SHIFT jz 0f - l %r15,__LC_PANIC_STACK - j .Lmcck_skip -0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) + lg %r15,__LC_PANIC_STACK +0: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) j .Lmcck_skip # # PSW restart interrupt handler # ENTRY(restart_int_handler) - st %r15,__LC_SAVE_AREA_RESTART - l %r15,__LC_RESTART_STACK - ahi %r15,-__PT_SIZE # create pt_regs on stack + stg %r15,__LC_SAVE_AREA_RESTART + lg %r15,__LC_RESTART_STACK + aghi %r15,-__PT_SIZE # create pt_regs on stack xc 0(__PT_SIZE,%r15),0(%r15) - stm %r0,%r14,__PT_R0(%r15) - mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART - mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw - ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack + stmg %r0,%r14,__PT_R0(%r15) + mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART + mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw + aghi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) - l %r1,__LC_RESTART_FN # load fn, parm & source cpu - l %r2,__LC_RESTART_DATA - l %r3,__LC_RESTART_SOURCE - ltr %r3,%r3 # test source cpu address + lg %r1,__LC_RESTART_FN # load fn, parm & source cpu + lg %r2,__LC_RESTART_DATA + lg %r3,__LC_RESTART_SOURCE + ltgr %r3,%r3 # test source cpu address jm 1f # negative -> skip source stop 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu brc 10,0b # wait for status stored 1: basr %r14,%r1 # call function stap __SF_EMPTY(%r15) # store cpu address - lh %r3,__SF_EMPTY(%r15) + llgh %r3,__SF_EMPTY(%r15) 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu brc 2,2b 3: j 3b @@ -752,215 +803,257 @@ ENTRY(restart_int_handler) * Setup a pt_regs so that show_trace can provide a good call trace. */ stack_overflow: - l %r15,__LC_PANIC_STACK # change to panic stack + lg %r15,__LC_PANIC_STACK # change to panic stack la %r11,STACK_FRAME_OVERHEAD(%r15) - stm %r0,%r7,__PT_R0(%r11) - stm %r8,%r9,__PT_PSW(%r11) - mvc __PT_R8(32,%r11),0(%r14) - l %r1,BASED(1f) - xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) - lr %r2,%r11 # pass pointer to pt_regs - br %r1 # branch to kernel_stack_overflow -1: .long kernel_stack_overflow + stmg %r0,%r7,__PT_R0(%r11) + stmg %r8,%r9,__PT_PSW(%r11) + mvc __PT_R8(64,%r11),0(%r14) + stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 + xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) + lgr %r2,%r11 # pass pointer to pt_regs + jg kernel_stack_overflow #endif + .align 8 .Lcleanup_table: - .long system_call + 0x80000000 - .long .Lsysc_do_svc + 0x80000000 - .long .Lsysc_tif + 0x80000000 - .long .Lsysc_restore + 0x80000000 - .long .Lsysc_done + 0x80000000 - .long .Lio_tif + 0x80000000 - .long .Lio_restore + 0x80000000 - .long .Lio_done + 0x80000000 - .long psw_idle + 0x80000000 - .long .Lpsw_idle_end + 0x80000000 + .quad system_call + .quad .Lsysc_do_svc + .quad .Lsysc_tif + .quad .Lsysc_restore + .quad .Lsysc_done + .quad .Lio_tif + .quad .Lio_restore + .quad .Lio_done + .quad psw_idle + .quad .Lpsw_idle_end cleanup_critical: - cl %r9,BASED(.Lcleanup_table) # system_call + clg %r9,BASED(.Lcleanup_table) # system_call jl 0f - cl %r9,BASED(.Lcleanup_table+4) # .Lsysc_do_svc + clg %r9,BASED(.Lcleanup_table+8) # .Lsysc_do_svc jl .Lcleanup_system_call - cl %r9,BASED(.Lcleanup_table+8) # .Lsysc_tif + clg %r9,BASED(.Lcleanup_table+16) # .Lsysc_tif jl 0f - cl %r9,BASED(.Lcleanup_table+12) # .Lsysc_restore + clg %r9,BASED(.Lcleanup_table+24) # .Lsysc_restore jl .Lcleanup_sysc_tif - cl %r9,BASED(.Lcleanup_table+16) # .Lsysc_done + clg %r9,BASED(.Lcleanup_table+32) # .Lsysc_done jl .Lcleanup_sysc_restore - cl %r9,BASED(.Lcleanup_table+20) # .Lio_tif + clg %r9,BASED(.Lcleanup_table+40) # .Lio_tif jl 0f - cl %r9,BASED(.Lcleanup_table+24) # .Lio_restore + clg %r9,BASED(.Lcleanup_table+48) # .Lio_restore jl .Lcleanup_io_tif - cl %r9,BASED(.Lcleanup_table+28) # .Lio_done + clg %r9,BASED(.Lcleanup_table+56) # .Lio_done jl .Lcleanup_io_restore - cl %r9,BASED(.Lcleanup_table+32) # psw_idle + clg %r9,BASED(.Lcleanup_table+64) # psw_idle jl 0f - cl %r9,BASED(.Lcleanup_table+36) # .Lpsw_idle_end + clg %r9,BASED(.Lcleanup_table+72) # .Lpsw_idle_end jl .Lcleanup_idle 0: br %r14 + .Lcleanup_system_call: # check if stpt has been executed - cl %r9,BASED(.Lcleanup_system_call_insn) + clg %r9,BASED(.Lcleanup_system_call_insn) jh 0f mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER - chi %r11,__LC_SAVE_AREA_ASYNC + cghi %r11,__LC_SAVE_AREA_ASYNC je 0f mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER -0: # check if stm has been executed - cl %r9,BASED(.Lcleanup_system_call_insn+4) +0: # check if stmg has been executed + clg %r9,BASED(.Lcleanup_system_call_insn+8) jh 0f - mvc __LC_SAVE_AREA_SYNC(32),0(%r11) -0: # set up saved registers r12, and r13 - st %r12,16(%r11) # r12 thread-info pointer - st %r13,20(%r11) # r13 literal-pool pointer - # check if the user time calculation has been done - cl %r9,BASED(.Lcleanup_system_call_insn+8) + mvc __LC_SAVE_AREA_SYNC(64),0(%r11) +0: # check if base register setup + TIF bit load has been done + clg %r9,BASED(.Lcleanup_system_call_insn+16) + jhe 0f + # set up saved registers r10 and r12 + stg %r10,16(%r11) # r10 last break + stg %r12,32(%r11) # r12 thread-info pointer +0: # check if the user time update has been done + clg %r9,BASED(.Lcleanup_system_call_insn+24) jh 0f - l %r10,__LC_EXIT_TIMER - l %r15,__LC_EXIT_TIMER+4 - SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER - ADD64 %r10,%r15,__LC_USER_TIMER - st %r10,__LC_USER_TIMER - st %r15,__LC_USER_TIMER+4 -0: # check if the system time calculation has been done - cl %r9,BASED(.Lcleanup_system_call_insn+12) + lg %r15,__LC_EXIT_TIMER + slg %r15,__LC_SYNC_ENTER_TIMER + alg %r15,__LC_USER_TIMER + stg %r15,__LC_USER_TIMER +0: # check if the system time update has been done + clg %r9,BASED(.Lcleanup_system_call_insn+32) jh 0f - l %r10,__LC_LAST_UPDATE_TIMER - l %r15,__LC_LAST_UPDATE_TIMER+4 - SUB64 %r10,%r15,__LC_EXIT_TIMER - ADD64 %r10,%r15,__LC_SYSTEM_TIMER - st %r10,__LC_SYSTEM_TIMER - st %r15,__LC_SYSTEM_TIMER+4 + lg %r15,__LC_LAST_UPDATE_TIMER + slg %r15,__LC_EXIT_TIMER + alg %r15,__LC_SYSTEM_TIMER + stg %r15,__LC_SYSTEM_TIMER 0: # update accounting time stamp mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER - # set up saved register 11 - l %r15,__LC_KERNEL_STACK + # do LAST_BREAK + lg %r9,16(%r11) + srag %r9,%r9,23 + jz 0f + mvc __TI_last_break(8,%r12),16(%r11) +0: # set up saved register r11 + lg %r15,__LC_KERNEL_STACK la %r9,STACK_FRAME_OVERHEAD(%r15) - st %r9,12(%r11) # r11 pt_regs pointer + stg %r9,24(%r11) # r11 pt_regs pointer # fill pt_regs - mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC - stm %r0,%r7,__PT_R0(%r9) - mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW + mvc __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC + stmg %r0,%r7,__PT_R0(%r9) + mvc __PT_PSW(16,%r9),__LC_SVC_OLD_PSW mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC - xc __PT_FLAGS(4,%r9),__PT_FLAGS(%r9) - mvi __PT_FLAGS+3(%r9),_PIF_SYSCALL - # setup saved register 15 - st %r15,28(%r11) # r15 stack pointer + xc __PT_FLAGS(8,%r9),__PT_FLAGS(%r9) + mvi __PT_FLAGS+7(%r9),_PIF_SYSCALL + # setup saved register r15 + stg %r15,56(%r11) # r15 stack pointer # set new psw address and exit - l %r9,BASED(.Lcleanup_table+4) # .Lsysc_do_svc + 0x80000000 + larl %r9,.Lsysc_do_svc br %r14 .Lcleanup_system_call_insn: - .long system_call + 0x80000000 - .long .Lsysc_stm + 0x80000000 - .long .Lsysc_vtime + 0x80000000 + 36 - .long .Lsysc_vtime + 0x80000000 + 76 + .quad system_call + .quad .Lsysc_stmg + .quad .Lsysc_per + .quad .Lsysc_vtime+18 + .quad .Lsysc_vtime+42 .Lcleanup_sysc_tif: - l %r9,BASED(.Lcleanup_table+8) # .Lsysc_tif + 0x80000000 + larl %r9,.Lsysc_tif br %r14 .Lcleanup_sysc_restore: - cl %r9,BASED(.Lcleanup_sysc_restore_insn) - jhe 0f - l %r9,12(%r11) # get saved pointer to pt_regs - mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) - mvc 0(32,%r11),__PT_R8(%r9) - lm %r0,%r7,__PT_R0(%r9) -0: lm %r8,%r9,__LC_RETURN_PSW + clg %r9,BASED(.Lcleanup_sysc_restore_insn) + je 0f + lg %r9,24(%r11) # get saved pointer to pt_regs + mvc __LC_RETURN_PSW(16),__PT_PSW(%r9) + mvc 0(64,%r11),__PT_R8(%r9) + lmg %r0,%r7,__PT_R0(%r9) +0: lmg %r8,%r9,__LC_RETURN_PSW br %r14 .Lcleanup_sysc_restore_insn: - .long .Lsysc_done - 4 + 0x80000000 + .quad .Lsysc_done - 4 .Lcleanup_io_tif: - l %r9,BASED(.Lcleanup_table+20) # .Lio_tif + 0x80000000 + larl %r9,.Lio_tif br %r14 .Lcleanup_io_restore: - cl %r9,BASED(.Lcleanup_io_restore_insn) - jhe 0f - l %r9,12(%r11) # get saved r11 pointer to pt_regs - mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) - mvc 0(32,%r11),__PT_R8(%r9) - lm %r0,%r7,__PT_R0(%r9) -0: lm %r8,%r9,__LC_RETURN_PSW + clg %r9,BASED(.Lcleanup_io_restore_insn) + je 0f + lg %r9,24(%r11) # get saved r11 pointer to pt_regs + mvc __LC_RETURN_PSW(16),__PT_PSW(%r9) + mvc 0(64,%r11),__PT_R8(%r9) + lmg %r0,%r7,__PT_R0(%r9) +0: lmg %r8,%r9,__LC_RETURN_PSW br %r14 .Lcleanup_io_restore_insn: - .long .Lio_done - 4 + 0x80000000 + .quad .Lio_done - 4 .Lcleanup_idle: # copy interrupt clock & cpu timer mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER - chi %r11,__LC_SAVE_AREA_ASYNC + cghi %r11,__LC_SAVE_AREA_ASYNC je 0f mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER -0: # check if stck has been executed - cl %r9,BASED(.Lcleanup_idle_insn) +0: # check if stck & stpt have been executed + clg %r9,BASED(.Lcleanup_idle_insn) jhe 1f mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2) - mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3) + mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2) 1: # account system time going idle - lm %r9,%r10,__LC_STEAL_TIMER - ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2) - SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK - stm %r9,%r10,__LC_STEAL_TIMER + lg %r9,__LC_STEAL_TIMER + alg %r9,__CLOCK_IDLE_ENTER(%r2) + slg %r9,__LC_LAST_UPDATE_CLOCK + stg %r9,__LC_STEAL_TIMER mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2) - lm %r9,%r10,__LC_SYSTEM_TIMER - ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER - SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2) - stm %r9,%r10,__LC_SYSTEM_TIMER + lg %r9,__LC_SYSTEM_TIMER + alg %r9,__LC_LAST_UPDATE_TIMER + slg %r9,__TIMER_IDLE_ENTER(%r2) + stg %r9,__LC_SYSTEM_TIMER mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) # prepare return psw - n %r8,BASED(.Lcleanup_idle_wait) # clear irq & wait state bits - l %r9,24(%r11) # return from psw_idle + nihh %r8,0xfcfd # clear irq & wait state bits + lg %r9,48(%r11) # return from psw_idle br %r14 .Lcleanup_idle_insn: - .long .Lpsw_idle_lpsw + 0x80000000 -.Lcleanup_idle_wait: - .long 0xfcfdffff + .quad .Lpsw_idle_lpsw /* * Integer constants */ - .align 4 -.Lnr_syscalls: - .long NR_syscalls -.Lvtimer_max: - .quad 0x7fffffffffffffff + .align 8 +.Lcritical_start: + .quad .L__critical_start +.Lcritical_length: + .quad .L__critical_end - .L__critical_start + +#if IS_ENABLED(CONFIG_KVM) /* - * Symbol constants + * sie64a calling convention: + * %r2 pointer to sie control block + * %r3 guest register save area */ -.Lc_do_machine_check: .long s390_do_machine_check -.Lc_handle_mcck: .long s390_handle_mcck -.Lc_do_IRQ: .long do_IRQ -.Lc_do_signal: .long do_signal -.Lc_do_notify_resume: .long do_notify_resume -.Lc_do_per_trap: .long do_per_trap -.Lc_jump_table: .long pgm_check_table -.Lc_schedule: .long schedule -#ifdef CONFIG_PREEMPT -.Lc_preempt_irq: .long preempt_schedule_irq -#endif -.Lc_trace_enter: .long do_syscall_trace_enter -.Lc_trace_exit: .long do_syscall_trace_exit -.Lc_schedule_tail: .long schedule_tail -.Lc_sysc_per: .long .Lsysc_per + 0x80000000 -#ifdef CONFIG_TRACE_IRQFLAGS -.Lc_hardirqs_on: .long trace_hardirqs_on_caller -.Lc_hardirqs_off: .long trace_hardirqs_off_caller -#endif -#ifdef CONFIG_LOCKDEP -.Lc_lockdep_sys_exit: .long lockdep_sys_exit +ENTRY(sie64a) + stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers + stg %r2,__SF_EMPTY(%r15) # save control block pointer + stg %r3,__SF_EMPTY+8(%r15) # save guest register save area + xc __SF_EMPTY+16(16,%r15),__SF_EMPTY+16(%r15) # host id & reason + lmg %r0,%r13,0(%r3) # load guest gprs 0-13 + lg %r14,__LC_GMAP # get gmap pointer + ltgr %r14,%r14 + jz .Lsie_gmap + lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce +.Lsie_gmap: + lg %r14,__SF_EMPTY(%r15) # get control block pointer + oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now + tm __SIE_PROG20+3(%r14),1 # last exit... + jnz .Lsie_done + LPP __SF_EMPTY(%r15) # set guest id + sie 0(%r14) +.Lsie_done: + LPP __SF_EMPTY+16(%r15) # set host id + ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE + lctlg %c1,%c1,__LC_USER_ASCE # load primary asce +# some program checks are suppressing. C code (e.g. do_protection_exception) +# will rewind the PSW by the ILC, which is 4 bytes in case of SIE. Other +# instructions between sie64a and .Lsie_done should not cause program +# interrupts. So lets use a nop (47 00 00 00) as a landing pad. +# See also HANDLE_SIE_INTERCEPT +.Lrewind_pad: + nop 0 + .globl sie_exit +sie_exit: + lg %r14,__SF_EMPTY+8(%r15) # load guest register save area + stmg %r0,%r13,0(%r14) # save guest gprs 0-13 + lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers + lg %r2,__SF_EMPTY+24(%r15) # return exit reason code + br %r14 +.Lsie_fault: + lghi %r14,-EFAULT + stg %r14,__SF_EMPTY+24(%r15) # set exit reason code + j sie_exit + + .align 8 +.Lsie_critical: + .quad .Lsie_gmap +.Lsie_critical_length: + .quad .Lsie_done - .Lsie_gmap + + EX_TABLE(.Lrewind_pad,.Lsie_fault) + EX_TABLE(sie_exit,.Lsie_fault) #endif -.Lc_critical_start: .long .L__critical_start + 0x80000000 -.Lc_critical_length: .long .L__critical_end - .L__critical_start - .section .rodata, "a" -#define SYSCALL(esa,esame,emu) .long esa + .section .rodata, "a" +#define SYSCALL(esame,emu) .long esame .globl sys_call_table sys_call_table: #include "syscalls.S" #undef SYSCALL + +#ifdef CONFIG_COMPAT + +#define SYSCALL(esame,emu) .long emu + .globl sys_call_table_emu +sys_call_table_emu: +#include "syscalls.S" +#undef SYSCALL +#endif diff --git a/arch/s390/kernel/entry64.S b/arch/s390/kernel/entry64.S deleted file mode 100644 index c329446a951d..000000000000 --- a/arch/s390/kernel/entry64.S +++ /dev/null @@ -1,1059 +0,0 @@ -/* - * S390 low-level entry points. - * - * Copyright IBM Corp. 1999, 2012 - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), - * Hartmut Penner (hp@de.ibm.com), - * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), - * Heiko Carstens <heiko.carstens@de.ibm.com> - */ - -#include <linux/init.h> -#include <linux/linkage.h> -#include <asm/processor.h> -#include <asm/cache.h> -#include <asm/errno.h> -#include <asm/ptrace.h> -#include <asm/thread_info.h> -#include <asm/asm-offsets.h> -#include <asm/unistd.h> -#include <asm/page.h> -#include <asm/sigp.h> -#include <asm/irq.h> - -__PT_R0 = __PT_GPRS -__PT_R1 = __PT_GPRS + 8 -__PT_R2 = __PT_GPRS + 16 -__PT_R3 = __PT_GPRS + 24 -__PT_R4 = __PT_GPRS + 32 -__PT_R5 = __PT_GPRS + 40 -__PT_R6 = __PT_GPRS + 48 -__PT_R7 = __PT_GPRS + 56 -__PT_R8 = __PT_GPRS + 64 -__PT_R9 = __PT_GPRS + 72 -__PT_R10 = __PT_GPRS + 80 -__PT_R11 = __PT_GPRS + 88 -__PT_R12 = __PT_GPRS + 96 -__PT_R13 = __PT_GPRS + 104 -__PT_R14 = __PT_GPRS + 112 -__PT_R15 = __PT_GPRS + 120 - -STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER -STACK_SIZE = 1 << STACK_SHIFT -STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE - -_TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \ - _TIF_UPROBE) -_TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ - _TIF_SYSCALL_TRACEPOINT) -_CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE) -_PIF_WORK = (_PIF_PER_TRAP) - -#define BASED(name) name-system_call(%r13) - - .macro TRACE_IRQS_ON -#ifdef CONFIG_TRACE_IRQFLAGS - basr %r2,%r0 - brasl %r14,trace_hardirqs_on_caller -#endif - .endm - - .macro TRACE_IRQS_OFF -#ifdef CONFIG_TRACE_IRQFLAGS - basr %r2,%r0 - brasl %r14,trace_hardirqs_off_caller -#endif - .endm - - .macro LOCKDEP_SYS_EXIT -#ifdef CONFIG_LOCKDEP - tm __PT_PSW+1(%r11),0x01 # returning to user ? - jz .+10 - brasl %r14,lockdep_sys_exit -#endif - .endm - - .macro LPP newpp -#if IS_ENABLED(CONFIG_KVM) - tm __LC_MACHINE_FLAGS+6,0x20 # MACHINE_FLAG_LPP - jz .+8 - .insn s,0xb2800000,\newpp -#endif - .endm - - .macro HANDLE_SIE_INTERCEPT scratch,reason -#if IS_ENABLED(CONFIG_KVM) - tmhh %r8,0x0001 # interrupting from user ? - jnz .+62 - lgr \scratch,%r9 - slg \scratch,BASED(.Lsie_critical) - clg \scratch,BASED(.Lsie_critical_length) - .if \reason==1 - # Some program interrupts are suppressing (e.g. protection). - # We must also check the instruction after SIE in that case. - # do_protection_exception will rewind to .Lrewind_pad - jh .+42 - .else - jhe .+42 - .endif - lg %r14,__SF_EMPTY(%r15) # get control block pointer - LPP __SF_EMPTY+16(%r15) # set host id - ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - larl %r9,sie_exit # skip forward to sie_exit - mvi __SF_EMPTY+31(%r15),\reason # set exit reason -#endif - .endm - - .macro CHECK_STACK stacksize,savearea -#ifdef CONFIG_CHECK_STACK - tml %r15,\stacksize - CONFIG_STACK_GUARD - lghi %r14,\savearea - jz stack_overflow -#endif - .endm - - .macro SWITCH_ASYNC savearea,stack,shift - tmhh %r8,0x0001 # interrupting from user ? - jnz 1f - lgr %r14,%r9 - slg %r14,BASED(.Lcritical_start) - clg %r14,BASED(.Lcritical_length) - jhe 0f - lghi %r11,\savearea # inside critical section, do cleanup - brasl %r14,cleanup_critical - tmhh %r8,0x0001 # retest problem state after cleanup - jnz 1f -0: lg %r14,\stack # are we already on the target stack? - slgr %r14,%r15 - srag %r14,%r14,\shift - jnz 1f - CHECK_STACK 1<<\shift,\savearea - aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) - j 2f -1: lg %r15,\stack # load target stack -2: la %r11,STACK_FRAME_OVERHEAD(%r15) - .endm - - .macro UPDATE_VTIME scratch,enter_timer - lg \scratch,__LC_EXIT_TIMER - slg \scratch,\enter_timer - alg \scratch,__LC_USER_TIMER - stg \scratch,__LC_USER_TIMER - lg \scratch,__LC_LAST_UPDATE_TIMER - slg \scratch,__LC_EXIT_TIMER - alg \scratch,__LC_SYSTEM_TIMER - stg \scratch,__LC_SYSTEM_TIMER - mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer - .endm - - .macro LAST_BREAK scratch - srag \scratch,%r10,23 - jz .+10 - stg %r10,__TI_last_break(%r12) - .endm - - .macro REENABLE_IRQS - stg %r8,__LC_RETURN_PSW - ni __LC_RETURN_PSW,0xbf - ssm __LC_RETURN_PSW - .endm - - .macro STCK savearea -#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES - .insn s,0xb27c0000,\savearea # store clock fast -#else - .insn s,0xb2050000,\savearea # store clock -#endif - .endm - - .section .kprobes.text, "ax" - -/* - * Scheduler resume function, called by switch_to - * gpr2 = (task_struct *) prev - * gpr3 = (task_struct *) next - * Returns: - * gpr2 = prev - */ -ENTRY(__switch_to) - stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task - stg %r15,__THREAD_ksp(%r2) # store kernel stack of prev - lg %r4,__THREAD_info(%r2) # get thread_info of prev - lg %r5,__THREAD_info(%r3) # get thread_info of next - lgr %r15,%r5 - aghi %r15,STACK_INIT # end of kernel stack of next - stg %r3,__LC_CURRENT # store task struct of next - stg %r5,__LC_THREAD_INFO # store thread info of next - stg %r15,__LC_KERNEL_STACK # store end of kernel stack - lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 - mvc __LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next - lg %r15,__THREAD_ksp(%r3) # load kernel stack of next - lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task - br %r14 - -.L__critical_start: -/* - * SVC interrupt handler routine. System calls are synchronous events and - * are executed with interrupts enabled. - */ - -ENTRY(system_call) - stpt __LC_SYNC_ENTER_TIMER -.Lsysc_stmg: - stmg %r8,%r15,__LC_SAVE_AREA_SYNC - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO - lghi %r14,_PIF_SYSCALL -.Lsysc_per: - lg %r15,__LC_KERNEL_STACK - la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs -.Lsysc_vtime: - UPDATE_VTIME %r13,__LC_SYNC_ENTER_TIMER - LAST_BREAK %r13 - stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC - mvc __PT_PSW(16,%r11),__LC_SVC_OLD_PSW - mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC - stg %r14,__PT_FLAGS(%r11) -.Lsysc_do_svc: - lg %r10,__TI_sysc_table(%r12) # address of system call table - llgh %r8,__PT_INT_CODE+2(%r11) - slag %r8,%r8,2 # shift and test for svc 0 - jnz .Lsysc_nr_ok - # svc 0: system call number in %r1 - llgfr %r1,%r1 # clear high word in r1 - cghi %r1,NR_syscalls - jnl .Lsysc_nr_ok - sth %r1,__PT_INT_CODE+2(%r11) - slag %r8,%r1,2 -.Lsysc_nr_ok: - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - stg %r2,__PT_ORIG_GPR2(%r11) - stg %r7,STACK_FRAME_OVERHEAD(%r15) - lgf %r9,0(%r8,%r10) # get system call add. - tm __TI_flags+7(%r12),_TIF_TRACE - jnz .Lsysc_tracesys - basr %r14,%r9 # call sys_xxxx - stg %r2,__PT_R2(%r11) # store return value - -.Lsysc_return: - LOCKDEP_SYS_EXIT -.Lsysc_tif: - tm __PT_PSW+1(%r11),0x01 # returning to user ? - jno .Lsysc_restore - tm __PT_FLAGS+7(%r11),_PIF_WORK - jnz .Lsysc_work - tm __TI_flags+7(%r12),_TIF_WORK - jnz .Lsysc_work # check for work - tm __LC_CPU_FLAGS+7,_CIF_WORK - jnz .Lsysc_work -.Lsysc_restore: - lg %r14,__LC_VDSO_PER_CPU - lmg %r0,%r10,__PT_R0(%r11) - mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) - stpt __LC_EXIT_TIMER - mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER - lmg %r11,%r15,__PT_R11(%r11) - lpswe __LC_RETURN_PSW -.Lsysc_done: - -# -# One of the work bits is on. Find out which one. -# -.Lsysc_work: - tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING - jo .Lsysc_mcck_pending - tm __TI_flags+7(%r12),_TIF_NEED_RESCHED - jo .Lsysc_reschedule -#ifdef CONFIG_UPROBES - tm __TI_flags+7(%r12),_TIF_UPROBE - jo .Lsysc_uprobe_notify -#endif - tm __PT_FLAGS+7(%r11),_PIF_PER_TRAP - jo .Lsysc_singlestep - tm __TI_flags+7(%r12),_TIF_SIGPENDING - jo .Lsysc_sigpending - tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME - jo .Lsysc_notify_resume - tm __LC_CPU_FLAGS+7,_CIF_ASCE - jo .Lsysc_uaccess - j .Lsysc_return # beware of critical section cleanup - -# -# _TIF_NEED_RESCHED is set, call schedule -# -.Lsysc_reschedule: - larl %r14,.Lsysc_return - jg schedule - -# -# _CIF_MCCK_PENDING is set, call handler -# -.Lsysc_mcck_pending: - larl %r14,.Lsysc_return - jg s390_handle_mcck # TIF bit will be cleared by handler - -# -# _CIF_ASCE is set, load user space asce -# -.Lsysc_uaccess: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lsysc_return - -# -# _TIF_SIGPENDING is set, call do_signal -# -.Lsysc_sigpending: - lgr %r2,%r11 # pass pointer to pt_regs - brasl %r14,do_signal - tm __PT_FLAGS+7(%r11),_PIF_SYSCALL - jno .Lsysc_return - lmg %r2,%r7,__PT_R2(%r11) # load svc arguments - lg %r10,__TI_sysc_table(%r12) # address of system call table - lghi %r8,0 # svc 0 returns -ENOSYS - llgh %r1,__PT_INT_CODE+2(%r11) # load new svc number - cghi %r1,NR_syscalls - jnl .Lsysc_nr_ok # invalid svc number -> do svc 0 - slag %r8,%r1,2 - j .Lsysc_nr_ok # restart svc - -# -# _TIF_NOTIFY_RESUME is set, call do_notify_resume -# -.Lsysc_notify_resume: - lgr %r2,%r11 # pass pointer to pt_regs - larl %r14,.Lsysc_return - jg do_notify_resume - -# -# _TIF_UPROBE is set, call uprobe_notify_resume -# -#ifdef CONFIG_UPROBES -.Lsysc_uprobe_notify: - lgr %r2,%r11 # pass pointer to pt_regs - larl %r14,.Lsysc_return - jg uprobe_notify_resume -#endif - -# -# _PIF_PER_TRAP is set, call do_per_trap -# -.Lsysc_singlestep: - ni __PT_FLAGS+7(%r11),255-_PIF_PER_TRAP - lgr %r2,%r11 # pass pointer to pt_regs - larl %r14,.Lsysc_return - jg do_per_trap - -# -# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before -# and after the system call -# -.Lsysc_tracesys: - lgr %r2,%r11 # pass pointer to pt_regs - la %r3,0 - llgh %r0,__PT_INT_CODE+2(%r11) - stg %r0,__PT_R2(%r11) - brasl %r14,do_syscall_trace_enter - lghi %r0,NR_syscalls - clgr %r0,%r2 - jnh .Lsysc_tracenogo - sllg %r8,%r2,2 - lgf %r9,0(%r8,%r10) -.Lsysc_tracego: - lmg %r3,%r7,__PT_R3(%r11) - stg %r7,STACK_FRAME_OVERHEAD(%r15) - lg %r2,__PT_ORIG_GPR2(%r11) - basr %r14,%r9 # call sys_xxx - stg %r2,__PT_R2(%r11) # store return value -.Lsysc_tracenogo: - tm __TI_flags+7(%r12),_TIF_TRACE - jz .Lsysc_return - lgr %r2,%r11 # pass pointer to pt_regs - larl %r14,.Lsysc_return - jg do_syscall_trace_exit - -# -# a new process exits the kernel with ret_from_fork -# -ENTRY(ret_from_fork) - la %r11,STACK_FRAME_OVERHEAD(%r15) - lg %r12,__LC_THREAD_INFO - brasl %r14,schedule_tail - TRACE_IRQS_ON - ssm __LC_SVC_NEW_PSW # reenable interrupts - tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? - jne .Lsysc_tracenogo - # it's a kernel thread - lmg %r9,%r10,__PT_R9(%r11) # load gprs -ENTRY(kernel_thread_starter) - la %r2,0(%r10) - basr %r14,%r9 - j .Lsysc_tracenogo - -/* - * Program check handler routine - */ - -ENTRY(pgm_check_handler) - stpt __LC_SYNC_ENTER_TIMER - stmg %r8,%r15,__LC_SAVE_AREA_SYNC - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO - larl %r13,system_call - lmg %r8,%r9,__LC_PGM_OLD_PSW - HANDLE_SIE_INTERCEPT %r14,1 - tmhh %r8,0x0001 # test problem state bit - jnz 1f # -> fault in user space - tmhh %r8,0x4000 # PER bit set in old PSW ? - jnz 0f # -> enabled, can't be a double fault - tm __LC_PGM_ILC+3,0x80 # check for per exception - jnz .Lpgm_svcper # -> single stepped svc -0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC - aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) - j 2f -1: UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER - LAST_BREAK %r14 - lg %r15,__LC_KERNEL_STACK - lg %r14,__TI_task(%r12) - lghi %r13,__LC_PGM_TDB - tm __LC_PGM_ILC+2,0x02 # check for transaction abort - jz 2f - mvc __THREAD_trap_tdb(256,%r14),0(%r13) -2: la %r11,STACK_FRAME_OVERHEAD(%r15) - stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC - stmg %r8,%r9,__PT_PSW(%r11) - mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC - mvc __PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE - xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) - stg %r10,__PT_ARGS(%r11) - tm __LC_PGM_ILC+3,0x80 # check for per exception - jz 0f - tmhh %r8,0x0001 # kernel per event ? - jz .Lpgm_kprobe - oi __PT_FLAGS+7(%r11),_PIF_PER_TRAP - mvc __THREAD_per_address(8,%r14),__LC_PER_ADDRESS - mvc __THREAD_per_cause(2,%r14),__LC_PER_CODE - mvc __THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID -0: REENABLE_IRQS - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - larl %r1,pgm_check_table - llgh %r10,__PT_INT_CODE+2(%r11) - nill %r10,0x007f - sll %r10,2 - je .Lsysc_return - lgf %r1,0(%r10,%r1) # load address of handler routine - lgr %r2,%r11 # pass pointer to pt_regs - basr %r14,%r1 # branch to interrupt-handler - j .Lsysc_return - -# -# PER event in supervisor state, must be kprobes -# -.Lpgm_kprobe: - REENABLE_IRQS - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - lgr %r2,%r11 # pass pointer to pt_regs - brasl %r14,do_per_trap - j .Lsysc_return - -# -# single stepped system call -# -.Lpgm_svcper: - mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW - larl %r14,.Lsysc_per - stg %r14,__LC_RETURN_PSW+8 - lghi %r14,_PIF_SYSCALL | _PIF_PER_TRAP - lpswe __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs - -/* - * IO interrupt handler routine - */ -ENTRY(io_int_handler) - STCK __LC_INT_CLOCK - stpt __LC_ASYNC_ENTER_TIMER - stmg %r8,%r15,__LC_SAVE_AREA_ASYNC - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO - larl %r13,system_call - lmg %r8,%r9,__LC_IO_OLD_PSW - HANDLE_SIE_INTERCEPT %r14,2 - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT - tmhh %r8,0x0001 # interrupting from user? - jz .Lio_skip - UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER - LAST_BREAK %r14 -.Lio_skip: - stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC - stmg %r8,%r9,__PT_PSW(%r11) - mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID - xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) - TRACE_IRQS_OFF - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) -.Lio_loop: - lgr %r2,%r11 # pass pointer to pt_regs - lghi %r3,IO_INTERRUPT - tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? - jz .Lio_call - lghi %r3,THIN_INTERRUPT -.Lio_call: - brasl %r14,do_IRQ - tm __LC_MACHINE_FLAGS+6,0x10 # MACHINE_FLAG_LPAR - jz .Lio_return - tpi 0 - jz .Lio_return - mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID - j .Lio_loop -.Lio_return: - LOCKDEP_SYS_EXIT - TRACE_IRQS_ON -.Lio_tif: - tm __TI_flags+7(%r12),_TIF_WORK - jnz .Lio_work # there is work to do (signals etc.) - tm __LC_CPU_FLAGS+7,_CIF_WORK - jnz .Lio_work -.Lio_restore: - lg %r14,__LC_VDSO_PER_CPU - lmg %r0,%r10,__PT_R0(%r11) - mvc __LC_RETURN_PSW(16),__PT_PSW(%r11) - stpt __LC_EXIT_TIMER - mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER - lmg %r11,%r15,__PT_R11(%r11) - lpswe __LC_RETURN_PSW -.Lio_done: - -# -# There is work todo, find out in which context we have been interrupted: -# 1) if we return to user space we can do all _TIF_WORK work -# 2) if we return to kernel code and kvm is enabled check if we need to -# modify the psw to leave SIE -# 3) if we return to kernel code and preemptive scheduling is enabled check -# the preemption counter and if it is zero call preempt_schedule_irq -# Before any work can be done, a switch to the kernel stack is required. -# -.Lio_work: - tm __PT_PSW+1(%r11),0x01 # returning to user ? - jo .Lio_work_user # yes -> do resched & signal -#ifdef CONFIG_PREEMPT - # check for preemptive scheduling - icm %r0,15,__TI_precount(%r12) - jnz .Lio_restore # preemption is disabled - tm __TI_flags+7(%r12),_TIF_NEED_RESCHED - jno .Lio_restore - # switch to kernel stack - lg %r1,__PT_R15(%r11) - aghi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) - mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) - xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) - la %r11,STACK_FRAME_OVERHEAD(%r1) - lgr %r15,%r1 - # TRACE_IRQS_ON already done at .Lio_return, call - # TRACE_IRQS_OFF to keep things symmetrical - TRACE_IRQS_OFF - brasl %r14,preempt_schedule_irq - j .Lio_return -#else - j .Lio_restore -#endif - -# -# Need to do work before returning to userspace, switch to kernel stack -# -.Lio_work_user: - lg %r1,__LC_KERNEL_STACK - mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) - xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) - la %r11,STACK_FRAME_OVERHEAD(%r1) - lgr %r15,%r1 - -# -# One of the work bits is on. Find out which one. -# -.Lio_work_tif: - tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING - jo .Lio_mcck_pending - tm __TI_flags+7(%r12),_TIF_NEED_RESCHED - jo .Lio_reschedule - tm __TI_flags+7(%r12),_TIF_SIGPENDING - jo .Lio_sigpending - tm __TI_flags+7(%r12),_TIF_NOTIFY_RESUME - jo .Lio_notify_resume - tm __LC_CPU_FLAGS+7,_CIF_ASCE - jo .Lio_uaccess - j .Lio_return # beware of critical section cleanup - -# -# _CIF_MCCK_PENDING is set, call handler -# -.Lio_mcck_pending: - # TRACE_IRQS_ON already done at .Lio_return - brasl %r14,s390_handle_mcck # TIF bit will be cleared by handler - TRACE_IRQS_OFF - j .Lio_return - -# -# _CIF_ASCE is set, load user space asce -# -.Lio_uaccess: - ni __LC_CPU_FLAGS+7,255-_CIF_ASCE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce - j .Lio_return - -# -# _TIF_NEED_RESCHED is set, call schedule -# -.Lio_reschedule: - # TRACE_IRQS_ON already done at .Lio_return - ssm __LC_SVC_NEW_PSW # reenable interrupts - brasl %r14,schedule # call scheduler - ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts - TRACE_IRQS_OFF - j .Lio_return - -# -# _TIF_SIGPENDING or is set, call do_signal -# -.Lio_sigpending: - # TRACE_IRQS_ON already done at .Lio_return - ssm __LC_SVC_NEW_PSW # reenable interrupts - lgr %r2,%r11 # pass pointer to pt_regs - brasl %r14,do_signal - ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts - TRACE_IRQS_OFF - j .Lio_return - -# -# _TIF_NOTIFY_RESUME or is set, call do_notify_resume -# -.Lio_notify_resume: - # TRACE_IRQS_ON already done at .Lio_return - ssm __LC_SVC_NEW_PSW # reenable interrupts - lgr %r2,%r11 # pass pointer to pt_regs - brasl %r14,do_notify_resume - ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts - TRACE_IRQS_OFF - j .Lio_return - -/* - * External interrupt handler routine - */ -ENTRY(ext_int_handler) - STCK __LC_INT_CLOCK - stpt __LC_ASYNC_ENTER_TIMER - stmg %r8,%r15,__LC_SAVE_AREA_ASYNC - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO - larl %r13,system_call - lmg %r8,%r9,__LC_EXT_OLD_PSW - HANDLE_SIE_INTERCEPT %r14,3 - SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT - tmhh %r8,0x0001 # interrupting from user ? - jz .Lext_skip - UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER - LAST_BREAK %r14 -.Lext_skip: - stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC - stmg %r8,%r9,__PT_PSW(%r11) - lghi %r1,__LC_EXT_PARAMS2 - mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR - mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS - mvc __PT_INT_PARM_LONG(8,%r11),0(%r1) - xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) - TRACE_IRQS_OFF - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - lgr %r2,%r11 # pass pointer to pt_regs - lghi %r3,EXT_INTERRUPT - brasl %r14,do_IRQ - j .Lio_return - -/* - * Load idle PSW. The second "half" of this function is in .Lcleanup_idle. - */ -ENTRY(psw_idle) - stg %r3,__SF_EMPTY(%r15) - larl %r1,.Lpsw_idle_lpsw+4 - stg %r1,__SF_EMPTY+8(%r15) - STCK __CLOCK_IDLE_ENTER(%r2) - stpt __TIMER_IDLE_ENTER(%r2) -.Lpsw_idle_lpsw: - lpswe __SF_EMPTY(%r15) - br %r14 -.Lpsw_idle_end: - -.L__critical_end: - -/* - * Machine check handler routines - */ -ENTRY(mcck_int_handler) - STCK __LC_MCCK_CLOCK - la %r1,4095 # revalidate r1 - spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # revalidate cpu timer - lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs - lg %r10,__LC_LAST_BREAK - lg %r12,__LC_THREAD_INFO - larl %r13,system_call - lmg %r8,%r9,__LC_MCK_OLD_PSW - HANDLE_SIE_INTERCEPT %r14,4 - tm __LC_MCCK_CODE,0x80 # system damage? - jo .Lmcck_panic # yes -> rest of mcck code invalid - lghi %r14,__LC_CPU_TIMER_SAVE_AREA - mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) - tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid? - jo 3f - la %r14,__LC_SYNC_ENTER_TIMER - clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER - jl 0f - la %r14,__LC_ASYNC_ENTER_TIMER -0: clc 0(8,%r14),__LC_EXIT_TIMER - jl 1f - la %r14,__LC_EXIT_TIMER -1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER - jl 2f - la %r14,__LC_LAST_UPDATE_TIMER -2: spt 0(%r14) - mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) -3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid? - jno .Lmcck_panic # no -> skip cleanup critical - SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_PANIC_STACK,PAGE_SHIFT - tm %r8,0x0001 # interrupting from user ? - jz .Lmcck_skip - UPDATE_VTIME %r14,__LC_MCCK_ENTER_TIMER - LAST_BREAK %r14 -.Lmcck_skip: - lghi %r14,__LC_GPREGS_SAVE_AREA+64 - stmg %r0,%r7,__PT_R0(%r11) - mvc __PT_R8(64,%r11),0(%r14) - stmg %r8,%r9,__PT_PSW(%r11) - xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11) - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - lgr %r2,%r11 # pass pointer to pt_regs - brasl %r14,s390_do_machine_check - tm __PT_PSW+1(%r11),0x01 # returning to user ? - jno .Lmcck_return - lg %r1,__LC_KERNEL_STACK # switch to kernel stack - mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) - xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1) - la %r11,STACK_FRAME_OVERHEAD(%r1) - lgr %r15,%r1 - ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off - tm __LC_CPU_FLAGS+7,_CIF_MCCK_PENDING - jno .Lmcck_return - TRACE_IRQS_OFF - brasl %r14,s390_handle_mcck - TRACE_IRQS_ON -.Lmcck_return: - lg %r14,__LC_VDSO_PER_CPU - lmg %r0,%r10,__PT_R0(%r11) - mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW - tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? - jno 0f - stpt __LC_EXIT_TIMER - mvc __VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER -0: lmg %r11,%r15,__PT_R11(%r11) - lpswe __LC_RETURN_MCCK_PSW - -.Lmcck_panic: - lg %r14,__LC_PANIC_STACK - slgr %r14,%r15 - srag %r14,%r14,PAGE_SHIFT - jz 0f - lg %r15,__LC_PANIC_STACK -0: aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) - j .Lmcck_skip - -# -# PSW restart interrupt handler -# -ENTRY(restart_int_handler) - stg %r15,__LC_SAVE_AREA_RESTART - lg %r15,__LC_RESTART_STACK - aghi %r15,-__PT_SIZE # create pt_regs on stack - xc 0(__PT_SIZE,%r15),0(%r15) - stmg %r0,%r14,__PT_R0(%r15) - mvc __PT_R15(8,%r15),__LC_SAVE_AREA_RESTART - mvc __PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw - aghi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack - xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) - lg %r1,__LC_RESTART_FN # load fn, parm & source cpu - lg %r2,__LC_RESTART_DATA - lg %r3,__LC_RESTART_SOURCE - ltgr %r3,%r3 # test source cpu address - jm 1f # negative -> skip source stop -0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu - brc 10,0b # wait for status stored -1: basr %r14,%r1 # call function - stap __SF_EMPTY(%r15) # store cpu address - llgh %r3,__SF_EMPTY(%r15) -2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu - brc 2,2b -3: j 3b - - .section .kprobes.text, "ax" - -#ifdef CONFIG_CHECK_STACK -/* - * The synchronous or the asynchronous stack overflowed. We are dead. - * No need to properly save the registers, we are going to panic anyway. - * Setup a pt_regs so that show_trace can provide a good call trace. - */ -stack_overflow: - lg %r15,__LC_PANIC_STACK # change to panic stack - la %r11,STACK_FRAME_OVERHEAD(%r15) - stmg %r0,%r7,__PT_R0(%r11) - stmg %r8,%r9,__PT_PSW(%r11) - mvc __PT_R8(64,%r11),0(%r14) - stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2 - xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15) - lgr %r2,%r11 # pass pointer to pt_regs - jg kernel_stack_overflow -#endif - - .align 8 -.Lcleanup_table: - .quad system_call - .quad .Lsysc_do_svc - .quad .Lsysc_tif - .quad .Lsysc_restore - .quad .Lsysc_done - .quad .Lio_tif - .quad .Lio_restore - .quad .Lio_done - .quad psw_idle - .quad .Lpsw_idle_end - -cleanup_critical: - clg %r9,BASED(.Lcleanup_table) # system_call - jl 0f - clg %r9,BASED(.Lcleanup_table+8) # .Lsysc_do_svc - jl .Lcleanup_system_call - clg %r9,BASED(.Lcleanup_table+16) # .Lsysc_tif - jl 0f - clg %r9,BASED(.Lcleanup_table+24) # .Lsysc_restore - jl .Lcleanup_sysc_tif - clg %r9,BASED(.Lcleanup_table+32) # .Lsysc_done - jl .Lcleanup_sysc_restore - clg %r9,BASED(.Lcleanup_table+40) # .Lio_tif - jl 0f - clg %r9,BASED(.Lcleanup_table+48) # .Lio_restore - jl .Lcleanup_io_tif - clg %r9,BASED(.Lcleanup_table+56) # .Lio_done - jl .Lcleanup_io_restore - clg %r9,BASED(.Lcleanup_table+64) # psw_idle - jl 0f - clg %r9,BASED(.Lcleanup_table+72) # .Lpsw_idle_end - jl .Lcleanup_idle -0: br %r14 - - -.Lcleanup_system_call: - # check if stpt has been executed - clg %r9,BASED(.Lcleanup_system_call_insn) - jh 0f - mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER - cghi %r11,__LC_SAVE_AREA_ASYNC - je 0f - mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER -0: # check if stmg has been executed - clg %r9,BASED(.Lcleanup_system_call_insn+8) - jh 0f - mvc __LC_SAVE_AREA_SYNC(64),0(%r11) -0: # check if base register setup + TIF bit load has been done - clg %r9,BASED(.Lcleanup_system_call_insn+16) - jhe 0f - # set up saved registers r10 and r12 - stg %r10,16(%r11) # r10 last break - stg %r12,32(%r11) # r12 thread-info pointer -0: # check if the user time update has been done - clg %r9,BASED(.Lcleanup_system_call_insn+24) - jh 0f - lg %r15,__LC_EXIT_TIMER - slg %r15,__LC_SYNC_ENTER_TIMER - alg %r15,__LC_USER_TIMER - stg %r15,__LC_USER_TIMER -0: # check if the system time update has been done - clg %r9,BASED(.Lcleanup_system_call_insn+32) - jh 0f - lg %r15,__LC_LAST_UPDATE_TIMER - slg %r15,__LC_EXIT_TIMER - alg %r15,__LC_SYSTEM_TIMER - stg %r15,__LC_SYSTEM_TIMER -0: # update accounting time stamp - mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER - # do LAST_BREAK - lg %r9,16(%r11) - srag %r9,%r9,23 - jz 0f - mvc __TI_last_break(8,%r12),16(%r11) -0: # set up saved register r11 - lg %r15,__LC_KERNEL_STACK - la %r9,STACK_FRAME_OVERHEAD(%r15) - stg %r9,24(%r11) # r11 pt_regs pointer - # fill pt_regs - mvc __PT_R8(64,%r9),__LC_SAVE_AREA_SYNC - stmg %r0,%r7,__PT_R0(%r9) - mvc __PT_PSW(16,%r9),__LC_SVC_OLD_PSW - mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC - xc __PT_FLAGS(8,%r9),__PT_FLAGS(%r9) - mvi __PT_FLAGS+7(%r9),_PIF_SYSCALL - # setup saved register r15 - stg %r15,56(%r11) # r15 stack pointer - # set new psw address and exit - larl %r9,.Lsysc_do_svc - br %r14 -.Lcleanup_system_call_insn: - .quad system_call - .quad .Lsysc_stmg - .quad .Lsysc_per - .quad .Lsysc_vtime+18 - .quad .Lsysc_vtime+42 - -.Lcleanup_sysc_tif: - larl %r9,.Lsysc_tif - br %r14 - -.Lcleanup_sysc_restore: - clg %r9,BASED(.Lcleanup_sysc_restore_insn) - je 0f - lg %r9,24(%r11) # get saved pointer to pt_regs - mvc __LC_RETURN_PSW(16),__PT_PSW(%r9) - mvc 0(64,%r11),__PT_R8(%r9) - lmg %r0,%r7,__PT_R0(%r9) -0: lmg %r8,%r9,__LC_RETURN_PSW - br %r14 -.Lcleanup_sysc_restore_insn: - .quad .Lsysc_done - 4 - -.Lcleanup_io_tif: - larl %r9,.Lio_tif - br %r14 - -.Lcleanup_io_restore: - clg %r9,BASED(.Lcleanup_io_restore_insn) - je 0f - lg %r9,24(%r11) # get saved r11 pointer to pt_regs - mvc __LC_RETURN_PSW(16),__PT_PSW(%r9) - mvc 0(64,%r11),__PT_R8(%r9) - lmg %r0,%r7,__PT_R0(%r9) -0: lmg %r8,%r9,__LC_RETURN_PSW - br %r14 -.Lcleanup_io_restore_insn: - .quad .Lio_done - 4 - -.Lcleanup_idle: - # copy interrupt clock & cpu timer - mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK - mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER - cghi %r11,__LC_SAVE_AREA_ASYNC - je 0f - mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK - mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER -0: # check if stck & stpt have been executed - clg %r9,BASED(.Lcleanup_idle_insn) - jhe 1f - mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2) - mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2) -1: # account system time going idle - lg %r9,__LC_STEAL_TIMER - alg %r9,__CLOCK_IDLE_ENTER(%r2) - slg %r9,__LC_LAST_UPDATE_CLOCK - stg %r9,__LC_STEAL_TIMER - mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2) - lg %r9,__LC_SYSTEM_TIMER - alg %r9,__LC_LAST_UPDATE_TIMER - slg %r9,__TIMER_IDLE_ENTER(%r2) - stg %r9,__LC_SYSTEM_TIMER - mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) - # prepare return psw - nihh %r8,0xfcfd # clear irq & wait state bits - lg %r9,48(%r11) # return from psw_idle - br %r14 -.Lcleanup_idle_insn: - .quad .Lpsw_idle_lpsw - -/* - * Integer constants - */ - .align 8 -.Lcritical_start: - .quad .L__critical_start -.Lcritical_length: - .quad .L__critical_end - .L__critical_start - - -#if IS_ENABLED(CONFIG_KVM) -/* - * sie64a calling convention: - * %r2 pointer to sie control block - * %r3 guest register save area - */ -ENTRY(sie64a) - stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers - stg %r2,__SF_EMPTY(%r15) # save control block pointer - stg %r3,__SF_EMPTY+8(%r15) # save guest register save area - xc __SF_EMPTY+16(16,%r15),__SF_EMPTY+16(%r15) # host id & reason - lmg %r0,%r13,0(%r3) # load guest gprs 0-13 - lg %r14,__LC_GMAP # get gmap pointer - ltgr %r14,%r14 - jz .Lsie_gmap - lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce -.Lsie_gmap: - lg %r14,__SF_EMPTY(%r15) # get control block pointer - oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now - tm __SIE_PROG20+3(%r14),1 # last exit... - jnz .Lsie_done - LPP __SF_EMPTY(%r15) # set guest id - sie 0(%r14) -.Lsie_done: - LPP __SF_EMPTY+16(%r15) # set host id - ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE - lctlg %c1,%c1,__LC_USER_ASCE # load primary asce -# some program checks are suppressing. C code (e.g. do_protection_exception) -# will rewind the PSW by the ILC, which is 4 bytes in case of SIE. Other -# instructions between sie64a and .Lsie_done should not cause program -# interrupts. So lets use a nop (47 00 00 00) as a landing pad. -# See also HANDLE_SIE_INTERCEPT -.Lrewind_pad: - nop 0 - .globl sie_exit -sie_exit: - lg %r14,__SF_EMPTY+8(%r15) # load guest register save area - stmg %r0,%r13,0(%r14) # save guest gprs 0-13 - lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers - lg %r2,__SF_EMPTY+24(%r15) # return exit reason code - br %r14 -.Lsie_fault: - lghi %r14,-EFAULT - stg %r14,__SF_EMPTY+24(%r15) # set exit reason code - j sie_exit - - .align 8 -.Lsie_critical: - .quad .Lsie_gmap -.Lsie_critical_length: - .quad .Lsie_done - .Lsie_gmap - - EX_TABLE(.Lrewind_pad,.Lsie_fault) - EX_TABLE(sie_exit,.Lsie_fault) -#endif - - .section .rodata, "a" -#define SYSCALL(esa,esame,emu) .long esame - .globl sys_call_table -sys_call_table: -#include "syscalls.S" -#undef SYSCALL - -#ifdef CONFIG_COMPAT - -#define SYSCALL(esa,esame,emu) .long emu - .globl sys_call_table_emu -sys_call_table_emu: -#include "syscalls.S" -#undef SYSCALL -#endif diff --git a/arch/s390/kernel/ftrace.c b/arch/s390/kernel/ftrace.c index 6c79f1b44fe7..e0eaf11134b4 100644 --- a/arch/s390/kernel/ftrace.c +++ b/arch/s390/kernel/ftrace.c @@ -130,8 +130,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, /* Verify that the to be replaced code matches what we expect. */ if (memcmp(&orig, &old, sizeof(old))) return -EINVAL; - if (probe_kernel_write((void *) rec->ip, &new, sizeof(new))) - return -EPERM; + s390_kernel_write((void *) rec->ip, &new, sizeof(new)); return 0; } @@ -159,8 +158,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) /* Verify that the to be replaced code matches what we expect. */ if (memcmp(&orig, &old, sizeof(old))) return -EINVAL; - if (probe_kernel_write((void *) rec->ip, &new, sizeof(new))) - return -EPERM; + s390_kernel_write((void *) rec->ip, &new, sizeof(new)); return 0; } @@ -231,14 +229,16 @@ int ftrace_enable_ftrace_graph_caller(void) { u8 op = 0x04; /* set mask field to zero */ - return probe_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); + s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); + return 0; } int ftrace_disable_ftrace_graph_caller(void) { u8 op = 0xf4; /* set mask field to all ones */ - return probe_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); + s390_kernel_write(__va(ftrace_graph_caller)+1, &op, sizeof(op)); + return 0; } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/s390/kernel/head.S b/arch/s390/kernel/head.S index 132f4c9ade60..59b7c6470567 100644 --- a/arch/s390/kernel/head.S +++ b/arch/s390/kernel/head.S @@ -27,11 +27,7 @@ #include <asm/thread_info.h> #include <asm/page.h> -#ifdef CONFIG_64BIT #define ARCH_OFFSET 4 -#else -#define ARCH_OFFSET 0 -#endif __HEAD @@ -67,7 +63,6 @@ __HEAD # subroutine to set architecture mode # .Lsetmode: -#ifdef CONFIG_64BIT mvi __LC_AR_MODE_ID,1 # set esame flag slr %r0,%r0 # set cpuid to zero lhi %r1,2 # mode 2 = esame (dump) @@ -76,16 +71,12 @@ __HEAD .fill 16,4,0x0 0: lmh %r0,%r15,0(%r13) # clear high-order half of gprs sam31 # switch to 31 bit addressing mode -#else - mvi __LC_AR_MODE_ID,0 # set ESA flag (mode 0) -#endif br %r14 # # subroutine to wait for end I/O # .Lirqwait: -#ifdef CONFIG_64BIT mvc 0x1f0(16),.Lnewpsw # set up IO interrupt psw lpsw .Lwaitpsw .Lioint: @@ -93,15 +84,6 @@ __HEAD .align 8 .Lnewpsw: .quad 0x0000000080000000,.Lioint -#else - mvc 0x78(8),.Lnewpsw # set up IO interrupt psw - lpsw .Lwaitpsw -.Lioint: - br %r14 - .align 8 -.Lnewpsw: - .long 0x00080000,0x80000000+.Lioint -#endif .Lwaitpsw: .long 0x020a0000,0x80000000+.Lioint @@ -375,7 +357,6 @@ ENTRY(startup) ENTRY(startup_kdump) j .Lep_startup_kdump .Lep_startup_normal: -#ifdef CONFIG_64BIT mvi __LC_AR_MODE_ID,1 # set esame flag slr %r0,%r0 # set cpuid to zero lhi %r1,2 # mode 2 = esame (dump) @@ -384,9 +365,6 @@ ENTRY(startup_kdump) .fill 16,4,0x0 0: lmh %r0,%r15,0(%r13) # clear high-order half of gprs sam31 # switch to 31 bit addressing mode -#else - mvi __LC_AR_MODE_ID,0 # set ESA flag (mode 0) -#endif basr %r13,0 # get base .LPG0: xc 0x200(256),0x200 # partially clear lowcore @@ -396,7 +374,6 @@ ENTRY(startup_kdump) spt 6f-.LPG0(%r13) mvc __LC_LAST_UPDATE_TIMER(8),6f-.LPG0(%r13) xc __LC_STFL_FAC_LIST(8),__LC_STFL_FAC_LIST -#ifndef CONFIG_MARCH_G5 # check capabilities against MARCH_{G5,Z900,Z990,Z9_109,Z10} .insn s,0xb2b10000,0 # store facilities @ __LC_STFL_FAC_LIST tm __LC_STFL_FAC_LIST,0x01 # stfle available ? @@ -435,7 +412,6 @@ ENTRY(startup_kdump) # the kernel will crash. Format is number of facility words with bits set, # followed by the facility words. -#if defined(CONFIG_64BIT) #if defined(CONFIG_MARCH_Z13) .long 3, 0xc100eff2, 0xf46ce800, 0x00400000 #elif defined(CONFIG_MARCH_ZEC12) @@ -451,35 +427,10 @@ ENTRY(startup_kdump) #elif defined(CONFIG_MARCH_Z900) .long 1, 0xc0000000 #endif -#else -#if defined(CONFIG_MARCH_ZEC12) - .long 1, 0x8100c880 -#elif defined(CONFIG_MARCH_Z196) - .long 1, 0x8100c880 -#elif defined(CONFIG_MARCH_Z10) - .long 1, 0x8100c880 -#elif defined(CONFIG_MARCH_Z9_109) - .long 1, 0x8100c880 -#elif defined(CONFIG_MARCH_Z990) - .long 1, 0x80002000 -#elif defined(CONFIG_MARCH_Z900) - .long 1, 0x80000000 -#endif -#endif 4: -#endif - -#ifdef CONFIG_64BIT /* Continue with 64bit startup code in head64.S */ sam64 # switch to 64 bit mode jg startup_continue -#else - /* Continue with 31bit startup code in head31.S */ - l %r13,5f-.LPG0(%r13) - b 0(%r13) - .align 8 -5: .long startup_continue -#endif .align 8 6: .long 0x7fffffff,0xffffffff diff --git a/arch/s390/kernel/head31.S b/arch/s390/kernel/head31.S deleted file mode 100644 index 6dbe80983a24..000000000000 --- a/arch/s390/kernel/head31.S +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright IBM Corp. 2005, 2010 - * - * Author(s): Hartmut Penner <hp@de.ibm.com> - * Martin Schwidefsky <schwidefsky@de.ibm.com> - * Rob van der Heij <rvdhei@iae.nl> - * Heiko Carstens <heiko.carstens@de.ibm.com> - * - */ - -#include <linux/init.h> -#include <linux/linkage.h> -#include <asm/asm-offsets.h> -#include <asm/thread_info.h> -#include <asm/page.h> - -__HEAD -ENTRY(startup_continue) - basr %r13,0 # get base -.LPG1: - - l %r1,.Lbase_cc-.LPG1(%r13) - mvc 0(8,%r1),__LC_LAST_UPDATE_CLOCK - lctl %c0,%c15,.Lctl-.LPG1(%r13) # load control registers - l %r12,.Lparmaddr-.LPG1(%r13) # pointer to parameter area - # move IPL device to lowcore -# -# Setup stack -# - l %r15,.Linittu-.LPG1(%r13) - st %r15,__LC_THREAD_INFO # cache thread info in lowcore - mvc __LC_CURRENT(4),__TI_task(%r15) - ahi %r15,1<<(PAGE_SHIFT+THREAD_ORDER) # init_task_union+THREAD_SIZE - st %r15,__LC_KERNEL_STACK # set end of kernel stack - ahi %r15,-96 -# -# Save ipl parameters, clear bss memory, initialize storage key for kernel pages, -# and create a kernel NSS if the SAVESYS= parm is defined -# - l %r14,.Lstartup_init-.LPG1(%r13) - basr %r14,%r14 - lpsw .Lentry-.LPG1(13) # jump to _stext in primary-space, - # virtual and never return ... - .align 8 -.Lentry:.long 0x00080000,0x80000000 + _stext -.Lctl: .long 0x04b50000 # cr0: various things - .long 0 # cr1: primary space segment table - .long .Lduct # cr2: dispatchable unit control table - .long 0 # cr3: instruction authorization - .long 0 # cr4: instruction authorization - .long .Lduct # cr5: primary-aste origin - .long 0 # cr6: I/O interrupts - .long 0 # cr7: secondary space segment table - .long 0 # cr8: access registers translation - .long 0 # cr9: tracing off - .long 0 # cr10: tracing off - .long 0 # cr11: tracing off - .long 0 # cr12: tracing off - .long 0 # cr13: home space segment table - .long 0xc0000000 # cr14: machine check handling off - .long 0 # cr15: linkage stack operations -.Lbss_bgn: .long __bss_start -.Lbss_end: .long _end -.Lparmaddr: .long PARMAREA -.Linittu: .long init_thread_union -.Lstartup_init: - .long startup_init - .align 64 -.Lduct: .long 0,0,0,0,.Lduald,0,0,0 - .long 0,0,0,0,0,0,0,0 - .align 128 -.Lduald:.rept 8 - .long 0x80000000,0,0,0 # invalid access-list entries - .endr -.Lbase_cc: - .long sched_clock_base_cc - -ENTRY(_ehead) - - .org 0x100000 - 0x11000 # head.o ends at 0x11000 -# -# startup-code, running in absolute addressing mode -# -ENTRY(_stext) - basr %r13,0 # get base -.LPG3: -# check control registers - stctl %c0,%c15,0(%r15) - oi 2(%r15),0x60 # enable sigp emergency & external call - oi 0(%r15),0x10 # switch on low address protection - lctl %c0,%c15,0(%r15) - -# - lam 0,15,.Laregs-.LPG3(%r13) # load access regs needed by uaccess - l %r14,.Lstart-.LPG3(%r13) - basr %r14,%r14 # call start_kernel -# -# We returned from start_kernel ?!? PANIK -# - basr %r13,0 - lpsw .Ldw-.(%r13) # load disabled wait psw -# - .align 8 -.Ldw: .long 0x000a0000,0x00000000 -.Lstart:.long start_kernel -.Laregs:.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/arch/s390/kernel/head_kdump.S b/arch/s390/kernel/head_kdump.S index 085a95eb315f..d05950f02c34 100644 --- a/arch/s390/kernel/head_kdump.S +++ b/arch/s390/kernel/head_kdump.S @@ -92,17 +92,9 @@ startup_kdump_relocated: #else .align 2 .Lep_startup_kdump: -#ifdef CONFIG_64BIT larl %r13,startup_kdump_crash lpswe 0(%r13) .align 8 startup_kdump_crash: .quad 0x0002000080000000,0x0000000000000000 + startup_kdump_crash -#else - basr %r13,0 -0: lpsw startup_kdump_crash-0b(%r13) -.align 8 -startup_kdump_crash: - .long 0x000a0000,0x00000000 + startup_kdump_crash -#endif /* CONFIG_64BIT */ #endif /* CONFIG_CRASH_DUMP */ diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index 5c8651f36509..52fbef91d1d9 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -182,24 +182,21 @@ EXPORT_SYMBOL_GPL(diag308); /* SYSFS */ -#define DEFINE_IPL_ATTR_RO(_prefix, _name, _format, _value) \ +#define IPL_ATTR_SHOW_FN(_prefix, _name, _format, args...) \ static ssize_t sys_##_prefix##_##_name##_show(struct kobject *kobj, \ struct kobj_attribute *attr, \ char *page) \ { \ - return sprintf(page, _format, _value); \ -} \ + return snprintf(page, PAGE_SIZE, _format, ##args); \ +} + +#define DEFINE_IPL_ATTR_RO(_prefix, _name, _format, _value) \ +IPL_ATTR_SHOW_FN(_prefix, _name, _format, _value) \ static struct kobj_attribute sys_##_prefix##_##_name##_attr = \ - __ATTR(_name, S_IRUGO, sys_##_prefix##_##_name##_show, NULL); + __ATTR(_name, S_IRUGO, sys_##_prefix##_##_name##_show, NULL) #define DEFINE_IPL_ATTR_RW(_prefix, _name, _fmt_out, _fmt_in, _value) \ -static ssize_t sys_##_prefix##_##_name##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, \ - char *page) \ -{ \ - return sprintf(page, _fmt_out, \ - (unsigned long long) _value); \ -} \ +IPL_ATTR_SHOW_FN(_prefix, _name, _fmt_out, (unsigned long long) _value) \ static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj, \ struct kobj_attribute *attr, \ const char *buf, size_t len) \ @@ -213,15 +210,10 @@ static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj, \ static struct kobj_attribute sys_##_prefix##_##_name##_attr = \ __ATTR(_name,(S_IRUGO | S_IWUSR), \ sys_##_prefix##_##_name##_show, \ - sys_##_prefix##_##_name##_store); + sys_##_prefix##_##_name##_store) #define DEFINE_IPL_ATTR_STR_RW(_prefix, _name, _fmt_out, _fmt_in, _value)\ -static ssize_t sys_##_prefix##_##_name##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, \ - char *page) \ -{ \ - return sprintf(page, _fmt_out, _value); \ -} \ +IPL_ATTR_SHOW_FN(_prefix, _name, _fmt_out, _value) \ static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj, \ struct kobj_attribute *attr, \ const char *buf, size_t len) \ @@ -233,7 +225,7 @@ static ssize_t sys_##_prefix##_##_name##_store(struct kobject *kobj, \ static struct kobj_attribute sys_##_prefix##_##_name##_attr = \ __ATTR(_name,(S_IRUGO | S_IWUSR), \ sys_##_prefix##_##_name##_show, \ - sys_##_prefix##_##_name##_store); + sys_##_prefix##_##_name##_store) static void make_attrs_ro(struct attribute **attrs) { @@ -415,15 +407,9 @@ static ssize_t ipl_parameter_read(struct file *filp, struct kobject *kobj, return memory_read_from_buffer(buf, count, &off, IPL_PARMBLOCK_START, IPL_PARMBLOCK_SIZE); } - -static struct bin_attribute ipl_parameter_attr = { - .attr = { - .name = "binary_parameter", - .mode = S_IRUGO, - }, - .size = PAGE_SIZE, - .read = &ipl_parameter_read, -}; +static struct bin_attribute ipl_parameter_attr = + __BIN_ATTR(binary_parameter, S_IRUGO, ipl_parameter_read, NULL, + PAGE_SIZE); static ssize_t ipl_scp_data_read(struct file *filp, struct kobject *kobj, struct bin_attribute *attr, char *buf, @@ -434,14 +420,13 @@ static ssize_t ipl_scp_data_read(struct file *filp, struct kobject *kobj, return memory_read_from_buffer(buf, count, &off, scp_data, size); } +static struct bin_attribute ipl_scp_data_attr = + __BIN_ATTR(scp_data, S_IRUGO, ipl_scp_data_read, NULL, PAGE_SIZE); -static struct bin_attribute ipl_scp_data_attr = { - .attr = { - .name = "scp_data", - .mode = S_IRUGO, - }, - .size = PAGE_SIZE, - .read = ipl_scp_data_read, +static struct bin_attribute *ipl_fcp_bin_attrs[] = { + &ipl_parameter_attr, + &ipl_scp_data_attr, + NULL, }; /* FCP ipl device attributes */ @@ -484,6 +469,7 @@ static struct attribute *ipl_fcp_attrs[] = { static struct attribute_group ipl_fcp_attr_group = { .attrs = ipl_fcp_attrs, + .bin_attrs = ipl_fcp_bin_attrs, }; /* CCW ipl device attributes */ @@ -540,28 +526,6 @@ static struct attribute_group ipl_unknown_attr_group = { static struct kset *ipl_kset; -static int __init ipl_register_fcp_files(void) -{ - int rc; - - rc = sysfs_create_group(&ipl_kset->kobj, &ipl_fcp_attr_group); - if (rc) - goto out; - rc = sysfs_create_bin_file(&ipl_kset->kobj, &ipl_parameter_attr); - if (rc) - goto out_ipl_parm; - rc = sysfs_create_bin_file(&ipl_kset->kobj, &ipl_scp_data_attr); - if (!rc) - goto out; - - sysfs_remove_bin_file(&ipl_kset->kobj, &ipl_parameter_attr); - -out_ipl_parm: - sysfs_remove_group(&ipl_kset->kobj, &ipl_fcp_attr_group); -out: - return rc; -} - static void __ipl_run(void *unused) { diag308(DIAG308_IPL, NULL); @@ -596,7 +560,7 @@ static int __init ipl_init(void) break; case IPL_TYPE_FCP: case IPL_TYPE_FCP_DUMP: - rc = ipl_register_fcp_files(); + rc = sysfs_create_group(&ipl_kset->kobj, &ipl_fcp_attr_group); break; case IPL_TYPE_NSS: rc = sysfs_create_group(&ipl_kset->kobj, &ipl_nss_attr_group); @@ -744,15 +708,13 @@ static ssize_t reipl_fcp_scpdata_write(struct file *filp, struct kobject *kobj, return count; } +static struct bin_attribute sys_reipl_fcp_scp_data_attr = + __BIN_ATTR(scp_data, (S_IRUGO | S_IWUSR), reipl_fcp_scpdata_read, + reipl_fcp_scpdata_write, PAGE_SIZE); -static struct bin_attribute sys_reipl_fcp_scp_data_attr = { - .attr = { - .name = "scp_data", - .mode = S_IRUGO | S_IWUSR, - }, - .size = PAGE_SIZE, - .read = reipl_fcp_scpdata_read, - .write = reipl_fcp_scpdata_write, +static struct bin_attribute *reipl_fcp_bin_attrs[] = { + &sys_reipl_fcp_scp_data_attr, + NULL, }; DEFINE_IPL_ATTR_RW(reipl_fcp, wwpn, "0x%016llx\n", "%llx\n", @@ -841,6 +803,7 @@ static struct attribute *reipl_fcp_attrs[] = { static struct attribute_group reipl_fcp_attr_group = { .attrs = reipl_fcp_attrs, + .bin_attrs = reipl_fcp_bin_attrs, }; /* CCW reipl device attributes */ @@ -1261,15 +1224,6 @@ static int __init reipl_fcp_init(void) return rc; } - rc = sysfs_create_bin_file(&reipl_fcp_kset->kobj, - &sys_reipl_fcp_scp_data_attr); - if (rc) { - sysfs_remove_group(&reipl_fcp_kset->kobj, &reipl_fcp_attr_group); - kset_unregister(reipl_fcp_kset); - free_page((unsigned long) reipl_block_fcp); - return rc; - } - if (ipl_info.type == IPL_TYPE_FCP) { memcpy(reipl_block_fcp, IPL_PARMBLOCK_START, PAGE_SIZE); /* @@ -1713,9 +1667,7 @@ static ssize_t on_reboot_store(struct kobject *kobj, { return set_trigger(buf, &on_reboot_trigger, len); } - -static struct kobj_attribute on_reboot_attr = - __ATTR(on_reboot, 0644, on_reboot_show, on_reboot_store); +static struct kobj_attribute on_reboot_attr = __ATTR_RW(on_reboot); static void do_machine_restart(char *__unused) { @@ -1741,9 +1693,7 @@ static ssize_t on_panic_store(struct kobject *kobj, { return set_trigger(buf, &on_panic_trigger, len); } - -static struct kobj_attribute on_panic_attr = - __ATTR(on_panic, 0644, on_panic_show, on_panic_store); +static struct kobj_attribute on_panic_attr = __ATTR_RW(on_panic); static void do_panic(void) { @@ -1769,9 +1719,7 @@ static ssize_t on_restart_store(struct kobject *kobj, { return set_trigger(buf, &on_restart_trigger, len); } - -static struct kobj_attribute on_restart_attr = - __ATTR(on_restart, 0644, on_restart_show, on_restart_store); +static struct kobj_attribute on_restart_attr = __ATTR_RW(on_restart); static void __do_restart(void *ignore) { @@ -1808,10 +1756,7 @@ static ssize_t on_halt_store(struct kobject *kobj, { return set_trigger(buf, &on_halt_trigger, len); } - -static struct kobj_attribute on_halt_attr = - __ATTR(on_halt, 0644, on_halt_show, on_halt_store); - +static struct kobj_attribute on_halt_attr = __ATTR_RW(on_halt); static void do_machine_halt(void) { @@ -1837,10 +1782,7 @@ static ssize_t on_poff_store(struct kobject *kobj, { return set_trigger(buf, &on_poff_trigger, len); } - -static struct kobj_attribute on_poff_attr = - __ATTR(on_poff, 0644, on_poff_show, on_poff_store); - +static struct kobj_attribute on_poff_attr = __ATTR_RW(on_poff); static void do_machine_power_off(void) { @@ -1850,26 +1792,27 @@ static void do_machine_power_off(void) } void (*_machine_power_off)(void) = do_machine_power_off; +static struct attribute *shutdown_action_attrs[] = { + &on_restart_attr.attr, + &on_reboot_attr.attr, + &on_panic_attr.attr, + &on_halt_attr.attr, + &on_poff_attr.attr, + NULL, +}; + +static struct attribute_group shutdown_action_attr_group = { + .attrs = shutdown_action_attrs, +}; + static void __init shutdown_triggers_init(void) { shutdown_actions_kset = kset_create_and_add("shutdown_actions", NULL, firmware_kobj); if (!shutdown_actions_kset) goto fail; - if (sysfs_create_file(&shutdown_actions_kset->kobj, - &on_reboot_attr.attr)) - goto fail; - if (sysfs_create_file(&shutdown_actions_kset->kobj, - &on_panic_attr.attr)) - goto fail; - if (sysfs_create_file(&shutdown_actions_kset->kobj, - &on_halt_attr.attr)) - goto fail; - if (sysfs_create_file(&shutdown_actions_kset->kobj, - &on_poff_attr.attr)) - goto fail; - if (sysfs_create_file(&shutdown_actions_kset->kobj, - &on_restart_attr.attr)) + if (sysfs_create_group(&shutdown_actions_kset->kobj, + &shutdown_action_attr_group)) goto fail; return; fail: @@ -2062,12 +2005,10 @@ static void do_reset_calls(void) { struct reset_call *reset; -#ifdef CONFIG_64BIT if (diag308_set_works) { diag308_reset(); return; } -#endif list_for_each_entry(reset, &rcall, list) reset->fn(); } diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index f238720690f3..e9d9addfaa44 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -56,7 +56,7 @@ static const struct irq_class irqclass_main_desc[NR_IRQS_BASE] = { * /proc/interrupts. * In addition this list contains non external / I/O events like NMIs. */ -static const struct irq_class irqclass_sub_desc[NR_ARCH_IRQS] = { +static const struct irq_class irqclass_sub_desc[] = { {.irq = IRQEXT_CLK, .name = "CLK", .desc = "[EXT] Clock Comparator"}, {.irq = IRQEXT_EXC, .name = "EXC", .desc = "[EXT] External Call"}, {.irq = IRQEXT_EMS, .name = "EMS", .desc = "[EXT] Emergency Signal"}, @@ -79,7 +79,6 @@ static const struct irq_class irqclass_sub_desc[NR_ARCH_IRQS] = { {.irq = IRQIO_TAP, .name = "TAP", .desc = "[I/O] Tape"}, {.irq = IRQIO_VMR, .name = "VMR", .desc = "[I/O] Unit Record Devices"}, {.irq = IRQIO_LCS, .name = "LCS", .desc = "[I/O] LCS"}, - {.irq = IRQIO_CLW, .name = "CLW", .desc = "[I/O] CLAW"}, {.irq = IRQIO_CTC, .name = "CTC", .desc = "[I/O] CTC"}, {.irq = IRQIO_APB, .name = "APB", .desc = "[I/O] AP Bus"}, {.irq = IRQIO_ADM, .name = "ADM", .desc = "[I/O] EADM Subchannel"}, @@ -94,6 +93,7 @@ static const struct irq_class irqclass_sub_desc[NR_ARCH_IRQS] = { void __init init_IRQ(void) { + BUILD_BUG_ON(ARRAY_SIZE(irqclass_sub_desc) != NR_ARCH_IRQS); init_cio_interrupts(); init_airq_interrupts(); init_ext_interrupts(); diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c index 830066f936c8..a90299600483 100644 --- a/arch/s390/kernel/jump_label.c +++ b/arch/s390/kernel/jump_label.c @@ -78,7 +78,7 @@ static void __jump_label_transform(struct jump_entry *entry, if (memcmp((void *)entry->code, &old, sizeof(old))) jump_label_bug(entry, &old, &new); } - probe_kernel_write((void *)entry->code, &new, sizeof(new)); + s390_kernel_write((void *)entry->code, &new, sizeof(new)); } static int __sm_arch_jump_label_transform(void *data) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index f516edc1fbe3..389db56a2208 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -178,7 +178,7 @@ static int swap_instruction(void *data) } skip_ftrace: kcb->kprobe_status = KPROBE_SWAP_INST; - probe_kernel_write(p->addr, &new_insn, len); + s390_kernel_write(p->addr, &new_insn, len); kcb->kprobe_status = status; return 0; } diff --git a/arch/s390/kernel/module.c b/arch/s390/kernel/module.c index 2ca95862e336..0c1a679314dd 100644 --- a/arch/s390/kernel/module.c +++ b/arch/s390/kernel/module.c @@ -38,13 +38,8 @@ #define DEBUGP(fmt , ...) #endif -#ifndef CONFIG_64BIT -#define PLT_ENTRY_SIZE 12 -#else /* CONFIG_64BIT */ #define PLT_ENTRY_SIZE 20 -#endif /* CONFIG_64BIT */ -#ifdef CONFIG_64BIT void *module_alloc(unsigned long size) { if (PAGE_ALIGN(size) > MODULES_LEN) @@ -53,7 +48,6 @@ void *module_alloc(unsigned long size) GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); } -#endif void module_arch_freeing_init(struct module *mod) { @@ -323,17 +317,11 @@ static int apply_rela(Elf_Rela *rela, Elf_Addr base, Elf_Sym *symtab, unsigned int *ip; ip = me->module_core + me->arch.plt_offset + info->plt_offset; -#ifndef CONFIG_64BIT - ip[0] = 0x0d105810; /* basr 1,0; l 1,6(1); br 1 */ - ip[1] = 0x100607f1; - ip[2] = val; -#else /* CONFIG_64BIT */ ip[0] = 0x0d10e310; /* basr 1,0; lg 1,10(1); br 1 */ ip[1] = 0x100a0004; ip[2] = 0x07f10000; ip[3] = (unsigned int) (val >> 32); ip[4] = (unsigned int) val; -#endif /* CONFIG_64BIT */ info->plt_initialized = 1; } if (r_type == R_390_PLTOFF16 || diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 3f51cf4e8f02..505c17c0ae1a 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -117,55 +117,36 @@ static int notrace s390_revalidate_registers(struct mci *mci) */ kill_task = 1; } -#ifndef CONFIG_64BIT + fpt_save_area = &S390_lowcore.floating_pt_save_area; + fpt_creg_save_area = &S390_lowcore.fpt_creg_save_area; + if (!mci->fc) { + /* + * Floating point control register can't be restored. + * Task will be terminated. + */ + asm volatile("lfpc 0(%0)" : : "a" (&zero), "m" (zero)); + kill_task = 1; + } else + asm volatile("lfpc 0(%0)" : : "a" (fpt_creg_save_area)); + asm volatile( " ld 0,0(%0)\n" - " ld 2,8(%0)\n" - " ld 4,16(%0)\n" - " ld 6,24(%0)" - : : "a" (&S390_lowcore.floating_pt_save_area)); -#endif - - if (MACHINE_HAS_IEEE) { -#ifdef CONFIG_64BIT - fpt_save_area = &S390_lowcore.floating_pt_save_area; - fpt_creg_save_area = &S390_lowcore.fpt_creg_save_area; -#else - fpt_save_area = (void *) S390_lowcore.extended_save_area_addr; - fpt_creg_save_area = fpt_save_area + 128; -#endif - if (!mci->fc) { - /* - * Floating point control register can't be restored. - * Task will be terminated. - */ - asm volatile("lfpc 0(%0)" : : "a" (&zero), "m" (zero)); - kill_task = 1; - - } else - asm volatile("lfpc 0(%0)" : : "a" (fpt_creg_save_area)); - - asm volatile( - " ld 0,0(%0)\n" - " ld 1,8(%0)\n" - " ld 2,16(%0)\n" - " ld 3,24(%0)\n" - " ld 4,32(%0)\n" - " ld 5,40(%0)\n" - " ld 6,48(%0)\n" - " ld 7,56(%0)\n" - " ld 8,64(%0)\n" - " ld 9,72(%0)\n" - " ld 10,80(%0)\n" - " ld 11,88(%0)\n" - " ld 12,96(%0)\n" - " ld 13,104(%0)\n" - " ld 14,112(%0)\n" - " ld 15,120(%0)\n" - : : "a" (fpt_save_area)); - } - -#ifdef CONFIG_64BIT + " ld 1,8(%0)\n" + " ld 2,16(%0)\n" + " ld 3,24(%0)\n" + " ld 4,32(%0)\n" + " ld 5,40(%0)\n" + " ld 6,48(%0)\n" + " ld 7,56(%0)\n" + " ld 8,64(%0)\n" + " ld 9,72(%0)\n" + " ld 10,80(%0)\n" + " ld 11,88(%0)\n" + " ld 12,96(%0)\n" + " ld 13,104(%0)\n" + " ld 14,112(%0)\n" + " ld 15,120(%0)\n" + : : "a" (fpt_save_area)); /* Revalidate vector registers */ if (MACHINE_HAS_VX && current->thread.vxrs) { if (!mci->vr) { @@ -178,7 +159,6 @@ static int notrace s390_revalidate_registers(struct mci *mci) restore_vx_regs((__vector128 *) S390_lowcore.vector_save_area_addr); } -#endif /* Revalidate access registers */ asm volatile( " lam 0,15,0(%0)" @@ -198,21 +178,14 @@ static int notrace s390_revalidate_registers(struct mci *mci) */ s390_handle_damage("invalid control registers."); } else { -#ifdef CONFIG_64BIT asm volatile( " lctlg 0,15,0(%0)" : : "a" (&S390_lowcore.cregs_save_area)); -#else - asm volatile( - " lctl 0,15,0(%0)" - : : "a" (&S390_lowcore.cregs_save_area)); -#endif } /* * We don't even try to revalidate the TOD register, since we simply * can't write something sensible into that register. */ -#ifdef CONFIG_64BIT /* * See if we can revalidate the TOD programmable register with its * old contents (should be zero) otherwise set it to zero. @@ -228,7 +201,6 @@ static int notrace s390_revalidate_registers(struct mci *mci) " sckpf" : : "a" (&S390_lowcore.tod_progreg_save_area) : "0", "cc"); -#endif /* Revalidate clock comparator register */ set_clock_comparator(S390_lowcore.clock_comparator); /* Check if old PSW is valid */ @@ -280,19 +252,11 @@ void notrace s390_do_machine_check(struct pt_regs *regs) if (mci->b) { /* Processing backup -> verify if we can survive this */ u64 z_mcic, o_mcic, t_mcic; -#ifdef CONFIG_64BIT z_mcic = (1ULL<<63 | 1ULL<<59 | 1ULL<<29); o_mcic = (1ULL<<43 | 1ULL<<42 | 1ULL<<41 | 1ULL<<40 | 1ULL<<36 | 1ULL<<35 | 1ULL<<34 | 1ULL<<32 | 1ULL<<30 | 1ULL<<21 | 1ULL<<20 | 1ULL<<17 | 1ULL<<16); -#else - z_mcic = (1ULL<<63 | 1ULL<<59 | 1ULL<<57 | 1ULL<<50 | - 1ULL<<29); - o_mcic = (1ULL<<43 | 1ULL<<42 | 1ULL<<41 | 1ULL<<40 | - 1ULL<<36 | 1ULL<<35 | 1ULL<<34 | 1ULL<<32 | - 1ULL<<30 | 1ULL<<20 | 1ULL<<17 | 1ULL<<16); -#endif t_mcic = *(u64 *)mci; if (((t_mcic & z_mcic) != 0) || diff --git a/arch/s390/kernel/pgm_check.S b/arch/s390/kernel/pgm_check.S index f6f8886399f6..036aa01d06a9 100644 --- a/arch/s390/kernel/pgm_check.S +++ b/arch/s390/kernel/pgm_check.S @@ -6,19 +6,13 @@ #include <linux/linkage.h> -#ifdef CONFIG_32BIT -#define PGM_CHECK_64BIT(handler) .long default_trap_handler -#else -#define PGM_CHECK_64BIT(handler) .long handler -#endif - #define PGM_CHECK(handler) .long handler #define PGM_CHECK_DEFAULT PGM_CHECK(default_trap_handler) /* * The program check table contains exactly 128 (0x00-0x7f) entries. Each - * line defines the 31 and/or 64 bit function to be called corresponding - * to the program check interruption code. + * line defines the function to be called corresponding to the program check + * interruption code. */ .section .rodata, "a" ENTRY(pgm_check_table) @@ -46,10 +40,10 @@ PGM_CHECK_DEFAULT /* 14 */ PGM_CHECK(operand_exception) /* 15 */ PGM_CHECK_DEFAULT /* 16 */ PGM_CHECK_DEFAULT /* 17 */ -PGM_CHECK_64BIT(transaction_exception) /* 18 */ +PGM_CHECK(transaction_exception) /* 18 */ PGM_CHECK_DEFAULT /* 19 */ PGM_CHECK_DEFAULT /* 1a */ -PGM_CHECK_64BIT(vector_exception) /* 1b */ +PGM_CHECK(vector_exception) /* 1b */ PGM_CHECK(space_switch_exception) /* 1c */ PGM_CHECK(hfp_sqrt_exception) /* 1d */ PGM_CHECK_DEFAULT /* 1e */ @@ -78,10 +72,10 @@ PGM_CHECK_DEFAULT /* 34 */ PGM_CHECK_DEFAULT /* 35 */ PGM_CHECK_DEFAULT /* 36 */ PGM_CHECK_DEFAULT /* 37 */ -PGM_CHECK_64BIT(do_dat_exception) /* 38 */ -PGM_CHECK_64BIT(do_dat_exception) /* 39 */ -PGM_CHECK_64BIT(do_dat_exception) /* 3a */ -PGM_CHECK_64BIT(do_dat_exception) /* 3b */ +PGM_CHECK(do_dat_exception) /* 38 */ +PGM_CHECK(do_dat_exception) /* 39 */ +PGM_CHECK(do_dat_exception) /* 3a */ +PGM_CHECK(do_dat_exception) /* 3b */ PGM_CHECK_DEFAULT /* 3c */ PGM_CHECK_DEFAULT /* 3d */ PGM_CHECK_DEFAULT /* 3e */ diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 13fc0978ca7e..dc5edc29b73a 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -79,13 +79,11 @@ void release_thread(struct task_struct *dead_task) { } -#ifdef CONFIG_64BIT void arch_release_task_struct(struct task_struct *tsk) { if (tsk->thread.vxrs) kfree(tsk->thread.vxrs); } -#endif int copy_thread(unsigned long clone_flags, unsigned long new_stackp, unsigned long arg, struct task_struct *p) @@ -144,19 +142,6 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, p->thread.ri_signum = 0; frame->childregs.psw.mask &= ~PSW_MASK_RI; -#ifndef CONFIG_64BIT - /* - * save fprs to current->thread.fp_regs to merge them with - * the emulated registers and then copy the result to the child. - */ - save_fp_ctl(¤t->thread.fp_regs.fpc); - save_fp_regs(current->thread.fp_regs.fprs); - memcpy(&p->thread.fp_regs, ¤t->thread.fp_regs, - sizeof(s390_fp_regs)); - /* Set a new TLS ? */ - if (clone_flags & CLONE_SETTLS) - p->thread.acrs[0] = frame->childregs.gprs[6]; -#else /* CONFIG_64BIT */ /* Save the fpu registers to new thread structure. */ save_fp_ctl(&p->thread.fp_regs.fpc); save_fp_regs(p->thread.fp_regs.fprs); @@ -172,15 +157,13 @@ int copy_thread(unsigned long clone_flags, unsigned long new_stackp, p->thread.acrs[1] = (unsigned int)tls; } } -#endif /* CONFIG_64BIT */ return 0; } asmlinkage void execve_tail(void) { current->thread.fp_regs.fpc = 0; - if (MACHINE_HAS_IEEE) - asm volatile("sfpc %0,%0" : : "d" (0)); + asm volatile("sfpc %0,%0" : : "d" (0)); } /* @@ -188,18 +171,8 @@ asmlinkage void execve_tail(void) */ int dump_fpu (struct pt_regs * regs, s390_fp_regs *fpregs) { -#ifndef CONFIG_64BIT - /* - * save fprs to current->thread.fp_regs to merge them with - * the emulated registers and then copy the result to the dump. - */ - save_fp_ctl(¤t->thread.fp_regs.fpc); - save_fp_regs(current->thread.fp_regs.fprs); - memcpy(fpregs, ¤t->thread.fp_regs, sizeof(s390_fp_regs)); -#else /* CONFIG_64BIT */ save_fp_ctl(&fpregs->fpc); save_fp_regs(fpregs->fprs); -#endif /* CONFIG_64BIT */ return 1; } EXPORT_SYMBOL(dump_fpu); diff --git a/arch/s390/kernel/ptrace.c b/arch/s390/kernel/ptrace.c index eabfb4594517..d363c9c322a1 100644 --- a/arch/s390/kernel/ptrace.c +++ b/arch/s390/kernel/ptrace.c @@ -44,7 +44,6 @@ void update_cr_regs(struct task_struct *task) struct thread_struct *thread = &task->thread; struct per_regs old, new; -#ifdef CONFIG_64BIT /* Take care of the enable/disable of transactional execution. */ if (MACHINE_HAS_TE || MACHINE_HAS_VX) { unsigned long cr, cr_new; @@ -80,7 +79,6 @@ void update_cr_regs(struct task_struct *task) __ctl_load(cr_new, 2, 2); } } -#endif /* Copy user specified PER registers */ new.control = thread->per_user.control; new.start = thread->per_user.start; @@ -93,10 +91,8 @@ void update_cr_regs(struct task_struct *task) new.control |= PER_EVENT_BRANCH; else new.control |= PER_EVENT_IFETCH; -#ifdef CONFIG_64BIT new.control |= PER_CONTROL_SUSPENSION; new.control |= PER_EVENT_TRANSACTION_END; -#endif if (test_tsk_thread_flag(task, TIF_UPROBE_SINGLESTEP)) new.control |= PER_EVENT_IFETCH; new.start = 0; @@ -146,11 +142,7 @@ void ptrace_disable(struct task_struct *task) task->thread.per_flags = 0; } -#ifndef CONFIG_64BIT -# define __ADDR_MASK 3 -#else -# define __ADDR_MASK 7 -#endif +#define __ADDR_MASK 7 static inline unsigned long __peek_user_per(struct task_struct *child, addr_t addr) @@ -223,7 +215,6 @@ static unsigned long __peek_user(struct task_struct *child, addr_t addr) * access registers are stored in the thread structure */ offset = addr - (addr_t) &dummy->regs.acrs; -#ifdef CONFIG_64BIT /* * Very special case: old & broken 64 bit gdb reading * from acrs[15]. Result is a 64 bit value. Read the @@ -232,8 +223,7 @@ static unsigned long __peek_user(struct task_struct *child, addr_t addr) if (addr == (addr_t) &dummy->regs.acrs[15]) tmp = ((unsigned long) child->thread.acrs[15]) << 32; else -#endif - tmp = *(addr_t *)((addr_t) &child->thread.acrs + offset); + tmp = *(addr_t *)((addr_t) &child->thread.acrs + offset); } else if (addr == (addr_t) &dummy->regs.orig_gpr2) { /* @@ -261,12 +251,10 @@ static unsigned long __peek_user(struct task_struct *child, addr_t addr) * or the child->thread.vxrs array */ offset = addr - (addr_t) &dummy->regs.fp_regs.fprs; -#ifdef CONFIG_64BIT if (child->thread.vxrs) tmp = *(addr_t *) ((addr_t) child->thread.vxrs + 2*offset); else -#endif tmp = *(addr_t *) ((addr_t) &child->thread.fp_regs.fprs + offset); @@ -293,11 +281,9 @@ peek_user(struct task_struct *child, addr_t addr, addr_t data) * an alignment of 4. Programmers from hell... */ mask = __ADDR_MASK; -#ifdef CONFIG_64BIT if (addr >= (addr_t) &((struct user *) NULL)->regs.acrs && addr < (addr_t) &((struct user *) NULL)->regs.orig_gpr2) mask = 3; -#endif if ((addr & mask) || addr > sizeof(struct user) - __ADDR_MASK) return -EIO; @@ -370,7 +356,6 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) * access registers are stored in the thread structure */ offset = addr - (addr_t) &dummy->regs.acrs; -#ifdef CONFIG_64BIT /* * Very special case: old & broken 64 bit gdb writing * to acrs[15] with a 64 bit value. Ignore the lower @@ -380,8 +365,7 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) if (addr == (addr_t) &dummy->regs.acrs[15]) child->thread.acrs[15] = (unsigned int) (data >> 32); else -#endif - *(addr_t *)((addr_t) &child->thread.acrs + offset) = data; + *(addr_t *)((addr_t) &child->thread.acrs + offset) = data; } else if (addr == (addr_t) &dummy->regs.orig_gpr2) { /* @@ -411,12 +395,10 @@ static int __poke_user(struct task_struct *child, addr_t addr, addr_t data) * or the child->thread.vxrs array */ offset = addr - (addr_t) &dummy->regs.fp_regs.fprs; -#ifdef CONFIG_64BIT if (child->thread.vxrs) *(addr_t *)((addr_t) child->thread.vxrs + 2*offset) = data; else -#endif *(addr_t *)((addr_t) &child->thread.fp_regs.fprs + offset) = data; @@ -441,11 +423,9 @@ static int poke_user(struct task_struct *child, addr_t addr, addr_t data) * an alignment of 4. Programmers from hell indeed... */ mask = __ADDR_MASK; -#ifdef CONFIG_64BIT if (addr >= (addr_t) &((struct user *) NULL)->regs.acrs && addr < (addr_t) &((struct user *) NULL)->regs.orig_gpr2) mask = 3; -#endif if ((addr & mask) || addr > sizeof(struct user) - __ADDR_MASK) return -EIO; @@ -649,12 +629,10 @@ static u32 __peek_user_compat(struct task_struct *child, addr_t addr) * or the child->thread.vxrs array */ offset = addr - (addr_t) &dummy32->regs.fp_regs.fprs; -#ifdef CONFIG_64BIT if (child->thread.vxrs) tmp = *(__u32 *) ((addr_t) child->thread.vxrs + 2*offset); else -#endif tmp = *(__u32 *) ((addr_t) &child->thread.fp_regs.fprs + offset); @@ -776,12 +754,10 @@ static int __poke_user_compat(struct task_struct *child, * or the child->thread.vxrs array */ offset = addr - (addr_t) &dummy32->regs.fp_regs.fprs; -#ifdef CONFIG_64BIT if (child->thread.vxrs) *(__u32 *)((addr_t) child->thread.vxrs + 2*offset) = tmp; else -#endif *(__u32 *)((addr_t) &child->thread.fp_regs.fprs + offset) = tmp; @@ -979,16 +955,13 @@ static int s390_fpregs_get(struct task_struct *target, if (target == current) { save_fp_ctl(&target->thread.fp_regs.fpc); save_fp_regs(target->thread.fp_regs.fprs); - } -#ifdef CONFIG_64BIT - else if (target->thread.vxrs) { + } else if (target->thread.vxrs) { int i; for (i = 0; i < __NUM_VXRS_LOW; i++) target->thread.fp_regs.fprs[i] = *(freg_t *)(target->thread.vxrs + i); } -#endif return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &target->thread.fp_regs, 0, -1); } @@ -1026,23 +999,18 @@ static int s390_fpregs_set(struct task_struct *target, if (target == current) { restore_fp_ctl(&target->thread.fp_regs.fpc); restore_fp_regs(target->thread.fp_regs.fprs); - } -#ifdef CONFIG_64BIT - else if (target->thread.vxrs) { + } else if (target->thread.vxrs) { int i; for (i = 0; i < __NUM_VXRS_LOW; i++) *(freg_t *)(target->thread.vxrs + i) = target->thread.fp_regs.fprs[i]; } -#endif } return rc; } -#ifdef CONFIG_64BIT - static int s390_last_break_get(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, @@ -1182,8 +1150,6 @@ static int s390_vxrs_high_set(struct task_struct *target, return rc; } -#endif - static int s390_system_call_get(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, @@ -1229,7 +1195,6 @@ static const struct user_regset s390_regsets[] = { .get = s390_system_call_get, .set = s390_system_call_set, }, -#ifdef CONFIG_64BIT { .core_note_type = NT_S390_LAST_BREAK, .n = 1, @@ -1262,7 +1227,6 @@ static const struct user_regset s390_regsets[] = { .get = s390_vxrs_high_get, .set = s390_vxrs_high_set, }, -#endif }; static const struct user_regset_view user_s390_view = { diff --git a/arch/s390/kernel/reipl.S b/arch/s390/kernel/reipl.S index dd8016b0477e..52aab0bd84f8 100644 --- a/arch/s390/kernel/reipl.S +++ b/arch/s390/kernel/reipl.S @@ -1,7 +1,7 @@ /* - * S390 version - * Copyright IBM Corp. 2000 - * Author(s): Holger Smolinski (Holger.Smolinski@de.ibm.com) + * Copyright IBM Corp 2000, 2011 + * Author(s): Holger Smolinski <Holger.Smolinski@de.ibm.com>, + * Denis Joseph Barrow, */ #include <linux/linkage.h> @@ -9,43 +9,90 @@ #include <asm/sigp.h> # -# store_status: Empty implementation until kdump is supported on 31 bit +# store_status +# +# Prerequisites to run this function: +# - Prefix register is set to zero +# - Original prefix register is stored in "dump_prefix_page" +# - Lowcore protection is off # ENTRY(store_status) - br %r14 + /* Save register one and load save area base */ + stg %r1,__LC_SAVE_AREA_RESTART + lghi %r1,SAVE_AREA_BASE + /* General purpose registers */ + stmg %r0,%r15,__LC_GPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + lg %r2,__LC_SAVE_AREA_RESTART + stg %r2,__LC_GPREGS_SAVE_AREA-SAVE_AREA_BASE+8(%r1) + /* Control registers */ + stctg %c0,%c15,__LC_CREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Access registers */ + stam %a0,%a15,__LC_AREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Floating point registers */ + std %f0, 0x00 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f1, 0x08 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f2, 0x10 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f3, 0x18 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f4, 0x20 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f5, 0x28 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f6, 0x30 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f7, 0x38 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f8, 0x40 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f9, 0x48 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f10,0x50 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f11,0x58 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f12,0x60 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f13,0x68 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f14,0x70 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + std %f15,0x78 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Floating point control register */ + stfpc __LC_FP_CREG_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* CPU timer */ + stpt __LC_CPU_TIMER_SAVE_AREA-SAVE_AREA_BASE(%r1) + /* Saved prefix register */ + larl %r2,dump_prefix_page + mvc __LC_PREFIX_SAVE_AREA-SAVE_AREA_BASE(4,%r1),0(%r2) + /* Clock comparator - seven bytes */ + larl %r2,.Lclkcmp + stckc 0(%r2) + mvc __LC_CLOCK_COMP_SAVE_AREA-SAVE_AREA_BASE + 1(7,%r1),1(%r2) + /* Program status word */ + epsw %r2,%r3 + st %r2,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 0(%r1) + st %r3,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 4(%r1) + larl %r2,store_status + stg %r2,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 8(%r1) + br %r14 + + .section .bss + .align 8 +.Lclkcmp: .quad 0x0000000000000000 + .previous # # do_reipl_asm # Parameter: r2 = schid of reipl device # + ENTRY(do_reipl_asm) basr %r13,0 -.Lpg0: lpsw .Lnewpsw-.Lpg0(%r13) -.Lpg1: # do store status of all registers +.Lpg0: lpswe .Lnewpsw-.Lpg0(%r13) +.Lpg1: brasl %r14,store_status - stm %r0,%r15,__LC_GPREGS_SAVE_AREA - stctl %c0,%c15,__LC_CREGS_SAVE_AREA - stam %a0,%a15,__LC_AREGS_SAVE_AREA - l %r10,.Ldump_pfx-.Lpg0(%r13) - mvc __LC_PREFIX_SAVE_AREA(4),0(%r10) - stckc .Lclkcmp-.Lpg0(%r13) - mvc __LC_CLOCK_COMP_SAVE_AREA(8),.Lclkcmp-.Lpg0(%r13) - stpt __LC_CPU_TIMER_SAVE_AREA - st %r13, __LC_PSW_SAVE_AREA+4 - lctl %c6,%c6,.Lall-.Lpg0(%r13) - lr %r1,%r2 - mvc __LC_PGM_NEW_PSW(8),.Lpcnew-.Lpg0(%r13) + lctlg %c6,%c6,.Lall-.Lpg0(%r13) + lgr %r1,%r2 + mvc __LC_PGM_NEW_PSW(16),.Lpcnew-.Lpg0(%r13) stsch .Lschib-.Lpg0(%r13) oi .Lschib+5-.Lpg0(%r13),0x84 -.Lecs: xi .Lschib+27-.Lpg0(%r13),0x01 +.Lecs: xi .Lschib+27-.Lpg0(%r13),0x01 msch .Lschib-.Lpg0(%r13) - lhi %r0,5 + lghi %r0,5 .Lssch: ssch .Liplorb-.Lpg0(%r13) jz .L001 brct %r0,.Lssch bas %r14,.Ldisab-.Lpg0(%r13) -.L001: mvc __LC_IO_NEW_PSW(8),.Lionew-.Lpg0(%r13) -.Ltpi: lpsw .Lwaitpsw-.Lpg0(%r13) +.L001: mvc __LC_IO_NEW_PSW(16),.Lionew-.Lpg0(%r13) +.Ltpi: lpswe .Lwaitpsw-.Lpg0(%r13) .Lcont: c %r1,__LC_SUBCHANNEL_ID jnz .Ltpi clc __LC_IO_INT_PARM(4),.Liplorb-.Lpg0(%r13) @@ -58,20 +105,36 @@ ENTRY(do_reipl_asm) jz .L003 bas %r14,.Ldisab-.Lpg0(%r13) .L003: st %r1,__LC_SUBCHANNEL_ID + lhi %r1,0 # mode 0 = esa + slr %r0,%r0 # set cpuid to zero + sigp %r1,%r0,SIGP_SET_ARCHITECTURE # switch to esa mode lpsw 0 - sigp 0,0,SIGP_RESTART -.Ldisab: st %r14,.Ldispsw+4-.Lpg0(%r13) - lpsw .Ldispsw-.Lpg0(%r13) +.Ldisab: sll %r14,1 + srl %r14,1 # need to kill hi bit to avoid specification exceptions. + st %r14,.Ldispsw+12-.Lpg0(%r13) + lpswe .Ldispsw-.Lpg0(%r13) .align 8 -.Lclkcmp: .quad 0x0000000000000000 -.Lall: .long 0xff000000 -.Ldump_pfx: .long dump_prefix_page - .align 8 -.Lnewpsw: .long 0x00080000,0x80000000+.Lpg1 -.Lpcnew: .long 0x00080000,0x80000000+.Lecs -.Lionew: .long 0x00080000,0x80000000+.Lcont -.Lwaitpsw: .long 0x020a0000,0x00000000+.Ltpi -.Ldispsw: .long 0x000a0000,0x00000000 +.Lall: .quad 0x00000000ff000000 + .align 16 +/* + * These addresses have to be 31 bit otherwise + * the sigp will throw a specifcation exception + * when switching to ESA mode as bit 31 be set + * in the ESA psw. + * Bit 31 of the addresses has to be 0 for the + * 31bit lpswe instruction a fact they appear to have + * omitted from the pop. + */ +.Lnewpsw: .quad 0x0000000080000000 + .quad .Lpg1 +.Lpcnew: .quad 0x0000000080000000 + .quad .Lecs +.Lionew: .quad 0x0000000080000000 + .quad .Lcont +.Lwaitpsw: .quad 0x0202000080000000 + .quad .Ltpi +.Ldispsw: .quad 0x0002000080000000 + .quad 0x0000000000000000 .Liplccws: .long 0x02000000,0x60000018 .long 0x08000008,0x20000001 .Liplorb: .long 0x0049504c,0x0040ff80 diff --git a/arch/s390/kernel/reipl64.S b/arch/s390/kernel/reipl64.S deleted file mode 100644 index dc3b1273c4dc..000000000000 --- a/arch/s390/kernel/reipl64.S +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright IBM Corp 2000, 2011 - * Author(s): Holger Smolinski <Holger.Smolinski@de.ibm.com>, - * Denis Joseph Barrow, - */ - -#include <linux/linkage.h> -#include <asm/asm-offsets.h> -#include <asm/sigp.h> - -# -# store_status -# -# Prerequisites to run this function: -# - Prefix register is set to zero -# - Original prefix register is stored in "dump_prefix_page" -# - Lowcore protection is off -# -ENTRY(store_status) - /* Save register one and load save area base */ - stg %r1,__LC_SAVE_AREA_RESTART - lghi %r1,SAVE_AREA_BASE - /* General purpose registers */ - stmg %r0,%r15,__LC_GPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - lg %r2,__LC_SAVE_AREA_RESTART - stg %r2,__LC_GPREGS_SAVE_AREA-SAVE_AREA_BASE+8(%r1) - /* Control registers */ - stctg %c0,%c15,__LC_CREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - /* Access registers */ - stam %a0,%a15,__LC_AREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - /* Floating point registers */ - std %f0, 0x00 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f1, 0x08 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f2, 0x10 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f3, 0x18 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f4, 0x20 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f5, 0x28 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f6, 0x30 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f7, 0x38 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f8, 0x40 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f9, 0x48 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f10,0x50 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f11,0x58 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f12,0x60 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f13,0x68 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f14,0x70 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - std %f15,0x78 + __LC_FPREGS_SAVE_AREA-SAVE_AREA_BASE(%r1) - /* Floating point control register */ - stfpc __LC_FP_CREG_SAVE_AREA-SAVE_AREA_BASE(%r1) - /* CPU timer */ - stpt __LC_CPU_TIMER_SAVE_AREA-SAVE_AREA_BASE(%r1) - /* Saved prefix register */ - larl %r2,dump_prefix_page - mvc __LC_PREFIX_SAVE_AREA-SAVE_AREA_BASE(4,%r1),0(%r2) - /* Clock comparator - seven bytes */ - larl %r2,.Lclkcmp - stckc 0(%r2) - mvc __LC_CLOCK_COMP_SAVE_AREA-SAVE_AREA_BASE + 1(7,%r1),1(%r2) - /* Program status word */ - epsw %r2,%r3 - st %r2,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 0(%r1) - st %r3,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 4(%r1) - larl %r2,store_status - stg %r2,__LC_PSW_SAVE_AREA-SAVE_AREA_BASE + 8(%r1) - br %r14 - - .section .bss - .align 8 -.Lclkcmp: .quad 0x0000000000000000 - .previous - -# -# do_reipl_asm -# Parameter: r2 = schid of reipl device -# - -ENTRY(do_reipl_asm) - basr %r13,0 -.Lpg0: lpswe .Lnewpsw-.Lpg0(%r13) -.Lpg1: brasl %r14,store_status - - lctlg %c6,%c6,.Lall-.Lpg0(%r13) - lgr %r1,%r2 - mvc __LC_PGM_NEW_PSW(16),.Lpcnew-.Lpg0(%r13) - stsch .Lschib-.Lpg0(%r13) - oi .Lschib+5-.Lpg0(%r13),0x84 -.Lecs: xi .Lschib+27-.Lpg0(%r13),0x01 - msch .Lschib-.Lpg0(%r13) - lghi %r0,5 -.Lssch: ssch .Liplorb-.Lpg0(%r13) - jz .L001 - brct %r0,.Lssch - bas %r14,.Ldisab-.Lpg0(%r13) -.L001: mvc __LC_IO_NEW_PSW(16),.Lionew-.Lpg0(%r13) -.Ltpi: lpswe .Lwaitpsw-.Lpg0(%r13) -.Lcont: c %r1,__LC_SUBCHANNEL_ID - jnz .Ltpi - clc __LC_IO_INT_PARM(4),.Liplorb-.Lpg0(%r13) - jnz .Ltpi - tsch .Liplirb-.Lpg0(%r13) - tm .Liplirb+9-.Lpg0(%r13),0xbf - jz .L002 - bas %r14,.Ldisab-.Lpg0(%r13) -.L002: tm .Liplirb+8-.Lpg0(%r13),0xf3 - jz .L003 - bas %r14,.Ldisab-.Lpg0(%r13) -.L003: st %r1,__LC_SUBCHANNEL_ID - lhi %r1,0 # mode 0 = esa - slr %r0,%r0 # set cpuid to zero - sigp %r1,%r0,SIGP_SET_ARCHITECTURE # switch to esa mode - lpsw 0 -.Ldisab: sll %r14,1 - srl %r14,1 # need to kill hi bit to avoid specification exceptions. - st %r14,.Ldispsw+12-.Lpg0(%r13) - lpswe .Ldispsw-.Lpg0(%r13) - .align 8 -.Lall: .quad 0x00000000ff000000 - .align 16 -/* - * These addresses have to be 31 bit otherwise - * the sigp will throw a specifcation exception - * when switching to ESA mode as bit 31 be set - * in the ESA psw. - * Bit 31 of the addresses has to be 0 for the - * 31bit lpswe instruction a fact they appear to have - * omitted from the pop. - */ -.Lnewpsw: .quad 0x0000000080000000 - .quad .Lpg1 -.Lpcnew: .quad 0x0000000080000000 - .quad .Lecs -.Lionew: .quad 0x0000000080000000 - .quad .Lcont -.Lwaitpsw: .quad 0x0202000080000000 - .quad .Ltpi -.Ldispsw: .quad 0x0002000080000000 - .quad 0x0000000000000000 -.Liplccws: .long 0x02000000,0x60000018 - .long 0x08000008,0x20000001 -.Liplorb: .long 0x0049504c,0x0040ff80 - .long 0x00000000+.Liplccws -.Lschib: .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 -.Liplirb: .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 - .long 0x00000000,0x00000000 diff --git a/arch/s390/kernel/relocate_kernel.S b/arch/s390/kernel/relocate_kernel.S index f4e6f20e117a..cfac28330b03 100644 --- a/arch/s390/kernel/relocate_kernel.S +++ b/arch/s390/kernel/relocate_kernel.S @@ -19,7 +19,8 @@ * %r7 = PAGE_SIZE * %r8 holds the source address * %r9 = PAGE_SIZE - * %r10 is a page mask + * + * 0xf000 is a page_mask */ .text @@ -27,46 +28,47 @@ ENTRY(relocate_kernel) basr %r13,0 # base address .base: stnsm sys_msk-.base(%r13),0xfb # disable DAT - stctl %c0,%c15,ctlregs-.base(%r13) - stm %r0,%r15,gprregs-.base(%r13) + stctg %c0,%c15,ctlregs-.base(%r13) + stmg %r0,%r15,gprregs-.base(%r13) + lghi %r0,3 + sllg %r0,%r0,31 + stg %r0,0x1d0(%r0) + la %r0,.back_pgm-.base(%r13) + stg %r0,0x1d8(%r0) la %r1,load_psw-.base(%r13) mvc 0(8,%r0),0(%r1) la %r0,.back-.base(%r13) st %r0,4(%r0) oi 4(%r0),0x80 - mvc 0x68(8,%r0),0(%r1) - la %r0,.back_pgm-.base(%r13) - st %r0,0x6c(%r0) - oi 0x6c(%r0),0x80 - lhi %r0,0 + lghi %r0,0 diag %r0,%r0,0x308 .back: + lhi %r1,1 # mode 1 = esame + sigp %r1,%r0,SIGP_SET_ARCHITECTURE # switch to esame mode + sam64 # switch to 64 bit addressing mode basr %r13,0 .back_base: oi have_diag308-.back_base(%r13),0x01 - lctl %c0,%c15,ctlregs-.back_base(%r13) - lm %r0,%r15,gprregs-.back_base(%r13) - j .start_reloc + lctlg %c0,%c15,ctlregs-.back_base(%r13) + lmg %r0,%r15,gprregs-.back_base(%r13) + j .top .back_pgm: - lm %r0,%r15,gprregs-.base(%r13) - .start_reloc: - lhi %r10,-1 # preparing the mask - sll %r10,12 # shift it such that it becomes 0xf000 + lmg %r0,%r15,gprregs-.base(%r13) .top: - lhi %r7,4096 # load PAGE_SIZE in r7 - lhi %r9,4096 # load PAGE_SIZE in r9 - l %r5,0(%r2) # read another word for indirection page - ahi %r2,4 # increment pointer + lghi %r7,4096 # load PAGE_SIZE in r7 + lghi %r9,4096 # load PAGE_SIZE in r9 + lg %r5,0(%r2) # read another word for indirection page + aghi %r2,8 # increment pointer tml %r5,0x1 # is it a destination page? je .indir_check # NO, goto "indir_check" - lr %r6,%r5 # r6 = r5 - nr %r6,%r10 # mask it out and... + lgr %r6,%r5 # r6 = r5 + nill %r6,0xf000 # mask it out and... j .top # ...next iteration .indir_check: tml %r5,0x2 # is it a indirection page? je .done_test # NO, goto "done_test" - nr %r5,%r10 # YES, mask out, - lr %r2,%r5 # move it into the right register, + nill %r5,0xf000 # YES, mask out, + lgr %r2,%r5 # move it into the right register, j .top # and read next... .done_test: tml %r5,0x4 # is it the done indicator? @@ -75,13 +77,13 @@ ENTRY(relocate_kernel) .source_test: tml %r5,0x8 # it should be a source indicator... je .top # NO, ignore it... - lr %r8,%r5 # r8 = r5 - nr %r8,%r10 # masking + lgr %r8,%r5 # r8 = r5 + nill %r8,0xf000 # masking 0: mvcle %r6,%r8,0x0 # copy PAGE_SIZE bytes from r8 to r6 - pad with 0 jo 0b j .top .done: - sr %r0,%r0 # clear register r0 + sgr %r0,%r0 # clear register r0 la %r4,load_psw-.base(%r13) # load psw-address into the register o %r3,4(%r4) # or load address into psw st %r3,4(%r4) @@ -90,8 +92,9 @@ ENTRY(relocate_kernel) jno .no_diag308 diag %r0,%r0,0x308 .no_diag308: - sr %r1,%r1 # clear %r1 - sr %r2,%r2 # clear %r2 + sam31 # 31 bit mode + sr %r1,%r1 # erase register r1 + sr %r2,%r2 # erase register r2 sigp %r1,%r2,SIGP_SET_ARCHITECTURE # set cpuid to zero lpsw 0 # hopefully start new kernel... @@ -102,11 +105,11 @@ ENTRY(relocate_kernel) .quad 0 ctlregs: .rept 16 - .long 0 + .quad 0 .endr gprregs: .rept 16 - .long 0 + .quad 0 .endr have_diag308: .byte 0 diff --git a/arch/s390/kernel/relocate_kernel64.S b/arch/s390/kernel/relocate_kernel64.S deleted file mode 100644 index cfac28330b03..000000000000 --- a/arch/s390/kernel/relocate_kernel64.S +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright IBM Corp. 2005 - * - * Author(s): Rolf Adelsberger, - * Heiko Carstens <heiko.carstens@de.ibm.com> - * - */ - -#include <linux/linkage.h> -#include <asm/sigp.h> - -/* - * moves the new kernel to its destination... - * %r2 = pointer to first kimage_entry_t - * %r3 = start address - where to jump to after the job is done... - * - * %r5 will be used as temp. storage - * %r6 holds the destination address - * %r7 = PAGE_SIZE - * %r8 holds the source address - * %r9 = PAGE_SIZE - * - * 0xf000 is a page_mask - */ - - .text -ENTRY(relocate_kernel) - basr %r13,0 # base address - .base: - stnsm sys_msk-.base(%r13),0xfb # disable DAT - stctg %c0,%c15,ctlregs-.base(%r13) - stmg %r0,%r15,gprregs-.base(%r13) - lghi %r0,3 - sllg %r0,%r0,31 - stg %r0,0x1d0(%r0) - la %r0,.back_pgm-.base(%r13) - stg %r0,0x1d8(%r0) - la %r1,load_psw-.base(%r13) - mvc 0(8,%r0),0(%r1) - la %r0,.back-.base(%r13) - st %r0,4(%r0) - oi 4(%r0),0x80 - lghi %r0,0 - diag %r0,%r0,0x308 - .back: - lhi %r1,1 # mode 1 = esame - sigp %r1,%r0,SIGP_SET_ARCHITECTURE # switch to esame mode - sam64 # switch to 64 bit addressing mode - basr %r13,0 - .back_base: - oi have_diag308-.back_base(%r13),0x01 - lctlg %c0,%c15,ctlregs-.back_base(%r13) - lmg %r0,%r15,gprregs-.back_base(%r13) - j .top - .back_pgm: - lmg %r0,%r15,gprregs-.base(%r13) - .top: - lghi %r7,4096 # load PAGE_SIZE in r7 - lghi %r9,4096 # load PAGE_SIZE in r9 - lg %r5,0(%r2) # read another word for indirection page - aghi %r2,8 # increment pointer - tml %r5,0x1 # is it a destination page? - je .indir_check # NO, goto "indir_check" - lgr %r6,%r5 # r6 = r5 - nill %r6,0xf000 # mask it out and... - j .top # ...next iteration - .indir_check: - tml %r5,0x2 # is it a indirection page? - je .done_test # NO, goto "done_test" - nill %r5,0xf000 # YES, mask out, - lgr %r2,%r5 # move it into the right register, - j .top # and read next... - .done_test: - tml %r5,0x4 # is it the done indicator? - je .source_test # NO! Well, then it should be the source indicator... - j .done # ok, lets finish it here... - .source_test: - tml %r5,0x8 # it should be a source indicator... - je .top # NO, ignore it... - lgr %r8,%r5 # r8 = r5 - nill %r8,0xf000 # masking - 0: mvcle %r6,%r8,0x0 # copy PAGE_SIZE bytes from r8 to r6 - pad with 0 - jo 0b - j .top - .done: - sgr %r0,%r0 # clear register r0 - la %r4,load_psw-.base(%r13) # load psw-address into the register - o %r3,4(%r4) # or load address into psw - st %r3,4(%r4) - mvc 0(8,%r0),0(%r4) # copy psw to absolute address 0 - tm have_diag308-.base(%r13),0x01 - jno .no_diag308 - diag %r0,%r0,0x308 - .no_diag308: - sam31 # 31 bit mode - sr %r1,%r1 # erase register r1 - sr %r2,%r2 # erase register r2 - sigp %r1,%r2,SIGP_SET_ARCHITECTURE # set cpuid to zero - lpsw 0 # hopefully start new kernel... - - .align 8 - load_psw: - .long 0x00080000,0x80000000 - sys_msk: - .quad 0 - ctlregs: - .rept 16 - .quad 0 - .endr - gprregs: - .rept 16 - .quad 0 - .endr - have_diag308: - .byte 0 - .align 8 - relocate_kernel_end: - .align 8 - .globl relocate_kernel_len - relocate_kernel_len: - .quad relocate_kernel_end - relocate_kernel diff --git a/arch/s390/kernel/sclp.S b/arch/s390/kernel/sclp.S index 7e77e03378f3..43c3169ea49c 100644 --- a/arch/s390/kernel/sclp.S +++ b/arch/s390/kernel/sclp.S @@ -36,21 +36,17 @@ _sclp_wait_int: ahi %r15,-96 # create stack frame la %r8,LC_EXT_NEW_PSW # register int handler la %r9,.LextpswS1-.LbaseS1(%r13) -#ifdef CONFIG_64BIT tm LC_AR_MODE_ID,1 jno .Lesa1 la %r8,LC_EXT_NEW_PSW_64 # register int handler 64 bit la %r9,.LextpswS1_64-.LbaseS1(%r13) .Lesa1: -#endif mvc .LoldpswS1-.LbaseS1(16,%r13),0(%r8) mvc 0(16,%r8),0(%r9) -#ifdef CONFIG_64BIT epsw %r6,%r7 # set current addressing mode nill %r6,0x1 # in new psw (31 or 64 bit mode) nilh %r7,0x8000 stm %r6,%r7,0(%r8) -#endif lhi %r6,0x0200 # cr mask for ext int (cr0.54) ltr %r2,%r2 jz .LsetctS1 @@ -92,10 +88,8 @@ _sclp_wait_int: .long 0, 0, 0, 0 # old ext int PSW .LextpswS1: .long 0x00080000, 0x80000000+.LwaitS1 # PSW to handle ext int -#ifdef CONFIG_64BIT .LextpswS1_64: .quad 0, .LwaitS1 # PSW to handle ext int, 64 bit -#endif .LwaitpswS1: .long 0x010a0000, 0x00000000+.LloopS1 # PSW to wait for ext int .LtimeS1: @@ -272,13 +266,11 @@ _sclp_print: ENTRY(_sclp_print_early) stm %r6,%r15,24(%r15) # save registers ahi %r15,-96 # create stack frame -#ifdef CONFIG_64BIT tm LC_AR_MODE_ID,1 jno .Lesa2 ahi %r15,-80 stmh %r6,%r15,96(%r15) # store upper register halves .Lesa2: -#endif lr %r10,%r2 # save string pointer lhi %r2,0 bras %r14,_sclp_setup # enable console @@ -291,14 +283,12 @@ ENTRY(_sclp_print_early) lhi %r2,1 bras %r14,_sclp_setup # disable console .LendS5: -#ifdef CONFIG_64BIT tm LC_AR_MODE_ID,1 jno .Lesa3 lgfr %r2,%r2 # sign extend return value lmh %r6,%r15,96(%r15) # restore upper register halves ahi %r15,80 .Lesa3: -#endif lm %r6,%r15,120(%r15) # restore registers br %r14 diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index a5ea8bc17cb3..7262fe438c99 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -92,10 +92,8 @@ EXPORT_SYMBOL(VMALLOC_END); struct page *vmemmap; EXPORT_SYMBOL(vmemmap); -#ifdef CONFIG_64BIT unsigned long MODULES_VADDR; unsigned long MODULES_END; -#endif /* An array with a pointer to the lowcore of every CPU. */ struct _lowcore *lowcore_ptr[NR_CPUS]; @@ -334,19 +332,10 @@ static void __init setup_lowcore(void) lc->stfl_fac_list = S390_lowcore.stfl_fac_list; memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list, MAX_FACILITY_BIT/8); -#ifndef CONFIG_64BIT - if (MACHINE_HAS_IEEE) { - lc->extended_save_area_addr = (__u32) - __alloc_bootmem_low(PAGE_SIZE, PAGE_SIZE, 0); - /* enable extended save area */ - __ctl_set_bit(14, 29); - } -#else if (MACHINE_HAS_VX) lc->vector_save_area_addr = (unsigned long) &lc->vector_save_area; lc->vdso_per_cpu_data = (unsigned long) &lc->paste[0]; -#endif lc->sync_enter_timer = S390_lowcore.sync_enter_timer; lc->async_enter_timer = S390_lowcore.async_enter_timer; lc->exit_timer = S390_lowcore.exit_timer; @@ -450,7 +439,6 @@ static void __init setup_memory_end(void) unsigned long vmax, vmalloc_size, tmp; /* Choose kernel address space layout: 2, 3, or 4 levels. */ -#ifdef CONFIG_64BIT vmalloc_size = VMALLOC_END ?: (128UL << 30) - MODULES_LEN; tmp = (memory_end ?: max_physmem_end) / PAGE_SIZE; tmp = tmp * (sizeof(struct page) + PAGE_SIZE); @@ -462,12 +450,6 @@ static void __init setup_memory_end(void) MODULES_END = vmax; MODULES_VADDR = MODULES_END - MODULES_LEN; VMALLOC_END = MODULES_VADDR; -#else - vmalloc_size = VMALLOC_END ?: 96UL << 20; - vmax = 1UL << 31; /* 2-level kernel page table */ - /* vmalloc area is at the end of the kernel address space. */ - VMALLOC_END = vmax; -#endif VMALLOC_START = vmax - vmalloc_size; /* Split remaining virtual space between 1:1 mapping & vmemmap array */ @@ -754,7 +736,6 @@ static void __init setup_hwcaps(void) if (MACHINE_HAS_HPAGE) elf_hwcap |= HWCAP_S390_HPAGE; -#if defined(CONFIG_64BIT) /* * 64-bit register support for 31-bit processes * HWCAP_S390_HIGH_GPRS is bit 9. @@ -772,22 +753,15 @@ static void __init setup_hwcaps(void) */ if (test_facility(129)) elf_hwcap |= HWCAP_S390_VXRS; -#endif - get_cpu_id(&cpu_id); add_device_randomness(&cpu_id, sizeof(cpu_id)); switch (cpu_id.machine) { case 0x9672: -#if !defined(CONFIG_64BIT) - default: /* Use "g5" as default for 31 bit kernels. */ -#endif strcpy(elf_platform, "g5"); break; case 0x2064: case 0x2066: -#if defined(CONFIG_64BIT) default: /* Use "z900" as default for 64 bit kernels. */ -#endif strcpy(elf_platform, "z900"); break; case 0x2084: @@ -839,19 +813,6 @@ void __init setup_arch(char **cmdline_p) /* * print what head.S has found out about the machine */ -#ifndef CONFIG_64BIT - if (MACHINE_IS_VM) - pr_info("Linux is running as a z/VM " - "guest operating system in 31-bit mode\n"); - else if (MACHINE_IS_LPAR) - pr_info("Linux is running natively in 31-bit mode\n"); - if (MACHINE_HAS_IEEE) - pr_info("The hardware system has IEEE compatible " - "floating point units\n"); - else - pr_info("The hardware system has no IEEE compatible " - "floating point units\n"); -#else /* CONFIG_64BIT */ if (MACHINE_IS_VM) pr_info("Linux is running as a z/VM " "guest operating system in 64-bit mode\n"); @@ -859,7 +820,6 @@ void __init setup_arch(char **cmdline_p) pr_info("Linux is running under KVM in 64-bit mode\n"); else if (MACHINE_IS_LPAR) pr_info("Linux is running natively in 64-bit mode\n"); -#endif /* CONFIG_64BIT */ /* Have one command line that is parsed and saved in /proc/cmdline */ /* boot_command_line has been already set up in early.c */ @@ -930,35 +890,3 @@ void __init setup_arch(char **cmdline_p) /* Add system specific data to the random pool */ setup_randomness(); } - -#ifdef CONFIG_32BIT -static int no_removal_warning __initdata; - -static int __init parse_no_removal_warning(char *str) -{ - no_removal_warning = 1; - return 0; -} -__setup("no_removal_warning", parse_no_removal_warning); - -static int __init removal_warning(void) -{ - if (no_removal_warning) - return 0; - printk(KERN_ALERT "\n\n"); - printk(KERN_CONT "Warning - you are using a 31 bit kernel!\n\n"); - printk(KERN_CONT "We plan to remove 31 bit kernel support from the kernel sources in March 2015.\n"); - printk(KERN_CONT "Currently we assume that nobody is using the 31 bit kernel on old 31 bit\n"); - printk(KERN_CONT "hardware anymore. If you think that the code should not be removed and also\n"); - printk(KERN_CONT "future versions of the Linux kernel should be able to run in 31 bit mode\n"); - printk(KERN_CONT "please let us know. Please write to:\n"); - printk(KERN_CONT "linux390@de.ibm.com (mail address) and/or\n"); - printk(KERN_CONT "linux-s390@vger.kernel.org (mailing list).\n\n"); - printk(KERN_CONT "Thank you!\n\n"); - printk(KERN_CONT "If this kernel runs on a 64 bit machine you may consider using a 64 bit kernel.\n"); - printk(KERN_CONT "This message can be disabled with the \"no_removal_warning\" kernel parameter.\n"); - schedule_timeout_uninterruptible(300 * HZ); - return 0; -} -early_initcall(removal_warning); -#endif diff --git a/arch/s390/kernel/signal.c b/arch/s390/kernel/signal.c index b3ae6f70c6d6..c551f22ce066 100644 --- a/arch/s390/kernel/signal.c +++ b/arch/s390/kernel/signal.c @@ -106,7 +106,6 @@ static void store_sigregs(void) { save_access_regs(current->thread.acrs); save_fp_ctl(¤t->thread.fp_regs.fpc); -#ifdef CONFIG_64BIT if (current->thread.vxrs) { int i; @@ -115,7 +114,6 @@ static void store_sigregs(void) current->thread.fp_regs.fprs[i] = *(freg_t *)(current->thread.vxrs + i); } else -#endif save_fp_regs(current->thread.fp_regs.fprs); } @@ -124,7 +122,6 @@ static void load_sigregs(void) { restore_access_regs(current->thread.acrs); /* restore_fp_ctl is done in restore_sigregs */ -#ifdef CONFIG_64BIT if (current->thread.vxrs) { int i; @@ -133,7 +130,6 @@ static void load_sigregs(void) current->thread.fp_regs.fprs[i]; restore_vx_regs(current->thread.vxrs); } else -#endif restore_fp_regs(current->thread.fp_regs.fprs); } @@ -200,7 +196,6 @@ static int restore_sigregs(struct pt_regs *regs, _sigregs __user *sregs) static int save_sigregs_ext(struct pt_regs *regs, _sigregs_ext __user *sregs_ext) { -#ifdef CONFIG_64BIT __u64 vxrs[__NUM_VXRS_LOW]; int i; @@ -215,14 +210,12 @@ static int save_sigregs_ext(struct pt_regs *regs, sizeof(sregs_ext->vxrs_high))) return -EFAULT; } -#endif return 0; } static int restore_sigregs_ext(struct pt_regs *regs, _sigregs_ext __user *sregs_ext) { -#ifdef CONFIG_64BIT __u64 vxrs[__NUM_VXRS_LOW]; int i; @@ -237,7 +230,6 @@ static int restore_sigregs_ext(struct pt_regs *regs, for (i = 0; i < __NUM_VXRS_LOW; i++) *((__u64 *)(current->thread.vxrs + i) + 1) = vxrs[i]; } -#endif return 0; } @@ -309,16 +301,6 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs * regs, size_t frame_size) return (void __user *)((sp - frame_size) & -8ul); } -static inline int map_signal(int sig) -{ - if (current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32) - return current_thread_info()->exec_domain->signal_invmap[sig]; - else - return sig; -} - static int setup_frame(int sig, struct k_sigaction *ka, sigset_t *set, struct pt_regs * regs) { @@ -386,7 +368,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, (regs->psw.mask & ~PSW_MASK_ASC); regs->psw.addr = (unsigned long) ka->sa.sa_handler | PSW_ADDR_AMODE; - regs->gprs[2] = map_signal(sig); + regs->gprs[2] = sig; regs->gprs[3] = (unsigned long) &frame->sc; /* We forgot to include these in the sigcontext. @@ -416,13 +398,11 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, * included in the signal frame on a 31-bit system. */ uc_flags = 0; -#ifdef CONFIG_64BIT if (MACHINE_HAS_VX) { frame_size += sizeof(_sigregs_ext); if (current->thread.vxrs) uc_flags |= UC_VXRS; } -#endif frame = get_sigframe(&ksig->ka, regs, frame_size); if (frame == (void __user *) -1UL) return -EFAULT; @@ -468,7 +448,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, (regs->psw.mask & ~PSW_MASK_ASC); regs->psw.addr = (unsigned long) ksig->ka.sa.sa_handler | PSW_ADDR_AMODE; - regs->gprs[2] = map_signal(ksig->sig); + regs->gprs[2] = ksig->sig; regs->gprs[3] = (unsigned long) &frame->info; regs->gprs[4] = (unsigned long) &frame->uc; regs->gprs[5] = task_thread_info(current)->last_break; diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index db8f1115a3bf..efd2c1968000 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -198,19 +198,11 @@ static int pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu) lc->panic_stack = panic_stack + PANIC_FRAME_OFFSET; lc->cpu_nr = cpu; lc->spinlock_lockval = arch_spin_lockval(cpu); -#ifndef CONFIG_64BIT - if (MACHINE_HAS_IEEE) { - lc->extended_save_area_addr = get_zeroed_page(GFP_KERNEL); - if (!lc->extended_save_area_addr) - goto out; - } -#else if (MACHINE_HAS_VX) lc->vector_save_area_addr = (unsigned long) &lc->vector_save_area; if (vdso_alloc_per_cpu(lc)) goto out; -#endif lowcore_ptr[cpu] = lc; pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, (u32)(unsigned long) lc); return 0; @@ -229,16 +221,7 @@ static void pcpu_free_lowcore(struct pcpu *pcpu) { pcpu_sigp_retry(pcpu, SIGP_SET_PREFIX, 0); lowcore_ptr[pcpu - pcpu_devices] = NULL; -#ifndef CONFIG_64BIT - if (MACHINE_HAS_IEEE) { - struct _lowcore *lc = pcpu->lowcore; - - free_page((unsigned long) lc->extended_save_area_addr); - lc->extended_save_area_addr = 0; - } -#else vdso_free_per_cpu(pcpu->lowcore); -#endif if (pcpu == &pcpu_devices[0]) return; free_page(pcpu->lowcore->panic_stack-PANIC_FRAME_OFFSET); @@ -492,22 +475,6 @@ void arch_send_call_function_single_ipi(int cpu) pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single); } -#ifndef CONFIG_64BIT -/* - * this function sends a 'purge tlb' signal to another CPU. - */ -static void smp_ptlb_callback(void *info) -{ - __tlb_flush_local(); -} - -void smp_ptlb_all(void) -{ - on_each_cpu(smp_ptlb_callback, NULL, 1); -} -EXPORT_SYMBOL(smp_ptlb_all); -#endif /* ! CONFIG_64BIT */ - /* * this function sends a 'reschedule' IPI to another CPU. * it goes straight through and wastes no time serializing @@ -851,7 +818,8 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) pcpu_prepare_secondary(pcpu, cpu); pcpu_attach_task(pcpu, tidle); pcpu_start_fn(pcpu, smp_start_secondary, NULL); - while (!cpu_online(cpu)) + /* Wait until cpu puts itself in the online & active maps */ + while (!cpu_online(cpu) || !cpu_active(cpu)) cpu_relax(); return 0; } diff --git a/arch/s390/kernel/suspend.c b/arch/s390/kernel/suspend.c index 1c4c5accd220..d3236c9e226b 100644 --- a/arch/s390/kernel/suspend.c +++ b/arch/s390/kernel/suspend.c @@ -138,6 +138,8 @@ int pfn_is_nosave(unsigned long pfn) { unsigned long nosave_begin_pfn = PFN_DOWN(__pa(&__nosave_begin)); unsigned long nosave_end_pfn = PFN_DOWN(__pa(&__nosave_end)); + unsigned long eshared_pfn = PFN_DOWN(__pa(&_eshared)) - 1; + unsigned long stext_pfn = PFN_DOWN(__pa(&_stext)); /* Always save lowcore pages (LC protection might be enabled). */ if (pfn <= LC_PAGES) @@ -145,6 +147,8 @@ int pfn_is_nosave(unsigned long pfn) if (pfn >= nosave_begin_pfn && pfn < nosave_end_pfn) return 1; /* Skip memory holes and read-only pages (NSS, DCSS, ...). */ + if (pfn >= stext_pfn && pfn <= eshared_pfn) + return ipl_info.type == IPL_TYPE_NSS ? 1 : 0; if (tprot(PFN_PHYS(pfn))) return 1; return 0; diff --git a/arch/s390/kernel/swsusp_asm64.S b/arch/s390/kernel/swsusp.S index ca6294645dd3..ca6294645dd3 100644 --- a/arch/s390/kernel/swsusp_asm64.S +++ b/arch/s390/kernel/swsusp.S diff --git a/arch/s390/kernel/sys_s390.c b/arch/s390/kernel/sys_s390.c index 23eb222c1658..f145490cce54 100644 --- a/arch/s390/kernel/sys_s390.c +++ b/arch/s390/kernel/sys_s390.c @@ -76,7 +76,6 @@ SYSCALL_DEFINE5(s390_ipc, uint, call, int, first, unsigned long, second, return sys_ipc(call, first, second, third, ptr, third); } -#ifdef CONFIG_64BIT SYSCALL_DEFINE1(s390_personality, unsigned int, personality) { unsigned int ret; @@ -90,51 +89,3 @@ SYSCALL_DEFINE1(s390_personality, unsigned int, personality) return ret; } -#endif /* CONFIG_64BIT */ - -/* - * Wrapper function for sys_fadvise64/fadvise64_64 - */ -#ifndef CONFIG_64BIT - -SYSCALL_DEFINE5(s390_fadvise64, int, fd, u32, offset_high, u32, offset_low, - size_t, len, int, advice) -{ - return sys_fadvise64(fd, (u64) offset_high << 32 | offset_low, - len, advice); -} - -struct fadvise64_64_args { - int fd; - long long offset; - long long len; - int advice; -}; - -SYSCALL_DEFINE1(s390_fadvise64_64, struct fadvise64_64_args __user *, args) -{ - struct fadvise64_64_args a; - - if ( copy_from_user(&a, args, sizeof(a)) ) - return -EFAULT; - return sys_fadvise64_64(a.fd, a.offset, a.len, a.advice); -} - -/* - * This is a wrapper to call sys_fallocate(). For 31 bit s390 the last - * 64 bit argument "len" is split into the upper and lower 32 bits. The - * system call wrapper in the user space loads the value to %r6/%r7. - * The code in entry.S keeps the values in %r2 - %r6 where they are and - * stores %r7 to 96(%r15). But the standard C linkage requires that - * the whole 64 bit value for len is stored on the stack and doesn't - * use %r6 at all. So s390_fallocate has to convert the arguments from - * %r2: fd, %r3: mode, %r4/%r5: offset, %r6/96(%r15)-99(%r15): len - * to - * %r2: fd, %r3: mode, %r4/%r5: offset, 96(%r15)-103(%r15): len - */ -SYSCALL_DEFINE5(s390_fallocate, int, fd, int, mode, loff_t, offset, - u32, len_high, u32, len_low) -{ - return sys_fallocate(fd, mode, offset, ((u64)len_high << 32) | len_low); -} -#endif diff --git a/arch/s390/kernel/syscalls.S b/arch/s390/kernel/syscalls.S index 939ec474b1dd..1acad02681c4 100644 --- a/arch/s390/kernel/syscalls.S +++ b/arch/s390/kernel/syscalls.S @@ -1,365 +1,365 @@ /* * definitions for sys_call_table, each line represents an - * entry in the table in the form - * SYSCALL(31 bit syscall, 64 bit syscall, 31 bit emulated syscall) + * entry in the table in the form + * SYSCALL(64 bit syscall, 31 bit emulated syscall) * - * this file is meant to be included from entry.S and entry64.S + * this file is meant to be included from entry.S */ -#define NI_SYSCALL SYSCALL(sys_ni_syscall,sys_ni_syscall,sys_ni_syscall) +#define NI_SYSCALL SYSCALL(sys_ni_syscall,sys_ni_syscall) -NI_SYSCALL /* 0 */ -SYSCALL(sys_exit,sys_exit,compat_sys_exit) -SYSCALL(sys_fork,sys_fork,sys_fork) -SYSCALL(sys_read,sys_read,compat_sys_s390_read) -SYSCALL(sys_write,sys_write,compat_sys_s390_write) -SYSCALL(sys_open,sys_open,compat_sys_open) /* 5 */ -SYSCALL(sys_close,sys_close,compat_sys_close) -SYSCALL(sys_restart_syscall,sys_restart_syscall,sys_restart_syscall) -SYSCALL(sys_creat,sys_creat,compat_sys_creat) -SYSCALL(sys_link,sys_link,compat_sys_link) -SYSCALL(sys_unlink,sys_unlink,compat_sys_unlink) /* 10 */ -SYSCALL(sys_execve,sys_execve,compat_sys_execve) -SYSCALL(sys_chdir,sys_chdir,compat_sys_chdir) -SYSCALL(sys_time,sys_ni_syscall,compat_sys_time) /* old time syscall */ -SYSCALL(sys_mknod,sys_mknod,compat_sys_mknod) -SYSCALL(sys_chmod,sys_chmod,compat_sys_chmod) /* 15 */ -SYSCALL(sys_lchown16,sys_ni_syscall,compat_sys_s390_lchown16) /* old lchown16 syscall*/ -NI_SYSCALL /* old break syscall holder */ -NI_SYSCALL /* old stat syscall holder */ -SYSCALL(sys_lseek,sys_lseek,compat_sys_lseek) -SYSCALL(sys_getpid,sys_getpid,sys_getpid) /* 20 */ -SYSCALL(sys_mount,sys_mount,compat_sys_mount) -SYSCALL(sys_oldumount,sys_oldumount,compat_sys_oldumount) -SYSCALL(sys_setuid16,sys_ni_syscall,compat_sys_s390_setuid16) /* old setuid16 syscall*/ -SYSCALL(sys_getuid16,sys_ni_syscall,compat_sys_s390_getuid16) /* old getuid16 syscall*/ -SYSCALL(sys_stime,sys_ni_syscall,compat_sys_stime) /* 25 old stime syscall */ -SYSCALL(sys_ptrace,sys_ptrace,compat_sys_ptrace) -SYSCALL(sys_alarm,sys_alarm,compat_sys_alarm) -NI_SYSCALL /* old fstat syscall */ -SYSCALL(sys_pause,sys_pause,sys_pause) -SYSCALL(sys_utime,sys_utime,compat_sys_utime) /* 30 */ -NI_SYSCALL /* old stty syscall */ -NI_SYSCALL /* old gtty syscall */ -SYSCALL(sys_access,sys_access,compat_sys_access) -SYSCALL(sys_nice,sys_nice,compat_sys_nice) -NI_SYSCALL /* 35 old ftime syscall */ -SYSCALL(sys_sync,sys_sync,sys_sync) -SYSCALL(sys_kill,sys_kill,compat_sys_kill) -SYSCALL(sys_rename,sys_rename,compat_sys_rename) -SYSCALL(sys_mkdir,sys_mkdir,compat_sys_mkdir) -SYSCALL(sys_rmdir,sys_rmdir,compat_sys_rmdir) /* 40 */ -SYSCALL(sys_dup,sys_dup,compat_sys_dup) -SYSCALL(sys_pipe,sys_pipe,compat_sys_pipe) -SYSCALL(sys_times,sys_times,compat_sys_times) -NI_SYSCALL /* old prof syscall */ -SYSCALL(sys_brk,sys_brk,compat_sys_brk) /* 45 */ -SYSCALL(sys_setgid16,sys_ni_syscall,compat_sys_s390_setgid16) /* old setgid16 syscall*/ -SYSCALL(sys_getgid16,sys_ni_syscall,compat_sys_s390_getgid16) /* old getgid16 syscall*/ -SYSCALL(sys_signal,sys_signal,compat_sys_signal) -SYSCALL(sys_geteuid16,sys_ni_syscall,compat_sys_s390_geteuid16) /* old geteuid16 syscall */ -SYSCALL(sys_getegid16,sys_ni_syscall,compat_sys_s390_getegid16) /* 50 old getegid16 syscall */ -SYSCALL(sys_acct,sys_acct,compat_sys_acct) -SYSCALL(sys_umount,sys_umount,compat_sys_umount) -NI_SYSCALL /* old lock syscall */ -SYSCALL(sys_ioctl,sys_ioctl,compat_sys_ioctl) -SYSCALL(sys_fcntl,sys_fcntl,compat_sys_fcntl) /* 55 */ -NI_SYSCALL /* intel mpx syscall */ -SYSCALL(sys_setpgid,sys_setpgid,compat_sys_setpgid) -NI_SYSCALL /* old ulimit syscall */ -NI_SYSCALL /* old uname syscall */ -SYSCALL(sys_umask,sys_umask,compat_sys_umask) /* 60 */ -SYSCALL(sys_chroot,sys_chroot,compat_sys_chroot) -SYSCALL(sys_ustat,sys_ustat,compat_sys_ustat) -SYSCALL(sys_dup2,sys_dup2,compat_sys_dup2) -SYSCALL(sys_getppid,sys_getppid,sys_getppid) -SYSCALL(sys_getpgrp,sys_getpgrp,sys_getpgrp) /* 65 */ -SYSCALL(sys_setsid,sys_setsid,sys_setsid) -SYSCALL(sys_sigaction,sys_sigaction,compat_sys_sigaction) -NI_SYSCALL /* old sgetmask syscall*/ -NI_SYSCALL /* old ssetmask syscall*/ -SYSCALL(sys_setreuid16,sys_ni_syscall,compat_sys_s390_setreuid16) /* old setreuid16 syscall */ -SYSCALL(sys_setregid16,sys_ni_syscall,compat_sys_s390_setregid16) /* old setregid16 syscall */ -SYSCALL(sys_sigsuspend,sys_sigsuspend,compat_sys_sigsuspend) -SYSCALL(sys_sigpending,sys_sigpending,compat_sys_sigpending) -SYSCALL(sys_sethostname,sys_sethostname,compat_sys_sethostname) -SYSCALL(sys_setrlimit,sys_setrlimit,compat_sys_setrlimit) /* 75 */ -SYSCALL(sys_old_getrlimit,sys_getrlimit,compat_sys_old_getrlimit) -SYSCALL(sys_getrusage,sys_getrusage,compat_sys_getrusage) -SYSCALL(sys_gettimeofday,sys_gettimeofday,compat_sys_gettimeofday) -SYSCALL(sys_settimeofday,sys_settimeofday,compat_sys_settimeofday) -SYSCALL(sys_getgroups16,sys_ni_syscall,compat_sys_s390_getgroups16) /* 80 old getgroups16 syscall */ -SYSCALL(sys_setgroups16,sys_ni_syscall,compat_sys_s390_setgroups16) /* old setgroups16 syscall */ -NI_SYSCALL /* old select syscall */ -SYSCALL(sys_symlink,sys_symlink,compat_sys_symlink) -NI_SYSCALL /* old lstat syscall */ -SYSCALL(sys_readlink,sys_readlink,compat_sys_readlink) /* 85 */ -SYSCALL(sys_uselib,sys_uselib,compat_sys_uselib) -SYSCALL(sys_swapon,sys_swapon,compat_sys_swapon) -SYSCALL(sys_reboot,sys_reboot,compat_sys_reboot) -SYSCALL(sys_ni_syscall,sys_ni_syscall,compat_sys_old_readdir) /* old readdir syscall */ -SYSCALL(sys_old_mmap,sys_old_mmap,compat_sys_s390_old_mmap) /* 90 */ -SYSCALL(sys_munmap,sys_munmap,compat_sys_munmap) -SYSCALL(sys_truncate,sys_truncate,compat_sys_truncate) -SYSCALL(sys_ftruncate,sys_ftruncate,compat_sys_ftruncate) -SYSCALL(sys_fchmod,sys_fchmod,compat_sys_fchmod) -SYSCALL(sys_fchown16,sys_ni_syscall,compat_sys_s390_fchown16) /* 95 old fchown16 syscall*/ -SYSCALL(sys_getpriority,sys_getpriority,compat_sys_getpriority) -SYSCALL(sys_setpriority,sys_setpriority,compat_sys_setpriority) -NI_SYSCALL /* old profil syscall */ -SYSCALL(sys_statfs,sys_statfs,compat_sys_statfs) -SYSCALL(sys_fstatfs,sys_fstatfs,compat_sys_fstatfs) /* 100 */ -NI_SYSCALL /* ioperm for i386 */ -SYSCALL(sys_socketcall,sys_socketcall,compat_sys_socketcall) -SYSCALL(sys_syslog,sys_syslog,compat_sys_syslog) -SYSCALL(sys_setitimer,sys_setitimer,compat_sys_setitimer) -SYSCALL(sys_getitimer,sys_getitimer,compat_sys_getitimer) /* 105 */ -SYSCALL(sys_newstat,sys_newstat,compat_sys_newstat) -SYSCALL(sys_newlstat,sys_newlstat,compat_sys_newlstat) -SYSCALL(sys_newfstat,sys_newfstat,compat_sys_newfstat) -NI_SYSCALL /* old uname syscall */ -SYSCALL(sys_lookup_dcookie,sys_lookup_dcookie,compat_sys_lookup_dcookie) /* 110 */ -SYSCALL(sys_vhangup,sys_vhangup,sys_vhangup) -NI_SYSCALL /* old "idle" system call */ -NI_SYSCALL /* vm86old for i386 */ -SYSCALL(sys_wait4,sys_wait4,compat_sys_wait4) -SYSCALL(sys_swapoff,sys_swapoff,compat_sys_swapoff) /* 115 */ -SYSCALL(sys_sysinfo,sys_sysinfo,compat_sys_sysinfo) -SYSCALL(sys_s390_ipc,sys_s390_ipc,compat_sys_s390_ipc) -SYSCALL(sys_fsync,sys_fsync,compat_sys_fsync) -SYSCALL(sys_sigreturn,sys_sigreturn,compat_sys_sigreturn) -SYSCALL(sys_clone,sys_clone,compat_sys_clone) /* 120 */ -SYSCALL(sys_setdomainname,sys_setdomainname,compat_sys_setdomainname) -SYSCALL(sys_newuname,sys_newuname,compat_sys_newuname) -NI_SYSCALL /* modify_ldt for i386 */ -SYSCALL(sys_adjtimex,sys_adjtimex,compat_sys_adjtimex) -SYSCALL(sys_mprotect,sys_mprotect,compat_sys_mprotect) /* 125 */ -SYSCALL(sys_sigprocmask,sys_sigprocmask,compat_sys_sigprocmask) -NI_SYSCALL /* old "create module" */ -SYSCALL(sys_init_module,sys_init_module,compat_sys_init_module) -SYSCALL(sys_delete_module,sys_delete_module,compat_sys_delete_module) -NI_SYSCALL /* 130: old get_kernel_syms */ -SYSCALL(sys_quotactl,sys_quotactl,compat_sys_quotactl) -SYSCALL(sys_getpgid,sys_getpgid,compat_sys_getpgid) -SYSCALL(sys_fchdir,sys_fchdir,compat_sys_fchdir) -SYSCALL(sys_bdflush,sys_bdflush,compat_sys_bdflush) -SYSCALL(sys_sysfs,sys_sysfs,compat_sys_sysfs) /* 135 */ -SYSCALL(sys_personality,sys_s390_personality,compat_sys_s390_personality) -NI_SYSCALL /* for afs_syscall */ -SYSCALL(sys_setfsuid16,sys_ni_syscall,compat_sys_s390_setfsuid16) /* old setfsuid16 syscall */ -SYSCALL(sys_setfsgid16,sys_ni_syscall,compat_sys_s390_setfsgid16) /* old setfsgid16 syscall */ -SYSCALL(sys_llseek,sys_llseek,compat_sys_llseek) /* 140 */ -SYSCALL(sys_getdents,sys_getdents,compat_sys_getdents) -SYSCALL(sys_select,sys_select,compat_sys_select) -SYSCALL(sys_flock,sys_flock,compat_sys_flock) -SYSCALL(sys_msync,sys_msync,compat_sys_msync) -SYSCALL(sys_readv,sys_readv,compat_sys_readv) /* 145 */ -SYSCALL(sys_writev,sys_writev,compat_sys_writev) -SYSCALL(sys_getsid,sys_getsid,compat_sys_getsid) -SYSCALL(sys_fdatasync,sys_fdatasync,compat_sys_fdatasync) -SYSCALL(sys_sysctl,sys_sysctl,compat_sys_sysctl) -SYSCALL(sys_mlock,sys_mlock,compat_sys_mlock) /* 150 */ -SYSCALL(sys_munlock,sys_munlock,compat_sys_munlock) -SYSCALL(sys_mlockall,sys_mlockall,compat_sys_mlockall) -SYSCALL(sys_munlockall,sys_munlockall,sys_munlockall) -SYSCALL(sys_sched_setparam,sys_sched_setparam,compat_sys_sched_setparam) -SYSCALL(sys_sched_getparam,sys_sched_getparam,compat_sys_sched_getparam) /* 155 */ -SYSCALL(sys_sched_setscheduler,sys_sched_setscheduler,compat_sys_sched_setscheduler) -SYSCALL(sys_sched_getscheduler,sys_sched_getscheduler,compat_sys_sched_getscheduler) -SYSCALL(sys_sched_yield,sys_sched_yield,sys_sched_yield) -SYSCALL(sys_sched_get_priority_max,sys_sched_get_priority_max,compat_sys_sched_get_priority_max) -SYSCALL(sys_sched_get_priority_min,sys_sched_get_priority_min,compat_sys_sched_get_priority_min) /* 160 */ -SYSCALL(sys_sched_rr_get_interval,sys_sched_rr_get_interval,compat_sys_sched_rr_get_interval) -SYSCALL(sys_nanosleep,sys_nanosleep,compat_sys_nanosleep) -SYSCALL(sys_mremap,sys_mremap,compat_sys_mremap) -SYSCALL(sys_setresuid16,sys_ni_syscall,compat_sys_s390_setresuid16) /* old setresuid16 syscall */ -SYSCALL(sys_getresuid16,sys_ni_syscall,compat_sys_s390_getresuid16) /* 165 old getresuid16 syscall */ -NI_SYSCALL /* for vm86 */ -NI_SYSCALL /* old sys_query_module */ -SYSCALL(sys_poll,sys_poll,compat_sys_poll) -NI_SYSCALL /* old nfsservctl */ -SYSCALL(sys_setresgid16,sys_ni_syscall,compat_sys_s390_setresgid16) /* 170 old setresgid16 syscall */ -SYSCALL(sys_getresgid16,sys_ni_syscall,compat_sys_s390_getresgid16) /* old getresgid16 syscall */ -SYSCALL(sys_prctl,sys_prctl,compat_sys_prctl) -SYSCALL(sys_rt_sigreturn,sys_rt_sigreturn,compat_sys_rt_sigreturn) -SYSCALL(sys_rt_sigaction,sys_rt_sigaction,compat_sys_rt_sigaction) -SYSCALL(sys_rt_sigprocmask,sys_rt_sigprocmask,compat_sys_rt_sigprocmask) /* 175 */ -SYSCALL(sys_rt_sigpending,sys_rt_sigpending,compat_sys_rt_sigpending) -SYSCALL(sys_rt_sigtimedwait,sys_rt_sigtimedwait,compat_sys_rt_sigtimedwait) -SYSCALL(sys_rt_sigqueueinfo,sys_rt_sigqueueinfo,compat_sys_rt_sigqueueinfo) -SYSCALL(sys_rt_sigsuspend,sys_rt_sigsuspend,compat_sys_rt_sigsuspend) -SYSCALL(sys_pread64,sys_pread64,compat_sys_s390_pread64) /* 180 */ -SYSCALL(sys_pwrite64,sys_pwrite64,compat_sys_s390_pwrite64) -SYSCALL(sys_chown16,sys_ni_syscall,compat_sys_s390_chown16) /* old chown16 syscall */ -SYSCALL(sys_getcwd,sys_getcwd,compat_sys_getcwd) -SYSCALL(sys_capget,sys_capget,compat_sys_capget) -SYSCALL(sys_capset,sys_capset,compat_sys_capset) /* 185 */ -SYSCALL(sys_sigaltstack,sys_sigaltstack,compat_sys_sigaltstack) -SYSCALL(sys_sendfile,sys_sendfile64,compat_sys_sendfile) -NI_SYSCALL /* streams1 */ -NI_SYSCALL /* streams2 */ -SYSCALL(sys_vfork,sys_vfork,sys_vfork) /* 190 */ -SYSCALL(sys_getrlimit,sys_getrlimit,compat_sys_getrlimit) -SYSCALL(sys_mmap2,sys_mmap2,compat_sys_s390_mmap2) -SYSCALL(sys_truncate64,sys_ni_syscall,compat_sys_s390_truncate64) -SYSCALL(sys_ftruncate64,sys_ni_syscall,compat_sys_s390_ftruncate64) -SYSCALL(sys_stat64,sys_ni_syscall,compat_sys_s390_stat64) /* 195 */ -SYSCALL(sys_lstat64,sys_ni_syscall,compat_sys_s390_lstat64) -SYSCALL(sys_fstat64,sys_ni_syscall,compat_sys_s390_fstat64) -SYSCALL(sys_lchown,sys_lchown,compat_sys_lchown) -SYSCALL(sys_getuid,sys_getuid,sys_getuid) -SYSCALL(sys_getgid,sys_getgid,sys_getgid) /* 200 */ -SYSCALL(sys_geteuid,sys_geteuid,sys_geteuid) -SYSCALL(sys_getegid,sys_getegid,sys_getegid) -SYSCALL(sys_setreuid,sys_setreuid,compat_sys_setreuid) -SYSCALL(sys_setregid,sys_setregid,compat_sys_setregid) -SYSCALL(sys_getgroups,sys_getgroups,compat_sys_getgroups) /* 205 */ -SYSCALL(sys_setgroups,sys_setgroups,compat_sys_setgroups) -SYSCALL(sys_fchown,sys_fchown,compat_sys_fchown) -SYSCALL(sys_setresuid,sys_setresuid,compat_sys_setresuid) -SYSCALL(sys_getresuid,sys_getresuid,compat_sys_getresuid) -SYSCALL(sys_setresgid,sys_setresgid,compat_sys_setresgid) /* 210 */ -SYSCALL(sys_getresgid,sys_getresgid,compat_sys_getresgid) -SYSCALL(sys_chown,sys_chown,compat_sys_chown) -SYSCALL(sys_setuid,sys_setuid,compat_sys_setuid) -SYSCALL(sys_setgid,sys_setgid,compat_sys_setgid) -SYSCALL(sys_setfsuid,sys_setfsuid,compat_sys_setfsuid) /* 215 */ -SYSCALL(sys_setfsgid,sys_setfsgid,compat_sys_setfsgid) -SYSCALL(sys_pivot_root,sys_pivot_root,compat_sys_pivot_root) -SYSCALL(sys_mincore,sys_mincore,compat_sys_mincore) -SYSCALL(sys_madvise,sys_madvise,compat_sys_madvise) -SYSCALL(sys_getdents64,sys_getdents64,compat_sys_getdents64) /* 220 */ -SYSCALL(sys_fcntl64,sys_ni_syscall,compat_sys_fcntl64) -SYSCALL(sys_readahead,sys_readahead,compat_sys_s390_readahead) -SYSCALL(sys_sendfile64,sys_ni_syscall,compat_sys_sendfile64) -SYSCALL(sys_setxattr,sys_setxattr,compat_sys_setxattr) -SYSCALL(sys_lsetxattr,sys_lsetxattr,compat_sys_lsetxattr) /* 225 */ -SYSCALL(sys_fsetxattr,sys_fsetxattr,compat_sys_fsetxattr) -SYSCALL(sys_getxattr,sys_getxattr,compat_sys_getxattr) -SYSCALL(sys_lgetxattr,sys_lgetxattr,compat_sys_lgetxattr) -SYSCALL(sys_fgetxattr,sys_fgetxattr,compat_sys_fgetxattr) -SYSCALL(sys_listxattr,sys_listxattr,compat_sys_listxattr) /* 230 */ -SYSCALL(sys_llistxattr,sys_llistxattr,compat_sys_llistxattr) -SYSCALL(sys_flistxattr,sys_flistxattr,compat_sys_flistxattr) -SYSCALL(sys_removexattr,sys_removexattr,compat_sys_removexattr) -SYSCALL(sys_lremovexattr,sys_lremovexattr,compat_sys_lremovexattr) -SYSCALL(sys_fremovexattr,sys_fremovexattr,compat_sys_fremovexattr) /* 235 */ -SYSCALL(sys_gettid,sys_gettid,sys_gettid) -SYSCALL(sys_tkill,sys_tkill,compat_sys_tkill) -SYSCALL(sys_futex,sys_futex,compat_sys_futex) -SYSCALL(sys_sched_setaffinity,sys_sched_setaffinity,compat_sys_sched_setaffinity) -SYSCALL(sys_sched_getaffinity,sys_sched_getaffinity,compat_sys_sched_getaffinity) /* 240 */ -SYSCALL(sys_tgkill,sys_tgkill,compat_sys_tgkill) -NI_SYSCALL /* reserved for TUX */ -SYSCALL(sys_io_setup,sys_io_setup,compat_sys_io_setup) -SYSCALL(sys_io_destroy,sys_io_destroy,compat_sys_io_destroy) -SYSCALL(sys_io_getevents,sys_io_getevents,compat_sys_io_getevents) /* 245 */ -SYSCALL(sys_io_submit,sys_io_submit,compat_sys_io_submit) -SYSCALL(sys_io_cancel,sys_io_cancel,compat_sys_io_cancel) -SYSCALL(sys_exit_group,sys_exit_group,compat_sys_exit_group) -SYSCALL(sys_epoll_create,sys_epoll_create,compat_sys_epoll_create) -SYSCALL(sys_epoll_ctl,sys_epoll_ctl,compat_sys_epoll_ctl) /* 250 */ -SYSCALL(sys_epoll_wait,sys_epoll_wait,compat_sys_epoll_wait) -SYSCALL(sys_set_tid_address,sys_set_tid_address,compat_sys_set_tid_address) -SYSCALL(sys_s390_fadvise64,sys_fadvise64_64,compat_sys_s390_fadvise64) -SYSCALL(sys_timer_create,sys_timer_create,compat_sys_timer_create) -SYSCALL(sys_timer_settime,sys_timer_settime,compat_sys_timer_settime) /* 255 */ -SYSCALL(sys_timer_gettime,sys_timer_gettime,compat_sys_timer_gettime) -SYSCALL(sys_timer_getoverrun,sys_timer_getoverrun,compat_sys_timer_getoverrun) -SYSCALL(sys_timer_delete,sys_timer_delete,compat_sys_timer_delete) -SYSCALL(sys_clock_settime,sys_clock_settime,compat_sys_clock_settime) -SYSCALL(sys_clock_gettime,sys_clock_gettime,compat_sys_clock_gettime) /* 260 */ -SYSCALL(sys_clock_getres,sys_clock_getres,compat_sys_clock_getres) -SYSCALL(sys_clock_nanosleep,sys_clock_nanosleep,compat_sys_clock_nanosleep) -NI_SYSCALL /* reserved for vserver */ -SYSCALL(sys_s390_fadvise64_64,sys_ni_syscall,compat_sys_s390_fadvise64_64) -SYSCALL(sys_statfs64,sys_statfs64,compat_sys_statfs64) -SYSCALL(sys_fstatfs64,sys_fstatfs64,compat_sys_fstatfs64) -SYSCALL(sys_remap_file_pages,sys_remap_file_pages,compat_sys_remap_file_pages) -NI_SYSCALL /* 268 sys_mbind */ -NI_SYSCALL /* 269 sys_get_mempolicy */ -NI_SYSCALL /* 270 sys_set_mempolicy */ -SYSCALL(sys_mq_open,sys_mq_open,compat_sys_mq_open) -SYSCALL(sys_mq_unlink,sys_mq_unlink,compat_sys_mq_unlink) -SYSCALL(sys_mq_timedsend,sys_mq_timedsend,compat_sys_mq_timedsend) -SYSCALL(sys_mq_timedreceive,sys_mq_timedreceive,compat_sys_mq_timedreceive) -SYSCALL(sys_mq_notify,sys_mq_notify,compat_sys_mq_notify) /* 275 */ -SYSCALL(sys_mq_getsetattr,sys_mq_getsetattr,compat_sys_mq_getsetattr) -SYSCALL(sys_kexec_load,sys_kexec_load,compat_sys_kexec_load) -SYSCALL(sys_add_key,sys_add_key,compat_sys_add_key) -SYSCALL(sys_request_key,sys_request_key,compat_sys_request_key) -SYSCALL(sys_keyctl,sys_keyctl,compat_sys_keyctl) /* 280 */ -SYSCALL(sys_waitid,sys_waitid,compat_sys_waitid) -SYSCALL(sys_ioprio_set,sys_ioprio_set,compat_sys_ioprio_set) -SYSCALL(sys_ioprio_get,sys_ioprio_get,compat_sys_ioprio_get) -SYSCALL(sys_inotify_init,sys_inotify_init,sys_inotify_init) -SYSCALL(sys_inotify_add_watch,sys_inotify_add_watch,compat_sys_inotify_add_watch) /* 285 */ -SYSCALL(sys_inotify_rm_watch,sys_inotify_rm_watch,compat_sys_inotify_rm_watch) -NI_SYSCALL /* 287 sys_migrate_pages */ -SYSCALL(sys_openat,sys_openat,compat_sys_openat) -SYSCALL(sys_mkdirat,sys_mkdirat,compat_sys_mkdirat) -SYSCALL(sys_mknodat,sys_mknodat,compat_sys_mknodat) /* 290 */ -SYSCALL(sys_fchownat,sys_fchownat,compat_sys_fchownat) -SYSCALL(sys_futimesat,sys_futimesat,compat_sys_futimesat) -SYSCALL(sys_fstatat64,sys_newfstatat,compat_sys_s390_fstatat64) -SYSCALL(sys_unlinkat,sys_unlinkat,compat_sys_unlinkat) -SYSCALL(sys_renameat,sys_renameat,compat_sys_renameat) /* 295 */ -SYSCALL(sys_linkat,sys_linkat,compat_sys_linkat) -SYSCALL(sys_symlinkat,sys_symlinkat,compat_sys_symlinkat) -SYSCALL(sys_readlinkat,sys_readlinkat,compat_sys_readlinkat) -SYSCALL(sys_fchmodat,sys_fchmodat,compat_sys_fchmodat) -SYSCALL(sys_faccessat,sys_faccessat,compat_sys_faccessat) /* 300 */ -SYSCALL(sys_pselect6,sys_pselect6,compat_sys_pselect6) -SYSCALL(sys_ppoll,sys_ppoll,compat_sys_ppoll) -SYSCALL(sys_unshare,sys_unshare,compat_sys_unshare) -SYSCALL(sys_set_robust_list,sys_set_robust_list,compat_sys_set_robust_list) -SYSCALL(sys_get_robust_list,sys_get_robust_list,compat_sys_get_robust_list) -SYSCALL(sys_splice,sys_splice,compat_sys_splice) -SYSCALL(sys_sync_file_range,sys_sync_file_range,compat_sys_s390_sync_file_range) -SYSCALL(sys_tee,sys_tee,compat_sys_tee) -SYSCALL(sys_vmsplice,sys_vmsplice,compat_sys_vmsplice) -NI_SYSCALL /* 310 sys_move_pages */ -SYSCALL(sys_getcpu,sys_getcpu,compat_sys_getcpu) -SYSCALL(sys_epoll_pwait,sys_epoll_pwait,compat_sys_epoll_pwait) -SYSCALL(sys_utimes,sys_utimes,compat_sys_utimes) -SYSCALL(sys_s390_fallocate,sys_fallocate,compat_sys_s390_fallocate) -SYSCALL(sys_utimensat,sys_utimensat,compat_sys_utimensat) /* 315 */ -SYSCALL(sys_signalfd,sys_signalfd,compat_sys_signalfd) +NI_SYSCALL /* 0 */ +SYSCALL(sys_exit,compat_sys_exit) +SYSCALL(sys_fork,sys_fork) +SYSCALL(sys_read,compat_sys_s390_read) +SYSCALL(sys_write,compat_sys_s390_write) +SYSCALL(sys_open,compat_sys_open) /* 5 */ +SYSCALL(sys_close,compat_sys_close) +SYSCALL(sys_restart_syscall,sys_restart_syscall) +SYSCALL(sys_creat,compat_sys_creat) +SYSCALL(sys_link,compat_sys_link) +SYSCALL(sys_unlink,compat_sys_unlink) /* 10 */ +SYSCALL(sys_execve,compat_sys_execve) +SYSCALL(sys_chdir,compat_sys_chdir) +SYSCALL(sys_ni_syscall,compat_sys_time) /* old time syscall */ +SYSCALL(sys_mknod,compat_sys_mknod) +SYSCALL(sys_chmod,compat_sys_chmod) /* 15 */ +SYSCALL(sys_ni_syscall,compat_sys_s390_lchown16) /* old lchown16 syscall*/ +NI_SYSCALL /* old break syscall holder */ +NI_SYSCALL /* old stat syscall holder */ +SYSCALL(sys_lseek,compat_sys_lseek) +SYSCALL(sys_getpid,sys_getpid) /* 20 */ +SYSCALL(sys_mount,compat_sys_mount) +SYSCALL(sys_oldumount,compat_sys_oldumount) +SYSCALL(sys_ni_syscall,compat_sys_s390_setuid16) /* old setuid16 syscall*/ +SYSCALL(sys_ni_syscall,compat_sys_s390_getuid16) /* old getuid16 syscall*/ +SYSCALL(sys_ni_syscall,compat_sys_stime) /* 25 old stime syscall */ +SYSCALL(sys_ptrace,compat_sys_ptrace) +SYSCALL(sys_alarm,compat_sys_alarm) +NI_SYSCALL /* old fstat syscall */ +SYSCALL(sys_pause,sys_pause) +SYSCALL(sys_utime,compat_sys_utime) /* 30 */ +NI_SYSCALL /* old stty syscall */ +NI_SYSCALL /* old gtty syscall */ +SYSCALL(sys_access,compat_sys_access) +SYSCALL(sys_nice,compat_sys_nice) +NI_SYSCALL /* 35 old ftime syscall */ +SYSCALL(sys_sync,sys_sync) +SYSCALL(sys_kill,compat_sys_kill) +SYSCALL(sys_rename,compat_sys_rename) +SYSCALL(sys_mkdir,compat_sys_mkdir) +SYSCALL(sys_rmdir,compat_sys_rmdir) /* 40 */ +SYSCALL(sys_dup,compat_sys_dup) +SYSCALL(sys_pipe,compat_sys_pipe) +SYSCALL(sys_times,compat_sys_times) +NI_SYSCALL /* old prof syscall */ +SYSCALL(sys_brk,compat_sys_brk) /* 45 */ +SYSCALL(sys_ni_syscall,compat_sys_s390_setgid16) /* old setgid16 syscall*/ +SYSCALL(sys_ni_syscall,compat_sys_s390_getgid16) /* old getgid16 syscall*/ +SYSCALL(sys_signal,compat_sys_signal) +SYSCALL(sys_ni_syscall,compat_sys_s390_geteuid16) /* old geteuid16 syscall */ +SYSCALL(sys_ni_syscall,compat_sys_s390_getegid16) /* 50 old getegid16 syscall */ +SYSCALL(sys_acct,compat_sys_acct) +SYSCALL(sys_umount,compat_sys_umount) +NI_SYSCALL /* old lock syscall */ +SYSCALL(sys_ioctl,compat_sys_ioctl) +SYSCALL(sys_fcntl,compat_sys_fcntl) /* 55 */ +NI_SYSCALL /* intel mpx syscall */ +SYSCALL(sys_setpgid,compat_sys_setpgid) +NI_SYSCALL /* old ulimit syscall */ +NI_SYSCALL /* old uname syscall */ +SYSCALL(sys_umask,compat_sys_umask) /* 60 */ +SYSCALL(sys_chroot,compat_sys_chroot) +SYSCALL(sys_ustat,compat_sys_ustat) +SYSCALL(sys_dup2,compat_sys_dup2) +SYSCALL(sys_getppid,sys_getppid) +SYSCALL(sys_getpgrp,sys_getpgrp) /* 65 */ +SYSCALL(sys_setsid,sys_setsid) +SYSCALL(sys_sigaction,compat_sys_sigaction) +NI_SYSCALL /* old sgetmask syscall*/ +NI_SYSCALL /* old ssetmask syscall*/ +SYSCALL(sys_ni_syscall,compat_sys_s390_setreuid16) /* old setreuid16 syscall */ +SYSCALL(sys_ni_syscall,compat_sys_s390_setregid16) /* old setregid16 syscall */ +SYSCALL(sys_sigsuspend,compat_sys_sigsuspend) +SYSCALL(sys_sigpending,compat_sys_sigpending) +SYSCALL(sys_sethostname,compat_sys_sethostname) +SYSCALL(sys_setrlimit,compat_sys_setrlimit) /* 75 */ +SYSCALL(sys_getrlimit,compat_sys_old_getrlimit) +SYSCALL(sys_getrusage,compat_sys_getrusage) +SYSCALL(sys_gettimeofday,compat_sys_gettimeofday) +SYSCALL(sys_settimeofday,compat_sys_settimeofday) +SYSCALL(sys_ni_syscall,compat_sys_s390_getgroups16) /* 80 old getgroups16 syscall */ +SYSCALL(sys_ni_syscall,compat_sys_s390_setgroups16) /* old setgroups16 syscall */ +NI_SYSCALL /* old select syscall */ +SYSCALL(sys_symlink,compat_sys_symlink) +NI_SYSCALL /* old lstat syscall */ +SYSCALL(sys_readlink,compat_sys_readlink) /* 85 */ +SYSCALL(sys_uselib,compat_sys_uselib) +SYSCALL(sys_swapon,compat_sys_swapon) +SYSCALL(sys_reboot,compat_sys_reboot) +SYSCALL(sys_ni_syscall,compat_sys_old_readdir) /* old readdir syscall */ +SYSCALL(sys_old_mmap,compat_sys_s390_old_mmap) /* 90 */ +SYSCALL(sys_munmap,compat_sys_munmap) +SYSCALL(sys_truncate,compat_sys_truncate) +SYSCALL(sys_ftruncate,compat_sys_ftruncate) +SYSCALL(sys_fchmod,compat_sys_fchmod) +SYSCALL(sys_ni_syscall,compat_sys_s390_fchown16) /* 95 old fchown16 syscall*/ +SYSCALL(sys_getpriority,compat_sys_getpriority) +SYSCALL(sys_setpriority,compat_sys_setpriority) +NI_SYSCALL /* old profil syscall */ +SYSCALL(sys_statfs,compat_sys_statfs) +SYSCALL(sys_fstatfs,compat_sys_fstatfs) /* 100 */ +NI_SYSCALL /* ioperm for i386 */ +SYSCALL(sys_socketcall,compat_sys_socketcall) +SYSCALL(sys_syslog,compat_sys_syslog) +SYSCALL(sys_setitimer,compat_sys_setitimer) +SYSCALL(sys_getitimer,compat_sys_getitimer) /* 105 */ +SYSCALL(sys_newstat,compat_sys_newstat) +SYSCALL(sys_newlstat,compat_sys_newlstat) +SYSCALL(sys_newfstat,compat_sys_newfstat) +NI_SYSCALL /* old uname syscall */ +SYSCALL(sys_lookup_dcookie,compat_sys_lookup_dcookie) /* 110 */ +SYSCALL(sys_vhangup,sys_vhangup) +NI_SYSCALL /* old "idle" system call */ +NI_SYSCALL /* vm86old for i386 */ +SYSCALL(sys_wait4,compat_sys_wait4) +SYSCALL(sys_swapoff,compat_sys_swapoff) /* 115 */ +SYSCALL(sys_sysinfo,compat_sys_sysinfo) +SYSCALL(sys_s390_ipc,compat_sys_s390_ipc) +SYSCALL(sys_fsync,compat_sys_fsync) +SYSCALL(sys_sigreturn,compat_sys_sigreturn) +SYSCALL(sys_clone,compat_sys_clone) /* 120 */ +SYSCALL(sys_setdomainname,compat_sys_setdomainname) +SYSCALL(sys_newuname,compat_sys_newuname) +NI_SYSCALL /* modify_ldt for i386 */ +SYSCALL(sys_adjtimex,compat_sys_adjtimex) +SYSCALL(sys_mprotect,compat_sys_mprotect) /* 125 */ +SYSCALL(sys_sigprocmask,compat_sys_sigprocmask) +NI_SYSCALL /* old "create module" */ +SYSCALL(sys_init_module,compat_sys_init_module) +SYSCALL(sys_delete_module,compat_sys_delete_module) +NI_SYSCALL /* 130: old get_kernel_syms */ +SYSCALL(sys_quotactl,compat_sys_quotactl) +SYSCALL(sys_getpgid,compat_sys_getpgid) +SYSCALL(sys_fchdir,compat_sys_fchdir) +SYSCALL(sys_bdflush,compat_sys_bdflush) +SYSCALL(sys_sysfs,compat_sys_sysfs) /* 135 */ +SYSCALL(sys_s390_personality,compat_sys_s390_personality) +NI_SYSCALL /* for afs_syscall */ +SYSCALL(sys_ni_syscall,compat_sys_s390_setfsuid16) /* old setfsuid16 syscall */ +SYSCALL(sys_ni_syscall,compat_sys_s390_setfsgid16) /* old setfsgid16 syscall */ +SYSCALL(sys_llseek,compat_sys_llseek) /* 140 */ +SYSCALL(sys_getdents,compat_sys_getdents) +SYSCALL(sys_select,compat_sys_select) +SYSCALL(sys_flock,compat_sys_flock) +SYSCALL(sys_msync,compat_sys_msync) +SYSCALL(sys_readv,compat_sys_readv) /* 145 */ +SYSCALL(sys_writev,compat_sys_writev) +SYSCALL(sys_getsid,compat_sys_getsid) +SYSCALL(sys_fdatasync,compat_sys_fdatasync) +SYSCALL(sys_sysctl,compat_sys_sysctl) +SYSCALL(sys_mlock,compat_sys_mlock) /* 150 */ +SYSCALL(sys_munlock,compat_sys_munlock) +SYSCALL(sys_mlockall,compat_sys_mlockall) +SYSCALL(sys_munlockall,sys_munlockall) +SYSCALL(sys_sched_setparam,compat_sys_sched_setparam) +SYSCALL(sys_sched_getparam,compat_sys_sched_getparam) /* 155 */ +SYSCALL(sys_sched_setscheduler,compat_sys_sched_setscheduler) +SYSCALL(sys_sched_getscheduler,compat_sys_sched_getscheduler) +SYSCALL(sys_sched_yield,sys_sched_yield) +SYSCALL(sys_sched_get_priority_max,compat_sys_sched_get_priority_max) +SYSCALL(sys_sched_get_priority_min,compat_sys_sched_get_priority_min) /* 160 */ +SYSCALL(sys_sched_rr_get_interval,compat_sys_sched_rr_get_interval) +SYSCALL(sys_nanosleep,compat_sys_nanosleep) +SYSCALL(sys_mremap,compat_sys_mremap) +SYSCALL(sys_ni_syscall,compat_sys_s390_setresuid16) /* old setresuid16 syscall */ +SYSCALL(sys_ni_syscall,compat_sys_s390_getresuid16) /* 165 old getresuid16 syscall */ +NI_SYSCALL /* for vm86 */ +NI_SYSCALL /* old sys_query_module */ +SYSCALL(sys_poll,compat_sys_poll) +NI_SYSCALL /* old nfsservctl */ +SYSCALL(sys_ni_syscall,compat_sys_s390_setresgid16) /* 170 old setresgid16 syscall */ +SYSCALL(sys_ni_syscall,compat_sys_s390_getresgid16) /* old getresgid16 syscall */ +SYSCALL(sys_prctl,compat_sys_prctl) +SYSCALL(sys_rt_sigreturn,compat_sys_rt_sigreturn) +SYSCALL(sys_rt_sigaction,compat_sys_rt_sigaction) +SYSCALL(sys_rt_sigprocmask,compat_sys_rt_sigprocmask) /* 175 */ +SYSCALL(sys_rt_sigpending,compat_sys_rt_sigpending) +SYSCALL(sys_rt_sigtimedwait,compat_sys_rt_sigtimedwait) +SYSCALL(sys_rt_sigqueueinfo,compat_sys_rt_sigqueueinfo) +SYSCALL(sys_rt_sigsuspend,compat_sys_rt_sigsuspend) +SYSCALL(sys_pread64,compat_sys_s390_pread64) /* 180 */ +SYSCALL(sys_pwrite64,compat_sys_s390_pwrite64) +SYSCALL(sys_ni_syscall,compat_sys_s390_chown16) /* old chown16 syscall */ +SYSCALL(sys_getcwd,compat_sys_getcwd) +SYSCALL(sys_capget,compat_sys_capget) +SYSCALL(sys_capset,compat_sys_capset) /* 185 */ +SYSCALL(sys_sigaltstack,compat_sys_sigaltstack) +SYSCALL(sys_sendfile64,compat_sys_sendfile) +NI_SYSCALL /* streams1 */ +NI_SYSCALL /* streams2 */ +SYSCALL(sys_vfork,sys_vfork) /* 190 */ +SYSCALL(sys_getrlimit,compat_sys_getrlimit) +SYSCALL(sys_mmap2,compat_sys_s390_mmap2) +SYSCALL(sys_ni_syscall,compat_sys_s390_truncate64) +SYSCALL(sys_ni_syscall,compat_sys_s390_ftruncate64) +SYSCALL(sys_ni_syscall,compat_sys_s390_stat64) /* 195 */ +SYSCALL(sys_ni_syscall,compat_sys_s390_lstat64) +SYSCALL(sys_ni_syscall,compat_sys_s390_fstat64) +SYSCALL(sys_lchown,compat_sys_lchown) +SYSCALL(sys_getuid,sys_getuid) +SYSCALL(sys_getgid,sys_getgid) /* 200 */ +SYSCALL(sys_geteuid,sys_geteuid) +SYSCALL(sys_getegid,sys_getegid) +SYSCALL(sys_setreuid,compat_sys_setreuid) +SYSCALL(sys_setregid,compat_sys_setregid) +SYSCALL(sys_getgroups,compat_sys_getgroups) /* 205 */ +SYSCALL(sys_setgroups,compat_sys_setgroups) +SYSCALL(sys_fchown,compat_sys_fchown) +SYSCALL(sys_setresuid,compat_sys_setresuid) +SYSCALL(sys_getresuid,compat_sys_getresuid) +SYSCALL(sys_setresgid,compat_sys_setresgid) /* 210 */ +SYSCALL(sys_getresgid,compat_sys_getresgid) +SYSCALL(sys_chown,compat_sys_chown) +SYSCALL(sys_setuid,compat_sys_setuid) +SYSCALL(sys_setgid,compat_sys_setgid) +SYSCALL(sys_setfsuid,compat_sys_setfsuid) /* 215 */ +SYSCALL(sys_setfsgid,compat_sys_setfsgid) +SYSCALL(sys_pivot_root,compat_sys_pivot_root) +SYSCALL(sys_mincore,compat_sys_mincore) +SYSCALL(sys_madvise,compat_sys_madvise) +SYSCALL(sys_getdents64,compat_sys_getdents64) /* 220 */ +SYSCALL(sys_ni_syscall,compat_sys_fcntl64) +SYSCALL(sys_readahead,compat_sys_s390_readahead) +SYSCALL(sys_ni_syscall,compat_sys_sendfile64) +SYSCALL(sys_setxattr,compat_sys_setxattr) +SYSCALL(sys_lsetxattr,compat_sys_lsetxattr) /* 225 */ +SYSCALL(sys_fsetxattr,compat_sys_fsetxattr) +SYSCALL(sys_getxattr,compat_sys_getxattr) +SYSCALL(sys_lgetxattr,compat_sys_lgetxattr) +SYSCALL(sys_fgetxattr,compat_sys_fgetxattr) +SYSCALL(sys_listxattr,compat_sys_listxattr) /* 230 */ +SYSCALL(sys_llistxattr,compat_sys_llistxattr) +SYSCALL(sys_flistxattr,compat_sys_flistxattr) +SYSCALL(sys_removexattr,compat_sys_removexattr) +SYSCALL(sys_lremovexattr,compat_sys_lremovexattr) +SYSCALL(sys_fremovexattr,compat_sys_fremovexattr) /* 235 */ +SYSCALL(sys_gettid,sys_gettid) +SYSCALL(sys_tkill,compat_sys_tkill) +SYSCALL(sys_futex,compat_sys_futex) +SYSCALL(sys_sched_setaffinity,compat_sys_sched_setaffinity) +SYSCALL(sys_sched_getaffinity,compat_sys_sched_getaffinity) /* 240 */ +SYSCALL(sys_tgkill,compat_sys_tgkill) +NI_SYSCALL /* reserved for TUX */ +SYSCALL(sys_io_setup,compat_sys_io_setup) +SYSCALL(sys_io_destroy,compat_sys_io_destroy) +SYSCALL(sys_io_getevents,compat_sys_io_getevents) /* 245 */ +SYSCALL(sys_io_submit,compat_sys_io_submit) +SYSCALL(sys_io_cancel,compat_sys_io_cancel) +SYSCALL(sys_exit_group,compat_sys_exit_group) +SYSCALL(sys_epoll_create,compat_sys_epoll_create) +SYSCALL(sys_epoll_ctl,compat_sys_epoll_ctl) /* 250 */ +SYSCALL(sys_epoll_wait,compat_sys_epoll_wait) +SYSCALL(sys_set_tid_address,compat_sys_set_tid_address) +SYSCALL(sys_fadvise64_64,compat_sys_s390_fadvise64) +SYSCALL(sys_timer_create,compat_sys_timer_create) +SYSCALL(sys_timer_settime,compat_sys_timer_settime) /* 255 */ +SYSCALL(sys_timer_gettime,compat_sys_timer_gettime) +SYSCALL(sys_timer_getoverrun,compat_sys_timer_getoverrun) +SYSCALL(sys_timer_delete,compat_sys_timer_delete) +SYSCALL(sys_clock_settime,compat_sys_clock_settime) +SYSCALL(sys_clock_gettime,compat_sys_clock_gettime) /* 260 */ +SYSCALL(sys_clock_getres,compat_sys_clock_getres) +SYSCALL(sys_clock_nanosleep,compat_sys_clock_nanosleep) +NI_SYSCALL /* reserved for vserver */ +SYSCALL(sys_ni_syscall,compat_sys_s390_fadvise64_64) +SYSCALL(sys_statfs64,compat_sys_statfs64) +SYSCALL(sys_fstatfs64,compat_sys_fstatfs64) +SYSCALL(sys_remap_file_pages,compat_sys_remap_file_pages) +NI_SYSCALL /* 268 sys_mbind */ +NI_SYSCALL /* 269 sys_get_mempolicy */ +NI_SYSCALL /* 270 sys_set_mempolicy */ +SYSCALL(sys_mq_open,compat_sys_mq_open) +SYSCALL(sys_mq_unlink,compat_sys_mq_unlink) +SYSCALL(sys_mq_timedsend,compat_sys_mq_timedsend) +SYSCALL(sys_mq_timedreceive,compat_sys_mq_timedreceive) +SYSCALL(sys_mq_notify,compat_sys_mq_notify) /* 275 */ +SYSCALL(sys_mq_getsetattr,compat_sys_mq_getsetattr) +SYSCALL(sys_kexec_load,compat_sys_kexec_load) +SYSCALL(sys_add_key,compat_sys_add_key) +SYSCALL(sys_request_key,compat_sys_request_key) +SYSCALL(sys_keyctl,compat_sys_keyctl) /* 280 */ +SYSCALL(sys_waitid,compat_sys_waitid) +SYSCALL(sys_ioprio_set,compat_sys_ioprio_set) +SYSCALL(sys_ioprio_get,compat_sys_ioprio_get) +SYSCALL(sys_inotify_init,sys_inotify_init) +SYSCALL(sys_inotify_add_watch,compat_sys_inotify_add_watch) /* 285 */ +SYSCALL(sys_inotify_rm_watch,compat_sys_inotify_rm_watch) +NI_SYSCALL /* 287 sys_migrate_pages */ +SYSCALL(sys_openat,compat_sys_openat) +SYSCALL(sys_mkdirat,compat_sys_mkdirat) +SYSCALL(sys_mknodat,compat_sys_mknodat) /* 290 */ +SYSCALL(sys_fchownat,compat_sys_fchownat) +SYSCALL(sys_futimesat,compat_sys_futimesat) +SYSCALL(sys_newfstatat,compat_sys_s390_fstatat64) +SYSCALL(sys_unlinkat,compat_sys_unlinkat) +SYSCALL(sys_renameat,compat_sys_renameat) /* 295 */ +SYSCALL(sys_linkat,compat_sys_linkat) +SYSCALL(sys_symlinkat,compat_sys_symlinkat) +SYSCALL(sys_readlinkat,compat_sys_readlinkat) +SYSCALL(sys_fchmodat,compat_sys_fchmodat) +SYSCALL(sys_faccessat,compat_sys_faccessat) /* 300 */ +SYSCALL(sys_pselect6,compat_sys_pselect6) +SYSCALL(sys_ppoll,compat_sys_ppoll) +SYSCALL(sys_unshare,compat_sys_unshare) +SYSCALL(sys_set_robust_list,compat_sys_set_robust_list) +SYSCALL(sys_get_robust_list,compat_sys_get_robust_list) +SYSCALL(sys_splice,compat_sys_splice) +SYSCALL(sys_sync_file_range,compat_sys_s390_sync_file_range) +SYSCALL(sys_tee,compat_sys_tee) +SYSCALL(sys_vmsplice,compat_sys_vmsplice) +NI_SYSCALL /* 310 sys_move_pages */ +SYSCALL(sys_getcpu,compat_sys_getcpu) +SYSCALL(sys_epoll_pwait,compat_sys_epoll_pwait) +SYSCALL(sys_utimes,compat_sys_utimes) +SYSCALL(sys_fallocate,compat_sys_s390_fallocate) +SYSCALL(sys_utimensat,compat_sys_utimensat) /* 315 */ +SYSCALL(sys_signalfd,compat_sys_signalfd) NI_SYSCALL /* 317 old sys_timer_fd */ -SYSCALL(sys_eventfd,sys_eventfd,compat_sys_eventfd) -SYSCALL(sys_timerfd_create,sys_timerfd_create,compat_sys_timerfd_create) -SYSCALL(sys_timerfd_settime,sys_timerfd_settime,compat_sys_timerfd_settime) /* 320 */ -SYSCALL(sys_timerfd_gettime,sys_timerfd_gettime,compat_sys_timerfd_gettime) -SYSCALL(sys_signalfd4,sys_signalfd4,compat_sys_signalfd4) -SYSCALL(sys_eventfd2,sys_eventfd2,compat_sys_eventfd2) -SYSCALL(sys_inotify_init1,sys_inotify_init1,compat_sys_inotify_init1) -SYSCALL(sys_pipe2,sys_pipe2,compat_sys_pipe2) /* 325 */ -SYSCALL(sys_dup3,sys_dup3,compat_sys_dup3) -SYSCALL(sys_epoll_create1,sys_epoll_create1,compat_sys_epoll_create1) -SYSCALL(sys_preadv,sys_preadv,compat_sys_preadv) -SYSCALL(sys_pwritev,sys_pwritev,compat_sys_pwritev) -SYSCALL(sys_rt_tgsigqueueinfo,sys_rt_tgsigqueueinfo,compat_sys_rt_tgsigqueueinfo) /* 330 */ -SYSCALL(sys_perf_event_open,sys_perf_event_open,compat_sys_perf_event_open) -SYSCALL(sys_fanotify_init,sys_fanotify_init,compat_sys_fanotify_init) -SYSCALL(sys_fanotify_mark,sys_fanotify_mark,compat_sys_fanotify_mark) -SYSCALL(sys_prlimit64,sys_prlimit64,compat_sys_prlimit64) -SYSCALL(sys_name_to_handle_at,sys_name_to_handle_at,compat_sys_name_to_handle_at) /* 335 */ -SYSCALL(sys_open_by_handle_at,sys_open_by_handle_at,compat_sys_open_by_handle_at) -SYSCALL(sys_clock_adjtime,sys_clock_adjtime,compat_sys_clock_adjtime) -SYSCALL(sys_syncfs,sys_syncfs,compat_sys_syncfs) -SYSCALL(sys_setns,sys_setns,compat_sys_setns) -SYSCALL(sys_process_vm_readv,sys_process_vm_readv,compat_sys_process_vm_readv) /* 340 */ -SYSCALL(sys_process_vm_writev,sys_process_vm_writev,compat_sys_process_vm_writev) -SYSCALL(sys_ni_syscall,sys_s390_runtime_instr,compat_sys_s390_runtime_instr) -SYSCALL(sys_kcmp,sys_kcmp,compat_sys_kcmp) -SYSCALL(sys_finit_module,sys_finit_module,compat_sys_finit_module) -SYSCALL(sys_sched_setattr,sys_sched_setattr,compat_sys_sched_setattr) /* 345 */ -SYSCALL(sys_sched_getattr,sys_sched_getattr,compat_sys_sched_getattr) -SYSCALL(sys_renameat2,sys_renameat2,compat_sys_renameat2) -SYSCALL(sys_seccomp,sys_seccomp,compat_sys_seccomp) -SYSCALL(sys_getrandom,sys_getrandom,compat_sys_getrandom) -SYSCALL(sys_memfd_create,sys_memfd_create,compat_sys_memfd_create) /* 350 */ -SYSCALL(sys_bpf,sys_bpf,compat_sys_bpf) -SYSCALL(sys_ni_syscall,sys_s390_pci_mmio_write,compat_sys_s390_pci_mmio_write) -SYSCALL(sys_ni_syscall,sys_s390_pci_mmio_read,compat_sys_s390_pci_mmio_read) -SYSCALL(sys_execveat,sys_execveat,compat_sys_execveat) +SYSCALL(sys_eventfd,compat_sys_eventfd) +SYSCALL(sys_timerfd_create,compat_sys_timerfd_create) +SYSCALL(sys_timerfd_settime,compat_sys_timerfd_settime) /* 320 */ +SYSCALL(sys_timerfd_gettime,compat_sys_timerfd_gettime) +SYSCALL(sys_signalfd4,compat_sys_signalfd4) +SYSCALL(sys_eventfd2,compat_sys_eventfd2) +SYSCALL(sys_inotify_init1,compat_sys_inotify_init1) +SYSCALL(sys_pipe2,compat_sys_pipe2) /* 325 */ +SYSCALL(sys_dup3,compat_sys_dup3) +SYSCALL(sys_epoll_create1,compat_sys_epoll_create1) +SYSCALL(sys_preadv,compat_sys_preadv) +SYSCALL(sys_pwritev,compat_sys_pwritev) +SYSCALL(sys_rt_tgsigqueueinfo,compat_sys_rt_tgsigqueueinfo) /* 330 */ +SYSCALL(sys_perf_event_open,compat_sys_perf_event_open) +SYSCALL(sys_fanotify_init,compat_sys_fanotify_init) +SYSCALL(sys_fanotify_mark,compat_sys_fanotify_mark) +SYSCALL(sys_prlimit64,compat_sys_prlimit64) +SYSCALL(sys_name_to_handle_at,compat_sys_name_to_handle_at) /* 335 */ +SYSCALL(sys_open_by_handle_at,compat_sys_open_by_handle_at) +SYSCALL(sys_clock_adjtime,compat_sys_clock_adjtime) +SYSCALL(sys_syncfs,compat_sys_syncfs) +SYSCALL(sys_setns,compat_sys_setns) +SYSCALL(sys_process_vm_readv,compat_sys_process_vm_readv) /* 340 */ +SYSCALL(sys_process_vm_writev,compat_sys_process_vm_writev) +SYSCALL(sys_s390_runtime_instr,compat_sys_s390_runtime_instr) +SYSCALL(sys_kcmp,compat_sys_kcmp) +SYSCALL(sys_finit_module,compat_sys_finit_module) +SYSCALL(sys_sched_setattr,compat_sys_sched_setattr) /* 345 */ +SYSCALL(sys_sched_getattr,compat_sys_sched_getattr) +SYSCALL(sys_renameat2,compat_sys_renameat2) +SYSCALL(sys_seccomp,compat_sys_seccomp) +SYSCALL(sys_getrandom,compat_sys_getrandom) +SYSCALL(sys_memfd_create,compat_sys_memfd_create) /* 350 */ +SYSCALL(sys_bpf,compat_sys_bpf) +SYSCALL(sys_s390_pci_mmio_write,compat_sys_s390_pci_mmio_write) +SYSCALL(sys_s390_pci_mmio_read,compat_sys_s390_pci_mmio_read) +SYSCALL(sys_execveat,compat_sys_execveat) diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c index 14da43b801d9..5728c5bd44a8 100644 --- a/arch/s390/kernel/topology.c +++ b/arch/s390/kernel/topology.c @@ -421,7 +421,7 @@ int topology_cpu_init(struct cpu *cpu) return sysfs_create_group(&cpu->dev.kobj, &topology_cpu_attr_group); } -const struct cpumask *cpu_thread_mask(int cpu) +static const struct cpumask *cpu_thread_mask(int cpu) { return &per_cpu(cpu_topology, cpu).thread_mask; } diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index f081cf1157c3..4d96c9f53455 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -26,7 +26,6 @@ int show_unhandled_signals = 1; static inline void __user *get_trap_ip(struct pt_regs *regs) { -#ifdef CONFIG_64BIT unsigned long address; if (regs->int_code & 0x200) @@ -35,10 +34,6 @@ static inline void __user *get_trap_ip(struct pt_regs *regs) address = regs->psw.addr; return (void __user *) ((address - (regs->int_code >> 16)) & PSW_ADDR_INSN); -#else - return (void __user *) - ((regs->psw.addr - (regs->int_code >> 16)) & PSW_ADDR_INSN); -#endif } static inline void report_user_fault(struct pt_regs *regs, int signr) @@ -153,11 +148,8 @@ DO_ERROR_INFO(privileged_op, SIGILL, ILL_PRVOPC, "privileged operation") DO_ERROR_INFO(special_op_exception, SIGILL, ILL_ILLOPN, "special operation exception") - -#ifdef CONFIG_64BIT DO_ERROR_INFO(transaction_exception, SIGILL, ILL_ILLOPN, "transaction constraint exception") -#endif static inline void do_fp_trap(struct pt_regs *regs, int fpc) { @@ -182,7 +174,7 @@ static inline void do_fp_trap(struct pt_regs *regs, int fpc) void translation_exception(struct pt_regs *regs) { /* May never happen. */ - die(regs, "Translation exception"); + panic("Translation exception"); } void illegal_op(struct pt_regs *regs) @@ -211,29 +203,6 @@ void illegal_op(struct pt_regs *regs) } else if (*((__u16 *) opcode) == UPROBE_SWBP_INSN) { is_uprobe_insn = 1; #endif -#ifdef CONFIG_MATHEMU - } else if (opcode[0] == 0xb3) { - if (get_user(*((__u16 *) (opcode+2)), location+1)) - return; - signal = math_emu_b3(opcode, regs); - } else if (opcode[0] == 0xed) { - if (get_user(*((__u32 *) (opcode+2)), - (__u32 __user *)(location+1))) - return; - signal = math_emu_ed(opcode, regs); - } else if (*((__u16 *) opcode) == 0xb299) { - if (get_user(*((__u16 *) (opcode+2)), location+1)) - return; - signal = math_emu_srnm(opcode, regs); - } else if (*((__u16 *) opcode) == 0xb29c) { - if (get_user(*((__u16 *) (opcode+2)), location+1)) - return; - signal = math_emu_stfpc(opcode, regs); - } else if (*((__u16 *) opcode) == 0xb29d) { - if (get_user(*((__u16 *) (opcode+2)), location+1)) - return; - signal = math_emu_lfpc(opcode, regs); -#endif } else signal = SIGILL; } @@ -247,71 +216,14 @@ void illegal_op(struct pt_regs *regs) 3, SIGTRAP) != NOTIFY_STOP) signal = SIGILL; } - -#ifdef CONFIG_MATHEMU - if (signal == SIGFPE) - do_fp_trap(regs, current->thread.fp_regs.fpc); - else if (signal == SIGSEGV) - do_trap(regs, signal, SEGV_MAPERR, "user address fault"); - else -#endif if (signal) do_trap(regs, signal, ILL_ILLOPC, "illegal operation"); } NOKPROBE_SYMBOL(illegal_op); -#ifdef CONFIG_MATHEMU -void specification_exception(struct pt_regs *regs) -{ - __u8 opcode[6]; - __u16 __user *location = NULL; - int signal = 0; - - location = (__u16 __user *) get_trap_ip(regs); - - if (user_mode(regs)) { - get_user(*((__u16 *) opcode), location); - switch (opcode[0]) { - case 0x28: /* LDR Rx,Ry */ - signal = math_emu_ldr(opcode); - break; - case 0x38: /* LER Rx,Ry */ - signal = math_emu_ler(opcode); - break; - case 0x60: /* STD R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_std(opcode, regs); - break; - case 0x68: /* LD R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_ld(opcode, regs); - break; - case 0x70: /* STE R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_ste(opcode, regs); - break; - case 0x78: /* LE R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_le(opcode, regs); - break; - default: - signal = SIGILL; - break; - } - } else - signal = SIGILL; - - if (signal == SIGFPE) - do_fp_trap(regs, current->thread.fp_regs.fpc); - else if (signal) - do_trap(regs, signal, ILL_ILLOPN, "specification exception"); -} -#else DO_ERROR_INFO(specification_exception, SIGILL, ILL_ILLOPN, "specification exception"); -#endif -#ifdef CONFIG_64BIT int alloc_vector_registers(struct task_struct *tsk) { __vector128 *vxrs; @@ -377,7 +289,6 @@ static int __init disable_vector_extension(char *str) return 1; } __setup("novx", disable_vector_extension); -#endif void data_exception(struct pt_regs *regs) { @@ -386,65 +297,7 @@ void data_exception(struct pt_regs *regs) location = get_trap_ip(regs); - if (MACHINE_HAS_IEEE) - asm volatile("stfpc %0" : "=m" (current->thread.fp_regs.fpc)); - -#ifdef CONFIG_MATHEMU - else if (user_mode(regs)) { - __u8 opcode[6]; - get_user(*((__u16 *) opcode), location); - switch (opcode[0]) { - case 0x28: /* LDR Rx,Ry */ - signal = math_emu_ldr(opcode); - break; - case 0x38: /* LER Rx,Ry */ - signal = math_emu_ler(opcode); - break; - case 0x60: /* STD R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_std(opcode, regs); - break; - case 0x68: /* LD R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_ld(opcode, regs); - break; - case 0x70: /* STE R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_ste(opcode, regs); - break; - case 0x78: /* LE R,D(X,B) */ - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_le(opcode, regs); - break; - case 0xb3: - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_b3(opcode, regs); - break; - case 0xed: - get_user(*((__u32 *) (opcode+2)), - (__u32 __user *)(location+1)); - signal = math_emu_ed(opcode, regs); - break; - case 0xb2: - if (opcode[1] == 0x99) { - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_srnm(opcode, regs); - } else if (opcode[1] == 0x9c) { - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_stfpc(opcode, regs); - } else if (opcode[1] == 0x9d) { - get_user(*((__u16 *) (opcode+2)), location+1); - signal = math_emu_lfpc(opcode, regs); - } else - signal = SIGILL; - break; - default: - signal = SIGILL; - break; - } - } -#endif -#ifdef CONFIG_64BIT + asm volatile("stfpc %0" : "=m" (current->thread.fp_regs.fpc)); /* Check for vector register enablement */ if (MACHINE_HAS_VX && !current->thread.vxrs && (current->thread.fp_regs.fpc & FPC_DXC_MASK) == 0xfe00) { @@ -454,13 +307,11 @@ void data_exception(struct pt_regs *regs) clear_pt_regs_flag(regs, PIF_PER_TRAP); return; } -#endif - if (current->thread.fp_regs.fpc & FPC_DXC_MASK) signal = SIGFPE; else signal = SIGILL; - if (signal == SIGFPE) + if (signal == SIGFPE) do_fp_trap(regs, current->thread.fp_regs.fpc); else if (signal) do_trap(regs, signal, ILL_ILLOPN, "data exception"); diff --git a/arch/s390/kernel/uprobes.c b/arch/s390/kernel/uprobes.c index cc7328080b60..66956c09d5bf 100644 --- a/arch/s390/kernel/uprobes.c +++ b/arch/s390/kernel/uprobes.c @@ -188,7 +188,9 @@ static void adjust_psw_addr(psw_t *psw, unsigned long len) else if (put_user(*(input), __ptr)) \ __rc = EMU_ADDRESSING; \ if (__rc == 0) \ - sim_stor_event(regs, __ptr, mask + 1); \ + sim_stor_event(regs, \ + (void __force *)__ptr, \ + mask + 1); \ __rc; \ }) diff --git a/arch/s390/kernel/vdso.c b/arch/s390/kernel/vdso.c index 0bbb7e027c5a..0d58269ff425 100644 --- a/arch/s390/kernel/vdso.c +++ b/arch/s390/kernel/vdso.c @@ -32,19 +32,17 @@ #include <asm/vdso.h> #include <asm/facility.h> -#if defined(CONFIG_32BIT) || defined(CONFIG_COMPAT) +#ifdef CONFIG_COMPAT extern char vdso32_start, vdso32_end; static void *vdso32_kbase = &vdso32_start; static unsigned int vdso32_pages; static struct page **vdso32_pagelist; #endif -#ifdef CONFIG_64BIT extern char vdso64_start, vdso64_end; static void *vdso64_kbase = &vdso64_start; static unsigned int vdso64_pages; static struct page **vdso64_pagelist; -#endif /* CONFIG_64BIT */ /* * Should the kernel map a VDSO page into processes and pass its @@ -87,7 +85,6 @@ static void vdso_init_data(struct vdso_data *vd) vd->ectg_available = test_facility(31); } -#ifdef CONFIG_64BIT /* * Allocate/free per cpu vdso data. */ @@ -169,7 +166,6 @@ static void vdso_init_cr5(void) cr5 = offsetof(struct _lowcore, paste); __ctl_load(cr5, 5, 5); } -#endif /* CONFIG_64BIT */ /* * This is called from binfmt_elf, we create the special vma for the @@ -191,7 +187,6 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) if (!uses_interp) return 0; -#ifdef CONFIG_64BIT vdso_pagelist = vdso64_pagelist; vdso_pages = vdso64_pages; #ifdef CONFIG_COMPAT @@ -200,11 +195,6 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) vdso_pages = vdso32_pages; } #endif -#else - vdso_pagelist = vdso32_pagelist; - vdso_pages = vdso32_pages; -#endif - /* * vDSO has a problem and was disabled, just don't "enable" it for * the process @@ -268,7 +258,7 @@ static int __init vdso_init(void) if (!vdso_enabled) return 0; vdso_init_data(vdso_data); -#if defined(CONFIG_32BIT) || defined(CONFIG_COMPAT) +#ifdef CONFIG_COMPAT /* Calculate the size of the 32 bit vDSO */ vdso32_pages = ((&vdso32_end - &vdso32_start + PAGE_SIZE - 1) >> PAGE_SHIFT) + 1; @@ -287,7 +277,6 @@ static int __init vdso_init(void) vdso32_pagelist[vdso32_pages] = NULL; #endif -#ifdef CONFIG_64BIT /* Calculate the size of the 64 bit vDSO */ vdso64_pages = ((&vdso64_end - &vdso64_start + PAGE_SIZE - 1) >> PAGE_SHIFT) + 1; @@ -307,7 +296,6 @@ static int __init vdso_init(void) if (vdso_alloc_per_cpu(&S390_lowcore)) BUG(); vdso_init_cr5(); -#endif /* CONFIG_64BIT */ get_page(virt_to_page(vdso_data)); diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 35b13ed0af5f..445657fe658c 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -6,17 +6,10 @@ #include <asm/page.h> #include <asm-generic/vmlinux.lds.h> -#ifndef CONFIG_64BIT -OUTPUT_FORMAT("elf32-s390", "elf32-s390", "elf32-s390") -OUTPUT_ARCH(s390:31-bit) -ENTRY(startup) -jiffies = jiffies_64 + 4; -#else OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") OUTPUT_ARCH(s390:64-bit) ENTRY(startup) jiffies = jiffies_64; -#endif PHDRS { text PT_LOAD FLAGS(5); /* R_E */ diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index a01df233856f..0e8fefe5b0ce 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,8 +3,7 @@ # lib-y += delay.o string.o uaccess.o find.o -obj-$(CONFIG_32BIT) += div64.o qrnnd.o ucmpdi2.o mem32.o -obj-$(CONFIG_64BIT) += mem64.o +obj-y += mem.o lib-$(CONFIG_SMP) += spinlock.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o diff --git a/arch/s390/lib/div64.c b/arch/s390/lib/div64.c deleted file mode 100644 index 261152f83242..000000000000 --- a/arch/s390/lib/div64.c +++ /dev/null @@ -1,147 +0,0 @@ -/* - * __div64_32 implementation for 31 bit. - * - * Copyright IBM Corp. 2006 - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), - */ - -#include <linux/types.h> -#include <linux/module.h> - -#ifdef CONFIG_MARCH_G5 - -/* - * Function to divide an unsigned 64 bit integer by an unsigned - * 31 bit integer using signed 64/32 bit division. - */ -static uint32_t __div64_31(uint64_t *n, uint32_t base) -{ - register uint32_t reg2 asm("2"); - register uint32_t reg3 asm("3"); - uint32_t *words = (uint32_t *) n; - uint32_t tmp; - - /* Special case base==1, remainder = 0, quotient = n */ - if (base == 1) - return 0; - /* - * Special case base==0 will cause a fixed point divide exception - * on the dr instruction and may not happen anyway. For the - * following calculation we can assume base > 1. The first - * signed 64 / 32 bit division with an upper half of 0 will - * give the correct upper half of the 64 bit quotient. - */ - reg2 = 0UL; - reg3 = words[0]; - asm volatile( - " dr %0,%2\n" - : "+d" (reg2), "+d" (reg3) : "d" (base) : "cc" ); - words[0] = reg3; - reg3 = words[1]; - /* - * To get the lower half of the 64 bit quotient and the 32 bit - * remainder we have to use a little trick. Since we only have - * a signed division the quotient can get too big. To avoid this - * the 64 bit dividend is halved, then the signed division will - * work. Afterwards the quotient and the remainder are doubled. - * If the last bit of the dividend has been one the remainder - * is increased by one then checked against the base. If the - * remainder has overflown subtract base and increase the - * quotient. Simple, no ? - */ - asm volatile( - " nr %2,%1\n" - " srdl %0,1\n" - " dr %0,%3\n" - " alr %0,%0\n" - " alr %1,%1\n" - " alr %0,%2\n" - " clr %0,%3\n" - " jl 0f\n" - " slr %0,%3\n" - " ahi %1,1\n" - "0:\n" - : "+d" (reg2), "+d" (reg3), "=d" (tmp) - : "d" (base), "2" (1UL) : "cc" ); - words[1] = reg3; - return reg2; -} - -/* - * Function to divide an unsigned 64 bit integer by an unsigned - * 32 bit integer using the unsigned 64/31 bit division. - */ -uint32_t __div64_32(uint64_t *n, uint32_t base) -{ - uint32_t r; - - /* - * If the most significant bit of base is set, divide n by - * (base/2). That allows to use 64/31 bit division and gives a - * good approximation of the result: n = (base/2)*q + r. The - * result needs to be corrected with two simple transformations. - * If base is already < 2^31-1 __div64_31 can be used directly. - */ - r = __div64_31(n, ((signed) base < 0) ? (base/2) : base); - if ((signed) base < 0) { - uint64_t q = *n; - /* - * First transformation: - * n = (base/2)*q + r - * = ((base/2)*2)*(q/2) + ((q&1) ? (base/2) : 0) + r - * Since r < (base/2), r + (base/2) < base. - * With q1 = (q/2) and r1 = r + ((q&1) ? (base/2) : 0) - * n = ((base/2)*2)*q1 + r1 with r1 < base. - */ - if (q & 1) - r += base/2; - q >>= 1; - /* - * Second transformation. ((base/2)*2) could have lost the - * last bit. - * n = ((base/2)*2)*q1 + r1 - * = base*q1 - ((base&1) ? q1 : 0) + r1 - */ - if (base & 1) { - int64_t rx = r - q; - /* - * base is >= 2^31. The worst case for the while - * loop is n=2^64-1 base=2^31+1. That gives a - * maximum for q=(2^64-1)/2^31 = 0x1ffffffff. Since - * base >= 2^31 the loop is finished after a maximum - * of three iterations. - */ - while (rx < 0) { - rx += base; - q--; - } - r = rx; - } - *n = q; - } - return r; -} - -#else /* MARCH_G5 */ - -uint32_t __div64_32(uint64_t *n, uint32_t base) -{ - register uint32_t reg2 asm("2"); - register uint32_t reg3 asm("3"); - uint32_t *words = (uint32_t *) n; - - reg2 = 0UL; - reg3 = words[0]; - asm volatile( - " dlr %0,%2\n" - : "+d" (reg2), "+d" (reg3) : "d" (base) : "cc" ); - words[0] = reg3; - reg3 = words[1]; - asm volatile( - " dlr %0,%2\n" - : "+d" (reg2), "+d" (reg3) : "d" (base) : "cc" ); - words[1] = reg3; - return reg2; -} - -#endif /* MARCH_G5 */ diff --git a/arch/s390/lib/mem64.S b/arch/s390/lib/mem.S index c6d553e85ab1..c6d553e85ab1 100644 --- a/arch/s390/lib/mem64.S +++ b/arch/s390/lib/mem.S diff --git a/arch/s390/lib/mem32.S b/arch/s390/lib/mem32.S deleted file mode 100644 index 14ca9244b615..000000000000 --- a/arch/s390/lib/mem32.S +++ /dev/null @@ -1,92 +0,0 @@ -/* - * String handling functions. - * - * Copyright IBM Corp. 2012 - */ - -#include <linux/linkage.h> - -/* - * memset implementation - * - * This code corresponds to the C construct below. We do distinguish - * between clearing (c == 0) and setting a memory array (c != 0) simply - * because nearly all memset invocations in the kernel clear memory and - * the xc instruction is preferred in such cases. - * - * void *memset(void *s, int c, size_t n) - * { - * if (likely(c == 0)) - * return __builtin_memset(s, 0, n); - * return __builtin_memset(s, c, n); - * } - */ -ENTRY(memset) - basr %r5,%r0 -.Lmemset_base: - ltr %r4,%r4 - bzr %r14 - ltr %r3,%r3 - jnz .Lmemset_fill - ahi %r4,-1 - lr %r3,%r4 - srl %r3,8 - ltr %r3,%r3 - lr %r1,%r2 - je .Lmemset_clear_rest -.Lmemset_clear_loop: - xc 0(256,%r1),0(%r1) - la %r1,256(%r1) - brct %r3,.Lmemset_clear_loop -.Lmemset_clear_rest: - ex %r4,.Lmemset_xc-.Lmemset_base(%r5) - br %r14 -.Lmemset_fill: - stc %r3,0(%r2) - chi %r4,1 - lr %r1,%r2 - ber %r14 - ahi %r4,-2 - lr %r3,%r4 - srl %r3,8 - ltr %r3,%r3 - je .Lmemset_fill_rest -.Lmemset_fill_loop: - mvc 1(256,%r1),0(%r1) - la %r1,256(%r1) - brct %r3,.Lmemset_fill_loop -.Lmemset_fill_rest: - ex %r4,.Lmemset_mvc-.Lmemset_base(%r5) - br %r14 -.Lmemset_xc: - xc 0(1,%r1),0(%r1) -.Lmemset_mvc: - mvc 1(1,%r1),0(%r1) - -/* - * memcpy implementation - * - * void *memcpy(void *dest, const void *src, size_t n) - */ -ENTRY(memcpy) - basr %r5,%r0 -.Lmemcpy_base: - ltr %r4,%r4 - bzr %r14 - ahi %r4,-1 - lr %r0,%r4 - srl %r0,8 - ltr %r0,%r0 - lr %r1,%r2 - jnz .Lmemcpy_loop -.Lmemcpy_rest: - ex %r4,.Lmemcpy_mvc-.Lmemcpy_base(%r5) - br %r14 -.Lmemcpy_loop: - mvc 0(256,%r1),0(%r3) - la %r1,256(%r1) - la %r3,256(%r3) - brct %r0,.Lmemcpy_loop - j .Lmemcpy_rest -.Lmemcpy_mvc: - mvc 0(1,%r1),0(%r3) diff --git a/arch/s390/lib/qrnnd.S b/arch/s390/lib/qrnnd.S deleted file mode 100644 index d321329130ec..000000000000 --- a/arch/s390/lib/qrnnd.S +++ /dev/null @@ -1,78 +0,0 @@ -# S/390 __udiv_qrnnd - -#include <linux/linkage.h> - -# r2 : &__r -# r3 : upper half of 64 bit word n -# r4 : lower half of 64 bit word n -# r5 : divisor d -# the reminder r of the division is to be stored to &__r and -# the quotient q is to be returned - - .text -ENTRY(__udiv_qrnnd) - st %r2,24(%r15) # store pointer to reminder for later - lr %r0,%r3 # reload n - lr %r1,%r4 - ltr %r2,%r5 # reload and test divisor - jp 5f - # divisor >= 0x80000000 - srdl %r0,2 # n/4 - srl %r2,1 # d/2 - slr %r1,%r2 # special case if last bit of d is set - brc 3,0f # (n/4) div (n/2) can overflow by 1 - ahi %r0,-1 # trick: subtract n/2, then divide -0: dr %r0,%r2 # signed division - ahi %r1,1 # trick part 2: add 1 to the quotient - # now (n >> 2) = (d >> 1) * %r1 + %r0 - lhi %r3,1 - nr %r3,%r1 # test last bit of q - jz 1f - alr %r0,%r2 # add (d>>1) to r -1: srl %r1,1 # q >>= 1 - # now (n >> 2) = (d&-2) * %r1 + %r0 - lhi %r3,1 - nr %r3,%r5 # test last bit of d - jz 2f - slr %r0,%r1 # r -= q - brc 3,2f # borrow ? - alr %r0,%r5 # r += d - ahi %r1,-1 -2: # now (n >> 2) = d * %r1 + %r0 - alr %r1,%r1 # q <<= 1 - alr %r0,%r0 # r <<= 1 - brc 12,3f # overflow on r ? - slr %r0,%r5 # r -= d - ahi %r1,1 # q += 1 -3: lhi %r3,2 - nr %r3,%r4 # test next to last bit of n - jz 4f - ahi %r0,1 # r += 1 -4: clr %r0,%r5 # r >= d ? - jl 6f - slr %r0,%r5 # r -= d - ahi %r1,1 # q += 1 - # now (n >> 1) = d * %r1 + %r0 - j 6f -5: # divisor < 0x80000000 - srdl %r0,1 - dr %r0,%r2 # signed division - # now (n >> 1) = d * %r1 + %r0 -6: alr %r1,%r1 # q <<= 1 - alr %r0,%r0 # r <<= 1 - brc 12,7f # overflow on r ? - slr %r0,%r5 # r -= d - ahi %r1,1 # q += 1 -7: lhi %r3,1 - nr %r3,%r4 # isolate last bit of n - alr %r0,%r3 # r += (n & 1) - clr %r0,%r5 # r >= d ? - jl 8f - slr %r0,%r5 # r -= d - ahi %r1,1 # q += 1 -8: # now n = d * %r1 + %r0 - l %r2,24(%r15) - st %r0,0(%r2) - lr %r2,%r1 - br %r14 - .end __udiv_qrnnd diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 53dd5d7a0c96..4614d415bb58 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -15,20 +15,6 @@ #include <asm/mmu_context.h> #include <asm/facility.h> -#ifndef CONFIG_64BIT -#define AHI "ahi" -#define ALR "alr" -#define CLR "clr" -#define LHI "lhi" -#define SLR "slr" -#else -#define AHI "aghi" -#define ALR "algr" -#define CLR "clgr" -#define LHI "lghi" -#define SLR "slgr" -#endif - static struct static_key have_mvcos = STATIC_KEY_INIT_FALSE; static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr, @@ -41,29 +27,29 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr asm volatile( "0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n" "9: jz 7f\n" - "1:"ALR" %0,%3\n" - " "SLR" %1,%3\n" - " "SLR" %2,%3\n" + "1: algr %0,%3\n" + " slgr %1,%3\n" + " slgr %2,%3\n" " j 0b\n" "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ - " "SLR" %4,%1\n" - " "CLR" %0,%4\n" /* copy crosses next page boundary? */ + " slgr %4,%1\n" + " clgr %0,%4\n" /* copy crosses next page boundary? */ " jnh 4f\n" "3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n" - "10:"SLR" %0,%4\n" - " "ALR" %2,%4\n" - "4:"LHI" %4,-1\n" - " "ALR" %4,%0\n" /* copy remaining size, subtract 1 */ + "10:slgr %0,%4\n" + " algr %2,%4\n" + "4: lghi %4,-1\n" + " algr %4,%0\n" /* copy remaining size, subtract 1 */ " bras %3,6f\n" /* memset loop */ " xc 0(1,%2),0(%2)\n" "5: xc 0(256,%2),0(%2)\n" " la %2,256(%2)\n" - "6:"AHI" %4,-256\n" + "6: aghi %4,-256\n" " jnm 5b\n" " ex %4,0(%3)\n" " j 8f\n" - "7:"SLR" %0,%0\n" + "7:slgr %0,%0\n" "8:\n" EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) @@ -82,32 +68,32 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr, " sacf 0\n" "0: mvcp 0(%0,%2),0(%1),%3\n" "10:jz 8f\n" - "1:"ALR" %0,%3\n" + "1: algr %0,%3\n" " la %1,256(%1)\n" " la %2,256(%2)\n" "2: mvcp 0(%0,%2),0(%1),%3\n" "11:jnz 1b\n" " j 8f\n" "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ - " "LHI" %3,-4096\n" + " lghi %3,-4096\n" " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ - " "SLR" %4,%1\n" - " "CLR" %0,%4\n" /* copy crosses next page boundary? */ + " slgr %4,%1\n" + " clgr %0,%4\n" /* copy crosses next page boundary? */ " jnh 5f\n" "4: mvcp 0(%4,%2),0(%1),%3\n" - "12:"SLR" %0,%4\n" - " "ALR" %2,%4\n" - "5:"LHI" %4,-1\n" - " "ALR" %4,%0\n" /* copy remaining size, subtract 1 */ + "12:slgr %0,%4\n" + " algr %2,%4\n" + "5: lghi %4,-1\n" + " algr %4,%0\n" /* copy remaining size, subtract 1 */ " bras %3,7f\n" /* memset loop */ " xc 0(1,%2),0(%2)\n" "6: xc 0(256,%2),0(%2)\n" " la %2,256(%2)\n" - "7:"AHI" %4,-256\n" + "7: aghi %4,-256\n" " jnm 6b\n" " ex %4,0(%3)\n" " j 9f\n" - "8:"SLR" %0,%0\n" + "8:slgr %0,%0\n" "9: sacf 768\n" EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b) EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b) @@ -134,19 +120,19 @@ static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x, asm volatile( "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" "6: jz 4f\n" - "1:"ALR" %0,%3\n" - " "SLR" %1,%3\n" - " "SLR" %2,%3\n" + "1: algr %0,%3\n" + " slgr %1,%3\n" + " slgr %2,%3\n" " j 0b\n" "2: la %4,4095(%1)\n"/* %4 = ptr + 4095 */ " nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */ - " "SLR" %4,%1\n" - " "CLR" %0,%4\n" /* copy crosses next page boundary? */ + " slgr %4,%1\n" + " clgr %0,%4\n" /* copy crosses next page boundary? */ " jnh 5f\n" "3: .insn ss,0xc80000000000,0(%4,%1),0(%2),0\n" - "7:"SLR" %0,%4\n" + "7: slgr %0,%4\n" " j 5f\n" - "4:"SLR" %0,%0\n" + "4: slgr %0,%0\n" "5:\n" EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b) : "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2) @@ -165,22 +151,22 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x, " sacf 0\n" "0: mvcs 0(%0,%1),0(%2),%3\n" "7: jz 5f\n" - "1:"ALR" %0,%3\n" + "1: algr %0,%3\n" " la %1,256(%1)\n" " la %2,256(%2)\n" "2: mvcs 0(%0,%1),0(%2),%3\n" "8: jnz 1b\n" " j 5f\n" "3: la %4,255(%1)\n" /* %4 = ptr + 255 */ - " "LHI" %3,-4096\n" + " lghi %3,-4096\n" " nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */ - " "SLR" %4,%1\n" - " "CLR" %0,%4\n" /* copy crosses next page boundary? */ + " slgr %4,%1\n" + " clgr %0,%4\n" /* copy crosses next page boundary? */ " jnh 6f\n" "4: mvcs 0(%4,%1),0(%2),%3\n" - "9:"SLR" %0,%4\n" + "9: slgr %0,%4\n" " j 6f\n" - "5:"SLR" %0,%0\n" + "5: slgr %0,%0\n" "6: sacf 768\n" EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b) EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b) @@ -208,11 +194,11 @@ static inline unsigned long copy_in_user_mvcos(void __user *to, const void __use asm volatile( "0: .insn ss,0xc80000000000,0(%0,%1),0(%2),0\n" " jz 2f\n" - "1:"ALR" %0,%3\n" - " "SLR" %1,%3\n" - " "SLR" %2,%3\n" + "1: algr %0,%3\n" + " slgr %1,%3\n" + " slgr %2,%3\n" " j 0b\n" - "2:"SLR" %0,%0\n" + "2:slgr %0,%0\n" "3: \n" EX_TABLE(0b,3b) : "+a" (size), "+a" (to), "+a" (from), "+a" (tmp1), "=a" (tmp2) @@ -228,23 +214,23 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user load_kernel_asce(); asm volatile( " sacf 256\n" - " "AHI" %0,-1\n" + " aghi %0,-1\n" " jo 5f\n" " bras %3,3f\n" - "0:"AHI" %0,257\n" + "0: aghi %0,257\n" "1: mvc 0(1,%1),0(%2)\n" " la %1,1(%1)\n" " la %2,1(%2)\n" - " "AHI" %0,-1\n" + " aghi %0,-1\n" " jnz 1b\n" " j 5f\n" "2: mvc 0(256,%1),0(%2)\n" " la %1,256(%1)\n" " la %2,256(%2)\n" - "3:"AHI" %0,-256\n" + "3: aghi %0,-256\n" " jnm 2b\n" "4: ex %0,1b-0b(%3)\n" - "5: "SLR" %0,%0\n" + "5: slgr %0,%0\n" "6: sacf 768\n" EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) : "+a" (size), "+a" (to), "+a" (from), "=a" (tmp1) @@ -269,18 +255,18 @@ static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size asm volatile( "0: .insn ss,0xc80000000000,0(%0,%1),0(%4),0\n" " jz 4f\n" - "1:"ALR" %0,%2\n" - " "SLR" %1,%2\n" + "1: algr %0,%2\n" + " slgr %1,%2\n" " j 0b\n" "2: la %3,4095(%1)\n"/* %4 = to + 4095 */ " nr %3,%2\n" /* %4 = (to + 4095) & -4096 */ - " "SLR" %3,%1\n" - " "CLR" %0,%3\n" /* copy crosses next page boundary? */ + " slgr %3,%1\n" + " clgr %0,%3\n" /* copy crosses next page boundary? */ " jnh 5f\n" "3: .insn ss,0xc80000000000,0(%3,%1),0(%4),0\n" - " "SLR" %0,%3\n" + " slgr %0,%3\n" " j 5f\n" - "4:"SLR" %0,%0\n" + "4:slgr %0,%0\n" "5:\n" EX_TABLE(0b,2b) EX_TABLE(3b,5b) : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2) @@ -295,28 +281,28 @@ static inline unsigned long clear_user_xc(void __user *to, unsigned long size) load_kernel_asce(); asm volatile( " sacf 256\n" - " "AHI" %0,-1\n" + " aghi %0,-1\n" " jo 5f\n" " bras %3,3f\n" " xc 0(1,%1),0(%1)\n" - "0:"AHI" %0,257\n" + "0: aghi %0,257\n" " la %2,255(%1)\n" /* %2 = ptr + 255 */ " srl %2,12\n" " sll %2,12\n" /* %2 = (ptr + 255) & -4096 */ - " "SLR" %2,%1\n" - " "CLR" %0,%2\n" /* clear crosses next page boundary? */ + " slgr %2,%1\n" + " clgr %0,%2\n" /* clear crosses next page boundary? */ " jnh 5f\n" - " "AHI" %2,-1\n" + " aghi %2,-1\n" "1: ex %2,0(%3)\n" - " "AHI" %2,1\n" - " "SLR" %0,%2\n" + " aghi %2,1\n" + " slgr %0,%2\n" " j 5f\n" "2: xc 0(256,%1),0(%1)\n" " la %1,256(%1)\n" - "3:"AHI" %0,-256\n" + "3: aghi %0,-256\n" " jnm 2b\n" "4: ex %0,0(%3)\n" - "5: "SLR" %0,%0\n" + "5: slgr %0,%0\n" "6: sacf 768\n" EX_TABLE(1b,6b) EX_TABLE(2b,0b) EX_TABLE(4b,0b) : "+a" (size), "+a" (to), "=a" (tmp1), "=a" (tmp2) @@ -341,12 +327,12 @@ static inline unsigned long strnlen_user_srst(const char __user *src, asm volatile( " la %2,0(%1)\n" " la %3,0(%0,%1)\n" - " "SLR" %0,%0\n" + " slgr %0,%0\n" " sacf 256\n" "0: srst %3,%2\n" " jo 0b\n" " la %0,1(%3)\n" /* strnlen_user results includes \0 */ - " "SLR" %0,%1\n" + " slgr %0,%1\n" "1: sacf 768\n" EX_TABLE(0b,1b) : "+a" (size), "+a" (src), "=a" (tmp1), "=a" (tmp2) @@ -399,7 +385,7 @@ early_param("uaccess_primary", parse_uaccess_pt); static int __init uaccess_init(void) { - if (IS_ENABLED(CONFIG_64BIT) && !uaccess_primary && test_facility(27)) + if (!uaccess_primary && test_facility(27)) static_key_slow_inc(&have_mvcos); return 0; } diff --git a/arch/s390/lib/ucmpdi2.c b/arch/s390/lib/ucmpdi2.c deleted file mode 100644 index 3e05ff532582..000000000000 --- a/arch/s390/lib/ucmpdi2.c +++ /dev/null @@ -1,26 +0,0 @@ -#include <linux/module.h> - -union ull_union { - unsigned long long ull; - struct { - unsigned int high; - unsigned int low; - } ui; -}; - -int __ucmpdi2(unsigned long long a, unsigned long long b) -{ - union ull_union au = {.ull = a}; - union ull_union bu = {.ull = b}; - - if (au.ui.high < bu.ui.high) - return 0; - else if (au.ui.high > bu.ui.high) - return 2; - if (au.ui.low < bu.ui.low) - return 0; - else if (au.ui.low > bu.ui.low) - return 2; - return 1; -} -EXPORT_SYMBOL(__ucmpdi2); diff --git a/arch/s390/math-emu/Makefile b/arch/s390/math-emu/Makefile deleted file mode 100644 index 51d399549f60..000000000000 --- a/arch/s390/math-emu/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# -# Makefile for the FPU instruction emulation. -# - -obj-$(CONFIG_MATHEMU) := math.o - -ccflags-y := -I$(src) -Iinclude/math-emu -w diff --git a/arch/s390/math-emu/math.c b/arch/s390/math-emu/math.c deleted file mode 100644 index a6ba0d724335..000000000000 --- a/arch/s390/math-emu/math.c +++ /dev/null @@ -1,2255 +0,0 @@ -/* - * S390 version - * Copyright IBM Corp. 1999, 2001 - * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), - * - * 'math.c' emulates IEEE instructions on a S390 processor - * that does not have the IEEE fpu (all processors before G5). - */ - -#include <linux/types.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <asm/uaccess.h> -#include <asm/lowcore.h> - -#include <asm/sfp-util.h> -#include <math-emu/soft-fp.h> -#include <math-emu/single.h> -#include <math-emu/double.h> -#include <math-emu/quad.h> - -#define FPC_VALID_MASK 0xF8F8FF03 - -/* - * I miss a macro to round a floating point number to the - * nearest integer in the same floating point format. - */ -#define _FP_TO_FPINT_ROUND(fs, wc, X) \ - do { \ - switch (X##_c) \ - { \ - case FP_CLS_NORMAL: \ - if (X##_e > _FP_FRACBITS_##fs + _FP_EXPBIAS_##fs) \ - { /* floating point number has no bits after the dot. */ \ - } \ - else if (X##_e <= _FP_FRACBITS_##fs + _FP_EXPBIAS_##fs && \ - X##_e > _FP_EXPBIAS_##fs) \ - { /* some bits before the dot, some after it. */ \ - _FP_FRAC_SRS_##wc(X, _FP_WFRACBITS_##fs, \ - X##_e - _FP_EXPBIAS_##fs \ - + _FP_FRACBITS_##fs); \ - _FP_ROUND(wc, X); \ - _FP_FRAC_SLL_##wc(X, X##_e - _FP_EXPBIAS_##fs \ - + _FP_FRACBITS_##fs); \ - } \ - else \ - { /* all bits after the dot. */ \ - FP_SET_EXCEPTION(FP_EX_INEXACT); \ - X##_c = FP_CLS_ZERO; \ - } \ - break; \ - case FP_CLS_NAN: \ - case FP_CLS_INF: \ - case FP_CLS_ZERO: \ - break; \ - } \ - } while (0) - -#define FP_TO_FPINT_ROUND_S(X) _FP_TO_FPINT_ROUND(S,1,X) -#define FP_TO_FPINT_ROUND_D(X) _FP_TO_FPINT_ROUND(D,2,X) -#define FP_TO_FPINT_ROUND_Q(X) _FP_TO_FPINT_ROUND(Q,4,X) - -typedef union { - long double ld; - struct { - __u64 high; - __u64 low; - } w; -} mathemu_ldcv; - -#ifdef CONFIG_SYSCTL -int sysctl_ieee_emulation_warnings=1; -#endif - -#define mathemu_put_user(x, p) \ - do { \ - if (put_user((x),(p))) \ - return SIGSEGV; \ - } while (0) - -#define mathemu_get_user(x, p) \ - do { \ - if (get_user((x),(p))) \ - return SIGSEGV; \ - } while (0) - -#define mathemu_copy_from_user(d, s, n)\ - do { \ - if (copy_from_user((d),(s),(n)) != 0) \ - return SIGSEGV; \ - } while (0) - -#define mathemu_copy_to_user(d, s, n) \ - do { \ - if (copy_to_user((d),(s),(n)) != 0) \ - return SIGSEGV; \ - } while (0) - -static void display_emulation_not_implemented(struct pt_regs *regs, char *instr) -{ - __u16 *location; - -#ifdef CONFIG_SYSCTL - if(sysctl_ieee_emulation_warnings) -#endif - { - location = (__u16 *)(regs->psw.addr-S390_lowcore.pgm_ilc); - printk("%s ieee fpu instruction not emulated " - "process name: %s pid: %d \n", - instr, current->comm, current->pid); - printk("%s's PSW: %08lx %08lx\n", instr, - (unsigned long) regs->psw.mask, - (unsigned long) location); - } -} - -static inline void emu_set_CC (struct pt_regs *regs, int cc) -{ - regs->psw.mask = (regs->psw.mask & 0xFFFFCFFF) | ((cc&3) << 12); -} - -/* - * Set the condition code in the user psw. - * 0 : Result is zero - * 1 : Result is less than zero - * 2 : Result is greater than zero - * 3 : Result is NaN or INF - */ -static inline void emu_set_CC_cs(struct pt_regs *regs, int class, int sign) -{ - switch (class) { - case FP_CLS_NORMAL: - case FP_CLS_INF: - emu_set_CC(regs, sign ? 1 : 2); - break; - case FP_CLS_ZERO: - emu_set_CC(regs, 0); - break; - case FP_CLS_NAN: - emu_set_CC(regs, 3); - break; - } -} - -/* Add long double */ -static int emu_axbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QB, &cvt.ld); - FP_ADD_Q(QR, QA, QB); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - emu_set_CC_cs(regs, QR_c, QR_s); - return _fex; -} - -/* Add double */ -static int emu_adbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_ADD_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Add double */ -static int emu_adb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, val); - FP_ADD_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Add float */ -static int emu_aebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_ADD_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Add float */ -static int emu_aeb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, val); - FP_ADD_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Compare long double */ -static int emu_cxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QB); - mathemu_ldcv cvt; - int IR; - - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_RAW_QP(QA, &cvt.ld); - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_RAW_QP(QB, &cvt.ld); - FP_CMP_Q(IR, QA, QB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - return 0; -} - -/* Compare double */ -static int emu_cdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DB); - int IR; - - FP_UNPACK_RAW_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_RAW_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_CMP_D(IR, DA, DB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - return 0; -} - -/* Compare double */ -static int emu_cdb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_D(DB); - int IR; - - FP_UNPACK_RAW_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_RAW_DP(DB, val); - FP_CMP_D(IR, DA, DB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - return 0; -} - -/* Compare float */ -static int emu_cebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SB); - int IR; - - FP_UNPACK_RAW_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_RAW_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_CMP_S(IR, SA, SB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - return 0; -} - -/* Compare float */ -static int emu_ceb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_S(SB); - int IR; - - FP_UNPACK_RAW_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_RAW_SP(SB, val); - FP_CMP_S(IR, SA, SB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - return 0; -} - -/* Compare and signal long double */ -static int emu_kxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QB); - FP_DECL_EX; - mathemu_ldcv cvt; - int IR; - - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_RAW_QP(QA, &cvt.ld); - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QB, &cvt.ld); - FP_CMP_Q(IR, QA, QB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - if (IR == 3) - FP_SET_EXCEPTION (FP_EX_INVALID); - return _fex; -} - -/* Compare and signal double */ -static int emu_kdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DB); - FP_DECL_EX; - int IR; - - FP_UNPACK_RAW_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_RAW_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_CMP_D(IR, DA, DB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - if (IR == 3) - FP_SET_EXCEPTION (FP_EX_INVALID); - return _fex; -} - -/* Compare and signal double */ -static int emu_kdb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_D(DB); - FP_DECL_EX; - int IR; - - FP_UNPACK_RAW_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_RAW_DP(DB, val); - FP_CMP_D(IR, DA, DB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - if (IR == 3) - FP_SET_EXCEPTION (FP_EX_INVALID); - return _fex; -} - -/* Compare and signal float */ -static int emu_kebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SB); - FP_DECL_EX; - int IR; - - FP_UNPACK_RAW_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_RAW_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_CMP_S(IR, SA, SB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - if (IR == 3) - FP_SET_EXCEPTION (FP_EX_INVALID); - return _fex; -} - -/* Compare and signal float */ -static int emu_keb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_S(SB); - FP_DECL_EX; - int IR; - - FP_UNPACK_RAW_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_RAW_SP(SB, val); - FP_CMP_S(IR, SA, SB, 3); - /* - * IR == -1 if DA < DB, IR == 0 if DA == DB, - * IR == 1 if DA > DB and IR == 3 if unorderded - */ - emu_set_CC(regs, (IR == -1) ? 1 : (IR == 1) ? 2 : IR); - if (IR == 3) - FP_SET_EXCEPTION (FP_EX_INVALID); - return _fex; -} - -/* Convert from fixed long double */ -static int emu_cxfbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - __s32 si; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - si = regs->gprs[ry]; - FP_FROM_INT_Q(QR, si, 32, int); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Convert from fixed double */ -static int emu_cdfbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DR); - FP_DECL_EX; - __s32 si; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - si = regs->gprs[ry]; - FP_FROM_INT_D(DR, si, 32, int); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Convert from fixed float */ -static int emu_cefbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SR); - FP_DECL_EX; - __s32 si; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - si = regs->gprs[ry]; - FP_FROM_INT_S(SR, si, 32, int); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - return _fex; -} - -/* Convert to fixed long double */ -static int emu_cfxbr (struct pt_regs *regs, int rx, int ry, int mask) { - FP_DECL_Q(QA); - FP_DECL_EX; - mathemu_ldcv cvt; - __s32 si; - int mode; - - if (mask == 0) - mode = current->thread.fp_regs.fpc & 3; - else if (mask == 1) - mode = FP_RND_NEAREST; - else - mode = mask - 4; - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - FP_TO_INT_ROUND_Q(si, QA, 32, 1); - regs->gprs[rx] = si; - emu_set_CC_cs(regs, QA_c, QA_s); - return _fex; -} - -/* Convert to fixed double */ -static int emu_cfdbr (struct pt_regs *regs, int rx, int ry, int mask) { - FP_DECL_D(DA); - FP_DECL_EX; - __s32 si; - int mode; - - if (mask == 0) - mode = current->thread.fp_regs.fpc & 3; - else if (mask == 1) - mode = FP_RND_NEAREST; - else - mode = mask - 4; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - FP_TO_INT_ROUND_D(si, DA, 32, 1); - regs->gprs[rx] = si; - emu_set_CC_cs(regs, DA_c, DA_s); - return _fex; -} - -/* Convert to fixed float */ -static int emu_cfebr (struct pt_regs *regs, int rx, int ry, int mask) { - FP_DECL_S(SA); - FP_DECL_EX; - __s32 si; - int mode; - - if (mask == 0) - mode = current->thread.fp_regs.fpc & 3; - else if (mask == 1) - mode = FP_RND_NEAREST; - else - mode = mask - 4; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - FP_TO_INT_ROUND_S(si, SA, 32, 1); - regs->gprs[rx] = si; - emu_set_CC_cs(regs, SA_c, SA_s); - return _fex; -} - -/* Divide long double */ -static int emu_dxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QB, &cvt.ld); - FP_DIV_Q(QR, QA, QB); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Divide double */ -static int emu_ddbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_DIV_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Divide double */ -static int emu_ddb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, val); - FP_DIV_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Divide float */ -static int emu_debr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_DIV_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - return _fex; -} - -/* Divide float */ -static int emu_deb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, val); - FP_DIV_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - return _fex; -} - -/* Divide to integer double */ -static int emu_didbr (struct pt_regs *regs, int rx, int ry, int mask) { - display_emulation_not_implemented(regs, "didbr"); - return 0; -} - -/* Divide to integer float */ -static int emu_diebr (struct pt_regs *regs, int rx, int ry, int mask) { - display_emulation_not_implemented(regs, "diebr"); - return 0; -} - -/* Extract fpc */ -static int emu_efpc (struct pt_regs *regs, int rx, int ry) { - regs->gprs[rx] = current->thread.fp_regs.fpc; - return 0; -} - -/* Load and test long double */ -static int emu_ltxbr (struct pt_regs *regs, int rx, int ry) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - mathemu_ldcv cvt; - FP_DECL_Q(QA); - FP_DECL_EX; - - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - fp_regs->fprs[rx].ui = fp_regs->fprs[ry].ui; - fp_regs->fprs[rx+2].ui = fp_regs->fprs[ry+2].ui; - emu_set_CC_cs(regs, QA_c, QA_s); - return _fex; -} - -/* Load and test double */ -static int emu_ltdbr (struct pt_regs *regs, int rx, int ry) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - FP_DECL_D(DA); - FP_DECL_EX; - - FP_UNPACK_DP(DA, &fp_regs->fprs[ry].d); - fp_regs->fprs[rx].ui = fp_regs->fprs[ry].ui; - emu_set_CC_cs(regs, DA_c, DA_s); - return _fex; -} - -/* Load and test double */ -static int emu_ltebr (struct pt_regs *regs, int rx, int ry) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - FP_DECL_S(SA); - FP_DECL_EX; - - FP_UNPACK_SP(SA, &fp_regs->fprs[ry].f); - fp_regs->fprs[rx].ui = fp_regs->fprs[ry].ui; - emu_set_CC_cs(regs, SA_c, SA_s); - return _fex; -} - -/* Load complement long double */ -static int emu_lcxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - FP_NEG_Q(QR, QA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - emu_set_CC_cs(regs, QR_c, QR_s); - return _fex; -} - -/* Load complement double */ -static int emu_lcdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - FP_NEG_D(DR, DA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Load complement float */ -static int emu_lcebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - FP_NEG_S(SR, SA); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Load floating point integer long double */ -static int emu_fixbr (struct pt_regs *regs, int rx, int ry, int mask) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - FP_DECL_Q(QA); - FP_DECL_EX; - mathemu_ldcv cvt; - __s32 si; - int mode; - - if (mask == 0) - mode = fp_regs->fpc & 3; - else if (mask == 1) - mode = FP_RND_NEAREST; - else - mode = mask - 4; - cvt.w.high = fp_regs->fprs[ry].ui; - cvt.w.low = fp_regs->fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - FP_TO_FPINT_ROUND_Q(QA); - FP_PACK_QP(&cvt.ld, QA); - fp_regs->fprs[rx].ui = cvt.w.high; - fp_regs->fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Load floating point integer double */ -static int emu_fidbr (struct pt_regs *regs, int rx, int ry, int mask) { - /* FIXME: rounding mode !! */ - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - FP_DECL_D(DA); - FP_DECL_EX; - __s32 si; - int mode; - - if (mask == 0) - mode = fp_regs->fpc & 3; - else if (mask == 1) - mode = FP_RND_NEAREST; - else - mode = mask - 4; - FP_UNPACK_DP(DA, &fp_regs->fprs[ry].d); - FP_TO_FPINT_ROUND_D(DA); - FP_PACK_DP(&fp_regs->fprs[rx].d, DA); - return _fex; -} - -/* Load floating point integer float */ -static int emu_fiebr (struct pt_regs *regs, int rx, int ry, int mask) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - FP_DECL_S(SA); - FP_DECL_EX; - __s32 si; - int mode; - - if (mask == 0) - mode = fp_regs->fpc & 3; - else if (mask == 1) - mode = FP_RND_NEAREST; - else - mode = mask - 4; - FP_UNPACK_SP(SA, &fp_regs->fprs[ry].f); - FP_TO_FPINT_ROUND_S(SA); - FP_PACK_SP(&fp_regs->fprs[rx].f, SA); - return _fex; -} - -/* Load lengthened double to long double */ -static int emu_lxdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - FP_CONV (Q, D, 4, 2, QR, DA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Load lengthened double to long double */ -static int emu_lxdb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, val); - FP_CONV (Q, D, 4, 2, QR, DA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Load lengthened float to long double */ -static int emu_lxebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - FP_CONV (Q, S, 4, 1, QR, SA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Load lengthened float to long double */ -static int emu_lxeb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, val); - FP_CONV (Q, S, 4, 1, QR, SA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Load lengthened float to double */ -static int emu_ldebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - FP_CONV (D, S, 2, 1, DR, SA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Load lengthened float to double */ -static int emu_ldeb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, val); - FP_CONV (D, S, 2, 1, DR, SA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Load negative long double */ -static int emu_lnxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - if (QA_s == 0) { - FP_NEG_Q(QR, QA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - } else { - current->thread.fp_regs.fprs[rx].ui = - current->thread.fp_regs.fprs[ry].ui; - current->thread.fp_regs.fprs[rx+2].ui = - current->thread.fp_regs.fprs[ry+2].ui; - } - emu_set_CC_cs(regs, QR_c, QR_s); - return _fex; -} - -/* Load negative double */ -static int emu_lndbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - if (DA_s == 0) { - FP_NEG_D(DR, DA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - } else - current->thread.fp_regs.fprs[rx].ui = - current->thread.fp_regs.fprs[ry].ui; - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Load negative float */ -static int emu_lnebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - if (SA_s == 0) { - FP_NEG_S(SR, SA); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - } else - current->thread.fp_regs.fprs[rx].ui = - current->thread.fp_regs.fprs[ry].ui; - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Load positive long double */ -static int emu_lpxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - if (QA_s != 0) { - FP_NEG_Q(QR, QA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - } else{ - current->thread.fp_regs.fprs[rx].ui = - current->thread.fp_regs.fprs[ry].ui; - current->thread.fp_regs.fprs[rx+2].ui = - current->thread.fp_regs.fprs[ry+2].ui; - } - emu_set_CC_cs(regs, QR_c, QR_s); - return _fex; -} - -/* Load positive double */ -static int emu_lpdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - if (DA_s != 0) { - FP_NEG_D(DR, DA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - } else - current->thread.fp_regs.fprs[rx].ui = - current->thread.fp_regs.fprs[ry].ui; - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Load positive float */ -static int emu_lpebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - if (SA_s != 0) { - FP_NEG_S(SR, SA); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - } else - current->thread.fp_regs.fprs[rx].ui = - current->thread.fp_regs.fprs[ry].ui; - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Load rounded long double to double */ -static int emu_ldxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_D(DR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - FP_CONV (D, Q, 2, 4, DR, QA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].f, DR); - return _fex; -} - -/* Load rounded long double to float */ -static int emu_lexbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_S(SR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - FP_CONV (S, Q, 1, 4, SR, QA); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - return _fex; -} - -/* Load rounded double to float */ -static int emu_ledbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - FP_CONV (S, D, 1, 2, SR, DA); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - return _fex; -} - -/* Multiply long double */ -static int emu_mxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QB, &cvt.ld); - FP_MUL_Q(QR, QA, QB); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Multiply double */ -static int emu_mdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_MUL_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Multiply double */ -static int emu_mdb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, val); - FP_MUL_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Multiply double to long double */ -static int emu_mxdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_CONV (Q, D, 4, 2, QA, DA); - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - FP_CONV (Q, D, 4, 2, QB, DA); - FP_MUL_Q(QR, QA, QB); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Multiply double to long double */ -static int emu_mxdb (struct pt_regs *regs, int rx, long double *val) { - FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - FP_UNPACK_QP(QB, val); - FP_MUL_Q(QR, QA, QB); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - return _fex; -} - -/* Multiply float */ -static int emu_meebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_MUL_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - return _fex; -} - -/* Multiply float */ -static int emu_meeb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, val); - FP_MUL_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - return _fex; -} - -/* Multiply float to double */ -static int emu_mdebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_CONV (D, S, 2, 1, DA, SA); - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - FP_CONV (D, S, 2, 1, DB, SA); - FP_MUL_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Multiply float to double */ -static int emu_mdeb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_CONV (D, S, 2, 1, DA, SA); - FP_UNPACK_SP(SA, val); - FP_CONV (D, S, 2, 1, DB, SA); - FP_MUL_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - return _fex; -} - -/* Multiply and add double */ -static int emu_madbr (struct pt_regs *regs, int rx, int ry, int rz) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DC); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_UNPACK_DP(DC, ¤t->thread.fp_regs.fprs[rz].d); - FP_MUL_D(DR, DA, DB); - FP_ADD_D(DR, DR, DC); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rz].d, DR); - return _fex; -} - -/* Multiply and add double */ -static int emu_madb (struct pt_regs *regs, int rx, double *val, int rz) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DC); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, val); - FP_UNPACK_DP(DC, ¤t->thread.fp_regs.fprs[rz].d); - FP_MUL_D(DR, DA, DB); - FP_ADD_D(DR, DR, DC); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rz].d, DR); - return _fex; -} - -/* Multiply and add float */ -static int emu_maebr (struct pt_regs *regs, int rx, int ry, int rz) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SC); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_UNPACK_SP(SC, ¤t->thread.fp_regs.fprs[rz].f); - FP_MUL_S(SR, SA, SB); - FP_ADD_S(SR, SR, SC); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rz].f, SR); - return _fex; -} - -/* Multiply and add float */ -static int emu_maeb (struct pt_regs *regs, int rx, float *val, int rz) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SC); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, val); - FP_UNPACK_SP(SC, ¤t->thread.fp_regs.fprs[rz].f); - FP_MUL_S(SR, SA, SB); - FP_ADD_S(SR, SR, SC); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rz].f, SR); - return _fex; -} - -/* Multiply and subtract double */ -static int emu_msdbr (struct pt_regs *regs, int rx, int ry, int rz) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DC); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_UNPACK_DP(DC, ¤t->thread.fp_regs.fprs[rz].d); - FP_MUL_D(DR, DA, DB); - FP_SUB_D(DR, DR, DC); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rz].d, DR); - return _fex; -} - -/* Multiply and subtract double */ -static int emu_msdb (struct pt_regs *regs, int rx, double *val, int rz) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DC); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, val); - FP_UNPACK_DP(DC, ¤t->thread.fp_regs.fprs[rz].d); - FP_MUL_D(DR, DA, DB); - FP_SUB_D(DR, DR, DC); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rz].d, DR); - return _fex; -} - -/* Multiply and subtract float */ -static int emu_msebr (struct pt_regs *regs, int rx, int ry, int rz) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SC); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_UNPACK_SP(SC, ¤t->thread.fp_regs.fprs[rz].f); - FP_MUL_S(SR, SA, SB); - FP_SUB_S(SR, SR, SC); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rz].f, SR); - return _fex; -} - -/* Multiply and subtract float */ -static int emu_mseb (struct pt_regs *regs, int rx, float *val, int rz) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SC); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, val); - FP_UNPACK_SP(SC, ¤t->thread.fp_regs.fprs[rz].f); - FP_MUL_S(SR, SA, SB); - FP_SUB_S(SR, SR, SC); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rz].f, SR); - return _fex; -} - -/* Set floating point control word */ -static int emu_sfpc (struct pt_regs *regs, int rx, int ry) { - __u32 temp; - - temp = regs->gprs[rx]; - if ((temp & ~FPC_VALID_MASK) != 0) - return SIGILL; - current->thread.fp_regs.fpc = temp; - return 0; -} - -/* Square root long double */ -static int emu_sqxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - FP_SQRT_Q(QR, QA); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - emu_set_CC_cs(regs, QR_c, QR_s); - return _fex; -} - -/* Square root double */ -static int emu_sqdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[ry].d); - FP_SQRT_D(DR, DA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Square root double */ -static int emu_sqdb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, val); - FP_SQRT_D(DR, DA); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Square root float */ -static int emu_sqebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[ry].f); - FP_SQRT_S(SR, SA); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Square root float */ -static int emu_sqeb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, val); - FP_SQRT_S(SR, SA); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Subtract long double */ -static int emu_sxbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); - FP_DECL_EX; - mathemu_ldcv cvt; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_QP(QA, &cvt.ld); - cvt.w.high = current->thread.fp_regs.fprs[ry].ui; - cvt.w.low = current->thread.fp_regs.fprs[ry+2].ui; - FP_UNPACK_QP(QB, &cvt.ld); - FP_SUB_Q(QR, QA, QB); - FP_PACK_QP(&cvt.ld, QR); - current->thread.fp_regs.fprs[rx].ui = cvt.w.high; - current->thread.fp_regs.fprs[rx+2].ui = cvt.w.low; - emu_set_CC_cs(regs, QR_c, QR_s); - return _fex; -} - -/* Subtract double */ -static int emu_sdbr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, ¤t->thread.fp_regs.fprs[ry].d); - FP_SUB_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Subtract double */ -static int emu_sdb (struct pt_regs *regs, int rx, double *val) { - FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - FP_UNPACK_DP(DB, val); - FP_SUB_D(DR, DA, DB); - FP_PACK_DP(¤t->thread.fp_regs.fprs[rx].d, DR); - emu_set_CC_cs(regs, DR_c, DR_s); - return _fex; -} - -/* Subtract float */ -static int emu_sebr (struct pt_regs *regs, int rx, int ry) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, ¤t->thread.fp_regs.fprs[ry].f); - FP_SUB_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Subtract float */ -static int emu_seb (struct pt_regs *regs, int rx, float *val) { - FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); - FP_DECL_EX; - int mode; - - mode = current->thread.fp_regs.fpc & 3; - FP_UNPACK_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - FP_UNPACK_SP(SB, val); - FP_SUB_S(SR, SA, SB); - FP_PACK_SP(¤t->thread.fp_regs.fprs[rx].f, SR); - emu_set_CC_cs(regs, SR_c, SR_s); - return _fex; -} - -/* Test data class long double */ -static int emu_tcxb (struct pt_regs *regs, int rx, long val) { - FP_DECL_Q(QA); - mathemu_ldcv cvt; - int bit; - - cvt.w.high = current->thread.fp_regs.fprs[rx].ui; - cvt.w.low = current->thread.fp_regs.fprs[rx+2].ui; - FP_UNPACK_RAW_QP(QA, &cvt.ld); - switch (QA_e) { - default: - bit = 8; /* normalized number */ - break; - case 0: - if (_FP_FRAC_ZEROP_4(QA)) - bit = 10; /* zero */ - else - bit = 6; /* denormalized number */ - break; - case _FP_EXPMAX_Q: - if (_FP_FRAC_ZEROP_4(QA)) - bit = 4; /* infinity */ - else if (_FP_FRAC_HIGH_RAW_Q(QA) & _FP_QNANBIT_Q) - bit = 2; /* quiet NAN */ - else - bit = 0; /* signaling NAN */ - break; - } - if (!QA_s) - bit++; - emu_set_CC(regs, ((__u32) val >> bit) & 1); - return 0; -} - -/* Test data class double */ -static int emu_tcdb (struct pt_regs *regs, int rx, long val) { - FP_DECL_D(DA); - int bit; - - FP_UNPACK_RAW_DP(DA, ¤t->thread.fp_regs.fprs[rx].d); - switch (DA_e) { - default: - bit = 8; /* normalized number */ - break; - case 0: - if (_FP_FRAC_ZEROP_2(DA)) - bit = 10; /* zero */ - else - bit = 6; /* denormalized number */ - break; - case _FP_EXPMAX_D: - if (_FP_FRAC_ZEROP_2(DA)) - bit = 4; /* infinity */ - else if (_FP_FRAC_HIGH_RAW_D(DA) & _FP_QNANBIT_D) - bit = 2; /* quiet NAN */ - else - bit = 0; /* signaling NAN */ - break; - } - if (!DA_s) - bit++; - emu_set_CC(regs, ((__u32) val >> bit) & 1); - return 0; -} - -/* Test data class float */ -static int emu_tceb (struct pt_regs *regs, int rx, long val) { - FP_DECL_S(SA); - int bit; - - FP_UNPACK_RAW_SP(SA, ¤t->thread.fp_regs.fprs[rx].f); - switch (SA_e) { - default: - bit = 8; /* normalized number */ - break; - case 0: - if (_FP_FRAC_ZEROP_1(SA)) - bit = 10; /* zero */ - else - bit = 6; /* denormalized number */ - break; - case _FP_EXPMAX_S: - if (_FP_FRAC_ZEROP_1(SA)) - bit = 4; /* infinity */ - else if (_FP_FRAC_HIGH_RAW_S(SA) & _FP_QNANBIT_S) - bit = 2; /* quiet NAN */ - else - bit = 0; /* signaling NAN */ - break; - } - if (!SA_s) - bit++; - emu_set_CC(regs, ((__u32) val >> bit) & 1); - return 0; -} - -static inline void emu_load_regd(int reg) { - if ((reg&9) != 0) /* test if reg in {0,2,4,6} */ - return; - asm volatile( /* load reg from fp_regs.fprs[reg] */ - " bras 1,0f\n" - " ld 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" (reg<<4),"a" (¤t->thread.fp_regs.fprs[reg].d) - : "1"); -} - -static inline void emu_load_rege(int reg) { - if ((reg&9) != 0) /* test if reg in {0,2,4,6} */ - return; - asm volatile( /* load reg from fp_regs.fprs[reg] */ - " bras 1,0f\n" - " le 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" (reg<<4), "a" (¤t->thread.fp_regs.fprs[reg].f) - : "1"); -} - -static inline void emu_store_regd(int reg) { - if ((reg&9) != 0) /* test if reg in {0,2,4,6} */ - return; - asm volatile( /* store reg to fp_regs.fprs[reg] */ - " bras 1,0f\n" - " std 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" (reg<<4), "a" (¤t->thread.fp_regs.fprs[reg].d) - : "1"); -} - - -static inline void emu_store_rege(int reg) { - if ((reg&9) != 0) /* test if reg in {0,2,4,6} */ - return; - asm volatile( /* store reg to fp_regs.fprs[reg] */ - " bras 1,0f\n" - " ste 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" (reg<<4), "a" (¤t->thread.fp_regs.fprs[reg].f) - : "1"); -} - -int math_emu_b3(__u8 *opcode, struct pt_regs * regs) { - int _fex = 0; - static const __u8 format_table[256] = { - [0x00] = 0x03,[0x01] = 0x03,[0x02] = 0x03,[0x03] = 0x03, - [0x04] = 0x0f,[0x05] = 0x0d,[0x06] = 0x0e,[0x07] = 0x0d, - [0x08] = 0x03,[0x09] = 0x03,[0x0a] = 0x03,[0x0b] = 0x03, - [0x0c] = 0x0f,[0x0d] = 0x03,[0x0e] = 0x06,[0x0f] = 0x06, - [0x10] = 0x02,[0x11] = 0x02,[0x12] = 0x02,[0x13] = 0x02, - [0x14] = 0x03,[0x15] = 0x02,[0x16] = 0x01,[0x17] = 0x03, - [0x18] = 0x02,[0x19] = 0x02,[0x1a] = 0x02,[0x1b] = 0x02, - [0x1c] = 0x02,[0x1d] = 0x02,[0x1e] = 0x05,[0x1f] = 0x05, - [0x40] = 0x01,[0x41] = 0x01,[0x42] = 0x01,[0x43] = 0x01, - [0x44] = 0x12,[0x45] = 0x0d,[0x46] = 0x11,[0x47] = 0x04, - [0x48] = 0x01,[0x49] = 0x01,[0x4a] = 0x01,[0x4b] = 0x01, - [0x4c] = 0x01,[0x4d] = 0x01,[0x53] = 0x06,[0x57] = 0x06, - [0x5b] = 0x05,[0x5f] = 0x05,[0x84] = 0x13,[0x8c] = 0x13, - [0x94] = 0x09,[0x95] = 0x08,[0x96] = 0x07,[0x98] = 0x0c, - [0x99] = 0x0b,[0x9a] = 0x0a - }; - static const void *jump_table[256]= { - [0x00] = emu_lpebr,[0x01] = emu_lnebr,[0x02] = emu_ltebr, - [0x03] = emu_lcebr,[0x04] = emu_ldebr,[0x05] = emu_lxdbr, - [0x06] = emu_lxebr,[0x07] = emu_mxdbr,[0x08] = emu_kebr, - [0x09] = emu_cebr, [0x0a] = emu_aebr, [0x0b] = emu_sebr, - [0x0c] = emu_mdebr,[0x0d] = emu_debr, [0x0e] = emu_maebr, - [0x0f] = emu_msebr,[0x10] = emu_lpdbr,[0x11] = emu_lndbr, - [0x12] = emu_ltdbr,[0x13] = emu_lcdbr,[0x14] = emu_sqebr, - [0x15] = emu_sqdbr,[0x16] = emu_sqxbr,[0x17] = emu_meebr, - [0x18] = emu_kdbr, [0x19] = emu_cdbr, [0x1a] = emu_adbr, - [0x1b] = emu_sdbr, [0x1c] = emu_mdbr, [0x1d] = emu_ddbr, - [0x1e] = emu_madbr,[0x1f] = emu_msdbr,[0x40] = emu_lpxbr, - [0x41] = emu_lnxbr,[0x42] = emu_ltxbr,[0x43] = emu_lcxbr, - [0x44] = emu_ledbr,[0x45] = emu_ldxbr,[0x46] = emu_lexbr, - [0x47] = emu_fixbr,[0x48] = emu_kxbr, [0x49] = emu_cxbr, - [0x4a] = emu_axbr, [0x4b] = emu_sxbr, [0x4c] = emu_mxbr, - [0x4d] = emu_dxbr, [0x53] = emu_diebr,[0x57] = emu_fiebr, - [0x5b] = emu_didbr,[0x5f] = emu_fidbr,[0x84] = emu_sfpc, - [0x8c] = emu_efpc, [0x94] = emu_cefbr,[0x95] = emu_cdfbr, - [0x96] = emu_cxfbr,[0x98] = emu_cfebr,[0x99] = emu_cfdbr, - [0x9a] = emu_cfxbr - }; - - switch (format_table[opcode[1]]) { - case 1: /* RRE format, long double operation */ - if (opcode[3] & 0x22) - return SIGILL; - emu_store_regd((opcode[3] >> 4) & 15); - emu_store_regd(((opcode[3] >> 4) & 15) + 2); - emu_store_regd(opcode[3] & 15); - emu_store_regd((opcode[3] & 15) + 2); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *,int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - emu_load_regd(((opcode[3] >> 4) & 15) + 2); - emu_load_regd(opcode[3] & 15); - emu_load_regd((opcode[3] & 15) + 2); - break; - case 2: /* RRE format, double operation */ - emu_store_regd((opcode[3] >> 4) & 15); - emu_store_regd(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - emu_load_regd(opcode[3] & 15); - break; - case 3: /* RRE format, float operation */ - emu_store_rege((opcode[3] >> 4) & 15); - emu_store_rege(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_rege((opcode[3] >> 4) & 15); - emu_load_rege(opcode[3] & 15); - break; - case 4: /* RRF format, long double operation */ - if (opcode[3] & 0x22) - return SIGILL; - emu_store_regd((opcode[3] >> 4) & 15); - emu_store_regd(((opcode[3] >> 4) & 15) + 2); - emu_store_regd(opcode[3] & 15); - emu_store_regd((opcode[3] & 15) + 2); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15, opcode[2] >> 4); - emu_load_regd((opcode[3] >> 4) & 15); - emu_load_regd(((opcode[3] >> 4) & 15) + 2); - emu_load_regd(opcode[3] & 15); - emu_load_regd((opcode[3] & 15) + 2); - break; - case 5: /* RRF format, double operation */ - emu_store_regd((opcode[2] >> 4) & 15); - emu_store_regd((opcode[3] >> 4) & 15); - emu_store_regd(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15, opcode[2] >> 4); - emu_load_regd((opcode[2] >> 4) & 15); - emu_load_regd((opcode[3] >> 4) & 15); - emu_load_regd(opcode[3] & 15); - break; - case 6: /* RRF format, float operation */ - emu_store_rege((opcode[2] >> 4) & 15); - emu_store_rege((opcode[3] >> 4) & 15); - emu_store_rege(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15, opcode[2] >> 4); - emu_load_rege((opcode[2] >> 4) & 15); - emu_load_rege((opcode[3] >> 4) & 15); - emu_load_rege(opcode[3] & 15); - break; - case 7: /* RRE format, cxfbr instruction */ - /* call the emulation function */ - if (opcode[3] & 0x20) - return SIGILL; - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - emu_load_regd(((opcode[3] >> 4) & 15) + 2); - break; - case 8: /* RRE format, cdfbr instruction */ - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - break; - case 9: /* RRE format, cefbr instruction */ - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_rege((opcode[3] >> 4) & 15); - break; - case 10: /* RRF format, cfxbr instruction */ - if ((opcode[2] & 128) == 128 || (opcode[2] & 96) == 32) - /* mask of { 2,3,8-15 } is invalid */ - return SIGILL; - if (opcode[3] & 2) - return SIGILL; - emu_store_regd(opcode[3] & 15); - emu_store_regd((opcode[3] & 15) + 2); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15, opcode[2] >> 4); - break; - case 11: /* RRF format, cfdbr instruction */ - if ((opcode[2] & 128) == 128 || (opcode[2] & 96) == 32) - /* mask of { 2,3,8-15 } is invalid */ - return SIGILL; - emu_store_regd(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15, opcode[2] >> 4); - break; - case 12: /* RRF format, cfebr instruction */ - if ((opcode[2] & 128) == 128 || (opcode[2] & 96) == 32) - /* mask of { 2,3,8-15 } is invalid */ - return SIGILL; - emu_store_rege(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15, opcode[2] >> 4); - break; - case 13: /* RRE format, ldxbr & mdxbr instruction */ - /* double store but long double load */ - if (opcode[3] & 0x20) - return SIGILL; - emu_store_regd((opcode[3] >> 4) & 15); - emu_store_regd(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - emu_load_regd(((opcode[3] >> 4) & 15) + 2); - break; - case 14: /* RRE format, ldxbr & mdxbr instruction */ - /* float store but long double load */ - if (opcode[3] & 0x20) - return SIGILL; - emu_store_rege((opcode[3] >> 4) & 15); - emu_store_rege(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - emu_load_regd(((opcode[3] >> 4) & 15) + 2); - break; - case 15: /* RRE format, ldebr & mdebr instruction */ - /* float store but double load */ - emu_store_rege((opcode[3] >> 4) & 15); - emu_store_rege(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - break; - case 16: /* RRE format, ldxbr instruction */ - /* long double store but double load */ - if (opcode[3] & 2) - return SIGILL; - emu_store_regd(opcode[3] & 15); - emu_store_regd((opcode[3] & 15) + 2); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_regd((opcode[3] >> 4) & 15); - break; - case 17: /* RRE format, ldxbr instruction */ - /* long double store but float load */ - if (opcode[3] & 2) - return SIGILL; - emu_store_regd(opcode[3] & 15); - emu_store_regd((opcode[3] & 15) + 2); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_rege((opcode[3] >> 4) & 15); - break; - case 18: /* RRE format, ledbr instruction */ - /* double store but float load */ - emu_store_regd(opcode[3] & 15); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - emu_load_rege((opcode[3] >> 4) & 15); - break; - case 19: /* RRE format, efpc & sfpc instruction */ - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, int)) - jump_table[opcode[1]]) - (regs, opcode[3] >> 4, opcode[3] & 15); - break; - default: /* invalid operation */ - return SIGILL; - } - if (_fex != 0) { - current->thread.fp_regs.fpc |= _fex; - if (current->thread.fp_regs.fpc & (_fex << 8)) - return SIGFPE; - } - return 0; -} - -static void* calc_addr(struct pt_regs *regs, int rx, int rb, int disp) -{ - addr_t addr; - - rx &= 15; - rb &= 15; - addr = disp & 0xfff; - addr += (rx != 0) ? regs->gprs[rx] : 0; /* + index */ - addr += (rb != 0) ? regs->gprs[rb] : 0; /* + base */ - return (void*) addr; -} - -int math_emu_ed(__u8 *opcode, struct pt_regs * regs) { - int _fex = 0; - - static const __u8 format_table[256] = { - [0x04] = 0x06,[0x05] = 0x05,[0x06] = 0x07,[0x07] = 0x05, - [0x08] = 0x02,[0x09] = 0x02,[0x0a] = 0x02,[0x0b] = 0x02, - [0x0c] = 0x06,[0x0d] = 0x02,[0x0e] = 0x04,[0x0f] = 0x04, - [0x10] = 0x08,[0x11] = 0x09,[0x12] = 0x0a,[0x14] = 0x02, - [0x15] = 0x01,[0x17] = 0x02,[0x18] = 0x01,[0x19] = 0x01, - [0x1a] = 0x01,[0x1b] = 0x01,[0x1c] = 0x01,[0x1d] = 0x01, - [0x1e] = 0x03,[0x1f] = 0x03, - }; - static const void *jump_table[]= { - [0x04] = emu_ldeb,[0x05] = emu_lxdb,[0x06] = emu_lxeb, - [0x07] = emu_mxdb,[0x08] = emu_keb, [0x09] = emu_ceb, - [0x0a] = emu_aeb, [0x0b] = emu_seb, [0x0c] = emu_mdeb, - [0x0d] = emu_deb, [0x0e] = emu_maeb,[0x0f] = emu_mseb, - [0x10] = emu_tceb,[0x11] = emu_tcdb,[0x12] = emu_tcxb, - [0x14] = emu_sqeb,[0x15] = emu_sqdb,[0x17] = emu_meeb, - [0x18] = emu_kdb, [0x19] = emu_cdb, [0x1a] = emu_adb, - [0x1b] = emu_sdb, [0x1c] = emu_mdb, [0x1d] = emu_ddb, - [0x1e] = emu_madb,[0x1f] = emu_msdb - }; - - switch (format_table[opcode[5]]) { - case 1: /* RXE format, double constant */ { - __u64 *dxb, temp; - __u32 opc; - - emu_store_regd((opcode[1] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u64 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_copy_from_user(&temp, dxb, 8); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, double *)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, (double *) &temp); - emu_load_regd((opcode[1] >> 4) & 15); - break; - } - case 2: /* RXE format, float constant */ { - __u32 *dxb, temp; - __u32 opc; - - emu_store_rege((opcode[1] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u32 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_get_user(temp, dxb); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, float *)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, (float *) &temp); - emu_load_rege((opcode[1] >> 4) & 15); - break; - } - case 3: /* RXF format, double constant */ { - __u64 *dxb, temp; - __u32 opc; - - emu_store_regd((opcode[1] >> 4) & 15); - emu_store_regd((opcode[4] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u64 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_copy_from_user(&temp, dxb, 8); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, double *, int)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, (double *) &temp, opcode[4] >> 4); - emu_load_regd((opcode[1] >> 4) & 15); - break; - } - case 4: /* RXF format, float constant */ { - __u32 *dxb, temp; - __u32 opc; - - emu_store_rege((opcode[1] >> 4) & 15); - emu_store_rege((opcode[4] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u32 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_get_user(temp, dxb); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, float *, int)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, (float *) &temp, opcode[4] >> 4); - emu_load_rege((opcode[4] >> 4) & 15); - break; - } - case 5: /* RXE format, double constant */ - /* store double and load long double */ - { - __u64 *dxb, temp; - __u32 opc; - if ((opcode[1] >> 4) & 0x20) - return SIGILL; - emu_store_regd((opcode[1] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u64 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_copy_from_user(&temp, dxb, 8); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, double *)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, (double *) &temp); - emu_load_regd((opcode[1] >> 4) & 15); - emu_load_regd(((opcode[1] >> 4) & 15) + 2); - break; - } - case 6: /* RXE format, float constant */ - /* store float and load double */ - { - __u32 *dxb, temp; - __u32 opc; - emu_store_rege((opcode[1] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u32 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_get_user(temp, dxb); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, float *)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, (float *) &temp); - emu_load_regd((opcode[1] >> 4) & 15); - break; - } - case 7: /* RXE format, float constant */ - /* store float and load long double */ - { - __u32 *dxb, temp; - __u32 opc; - if ((opcode[1] >> 4) & 0x20) - return SIGILL; - emu_store_rege((opcode[1] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u32 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_get_user(temp, dxb); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, float *)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, (float *) &temp); - emu_load_regd((opcode[1] >> 4) & 15); - emu_load_regd(((opcode[1] >> 4) & 15) + 2); - break; - } - case 8: /* RXE format, RX address used as int value */ { - __u64 dxb; - __u32 opc; - - emu_store_rege((opcode[1] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u64) calc_addr(regs, opc >> 16, opc >> 12, opc); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, long)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, dxb); - break; - } - case 9: /* RXE format, RX address used as int value */ { - __u64 dxb; - __u32 opc; - - emu_store_regd((opcode[1] >> 4) & 15); - opc = *((__u32 *) opcode); - dxb = (__u64) calc_addr(regs, opc >> 16, opc >> 12, opc); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, long)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, dxb); - break; - } - case 10: /* RXE format, RX address used as int value */ { - __u64 dxb; - __u32 opc; - - if ((opcode[1] >> 4) & 2) - return SIGILL; - emu_store_regd((opcode[1] >> 4) & 15); - emu_store_regd(((opcode[1] >> 4) & 15) + 2); - opc = *((__u32 *) opcode); - dxb = (__u64) calc_addr(regs, opc >> 16, opc >> 12, opc); - /* call the emulation function */ - _fex = ((int (*)(struct pt_regs *, int, long)) - jump_table[opcode[5]]) - (regs, opcode[1] >> 4, dxb); - break; - } - default: /* invalid operation */ - return SIGILL; - } - if (_fex != 0) { - current->thread.fp_regs.fpc |= _fex; - if (current->thread.fp_regs.fpc & (_fex << 8)) - return SIGFPE; - } - return 0; -} - -/* - * Emulate LDR Rx,Ry with Rx or Ry not in {0, 2, 4, 6} - */ -int math_emu_ldr(__u8 *opcode) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - __u16 opc = *((__u16 *) opcode); - - if ((opc & 0x90) == 0) { /* test if rx in {0,2,4,6} */ - /* we got an exception therefore ry can't be in {0,2,4,6} */ - asm volatile( /* load rx from fp_regs.fprs[ry] */ - " bras 1,0f\n" - " ld 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" (opc & 0xf0), "a" (&fp_regs->fprs[opc & 0xf].d) - : "1"); - } else if ((opc & 0x9) == 0) { /* test if ry in {0,2,4,6} */ - asm volatile ( /* store ry to fp_regs.fprs[rx] */ - " bras 1,0f\n" - " std 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" ((opc & 0xf) << 4), - "a" (&fp_regs->fprs[(opc & 0xf0)>>4].d) - : "1"); - } else /* move fp_regs.fprs[ry] to fp_regs.fprs[rx] */ - fp_regs->fprs[(opc & 0xf0) >> 4] = fp_regs->fprs[opc & 0xf]; - return 0; -} - -/* - * Emulate LER Rx,Ry with Rx or Ry not in {0, 2, 4, 6} - */ -int math_emu_ler(__u8 *opcode) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - __u16 opc = *((__u16 *) opcode); - - if ((opc & 0x90) == 0) { /* test if rx in {0,2,4,6} */ - /* we got an exception therefore ry can't be in {0,2,4,6} */ - asm volatile( /* load rx from fp_regs.fprs[ry] */ - " bras 1,0f\n" - " le 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" (opc & 0xf0), "a" (&fp_regs->fprs[opc & 0xf].f) - : "1"); - } else if ((opc & 0x9) == 0) { /* test if ry in {0,2,4,6} */ - asm volatile( /* store ry to fp_regs.fprs[rx] */ - " bras 1,0f\n" - " ste 0,0(%1)\n" - "0: ex %0,0(1)" - : /* no output */ - : "a" ((opc & 0xf) << 4), - "a" (&fp_regs->fprs[(opc & 0xf0) >> 4].f) - : "1"); - } else /* move fp_regs.fprs[ry] to fp_regs.fprs[rx] */ - fp_regs->fprs[(opc & 0xf0) >> 4] = fp_regs->fprs[opc & 0xf]; - return 0; -} - -/* - * Emulate LD R,D(X,B) with R not in {0, 2, 4, 6} - */ -int math_emu_ld(__u8 *opcode, struct pt_regs * regs) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - __u32 opc = *((__u32 *) opcode); - __u64 *dxb; - - dxb = (__u64 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_copy_from_user(&fp_regs->fprs[(opc >> 20) & 0xf].d, dxb, 8); - return 0; -} - -/* - * Emulate LE R,D(X,B) with R not in {0, 2, 4, 6} - */ -int math_emu_le(__u8 *opcode, struct pt_regs * regs) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - __u32 opc = *((__u32 *) opcode); - __u32 *mem, *dxb; - - dxb = (__u32 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mem = (__u32 *) (&fp_regs->fprs[(opc >> 20) & 0xf].f); - mathemu_get_user(mem[0], dxb); - return 0; -} - -/* - * Emulate STD R,D(X,B) with R not in {0, 2, 4, 6} - */ -int math_emu_std(__u8 *opcode, struct pt_regs * regs) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - __u32 opc = *((__u32 *) opcode); - __u64 *dxb; - - dxb = (__u64 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mathemu_copy_to_user(dxb, &fp_regs->fprs[(opc >> 20) & 0xf].d, 8); - return 0; -} - -/* - * Emulate STE R,D(X,B) with R not in {0, 2, 4, 6} - */ -int math_emu_ste(__u8 *opcode, struct pt_regs * regs) { - s390_fp_regs *fp_regs = ¤t->thread.fp_regs; - __u32 opc = *((__u32 *) opcode); - __u32 *mem, *dxb; - - dxb = (__u32 *) calc_addr(regs, opc >> 16, opc >> 12, opc); - mem = (__u32 *) (&fp_regs->fprs[(opc >> 20) & 0xf].f); - mathemu_put_user(mem[0], dxb); - return 0; -} - -/* - * Emulate LFPC D(B) - */ -int math_emu_lfpc(__u8 *opcode, struct pt_regs *regs) { - __u32 opc = *((__u32 *) opcode); - __u32 *dxb, temp; - - dxb= (__u32 *) calc_addr(regs, 0, opc>>12, opc); - mathemu_get_user(temp, dxb); - if ((temp & ~FPC_VALID_MASK) != 0) - return SIGILL; - current->thread.fp_regs.fpc = temp; - return 0; -} - -/* - * Emulate STFPC D(B) - */ -int math_emu_stfpc(__u8 *opcode, struct pt_regs *regs) { - __u32 opc = *((__u32 *) opcode); - __u32 *dxb; - - dxb= (__u32 *) calc_addr(regs, 0, opc>>12, opc); - mathemu_put_user(current->thread.fp_regs.fpc, dxb); - return 0; -} - -/* - * Emulate SRNM D(B) - */ -int math_emu_srnm(__u8 *opcode, struct pt_regs *regs) { - __u32 opc = *((__u32 *) opcode); - __u32 temp; - - temp = calc_addr(regs, 0, opc>>12, opc); - current->thread.fp_regs.fpc &= ~3; - current->thread.fp_regs.fpc |= (temp & 3); - return 0; -} - -/* broken compiler ... */ -long long -__negdi2 (long long u) -{ - - union lll { - long long ll; - long s[2]; - }; - - union lll w,uu; - - uu.ll = u; - - w.s[1] = -uu.s[1]; - w.s[0] = -uu.s[0] - ((int) w.s[1] != 0); - - return w.ll; -} diff --git a/arch/s390/mm/dump_pagetables.c b/arch/s390/mm/dump_pagetables.c index d46cadeda204..8556d6be9b54 100644 --- a/arch/s390/mm/dump_pagetables.c +++ b/arch/s390/mm/dump_pagetables.c @@ -18,9 +18,7 @@ enum address_markers_idx { KERNEL_END_NR, VMEMMAP_NR, VMALLOC_NR, -#ifdef CONFIG_64BIT MODULES_NR, -#endif }; static struct addr_marker address_markers[] = { @@ -29,9 +27,7 @@ static struct addr_marker address_markers[] = { [KERNEL_END_NR] = {(unsigned long)&_end, "Kernel Image End"}, [VMEMMAP_NR] = {0, "vmemmap Area"}, [VMALLOC_NR] = {0, "vmalloc Area"}, -#ifdef CONFIG_64BIT [MODULES_NR] = {0, "Modules Area"}, -#endif { -1, NULL } }; @@ -127,12 +123,6 @@ static void walk_pte_level(struct seq_file *m, struct pg_state *st, } } -#ifdef CONFIG_64BIT -#define _PMD_PROT_MASK _SEGMENT_ENTRY_PROTECT -#else -#define _PMD_PROT_MASK 0 -#endif - static void walk_pmd_level(struct seq_file *m, struct pg_state *st, pud_t *pud, unsigned long addr) { @@ -145,7 +135,7 @@ static void walk_pmd_level(struct seq_file *m, struct pg_state *st, pmd = pmd_offset(pud, addr); if (!pmd_none(*pmd)) { if (pmd_large(*pmd)) { - prot = pmd_val(*pmd) & _PMD_PROT_MASK; + prot = pmd_val(*pmd) & _SEGMENT_ENTRY_PROTECT; note_page(m, st, prot, 3); } else walk_pte_level(m, st, pmd, addr); @@ -155,12 +145,6 @@ static void walk_pmd_level(struct seq_file *m, struct pg_state *st, } } -#ifdef CONFIG_64BIT -#define _PUD_PROT_MASK _REGION3_ENTRY_RO -#else -#define _PUD_PROT_MASK 0 -#endif - static void walk_pud_level(struct seq_file *m, struct pg_state *st, pgd_t *pgd, unsigned long addr) { @@ -173,7 +157,7 @@ static void walk_pud_level(struct seq_file *m, struct pg_state *st, pud = pud_offset(pgd, addr); if (!pud_none(*pud)) if (pud_large(*pud)) { - prot = pud_val(*pud) & _PUD_PROT_MASK; + prot = pud_val(*pud) & _REGION3_ENTRY_RO; note_page(m, st, prot, 2); } else walk_pmd_level(m, st, pud, addr); @@ -230,13 +214,9 @@ static int pt_dump_init(void) * kernel ASCE. We need this to keep the page table walker functions * from accessing non-existent entries. */ -#ifdef CONFIG_32BIT - max_addr = 1UL << 31; -#else max_addr = (S390_lowcore.kernel_asce & _REGION_ENTRY_TYPE_MASK) >> 2; max_addr = 1UL << (max_addr * 11 + 31); address_markers[MODULES_NR].start_address = MODULES_VADDR; -#endif address_markers[VMEMMAP_NR].start_address = (unsigned long) vmemmap; address_markers[VMALLOC_NR].start_address = VMALLOC_START; debugfs_create_file("kernel_page_tables", 0400, NULL, NULL, &ptdump_fops); diff --git a/arch/s390/mm/extmem.c b/arch/s390/mm/extmem.c index 519bba716cc3..23c496957c22 100644 --- a/arch/s390/mm/extmem.c +++ b/arch/s390/mm/extmem.c @@ -51,7 +51,6 @@ struct qout64 { struct qrange range[6]; }; -#ifdef CONFIG_64BIT struct qrange_old { unsigned int start; /* last byte type */ unsigned int end; /* last byte reserved */ @@ -65,7 +64,6 @@ struct qout64_old { int segrcnt; struct qrange_old range[6]; }; -#endif struct qin64 { char qopcode; @@ -103,7 +101,6 @@ static int scode_set; static int dcss_set_subcodes(void) { -#ifdef CONFIG_64BIT char *name = kmalloc(8 * sizeof(char), GFP_KERNEL | GFP_DMA); unsigned long rx, ry; int rc; @@ -135,7 +132,6 @@ dcss_set_subcodes(void) segext_scode = DCSS_SEGEXTX; return 0; } -#endif /* Diag x'64' new subcodes are not supported, set to old subcodes */ loadshr_scode = DCSS_LOADNOLY; loadnsr_scode = DCSS_LOADNSR; @@ -208,7 +204,6 @@ dcss_diag(int *func, void *parameter, rx = (unsigned long) parameter; ry = (unsigned long) *func; -#ifdef CONFIG_64BIT /* 64-bit Diag x'64' new subcode, keep in 64-bit addressing mode */ if (*func > DCSS_SEGEXT) asm volatile( @@ -225,13 +220,6 @@ dcss_diag(int *func, void *parameter, " ipm %2\n" " srl %2,28\n" : "+d" (rx), "+d" (ry), "=d" (rc) : : "cc"); -#else - asm volatile( - " diag %0,%1,0x64\n" - " ipm %2\n" - " srl %2,28\n" - : "+d" (rx), "+d" (ry), "=d" (rc) : : "cc"); -#endif *ret1 = rx; *ret2 = ry; return rc; @@ -281,7 +269,6 @@ query_segment_type (struct dcss_segment *seg) goto out_free; } -#ifdef CONFIG_64BIT /* Only old format of output area of Diagnose x'64' is supported, copy data for the new format. */ if (segext_scode == DCSS_SEGEXT) { @@ -307,7 +294,6 @@ query_segment_type (struct dcss_segment *seg) } kfree(qout_old); } -#endif if (qout->segcnt > 6) { rc = -EOPNOTSUPP; goto out_free; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 3ff86533f7db..76515bcea2f1 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -36,15 +36,9 @@ #include <asm/facility.h> #include "../kernel/entry.h" -#ifndef CONFIG_64BIT -#define __FAIL_ADDR_MASK 0x7ffff000 -#define __SUBCODE_MASK 0x0200 -#define __PF_RES_FIELD 0ULL -#else /* CONFIG_64BIT */ #define __FAIL_ADDR_MASK -4096L #define __SUBCODE_MASK 0x0600 #define __PF_RES_FIELD 0x8000000000000000ULL -#endif /* CONFIG_64BIT */ #define VM_FAULT_BADCONTEXT 0x010000 #define VM_FAULT_BADMAP 0x020000 @@ -54,7 +48,6 @@ static unsigned long store_indication __read_mostly; -#ifdef CONFIG_64BIT static int __init fault_init(void) { if (test_facility(75)) @@ -62,7 +55,6 @@ static int __init fault_init(void) return 0; } early_initcall(fault_init); -#endif static inline int notify_page_fault(struct pt_regs *regs) { @@ -133,7 +125,6 @@ static int bad_address(void *p) return probe_kernel_address((unsigned long *)p, dummy); } -#ifdef CONFIG_64BIT static void dump_pagetable(unsigned long asce, unsigned long address) { unsigned long *table = __va(asce & PAGE_MASK); @@ -187,33 +178,6 @@ bad: pr_cont("BAD\n"); } -#else /* CONFIG_64BIT */ - -static void dump_pagetable(unsigned long asce, unsigned long address) -{ - unsigned long *table = __va(asce & PAGE_MASK); - - pr_alert("AS:%08lx ", asce); - table = table + ((address >> 20) & 0x7ff); - if (bad_address(table)) - goto bad; - pr_cont("S:%08lx ", *table); - if (*table & _SEGMENT_ENTRY_INVALID) - goto out; - table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN); - table = table + ((address >> 12) & 0xff); - if (bad_address(table)) - goto bad; - pr_cont("P:%08lx ", *table); -out: - pr_cont("\n"); - return; -bad: - pr_cont("BAD\n"); -} - -#endif /* CONFIG_64BIT */ - static void dump_fault_info(struct pt_regs *regs) { unsigned long asce; diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c index 5c586c78ca8d..1eb41bb3010c 100644 --- a/arch/s390/mm/gup.c +++ b/arch/s390/mm/gup.c @@ -106,11 +106,9 @@ static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, pmd_t *pmdp, pmd; pmdp = (pmd_t *) pudp; -#ifdef CONFIG_64BIT if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) pmdp = (pmd_t *) pud_deref(pud); pmdp += pmd_index(addr); -#endif do { pmd = *pmdp; barrier(); @@ -145,11 +143,9 @@ static inline int gup_pud_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, pud_t *pudp, pud; pudp = (pud_t *) pgdp; -#ifdef CONFIG_64BIT if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) pudp = (pud_t *) pgd_deref(pgd); pudp += pud_index(addr); -#endif do { pud = *pudp; barrier(); diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index d35b15113b17..80875c43a4a4 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -105,7 +105,6 @@ void __init paging_init(void) unsigned long pgd_type, asce_bits; init_mm.pgd = swapper_pg_dir; -#ifdef CONFIG_64BIT if (VMALLOC_END > (1UL << 42)) { asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; pgd_type = _REGION2_ENTRY_EMPTY; @@ -113,10 +112,6 @@ void __init paging_init(void) asce_bits = _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH; pgd_type = _REGION3_ENTRY_EMPTY; } -#else - asce_bits = _ASCE_TABLE_LENGTH; - pgd_type = _SEGMENT_ENTRY_EMPTY; -#endif S390_lowcore.kernel_asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; clear_table((unsigned long *) init_mm.pgd, pgd_type, sizeof(unsigned long)*2048); diff --git a/arch/s390/mm/maccess.c b/arch/s390/mm/maccess.c index 2eb34bdfc613..8a993a53fcd6 100644 --- a/arch/s390/mm/maccess.c +++ b/arch/s390/mm/maccess.c @@ -1,7 +1,7 @@ /* * Access kernel memory without faulting -- s390 specific implementation. * - * Copyright IBM Corp. 2009 + * Copyright IBM Corp. 2009, 2015 * * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>, * @@ -16,51 +16,55 @@ #include <asm/ctl_reg.h> #include <asm/io.h> -/* - * This function writes to kernel memory bypassing DAT and possible - * write protection. It copies one to four bytes from src to dst - * using the stura instruction. - * Returns the number of bytes copied or -EFAULT. - */ -static long probe_kernel_write_odd(void *dst, const void *src, size_t size) +static notrace long s390_kernel_write_odd(void *dst, const void *src, size_t size) { - unsigned long count, aligned; - int offset, mask; - int rc = -EFAULT; + unsigned long aligned, offset, count; + char tmp[8]; - aligned = (unsigned long) dst & ~3UL; - offset = (unsigned long) dst & 3; - count = min_t(unsigned long, 4 - offset, size); - mask = (0xf << (4 - count)) & 0xf; - mask >>= offset; + aligned = (unsigned long) dst & ~7UL; + offset = (unsigned long) dst & 7UL; + size = min(8UL - offset, size); + count = size - 1; asm volatile( " bras 1,0f\n" - " icm 0,0,0(%3)\n" - "0: l 0,0(%1)\n" - " lra %1,0(%1)\n" - "1: ex %2,0(1)\n" - "2: stura 0,%1\n" - " la %0,0\n" - "3:\n" - EX_TABLE(0b,3b) EX_TABLE(1b,3b) EX_TABLE(2b,3b) - : "+d" (rc), "+a" (aligned) - : "a" (mask), "a" (src) : "cc", "memory", "0", "1"); - return rc ? rc : count; + " mvc 0(1,%4),0(%5)\n" + "0: mvc 0(8,%3),0(%0)\n" + " ex %1,0(1)\n" + " lg %1,0(%3)\n" + " lra %0,0(%0)\n" + " sturg %1,%0\n" + : "+&a" (aligned), "+&a" (count), "=m" (tmp) + : "a" (&tmp), "a" (&tmp[offset]), "a" (src) + : "cc", "memory", "1"); + return size; } -long probe_kernel_write(void *dst, const void *src, size_t size) +/* + * s390_kernel_write - write to kernel memory bypassing DAT + * @dst: destination address + * @src: source address + * @size: number of bytes to copy + * + * This function writes to kernel memory bypassing DAT and possible page table + * write protection. It writes to the destination using the sturg instruction. + * Therefore we have a read-modify-write sequence: the function reads eight + * bytes from destination at an eight byte boundary, modifies the bytes + * requested and writes the result back in a loop. + * + * Note: this means that this function may not be called concurrently on + * several cpus with overlapping words, since this may potentially + * cause data corruption. + */ +void notrace s390_kernel_write(void *dst, const void *src, size_t size) { - long copied = 0; + long copied; while (size) { - copied = probe_kernel_write_odd(dst, src, size); - if (copied < 0) - break; + copied = s390_kernel_write_odd(dst, src, size); dst += copied; src += copied; size -= copied; } - return copied < 0 ? -EFAULT : 0; } static int __memcpy_real(void *dest, void *src, size_t count) diff --git a/arch/s390/mm/mem_detect.c b/arch/s390/mm/mem_detect.c index 5535cfe0ee11..0f3604395805 100644 --- a/arch/s390/mm/mem_detect.c +++ b/arch/s390/mm/mem_detect.c @@ -36,10 +36,6 @@ void __init detect_memory_memblock(void) memsize = rzm * rnmax; if (!rzm) rzm = 1ULL << 17; - if (IS_ENABLED(CONFIG_32BIT)) { - rzm = min(ADDR2G, rzm); - memsize = min(ADDR2G, memsize); - } max_physmem_end = memsize; addr = 0; /* keep memblock lists close to the kernel */ diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index bb3367c5cb0b..6e552af08c76 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -32,7 +32,7 @@ #include <asm/pgalloc.h> unsigned long mmap_rnd_mask; -unsigned long mmap_align_mask; +static unsigned long mmap_align_mask; static unsigned long stack_maxrandom_size(void) { @@ -177,34 +177,6 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, return addr; } -#ifndef CONFIG_64BIT - -/* - * This function, called very early during the creation of a new - * process VM image, sets up which VM layout function to use: - */ -void arch_pick_mmap_layout(struct mm_struct *mm) -{ - unsigned long random_factor = 0UL; - - if (current->flags & PF_RANDOMIZE) - random_factor = arch_mmap_rnd(); - - /* - * Fall back to the standard layout if the personality - * bit is set, or if the expected stack growth is unlimited: - */ - if (mmap_is_legacy()) { - mm->mmap_base = mmap_base_legacy(random_factor); - mm->get_unmapped_area = arch_get_unmapped_area; - } else { - mm->mmap_base = mmap_base(random_factor); - mm->get_unmapped_area = arch_get_unmapped_area_topdown; - } -} - -#else - int s390_mmap_check(unsigned long addr, unsigned long len, unsigned long flags) { if (is_compat_task() || (TASK_SIZE >= (1UL << 53))) @@ -314,5 +286,3 @@ static int __init setup_mmap_rnd(void) return 0; } early_initcall(setup_mmap_rnd); - -#endif diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 426c9d462d1c..749c98407b41 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -109,7 +109,7 @@ static void ipte_range(pte_t *pte, unsigned long address, int nr) { int i; - if (test_facility(13) && IS_ENABLED(CONFIG_64BIT)) { + if (test_facility(13)) { __ptep_ipte_range(address, nr - 1, pte); return; } diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c index b2c1542f2ba2..33f589459113 100644 --- a/arch/s390/mm/pgtable.c +++ b/arch/s390/mm/pgtable.c @@ -27,14 +27,8 @@ #include <asm/tlbflush.h> #include <asm/mmu_context.h> -#ifndef CONFIG_64BIT -#define ALLOC_ORDER 1 -#define FRAG_MASK 0x0f -#else #define ALLOC_ORDER 2 #define FRAG_MASK 0x03 -#endif - unsigned long *crst_table_alloc(struct mm_struct *mm) { @@ -50,7 +44,6 @@ void crst_table_free(struct mm_struct *mm, unsigned long *table) free_pages((unsigned long) table, ALLOC_ORDER); } -#ifdef CONFIG_64BIT static void __crst_table_upgrade(void *arg) { struct mm_struct *mm = arg; @@ -140,7 +133,6 @@ void crst_table_downgrade(struct mm_struct *mm, unsigned long limit) if (current->active_mm == mm) set_user_asce(mm); } -#endif #ifdef CONFIG_PGSTE diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index b1593c2f751a..ef7d6c8fea66 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -38,12 +38,10 @@ static inline pud_t *vmem_pud_alloc(void) { pud_t *pud = NULL; -#ifdef CONFIG_64BIT pud = vmem_alloc_pages(2); if (!pud) return NULL; clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); -#endif return pud; } @@ -51,12 +49,10 @@ static inline pmd_t *vmem_pmd_alloc(void) { pmd_t *pmd = NULL; -#ifdef CONFIG_64BIT pmd = vmem_alloc_pages(2); if (!pmd) return NULL; clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); -#endif return pmd; } @@ -98,7 +94,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) pgd_populate(&init_mm, pg_dir, pu_dir); } pu_dir = pud_offset(pg_dir, address); -#if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC) +#ifndef CONFIG_DEBUG_PAGEALLOC if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) { pud_val(*pu_dir) = __pa(address) | @@ -115,7 +111,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro) pud_populate(&init_mm, pu_dir, pm_dir); } pm_dir = pmd_offset(pu_dir, address); -#if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC) +#ifndef CONFIG_DEBUG_PAGEALLOC if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) { pmd_val(*pm_dir) = __pa(address) | @@ -222,7 +218,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) pm_dir = pmd_offset(pu_dir, address); if (pmd_none(*pm_dir)) { -#ifdef CONFIG_64BIT /* Use 1MB frames for vmemmap if available. We always * use large frames even if they are only partially * used. @@ -240,7 +235,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) address = (address + PMD_SIZE) & PMD_MASK; continue; } -#endif pt_dir = vmem_pte_alloc(address); if (!pt_dir) goto out; diff --git a/arch/s390/oprofile/Makefile b/arch/s390/oprofile/Makefile index 524c4b615821..1bd23017191e 100644 --- a/arch/s390/oprofile/Makefile +++ b/arch/s390/oprofile/Makefile @@ -7,4 +7,4 @@ DRIVER_OBJS = $(addprefix ../../../drivers/oprofile/, \ timer_int.o ) oprofile-y := $(DRIVER_OBJS) init.o backtrace.o -oprofile-$(CONFIG_64BIT) += hwsampler.o +oprofile-y += hwsampler.o diff --git a/arch/s390/oprofile/init.c b/arch/s390/oprofile/init.c index 9ffe645d5989..bc927a09a172 100644 --- a/arch/s390/oprofile/init.c +++ b/arch/s390/oprofile/init.c @@ -21,8 +21,6 @@ extern void s390_backtrace(struct pt_regs * const regs, unsigned int depth); -#ifdef CONFIG_64BIT - #include "hwsampler.h" #include "op_counter.h" @@ -495,14 +493,10 @@ static void oprofile_hwsampler_exit(void) hwsampler_shutdown(); } -#endif /* CONFIG_64BIT */ - int __init oprofile_arch_init(struct oprofile_operations *ops) { ops->backtrace = s390_backtrace; -#ifdef CONFIG_64BIT - /* * -ENODEV is not reported to the caller. The module itself * will use the timer mode sampling as fallback and this is @@ -511,14 +505,9 @@ int __init oprofile_arch_init(struct oprofile_operations *ops) hwsampler_available = oprofile_hwsampler_init(ops) == 0; return 0; -#else - return -ENODEV; -#endif } void oprofile_arch_exit(void) { -#ifdef CONFIG_64BIT oprofile_hwsampler_exit(); -#endif } diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index b2c76f64c530..98336200c7b2 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -913,8 +913,7 @@ static int __init pci_base_init(void) if (!s390_pci_probe) return 0; - if (!test_facility(2) || !test_facility(69) - || !test_facility(71) || !test_facility(72)) + if (!test_facility(69) || !test_facility(71) || !test_facility(72)) return 0; rc = zpci_debug_init(); diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c index 3229a2e570df..c22d4402ae45 100644 --- a/arch/s390/pci/pci_debug.c +++ b/arch/s390/pci/pci_debug.c @@ -45,8 +45,10 @@ static int pci_perf_show(struct seq_file *m, void *v) if (!zdev) return 0; - if (!zdev->fmb) - return seq_printf(m, "FMB statistics disabled\n"); + if (!zdev->fmb) { + seq_puts(m, "FMB statistics disabled\n"); + return 0; + } /* header */ seq_printf(m, "FMB @ %p\n", zdev->fmb); diff --git a/arch/score/include/asm/thread_info.h b/arch/score/include/asm/thread_info.h index 33864fa2a8d4..7d9ffb15c477 100644 --- a/arch/score/include/asm/thread_info.h +++ b/arch/score/include/asm/thread_info.h @@ -28,7 +28,6 @@ */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long tp_value; /* thread pointer */ __u32 cpu; /* current CPU */ @@ -53,7 +52,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .cpu = 0, \ .preempt_count = 1, \ .addr_limit = KERNEL_DS, \ diff --git a/arch/score/kernel/asm-offsets.c b/arch/score/kernel/asm-offsets.c index b4d5214a7a7e..52794f9421e2 100644 --- a/arch/score/kernel/asm-offsets.c +++ b/arch/score/kernel/asm-offsets.c @@ -100,7 +100,6 @@ void output_thread_info_defines(void) { COMMENT("SCORE thread_info offsets."); OFFSET(TI_TASK, thread_info, task); - OFFSET(TI_EXEC_DOMAIN, thread_info, exec_domain); OFFSET(TI_FLAGS, thread_info, flags); OFFSET(TI_TP_VALUE, thread_info, tp_value); OFFSET(TI_CPU, thread_info, cpu); diff --git a/arch/sh/include/asm/thread_info.h b/arch/sh/include/asm/thread_info.h index 657c03919627..2afa321157be 100644 --- a/arch/sh/include/asm/thread_info.h +++ b/arch/sh/include/asm/thread_info.h @@ -27,7 +27,6 @@ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ __u32 status; /* thread synchronous flags */ __u32 cpu; @@ -56,7 +55,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .status = 0, \ .cpu = 0, \ diff --git a/arch/sh/kernel/asm-offsets.c b/arch/sh/kernel/asm-offsets.c index 542225fedb11..4bd44da910f3 100644 --- a/arch/sh/kernel/asm-offsets.c +++ b/arch/sh/kernel/asm-offsets.c @@ -21,7 +21,6 @@ int main(void) { /* offsets into the thread_info struct */ DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain)); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_PRE_COUNT, offsetof(struct thread_info, preempt_count)); diff --git a/arch/sh/kernel/irq.c b/arch/sh/kernel/irq.c index 65a1ecd77f96..eb10ff84015c 100644 --- a/arch/sh/kernel/irq.c +++ b/arch/sh/kernel/irq.c @@ -124,7 +124,6 @@ void irq_ctx_init(int cpu) irqctx = (union irq_ctx *)&hardirq_stack[cpu * THREAD_SIZE]; irqctx->tinfo.task = NULL; - irqctx->tinfo.exec_domain = NULL; irqctx->tinfo.cpu = cpu; irqctx->tinfo.preempt_count = HARDIRQ_OFFSET; irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); @@ -133,7 +132,6 @@ void irq_ctx_init(int cpu) irqctx = (union irq_ctx *)&softirq_stack[cpu * THREAD_SIZE]; irqctx->tinfo.task = NULL; - irqctx->tinfo.exec_domain = NULL; irqctx->tinfo.cpu = cpu; irqctx->tinfo.preempt_count = 0; irqctx->tinfo.addr_limit = MAKE_MM_SEG(0); diff --git a/arch/sh/kernel/signal_32.c b/arch/sh/kernel/signal_32.c index 97292890b51b..f7c3d5c25caf 100644 --- a/arch/sh/kernel/signal_32.c +++ b/arch/sh/kernel/signal_32.c @@ -267,19 +267,12 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, { struct sigframe __user *frame; int err = 0, sig = ksig->sig; - int signal; frame = get_sigframe(&ksig->ka, regs->regs[15], sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - signal = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - err |= setup_sigcontext(&frame->sc, regs, set->sig[0]); if (_NSIG_WORDS > 1) @@ -313,7 +306,7 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, /* Set up registers for signal handler */ regs->regs[15] = (unsigned long) frame; - regs->regs[4] = signal; /* Arg for signal handler */ + regs->regs[4] = sig; /* Arg for signal handler */ regs->regs[5] = 0; regs->regs[6] = (unsigned long) &frame->sc; @@ -340,19 +333,12 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, { struct rt_sigframe __user *frame; int err = 0, sig = ksig->sig; - int signal; frame = get_sigframe(&ksig->ka, regs->regs[15], sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - signal = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - err |= copy_siginfo_to_user(&frame->info, &ksig->info); /* Create the ucontext. */ @@ -390,7 +376,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, /* Set up registers for signal handler */ regs->regs[15] = (unsigned long) frame; - regs->regs[4] = signal; /* Arg for signal handler */ + regs->regs[4] = sig; /* Arg for signal handler */ regs->regs[5] = (unsigned long) &frame->info; regs->regs[6] = (unsigned long) &frame->uc; diff --git a/arch/sh/kernel/signal_64.c b/arch/sh/kernel/signal_64.c index 0462995d4d7f..d8a3f0d22809 100644 --- a/arch/sh/kernel/signal_64.c +++ b/arch/sh/kernel/signal_64.c @@ -385,12 +385,6 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - signal = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - err |= setup_sigcontext(&frame->sc, regs, set->sig[0]); /* Give up earlier as i386, in case */ @@ -441,7 +435,7 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs * All edited pointers are subject to NEFF. */ regs->regs[REG_SP] = neff_sign_extend((unsigned long)frame); - regs->regs[REG_ARG1] = signal; /* Arg for signal handler */ + regs->regs[REG_ARG1] = sig; /* Arg for signal handler */ /* FIXME: The glibc profiling support for SH-5 needs to be passed a sigcontext @@ -459,7 +453,7 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, struct pt_regs *regs /* Broken %016Lx */ pr_debug("SIG deliver (#%d,%s:%d): sp=%p pc=%08Lx%08Lx link=%08Lx%08Lx\n", - signal, current->comm, current->pid, frame, + sig, current->comm, current->pid, frame, regs->pc >> 32, regs->pc & 0xffffffff, DEREF_REG_PR >> 32, DEREF_REG_PR & 0xffffffff); @@ -471,19 +465,12 @@ static int setup_rt_frame(struct ksignal *kig, sigset_t *set, { struct rt_sigframe __user *frame; int err = 0, sig = ksig->sig; - int signal; frame = get_sigframe(&ksig->ka, regs->regs[REG_SP], sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) return -EFAULT; - signal = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - err |= __put_user(&frame->info, &frame->pinfo); err |= __put_user(&frame->uc, &frame->puc); err |= copy_siginfo_to_user(&frame->info, &ksig->info); @@ -540,13 +527,13 @@ static int setup_rt_frame(struct ksignal *kig, sigset_t *set, * All edited pointers are subject to NEFF. */ regs->regs[REG_SP] = neff_sign_extend((unsigned long)frame); - regs->regs[REG_ARG1] = signal; /* Arg for signal handler */ + regs->regs[REG_ARG1] = sig; /* Arg for signal handler */ regs->regs[REG_ARG2] = (unsigned long long)(unsigned long)(signed long)&frame->info; regs->regs[REG_ARG3] = (unsigned long long)(unsigned long)(signed long)&frame->uc.uc_mcontext; regs->pc = neff_sign_extend((unsigned long)ksig->ka.sa.sa_handler); pr_debug("SIG deliver (#%d,%s:%d): sp=%p pc=%08Lx%08Lx link=%08Lx%08Lx\n", - signal, current->comm, current->pid, frame, + sig, current->comm, current->pid, frame, regs->pc >> 32, regs->pc & 0xffffffff, DEREF_REG_PR >> 32, DEREF_REG_PR & 0xffffffff); diff --git a/arch/sparc/include/asm/seccomp.h b/arch/sparc/include/asm/seccomp.h index adca1bce41d4..5ef8826d44f8 100644 --- a/arch/sparc/include/asm/seccomp.h +++ b/arch/sparc/include/asm/seccomp.h @@ -1,15 +1,10 @@ #ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H #include <linux/unistd.h> -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - -#define __NR_seccomp_read_32 __NR_read -#define __NR_seccomp_write_32 __NR_write -#define __NR_seccomp_exit_32 __NR_exit #define __NR_seccomp_sigreturn_32 __NR_sigreturn +#include <asm-generic/seccomp.h> + #endif /* _ASM_SECCOMP_H */ diff --git a/arch/sparc/include/asm/thread_info_32.h b/arch/sparc/include/asm/thread_info_32.h index fd7bd0a440ca..229475f0d7ce 100644 --- a/arch/sparc/include/asm/thread_info_32.h +++ b/arch/sparc/include/asm/thread_info_32.h @@ -27,7 +27,6 @@ struct thread_info { unsigned long uwinmask; struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, @@ -35,6 +34,8 @@ struct thread_info { int softirq_count; int hardirq_count; + u32 __unused; + /* Context switch saved kernel state. */ unsigned long ksp; /* ... ksp __attribute__ ((aligned (8))); */ unsigned long kpc; @@ -56,7 +57,6 @@ struct thread_info { { \ .uwinmask = 0, \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ @@ -85,12 +85,11 @@ register struct thread_info *current_thread_info_reg asm("g6"); */ #define TI_UWINMASK 0x00 /* uwinmask */ #define TI_TASK 0x04 -#define TI_EXECDOMAIN 0x08 /* exec_domain */ -#define TI_FLAGS 0x0c -#define TI_CPU 0x10 -#define TI_PREEMPT 0x14 /* preempt_count */ -#define TI_SOFTIRQ 0x18 /* softirq_count */ -#define TI_HARDIRQ 0x1c /* hardirq_count */ +#define TI_FLAGS 0x08 +#define TI_CPU 0x0c +#define TI_PREEMPT 0x10 /* preempt_count */ +#define TI_SOFTIRQ 0x14 /* softirq_count */ +#define TI_HARDIRQ 0x18 /* hardirq_count */ #define TI_KSP 0x20 /* ksp */ #define TI_KPC 0x24 /* kpc (ldd'ed with kpc) */ #define TI_KPSR 0x28 /* kpsr */ diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h index ff455164732a..bde59825d06c 100644 --- a/arch/sparc/include/asm/thread_info_64.h +++ b/arch/sparc/include/asm/thread_info_64.h @@ -31,7 +31,6 @@ #include <asm/types.h> struct task_struct; -struct exec_domain; struct thread_info { /* D$ line 1 */ @@ -44,7 +43,6 @@ struct thread_info { /* D$ line 2 */ unsigned long fault_address; struct pt_regs *kregs; - struct exec_domain *exec_domain; int preempt_count; /* 0 => preemptable, <0 => BUG */ __u8 new_child; __u8 current_ds; @@ -80,18 +78,17 @@ struct thread_info { #define TI_KSP 0x00000018 #define TI_FAULT_ADDR 0x00000020 #define TI_KREGS 0x00000028 -#define TI_EXEC_DOMAIN 0x00000030 -#define TI_PRE_COUNT 0x00000038 -#define TI_NEW_CHILD 0x0000003c -#define TI_CURRENT_DS 0x0000003d -#define TI_CPU 0x0000003e -#define TI_UTRAPS 0x00000040 -#define TI_REG_WINDOW 0x00000048 -#define TI_RWIN_SPTRS 0x000003c8 -#define TI_GSR 0x00000400 -#define TI_XFSR 0x00000438 -#define TI_KUNA_REGS 0x00000470 -#define TI_KUNA_INSN 0x00000478 +#define TI_PRE_COUNT 0x00000030 +#define TI_NEW_CHILD 0x00000034 +#define TI_CURRENT_DS 0x00000035 +#define TI_CPU 0x00000036 +#define TI_UTRAPS 0x00000038 +#define TI_REG_WINDOW 0x00000040 +#define TI_RWIN_SPTRS 0x000003c0 +#define TI_GSR 0x000003f8 +#define TI_XFSR 0x00000430 +#define TI_KUNA_REGS 0x00000468 +#define TI_KUNA_INSN 0x00000470 #define TI_FPREGS 0x00000480 /* We embed this in the uppermost byte of thread_info->flags */ @@ -119,7 +116,6 @@ struct thread_info { { \ .task = &tsk, \ .current_ds = ASI_P, \ - .exec_domain = &default_exec_domain, \ .preempt_count = INIT_PREEMPT_COUNT, \ } diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c index 6fd386c5232a..4f21df7d4f13 100644 --- a/arch/sparc/kernel/traps_32.c +++ b/arch/sparc/kernel/traps_32.c @@ -433,7 +433,6 @@ void trap_init(void) /* Force linker to barf if mismatched */ if (TI_UWINMASK != offsetof(struct thread_info, uwinmask) || TI_TASK != offsetof(struct thread_info, task) || - TI_EXECDOMAIN != offsetof(struct thread_info, exec_domain) || TI_FLAGS != offsetof(struct thread_info, flags) || TI_CPU != offsetof(struct thread_info, cpu) || TI_PREEMPT != offsetof(struct thread_info, preempt_count) || diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 0e699745d643..d21cd625c0de 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -2691,8 +2691,6 @@ void __init trap_init(void) fault_address) || TI_KREGS != offsetof(struct thread_info, kregs) || TI_UTRAPS != offsetof(struct thread_info, utraps) || - TI_EXEC_DOMAIN != offsetof(struct thread_info, - exec_domain) || TI_REG_WINDOW != offsetof(struct thread_info, reg_window) || TI_RWIN_SPTRS != offsetof(struct thread_info, diff --git a/arch/tile/gxio/mpipe.c b/arch/tile/gxio/mpipe.c index 6f00e9850636..ee186e13dfe6 100644 --- a/arch/tile/gxio/mpipe.c +++ b/arch/tile/gxio/mpipe.c @@ -456,7 +456,7 @@ int gxio_mpipe_equeue_init(gxio_mpipe_equeue_t *equeue, EXPORT_SYMBOL_GPL(gxio_mpipe_equeue_init); int gxio_mpipe_set_timestamp(gxio_mpipe_context_t *context, - const struct timespec *ts) + const struct timespec64 *ts) { cycles_t cycles = get_cycles(); return gxio_mpipe_set_timestamp_aux(context, (uint64_t)ts->tv_sec, @@ -466,7 +466,7 @@ int gxio_mpipe_set_timestamp(gxio_mpipe_context_t *context, EXPORT_SYMBOL_GPL(gxio_mpipe_set_timestamp); int gxio_mpipe_get_timestamp(gxio_mpipe_context_t *context, - struct timespec *ts) + struct timespec64 *ts) { int ret; cycles_t cycles_prev, cycles_now, clock_rate; diff --git a/arch/tile/include/asm/thread_info.h b/arch/tile/include/asm/thread_info.h index 96c14c1430d8..98ee10a0ae89 100644 --- a/arch/tile/include/asm/thread_info.h +++ b/arch/tile/include/asm/thread_info.h @@ -26,7 +26,6 @@ */ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ __u32 homecache_cpu; /* CPU we are homecached on */ @@ -51,7 +50,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/tile/include/gxio/mpipe.h b/arch/tile/include/gxio/mpipe.h index e37cf4f0cffd..73e83a187866 100644 --- a/arch/tile/include/gxio/mpipe.h +++ b/arch/tile/include/gxio/mpipe.h @@ -1830,7 +1830,7 @@ extern int gxio_mpipe_link_set_attr(gxio_mpipe_link_t *link, uint32_t attr, * code. */ extern int gxio_mpipe_get_timestamp(gxio_mpipe_context_t *context, - struct timespec *ts); + struct timespec64 *ts); /* Set the timestamp of mPIPE. * @@ -1840,7 +1840,7 @@ extern int gxio_mpipe_get_timestamp(gxio_mpipe_context_t *context, * code. */ extern int gxio_mpipe_set_timestamp(gxio_mpipe_context_t *context, - const struct timespec *ts); + const struct timespec64 *ts); /* Adjust the timestamp of mPIPE. * diff --git a/arch/tile/kernel/compat_signal.c b/arch/tile/kernel/compat_signal.c index 8c5abf2e4794..5cbc864398d3 100644 --- a/arch/tile/kernel/compat_signal.c +++ b/arch/tile/kernel/compat_signal.c @@ -196,19 +196,12 @@ int compat_setup_rt_frame(struct ksignal *ksig, sigset_t *set, unsigned long restorer; struct compat_rt_sigframe __user *frame; int err = 0, sig = ksig->sig; - int usig; frame = compat_get_sigframe(&ksig->ka, regs, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) goto err; - usig = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - /* Always write at least the signal number for the stack backtracer. */ if (ksig->ka.sa.sa_flags & SA_SIGINFO) { /* At sigreturn time, restore the callee-save registers too. */ @@ -243,7 +236,7 @@ int compat_setup_rt_frame(struct ksignal *ksig, sigset_t *set, regs->ex1 = PL_ICS_EX1(USER_PL, 1); /* set crit sec in handler */ regs->sp = ptr_to_compat_reg(frame); regs->lr = restorer; - regs->regs[0] = (unsigned long) usig; + regs->regs[0] = (unsigned long) sig; regs->regs[1] = ptr_to_compat_reg(&frame->info); regs->regs[2] = ptr_to_compat_reg(&frame->uc); regs->flags |= PT_FLAGS_CALLER_SAVES; diff --git a/arch/tile/kernel/signal.c b/arch/tile/kernel/signal.c index 8a524e332c1a..87299a6cfec8 100644 --- a/arch/tile/kernel/signal.c +++ b/arch/tile/kernel/signal.c @@ -151,19 +151,12 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, unsigned long restorer; struct rt_sigframe __user *frame; int err = 0, sig = ksig->sig; - int usig; frame = get_sigframe(&ksig->ka, regs, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame))) goto err; - usig = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - /* Always write at least the signal number for the stack backtracer. */ if (ksig->ka.sa.sa_flags & SA_SIGINFO) { /* At sigreturn time, restore the callee-save registers too. */ @@ -198,7 +191,7 @@ static int setup_rt_frame(struct ksignal *ksig, sigset_t *set, regs->ex1 = PL_ICS_EX1(USER_PL, 1); /* set crit sec in handler */ regs->sp = (unsigned long) frame; regs->lr = restorer; - regs->regs[0] = (unsigned long) usig; + regs->regs[0] = (unsigned long) sig; regs->regs[1] = (unsigned long) &frame->info; regs->regs[2] = (unsigned long) &frame->uc; regs->flags |= PT_FLAGS_CALLER_SAVES; diff --git a/arch/um/Kconfig.um b/arch/um/Kconfig.um index 5dbfe3d9107c..6e67847f5272 100644 --- a/arch/um/Kconfig.um +++ b/arch/um/Kconfig.um @@ -95,48 +95,6 @@ config MAGIC_SYSRQ The keys are documented in <file:Documentation/sysrq.txt>. Don't say Y unless you really know what this hack does. -config SMP - bool "Symmetric multi-processing support" - default n - depends on BROKEN - help - This option enables UML SMP support. - It is NOT related to having a real SMP box. Not directly, at least. - - UML implements virtual SMP by allowing as many processes to run - simultaneously on the host as there are virtual processors configured. - - Obviously, if the host is a uniprocessor, those processes will - timeshare, but, inside UML, will appear to be running simultaneously. - If the host is a multiprocessor, then UML processes may run - simultaneously, depending on the host scheduler. - - This, however, is supported only in TT mode. So, if you use the SKAS - patch on your host, switching to TT mode and enabling SMP usually - gives you worse performances. - Also, since the support for SMP has been under-developed, there could - be some bugs being exposed by enabling SMP. - - If you don't know what to do, say N. - -config NR_CPUS - int "Maximum number of CPUs (2-32)" - range 2 32 - depends on SMP - default "32" - -config HIGHMEM - bool "Highmem support" - depends on !64BIT && BROKEN - default n - help - This was used to allow UML to run with big amounts of memory. - Currently it is unstable, so if unsure say N. - - To use big amounts of memory, it is recommended enable static - linking (i.e. CONFIG_STATIC_LINK) - this should allow the - guest to use up to 2.75G of memory. - config KERNEL_STACK_ORDER int "Kernel stack size order" default 1 if 64BIT diff --git a/arch/um/Makefile b/arch/um/Makefile index e4b1a9639c4d..17d4460b1af3 100644 --- a/arch/um/Makefile +++ b/arch/um/Makefile @@ -43,8 +43,8 @@ endif HOST_DIR := arch/$(HEADER_ARCH) -include $(srctree)/$(ARCH_DIR)/Makefile-skas -include $(srctree)/$(HOST_DIR)/Makefile.um +include $(ARCH_DIR)/Makefile-skas +include $(HOST_DIR)/Makefile.um core-y += $(HOST_DIR)/um/ @@ -73,7 +73,7 @@ USER_CFLAGS = $(patsubst $(KERNEL_DEFINES),,$(patsubst -D__KERNEL__,,\ $(filter -I%,$(CFLAGS)) -D_FILE_OFFSET_BITS=64 -idirafter include #This will adjust *FLAGS accordingly to the platform. -include $(srctree)/$(ARCH_DIR)/Makefile-os-$(OS) +include $(ARCH_DIR)/Makefile-os-$(OS) KBUILD_CPPFLAGS += -I$(srctree)/$(HOST_DIR)/include \ -I$(srctree)/$(HOST_DIR)/include/uapi \ diff --git a/arch/um/Makefile-ia64 b/arch/um/Makefile-ia64 deleted file mode 100644 index f84dc23b0f6e..000000000000 --- a/arch/um/Makefile-ia64 +++ /dev/null @@ -1 +0,0 @@ -START_ADDR = 0x1000000000000000 diff --git a/arch/um/Makefile-ppc b/arch/um/Makefile-ppc deleted file mode 100644 index 66fd2003e165..000000000000 --- a/arch/um/Makefile-ppc +++ /dev/null @@ -1,9 +0,0 @@ -ifeq ($(CONFIG_HOST_2G_2G), y) -START_ADDR = 0x80000000 -else -START_ADDR = 0xc0000000 -endif -ARCH_CFLAGS = -U__powerpc__ -D__UM_PPC__ - -# The arch is ppc, but the elf32 name is powerpc -ELF_SUBARCH = powerpc diff --git a/arch/um/include/asm/fixmap.h b/arch/um/include/asm/fixmap.h index 3094ea3c73b0..1761fd75bf13 100644 --- a/arch/um/include/asm/fixmap.h +++ b/arch/um/include/asm/fixmap.h @@ -33,10 +33,6 @@ * fix-mapped? */ enum fixed_addresses { -#ifdef CONFIG_HIGHMEM - FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */ - FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, -#endif __end_of_fixed_addresses }; diff --git a/arch/um/include/asm/pgtable.h b/arch/um/include/asm/pgtable.h index 2324b624f195..18eb9924dda3 100644 --- a/arch/um/include/asm/pgtable.h +++ b/arch/um/include/asm/pgtable.h @@ -47,11 +47,7 @@ extern unsigned long end_iomem; #define VMALLOC_OFFSET (__va_space) #define VMALLOC_START ((end_iomem + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)) #define PKMAP_BASE ((FIXADDR_START - LAST_PKMAP * PAGE_SIZE) & PMD_MASK) -#ifdef CONFIG_HIGHMEM -# define VMALLOC_END (PKMAP_BASE-2*PAGE_SIZE) -#else -# define VMALLOC_END (FIXADDR_START-2*PAGE_SIZE) -#endif +#define VMALLOC_END (FIXADDR_START-2*PAGE_SIZE) #define MODULES_VADDR VMALLOC_START #define MODULES_END VMALLOC_END #define MODULES_LEN (MODULES_VADDR - MODULES_END) diff --git a/arch/um/include/asm/processor-generic.h b/arch/um/include/asm/processor-generic.h index cbc5edd5a901..2d1e0dd5bb0b 100644 --- a/arch/um/include/asm/processor-generic.h +++ b/arch/um/include/asm/processor-generic.h @@ -98,16 +98,8 @@ struct cpuinfo_um { extern struct cpuinfo_um boot_cpu_data; -#define my_cpu_data cpu_data[smp_processor_id()] - -#ifdef CONFIG_SMP -extern struct cpuinfo_um cpu_data[]; -#define current_cpu_data cpu_data[smp_processor_id()] -#else #define cpu_data (&boot_cpu_data) #define current_cpu_data boot_cpu_data -#endif - #define KSTK_REG(tsk, reg) get_thread_reg(reg, &tsk->thread.switch_buf) extern unsigned long get_wchan(struct task_struct *p); diff --git a/arch/um/include/asm/smp.h b/arch/um/include/asm/smp.h index e4507938d8cf..9c3be355ed01 100644 --- a/arch/um/include/asm/smp.h +++ b/arch/um/include/asm/smp.h @@ -1,32 +1,6 @@ #ifndef __UM_SMP_H #define __UM_SMP_H -#ifdef CONFIG_SMP - -#include <linux/bitops.h> -#include <asm/current.h> -#include <linux/cpumask.h> - -#define raw_smp_processor_id() (current_thread->cpu) - -#define cpu_logical_map(n) (n) -#define cpu_number_map(n) (n) -extern int hard_smp_processor_id(void); -#define NO_PROC_ID -1 - -extern int ncpus; - - -static inline void smp_cpus_done(unsigned int maxcpus) -{ -} - -extern struct task_struct *idle_threads[NR_CPUS]; - -#else - #define hard_smp_processor_id() 0 #endif - -#endif diff --git a/arch/um/include/asm/thread_info.h b/arch/um/include/asm/thread_info.h index e04114c4fcd9..b30c85b141d9 100644 --- a/arch/um/include/asm/thread_info.h +++ b/arch/um/include/asm/thread_info.h @@ -14,7 +14,6 @@ struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ __u32 cpu; /* current CPU */ int preempt_count; /* 0 => preemptable, @@ -28,7 +27,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/um/include/shared/as-layout.h b/arch/um/include/shared/as-layout.h index 41c8c774ec10..ca1843e1df15 100644 --- a/arch/um/include/shared/as-layout.h +++ b/arch/um/include/shared/as-layout.h @@ -56,6 +56,7 @@ extern unsigned long brk_start; extern unsigned long host_task_size; extern int linux_main(int argc, char **argv); +extern void uml_finishsetup(void); struct siginfo; extern void (*sig_info[])(int, struct siginfo *si, struct uml_pt_regs *); diff --git a/arch/um/include/shared/os.h b/arch/um/include/shared/os.h index 08eec0b691b0..d824528f6f62 100644 --- a/arch/um/include/shared/os.h +++ b/arch/um/include/shared/os.h @@ -174,7 +174,6 @@ extern unsigned long long os_makedev(unsigned major, unsigned minor); /* start_up.c */ extern void os_early_checks(void); -extern void can_do_skas(void); extern void os_check_bugs(void); extern void check_host_supports_tls(int *supports_tls, int *tls_min); @@ -187,7 +186,6 @@ extern int os_process_parent(int pid); extern void os_stop_process(int pid); extern void os_kill_process(int pid, int reap_child); extern void os_kill_ptraced_process(int pid, int reap_child); -extern long os_ptrace_ldt(long pid, long addr, long data); extern int os_getpid(void); extern int os_getpgrp(void); diff --git a/arch/um/include/shared/skas/proc_mm.h b/arch/um/include/shared/skas/proc_mm.h deleted file mode 100644 index 902809209603..000000000000 --- a/arch/um/include/shared/skas/proc_mm.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2002 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SKAS_PROC_MM_H -#define __SKAS_PROC_MM_H - -#define MM_MMAP 54 -#define MM_MUNMAP 55 -#define MM_MPROTECT 56 -#define MM_COPY_SEGMENTS 57 - -struct mm_mmap { - unsigned long addr; - unsigned long len; - unsigned long prot; - unsigned long flags; - unsigned long fd; - unsigned long offset; -}; - -struct mm_munmap { - unsigned long addr; - unsigned long len; -}; - -struct mm_mprotect { - unsigned long addr; - unsigned long len; - unsigned int prot; -}; - -struct proc_mm_op { - int op; - union { - struct mm_mmap mmap; - struct mm_munmap munmap; - struct mm_mprotect mprotect; - int copy_segments; - } u; -}; - -#endif diff --git a/arch/um/include/shared/skas/skas.h b/arch/um/include/shared/skas/skas.h index c45df961c874..911f3c45ad1f 100644 --- a/arch/um/include/shared/skas/skas.h +++ b/arch/um/include/shared/skas/skas.h @@ -9,13 +9,10 @@ #include <sysdep/ptrace.h> extern int userspace_pid[]; -extern int proc_mm, ptrace_faultinfo, ptrace_ldt; -extern int skas_needs_stub; extern int user_thread(unsigned long stack, int flags); extern void new_thread_handler(void); extern void handle_syscall(struct uml_pt_regs *regs); -extern int new_mm(unsigned long stack); extern long execute_syscall_skas(void *r); extern unsigned long current_stub_stack(void); diff --git a/arch/um/include/shared/skas_ptrace.h b/arch/um/include/shared/skas_ptrace.h deleted file mode 100644 index 630a9c92b93c..000000000000 --- a/arch/um/include/shared/skas_ptrace.h +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) - * Licensed under the GPL - */ - -#ifndef __SKAS_PTRACE_H -#define __SKAS_PTRACE_H - -#define PTRACE_FAULTINFO 52 -#define PTRACE_SWITCH_MM 55 - -#include <sysdep/skas_ptrace.h> - -#endif diff --git a/arch/um/kernel/Makefile b/arch/um/kernel/Makefile index 2d840a070c8b..a6a5e42caaef 100644 --- a/arch/um/kernel/Makefile +++ b/arch/um/kernel/Makefile @@ -12,8 +12,8 @@ clean-files := obj-y = config.o exec.o exitcode.o irq.o ksyms.o mem.o \ physmem.o process.o ptrace.o reboot.o sigio.o \ - signal.o smp.o syscall.o sysrq.o time.o tlb.o trap.o \ - um_arch.o umid.o maccess.o skas/ + signal.o syscall.o sysrq.o time.o tlb.o trap.o \ + um_arch.o umid.o maccess.o kmsg_dump.o skas/ obj-$(CONFIG_BLK_DEV_INITRD) += initrd.o obj-$(CONFIG_GPROF) += gprof_syms.o diff --git a/arch/um/kernel/irq.c b/arch/um/kernel/irq.c index 1d8505b1e290..23cb9350d47e 100644 --- a/arch/um/kernel/irq.c +++ b/arch/um/kernel/irq.c @@ -35,9 +35,6 @@ void sigio_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs) struct irq_fd *irq_fd; int n; - if (smp_sigio_handler()) - return; - while (1) { n = os_waiting_for_events(active_fds); if (n <= 0) { diff --git a/arch/um/kernel/kmsg_dump.c b/arch/um/kernel/kmsg_dump.c new file mode 100644 index 000000000000..407d49251d6f --- /dev/null +++ b/arch/um/kernel/kmsg_dump.c @@ -0,0 +1,43 @@ +#include <linux/kmsg_dump.h> +#include <linux/console.h> +#include <shared/init.h> +#include <shared/kern.h> +#include <os.h> + +static void kmsg_dumper_stdout(struct kmsg_dumper *dumper, + enum kmsg_dump_reason reason) +{ + static char line[1024]; + + size_t len = 0; + bool con_available = false; + + /* only dump kmsg when no console is available */ + if (!console_trylock()) + return; + + if (console_drivers != NULL) + con_available = true; + + console_unlock(); + + if (con_available == true) + return; + + printf("kmsg_dump:\n"); + while (kmsg_dump_get_line(dumper, true, line, sizeof(line), &len)) { + line[len] = '\0'; + printf("%s", line); + } +} + +static struct kmsg_dumper kmsg_dumper = { + .dump = kmsg_dumper_stdout +}; + +int __init kmsg_dumper_stdout_init(void) +{ + return kmsg_dump_register(&kmsg_dumper); +} + +__uml_postsetup(kmsg_dumper_stdout_init); diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c index 8636e905426f..b2a2dff50b4e 100644 --- a/arch/um/kernel/mem.c +++ b/arch/um/kernel/mem.c @@ -38,19 +38,6 @@ int kmalloc_ok = 0; /* Used during early boot */ static unsigned long brk_end; -#ifdef CONFIG_HIGHMEM -static void setup_highmem(unsigned long highmem_start, - unsigned long highmem_len) -{ - unsigned long highmem_pfn; - int i; - - highmem_pfn = __pa(highmem_start) >> PAGE_SHIFT; - for (i = 0; i < highmem_len >> PAGE_SHIFT; i++) - free_highmem_page(&mem_map[highmem_pfn + i]); -} -#endif - void __init mem_init(void) { /* clear the zero-page */ @@ -67,9 +54,6 @@ void __init mem_init(void) /* this will put all low memory onto the freelists */ free_all_bootmem(); max_low_pfn = totalram_pages; -#ifdef CONFIG_HIGHMEM - setup_highmem(end_iomem, highmem); -#endif max_pfn = totalram_pages; mem_init_print_info(NULL); kmalloc_ok = 1; @@ -127,49 +111,6 @@ static void __init fixrange_init(unsigned long start, unsigned long end, } } -#ifdef CONFIG_HIGHMEM -pte_t *kmap_pte; -pgprot_t kmap_prot; - -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)),\ - (vaddr)), (vaddr)) - -static void __init kmap_init(void) -{ - unsigned long kmap_vstart; - - /* cache the first kmap pte */ - kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); - kmap_pte = kmap_get_fixmap_pte(kmap_vstart); - - kmap_prot = PAGE_KERNEL; -} - -static void __init init_highmem(void) -{ - pgd_t *pgd; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - unsigned long vaddr; - - /* - * Permanent kmaps: - */ - vaddr = PKMAP_BASE; - fixrange_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, swapper_pg_dir); - - pgd = swapper_pg_dir + pgd_index(vaddr); - pud = pud_offset(pgd, vaddr); - pmd = pmd_offset(pud, vaddr); - pte = pte_offset_kernel(pmd, vaddr); - pkmap_page_table = pte; - - kmap_init(); -} -#endif /* CONFIG_HIGHMEM */ - static void __init fixaddr_user_init( void) { #ifdef CONFIG_ARCH_REUSE_HOST_VSYSCALL_AREA @@ -211,9 +152,6 @@ void __init paging_init(void) zones_size[ZONE_NORMAL] = (end_iomem >> PAGE_SHIFT) - (uml_physmem >> PAGE_SHIFT); -#ifdef CONFIG_HIGHMEM - zones_size[ZONE_HIGHMEM] = highmem >> PAGE_SHIFT; -#endif free_area_init(zones_size); /* @@ -224,10 +162,6 @@ void __init paging_init(void) fixrange_init(vaddr, FIXADDR_TOP, swapper_pg_dir); fixaddr_user_init(); - -#ifdef CONFIG_HIGHMEM - init_highmem(); -#endif } /* diff --git a/arch/um/kernel/physmem.c b/arch/um/kernel/physmem.c index 549ecf3f5857..9034fc8056b4 100644 --- a/arch/um/kernel/physmem.c +++ b/arch/um/kernel/physmem.c @@ -57,22 +57,51 @@ void map_memory(unsigned long virt, unsigned long phys, unsigned long len, extern int __syscall_stub_start; +/** + * setup_physmem() - Setup physical memory for UML + * @start: Start address of the physical kernel memory, + * i.e start address of the executable image. + * @reserve_end: end address of the physical kernel memory. + * @len: Length of total physical memory that should be mapped/made + * available, in bytes. + * @highmem: Number of highmem bytes that should be mapped/made available. + * + * Creates an unlinked temporary file of size (len + highmem) and memory maps + * it on the last executable image address (uml_reserved). + * + * The offset is needed as the length of the total physical memory + * (len + highmem) includes the size of the memory used be the executable image, + * but the mapped-to address is the last address of the executable image + * (uml_reserved == end address of executable image). + * + * The memory mapped memory of the temporary file is used as backing memory + * of all user space processes/kernel tasks. + */ void __init setup_physmem(unsigned long start, unsigned long reserve_end, unsigned long len, unsigned long long highmem) { unsigned long reserve = reserve_end - start; - int pfn = PFN_UP(__pa(reserve_end)); - int delta = (len - reserve) >> PAGE_SHIFT; - int err, offset, bootmap_size; + unsigned long pfn = PFN_UP(__pa(reserve_end)); + unsigned long delta = (len - reserve) >> PAGE_SHIFT; + unsigned long offset, bootmap_size; + long map_size; + int err; + + offset = uml_reserved - uml_physmem; + map_size = len - offset; + if(map_size <= 0) { + printf("Too few physical memory! Needed=%d, given=%d\n", + offset, len); + exit(1); + } physmem_fd = create_mem_file(len + highmem); - offset = uml_reserved - uml_physmem; err = os_map_memory((void *) uml_reserved, physmem_fd, offset, - len - offset, 1, 1, 1); + map_size, 1, 1, 1); if (err < 0) { printf("setup_physmem - mapping %ld bytes of memory at 0x%p " - "failed - errno = %d\n", len - offset, + "failed - errno = %d\n", map_size, (void *) uml_reserved, err); exit(1); } diff --git a/arch/um/kernel/process.c b/arch/um/kernel/process.c index f17bca8ed2ce..68b9119841cd 100644 --- a/arch/um/kernel/process.c +++ b/arch/um/kernel/process.c @@ -259,17 +259,6 @@ int strlen_user_proc(char __user *str) return strlen_user(str); } -int smp_sigio_handler(void) -{ -#ifdef CONFIG_SMP - int cpu = current_thread_info()->cpu; - IPI_handler(cpu); - if (cpu != 0) - return 1; -#endif - return 0; -} - int cpu(void) { return current_thread_info()->cpu; diff --git a/arch/um/kernel/ptrace.c b/arch/um/kernel/ptrace.c index 62435ef003d9..174ee5017264 100644 --- a/arch/um/kernel/ptrace.c +++ b/arch/um/kernel/ptrace.c @@ -8,9 +8,6 @@ #include <linux/sched.h> #include <linux/tracehook.h> #include <asm/uaccess.h> -#include <skas_ptrace.h> - - void user_enable_single_step(struct task_struct *child) { @@ -104,35 +101,6 @@ long arch_ptrace(struct task_struct *child, long request, ret = ptrace_set_thread_area(child, addr, vp); break; - case PTRACE_FAULTINFO: { - /* - * Take the info from thread->arch->faultinfo, - * but transfer max. sizeof(struct ptrace_faultinfo). - * On i386, ptrace_faultinfo is smaller! - */ - ret = copy_to_user(p, &child->thread.arch.faultinfo, - sizeof(struct ptrace_faultinfo)) ? - -EIO : 0; - break; - } - -#ifdef PTRACE_LDT - case PTRACE_LDT: { - struct ptrace_ldt ldt; - - if (copy_from_user(&ldt, p, sizeof(ldt))) { - ret = -EIO; - break; - } - - /* - * This one is confusing, so just punt and return -EIO for - * now - */ - ret = -EIO; - break; - } -#endif default: ret = ptrace_request(child, request, addr, data); if (ret == -EIO) diff --git a/arch/um/kernel/reboot.c b/arch/um/kernel/reboot.c index ced8903921ae..9bdf67a092a5 100644 --- a/arch/um/kernel/reboot.c +++ b/arch/um/kernel/reboot.c @@ -15,28 +15,21 @@ void (*pm_power_off)(void); static void kill_off_processes(void) { - if (proc_mm) - /* - * FIXME: need to loop over userspace_pids - */ - os_kill_ptraced_process(userspace_pid[0], 1); - else { - struct task_struct *p; - int pid; - - read_lock(&tasklist_lock); - for_each_process(p) { - struct task_struct *t; - - t = find_lock_task_mm(p); - if (!t) - continue; - pid = t->mm->context.id.u.pid; - task_unlock(t); - os_kill_ptraced_process(pid, 1); - } - read_unlock(&tasklist_lock); + struct task_struct *p; + int pid; + + read_lock(&tasklist_lock); + for_each_process(p) { + struct task_struct *t; + + t = find_lock_task_mm(p); + if (!t) + continue; + pid = t->mm->context.id.u.pid; + task_unlock(t); + os_kill_ptraced_process(pid, 1); } + read_unlock(&tasklist_lock); } void uml_cleanup(void) diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c index 007d5503f49b..94abdcc1d6ad 100644 --- a/arch/um/kernel/skas/mmu.c +++ b/arch/um/kernel/skas/mmu.c @@ -54,35 +54,22 @@ int init_new_context(struct task_struct *task, struct mm_struct *mm) unsigned long stack = 0; int ret = -ENOMEM; - if (skas_needs_stub) { - stack = get_zeroed_page(GFP_KERNEL); - if (stack == 0) - goto out; - } + stack = get_zeroed_page(GFP_KERNEL); + if (stack == 0) + goto out; to_mm->id.stack = stack; if (current->mm != NULL && current->mm != &init_mm) from_mm = ¤t->mm->context; - if (proc_mm) { - ret = new_mm(stack); - if (ret < 0) { - printk(KERN_ERR "init_new_context_skas - " - "new_mm failed, errno = %d\n", ret); - goto out_free; - } - to_mm->id.u.mm_fd = ret; - } - else { - if (from_mm) - to_mm->id.u.pid = copy_context_skas0(stack, - from_mm->id.u.pid); - else to_mm->id.u.pid = start_userspace(stack); - - if (to_mm->id.u.pid < 0) { - ret = to_mm->id.u.pid; - goto out_free; - } + if (from_mm) + to_mm->id.u.pid = copy_context_skas0(stack, + from_mm->id.u.pid); + else to_mm->id.u.pid = start_userspace(stack); + + if (to_mm->id.u.pid < 0) { + ret = to_mm->id.u.pid; + goto out_free; } ret = init_new_ldt(to_mm, from_mm); @@ -105,9 +92,6 @@ void uml_setup_stubs(struct mm_struct *mm) { int err, ret; - if (!skas_needs_stub) - return; - ret = init_stub_pte(mm, STUB_CODE, (unsigned long) &__syscall_stub_start); if (ret) @@ -154,25 +138,19 @@ void destroy_context(struct mm_struct *mm) { struct mm_context *mmu = &mm->context; - if (proc_mm) - os_close_file(mmu->id.u.mm_fd); - else { - /* - * If init_new_context wasn't called, this will be - * zero, resulting in a kill(0), which will result in the - * whole UML suddenly dying. Also, cover negative and - * 1 cases, since they shouldn't happen either. - */ - if (mmu->id.u.pid < 2) { - printk(KERN_ERR "corrupt mm_context - pid = %d\n", - mmu->id.u.pid); - return; - } - os_kill_ptraced_process(mmu->id.u.pid, 1); + /* + * If init_new_context wasn't called, this will be + * zero, resulting in a kill(0), which will result in the + * whole UML suddenly dying. Also, cover negative and + * 1 cases, since they shouldn't happen either. + */ + if (mmu->id.u.pid < 2) { + printk(KERN_ERR "corrupt mm_context - pid = %d\n", + mmu->id.u.pid); + return; } + os_kill_ptraced_process(mmu->id.u.pid, 1); - if (skas_needs_stub) - free_page(mmu->id.stack); - + free_page(mmu->id.stack); free_ldt(mmu); } diff --git a/arch/um/kernel/skas/process.c b/arch/um/kernel/skas/process.c index 4da11b3c8ddb..527fa5881915 100644 --- a/arch/um/kernel/skas/process.c +++ b/arch/um/kernel/skas/process.c @@ -10,25 +10,6 @@ #include <os.h> #include <skas.h> -int new_mm(unsigned long stack) -{ - int fd, err; - - fd = os_open_file("/proc/mm", of_cloexec(of_write(OPENFLAGS())), 0); - if (fd < 0) - return fd; - - if (skas_needs_stub) { - err = map_stub_pages(fd, STUB_CODE, STUB_DATA, stack); - if (err) { - os_close_file(fd); - return err; - } - } - - return fd; -} - extern void start_kernel(void); static int __init start_kernel_proc(void *unused) @@ -40,9 +21,7 @@ static int __init start_kernel_proc(void *unused) cpu_tasks[0].pid = pid; cpu_tasks[0].task = current; -#ifdef CONFIG_SMP - init_cpu_online(get_cpu_mask(0)); -#endif + start_kernel(); return 0; } @@ -55,14 +34,6 @@ int __init start_uml(void) { stack_protections((unsigned long) &cpu0_irqstack); set_sigstack(cpu0_irqstack, THREAD_SIZE); - if (proc_mm) { - userspace_pid[0] = start_userspace(0); - if (userspace_pid[0] < 0) { - printf("start_uml - start_userspace returned %d\n", - userspace_pid[0]); - exit(1); - } - } init_new_thread_signals(); diff --git a/arch/um/kernel/smp.c b/arch/um/kernel/smp.c deleted file mode 100644 index 5c8c3ea7db7b..000000000000 --- a/arch/um/kernel/smp.c +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) - * Licensed under the GPL - */ - -#include <linux/percpu.h> -#include <asm/pgalloc.h> -#include <asm/tlb.h> - -#ifdef CONFIG_SMP - -#include <linux/sched.h> -#include <linux/module.h> -#include <linux/threads.h> -#include <linux/interrupt.h> -#include <linux/err.h> -#include <linux/hardirq.h> -#include <asm/smp.h> -#include <asm/processor.h> -#include <asm/spinlock.h> -#include <kern.h> -#include <irq_user.h> -#include <os.h> - -/* Per CPU bogomips and other parameters - * The only piece used here is the ipi pipe, which is set before SMP is - * started and never changed. - */ -struct cpuinfo_um cpu_data[NR_CPUS]; - -/* A statistic, can be a little off */ -int num_reschedules_sent = 0; - -/* Not changed after boot */ -struct task_struct *idle_threads[NR_CPUS]; - -void smp_send_reschedule(int cpu) -{ - os_write_file(cpu_data[cpu].ipi_pipe[1], "R", 1); - num_reschedules_sent++; -} - -void smp_send_stop(void) -{ - int i; - - printk(KERN_INFO "Stopping all CPUs..."); - for (i = 0; i < num_online_cpus(); i++) { - if (i == current_thread->cpu) - continue; - os_write_file(cpu_data[i].ipi_pipe[1], "S", 1); - } - printk(KERN_CONT "done\n"); -} - -static cpumask_t smp_commenced_mask = CPU_MASK_NONE; -static cpumask_t cpu_callin_map = CPU_MASK_NONE; - -static int idle_proc(void *cpup) -{ - int cpu = (int) cpup, err; - - err = os_pipe(cpu_data[cpu].ipi_pipe, 1, 1); - if (err < 0) - panic("CPU#%d failed to create IPI pipe, err = %d", cpu, -err); - - os_set_fd_async(cpu_data[cpu].ipi_pipe[0]); - - wmb(); - if (cpu_test_and_set(cpu, cpu_callin_map)) { - printk(KERN_ERR "huh, CPU#%d already present??\n", cpu); - BUG(); - } - - while (!cpu_isset(cpu, smp_commenced_mask)) - cpu_relax(); - - notify_cpu_starting(cpu); - set_cpu_online(cpu, true); - default_idle(); - return 0; -} - -static struct task_struct *idle_thread(int cpu) -{ - struct task_struct *new_task; - - current->thread.request.u.thread.proc = idle_proc; - current->thread.request.u.thread.arg = (void *) cpu; - new_task = fork_idle(cpu); - if (IS_ERR(new_task)) - panic("copy_process failed in idle_thread, error = %ld", - PTR_ERR(new_task)); - - cpu_tasks[cpu] = ((struct cpu_task) - { .pid = new_task->thread.mode.tt.extern_pid, - .task = new_task } ); - idle_threads[cpu] = new_task; - panic("skas mode doesn't support SMP"); - return new_task; -} - -void smp_prepare_cpus(unsigned int maxcpus) -{ - struct task_struct *idle; - unsigned long waittime; - int err, cpu, me = smp_processor_id(); - int i; - - for (i = 0; i < ncpus; ++i) - set_cpu_possible(i, true); - - set_cpu_online(me, true); - cpu_set(me, cpu_callin_map); - - err = os_pipe(cpu_data[me].ipi_pipe, 1, 1); - if (err < 0) - panic("CPU#0 failed to create IPI pipe, errno = %d", -err); - - os_set_fd_async(cpu_data[me].ipi_pipe[0]); - - for (cpu = 1; cpu < ncpus; cpu++) { - printk(KERN_INFO "Booting processor %d...\n", cpu); - - idle = idle_thread(cpu); - - init_idle(idle, cpu); - - waittime = 200000000; - while (waittime-- && !cpu_isset(cpu, cpu_callin_map)) - cpu_relax(); - - printk(KERN_INFO "%s\n", - cpu_isset(cpu, cpu_calling_map) ? "done" : "failed"); - } -} - -void smp_prepare_boot_cpu(void) -{ - set_cpu_online(smp_processor_id(), true); -} - -int __cpu_up(unsigned int cpu, struct task_struct *tidle) -{ - cpu_set(cpu, smp_commenced_mask); - while (!cpu_online(cpu)) - mb(); - return 0; -} - -int setup_profiling_timer(unsigned int multiplier) -{ - printk(KERN_INFO "setup_profiling_timer\n"); - return 0; -} - -void smp_call_function_slave(int cpu); - -void IPI_handler(int cpu) -{ - unsigned char c; - int fd; - - fd = cpu_data[cpu].ipi_pipe[0]; - while (os_read_file(fd, &c, 1) == 1) { - switch (c) { - case 'C': - smp_call_function_slave(cpu); - break; - - case 'R': - scheduler_ipi(); - break; - - case 'S': - printk(KERN_INFO "CPU#%d stopping\n", cpu); - while (1) - pause(); - break; - - default: - printk(KERN_ERR "CPU#%d received unknown IPI [%c]!\n", - cpu, c); - break; - } - } -} - -int hard_smp_processor_id(void) -{ - return pid_to_processor_id(os_getpid()); -} - -static DEFINE_SPINLOCK(call_lock); -static atomic_t scf_started; -static atomic_t scf_finished; -static void (*func)(void *info); -static void *info; - -void smp_call_function_slave(int cpu) -{ - atomic_inc(&scf_started); - (*func)(info); - atomic_inc(&scf_finished); -} - -int smp_call_function(void (*_func)(void *info), void *_info, int wait) -{ - int cpus = num_online_cpus() - 1; - int i; - - if (!cpus) - return 0; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - spin_lock_bh(&call_lock); - atomic_set(&scf_started, 0); - atomic_set(&scf_finished, 0); - func = _func; - info = _info; - - for_each_online_cpu(i) - os_write_file(cpu_data[i].ipi_pipe[1], "C", 1); - - while (atomic_read(&scf_started) != cpus) - barrier(); - - if (wait) - while (atomic_read(&scf_finished) != cpus) - barrier(); - - spin_unlock_bh(&call_lock); - return 0; -} - -#endif diff --git a/arch/um/kernel/sysrq.c b/arch/um/kernel/sysrq.c index 894c8d303cda..aa1b56f5ac68 100644 --- a/arch/um/kernel/sysrq.c +++ b/arch/um/kernel/sysrq.c @@ -29,7 +29,7 @@ static const struct stacktrace_ops stackops = { void show_stack(struct task_struct *task, unsigned long *stack) { - unsigned long *sp = stack, bp = 0; + unsigned long *sp = stack; struct pt_regs *segv_regs = current->thread.segv_regs; int i; @@ -39,10 +39,6 @@ void show_stack(struct task_struct *task, unsigned long *stack) return; } -#ifdef CONFIG_FRAME_POINTER - bp = get_frame_pointer(task, segv_regs); -#endif - if (!stack) sp = get_stack_pointer(task, segv_regs); diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 209617302df8..8e4daf44e980 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -220,7 +220,7 @@ unsigned long segv(struct faultinfo fi, unsigned long ip, int is_user, panic("Segfault with no mm"); } - if (SEGV_IS_FIXABLE(&fi) || SEGV_MAYBE_FIXABLE(&fi)) + if (SEGV_IS_FIXABLE(&fi)) err = handle_page_fault(address, ip, is_write, is_user, &si.si_code); else { diff --git a/arch/um/kernel/um_arch.c b/arch/um/kernel/um_arch.c index 9274eae6ae7b..07f798f4bcee 100644 --- a/arch/um/kernel/um_arch.c +++ b/arch/um/kernel/um_arch.c @@ -11,6 +11,7 @@ #include <linux/string.h> #include <linux/utsname.h> #include <linux/sched.h> +#include <linux/kmsg_dump.h> #include <asm/pgtable.h> #include <asm/processor.h> #include <asm/sections.h> @@ -66,12 +67,6 @@ static int show_cpuinfo(struct seq_file *m, void *v) { int index = 0; -#ifdef CONFIG_SMP - index = (struct cpuinfo_um *) v - cpu_data; - if (!cpu_online(index)) - return 0; -#endif - seq_printf(m, "processor\t: %d\n", index); seq_printf(m, "vendor_id\t: User Mode Linux\n"); seq_printf(m, "model name\t: UML\n"); @@ -168,23 +163,6 @@ __uml_setup("debug", no_skas_debug_setup, " this flag is not needed to run gdb on UML in skas mode\n\n" ); -#ifdef CONFIG_SMP -static int __init uml_ncpus_setup(char *line, int *add) -{ - if (!sscanf(line, "%d", &ncpus)) { - printf("Couldn't parse [%s]\n", line); - return -1; - } - - return 0; -} - -__uml_setup("ncpus=", uml_ncpus_setup, -"ncpus=<# of desired CPUs>\n" -" This tells an SMP kernel how many virtual processors to start.\n\n" -); -#endif - static int __init Usage(char *line, int *add) { const char **p; @@ -234,6 +212,7 @@ static void __init uml_postsetup(void) static int panic_exit(struct notifier_block *self, unsigned long unused1, void *unused2) { + kmsg_dump(KMSG_DUMP_PANIC); bust_spinlocks(1); bust_spinlocks(0); uml_exitcode = 1; @@ -247,6 +226,16 @@ static struct notifier_block panic_exit_notifier = { .priority = 0 }; +void uml_finishsetup(void) +{ + atomic_notifier_chain_register(&panic_notifier_list, + &panic_exit_notifier); + + uml_postsetup(); + + new_thread_handler(); +} + /* Set during early boot */ unsigned long task_size; EXPORT_SYMBOL(task_size); @@ -268,7 +257,6 @@ int __init linux_main(int argc, char **argv) unsigned long stack; unsigned int i; int add; - char * mode; for (i = 1; i < argc; i++) { if ((i == 1) && (argv[i][0] == ' ')) @@ -291,15 +279,6 @@ int __init linux_main(int argc, char **argv) /* OS sanity checks that need to happen before the kernel runs */ os_early_checks(); - can_do_skas(); - - if (proc_mm && ptrace_faultinfo) - mode = "SKAS3"; - else - mode = "SKAS0"; - - printf("UML running in %s mode\n", mode); - brk_start = (unsigned long) sbrk(0); /* @@ -334,11 +313,6 @@ int __init linux_main(int argc, char **argv) if (physmem_size + iomem_size > max_physmem) { highmem = physmem_size + iomem_size - max_physmem; physmem_size -= highmem; -#ifndef CONFIG_HIGHMEM - highmem = 0; - printf("CONFIG_HIGHMEM not enabled - physical memory shrunk " - "to %Lu bytes\n", physmem_size); -#endif } high_physmem = uml_physmem + physmem_size; @@ -362,11 +336,6 @@ int __init linux_main(int argc, char **argv) printf("Kernel virtual memory size shrunk to %lu bytes\n", virtmem_size); - atomic_notifier_chain_register(&panic_notifier_list, - &panic_exit_notifier); - - uml_postsetup(); - stack_protections((unsigned long) &init_thread_info); os_flush_stdout(); @@ -390,15 +359,3 @@ void __init check_bugs(void) void apply_alternatives(struct alt_instr *start, struct alt_instr *end) { } - -#ifdef CONFIG_SMP -void alternatives_smp_module_add(struct module *mod, char *name, - void *locks, void *locks_end, - void *text, void *text_end) -{ -} - -void alternatives_smp_module_del(struct module *mod) -{ -} -#endif diff --git a/arch/um/os-Linux/process.c b/arch/um/os-Linux/process.c index 33496fe2bb52..8408aba915b2 100644 --- a/arch/um/os-Linux/process.c +++ b/arch/um/os-Linux/process.c @@ -16,7 +16,6 @@ #include <init.h> #include <longjmp.h> #include <os.h> -#include <skas_ptrace.h> #define ARBITRARY_ADDR -1 #define FAILURE_PID -1 @@ -102,21 +101,6 @@ void os_kill_process(int pid, int reap_child) CATCH_EINTR(waitpid(pid, NULL, __WALL)); } -/* This is here uniquely to have access to the userspace errno, i.e. the one - * used by ptrace in case of error. - */ - -long os_ptrace_ldt(long pid, long addr, long data) -{ - int ret; - - ret = ptrace(PTRACE_LDT, pid, addr, data); - - if (ret < 0) - return -errno; - return ret; -} - /* Kill off a ptraced child by all means available. kill it normally first, * then PTRACE_KILL it, then PTRACE_CONT it in case it's in a run state from * which it can't exit directly. diff --git a/arch/um/os-Linux/skas/mem.c b/arch/um/os-Linux/skas/mem.c index 689b18db798f..e7f8c945a573 100644 --- a/arch/um/os-Linux/skas/mem.c +++ b/arch/um/os-Linux/skas/mem.c @@ -12,7 +12,6 @@ #include <as-layout.h> #include <mm_id.h> #include <os.h> -#include <proc_mm.h> #include <ptrace_user.h> #include <registers.h> #include <skas.h> @@ -46,8 +45,6 @@ static int __init init_syscall_regs(void) __initcall(init_syscall_regs); -extern int proc_mm; - static inline long do_syscall_stub(struct mm_id * mm_idp, void **addr) { int n, i; @@ -56,10 +53,6 @@ static inline long do_syscall_stub(struct mm_id * mm_idp, void **addr) unsigned long * syscall; int err, pid = mm_idp->u.pid; - if (proc_mm) - /* FIXME: Need to look up userspace_pid by cpu */ - pid = userspace_pid[0]; - n = ptrace_setregs(pid, syscall_regs); if (n < 0) { printk(UM_KERN_ERR "Registers - \n"); @@ -178,38 +171,12 @@ int map(struct mm_id * mm_idp, unsigned long virt, unsigned long len, int prot, int phys_fd, unsigned long long offset, int done, void **data) { int ret; + unsigned long args[] = { virt, len, prot, + MAP_SHARED | MAP_FIXED, phys_fd, + MMAP_OFFSET(offset) }; - if (proc_mm) { - struct proc_mm_op map; - int fd = mm_idp->u.mm_fd; - - map = ((struct proc_mm_op) { .op = MM_MMAP, - .u = - { .mmap = - { .addr = virt, - .len = len, - .prot = prot, - .flags = MAP_SHARED | - MAP_FIXED, - .fd = phys_fd, - .offset= offset - } } } ); - CATCH_EINTR(ret = write(fd, &map, sizeof(map))); - if (ret != sizeof(map)) { - ret = -errno; - printk(UM_KERN_ERR "map : /proc/mm map failed, " - "err = %d\n", -ret); - } - else ret = 0; - } - else { - unsigned long args[] = { virt, len, prot, - MAP_SHARED | MAP_FIXED, phys_fd, - MMAP_OFFSET(offset) }; - - ret = run_syscall_stub(mm_idp, STUB_MMAP_NR, args, virt, - data, done); - } + ret = run_syscall_stub(mm_idp, STUB_MMAP_NR, args, virt, + data, done); return ret; } @@ -218,32 +185,11 @@ int unmap(struct mm_id * mm_idp, unsigned long addr, unsigned long len, int done, void **data) { int ret; + unsigned long args[] = { (unsigned long) addr, len, 0, 0, 0, + 0 }; - if (proc_mm) { - struct proc_mm_op unmap; - int fd = mm_idp->u.mm_fd; - - unmap = ((struct proc_mm_op) { .op = MM_MUNMAP, - .u = - { .munmap = - { .addr = - (unsigned long) addr, - .len = len } } } ); - CATCH_EINTR(ret = write(fd, &unmap, sizeof(unmap))); - if (ret != sizeof(unmap)) { - ret = -errno; - printk(UM_KERN_ERR "unmap - proc_mm write returned " - "%d\n", ret); - } - else ret = 0; - } - else { - unsigned long args[] = { (unsigned long) addr, len, 0, 0, 0, - 0 }; - - ret = run_syscall_stub(mm_idp, __NR_munmap, args, 0, - data, done); - } + ret = run_syscall_stub(mm_idp, __NR_munmap, args, 0, + data, done); return ret; } @@ -251,33 +197,11 @@ int unmap(struct mm_id * mm_idp, unsigned long addr, unsigned long len, int protect(struct mm_id * mm_idp, unsigned long addr, unsigned long len, unsigned int prot, int done, void **data) { - struct proc_mm_op protect; int ret; + unsigned long args[] = { addr, len, prot, 0, 0, 0 }; - if (proc_mm) { - int fd = mm_idp->u.mm_fd; - - protect = ((struct proc_mm_op) { .op = MM_MPROTECT, - .u = - { .mprotect = - { .addr = - (unsigned long) addr, - .len = len, - .prot = prot } } } ); - - CATCH_EINTR(ret = write(fd, &protect, sizeof(protect))); - if (ret != sizeof(protect)) { - ret = -errno; - printk(UM_KERN_ERR "protect failed, err = %d", -ret); - } - else ret = 0; - } - else { - unsigned long args[] = { addr, len, prot, 0, 0, 0 }; - - ret = run_syscall_stub(mm_idp, __NR_mprotect, args, 0, - data, done); - } + ret = run_syscall_stub(mm_idp, __NR_mprotect, args, 0, + data, done); return ret; } diff --git a/arch/um/os-Linux/skas/process.c b/arch/um/os-Linux/skas/process.c index 908579f2b0ab..7a9777570a62 100644 --- a/arch/um/os-Linux/skas/process.c +++ b/arch/um/os-Linux/skas/process.c @@ -16,11 +16,9 @@ #include <kern_util.h> #include <mem.h> #include <os.h> -#include <proc_mm.h> #include <ptrace_user.h> #include <registers.h> #include <skas.h> -#include <skas_ptrace.h> #include <sysdep/stub.h> int is_skas_winch(int pid, int fd, void *data) @@ -91,50 +89,33 @@ extern unsigned long current_stub_stack(void); static void get_skas_faultinfo(int pid, struct faultinfo *fi) { int err; + unsigned long fpregs[FP_SIZE]; - if (ptrace_faultinfo) { - err = ptrace(PTRACE_FAULTINFO, pid, 0, fi); - if (err) { - printk(UM_KERN_ERR "get_skas_faultinfo - " - "PTRACE_FAULTINFO failed, errno = %d\n", errno); - fatal_sigsegv(); - } - - /* Special handling for i386, which has different structs */ - if (sizeof(struct ptrace_faultinfo) < sizeof(struct faultinfo)) - memset((char *)fi + sizeof(struct ptrace_faultinfo), 0, - sizeof(struct faultinfo) - - sizeof(struct ptrace_faultinfo)); + err = get_fp_registers(pid, fpregs); + if (err < 0) { + printk(UM_KERN_ERR "save_fp_registers returned %d\n", + err); + fatal_sigsegv(); } - else { - unsigned long fpregs[FP_SIZE]; - - err = get_fp_registers(pid, fpregs); - if (err < 0) { - printk(UM_KERN_ERR "save_fp_registers returned %d\n", - err); - fatal_sigsegv(); - } - err = ptrace(PTRACE_CONT, pid, 0, SIGSEGV); - if (err) { - printk(UM_KERN_ERR "Failed to continue stub, pid = %d, " - "errno = %d\n", pid, errno); - fatal_sigsegv(); - } - wait_stub_done(pid); + err = ptrace(PTRACE_CONT, pid, 0, SIGSEGV); + if (err) { + printk(UM_KERN_ERR "Failed to continue stub, pid = %d, " + "errno = %d\n", pid, errno); + fatal_sigsegv(); + } + wait_stub_done(pid); - /* - * faultinfo is prepared by the stub-segv-handler at start of - * the stub stack page. We just have to copy it. - */ - memcpy(fi, (void *)current_stub_stack(), sizeof(*fi)); + /* + * faultinfo is prepared by the stub-segv-handler at start of + * the stub stack page. We just have to copy it. + */ + memcpy(fi, (void *)current_stub_stack(), sizeof(*fi)); - err = put_fp_registers(pid, fpregs); - if (err < 0) { - printk(UM_KERN_ERR "put_fp_registers returned %d\n", - err); - fatal_sigsegv(); - } + err = put_fp_registers(pid, fpregs); + if (err < 0) { + printk(UM_KERN_ERR "put_fp_registers returned %d\n", + err); + fatal_sigsegv(); } } @@ -198,7 +179,8 @@ extern int __syscall_stub_start; static int userspace_tramp(void *stack) { void *addr; - int err; + int err, fd; + unsigned long long offset; ptrace(PTRACE_TRACEME, 0, 0, 0); @@ -211,36 +193,32 @@ static int userspace_tramp(void *stack) exit(1); } - if (!proc_mm) { - /* - * This has a pte, but it can't be mapped in with the usual - * tlb_flush mechanism because this is part of that mechanism - */ - int fd; - unsigned long long offset; - fd = phys_mapping(to_phys(&__syscall_stub_start), &offset); - addr = mmap64((void *) STUB_CODE, UM_KERN_PAGE_SIZE, - PROT_EXEC, MAP_FIXED | MAP_PRIVATE, fd, offset); + /* + * This has a pte, but it can't be mapped in with the usual + * tlb_flush mechanism because this is part of that mechanism + */ + fd = phys_mapping(to_phys(&__syscall_stub_start), &offset); + addr = mmap64((void *) STUB_CODE, UM_KERN_PAGE_SIZE, + PROT_EXEC, MAP_FIXED | MAP_PRIVATE, fd, offset); + if (addr == MAP_FAILED) { + printk(UM_KERN_ERR "mapping mmap stub at 0x%lx failed, " + "errno = %d\n", STUB_CODE, errno); + exit(1); + } + + if (stack != NULL) { + fd = phys_mapping(to_phys(stack), &offset); + addr = mmap((void *) STUB_DATA, + UM_KERN_PAGE_SIZE, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, fd, offset); if (addr == MAP_FAILED) { - printk(UM_KERN_ERR "mapping mmap stub at 0x%lx failed, " - "errno = %d\n", STUB_CODE, errno); + printk(UM_KERN_ERR "mapping segfault stack " + "at 0x%lx failed, errno = %d\n", + STUB_DATA, errno); exit(1); } - - if (stack != NULL) { - fd = phys_mapping(to_phys(stack), &offset); - addr = mmap((void *) STUB_DATA, - UM_KERN_PAGE_SIZE, PROT_READ | PROT_WRITE, - MAP_FIXED | MAP_SHARED, fd, offset); - if (addr == MAP_FAILED) { - printk(UM_KERN_ERR "mapping segfault stack " - "at 0x%lx failed, errno = %d\n", - STUB_DATA, errno); - exit(1); - } - } } - if (!ptrace_faultinfo && (stack != NULL)) { + if (stack != NULL) { struct sigaction sa; unsigned long v = STUB_CODE + @@ -286,11 +264,7 @@ int start_userspace(unsigned long stub_stack) sp = (unsigned long) stack + UM_KERN_PAGE_SIZE - sizeof(void *); - flags = CLONE_FILES; - if (proc_mm) - flags |= CLONE_VM; - else - flags |= SIGCHLD; + flags = CLONE_FILES | SIGCHLD; pid = clone(userspace_tramp, (void *) sp, flags, (void *) stub_stack); if (pid < 0) { @@ -413,8 +387,7 @@ void userspace(struct uml_pt_regs *regs) switch (sig) { case SIGSEGV: - if (PTRACE_FULL_FAULTINFO || - !ptrace_faultinfo) { + if (PTRACE_FULL_FAULTINFO) { get_skas_faultinfo(pid, ®s->faultinfo); (*sig_info[SIGSEGV])(SIGSEGV, (struct siginfo *)&si, @@ -571,67 +544,6 @@ int copy_context_skas0(unsigned long new_stack, int pid) return err; } -/* - * This is used only, if stub pages are needed, while proc_mm is - * available. Opening /proc/mm creates a new mm_context, which lacks - * the stub-pages. Thus, we map them using /proc/mm-fd - */ -int map_stub_pages(int fd, unsigned long code, unsigned long data, - unsigned long stack) -{ - struct proc_mm_op mmop; - int n; - unsigned long long code_offset; - int code_fd = phys_mapping(to_phys((void *) &__syscall_stub_start), - &code_offset); - - mmop = ((struct proc_mm_op) { .op = MM_MMAP, - .u = - { .mmap = - { .addr = code, - .len = UM_KERN_PAGE_SIZE, - .prot = PROT_EXEC, - .flags = MAP_FIXED | MAP_PRIVATE, - .fd = code_fd, - .offset = code_offset - } } }); - CATCH_EINTR(n = write(fd, &mmop, sizeof(mmop))); - if (n != sizeof(mmop)) { - n = errno; - printk(UM_KERN_ERR "mmap args - addr = 0x%lx, fd = %d, " - "offset = %llx\n", code, code_fd, - (unsigned long long) code_offset); - printk(UM_KERN_ERR "map_stub_pages : /proc/mm map for code " - "failed, err = %d\n", n); - return -n; - } - - if (stack) { - unsigned long long map_offset; - int map_fd = phys_mapping(to_phys((void *)stack), &map_offset); - mmop = ((struct proc_mm_op) - { .op = MM_MMAP, - .u = - { .mmap = - { .addr = data, - .len = UM_KERN_PAGE_SIZE, - .prot = PROT_READ | PROT_WRITE, - .flags = MAP_FIXED | MAP_SHARED, - .fd = map_fd, - .offset = map_offset - } } }); - CATCH_EINTR(n = write(fd, &mmop, sizeof(mmop))); - if (n != sizeof(mmop)) { - n = errno; - printk(UM_KERN_ERR "map_stub_pages : /proc/mm map for " - "data failed, err = %d\n", n); - return -n; - } - } - - return 0; -} - void new_thread(void *stack, jmp_buf *buf, void (*handler)(void)) { (*buf)[0].JB_IP = (unsigned long) handler; @@ -674,7 +586,7 @@ int start_idle_thread(void *stack, jmp_buf *switch_buf) n = setjmp(initial_jmpbuf); switch (n) { case INIT_JMP_NEW_THREAD: - (*switch_buf)[0].JB_IP = (unsigned long) new_thread_handler; + (*switch_buf)[0].JB_IP = (unsigned long) uml_finishsetup; (*switch_buf)[0].JB_SP = (unsigned long) stack + UM_THREAD_SIZE - sizeof(void *); break; @@ -728,17 +640,5 @@ void reboot_skas(void) void __switch_mm(struct mm_id *mm_idp) { - int err; - - /* FIXME: need cpu pid in __switch_mm */ - if (proc_mm) { - err = ptrace(PTRACE_SWITCH_MM, userspace_pid[0], 0, - mm_idp->u.mm_fd); - if (err) { - printk(UM_KERN_ERR "__switch_mm - PTRACE_SWITCH_MM " - "failed, errno = %d\n", errno); - fatal_sigsegv(); - } - } - else userspace_pid[0] = mm_idp->u.pid; + userspace_pid[0] = mm_idp->u.pid; } diff --git a/arch/um/os-Linux/start_up.c b/arch/um/os-Linux/start_up.c index 337518c5042a..47f1ff056a54 100644 --- a/arch/um/os-Linux/start_up.c +++ b/arch/um/os-Linux/start_up.c @@ -24,7 +24,6 @@ #include <ptrace_user.h> #include <registers.h> #include <skas.h> -#include <skas_ptrace.h> static void ptrace_child(void) { @@ -143,44 +142,6 @@ static int stop_ptraced_child(int pid, int exitcode, int mustexit) } /* Changed only during early boot */ -int ptrace_faultinfo; -static int disable_ptrace_faultinfo; - -int ptrace_ldt; -static int disable_ptrace_ldt; - -int proc_mm; -static int disable_proc_mm; - -int have_switch_mm; -static int disable_switch_mm; - -int skas_needs_stub; - -static int __init skas0_cmd_param(char *str, int* add) -{ - disable_ptrace_faultinfo = 1; - disable_ptrace_ldt = 1; - disable_proc_mm = 1; - disable_switch_mm = 1; - - return 0; -} - -/* The two __uml_setup would conflict, without this stupid alias. */ - -static int __init mode_skas0_cmd_param(char *str, int* add) - __attribute__((alias("skas0_cmd_param"))); - -__uml_setup("skas0", skas0_cmd_param, -"skas0\n" -" Disables SKAS3 and SKAS4 usage, so that SKAS0 is used\n\n"); - -__uml_setup("mode=skas0", mode_skas0_cmd_param, -"mode=skas0\n" -" Disables SKAS3 and SKAS4 usage, so that SKAS0 is used.\n\n"); - -/* Changed only during early boot */ static int force_sysemu_disabled = 0; static int __init nosysemu_cmd_param(char *str, int* add) @@ -376,121 +337,6 @@ void __init os_early_checks(void) stop_ptraced_child(pid, 1, 1); } -static int __init noprocmm_cmd_param(char *str, int* add) -{ - disable_proc_mm = 1; - return 0; -} - -__uml_setup("noprocmm", noprocmm_cmd_param, -"noprocmm\n" -" Turns off usage of /proc/mm, even if host supports it.\n" -" To support /proc/mm, the host needs to be patched using\n" -" the current skas3 patch.\n\n"); - -static int __init noptracefaultinfo_cmd_param(char *str, int* add) -{ - disable_ptrace_faultinfo = 1; - return 0; -} - -__uml_setup("noptracefaultinfo", noptracefaultinfo_cmd_param, -"noptracefaultinfo\n" -" Turns off usage of PTRACE_FAULTINFO, even if host supports\n" -" it. To support PTRACE_FAULTINFO, the host needs to be patched\n" -" using the current skas3 patch.\n\n"); - -static int __init noptraceldt_cmd_param(char *str, int* add) -{ - disable_ptrace_ldt = 1; - return 0; -} - -__uml_setup("noptraceldt", noptraceldt_cmd_param, -"noptraceldt\n" -" Turns off usage of PTRACE_LDT, even if host supports it.\n" -" To support PTRACE_LDT, the host needs to be patched using\n" -" the current skas3 patch.\n\n"); - -static inline void check_skas3_ptrace_faultinfo(void) -{ - struct ptrace_faultinfo fi; - int pid, n; - - non_fatal(" - PTRACE_FAULTINFO..."); - pid = start_ptraced_child(); - - n = ptrace(PTRACE_FAULTINFO, pid, 0, &fi); - if (n < 0) { - if (errno == EIO) - non_fatal("not found\n"); - else - perror("not found"); - } else if (disable_ptrace_faultinfo) - non_fatal("found but disabled on command line\n"); - else { - ptrace_faultinfo = 1; - non_fatal("found\n"); - } - - stop_ptraced_child(pid, 1, 1); -} - -static inline void check_skas3_ptrace_ldt(void) -{ -#ifdef PTRACE_LDT - int pid, n; - unsigned char ldtbuf[40]; - struct ptrace_ldt ldt_op = (struct ptrace_ldt) { - .func = 2, /* read default ldt */ - .ptr = ldtbuf, - .bytecount = sizeof(ldtbuf)}; - - non_fatal(" - PTRACE_LDT..."); - pid = start_ptraced_child(); - - n = ptrace(PTRACE_LDT, pid, 0, (unsigned long) &ldt_op); - if (n < 0) { - if (errno == EIO) - non_fatal("not found\n"); - else - perror("not found"); - } else if (disable_ptrace_ldt) - non_fatal("found, but use is disabled\n"); - else { - ptrace_ldt = 1; - non_fatal("found\n"); - } - - stop_ptraced_child(pid, 1, 1); -#endif -} - -static inline void check_skas3_proc_mm(void) -{ - non_fatal(" - /proc/mm..."); - if (access("/proc/mm", W_OK) < 0) - perror("not found"); - else if (disable_proc_mm) - non_fatal("found but disabled on command line\n"); - else { - proc_mm = 1; - non_fatal("found\n"); - } -} - -void can_do_skas(void) -{ - non_fatal("Checking for the skas3 patch in the host:\n"); - - check_skas3_proc_mm(); - check_skas3_ptrace_faultinfo(); - check_skas3_ptrace_ldt(); - - if (!proc_mm || !ptrace_faultinfo || !ptrace_ldt) - skas_needs_stub = 1; -} - int __init parse_iomem(char *str, int *add) { struct iomem_region *new; diff --git a/arch/um/sys-ia64/Makefile b/arch/um/sys-ia64/Makefile deleted file mode 100644 index d02f4c265232..000000000000 --- a/arch/um/sys-ia64/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -OBJ = built-in.o - -OBJS = - -all: $(OBJ) - -$(OBJ): $(OBJS) - rm -f $@ - $(LD) $(LINKFLAGS) --start-group $^ --end-group -o $@ - -clean-files := $(OBJS) link.ld diff --git a/arch/um/sys-ia64/sysdep/ptrace.h b/arch/um/sys-ia64/sysdep/ptrace.h deleted file mode 100644 index 0f0f4e6fd334..000000000000 --- a/arch/um/sys-ia64/sysdep/ptrace.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (C) 2000 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SYSDEP_IA64_PTRACE_H -#define __SYSDEP_IA64_PTRACE_H - -struct sys_pt_regs { - int foo; -}; - -#define EMPTY_REGS { 0 } - -#endif - diff --git a/arch/um/sys-ia64/sysdep/sigcontext.h b/arch/um/sys-ia64/sysdep/sigcontext.h deleted file mode 100644 index 76b43161e779..000000000000 --- a/arch/um/sys-ia64/sysdep/sigcontext.h +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright (C) 2000 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SYSDEP_IA64_SIGCONTEXT_H -#define __SYSDEP_IA64_SIGCONTEXT_H - -#endif - diff --git a/arch/um/sys-ia64/sysdep/skas_ptrace.h b/arch/um/sys-ia64/sysdep/skas_ptrace.h deleted file mode 100644 index 25a38e715702..000000000000 --- a/arch/um/sys-ia64/sysdep/skas_ptrace.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SYSDEP_IA64_SKAS_PTRACE_H -#define __SYSDEP_IA64_SKAS_PTRACE_H - -struct ptrace_faultinfo { - int is_write; - unsigned long addr; -}; - -struct ptrace_ldt { - int func; - void *ptr; - unsigned long bytecount; -}; - -#define PTRACE_LDT 54 - -#endif diff --git a/arch/um/sys-ia64/sysdep/syscalls.h b/arch/um/sys-ia64/sysdep/syscalls.h deleted file mode 100644 index 5f6700c41558..000000000000 --- a/arch/um/sys-ia64/sysdep/syscalls.h +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright (C) 2000 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SYSDEP_IA64_SYSCALLS_H -#define __SYSDEP_IA64_SYSCALLS_H - -#endif - diff --git a/arch/um/sys-ppc/Makefile b/arch/um/sys-ppc/Makefile deleted file mode 100644 index 20d363bd7004..000000000000 --- a/arch/um/sys-ppc/Makefile +++ /dev/null @@ -1,65 +0,0 @@ -OBJ = built-in.o - -.S.o: - $(CC) $(KBUILD_AFLAGS) -D__ASSEMBLY__ -D__UM_PPC__ -c $< -o $*.o - -OBJS = ptrace.o sigcontext.o checksum.o miscthings.o misc.o \ - ptrace_user.o sysrq.o - -asflags-y := -DCONFIG_PPC32 -I. -I$(srctree)/arch/ppc/kernel - -all: $(OBJ) - -$(OBJ): $(OBJS) - rm -f $@ - $(LD) $(LINKFLAGS) --start-group $^ --end-group -o $@ - -ptrace_user.o: ptrace_user.c - $(CC) -D__KERNEL__ $(USER_CFLAGS) $(ccflags-y) -c -o $@ $< - -sigcontext.o: sigcontext.c - $(CC) $(USER_CFLAGS) $(ccflags-y) -c -o $@ $< - -checksum.S: - rm -f $@ - ln -s $(srctree)/arch/ppc/lib/$@ $@ - -mk_defs.c: - rm -f $@ - ln -s $(srctree)/arch/ppc/kernel/$@ $@ - -ppc_defs.head: - rm -f $@ - ln -s $(srctree)/arch/ppc/kernel/$@ $@ - -ppc_defs.h: mk_defs.c ppc_defs.head \ - $(srctree)/include/asm-ppc/mmu.h \ - $(srctree)/include/asm-ppc/processor.h \ - $(srctree)/include/asm-ppc/pgtable.h \ - $(srctree)/include/asm-ppc/ptrace.h -# $(CC) $(CFLAGS) -S mk_defs.c - cp ppc_defs.head ppc_defs.h -# for bk, this way we can write to the file even if it's not checked out - echo '#define THREAD 608' >> ppc_defs.h - echo '#define PT_REGS 8' >> ppc_defs.h - echo '#define CLONE_VM 256' >> ppc_defs.h -# chmod u+w ppc_defs.h -# grep '^#define' mk_defs.s >> ppc_defs.h -# rm mk_defs.s - -# the asm link is horrible, and breaks the other targets. This is also -# not going to work with parallel makes. - -checksum.o: checksum.S - rm -f asm - ln -s $(srctree)/include/asm-ppc asm - $(CC) $(asflags-y) $(KBUILD_AFLAGS) -D__ASSEMBLY__ -D__UM_PPC__ -c $< -o $*.o - rm -f asm - -misc.o: misc.S ppc_defs.h - rm -f asm - ln -s $(srctree)/include/asm-ppc asm - $(CC) $(asflags-y) $(KBUILD_AFLAGS) -D__ASSEMBLY__ -D__UM_PPC__ -c $< -o $*.o - rm -f asm - -clean-files := $(OBJS) ppc_defs.h checksum.S mk_defs.c diff --git a/arch/um/sys-ppc/asm/archparam.h b/arch/um/sys-ppc/asm/archparam.h deleted file mode 100644 index 4269d8a37b4f..000000000000 --- a/arch/um/sys-ppc/asm/archparam.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef __UM_ARCHPARAM_PPC_H -#define __UM_ARCHPARAM_PPC_H - -/********* Bits for asm-um/string.h **********/ - -#define __HAVE_ARCH_STRRCHR - -#endif diff --git a/arch/um/sys-ppc/asm/elf.h b/arch/um/sys-ppc/asm/elf.h deleted file mode 100644 index 8aacaf56508d..000000000000 --- a/arch/um/sys-ppc/asm/elf.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef __UM_ELF_PPC_H -#define __UM_ELF_PPC_H - - -extern long elf_aux_hwcap; -#define ELF_HWCAP (elf_aux_hwcap) - -#define SET_PERSONALITY(ex) do ; while(0) - -#define ELF_EXEC_PAGESIZE 4096 - -#define elf_check_arch(x) (1) - -#ifdef CONFIG_64BIT -#define ELF_CLASS ELFCLASS64 -#else -#define ELF_CLASS ELFCLASS32 -#endif - -#define R_386_NONE 0 -#define R_386_32 1 -#define R_386_PC32 2 -#define R_386_GOT32 3 -#define R_386_PLT32 4 -#define R_386_COPY 5 -#define R_386_GLOB_DAT 6 -#define R_386_JMP_SLOT 7 -#define R_386_RELATIVE 8 -#define R_386_GOTOFF 9 -#define R_386_GOTPC 10 -#define R_386_NUM 11 - -#define ELF_PLATFORM (0) - -#define ELF_ET_DYN_BASE (0x08000000) - -/* the following stolen from asm-ppc/elf.h */ -#define ELF_NGREG 48 /* includes nip, msr, lr, etc. */ -#define ELF_NFPREG 33 /* includes fpscr */ -/* General registers */ -typedef unsigned long elf_greg_t; -typedef elf_greg_t elf_gregset_t[ELF_NGREG]; - -/* Floating point registers */ -typedef double elf_fpreg_t; -typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; - -#define ELF_DATA ELFDATA2MSB -#define ELF_ARCH EM_PPC - -#endif diff --git a/arch/um/sys-ppc/asm/processor.h b/arch/um/sys-ppc/asm/processor.h deleted file mode 100644 index 959323151229..000000000000 --- a/arch/um/sys-ppc/asm/processor.h +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef __UM_PROCESSOR_PPC_H -#define __UM_PROCESSOR_PPC_H - -#if defined(__ASSEMBLY__) - -#define CONFIG_PPC_MULTIPLATFORM -#include "arch/processor.h" - -#else - -#include "asm/processor-generic.h" - -#endif - -#endif diff --git a/arch/um/sys-ppc/misc.S b/arch/um/sys-ppc/misc.S deleted file mode 100644 index 1364b7da578c..000000000000 --- a/arch/um/sys-ppc/misc.S +++ /dev/null @@ -1,111 +0,0 @@ -/* - * This file contains miscellaneous low-level functions. - * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) - * - * Largely rewritten by Cort Dougan (cort@cs.nmt.edu) - * and Paul Mackerras. - * - * A couple of functions stolen from arch/ppc/kernel/misc.S for UML - * by Chris Emerson. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * - */ - -#include <asm/processor.h> -#include "ppc_asm.h" - -#if defined(CONFIG_4xx) || defined(CONFIG_8xx) -#define CACHE_LINE_SIZE 16 -#define LG_CACHE_LINE_SIZE 4 -#define MAX_COPY_PREFETCH 1 -#else -#define CACHE_LINE_SIZE 32 -#define LG_CACHE_LINE_SIZE 5 -#define MAX_COPY_PREFETCH 4 -#endif /* CONFIG_4xx || CONFIG_8xx */ - - .text - -/* - * Clear a page using the dcbz instruction, which doesn't cause any - * memory traffic (except to write out any cache lines which get - * displaced). This only works on cacheable memory. - */ -_GLOBAL(clear_page) - li r0,4096/CACHE_LINE_SIZE - mtctr r0 -#ifdef CONFIG_8xx - li r4, 0 -1: stw r4, 0(r3) - stw r4, 4(r3) - stw r4, 8(r3) - stw r4, 12(r3) -#else -1: dcbz 0,r3 -#endif - addi r3,r3,CACHE_LINE_SIZE - bdnz 1b - blr - -/* - * Copy a whole page. We use the dcbz instruction on the destination - * to reduce memory traffic (it eliminates the unnecessary reads of - * the destination into cache). This requires that the destination - * is cacheable. - */ -#define COPY_16_BYTES \ - lwz r6,4(r4); \ - lwz r7,8(r4); \ - lwz r8,12(r4); \ - lwzu r9,16(r4); \ - stw r6,4(r3); \ - stw r7,8(r3); \ - stw r8,12(r3); \ - stwu r9,16(r3) - -_GLOBAL(copy_page) - addi r3,r3,-4 - addi r4,r4,-4 - li r5,4 - -#ifndef CONFIG_8xx -#if MAX_COPY_PREFETCH > 1 - li r0,MAX_COPY_PREFETCH - li r11,4 - mtctr r0 -11: dcbt r11,r4 - addi r11,r11,CACHE_LINE_SIZE - bdnz 11b -#else /* MAX_COPY_PREFETCH == 1 */ - dcbt r5,r4 - li r11,CACHE_LINE_SIZE+4 -#endif /* MAX_COPY_PREFETCH */ -#endif /* CONFIG_8xx */ - - li r0,4096/CACHE_LINE_SIZE - mtctr r0 -1: -#ifndef CONFIG_8xx - dcbt r11,r4 - dcbz r5,r3 -#endif - COPY_16_BYTES -#if CACHE_LINE_SIZE >= 32 - COPY_16_BYTES -#if CACHE_LINE_SIZE >= 64 - COPY_16_BYTES - COPY_16_BYTES -#if CACHE_LINE_SIZE >= 128 - COPY_16_BYTES - COPY_16_BYTES - COPY_16_BYTES - COPY_16_BYTES -#endif -#endif -#endif - bdnz 1b - blr diff --git a/arch/um/sys-ppc/miscthings.c b/arch/um/sys-ppc/miscthings.c deleted file mode 100644 index 25908d26ce07..000000000000 --- a/arch/um/sys-ppc/miscthings.c +++ /dev/null @@ -1,42 +0,0 @@ -#include <linux/threads.h> -#include <linux/stddef.h> // for NULL -#include <linux/elf.h> // for AT_NULL - -/* The following function nicked from arch/ppc/kernel/process.c and - * adapted slightly */ -/* - * XXX ld.so expects the auxiliary table to start on - * a 16-byte boundary, so we have to find it and - * move it up. :-( - */ -void shove_aux_table(unsigned long sp) -{ - int argc; - char *p; - unsigned long e; - unsigned long aux_start, offset; - - argc = *(int *)sp; - sp += sizeof(int) + (argc + 1) * sizeof(char *); - /* skip over the environment pointers */ - do { - p = *(char **)sp; - sp += sizeof(char *); - } while (p != NULL); - aux_start = sp; - /* skip to the end of the auxiliary table */ - do { - e = *(unsigned long *)sp; - sp += 2 * sizeof(unsigned long); - } while (e != AT_NULL); - offset = ((aux_start + 15) & ~15) - aux_start; - if (offset != 0) { - do { - sp -= sizeof(unsigned long); - e = *(unsigned long *)sp; - *(unsigned long *)(sp + offset) = e; - } while (sp > aux_start); - } -} -/* END stuff taken from arch/ppc/kernel/process.c */ - diff --git a/arch/um/sys-ppc/ptrace.c b/arch/um/sys-ppc/ptrace.c deleted file mode 100644 index 8245df41b201..000000000000 --- a/arch/um/sys-ppc/ptrace.c +++ /dev/null @@ -1,58 +0,0 @@ -#include <linux/sched.h> -#include "asm/ptrace.h" - -int putreg(struct task_struct *child, unsigned long regno, - unsigned long value) -{ - child->thread.process_regs.regs[regno >> 2] = value; - return 0; -} - -int poke_user(struct task_struct *child, long addr, long data) -{ - if ((addr & 3) || addr < 0) - return -EIO; - - if (addr < MAX_REG_OFFSET) - return putreg(child, addr, data); - - else if((addr >= offsetof(struct user, u_debugreg[0])) && - (addr <= offsetof(struct user, u_debugreg[7]))){ - addr -= offsetof(struct user, u_debugreg[0]); - addr = addr >> 2; - if((addr == 4) || (addr == 5)) return -EIO; - child->thread.arch.debugregs[addr] = data; - return 0; - } - return -EIO; -} - -unsigned long getreg(struct task_struct *child, unsigned long regno) -{ - unsigned long retval = ~0UL; - - retval &= child->thread.process_regs.regs[regno >> 2]; - return retval; -} - -int peek_user(struct task_struct *child, long addr, long data) -{ - /* read the word at location addr in the USER area. */ - unsigned long tmp; - - if ((addr & 3) || addr < 0) - return -EIO; - - tmp = 0; /* Default return condition */ - if(addr < MAX_REG_OFFSET){ - tmp = getreg(child, addr); - } - else if((addr >= offsetof(struct user, u_debugreg[0])) && - (addr <= offsetof(struct user, u_debugreg[7]))){ - addr -= offsetof(struct user, u_debugreg[0]); - addr = addr >> 2; - tmp = child->thread.arch.debugregs[addr]; - } - return put_user(tmp, (unsigned long *) data); -} - diff --git a/arch/um/sys-ppc/ptrace_user.c b/arch/um/sys-ppc/ptrace_user.c deleted file mode 100644 index 4601b9296aa7..000000000000 --- a/arch/um/sys-ppc/ptrace_user.c +++ /dev/null @@ -1,29 +0,0 @@ -#include <errno.h> -#include <asm/ptrace.h> -#include <sysdep/ptrace.h> - -int ptrace_getregs(long pid, unsigned long *regs_out) -{ - int i; - for (i=0; i < sizeof(struct sys_pt_regs)/sizeof(PPC_REG); ++i) { - errno = 0; - regs_out->regs[i] = ptrace(PTRACE_PEEKUSR, pid, i*4, 0); - if (errno) { - return -errno; - } - } - return 0; -} - -int ptrace_setregs(long pid, unsigned long *regs_in) -{ - int i; - for (i=0; i < sizeof(struct sys_pt_regs)/sizeof(PPC_REG); ++i) { - if (i != 34 /* FIXME: PT_ORIG_R3 */ && i <= PT_MQ) { - if (ptrace(PTRACE_POKEUSR, pid, i*4, regs_in->regs[i]) < 0) { - return -errno; - } - } - } - return 0; -} diff --git a/arch/um/sys-ppc/shared/sysdep/ptrace.h b/arch/um/sys-ppc/shared/sysdep/ptrace.h deleted file mode 100644 index efe0c1a3ea9c..000000000000 --- a/arch/um/sys-ppc/shared/sysdep/ptrace.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed under the GPL - */ - -#ifndef __SYS_PTRACE_PPC_H -#define __SYS_PTRACE_PPC_H - -#include <linux/types.h> - -/* the following taken from <asm-ppc/ptrace.h> */ - -#ifdef CONFIG_PPC64 -#define PPC_REG unsigned long /*long*/ -#else -#define PPC_REG unsigned long -#endif -struct sys_pt_regs_s { - PPC_REG gpr[32]; - PPC_REG nip; - PPC_REG msr; - PPC_REG orig_gpr3; /* Used for restarting system calls */ - PPC_REG ctr; - PPC_REG link; - PPC_REG xer; - PPC_REG ccr; - PPC_REG mq; /* 601 only (not used at present) */ - /* Used on APUS to hold IPL value. */ - PPC_REG trap; /* Reason for being here */ - PPC_REG dar; /* Fault registers */ - PPC_REG dsisr; - PPC_REG result; /* Result of a system call */ -}; - -#define NUM_REGS (sizeof(struct sys_pt_regs_s) / sizeof(PPC_REG)) - -struct sys_pt_regs { - PPC_REG regs[sizeof(struct sys_pt_regs_s) / sizeof(PPC_REG)]; -}; - -#define UM_MAX_REG (PT_FPR0) -#define UM_MAX_REG_OFFSET (UM_MAX_REG * sizeof(PPC_REG)) - -#define EMPTY_REGS { { [ 0 ... NUM_REGS - 1] = 0 } } - -#define UM_REG(r, n) ((r)->regs[n]) - -#define UM_SYSCALL_RET(r) UM_REG(r, PT_R3) -#define UM_SP(r) UM_REG(r, PT_R1) -#define UM_IP(r) UM_REG(r, PT_NIP) -#define UM_ELF_ZERO(r) UM_REG(r, PT_FPSCR) -#define UM_SYSCALL_NR(r) UM_REG(r, PT_R0) -#define UM_SYSCALL_ARG1(r) UM_REG(r, PT_ORIG_R3) -#define UM_SYSCALL_ARG2(r) UM_REG(r, PT_R4) -#define UM_SYSCALL_ARG3(r) UM_REG(r, PT_R5) -#define UM_SYSCALL_ARG4(r) UM_REG(r, PT_R6) -#define UM_SYSCALL_ARG5(r) UM_REG(r, PT_R7) -#define UM_SYSCALL_ARG6(r) UM_REG(r, PT_R8) - -#define UM_SYSCALL_NR_OFFSET (PT_R0 * sizeof(PPC_REG)) -#define UM_SYSCALL_RET_OFFSET (PT_R3 * sizeof(PPC_REG)) -#define UM_SYSCALL_ARG1_OFFSET (PT_R3 * sizeof(PPC_REG)) -#define UM_SYSCALL_ARG2_OFFSET (PT_R4 * sizeof(PPC_REG)) -#define UM_SYSCALL_ARG3_OFFSET (PT_R5 * sizeof(PPC_REG)) -#define UM_SYSCALL_ARG4_OFFSET (PT_R6 * sizeof(PPC_REG)) -#define UM_SYSCALL_ARG5_OFFSET (PT_R7 * sizeof(PPC_REG)) -#define UM_SYSCALL_ARG6_OFFSET (PT_R8 * sizeof(PPC_REG)) -#define UM_SP_OFFSET (PT_R1 * sizeof(PPC_REG)) -#define UM_IP_OFFSET (PT_NIP * sizeof(PPC_REG)) -#define UM_ELF_ZERO_OFFSET (PT_R3 * sizeof(PPC_REG)) - -#define UM_SET_SYSCALL_RETURN(_regs, result) \ -do { \ - if (result < 0) { \ - (_regs)->regs[PT_CCR] |= 0x10000000; \ - UM_SYSCALL_RET((_regs)) = -result; \ - } else { \ - UM_SYSCALL_RET((_regs)) = result; \ - } \ -} while(0) - -extern void shove_aux_table(unsigned long sp); -#define UM_FIX_EXEC_STACK(sp) shove_aux_table(sp); - -/* These aren't actually defined. The undefs are just to make sure - * everyone's clear on the concept. - */ -#undef UML_HAVE_GETREGS -#undef UML_HAVE_GETFPREGS -#undef UML_HAVE_SETREGS -#undef UML_HAVE_SETFPREGS - -#endif - diff --git a/arch/um/sys-ppc/shared/sysdep/sigcontext.h b/arch/um/sys-ppc/shared/sysdep/sigcontext.h deleted file mode 100644 index b7286f0a1e00..000000000000 --- a/arch/um/sys-ppc/shared/sysdep/sigcontext.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2000 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SYS_SIGCONTEXT_PPC_H -#define __SYS_SIGCONTEXT_PPC_H - -#define DSISR_WRITE 0x02000000 - -#define SC_FAULT_ADDR(sc) ({ \ - struct sigcontext *_sc = (sc); \ - long retval = -1; \ - switch (_sc->regs->trap) { \ - case 0x300: \ - /* data exception */ \ - retval = _sc->regs->dar; \ - break; \ - case 0x400: \ - /* instruction exception */ \ - retval = _sc->regs->nip; \ - break; \ - default: \ - panic("SC_FAULT_ADDR: unhandled trap type\n"); \ - } \ - retval; \ - }) - -#define SC_FAULT_WRITE(sc) ({ \ - struct sigcontext *_sc = (sc); \ - long retval = -1; \ - switch (_sc->regs->trap) { \ - case 0x300: \ - /* data exception */ \ - retval = !!(_sc->regs->dsisr & DSISR_WRITE); \ - break; \ - case 0x400: \ - /* instruction exception: not a write */ \ - retval = 0; \ - break; \ - default: \ - panic("SC_FAULT_ADDR: unhandled trap type\n"); \ - } \ - retval; \ - }) - -#define SC_IP(sc) ((sc)->regs->nip) -#define SC_SP(sc) ((sc)->regs->gpr[1]) -#define SEGV_IS_FIXABLE(sc) (1) - -#endif - diff --git a/arch/um/sys-ppc/shared/sysdep/skas_ptrace.h b/arch/um/sys-ppc/shared/sysdep/skas_ptrace.h deleted file mode 100644 index d9fbbac10de0..000000000000 --- a/arch/um/sys-ppc/shared/sysdep/skas_ptrace.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SYSDEP_PPC_SKAS_PTRACE_H -#define __SYSDEP_PPC_SKAS_PTRACE_H - -struct ptrace_faultinfo { - int is_write; - unsigned long addr; -}; - -struct ptrace_ldt { - int func; - void *ptr; - unsigned long bytecount; -}; - -#define PTRACE_LDT 54 - -#endif diff --git a/arch/um/sys-ppc/shared/sysdep/syscalls.h b/arch/um/sys-ppc/shared/sysdep/syscalls.h deleted file mode 100644 index 1ff81552251c..000000000000 --- a/arch/um/sys-ppc/shared/sysdep/syscalls.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2000 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -typedef long syscall_handler_t(unsigned long arg1, unsigned long arg2, - unsigned long arg3, unsigned long arg4, - unsigned long arg5, unsigned long arg6); - -#define EXECUTE_SYSCALL(syscall, regs) \ - (*sys_call_table[syscall])(UM_SYSCALL_ARG1(®s), \ - UM_SYSCALL_ARG2(®s), \ - UM_SYSCALL_ARG3(®s), \ - UM_SYSCALL_ARG4(®s), \ - UM_SYSCALL_ARG5(®s), \ - UM_SYSCALL_ARG6(®s)) - -extern syscall_handler_t sys_mincore; -extern syscall_handler_t sys_madvise; - -/* old_mmap needs the correct prototype since syscall_kern.c includes - * this file. - */ -int old_mmap(unsigned long addr, unsigned long len, - unsigned long prot, unsigned long flags, - unsigned long fd, unsigned long offset); - -#define ARCH_SYSCALLS \ - [ __NR_modify_ldt ] = sys_ni_syscall, \ - [ __NR_pciconfig_read ] = sys_ni_syscall, \ - [ __NR_pciconfig_write ] = sys_ni_syscall, \ - [ __NR_pciconfig_iobase ] = sys_ni_syscall, \ - [ __NR_pivot_root ] = sys_ni_syscall, \ - [ __NR_multiplexer ] = sys_ni_syscall, \ - [ __NR_mmap ] = old_mmap, \ - [ __NR_madvise ] = sys_madvise, \ - [ __NR_mincore ] = sys_mincore, \ - [ __NR_iopl ] = (syscall_handler_t *) sys_ni_syscall, \ - [ __NR_utimes ] = (syscall_handler_t *) sys_utimes, \ - [ __NR_fadvise64 ] = (syscall_handler_t *) sys_fadvise64, - -#define LAST_ARCH_SYSCALL __NR_fadvise64 - diff --git a/arch/um/sys-ppc/sigcontext.c b/arch/um/sys-ppc/sigcontext.c deleted file mode 100644 index aac6c83fe44e..000000000000 --- a/arch/um/sys-ppc/sigcontext.c +++ /dev/null @@ -1,4 +0,0 @@ -#include "asm/ptrace.h" -#include "asm/sigcontext.h" -#include <sysdep/ptrace.h> - diff --git a/arch/um/sys-ppc/sysrq.c b/arch/um/sys-ppc/sysrq.c deleted file mode 100644 index 1ff1ad7f27da..000000000000 --- a/arch/um/sys-ppc/sysrq.c +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2001 Chris Emerson (cemerson@chiark.greenend.org.uk) - * Licensed under the GPL - */ - -#include <linux/kernel.h> -#include <linux/smp.h> -#include "asm/ptrace.h" -#include "sysrq.h" - -void show_regs(struct pt_regs_subarch *regs) -{ - printk("\n"); - show_regs_print_info(KERN_DEFAULT); - - printk("show_regs(): insert regs here.\n"); -#if 0 - printk("\n"); - printk("EIP: %04x:[<%08lx>] CPU: %d",0xffff & regs->xcs, regs->eip, - smp_processor_id()); - if (regs->xcs & 3) - printk(" ESP: %04x:%08lx",0xffff & regs->xss, regs->esp); - printk(" EFLAGS: %08lx\n", regs->eflags); - printk("EAX: %08lx EBX: %08lx ECX: %08lx EDX: %08lx\n", - regs->eax, regs->ebx, regs->ecx, regs->edx); - printk("ESI: %08lx EDI: %08lx EBP: %08lx", - regs->esi, regs->edi, regs->ebp); - printk(" DS: %04x ES: %04x\n", - 0xffff & regs->xds, 0xffff & regs->xes); -#endif - - show_trace(current, ®s->gpr[1]); -} diff --git a/arch/unicore32/include/asm/thread_info.h b/arch/unicore32/include/asm/thread_info.h index 63e2839dfeb8..e79ad6d5b5b2 100644 --- a/arch/unicore32/include/asm/thread_info.h +++ b/arch/unicore32/include/asm/thread_info.h @@ -24,7 +24,6 @@ #ifndef __ASSEMBLY__ struct task_struct; -struct exec_domain; #include <asm/types.h> @@ -71,7 +70,6 @@ struct thread_info { /* <0 => bug */ mm_segment_t addr_limit; /* address limit */ struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ __u32 cpu; /* cpu */ struct cpu_context_save cpu_context; /* cpu context */ __u32 syscall; /* syscall number */ @@ -84,7 +82,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ .addr_limit = KERNEL_DS, \ diff --git a/arch/unicore32/kernel/asm-offsets.c b/arch/unicore32/kernel/asm-offsets.c index ffcbe7536ca7..80d50c4651e3 100644 --- a/arch/unicore32/kernel/asm-offsets.c +++ b/arch/unicore32/kernel/asm-offsets.c @@ -42,7 +42,6 @@ int main(void) DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); - DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); DEFINE(TI_USED_CP, offsetof(struct thread_info, used_cp)); diff --git a/arch/unicore32/kernel/signal.c b/arch/unicore32/kernel/signal.c index d329f85766cc..4ae51cf15ade 100644 --- a/arch/unicore32/kernel/signal.c +++ b/arch/unicore32/kernel/signal.c @@ -330,13 +330,6 @@ static void handle_signal(struct ksignal *ksig, struct pt_regs *regs, } /* - * translate the signal - */ - if (usig < 32 && thread->exec_domain - && thread->exec_domain->signal_invmap) - usig = thread->exec_domain->signal_invmap[usig]; - - /* * Set up the stack frame */ if (ksig->ka.sa.sa_flags & SA_SIGINFO) diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 5ba2d9ce82dc..2fda005bb334 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -63,7 +63,7 @@ ifeq ($(CONFIG_X86_32),y) $(call cc-option,-fno-unit-at-a-time)) # CPU-specific tuning. Anything which can be shared with UML should go here. - include $(srctree)/arch/x86/Makefile_32.cpu + include arch/x86/Makefile_32.cpu KBUILD_CFLAGS += $(cflags-y) # temporary until string.h is fixed diff --git a/arch/x86/Makefile.um b/arch/x86/Makefile.um index 95eba554baf9..5b7e898ffd9a 100644 --- a/arch/x86/Makefile.um +++ b/arch/x86/Makefile.um @@ -18,7 +18,7 @@ LDS_EXTRA := -Ui386 export LDS_EXTRA # First of all, tune CFLAGS for the specific CPU. This actually sets cflags-y. -include $(srctree)/arch/x86/Makefile_32.cpu +include arch/x86/Makefile_32.cpu # prevent gcc from keeping the stack 16 byte aligned. Taken from i386. cflags-y += $(call cc-option,-mpreferred-stack-boundary=2) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index 54f60ab41c63..112cefacf2af 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -797,7 +797,9 @@ static int rfc4106_init(struct crypto_tfm *tfm) PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN); struct crypto_aead *cryptd_child; struct aesni_rfc4106_gcm_ctx *child_ctx; - cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0); + cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); if (IS_ERR(cryptd_tfm)) return PTR_ERR(cryptd_tfm); @@ -890,15 +892,12 @@ out_free_ablkcipher: return ret; } -static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, - unsigned int key_len) +static int common_rfc4106_set_key(struct crypto_aead *aead, const u8 *key, + unsigned int key_len) { int ret = 0; - struct crypto_tfm *tfm = crypto_aead_tfm(parent); - struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); - struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); - struct aesni_rfc4106_gcm_ctx *child_ctx = - aesni_rfc4106_gcm_ctx_get(cryptd_child); + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(aead); u8 *new_key_align, *new_key_mem = NULL; if (key_len < 4) { @@ -943,20 +942,31 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, goto exit; } ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len); - memcpy(child_ctx, ctx, sizeof(*ctx)); exit: kfree(new_key_mem); return ret; } -/* This is the Integrity Check Value (aka the authentication tag length and can - * be 8, 12 or 16 bytes long. */ -static int rfc4106_set_authsize(struct crypto_aead *parent, - unsigned int authsize) +static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, + unsigned int key_len) { struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); - struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); + struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm); + struct aesni_rfc4106_gcm_ctx *c_ctx = aesni_rfc4106_gcm_ctx_get(child); + struct cryptd_aead *cryptd_tfm = ctx->cryptd_tfm; + int ret; + ret = crypto_aead_setkey(child, key, key_len); + if (!ret) { + memcpy(ctx, c_ctx, sizeof(*ctx)); + ctx->cryptd_tfm = cryptd_tfm; + } + return ret; +} + +static int common_rfc4106_set_authsize(struct crypto_aead *aead, + unsigned int authsize) +{ switch (authsize) { case 8: case 12: @@ -965,51 +975,23 @@ static int rfc4106_set_authsize(struct crypto_aead *parent, default: return -EINVAL; } - crypto_aead_crt(parent)->authsize = authsize; - crypto_aead_crt(cryptd_child)->authsize = authsize; + crypto_aead_crt(aead)->authsize = authsize; return 0; } -static int rfc4106_encrypt(struct aead_request *req) -{ - int ret; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); - - if (!irq_fpu_usable()) { - struct aead_request *cryptd_req = - (struct aead_request *) aead_request_ctx(req); - memcpy(cryptd_req, req, sizeof(*req)); - aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); - return crypto_aead_encrypt(cryptd_req); - } else { - struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); - kernel_fpu_begin(); - ret = cryptd_child->base.crt_aead.encrypt(req); - kernel_fpu_end(); - return ret; - } -} - -static int rfc4106_decrypt(struct aead_request *req) +/* This is the Integrity Check Value (aka the authentication tag length and can + * be 8, 12 or 16 bytes long. */ +static int rfc4106_set_authsize(struct crypto_aead *parent, + unsigned int authsize) { + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent); + struct crypto_aead *child = cryptd_aead_child(ctx->cryptd_tfm); int ret; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); - struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); - if (!irq_fpu_usable()) { - struct aead_request *cryptd_req = - (struct aead_request *) aead_request_ctx(req); - memcpy(cryptd_req, req, sizeof(*req)); - aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); - return crypto_aead_decrypt(cryptd_req); - } else { - struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); - kernel_fpu_begin(); - ret = cryptd_child->base.crt_aead.decrypt(req); - kernel_fpu_end(); - return ret; - } + ret = crypto_aead_setauthsize(child, authsize); + if (!ret) + crypto_aead_crt(parent)->authsize = authsize; + return ret; } static int __driver_rfc4106_encrypt(struct aead_request *req) @@ -1185,6 +1167,78 @@ static int __driver_rfc4106_decrypt(struct aead_request *req) } return retval; } + +static int rfc4106_encrypt(struct aead_request *req) +{ + int ret; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + + if (!irq_fpu_usable()) { + struct aead_request *cryptd_req = + (struct aead_request *) aead_request_ctx(req); + + memcpy(cryptd_req, req, sizeof(*req)); + aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + ret = crypto_aead_encrypt(cryptd_req); + } else { + kernel_fpu_begin(); + ret = __driver_rfc4106_encrypt(req); + kernel_fpu_end(); + } + return ret; +} + +static int rfc4106_decrypt(struct aead_request *req) +{ + int ret; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + + if (!irq_fpu_usable()) { + struct aead_request *cryptd_req = + (struct aead_request *) aead_request_ctx(req); + + memcpy(cryptd_req, req, sizeof(*req)); + aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + ret = crypto_aead_decrypt(cryptd_req); + } else { + kernel_fpu_begin(); + ret = __driver_rfc4106_decrypt(req); + kernel_fpu_end(); + } + return ret; +} + +static int helper_rfc4106_encrypt(struct aead_request *req) +{ + int ret; + + if (unlikely(!irq_fpu_usable())) { + WARN_ONCE(1, "__gcm-aes-aesni alg used in invalid context"); + ret = -EINVAL; + } else { + kernel_fpu_begin(); + ret = __driver_rfc4106_encrypt(req); + kernel_fpu_end(); + } + return ret; +} + +static int helper_rfc4106_decrypt(struct aead_request *req) +{ + int ret; + + if (unlikely(!irq_fpu_usable())) { + WARN_ONCE(1, "__gcm-aes-aesni alg used in invalid context"); + ret = -EINVAL; + } else { + kernel_fpu_begin(); + ret = __driver_rfc4106_decrypt(req); + kernel_fpu_end(); + } + return ret; +} #endif static struct crypto_alg aesni_algs[] = { { @@ -1210,7 +1264,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_name = "__aes-aesni", .cra_driver_name = "__driver-aes-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1, @@ -1229,7 +1283,8 @@ static struct crypto_alg aesni_algs[] = { { .cra_name = "__ecb-aes-aesni", .cra_driver_name = "__driver-ecb-aes-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1, @@ -1249,7 +1304,8 @@ static struct crypto_alg aesni_algs[] = { { .cra_name = "__cbc-aes-aesni", .cra_driver_name = "__driver-cbc-aes-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1, @@ -1313,7 +1369,8 @@ static struct crypto_alg aesni_algs[] = { { .cra_name = "__ctr-aes-aesni", .cra_driver_name = "__driver-ctr-aes-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1, @@ -1357,7 +1414,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_name = "__gcm-aes-aesni", .cra_driver_name = "__driver-gcm-aes-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_AEAD, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN, @@ -1366,8 +1423,12 @@ static struct crypto_alg aesni_algs[] = { { .cra_module = THIS_MODULE, .cra_u = { .aead = { - .encrypt = __driver_rfc4106_encrypt, - .decrypt = __driver_rfc4106_decrypt, + .setkey = common_rfc4106_set_key, + .setauthsize = common_rfc4106_set_authsize, + .encrypt = helper_rfc4106_encrypt, + .decrypt = helper_rfc4106_decrypt, + .ivsize = 8, + .maxauthsize = 16, }, }, }, { @@ -1423,7 +1484,8 @@ static struct crypto_alg aesni_algs[] = { { .cra_name = "__lrw-aes-aesni", .cra_driver_name = "__driver-lrw-aes-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aesni_lrw_ctx), .cra_alignmask = 0, @@ -1444,7 +1506,8 @@ static struct crypto_alg aesni_algs[] = { { .cra_name = "__xts-aes-aesni", .cra_driver_name = "__driver-xts-aes-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct aesni_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/camellia_aesni_avx2_glue.c b/arch/x86/crypto/camellia_aesni_avx2_glue.c index 9a07fafe3831..baf0ac21ace5 100644 --- a/arch/x86/crypto/camellia_aesni_avx2_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx2_glue.c @@ -343,7 +343,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__ecb-camellia-aesni-avx2", .cra_driver_name = "__driver-ecb-camellia-aesni-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_ctx), .cra_alignmask = 0, @@ -362,7 +363,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__cbc-camellia-aesni-avx2", .cra_driver_name = "__driver-cbc-camellia-aesni-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_ctx), .cra_alignmask = 0, @@ -381,7 +383,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__ctr-camellia-aesni-avx2", .cra_driver_name = "__driver-ctr-camellia-aesni-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct camellia_ctx), .cra_alignmask = 0, @@ -401,7 +404,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__lrw-camellia-aesni-avx2", .cra_driver_name = "__driver-lrw-camellia-aesni-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_lrw_ctx), .cra_alignmask = 0, @@ -424,7 +428,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__xts-camellia-aesni-avx2", .cra_driver_name = "__driver-xts-camellia-aesni-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/camellia_aesni_avx_glue.c b/arch/x86/crypto/camellia_aesni_avx_glue.c index ed38d959add6..78818a1e73e3 100644 --- a/arch/x86/crypto/camellia_aesni_avx_glue.c +++ b/arch/x86/crypto/camellia_aesni_avx_glue.c @@ -335,7 +335,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__ecb-camellia-aesni", .cra_driver_name = "__driver-ecb-camellia-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_ctx), .cra_alignmask = 0, @@ -354,7 +355,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__cbc-camellia-aesni", .cra_driver_name = "__driver-cbc-camellia-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_ctx), .cra_alignmask = 0, @@ -373,7 +375,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__ctr-camellia-aesni", .cra_driver_name = "__driver-ctr-camellia-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct camellia_ctx), .cra_alignmask = 0, @@ -393,7 +396,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__lrw-camellia-aesni", .cra_driver_name = "__driver-lrw-camellia-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_lrw_ctx), .cra_alignmask = 0, @@ -416,7 +420,8 @@ static struct crypto_alg cmll_algs[10] = { { .cra_name = "__xts-camellia-aesni", .cra_driver_name = "__driver-xts-camellia-aesni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAMELLIA_BLOCK_SIZE, .cra_ctxsize = sizeof(struct camellia_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c index 60ada677a928..236c80974457 100644 --- a/arch/x86/crypto/cast5_avx_glue.c +++ b/arch/x86/crypto/cast5_avx_glue.c @@ -341,7 +341,8 @@ static struct crypto_alg cast5_algs[6] = { { .cra_name = "__ecb-cast5-avx", .cra_driver_name = "__driver-ecb-cast5-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAST5_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast5_ctx), .cra_alignmask = 0, @@ -360,7 +361,8 @@ static struct crypto_alg cast5_algs[6] = { { .cra_name = "__cbc-cast5-avx", .cra_driver_name = "__driver-cbc-cast5-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAST5_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast5_ctx), .cra_alignmask = 0, @@ -379,7 +381,8 @@ static struct crypto_alg cast5_algs[6] = { { .cra_name = "__ctr-cast5-avx", .cra_driver_name = "__driver-ctr-cast5-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct cast5_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c index 0160f68a57ff..f448810ca4ac 100644 --- a/arch/x86/crypto/cast6_avx_glue.c +++ b/arch/x86/crypto/cast6_avx_glue.c @@ -372,7 +372,8 @@ static struct crypto_alg cast6_algs[10] = { { .cra_name = "__ecb-cast6-avx", .cra_driver_name = "__driver-ecb-cast6-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, @@ -391,7 +392,8 @@ static struct crypto_alg cast6_algs[10] = { { .cra_name = "__cbc-cast6-avx", .cra_driver_name = "__driver-cbc-cast6-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, @@ -410,7 +412,8 @@ static struct crypto_alg cast6_algs[10] = { { .cra_name = "__ctr-cast6-avx", .cra_driver_name = "__driver-ctr-cast6-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct cast6_ctx), .cra_alignmask = 0, @@ -430,7 +433,8 @@ static struct crypto_alg cast6_algs[10] = { { .cra_name = "__lrw-cast6-avx", .cra_driver_name = "__driver-lrw-cast6-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_lrw_ctx), .cra_alignmask = 0, @@ -453,7 +457,8 @@ static struct crypto_alg cast6_algs[10] = { { .cra_name = "__xts-cast6-avx", .cra_driver_name = "__driver-xts-cast6-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = CAST6_BLOCK_SIZE, .cra_ctxsize = sizeof(struct cast6_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c index 8253d85aa165..2079baf06bdd 100644 --- a/arch/x86/crypto/ghash-clmulni-intel_glue.c +++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c @@ -154,7 +154,8 @@ static struct shash_alg ghash_alg = { .cra_name = "__ghash", .cra_driver_name = "__ghash-pclmulqdqni", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_flags = CRYPTO_ALG_TYPE_SHASH | + CRYPTO_ALG_INTERNAL, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct ghash_ctx), .cra_module = THIS_MODULE, @@ -261,7 +262,9 @@ static int ghash_async_init_tfm(struct crypto_tfm *tfm) struct cryptd_ahash *cryptd_tfm; struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); - cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", 0, 0); + cryptd_tfm = cryptd_alloc_ahash("__ghash-pclmulqdqni", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); if (IS_ERR(cryptd_tfm)) return PTR_ERR(cryptd_tfm); ctx->cryptd_tfm = cryptd_tfm; diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c index 432f1d76ceb8..6a85598931b5 100644 --- a/arch/x86/crypto/glue_helper.c +++ b/arch/x86/crypto/glue_helper.c @@ -232,7 +232,6 @@ static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr, le128_to_be128((be128 *)walk->iv, &ctrblk); } -EXPORT_SYMBOL_GPL(glue_ctr_crypt_final_128bit); static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, struct blkcipher_desc *desc, diff --git a/arch/x86/crypto/serpent_avx2_glue.c b/arch/x86/crypto/serpent_avx2_glue.c index 437e47a4d302..2f63dc89e7a9 100644 --- a/arch/x86/crypto/serpent_avx2_glue.c +++ b/arch/x86/crypto/serpent_avx2_glue.c @@ -309,7 +309,8 @@ static struct crypto_alg srp_algs[10] = { { .cra_name = "__ecb-serpent-avx2", .cra_driver_name = "__driver-ecb-serpent-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -329,7 +330,8 @@ static struct crypto_alg srp_algs[10] = { { .cra_name = "__cbc-serpent-avx2", .cra_driver_name = "__driver-cbc-serpent-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -349,7 +351,8 @@ static struct crypto_alg srp_algs[10] = { { .cra_name = "__ctr-serpent-avx2", .cra_driver_name = "__driver-ctr-serpent-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -370,7 +373,8 @@ static struct crypto_alg srp_algs[10] = { { .cra_name = "__lrw-serpent-avx2", .cra_driver_name = "__driver-lrw-serpent-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_lrw_ctx), .cra_alignmask = 0, @@ -394,7 +398,8 @@ static struct crypto_alg srp_algs[10] = { { .cra_name = "__xts-serpent-avx2", .cra_driver_name = "__driver-xts-serpent-avx2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c index 7e217398b4eb..c8d478af8456 100644 --- a/arch/x86/crypto/serpent_avx_glue.c +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -378,7 +378,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__ecb-serpent-avx", .cra_driver_name = "__driver-ecb-serpent-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -397,7 +398,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__cbc-serpent-avx", .cra_driver_name = "__driver-cbc-serpent-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -416,7 +418,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__ctr-serpent-avx", .cra_driver_name = "__driver-ctr-serpent-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -436,7 +439,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__lrw-serpent-avx", .cra_driver_name = "__driver-lrw-serpent-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_lrw_ctx), .cra_alignmask = 0, @@ -459,7 +463,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__xts-serpent-avx", .cra_driver_name = "__driver-xts-serpent-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index bf025adaea01..3643dd508f45 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -387,7 +387,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__ecb-serpent-sse2", .cra_driver_name = "__driver-ecb-serpent-sse2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -406,7 +407,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__cbc-serpent-sse2", .cra_driver_name = "__driver-cbc-serpent-sse2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -425,7 +427,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__ctr-serpent-sse2", .cra_driver_name = "__driver-ctr-serpent-sse2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct serpent_ctx), .cra_alignmask = 0, @@ -445,7 +448,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__lrw-serpent-sse2", .cra_driver_name = "__driver-lrw-serpent-sse2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_lrw_ctx), .cra_alignmask = 0, @@ -468,7 +472,8 @@ static struct crypto_alg serpent_algs[10] = { { .cra_name = "__xts-serpent-sse2", .cra_driver_name = "__driver-xts-serpent-sse2", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/crypto/sha-mb/sha1_mb.c b/arch/x86/crypto/sha-mb/sha1_mb.c index fd9f6b035b16..e510b1c5d690 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb.c +++ b/arch/x86/crypto/sha-mb/sha1_mb.c @@ -694,7 +694,8 @@ static struct shash_alg sha1_mb_shash_alg = { * use ASYNC flag as some buffers in multi-buffer * algo may not have completed before hashing thread sleep */ - .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_ASYNC, + .cra_flags = CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_INTERNAL, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(sha1_mb_shash_alg.base.cra_list), @@ -770,7 +771,9 @@ static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm) struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); struct mcryptd_hash_ctx *mctx; - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", 0, 0); + mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", + CRYPTO_ALG_INTERNAL, + CRYPTO_ALG_INTERNAL); if (IS_ERR(mcryptd_tfm)) return PTR_ERR(mcryptd_tfm); mctx = crypto_ahash_ctx(&mcryptd_tfm->base); @@ -828,7 +831,7 @@ static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate) while (!list_empty(&cstate->work_list)) { rctx = list_entry(cstate->work_list.next, struct mcryptd_hash_request_ctx, waiter); - if time_before(cur_time, rctx->tag.expire) + if (time_before(cur_time, rctx->tag.expire)) break; kernel_fpu_begin(); sha_ctx = (struct sha1_hash_ctx *) sha1_ctx_mgr_flush(cstate->mgr); diff --git a/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c b/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c index 4ca7e166a2aa..822acb5b464c 100644 --- a/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c +++ b/arch/x86/crypto/sha-mb/sha1_mb_mgr_init_avx2.c @@ -56,7 +56,7 @@ void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state) { unsigned int j; - state->unused_lanes = 0xF76543210; + state->unused_lanes = 0xF76543210ULL; for (j = 0; j < 8; j++) { state->lens[j] = 0xFFFFFFFF; state->ldata[j].job_in_lane = NULL; diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index 6c20fe04a738..33d1b9dc14cc 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -28,7 +28,7 @@ #include <linux/cryptohash.h> #include <linux/types.h> #include <crypto/sha.h> -#include <asm/byteorder.h> +#include <crypto/sha1_base.h> #include <asm/i387.h> #include <asm/xcr.h> #include <asm/xsave.h> @@ -44,132 +44,51 @@ asmlinkage void sha1_transform_avx(u32 *digest, const char *data, #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */ asmlinkage void sha1_transform_avx2(u32 *digest, const char *data, - unsigned int rounds); + unsigned int rounds); #endif -static asmlinkage void (*sha1_transform_asm)(u32 *, const char *, unsigned int); - - -static int sha1_ssse3_init(struct shash_desc *desc) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - *sctx = (struct sha1_state){ - .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - }; - - return 0; -} - -static int __sha1_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len, unsigned int partial) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int done = 0; - - sctx->count += len; - - if (partial) { - done = SHA1_BLOCK_SIZE - partial; - memcpy(sctx->buffer + partial, data, done); - sha1_transform_asm(sctx->state, sctx->buffer, 1); - } - - if (len - done >= SHA1_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; - - sha1_transform_asm(sctx->state, data + done, rounds); - done += rounds * SHA1_BLOCK_SIZE; - } - - memcpy(sctx->buffer, data + done, len - done); - - return 0; -} +static void (*sha1_transform_asm)(u32 *, const char *, unsigned int); static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; - int res; - /* Handle the fast case right here */ - if (partial + len < SHA1_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buffer + partial, data, len); + if (!irq_fpu_usable() || + (sctx->count % SHA1_BLOCK_SIZE) + len < SHA1_BLOCK_SIZE) + return crypto_sha1_update(desc, data, len); - return 0; - } + /* make sure casting to sha1_block_fn() is safe */ + BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0); - if (!irq_fpu_usable()) { - res = crypto_sha1_update(desc, data, len); - } else { - kernel_fpu_begin(); - res = __sha1_ssse3_update(desc, data, len, partial); - kernel_fpu_end(); - } - - return res; -} - - -/* Add padding and return the message digest. */ -static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA1_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); - if (!irq_fpu_usable()) { - crypto_sha1_update(desc, padding, padlen); - crypto_sha1_update(desc, (const u8 *)&bits, sizeof(bits)); - } else { - kernel_fpu_begin(); - /* We need to fill a whole block for __sha1_ssse3_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buffer + index, padding, padlen); - } else { - __sha1_ssse3_update(desc, padding, padlen, index); - } - __sha1_ssse3_update(desc, (const u8 *)&bits, sizeof(bits), 56); - kernel_fpu_end(); - } - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); + kernel_fpu_begin(); + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_transform_asm); + kernel_fpu_end(); return 0; } -static int sha1_ssse3_export(struct shash_desc *desc, void *out) +static int sha1_ssse3_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); + if (!irq_fpu_usable()) + return crypto_sha1_finup(desc, data, len, out); - memcpy(out, sctx, sizeof(*sctx)); + kernel_fpu_begin(); + if (len) + sha1_base_do_update(desc, data, len, + (sha1_block_fn *)sha1_transform_asm); + sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_transform_asm); + kernel_fpu_end(); - return 0; + return sha1_base_finish(desc, out); } -static int sha1_ssse3_import(struct shash_desc *desc, const void *in) +/* Add padding and return the message digest. */ +static int sha1_ssse3_final(struct shash_desc *desc, u8 *out) { - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; + return sha1_ssse3_finup(desc, NULL, 0, out); } #ifdef CONFIG_AS_AVX2 @@ -186,13 +105,11 @@ static void sha1_apply_transform_avx2(u32 *digest, const char *data, static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_ssse3_init, + .init = sha1_base_init, .update = sha1_ssse3_update, .final = sha1_ssse3_final, - .export = sha1_ssse3_export, - .import = sha1_ssse3_import, + .finup = sha1_ssse3_finup, .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name= "sha1-ssse3", diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/crypto/sha256-avx-asm.S index 642f15687a0a..92b3b5d75ba9 100644 --- a/arch/x86/crypto/sha256-avx-asm.S +++ b/arch/x86/crypto/sha256-avx-asm.S @@ -96,10 +96,10 @@ SHUF_DC00 = %xmm12 # shuffle xDxC -> DC00 BYTE_FLIP_MASK = %xmm13 NUM_BLKS = %rdx # 3rd arg -CTX = %rsi # 2nd arg -INP = %rdi # 1st arg +INP = %rsi # 2nd arg +CTX = %rdi # 1st arg -SRND = %rdi # clobbers INP +SRND = %rsi # clobbers INP c = %ecx d = %r8d e = %edx @@ -342,8 +342,8 @@ a = TMP_ ######################################################################## ## void sha256_transform_avx(void *input_data, UINT32 digest[8], UINT64 num_blks) -## arg 1 : pointer to input data -## arg 2 : pointer to digest +## arg 1 : pointer to digest +## arg 2 : pointer to input data ## arg 3 : Num blocks ######################################################################## .text diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/crypto/sha256-avx2-asm.S index 9e86944c539d..570ec5ec62d7 100644 --- a/arch/x86/crypto/sha256-avx2-asm.S +++ b/arch/x86/crypto/sha256-avx2-asm.S @@ -91,12 +91,12 @@ BYTE_FLIP_MASK = %ymm13 X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK NUM_BLKS = %rdx # 3rd arg -CTX = %rsi # 2nd arg -INP = %rdi # 1st arg +INP = %rsi # 2nd arg +CTX = %rdi # 1st arg c = %ecx d = %r8d e = %edx # clobbers NUM_BLKS -y3 = %edi # clobbers INP +y3 = %esi # clobbers INP TBL = %rbp @@ -523,8 +523,8 @@ STACK_SIZE = _RSP + _RSP_SIZE ######################################################################## ## void sha256_transform_rorx(void *input_data, UINT32 digest[8], UINT64 num_blks) -## arg 1 : pointer to input data -## arg 2 : pointer to digest +## arg 1 : pointer to digest +## arg 2 : pointer to input data ## arg 3 : Num blocks ######################################################################## .text diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/crypto/sha256-ssse3-asm.S index f833b74d902b..2cedc44e8121 100644 --- a/arch/x86/crypto/sha256-ssse3-asm.S +++ b/arch/x86/crypto/sha256-ssse3-asm.S @@ -88,10 +88,10 @@ SHUF_DC00 = %xmm11 # shuffle xDxC -> DC00 BYTE_FLIP_MASK = %xmm12 NUM_BLKS = %rdx # 3rd arg -CTX = %rsi # 2nd arg -INP = %rdi # 1st arg +INP = %rsi # 2nd arg +CTX = %rdi # 1st arg -SRND = %rdi # clobbers INP +SRND = %rsi # clobbers INP c = %ecx d = %r8d e = %edx @@ -348,8 +348,8 @@ a = TMP_ ######################################################################## ## void sha256_transform_ssse3(void *input_data, UINT32 digest[8], UINT64 num_blks) -## arg 1 : pointer to input data -## arg 2 : pointer to digest +## arg 1 : pointer to digest +## arg 2 : pointer to input data ## arg 3 : Num blocks ######################################################################## .text diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c index 8fad72f4dfd2..ccc338881ee8 100644 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ b/arch/x86/crypto/sha256_ssse3_glue.c @@ -36,195 +36,74 @@ #include <linux/cryptohash.h> #include <linux/types.h> #include <crypto/sha.h> -#include <asm/byteorder.h> +#include <crypto/sha256_base.h> #include <asm/i387.h> #include <asm/xcr.h> #include <asm/xsave.h> #include <linux/string.h> -asmlinkage void sha256_transform_ssse3(const char *data, u32 *digest, - u64 rounds); +asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data, + u64 rounds); #ifdef CONFIG_AS_AVX -asmlinkage void sha256_transform_avx(const char *data, u32 *digest, +asmlinkage void sha256_transform_avx(u32 *digest, const char *data, u64 rounds); #endif #ifdef CONFIG_AS_AVX2 -asmlinkage void sha256_transform_rorx(const char *data, u32 *digest, - u64 rounds); +asmlinkage void sha256_transform_rorx(u32 *digest, const char *data, + u64 rounds); #endif -static asmlinkage void (*sha256_transform_asm)(const char *, u32 *, u64); - - -static int sha256_ssse3_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; -} - -static int __sha256_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len, unsigned int partial) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int done = 0; - - sctx->count += len; - - if (partial) { - done = SHA256_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha256_transform_asm(sctx->buf, sctx->state, 1); - } - - if (len - done >= SHA256_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA256_BLOCK_SIZE; - - sha256_transform_asm(data + done, sctx->state, (u64) rounds); - - done += rounds * SHA256_BLOCK_SIZE; - } - - memcpy(sctx->buf, data + done, len - done); - - return 0; -} +static void (*sha256_transform_asm)(u32 *, const char *, u64); static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - int res; - /* Handle the fast case right here */ - if (partial + len < SHA256_BLOCK_SIZE) { - sctx->count += len; - memcpy(sctx->buf + partial, data, len); + if (!irq_fpu_usable() || + (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) + return crypto_sha256_update(desc, data, len); - return 0; - } - - if (!irq_fpu_usable()) { - res = crypto_sha256_update(desc, data, len); - } else { - kernel_fpu_begin(); - res = __sha256_ssse3_update(desc, data, len, partial); - kernel_fpu_end(); - } - - return res; -} + /* make sure casting to sha256_block_fn() is safe */ + BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); - -/* Add padding and return the message digest. */ -static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be32 *dst = (__be32 *)out; - __be64 bits; - static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 and append length */ - index = sctx->count % SHA256_BLOCK_SIZE; - padlen = (index < 56) ? (56 - index) : ((SHA256_BLOCK_SIZE+56)-index); - - if (!irq_fpu_usable()) { - crypto_sha256_update(desc, padding, padlen); - crypto_sha256_update(desc, (const u8 *)&bits, sizeof(bits)); - } else { - kernel_fpu_begin(); - /* We need to fill a whole block for __sha256_ssse3_update() */ - if (padlen <= 56) { - sctx->count += padlen; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha256_ssse3_update(desc, padding, padlen, index); - } - __sha256_ssse3_update(desc, (const u8 *)&bits, - sizeof(bits), 56); - kernel_fpu_end(); - } - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); + kernel_fpu_begin(); + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha256_transform_asm); + kernel_fpu_end(); return 0; } -static int sha256_ssse3_export(struct shash_desc *desc, void *out) +static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); + if (!irq_fpu_usable()) + return crypto_sha256_finup(desc, data, len, out); - memcpy(out, sctx, sizeof(*sctx)); + kernel_fpu_begin(); + if (len) + sha256_base_do_update(desc, data, len, + (sha256_block_fn *)sha256_transform_asm); + sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_transform_asm); + kernel_fpu_end(); - return 0; + return sha256_base_finish(desc, out); } -static int sha256_ssse3_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; -} - -static int sha224_ssse3_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; -} - -static int sha224_ssse3_final(struct shash_desc *desc, u8 *hash) +/* Add padding and return the message digest. */ +static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) { - u8 D[SHA256_DIGEST_SIZE]; - - sha256_ssse3_final(desc, D); - - memcpy(hash, D, SHA224_DIGEST_SIZE); - memzero_explicit(D, SHA256_DIGEST_SIZE); - - return 0; + return sha256_ssse3_finup(desc, NULL, 0, out); } static struct shash_alg algs[] = { { .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_ssse3_init, + .init = sha256_base_init, .update = sha256_ssse3_update, .final = sha256_ssse3_final, - .export = sha256_ssse3_export, - .import = sha256_ssse3_import, + .finup = sha256_ssse3_finup, .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-ssse3", @@ -235,13 +114,11 @@ static struct shash_alg algs[] = { { } }, { .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_ssse3_init, + .init = sha224_base_init, .update = sha256_ssse3_update, - .final = sha224_ssse3_final, - .export = sha256_ssse3_export, - .import = sha256_ssse3_import, + .final = sha256_ssse3_final, + .finup = sha256_ssse3_finup, .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name = "sha224-ssse3", diff --git a/arch/x86/crypto/sha512-avx-asm.S b/arch/x86/crypto/sha512-avx-asm.S index 974dde9bc6cd..565274d6a641 100644 --- a/arch/x86/crypto/sha512-avx-asm.S +++ b/arch/x86/crypto/sha512-avx-asm.S @@ -54,9 +54,9 @@ # Virtual Registers # ARG1 -msg = %rdi +digest = %rdi # ARG2 -digest = %rsi +msg = %rsi # ARG3 msglen = %rdx T1 = %rcx @@ -271,7 +271,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE .endm ######################################################################## -# void sha512_transform_avx(const void* M, void* D, u64 L) +# void sha512_transform_avx(void* D, const void* M, u64 L) # Purpose: Updates the SHA512 digest stored at D with the message stored in M. # The size of the message pointed to by M must be an integer multiple of SHA512 # message blocks. diff --git a/arch/x86/crypto/sha512-avx2-asm.S b/arch/x86/crypto/sha512-avx2-asm.S index 568b96105f5c..a4771dcd1fcf 100644 --- a/arch/x86/crypto/sha512-avx2-asm.S +++ b/arch/x86/crypto/sha512-avx2-asm.S @@ -70,9 +70,9 @@ XFER = YTMP0 BYTE_FLIP_MASK = %ymm9 # 1st arg -INP = %rdi +CTX = %rdi # 2nd arg -CTX = %rsi +INP = %rsi # 3rd arg NUM_BLKS = %rdx @@ -562,7 +562,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE .endm ######################################################################## -# void sha512_transform_rorx(const void* M, void* D, uint64_t L)# +# void sha512_transform_rorx(void* D, const void* M, uint64_t L)# # Purpose: Updates the SHA512 digest stored at D with the message stored in M. # The size of the message pointed to by M must be an integer multiple of SHA512 # message blocks. diff --git a/arch/x86/crypto/sha512-ssse3-asm.S b/arch/x86/crypto/sha512-ssse3-asm.S index fb56855d51f5..e610e29cbc81 100644 --- a/arch/x86/crypto/sha512-ssse3-asm.S +++ b/arch/x86/crypto/sha512-ssse3-asm.S @@ -53,9 +53,9 @@ # Virtual Registers # ARG1 -msg = %rdi +digest = %rdi # ARG2 -digest = %rsi +msg = %rsi # ARG3 msglen = %rdx T1 = %rcx @@ -269,7 +269,7 @@ frame_size = frame_GPRSAVE + GPRSAVE_SIZE .endm ######################################################################## -# void sha512_transform_ssse3(const void* M, void* D, u64 L)# +# void sha512_transform_ssse3(void* D, const void* M, u64 L)# # Purpose: Updates the SHA512 digest stored at D with the message stored in M. # The size of the message pointed to by M must be an integer multiple of SHA512 # message blocks. diff --git a/arch/x86/crypto/sha512_ssse3_glue.c b/arch/x86/crypto/sha512_ssse3_glue.c index 0b6af26832bf..d9fa4c1e063f 100644 --- a/arch/x86/crypto/sha512_ssse3_glue.c +++ b/arch/x86/crypto/sha512_ssse3_glue.c @@ -34,205 +34,75 @@ #include <linux/cryptohash.h> #include <linux/types.h> #include <crypto/sha.h> -#include <asm/byteorder.h> +#include <crypto/sha512_base.h> #include <asm/i387.h> #include <asm/xcr.h> #include <asm/xsave.h> #include <linux/string.h> -asmlinkage void sha512_transform_ssse3(const char *data, u64 *digest, - u64 rounds); +asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data, + u64 rounds); #ifdef CONFIG_AS_AVX -asmlinkage void sha512_transform_avx(const char *data, u64 *digest, +asmlinkage void sha512_transform_avx(u64 *digest, const char *data, u64 rounds); #endif #ifdef CONFIG_AS_AVX2 -asmlinkage void sha512_transform_rorx(const char *data, u64 *digest, - u64 rounds); +asmlinkage void sha512_transform_rorx(u64 *digest, const char *data, + u64 rounds); #endif -static asmlinkage void (*sha512_transform_asm)(const char *, u64 *, u64); - - -static int sha512_ssse3_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA512_H0; - sctx->state[1] = SHA512_H1; - sctx->state[2] = SHA512_H2; - sctx->state[3] = SHA512_H3; - sctx->state[4] = SHA512_H4; - sctx->state[5] = SHA512_H5; - sctx->state[6] = SHA512_H6; - sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} +static void (*sha512_transform_asm)(u64 *, const char *, u64); -static int __sha512_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len, unsigned int partial) +static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, + unsigned int len) { struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int done = 0; - - sctx->count[0] += len; - if (sctx->count[0] < len) - sctx->count[1]++; - if (partial) { - done = SHA512_BLOCK_SIZE - partial; - memcpy(sctx->buf + partial, data, done); - sha512_transform_asm(sctx->buf, sctx->state, 1); - } - - if (len - done >= SHA512_BLOCK_SIZE) { - const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; + if (!irq_fpu_usable() || + (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) + return crypto_sha512_update(desc, data, len); - sha512_transform_asm(data + done, sctx->state, (u64) rounds); - - done += rounds * SHA512_BLOCK_SIZE; - } + /* make sure casting to sha512_block_fn() is safe */ + BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); - memcpy(sctx->buf, data + done, len - done); + kernel_fpu_begin(); + sha512_base_do_update(desc, data, len, + (sha512_block_fn *)sha512_transform_asm); + kernel_fpu_end(); return 0; } -static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) +static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) { - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; - int res; - - /* Handle the fast case right here */ - if (partial + len < SHA512_BLOCK_SIZE) { - sctx->count[0] += len; - if (sctx->count[0] < len) - sctx->count[1]++; - memcpy(sctx->buf + partial, data, len); - - return 0; - } + if (!irq_fpu_usable()) + return crypto_sha512_finup(desc, data, len, out); - if (!irq_fpu_usable()) { - res = crypto_sha512_update(desc, data, len); - } else { - kernel_fpu_begin(); - res = __sha512_ssse3_update(desc, data, len, partial); - kernel_fpu_end(); - } + kernel_fpu_begin(); + if (len) + sha512_base_do_update(desc, data, len, + (sha512_block_fn *)sha512_transform_asm); + sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_transform_asm); + kernel_fpu_end(); - return res; + return sha512_base_finish(desc, out); } - /* Add padding and return the message digest. */ static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) { - struct sha512_state *sctx = shash_desc_ctx(desc); - unsigned int i, index, padlen; - __be64 *dst = (__be64 *)out; - __be64 bits[2]; - static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; - - /* save number of bits */ - bits[1] = cpu_to_be64(sctx->count[0] << 3); - bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); - - /* Pad out to 112 mod 128 and append length */ - index = sctx->count[0] & 0x7f; - padlen = (index < 112) ? (112 - index) : ((128+112) - index); - - if (!irq_fpu_usable()) { - crypto_sha512_update(desc, padding, padlen); - crypto_sha512_update(desc, (const u8 *)&bits, sizeof(bits)); - } else { - kernel_fpu_begin(); - /* We need to fill a whole block for __sha512_ssse3_update() */ - if (padlen <= 112) { - sctx->count[0] += padlen; - if (sctx->count[0] < padlen) - sctx->count[1]++; - memcpy(sctx->buf + index, padding, padlen); - } else { - __sha512_ssse3_update(desc, padding, padlen, index); - } - __sha512_ssse3_update(desc, (const u8 *)&bits, - sizeof(bits), 112); - kernel_fpu_end(); - } - - /* Store state in digest */ - for (i = 0; i < 8; i++) - dst[i] = cpu_to_be64(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof(*sctx)); - - return 0; -} - -static int sha512_ssse3_export(struct shash_desc *desc, void *out) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - - return 0; -} - -static int sha512_ssse3_import(struct shash_desc *desc, const void *in) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - - return 0; -} - -static int sha384_ssse3_init(struct shash_desc *desc) -{ - struct sha512_state *sctx = shash_desc_ctx(desc); - - sctx->state[0] = SHA384_H0; - sctx->state[1] = SHA384_H1; - sctx->state[2] = SHA384_H2; - sctx->state[3] = SHA384_H3; - sctx->state[4] = SHA384_H4; - sctx->state[5] = SHA384_H5; - sctx->state[6] = SHA384_H6; - sctx->state[7] = SHA384_H7; - - sctx->count[0] = sctx->count[1] = 0; - - return 0; -} - -static int sha384_ssse3_final(struct shash_desc *desc, u8 *hash) -{ - u8 D[SHA512_DIGEST_SIZE]; - - sha512_ssse3_final(desc, D); - - memcpy(hash, D, SHA384_DIGEST_SIZE); - memzero_explicit(D, SHA512_DIGEST_SIZE); - - return 0; + return sha512_ssse3_finup(desc, NULL, 0, out); } static struct shash_alg algs[] = { { .digestsize = SHA512_DIGEST_SIZE, - .init = sha512_ssse3_init, + .init = sha512_base_init, .update = sha512_ssse3_update, .final = sha512_ssse3_final, - .export = sha512_ssse3_export, - .import = sha512_ssse3_import, + .finup = sha512_ssse3_finup, .descsize = sizeof(struct sha512_state), - .statesize = sizeof(struct sha512_state), .base = { .cra_name = "sha512", .cra_driver_name = "sha512-ssse3", @@ -243,13 +113,11 @@ static struct shash_alg algs[] = { { } }, { .digestsize = SHA384_DIGEST_SIZE, - .init = sha384_ssse3_init, + .init = sha384_base_init, .update = sha512_ssse3_update, - .final = sha384_ssse3_final, - .export = sha512_ssse3_export, - .import = sha512_ssse3_import, + .final = sha512_ssse3_final, + .finup = sha512_ssse3_finup, .descsize = sizeof(struct sha512_state), - .statesize = sizeof(struct sha512_state), .base = { .cra_name = "sha384", .cra_driver_name = "sha384-ssse3", diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c index 1ac531ea9bcc..b5e2d5651851 100644 --- a/arch/x86/crypto/twofish_avx_glue.c +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -340,7 +340,8 @@ static struct crypto_alg twofish_algs[10] = { { .cra_name = "__ecb-twofish-avx", .cra_driver_name = "__driver-ecb-twofish-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_ctx), .cra_alignmask = 0, @@ -359,7 +360,8 @@ static struct crypto_alg twofish_algs[10] = { { .cra_name = "__cbc-twofish-avx", .cra_driver_name = "__driver-cbc-twofish-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_ctx), .cra_alignmask = 0, @@ -378,7 +380,8 @@ static struct crypto_alg twofish_algs[10] = { { .cra_name = "__ctr-twofish-avx", .cra_driver_name = "__driver-ctr-twofish-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = 1, .cra_ctxsize = sizeof(struct twofish_ctx), .cra_alignmask = 0, @@ -398,7 +401,8 @@ static struct crypto_alg twofish_algs[10] = { { .cra_name = "__lrw-twofish-avx", .cra_driver_name = "__driver-lrw-twofish-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_lrw_ctx), .cra_alignmask = 0, @@ -421,7 +425,8 @@ static struct crypto_alg twofish_algs[10] = { { .cra_name = "__xts-twofish-avx", .cra_driver_name = "__driver-xts-twofish-avx", .cra_priority = 0, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | + CRYPTO_ALG_INTERNAL, .cra_blocksize = TF_BLOCK_SIZE, .cra_ctxsize = sizeof(struct twofish_xts_ctx), .cra_alignmask = 0, diff --git a/arch/x86/include/asm/seccomp.h b/arch/x86/include/asm/seccomp.h index 0f3d7f099224..0c8c7c8861b4 100644 --- a/arch/x86/include/asm/seccomp.h +++ b/arch/x86/include/asm/seccomp.h @@ -1,5 +1,20 @@ +#ifndef _ASM_X86_SECCOMP_H +#define _ASM_X86_SECCOMP_H + +#include <asm/unistd.h> + #ifdef CONFIG_X86_32 -# include <asm/seccomp_32.h> -#else -# include <asm/seccomp_64.h> +#define __NR_seccomp_sigreturn __NR_sigreturn #endif + +#ifdef CONFIG_COMPAT +#include <asm/ia32_unistd.h> +#define __NR_seccomp_read_32 __NR_ia32_read +#define __NR_seccomp_write_32 __NR_ia32_write +#define __NR_seccomp_exit_32 __NR_ia32_exit +#define __NR_seccomp_sigreturn_32 __NR_ia32_sigreturn +#endif + +#include <asm-generic/seccomp.h> + +#endif /* _ASM_X86_SECCOMP_H */ diff --git a/arch/x86/include/asm/seccomp_32.h b/arch/x86/include/asm/seccomp_32.h deleted file mode 100644 index b811d6f5780c..000000000000 --- a/arch/x86/include/asm/seccomp_32.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef _ASM_X86_SECCOMP_32_H -#define _ASM_X86_SECCOMP_32_H - -#include <linux/unistd.h> - -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_sigreturn - -#endif /* _ASM_X86_SECCOMP_32_H */ diff --git a/arch/x86/include/asm/seccomp_64.h b/arch/x86/include/asm/seccomp_64.h deleted file mode 100644 index 84ec1bd161a5..000000000000 --- a/arch/x86/include/asm/seccomp_64.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef _ASM_X86_SECCOMP_64_H -#define _ASM_X86_SECCOMP_64_H - -#include <linux/unistd.h> -#include <asm/ia32_unistd.h> - -#define __NR_seccomp_read __NR_read -#define __NR_seccomp_write __NR_write -#define __NR_seccomp_exit __NR_exit -#define __NR_seccomp_sigreturn __NR_rt_sigreturn - -#define __NR_seccomp_read_32 __NR_ia32_read -#define __NR_seccomp_write_32 __NR_ia32_write -#define __NR_seccomp_exit_32 __NR_ia32_exit -#define __NR_seccomp_sigreturn_32 __NR_ia32_sigreturn - -#endif /* _ASM_X86_SECCOMP_64_H */ diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h index ea2dbe82cba3..b4bdec3e9523 100644 --- a/arch/x86/include/asm/thread_info.h +++ b/arch/x86/include/asm/thread_info.h @@ -46,13 +46,11 @@ */ #ifndef __ASSEMBLY__ struct task_struct; -struct exec_domain; #include <asm/processor.h> #include <linux/atomic.h> struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ __u32 flags; /* low level flags */ __u32 status; /* thread synchronous flags */ __u32 cpu; /* current CPU */ @@ -66,7 +64,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .saved_preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/x86/kernel/cpu/mtrr/if.c b/arch/x86/kernel/cpu/mtrr/if.c index a041e094b8b9..d76f13d6d8d6 100644 --- a/arch/x86/kernel/cpu/mtrr/if.c +++ b/arch/x86/kernel/cpu/mtrr/if.c @@ -404,11 +404,10 @@ static const struct file_operations mtrr_fops = { static int mtrr_seq_show(struct seq_file *seq, void *offset) { char factor; - int i, max, len; + int i, max; mtrr_type type; unsigned long base, size; - len = 0; max = num_var_ranges; for (i = 0; i < max; i++) { mtrr_if->get(i, &base, &size, &type); @@ -425,11 +424,10 @@ static int mtrr_seq_show(struct seq_file *seq, void *offset) size >>= 20 - PAGE_SHIFT; } /* Base can be > 32bit */ - len += seq_printf(seq, "reg%02i: base=0x%06lx000 " - "(%5luMB), size=%5lu%cB, count=%d: %s\n", - i, base, base >> (20 - PAGE_SHIFT), size, - factor, mtrr_usage_table[i], - mtrr_attrib_to_str(type)); + seq_printf(seq, "reg%02i: base=0x%06lx000 (%5luMB), size=%5lu%cB, count=%d: %s\n", + i, base, base >> (20 - PAGE_SHIFT), + size, factor, + mtrr_usage_table[i], mtrr_attrib_to_str(type)); } return 0; } diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index 3e581865c8e2..f9804080ccb3 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -593,24 +593,10 @@ badframe: return 0; } -/* - * OK, we're invoking a handler: - */ -static int signr_convert(int sig) -{ -#ifdef CONFIG_X86_32 - struct thread_info *info = current_thread_info(); - - if (info->exec_domain && info->exec_domain->signal_invmap && sig < 32) - return info->exec_domain->signal_invmap[sig]; -#endif /* CONFIG_X86_32 */ - return sig; -} - static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) { - int usig = signr_convert(ksig->sig); + int usig = ksig->sig; sigset_t *set = sigmask_to_save(); compat_sigset_t *cset = (compat_sigset_t *) set; diff --git a/arch/x86/syscalls/Makefile b/arch/x86/syscalls/Makefile index 3323c2745248..a55abb9f6c5e 100644 --- a/arch/x86/syscalls/Makefile +++ b/arch/x86/syscalls/Makefile @@ -19,6 +19,9 @@ quiet_cmd_syshdr = SYSHDR $@ quiet_cmd_systbl = SYSTBL $@ cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@ +quiet_cmd_hypercalls = HYPERCALLS $@ + cmd_hypercalls = $(CONFIG_SHELL) '$<' $@ $(filter-out $<,$^) + syshdr_abi_unistd_32 := i386 $(uapi)/unistd_32.h: $(syscall32) $(syshdr) $(call if_changed,syshdr) @@ -47,10 +50,16 @@ $(out)/syscalls_32.h: $(syscall32) $(systbl) $(out)/syscalls_64.h: $(syscall64) $(systbl) $(call if_changed,systbl) +$(out)/xen-hypercalls.h: $(srctree)/scripts/xen-hypercalls.sh + $(call if_changed,hypercalls) + +$(out)/xen-hypercalls.h: $(srctree)/include/xen/interface/xen*.h + uapisyshdr-y += unistd_32.h unistd_64.h unistd_x32.h syshdr-y += syscalls_32.h syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h syshdr-$(CONFIG_X86_64) += syscalls_64.h +syshdr-$(CONFIG_XEN) += xen-hypercalls.h targets += $(uapisyshdr-y) $(syshdr-y) diff --git a/arch/x86/um/Makefile b/arch/x86/um/Makefile index eafa324eb7a5..acb384d24669 100644 --- a/arch/x86/um/Makefile +++ b/arch/x86/um/Makefile @@ -21,7 +21,6 @@ obj-$(CONFIG_BINFMT_ELF) += elfcore.o subarch-y = ../lib/string_32.o ../lib/atomic64_32.o ../lib/atomic64_cx8_32.o subarch-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += ../lib/rwsem.o -subarch-$(CONFIG_HIGHMEM) += ../mm/highmem_32.o else diff --git a/arch/x86/um/asm/barrier.h b/arch/x86/um/asm/barrier.h index 8ffd2146fa6a..7e8a1a650435 100644 --- a/arch/x86/um/asm/barrier.h +++ b/arch/x86/um/asm/barrier.h @@ -36,22 +36,11 @@ #endif /* CONFIG_X86_PPRO_FENCE */ #define dma_wmb() barrier() -#ifdef CONFIG_SMP - -#define smp_mb() mb() -#define smp_rmb() dma_rmb() -#define smp_wmb() barrier() -#define set_mb(var, value) do { (void)xchg(&var, value); } while (0) - -#else /* CONFIG_SMP */ - #define smp_mb() barrier() #define smp_rmb() barrier() #define smp_wmb() barrier() #define set_mb(var, value) do { var = value; barrier(); } while (0) -#endif /* CONFIG_SMP */ - #define read_barrier_depends() do { } while (0) #define smp_read_barrier_depends() do { } while (0) diff --git a/arch/x86/um/asm/elf.h b/arch/x86/um/asm/elf.h index 25a1022dd793..0a656b727b1a 100644 --- a/arch/x86/um/asm/elf.h +++ b/arch/x86/um/asm/elf.h @@ -210,7 +210,7 @@ extern int elf_core_copy_fpregs(struct task_struct *t, elf_fpregset_t *fpu); #define ELF_EXEC_PAGESIZE 4096 -#define ELF_ET_DYN_BASE (2 * TASK_SIZE / 3) +#define ELF_ET_DYN_BASE (TASK_SIZE / 3 * 2) extern long elf_aux_hwcap; #define ELF_HWCAP (elf_aux_hwcap) diff --git a/arch/x86/um/ldt.c b/arch/x86/um/ldt.c index 8e08176f0bcb..5c0b711d2433 100644 --- a/arch/x86/um/ldt.c +++ b/arch/x86/um/ldt.c @@ -8,9 +8,7 @@ #include <linux/slab.h> #include <asm/unistd.h> #include <os.h> -#include <proc_mm.h> #include <skas.h> -#include <skas_ptrace.h> #include <sysdep/tls.h> extern int modify_ldt(int func, void *ptr, unsigned long bytecount); @@ -19,105 +17,20 @@ static long write_ldt_entry(struct mm_id *mm_idp, int func, struct user_desc *desc, void **addr, int done) { long res; - - if (proc_mm) { - /* - * This is a special handling for the case, that the mm to - * modify isn't current->active_mm. - * If this is called directly by modify_ldt, - * (current->active_mm->context.skas.u == mm_idp) - * will be true. So no call to __switch_mm(mm_idp) is done. - * If this is called in case of init_new_ldt or PTRACE_LDT, - * mm_idp won't belong to current->active_mm, but child->mm. - * So we need to switch child's mm into our userspace, then - * later switch back. - * - * Note: I'm unsure: should interrupts be disabled here? - */ - if (!current->active_mm || current->active_mm == &init_mm || - mm_idp != ¤t->active_mm->context.id) - __switch_mm(mm_idp); - } - - if (ptrace_ldt) { - struct ptrace_ldt ldt_op = (struct ptrace_ldt) { - .func = func, - .ptr = desc, - .bytecount = sizeof(*desc)}; - u32 cpu; - int pid; - - if (!proc_mm) - pid = mm_idp->u.pid; - else { - cpu = get_cpu(); - pid = userspace_pid[cpu]; - } - - res = os_ptrace_ldt(pid, 0, (unsigned long) &ldt_op); - - if (proc_mm) - put_cpu(); - } - else { - void *stub_addr; - res = syscall_stub_data(mm_idp, (unsigned long *)desc, - (sizeof(*desc) + sizeof(long) - 1) & - ~(sizeof(long) - 1), - addr, &stub_addr); - if (!res) { - unsigned long args[] = { func, - (unsigned long)stub_addr, - sizeof(*desc), - 0, 0, 0 }; - res = run_syscall_stub(mm_idp, __NR_modify_ldt, args, - 0, addr, done); - } + void *stub_addr; + res = syscall_stub_data(mm_idp, (unsigned long *)desc, + (sizeof(*desc) + sizeof(long) - 1) & + ~(sizeof(long) - 1), + addr, &stub_addr); + if (!res) { + unsigned long args[] = { func, + (unsigned long)stub_addr, + sizeof(*desc), + 0, 0, 0 }; + res = run_syscall_stub(mm_idp, __NR_modify_ldt, args, + 0, addr, done); } - if (proc_mm) { - /* - * This is the second part of special handling, that makes - * PTRACE_LDT possible to implement. - */ - if (current->active_mm && current->active_mm != &init_mm && - mm_idp != ¤t->active_mm->context.id) - __switch_mm(¤t->active_mm->context.id); - } - - return res; -} - -static long read_ldt_from_host(void __user * ptr, unsigned long bytecount) -{ - int res, n; - struct ptrace_ldt ptrace_ldt = (struct ptrace_ldt) { - .func = 0, - .bytecount = bytecount, - .ptr = kmalloc(bytecount, GFP_KERNEL)}; - u32 cpu; - - if (ptrace_ldt.ptr == NULL) - return -ENOMEM; - - /* - * This is called from sys_modify_ldt only, so userspace_pid gives - * us the right number - */ - - cpu = get_cpu(); - res = os_ptrace_ldt(userspace_pid[cpu], 0, (unsigned long) &ptrace_ldt); - put_cpu(); - if (res < 0) - goto out; - - n = copy_to_user(ptr, ptrace_ldt.ptr, res); - if (n != 0) - res = -EFAULT; - - out: - kfree(ptrace_ldt.ptr); - return res; } @@ -145,9 +58,6 @@ static int read_ldt(void __user * ptr, unsigned long bytecount) bytecount = LDT_ENTRY_SIZE*LDT_ENTRIES; err = bytecount; - if (ptrace_ldt) - return read_ldt_from_host(ptr, bytecount); - mutex_lock(&ldt->lock); if (ldt->entry_count <= LDT_DIRECT_ENTRIES) { size = LDT_ENTRY_SIZE*LDT_DIRECT_ENTRIES; @@ -229,17 +139,11 @@ static int write_ldt(void __user * ptr, unsigned long bytecount, int func) goto out; } - if (!ptrace_ldt) - mutex_lock(&ldt->lock); + mutex_lock(&ldt->lock); err = write_ldt_entry(mm_idp, func, &ldt_info, &addr, 1); if (err) goto out_unlock; - else if (ptrace_ldt) { - /* With PTRACE_LDT available, this is used as a flag only */ - ldt->entry_count = 1; - goto out; - } if (ldt_info.entry_number >= ldt->entry_count && ldt_info.entry_number >= LDT_DIRECT_ENTRIES) { @@ -393,91 +297,56 @@ long init_new_ldt(struct mm_context *new_mm, struct mm_context *from_mm) int i; long page, err=0; void *addr = NULL; - struct proc_mm_op copy; - if (!ptrace_ldt) - mutex_init(&new_mm->arch.ldt.lock); + mutex_init(&new_mm->arch.ldt.lock); if (!from_mm) { memset(&desc, 0, sizeof(desc)); /* - * We have to initialize a clean ldt. + * Now we try to retrieve info about the ldt, we + * inherited from the host. All ldt-entries found + * will be reset in the following loop */ - if (proc_mm) { - /* - * If the new mm was created using proc_mm, host's - * default-ldt currently is assigned, which normally - * contains the call-gates for lcall7 and lcall27. - * To remove these gates, we simply write an empty - * entry as number 0 to the host. - */ - err = write_ldt_entry(&new_mm->id, 1, &desc, &addr, 1); - } - else{ - /* - * Now we try to retrieve info about the ldt, we - * inherited from the host. All ldt-entries found - * will be reset in the following loop - */ - ldt_get_host_info(); - for (num_p=host_ldt_entries; *num_p != -1; num_p++) { - desc.entry_number = *num_p; - err = write_ldt_entry(&new_mm->id, 1, &desc, - &addr, *(num_p + 1) == -1); - if (err) - break; - } + ldt_get_host_info(); + for (num_p=host_ldt_entries; *num_p != -1; num_p++) { + desc.entry_number = *num_p; + err = write_ldt_entry(&new_mm->id, 1, &desc, + &addr, *(num_p + 1) == -1); + if (err) + break; } new_mm->arch.ldt.entry_count = 0; goto out; } - if (proc_mm) { - /* - * We have a valid from_mm, so we now have to copy the LDT of - * from_mm to new_mm, because using proc_mm an new mm with - * an empty/default LDT was created in new_mm() - */ - copy = ((struct proc_mm_op) { .op = MM_COPY_SEGMENTS, - .u = - { .copy_segments = - from_mm->id.u.mm_fd } } ); - i = os_write_file(new_mm->id.u.mm_fd, ©, sizeof(copy)); - if (i != sizeof(copy)) - printk(KERN_ERR "new_mm : /proc/mm copy_segments " - "failed, err = %d\n", -i); - } - - if (!ptrace_ldt) { - /* - * Our local LDT is used to supply the data for - * modify_ldt(READLDT), if PTRACE_LDT isn't available, - * i.e., we have to use the stub for modify_ldt, which - * can't handle the big read buffer of up to 64kB. - */ - mutex_lock(&from_mm->arch.ldt.lock); - if (from_mm->arch.ldt.entry_count <= LDT_DIRECT_ENTRIES) - memcpy(new_mm->arch.ldt.u.entries, from_mm->arch.ldt.u.entries, - sizeof(new_mm->arch.ldt.u.entries)); - else { - i = from_mm->arch.ldt.entry_count / LDT_ENTRIES_PER_PAGE; - while (i-->0) { - page = __get_free_page(GFP_KERNEL|__GFP_ZERO); - if (!page) { - err = -ENOMEM; - break; - } - new_mm->arch.ldt.u.pages[i] = - (struct ldt_entry *) page; - memcpy(new_mm->arch.ldt.u.pages[i], - from_mm->arch.ldt.u.pages[i], PAGE_SIZE); + /* + * Our local LDT is used to supply the data for + * modify_ldt(READLDT), if PTRACE_LDT isn't available, + * i.e., we have to use the stub for modify_ldt, which + * can't handle the big read buffer of up to 64kB. + */ + mutex_lock(&from_mm->arch.ldt.lock); + if (from_mm->arch.ldt.entry_count <= LDT_DIRECT_ENTRIES) + memcpy(new_mm->arch.ldt.u.entries, from_mm->arch.ldt.u.entries, + sizeof(new_mm->arch.ldt.u.entries)); + else { + i = from_mm->arch.ldt.entry_count / LDT_ENTRIES_PER_PAGE; + while (i-->0) { + page = __get_free_page(GFP_KERNEL|__GFP_ZERO); + if (!page) { + err = -ENOMEM; + break; } + new_mm->arch.ldt.u.pages[i] = + (struct ldt_entry *) page; + memcpy(new_mm->arch.ldt.u.pages[i], + from_mm->arch.ldt.u.pages[i], PAGE_SIZE); } - new_mm->arch.ldt.entry_count = from_mm->arch.ldt.entry_count; - mutex_unlock(&from_mm->arch.ldt.lock); } + new_mm->arch.ldt.entry_count = from_mm->arch.ldt.entry_count; + mutex_unlock(&from_mm->arch.ldt.lock); out: return err; @@ -488,7 +357,7 @@ void free_ldt(struct mm_context *mm) { int i; - if (!ptrace_ldt && mm->arch.ldt.entry_count > LDT_DIRECT_ENTRIES) { + if (mm->arch.ldt.entry_count > LDT_DIRECT_ENTRIES) { i = mm->arch.ldt.entry_count / LDT_ENTRIES_PER_PAGE; while (i-- > 0) free_page((long) mm->arch.ldt.u.pages[i]); diff --git a/arch/x86/um/shared/sysdep/faultinfo_32.h b/arch/x86/um/shared/sysdep/faultinfo_32.h index a26086b8a800..b6f2437ec29c 100644 --- a/arch/x86/um/shared/sysdep/faultinfo_32.h +++ b/arch/x86/um/shared/sysdep/faultinfo_32.h @@ -27,9 +27,6 @@ struct faultinfo { /* This is Page Fault */ #define SEGV_IS_FIXABLE(fi) ((fi)->trap_no == 14) -/* SKAS3 has no trap_no on i386, but get_skas_faultinfo() sets it to 0. */ -#define SEGV_MAYBE_FIXABLE(fi) ((fi)->trap_no == 0 && ptrace_faultinfo) - #define PTRACE_FULL_FAULTINFO 0 #endif diff --git a/arch/x86/um/shared/sysdep/faultinfo_64.h b/arch/x86/um/shared/sysdep/faultinfo_64.h index f811cbe15d62..ee88f88974ea 100644 --- a/arch/x86/um/shared/sysdep/faultinfo_64.h +++ b/arch/x86/um/shared/sysdep/faultinfo_64.h @@ -27,9 +27,6 @@ struct faultinfo { /* This is Page Fault */ #define SEGV_IS_FIXABLE(fi) ((fi)->trap_no == 14) -/* No broken SKAS API, which doesn't pass trap_no, here. */ -#define SEGV_MAYBE_FIXABLE(fi) 0 - #define PTRACE_FULL_FAULTINFO 1 #endif diff --git a/arch/x86/um/shared/sysdep/skas_ptrace.h b/arch/x86/um/shared/sysdep/skas_ptrace.h deleted file mode 100644 index 453febe98993..000000000000 --- a/arch/x86/um/shared/sysdep/skas_ptrace.h +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (C) 2000, 2001, 2002 Jeff Dike (jdike@karaya.com) - * Licensed under the GPL - */ - -#ifndef __SYSDEP_X86_SKAS_PTRACE_H -#define __SYSDEP_X86_SKAS_PTRACE_H - -struct ptrace_faultinfo { - int is_write; - unsigned long addr; -}; - -struct ptrace_ldt { - int func; - void *ptr; - unsigned long bytecount; -}; - -#define PTRACE_LDT 54 - -#endif diff --git a/arch/x86/um/signal.c b/arch/x86/um/signal.c index 0c8c32bfd792..592491d1d70d 100644 --- a/arch/x86/um/signal.c +++ b/arch/x86/um/signal.c @@ -549,13 +549,6 @@ int setup_signal_stack_si(unsigned long stack_top, struct ksignal *ksig, if (err) return err; - /* Set up registers for signal handler */ - { - struct exec_domain *ed = current_thread_info()->exec_domain; - if (unlikely(ed && ed->signal_invmap && sig < 32)) - sig = ed->signal_invmap[sig]; - } - PT_REGS_SP(regs) = (unsigned long) frame; PT_REGS_DI(regs) = sig; /* In case the signal handler was declared without prototypes */ diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c index 7005ced5d1ad..70e060ad879a 100644 --- a/arch/x86/xen/apic.c +++ b/arch/x86/xen/apic.c @@ -7,6 +7,7 @@ #include <xen/xen.h> #include <xen/interface/physdev.h> #include "xen-ops.h" +#include "smp.h" static unsigned int xen_io_apic_read(unsigned apic, unsigned reg) { @@ -28,7 +29,186 @@ static unsigned int xen_io_apic_read(unsigned apic, unsigned reg) return 0xfd; } +static unsigned long xen_set_apic_id(unsigned int x) +{ + WARN_ON(1); + return x; +} + +static unsigned int xen_get_apic_id(unsigned long x) +{ + return ((x)>>24) & 0xFFu; +} + +static u32 xen_apic_read(u32 reg) +{ + struct xen_platform_op op = { + .cmd = XENPF_get_cpuinfo, + .interface_version = XENPF_INTERFACE_VERSION, + .u.pcpu_info.xen_cpuid = 0, + }; + int ret = 0; + + /* Shouldn't need this as APIC is turned off for PV, and we only + * get called on the bootup processor. But just in case. */ + if (!xen_initial_domain() || smp_processor_id()) + return 0; + + if (reg == APIC_LVR) + return 0x10; +#ifdef CONFIG_X86_32 + if (reg == APIC_LDR) + return SET_APIC_LOGICAL_ID(1UL << smp_processor_id()); +#endif + if (reg != APIC_ID) + return 0; + + ret = HYPERVISOR_dom0_op(&op); + if (ret) + return 0; + + return op.u.pcpu_info.apic_id << 24; +} + +static void xen_apic_write(u32 reg, u32 val) +{ + /* Warn to see if there's any stray references */ + WARN(1,"register: %x, value: %x\n", reg, val); +} + +static u64 xen_apic_icr_read(void) +{ + return 0; +} + +static void xen_apic_icr_write(u32 low, u32 id) +{ + /* Warn to see if there's any stray references */ + WARN_ON(1); +} + +static u32 xen_safe_apic_wait_icr_idle(void) +{ + return 0; +} + +static int xen_apic_probe_pv(void) +{ + if (xen_pv_domain()) + return 1; + + return 0; +} + +static int xen_madt_oem_check(char *oem_id, char *oem_table_id) +{ + return xen_pv_domain(); +} + +static int xen_id_always_valid(int apicid) +{ + return 1; +} + +static int xen_id_always_registered(void) +{ + return 1; +} + +static int xen_phys_pkg_id(int initial_apic_id, int index_msb) +{ + return initial_apic_id >> index_msb; +} + +#ifdef CONFIG_X86_32 +static int xen_x86_32_early_logical_apicid(int cpu) +{ + /* Match with APIC_LDR read. Otherwise setup_local_APIC complains. */ + return 1 << cpu; +} +#endif + +static void xen_noop(void) +{ +} + +static void xen_silent_inquire(int apicid) +{ +} + +static struct apic xen_pv_apic = { + .name = "Xen PV", + .probe = xen_apic_probe_pv, + .acpi_madt_oem_check = xen_madt_oem_check, + .apic_id_valid = xen_id_always_valid, + .apic_id_registered = xen_id_always_registered, + + /* .irq_delivery_mode - used in native_compose_msi_msg only */ + /* .irq_dest_mode - used in native_compose_msi_msg only */ + + .target_cpus = default_target_cpus, + .disable_esr = 0, + /* .dest_logical - default_send_IPI_ use it but we use our own. */ + .check_apicid_used = default_check_apicid_used, /* Used on 32-bit */ + + .vector_allocation_domain = flat_vector_allocation_domain, + .init_apic_ldr = xen_noop, /* setup_local_APIC calls it */ + + .ioapic_phys_id_map = default_ioapic_phys_id_map, /* Used on 32-bit */ + .setup_apic_routing = NULL, + .cpu_present_to_apicid = default_cpu_present_to_apicid, + .apicid_to_cpu_present = physid_set_mask_of_physid, /* Used on 32-bit */ + .check_phys_apicid_present = default_check_phys_apicid_present, /* smp_sanity_check needs it */ + .phys_pkg_id = xen_phys_pkg_id, /* detect_ht */ + + .get_apic_id = xen_get_apic_id, + .set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */ + .apic_id_mask = 0xFF << 24, /* Used by verify_local_APIC. Match with what xen_get_apic_id does. */ + + .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, + +#ifdef CONFIG_SMP + .send_IPI_mask = xen_send_IPI_mask, + .send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself, + .send_IPI_allbutself = xen_send_IPI_allbutself, + .send_IPI_all = xen_send_IPI_all, + .send_IPI_self = xen_send_IPI_self, +#endif + /* .wait_for_init_deassert- used by AP bootup - smp_callin which we don't use */ + .inquire_remote_apic = xen_silent_inquire, + + .read = xen_apic_read, + .write = xen_apic_write, + .eoi_write = xen_apic_write, + + .icr_read = xen_apic_icr_read, + .icr_write = xen_apic_icr_write, + .wait_icr_idle = xen_noop, + .safe_wait_icr_idle = xen_safe_apic_wait_icr_idle, + +#ifdef CONFIG_X86_32 + /* generic_processor_info and setup_local_APIC. */ + .x86_32_early_logical_apicid = xen_x86_32_early_logical_apicid, +#endif +}; + +static void __init xen_apic_check(void) +{ + if (apic == &xen_pv_apic) + return; + + pr_info("Switched APIC routing from %s to %s.\n", apic->name, + xen_pv_apic.name); + apic = &xen_pv_apic; +} void __init xen_init_apic(void) { x86_io_apic_ops.read = xen_io_apic_read; + /* On PV guests the APIC CPUID bit is disabled so none of the + * routines end up executing. */ + if (!xen_initial_domain()) + apic = &xen_pv_apic; + + x86_platform.apic_post_init = xen_apic_check; } +apic_driver(xen_pv_apic); diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index 81665c9f2132..94578efd3067 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -928,92 +928,6 @@ static void xen_io_delay(void) { } -#ifdef CONFIG_X86_LOCAL_APIC -static unsigned long xen_set_apic_id(unsigned int x) -{ - WARN_ON(1); - return x; -} -static unsigned int xen_get_apic_id(unsigned long x) -{ - return ((x)>>24) & 0xFFu; -} -static u32 xen_apic_read(u32 reg) -{ - struct xen_platform_op op = { - .cmd = XENPF_get_cpuinfo, - .interface_version = XENPF_INTERFACE_VERSION, - .u.pcpu_info.xen_cpuid = 0, - }; - int ret = 0; - - /* Shouldn't need this as APIC is turned off for PV, and we only - * get called on the bootup processor. But just in case. */ - if (!xen_initial_domain() || smp_processor_id()) - return 0; - - if (reg == APIC_LVR) - return 0x10; - - if (reg != APIC_ID) - return 0; - - ret = HYPERVISOR_dom0_op(&op); - if (ret) - return 0; - - return op.u.pcpu_info.apic_id << 24; -} - -static void xen_apic_write(u32 reg, u32 val) -{ - /* Warn to see if there's any stray references */ - WARN_ON(1); -} - -static u64 xen_apic_icr_read(void) -{ - return 0; -} - -static void xen_apic_icr_write(u32 low, u32 id) -{ - /* Warn to see if there's any stray references */ - WARN_ON(1); -} - -static void xen_apic_wait_icr_idle(void) -{ - return; -} - -static u32 xen_safe_apic_wait_icr_idle(void) -{ - return 0; -} - -static void set_xen_basic_apic_ops(void) -{ - apic->read = xen_apic_read; - apic->write = xen_apic_write; - apic->icr_read = xen_apic_icr_read; - apic->icr_write = xen_apic_icr_write; - apic->wait_icr_idle = xen_apic_wait_icr_idle; - apic->safe_wait_icr_idle = xen_safe_apic_wait_icr_idle; - apic->set_apic_id = xen_set_apic_id; - apic->get_apic_id = xen_get_apic_id; - -#ifdef CONFIG_SMP - apic->send_IPI_allbutself = xen_send_IPI_allbutself; - apic->send_IPI_mask_allbutself = xen_send_IPI_mask_allbutself; - apic->send_IPI_mask = xen_send_IPI_mask; - apic->send_IPI_all = xen_send_IPI_all; - apic->send_IPI_self = xen_send_IPI_self; -#endif -} - -#endif - static void xen_clts(void) { struct multicall_space mcs; @@ -1619,7 +1533,7 @@ asmlinkage __visible void __init xen_start_kernel(void) /* * set up the basic apic ops. */ - set_xen_basic_apic_ops(); + xen_init_apic(); #endif if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) { @@ -1732,8 +1646,6 @@ asmlinkage __visible void __init xen_start_kernel(void) if (HYPERVISOR_dom0_op(&op) == 0) boot_params.kbd_status = op.u.firmware_info.u.kbd_shift_flags; - xen_init_apic(); - /* Make sure ACS will be enabled */ pci_request_acs(); diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index 65083ad63b6f..dd151b2045b0 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -2436,99 +2436,11 @@ void __init xen_hvm_init_mmu_ops(void) } #endif -#ifdef CONFIG_XEN_PVH -/* - * Map foreign gfn (fgfn), to local pfn (lpfn). This for the user - * space creating new guest on pvh dom0 and needing to map domU pages. - */ -static int xlate_add_to_p2m(unsigned long lpfn, unsigned long fgfn, - unsigned int domid) -{ - int rc, err = 0; - xen_pfn_t gpfn = lpfn; - xen_ulong_t idx = fgfn; - - struct xen_add_to_physmap_range xatp = { - .domid = DOMID_SELF, - .foreign_domid = domid, - .size = 1, - .space = XENMAPSPACE_gmfn_foreign, - }; - set_xen_guest_handle(xatp.idxs, &idx); - set_xen_guest_handle(xatp.gpfns, &gpfn); - set_xen_guest_handle(xatp.errs, &err); - - rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap_range, &xatp); - if (rc < 0) - return rc; - return err; -} - -static int xlate_remove_from_p2m(unsigned long spfn, int count) -{ - struct xen_remove_from_physmap xrp; - int i, rc; - - for (i = 0; i < count; i++) { - xrp.domid = DOMID_SELF; - xrp.gpfn = spfn+i; - rc = HYPERVISOR_memory_op(XENMEM_remove_from_physmap, &xrp); - if (rc) - break; - } - return rc; -} - -struct xlate_remap_data { - unsigned long fgfn; /* foreign domain's gfn */ - pgprot_t prot; - domid_t domid; - int index; - struct page **pages; -}; - -static int xlate_map_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr, - void *data) -{ - int rc; - struct xlate_remap_data *remap = data; - unsigned long pfn = page_to_pfn(remap->pages[remap->index++]); - pte_t pteval = pte_mkspecial(pfn_pte(pfn, remap->prot)); - - rc = xlate_add_to_p2m(pfn, remap->fgfn, remap->domid); - if (rc) - return rc; - native_set_pte(ptep, pteval); - - return 0; -} - -static int xlate_remap_gfn_range(struct vm_area_struct *vma, - unsigned long addr, unsigned long mfn, - int nr, pgprot_t prot, unsigned domid, - struct page **pages) -{ - int err; - struct xlate_remap_data pvhdata; - - BUG_ON(!pages); - - pvhdata.fgfn = mfn; - pvhdata.prot = prot; - pvhdata.domid = domid; - pvhdata.index = 0; - pvhdata.pages = pages; - err = apply_to_page_range(vma->vm_mm, addr, nr << PAGE_SHIFT, - xlate_map_pte_fn, &pvhdata); - flush_tlb_all(); - return err; -} -#endif - #define REMAP_BATCH_SIZE 16 struct remap_data { - unsigned long mfn; + xen_pfn_t *mfn; + bool contiguous; pgprot_t prot; struct mmu_update *mmu_update; }; @@ -2537,7 +2449,14 @@ static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token, unsigned long addr, void *data) { struct remap_data *rmd = data; - pte_t pte = pte_mkspecial(mfn_pte(rmd->mfn++, rmd->prot)); + pte_t pte = pte_mkspecial(mfn_pte(*rmd->mfn, rmd->prot)); + + /* If we have a contigious range, just update the mfn itself, + else update pointer to be "next mfn". */ + if (rmd->contiguous) + (*rmd->mfn)++; + else + rmd->mfn++; rmd->mmu_update->ptr = virt_to_machine(ptep).maddr; rmd->mmu_update->val = pte_val_ma(pte); @@ -2546,26 +2465,26 @@ static int remap_area_mfn_pte_fn(pte_t *ptep, pgtable_t token, return 0; } -int xen_remap_domain_mfn_range(struct vm_area_struct *vma, - unsigned long addr, - xen_pfn_t mfn, int nr, - pgprot_t prot, unsigned domid, - struct page **pages) - +static int do_remap_mfn(struct vm_area_struct *vma, + unsigned long addr, + xen_pfn_t *mfn, int nr, + int *err_ptr, pgprot_t prot, + unsigned domid, + struct page **pages) { + int err = 0; struct remap_data rmd; struct mmu_update mmu_update[REMAP_BATCH_SIZE]; - int batch; unsigned long range; - int err = 0; + int mapped = 0; BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO))); if (xen_feature(XENFEAT_auto_translated_physmap)) { #ifdef CONFIG_XEN_PVH /* We need to update the local page tables and the xen HAP */ - return xlate_remap_gfn_range(vma, addr, mfn, nr, prot, - domid, pages); + return xen_xlate_remap_gfn_array(vma, addr, mfn, nr, err_ptr, + prot, domid, pages); #else return -EINVAL; #endif @@ -2573,9 +2492,15 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma, rmd.mfn = mfn; rmd.prot = prot; + /* We use the err_ptr to indicate if there we are doing a contigious + * mapping or a discontigious mapping. */ + rmd.contiguous = !err_ptr; while (nr) { - batch = min(REMAP_BATCH_SIZE, nr); + int index = 0; + int done = 0; + int batch = min(REMAP_BATCH_SIZE, nr); + int batch_left = batch; range = (unsigned long)batch << PAGE_SHIFT; rmd.mmu_update = mmu_update; @@ -2584,23 +2509,72 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma, if (err) goto out; - err = HYPERVISOR_mmu_update(mmu_update, batch, NULL, domid); - if (err < 0) - goto out; + /* We record the error for each page that gives an error, but + * continue mapping until the whole set is done */ + do { + int i; + + err = HYPERVISOR_mmu_update(&mmu_update[index], + batch_left, &done, domid); + + /* + * @err_ptr may be the same buffer as @mfn, so + * only clear it after each chunk of @mfn is + * used. + */ + if (err_ptr) { + for (i = index; i < index + done; i++) + err_ptr[i] = 0; + } + if (err < 0) { + if (!err_ptr) + goto out; + err_ptr[i] = err; + done++; /* Skip failed frame. */ + } else + mapped += done; + batch_left -= done; + index += done; + } while (batch_left); nr -= batch; addr += range; + if (err_ptr) + err_ptr += batch; } - - err = 0; out: xen_flush_tlb_all(); - return err; + return err < 0 ? err : mapped; +} + +int xen_remap_domain_mfn_range(struct vm_area_struct *vma, + unsigned long addr, + xen_pfn_t mfn, int nr, + pgprot_t prot, unsigned domid, + struct page **pages) +{ + return do_remap_mfn(vma, addr, &mfn, nr, NULL, prot, domid, pages); } EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_range); +int xen_remap_domain_mfn_array(struct vm_area_struct *vma, + unsigned long addr, + xen_pfn_t *mfn, int nr, + int *err_ptr, pgprot_t prot, + unsigned domid, struct page **pages) +{ + /* We BUG_ON because it's a programmer error to pass a NULL err_ptr, + * and the consequences later is quite hard to detect what the actual + * cause of "wrong memory was mapped in". + */ + BUG_ON(err_ptr == NULL); + return do_remap_mfn(vma, addr, mfn, nr, err_ptr, prot, domid, pages); +} +EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array); + + /* Returns: 0 success */ int xen_unmap_domain_mfn_range(struct vm_area_struct *vma, int numpgs, struct page **pages) @@ -2609,22 +2583,7 @@ int xen_unmap_domain_mfn_range(struct vm_area_struct *vma, return 0; #ifdef CONFIG_XEN_PVH - while (numpgs--) { - /* - * The mmu has already cleaned up the process mmu - * resources at this point (lookup_address will return - * NULL). - */ - unsigned long pfn = page_to_pfn(pages[numpgs]); - - xlate_remove_from_p2m(pfn, 1); - } - /* - * We don't need to flush tlbs because as part of - * xlate_remove_from_p2m, the hypervisor will do tlb flushes - * after removing the p2m entries from the EPT/NPT - */ - return 0; + return xen_xlate_unmap_gfn_range(vma, numpgs, pages); #else return -EINVAL; #endif diff --git a/arch/x86/xen/trace.c b/arch/x86/xen/trace.c index 520022d1a181..a702ec2f5931 100644 --- a/arch/x86/xen/trace.c +++ b/arch/x86/xen/trace.c @@ -1,54 +1,12 @@ #include <linux/ftrace.h> #include <xen/interface/xen.h> +#include <xen/interface/xen-mca.h> -#define N(x) [__HYPERVISOR_##x] = "("#x")" +#define HYPERCALL(x) [__HYPERVISOR_##x] = "("#x")", static const char *xen_hypercall_names[] = { - N(set_trap_table), - N(mmu_update), - N(set_gdt), - N(stack_switch), - N(set_callbacks), - N(fpu_taskswitch), - N(sched_op_compat), - N(dom0_op), - N(set_debugreg), - N(get_debugreg), - N(update_descriptor), - N(memory_op), - N(multicall), - N(update_va_mapping), - N(set_timer_op), - N(event_channel_op_compat), - N(xen_version), - N(console_io), - N(physdev_op_compat), - N(grant_table_op), - N(vm_assist), - N(update_va_mapping_otherdomain), - N(iret), - N(vcpu_op), - N(set_segment_base), - N(mmuext_op), - N(acm_op), - N(nmi_op), - N(sched_op), - N(callback_op), - N(xenoprof_op), - N(event_channel_op), - N(physdev_op), - N(hvm_op), - -/* Architecture-specific hypercall definitions. */ - N(arch_0), - N(arch_1), - N(arch_2), - N(arch_3), - N(arch_4), - N(arch_5), - N(arch_6), - N(arch_7), +#include <asm/xen-hypercalls.h> }; -#undef N +#undef HYPERCALL static const char *xen_hypercall_name(unsigned op) { diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S index 674b222544b7..8afdfccf6086 100644 --- a/arch/x86/xen/xen-head.S +++ b/arch/x86/xen/xen-head.S @@ -12,6 +12,8 @@ #include <xen/interface/elfnote.h> #include <xen/interface/features.h> +#include <xen/interface/xen.h> +#include <xen/interface/xen-mca.h> #include <asm/xen/interface.h> #ifdef CONFIG_XEN_PVH @@ -85,59 +87,14 @@ ENTRY(xen_pvh_early_cpu_init) .pushsection .text .balign PAGE_SIZE ENTRY(hypercall_page) -#define NEXT_HYPERCALL(x) \ - ENTRY(xen_hypercall_##x) \ - .skip 32 - -NEXT_HYPERCALL(set_trap_table) -NEXT_HYPERCALL(mmu_update) -NEXT_HYPERCALL(set_gdt) -NEXT_HYPERCALL(stack_switch) -NEXT_HYPERCALL(set_callbacks) -NEXT_HYPERCALL(fpu_taskswitch) -NEXT_HYPERCALL(sched_op_compat) -NEXT_HYPERCALL(platform_op) -NEXT_HYPERCALL(set_debugreg) -NEXT_HYPERCALL(get_debugreg) -NEXT_HYPERCALL(update_descriptor) -NEXT_HYPERCALL(ni) -NEXT_HYPERCALL(memory_op) -NEXT_HYPERCALL(multicall) -NEXT_HYPERCALL(update_va_mapping) -NEXT_HYPERCALL(set_timer_op) -NEXT_HYPERCALL(event_channel_op_compat) -NEXT_HYPERCALL(xen_version) -NEXT_HYPERCALL(console_io) -NEXT_HYPERCALL(physdev_op_compat) -NEXT_HYPERCALL(grant_table_op) -NEXT_HYPERCALL(vm_assist) -NEXT_HYPERCALL(update_va_mapping_otherdomain) -NEXT_HYPERCALL(iret) -NEXT_HYPERCALL(vcpu_op) -NEXT_HYPERCALL(set_segment_base) -NEXT_HYPERCALL(mmuext_op) -NEXT_HYPERCALL(xsm_op) -NEXT_HYPERCALL(nmi_op) -NEXT_HYPERCALL(sched_op) -NEXT_HYPERCALL(callback_op) -NEXT_HYPERCALL(xenoprof_op) -NEXT_HYPERCALL(event_channel_op) -NEXT_HYPERCALL(physdev_op) -NEXT_HYPERCALL(hvm_op) -NEXT_HYPERCALL(sysctl) -NEXT_HYPERCALL(domctl) -NEXT_HYPERCALL(kexec_op) -NEXT_HYPERCALL(tmem_op) /* 38 */ -ENTRY(xen_hypercall_rsvr) - .skip 320 -NEXT_HYPERCALL(mca) /* 48 */ -NEXT_HYPERCALL(arch_1) -NEXT_HYPERCALL(arch_2) -NEXT_HYPERCALL(arch_3) -NEXT_HYPERCALL(arch_4) -NEXT_HYPERCALL(arch_5) -NEXT_HYPERCALL(arch_6) - .balign PAGE_SIZE + .skip PAGE_SIZE + +#define HYPERCALL(n) \ + .equ xen_hypercall_##n, hypercall_page + __HYPERVISOR_##n * 32; \ + .type xen_hypercall_##n, @function; .size xen_hypercall_##n, 32 +#include <asm/xen-hypercalls.h> +#undef HYPERCALL + .popsection ELFNOTE(Xen, XEN_ELFNOTE_GUEST_OS, .asciz "linux") diff --git a/arch/xtensa/include/asm/thread_info.h b/arch/xtensa/include/asm/thread_info.h index a9b5d3ba196c..9ad12c617184 100644 --- a/arch/xtensa/include/asm/thread_info.h +++ b/arch/xtensa/include/asm/thread_info.h @@ -44,7 +44,6 @@ typedef struct xtregs_coprocessor { struct thread_info { struct task_struct *task; /* main task structure */ - struct exec_domain *exec_domain; /* execution domain */ unsigned long flags; /* low level flags */ unsigned long status; /* thread-synchronous flags */ __u32 cpu; /* current CPU */ @@ -61,17 +60,6 @@ struct thread_info { xtregs_user_t xtregs_user; }; -#else /* !__ASSEMBLY__ */ - -/* offsets into the thread_info struct for assembly code access */ -#define TI_TASK 0x00000000 -#define TI_EXEC_DOMAIN 0x00000004 -#define TI_FLAGS 0x00000008 -#define TI_STATUS 0x0000000C -#define TI_CPU 0x00000010 -#define TI_PRE_COUNT 0x00000014 -#define TI_ADDR_LIMIT 0x00000018 - #endif /* @@ -83,7 +71,6 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .exec_domain = &default_exec_domain, \ .flags = 0, \ .cpu = 0, \ .preempt_count = INIT_PREEMPT_COUNT, \ diff --git a/arch/xtensa/kernel/asm-offsets.c b/arch/xtensa/kernel/asm-offsets.c index 1915c7c889ba..b123ace3b67c 100644 --- a/arch/xtensa/kernel/asm-offsets.c +++ b/arch/xtensa/kernel/asm-offsets.c @@ -77,6 +77,14 @@ int main(void) DEFINE(TASK_THREAD_INFO, offsetof (struct task_struct, stack)); DEFINE(TASK_STRUCT_SIZE, sizeof (struct task_struct)); + /* offsets in thread_info struct */ + OFFSET(TI_TASK, thread_info, task); + OFFSET(TI_FLAGS, thread_info, flags); + OFFSET(TI_STSTUS, thread_info, status); + OFFSET(TI_CPU, thread_info, cpu); + OFFSET(TI_PRE_COUNT, thread_info, preempt_count); + OFFSET(TI_ADDR_LIMIT, thread_info, addr_limit); + /* struct thread_info (offset from start_struct) */ DEFINE(THREAD_RA, offsetof (struct task_struct, thread.ra)); DEFINE(THREAD_SP, offsetof (struct task_struct, thread.sp)); diff --git a/arch/xtensa/kernel/signal.c b/arch/xtensa/kernel/signal.c index 6b3790445cbe..e87adaa07ff3 100644 --- a/arch/xtensa/kernel/signal.c +++ b/arch/xtensa/kernel/signal.c @@ -336,7 +336,6 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, { struct rt_sigframe *frame; int err = 0, sig = ksig->sig; - int signal; unsigned long sp, ra, tp; sp = regs->areg[1]; @@ -354,12 +353,6 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, return -EFAULT; } - signal = current_thread_info()->exec_domain - && current_thread_info()->exec_domain->signal_invmap - && sig < 32 - ? current_thread_info()->exec_domain->signal_invmap[sig] - : sig; - if (ksig->ka.sa.sa_flags & SA_SIGINFO) { err |= copy_siginfo_to_user(&frame->info, &ksig->info); } @@ -400,14 +393,14 @@ static int setup_frame(struct ksignal *ksig, sigset_t *set, * Note: PS.CALLINC is set to one by start_thread */ regs->areg[4] = (((unsigned long) ra) & 0x3fffffff) | 0x40000000; - regs->areg[6] = (unsigned long) signal; + regs->areg[6] = (unsigned long) sig; regs->areg[7] = (unsigned long) &frame->info; regs->areg[8] = (unsigned long) &frame->uc; regs->threadptr = tp; #if DEBUG_SIG printk("SIG rt deliver (%s:%d): signal=%d sp=%p pc=%08x\n", - current->comm, current->pid, signal, frame, regs->pc); + current->comm, current->pid, sig, frame, regs->pc); #endif return 0; |