diff options
Diffstat (limited to 'arch/powerpc')
366 files changed, 5287 insertions, 18010 deletions
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index a0ce777f9706..93402a1d9c9f 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -145,7 +145,8 @@ config PPC select ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE select ARCH_HAS_PHYS_TO_DMA select ARCH_HAS_PMEM_API - select ARCH_HAS_PTE_DEVMAP if PPC_BOOK3S_64 + select ARCH_HAS_PREEMPT_LAZY + select ARCH_HAS_PTDUMP select ARCH_HAS_PTE_SPECIAL select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE && PPC_BOOK3S_64 select ARCH_HAS_SET_MEMORY @@ -156,6 +157,7 @@ config PPC select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST select ARCH_HAS_UACCESS_FLUSHCACHE select ARCH_HAS_UBSAN + select ARCH_HAS_VDSO_ARCH_DATA select ARCH_HAVE_NMI_SAFE_CMPXCHG select ARCH_HAVE_EXTRA_ELF_NOTES if SPU_BASE select ARCH_KEEP_MEMBLOCK @@ -203,9 +205,9 @@ config PPC select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL select GENERIC_PCI_IOMAP if PCI - select GENERIC_PTDUMP select GENERIC_SMP_IDLE_THREAD select GENERIC_TIME_VSYSCALL + select GENERIC_VDSO_DATA_STORE select GENERIC_VDSO_TIME_NS select HAS_IOPORT if PCI select HAVE_ARCH_AUDITSYSCALL @@ -240,7 +242,7 @@ config PPC select HAVE_EBPF_JIT select HAVE_EFFICIENT_UNALIGNED_ACCESS select HAVE_GUP_FAST - select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FTRACE_GRAPH_FUNC select HAVE_FUNCTION_ARG_ACCESS_API select HAVE_FUNCTION_DESCRIPTORS if PPC64_ELF_ABI_V1 select HAVE_FUNCTION_ERROR_INJECTION @@ -271,6 +273,7 @@ config PPC select HAVE_PERF_EVENTS_NMI if PPC64 select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP + select HAVE_PREEMPT_DYNAMIC_KEY select HAVE_RETHOOK if KPROBES select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_RELIABLE_STACKTRACE @@ -282,6 +285,7 @@ config PPC select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,$(m32-flag) -mstack-protector-guard=tls -mstack-protector-guard-reg=r2 -mstack-protector-guard-offset=0) select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,$(m64-flag) -mstack-protector-guard=tls -mstack-protector-guard-reg=r13 -mstack-protector-guard-offset=0) select HAVE_STATIC_CALL if PPC32 + select HAVE_STATIC_CALL_INLINE if PPC32 select HAVE_SYSCALL_TRACEPOINTS select HAVE_VIRT_CPU_ACCOUNTING select HAVE_VIRT_CPU_ACCOUNTING_GEN @@ -407,12 +411,9 @@ config ARCH_HAS_ADD_PAGES config PPC_DCR_NATIVE bool -config PPC_DCR_MMIO - bool - config PPC_DCR bool - depends on PPC_DCR_NATIVE || PPC_DCR_MMIO + depends on PPC_DCR_NATIVE default y config PPC_PCI_OF_BUS_MAP @@ -438,11 +439,6 @@ config PPC_PCI_BUS_NUM_DOMAIN_DEPENDENT PCI domain dependent and each PCI controller on own domain can have 256 PCI buses, like it is on other Linux architectures. -config PPC_OF_PLATFORM_PCI - bool - depends on PCI - depends on PPC64 # not supported on 32 bits yet - config ARCH_SUPPORTS_UPROBES def_bool y @@ -717,6 +713,9 @@ config ARCH_SUPPORTS_CRASH_HOTPLUG def_bool y depends on PPC64 +config ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION + def_bool CRASH_RESERVE + config FA_DUMP bool "Firmware-assisted dump" depends on CRASH_DUMP && PPC64 && (PPC_RTAS || PPC_POWERNV) @@ -892,7 +891,7 @@ config DATA_SHIFT int "Data shift" if DATA_SHIFT_BOOL default 24 if STRICT_KERNEL_RWX && PPC64 range 17 28 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 - range 19 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx + range 14 23 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_8xx range 20 24 if (STRICT_KERNEL_RWX || DEBUG_PAGEALLOC || KFENCE) && PPC_85xx default 22 if STRICT_KERNEL_RWX && PPC_BOOK3S_32 default 18 if (DEBUG_PAGEALLOC || KFENCE) && PPC_BOOK3S_32 @@ -905,10 +904,10 @@ config DATA_SHIFT On Book3S 32 (603+), DBATs are used to map kernel text and rodata RO. Smaller is the alignment, greater is the number of necessary DBATs. - On 8xx, large pages (512kb or 8M) are used to map kernel linear - memory. Aligning to 8M reduces TLB misses as only 8M pages are used - in that case. If PIN_TLB is selected, it must be aligned to 8M as - 8M pages will be pinned. + On 8xx, large pages (16kb or 512kb or 8M) are used to map kernel + linear memory. Aligning to 8M reduces TLB misses as only 8M pages + are used in that case. If PIN_TLB is selected, it must be aligned + to 8M as 8M pages will be pinned. config ARCH_FORCE_MAX_ORDER int "Order of maximal physically contiguous allocations" diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index 20d05605fa83..f15e5920080b 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -216,13 +216,6 @@ config PPC_EARLY_DEBUG_RTAS_PANEL help Select this to enable early debugging via the RTAS panel. -config PPC_EARLY_DEBUG_RTAS_CONSOLE - bool "RTAS Console" - depends on PPC_RTAS - select UDBG_RTAS_CONSOLE - help - Select this to enable early debugging via the RTAS console. - config PPC_EARLY_DEBUG_PAS_REALMODE bool "PA Semi real mode" depends on PPC_PASEMI diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index f3804103c56c..9753fb87217c 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -101,7 +101,7 @@ KBUILD_LDFLAGS += -m elf$(BITS)$(LDEMULATION) endif LDFLAGS_vmlinux-y := -Bstatic -LDFLAGS_vmlinux-$(CONFIG_RELOCATABLE) := -pie +LDFLAGS_vmlinux-$(CONFIG_RELOCATABLE) := -pie --no-dynamic-linker LDFLAGS_vmlinux-$(CONFIG_RELOCATABLE) += -z notext LDFLAGS_vmlinux := $(LDFLAGS_vmlinux-y) diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index 1ff6ad4f6cd2..a7ab087d412c 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -70,6 +70,7 @@ BOOTCPPFLAGS := -nostdinc $(LINUXINCLUDE) BOOTCPPFLAGS += -isystem $(shell $(BOOTCC) -print-file-name=include) BOOTCFLAGS := $(BOOTTARGETFLAGS) \ + -std=gnu11 \ -Wall -Wundef -Wstrict-prototypes -Wno-trigraphs \ -fno-strict-aliasing -O2 \ -msoft-float -mno-altivec -mno-vsx \ @@ -173,7 +174,6 @@ src-plat-$(CONFIG_PPC_PS3) += ps3-head.S ps3-hvcall.S ps3.c src-plat-$(CONFIG_EPAPR_BOOT) += epapr.c epapr-wrapper.c src-plat-$(CONFIG_PPC_PSERIES) += pseries-head.S src-plat-$(CONFIG_PPC_POWERNV) += pseries-head.S -src-plat-$(CONFIG_PPC_IBM_CELL_BLADE) += pseries-head.S src-plat-$(CONFIG_MVME7100) += motload-head.S mvme7100.c src-plat-$(CONFIG_PPC_MICROWATT) += fixed-head.S microwatt.c @@ -276,7 +276,6 @@ quiet_cmd_wrap = WRAP $@ image-$(CONFIG_PPC_PSERIES) += zImage.pseries image-$(CONFIG_PPC_POWERNV) += zImage.pseries -image-$(CONFIG_PPC_IBM_CELL_BLADE) += zImage.pseries image-$(CONFIG_PPC_PS3) += dtbImage.ps3 image-$(CONFIG_PPC_CHRP) += zImage.chrp image-$(CONFIG_PPC_EFIKA) += zImage.chrp diff --git a/arch/powerpc/boot/dts/microwatt.dts b/arch/powerpc/boot/dts/microwatt.dts index 269e930b3b0b..292b909ca9ce 100644 --- a/arch/powerpc/boot/dts/microwatt.dts +++ b/arch/powerpc/boot/dts/microwatt.dts @@ -1,13 +1,15 @@ /dts-v1/; +#include <dt-bindings/gpio/gpio.h> / { #size-cells = <0x02>; #address-cells = <0x02>; - model-name = "microwatt"; + model = "microwatt"; compatible = "microwatt-soc"; aliases { serial0 = &UART0; + ethernet = &enet0; }; reserved-memory { @@ -35,40 +37,79 @@ ibm,powerpc-cpu-features { display-name = "Microwatt"; - isa = <3000>; + isa = <3100>; device_type = "cpu-features"; compatible = "ibm,powerpc-cpu-features"; mmu-radix { isa = <3000>; - usable-privilege = <2>; + usable-privilege = <6>; + os-support = <0>; }; little-endian { - isa = <2050>; - usable-privilege = <3>; + isa = <0>; + usable-privilege = <7>; + os-support = <0>; hwcap-bit-nr = <1>; }; cache-inhibited-large-page { - isa = <2040>; - usable-privilege = <2>; + isa = <0>; + usable-privilege = <6>; + os-support = <0>; }; fixed-point-v3 { isa = <3000>; - usable-privilege = <3>; + usable-privilege = <7>; }; no-execute { - isa = <2010>; + isa = <0x00>; usable-privilege = <2>; + os-support = <0>; }; floating-point { + hfscr-bit-nr = <0>; hwcap-bit-nr = <27>; isa = <0>; - usable-privilege = <3>; + usable-privilege = <7>; + hv-support = <1>; + os-support = <0>; + }; + + prefixed-instructions { + hfscr-bit-nr = <13>; + fscr-bit-nr = <13>; + isa = <3010>; + usable-privilege = <7>; + os-support = <1>; + hv-support = <1>; + }; + + tar { + hfscr-bit-nr = <8>; + fscr-bit-nr = <8>; + isa = <2070>; + usable-privilege = <7>; + os-support = <1>; + hv-support = <1>; + hwcap-bit-nr = <58>; + }; + + control-register { + isa = <0>; + usable-privilege = <7>; + }; + + system-call-vectored { + isa = <3000>; + usable-privilege = <7>; + os-support = <1>; + fscr-bit-nr = <12>; + hwcap-bit-nr = <52>; }; }; @@ -101,6 +142,36 @@ ibm,mmu-lpid-bits = <12>; ibm,mmu-pid-bits = <20>; }; + + PowerPC,Microwatt@1 { + i-cache-sets = <2>; + ibm,dec-bits = <64>; + reservation-granule-size = <64>; + clock-frequency = <100000000>; + timebase-frequency = <100000000>; + i-tlb-sets = <1>; + ibm,ppc-interrupt-server#s = <1>; + i-cache-block-size = <64>; + d-cache-block-size = <64>; + d-cache-sets = <2>; + i-tlb-size = <64>; + cpu-version = <0x990000>; + status = "okay"; + i-cache-size = <0x1000>; + ibm,processor-radix-AP-encodings = <0x0c 0xa0000010 0x20000015 0x4000001e>; + tlb-size = <0>; + tlb-sets = <0>; + device_type = "cpu"; + d-tlb-size = <128>; + d-tlb-sets = <2>; + reg = <1>; + general-purpose; + 64-bit; + d-cache-size = <0x1000>; + ibm,chip-id = <0>; + ibm,mmu-lpid-bits = <12>; + ibm,mmu-pid-bits = <20>; + }; }; soc@c0000000 { @@ -113,8 +184,8 @@ interrupt-controller@4000 { compatible = "openpower,xics-presentation", "ibm,ppc-xicp"; - ibm,interrupt-server-ranges = <0x0 0x1>; - reg = <0x4000 0x100>; + ibm,interrupt-server-ranges = <0x0 0x2>; + reg = <0x4000 0x10 0x4010 0x10>; }; ICS: interrupt-controller@5000 { @@ -138,7 +209,18 @@ interrupts = <0x10 0x1>; }; - ethernet@8020000 { + gpio: gpio@7000 { + device_type = "gpio"; + compatible = "faraday,ftgpio010"; + gpio-controller; + #gpio-cells = <2>; + reg = <0x7000 0x80>; + interrupts = <0x14 1>; + interrupt-controller; + #interrupt-cells = <2>; + }; + + enet0: ethernet@8020000 { compatible = "litex,liteeth"; reg = <0x8021000 0x100 0x8020800 0x100 @@ -160,7 +242,6 @@ reg-names = "phy", "core", "reader", "writer", "irq"; bus-width = <4>; interrupts = <0x13 1>; - cap-sd-highspeed; clocks = <&sys_clk>; }; }; diff --git a/arch/powerpc/boot/dts/mpc8315erdb.dts b/arch/powerpc/boot/dts/mpc8315erdb.dts index e09b37d7489d..a89cb3139ca8 100644 --- a/arch/powerpc/boot/dts/mpc8315erdb.dts +++ b/arch/powerpc/boot/dts/mpc8315erdb.dts @@ -6,6 +6,7 @@ */ /dts-v1/; +#include <dt-bindings/interrupt-controller/irq.h> / { compatible = "fsl,mpc8315erdb"; @@ -358,6 +359,15 @@ interrupt-parent = <&ipic>; fsl,mpc8313-wakeup-timer = <>m1>; }; + + gpio: gpio-controller@c00 { + compatible = "fsl,mpc8314-gpio"; + reg = <0xc00 0x100>; + interrupts = <74 IRQ_TYPE_LEVEL_LOW>; + interrupt-parent = <&ipic>; + gpio-controller; + #gpio-cells = <2>; + }; }; pci0: pci@e0008500 { diff --git a/arch/powerpc/boot/rs6000.h b/arch/powerpc/boot/rs6000.h index a9d879155ef9..16df8f3c43f1 100644 --- a/arch/powerpc/boot/rs6000.h +++ b/arch/powerpc/boot/rs6000.h @@ -1,11 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* IBM RS/6000 "XCOFF" file definitions for BFD. Copyright (C) 1990, 1991 Free Software Foundation, Inc. - FIXME: Can someone provide a transliteration of this name into ASCII? - Using the following chars caused a compiler warning on HIUX (so I replaced - them with octal escapes), and isn't useful without an understanding of what - character set it is. - Written by Mimi Ph\373\364ng-Th\345o V\365 of IBM + Written by Mimi Phuong-Thao Vo of IBM and John Gilmore of Cygnus Support. */ /********************** FILE HEADER **********************/ diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper index 1db60fe13802..3d8dc822282a 100755 --- a/arch/powerpc/boot/wrapper +++ b/arch/powerpc/boot/wrapper @@ -234,10 +234,8 @@ fi # suppress some warnings in recent ld versions nowarn="-z noexecstack" -if ! ld_is_lld; then - if [ "$LD_VERSION" -ge "$(echo 2.39 | ld_version)" ]; then - nowarn="$nowarn --no-warn-rwx-segments" - fi +if "${CROSS}ld" -v --no-warn-rwx-segments >/dev/null 2>&1; then + nowarn="$nowarn --no-warn-rwx-segments" fi platformo=$object/"$platform".o diff --git a/arch/powerpc/configs/44x/akebono_defconfig b/arch/powerpc/configs/44x/akebono_defconfig index fde4824f235e..1882eb2da354 100644 --- a/arch/powerpc/configs/44x/akebono_defconfig +++ b/arch/powerpc/configs/44x/akebono_defconfig @@ -128,6 +128,5 @@ CONFIG_PPC_EARLY_DEBUG_44x_PHYSLOW=0x00010000 CONFIG_PPC_EARLY_DEBUG_44x_PHYSHIGH=0x33f CONFIG_CRYPTO_PCBC=y CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_SHA1_PPC=y CONFIG_CRYPTO_DES=y # CONFIG_CRYPTO_HW is not set diff --git a/arch/powerpc/configs/44x/sam440ep_defconfig b/arch/powerpc/configs/44x/sam440ep_defconfig index 2479ab62d12f..98221bda380d 100644 --- a/arch/powerpc/configs/44x/sam440ep_defconfig +++ b/arch/powerpc/configs/44x/sam440ep_defconfig @@ -91,5 +91,4 @@ CONFIG_AFFS_FS=m # CONFIG_NETWORK_FILESYSTEMS is not set CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_T10DIF=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/powerpc/configs/44x/warp_defconfig b/arch/powerpc/configs/44x/warp_defconfig index 20891c413149..5757625469c4 100644 --- a/arch/powerpc/configs/44x/warp_defconfig +++ b/arch/powerpc/configs/44x/warp_defconfig @@ -85,8 +85,6 @@ CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_15=y CONFIG_NLS_UTF8=y -CONFIG_CRC_CCITT=y -CONFIG_CRC_T10DIF=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_FS=y diff --git a/arch/powerpc/configs/83xx/mpc832x_rdb_defconfig b/arch/powerpc/configs/83xx/mpc832x_rdb_defconfig index 1715ff547442..b99caba8724a 100644 --- a/arch/powerpc/configs/83xx/mpc832x_rdb_defconfig +++ b/arch/powerpc/configs/83xx/mpc832x_rdb_defconfig @@ -73,6 +73,5 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_932=y CONFIG_NLS_ISO8859_8=y CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_T10DIF=y CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/83xx/mpc834x_itx_defconfig b/arch/powerpc/configs/83xx/mpc834x_itx_defconfig index e65c0057147f..11163052fdba 100644 --- a/arch/powerpc/configs/83xx/mpc834x_itx_defconfig +++ b/arch/powerpc/configs/83xx/mpc834x_itx_defconfig @@ -80,5 +80,4 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y -CONFIG_CRC_T10DIF=y CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/83xx/mpc834x_itxgp_defconfig b/arch/powerpc/configs/83xx/mpc834x_itxgp_defconfig index 17714bf0ed40..312d39e4242c 100644 --- a/arch/powerpc/configs/83xx/mpc834x_itxgp_defconfig +++ b/arch/powerpc/configs/83xx/mpc834x_itxgp_defconfig @@ -72,5 +72,4 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y -CONFIG_CRC_T10DIF=y CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig b/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig index 58fae5131fa7..ac27f99faab8 100644 --- a/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig +++ b/arch/powerpc/configs/83xx/mpc837x_rdb_defconfig @@ -75,6 +75,5 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y -CONFIG_CRC_T10DIF=y CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/85xx/ge_imp3a_defconfig b/arch/powerpc/configs/85xx/ge_imp3a_defconfig index da6fc203e2dc..7beb36a41d45 100644 --- a/arch/powerpc/configs/85xx/ge_imp3a_defconfig +++ b/arch/powerpc/configs/85xx/ge_imp3a_defconfig @@ -221,9 +221,6 @@ CONFIG_NLS_ISO8859_15=y CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_NLS_UTF8=y -CONFIG_CRC_CCITT=y -CONFIG_CRC_T10DIF=y -CONFIG_LIBCRC32C=y CONFIG_MAGIC_SYSRQ=y CONFIG_CRYPTO_CBC=y CONFIG_CRYPTO_MD5=y diff --git a/arch/powerpc/configs/85xx/stx_gp3_defconfig b/arch/powerpc/configs/85xx/stx_gp3_defconfig index e7080497048d..0a42072fa23c 100644 --- a/arch/powerpc/configs/85xx/stx_gp3_defconfig +++ b/arch/powerpc/configs/85xx/stx_gp3_defconfig @@ -60,8 +60,6 @@ CONFIG_CRAMFS=m CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NLS=y -CONFIG_CRC_CCITT=y -CONFIG_CRC_T10DIF=m CONFIG_DETECT_HUNG_TASK=y # CONFIG_DEBUG_BUGVERBOSE is not set CONFIG_BDI_SWITCH=y diff --git a/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig b/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig index 3a6381aa9fdc..488d03ae6d6c 100644 --- a/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig +++ b/arch/powerpc/configs/85xx/xes_mpc85xx_defconfig @@ -132,7 +132,6 @@ CONFIG_ROOT_NFS=y CONFIG_NFSD=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_T10DIF=y CONFIG_DETECT_HUNG_TASK=y # CONFIG_DEBUG_BUGVERBOSE is not set CONFIG_CRYPTO_HMAC=y diff --git a/arch/powerpc/configs/86xx-hw.config b/arch/powerpc/configs/86xx-hw.config index 0cb24b33c88e..e7bd265fae5a 100644 --- a/arch/powerpc/configs/86xx-hw.config +++ b/arch/powerpc/configs/86xx-hw.config @@ -5,7 +5,6 @@ CONFIG_BROADCOM_PHY=y # CONFIG_CARDBUS is not set CONFIG_CHR_DEV_SG=y CONFIG_CHR_DEV_ST=y -CONFIG_CRC_T10DIF=y CONFIG_CRYPTO_HMAC=y CONFIG_DS1682=y CONFIG_EEPROM_LEGACY=y diff --git a/arch/powerpc/configs/adder875_defconfig b/arch/powerpc/configs/adder875_defconfig index 97f4d4851735..3c6445c98a85 100644 --- a/arch/powerpc/configs/adder875_defconfig +++ b/arch/powerpc/configs/adder875_defconfig @@ -44,7 +44,6 @@ CONFIG_TMPFS=y CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y -CONFIG_CRC32_SLICEBY4=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_DEBUG_FS=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/powerpc/configs/amigaone_defconfig b/arch/powerpc/configs/amigaone_defconfig index 200bb1ecb560..69ef3dc31c4b 100644 --- a/arch/powerpc/configs/amigaone_defconfig +++ b/arch/powerpc/configs/amigaone_defconfig @@ -106,7 +106,6 @@ CONFIG_TMPFS=y CONFIG_AFFS_FS=m CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=m -CONFIG_CRC_T10DIF=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_MUTEXES=y diff --git a/arch/powerpc/configs/cell_defconfig b/arch/powerpc/configs/cell_defconfig index 53f43a34e1a9..7a31b52e92e1 100644 --- a/arch/powerpc/configs/cell_defconfig +++ b/arch/powerpc/configs/cell_defconfig @@ -25,7 +25,6 @@ CONFIG_PS3_DISK=y CONFIG_PS3_ROM=m CONFIG_PS3_FLASH=m CONFIG_PS3_LPM=m -CONFIG_PPC_IBM_CELL_BLADE=y CONFIG_RTAS_FLASH=y CONFIG_CPU_FREQ=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y @@ -63,7 +62,6 @@ CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m CONFIG_NETFILTER_XT_TARGET_TCPMSS=m CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m CONFIG_NETFILTER_XT_MATCH_DSCP=m CONFIG_NETFILTER_XT_MATCH_ESP=m CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m @@ -133,7 +131,6 @@ CONFIG_SKGE=m CONFIG_SKY2=m CONFIG_GELIC_NET=m CONFIG_GELIC_WIRELESS=y -CONFIG_SPIDER_NET=y # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set # CONFIG_SERIO_I8042 is not set @@ -168,7 +165,6 @@ CONFIG_INFINIBAND_MTHCA=m CONFIG_INFINIBAND_IPOIB=m CONFIG_INFINIBAND_IPOIB_DEBUG_DATA=y CONFIG_EDAC=y -CONFIG_EDAC_CELL=y CONFIG_UIO=m CONFIG_EXT2_FS=y CONFIG_EXT4_FS=y diff --git a/arch/powerpc/configs/chrp32_defconfig b/arch/powerpc/configs/chrp32_defconfig index fb314f75ad4b..b799c95480ae 100644 --- a/arch/powerpc/configs/chrp32_defconfig +++ b/arch/powerpc/configs/chrp32_defconfig @@ -110,7 +110,6 @@ CONFIG_PROC_KCORE=y CONFIG_TMPFS=y CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=m -CONFIG_CRC_T10DIF=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_MUTEXES=y diff --git a/arch/powerpc/configs/ep88xc_defconfig b/arch/powerpc/configs/ep88xc_defconfig index 50cc59eb36cf..354180ab94bc 100644 --- a/arch/powerpc/configs/ep88xc_defconfig +++ b/arch/powerpc/configs/ep88xc_defconfig @@ -47,7 +47,6 @@ CONFIG_TMPFS=y CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y -CONFIG_CRC32_SLICEBY4=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DETECT_HUNG_TASK=y diff --git a/arch/powerpc/configs/fsl-emb-nonhw.config b/arch/powerpc/configs/fsl-emb-nonhw.config index 3009b0efaf34..2f81bc2d819e 100644 --- a/arch/powerpc/configs/fsl-emb-nonhw.config +++ b/arch/powerpc/configs/fsl-emb-nonhw.config @@ -15,7 +15,6 @@ CONFIG_CGROUP_CPUACCT=y CONFIG_CGROUP_SCHED=y CONFIG_CGROUPS=y # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_CRC_T10DIF=y CONFIG_CPUSETS=y CONFIG_CRAMFS=y CONFIG_CRYPTO_MD4=y @@ -112,7 +111,6 @@ CONFIG_QNX4FS_FS=m CONFIG_RCU_TRACE=y CONFIG_RESET_CONTROLLER=y CONFIG_ROOT_NFS=y -CONFIG_SYSV_FS=m CONFIG_SYSVIPC=y CONFIG_TMPFS=y CONFIG_UBIFS_FS=y diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig index 9215bed53291..428f17b45513 100644 --- a/arch/powerpc/configs/g5_defconfig +++ b/arch/powerpc/configs/g5_defconfig @@ -231,12 +231,11 @@ CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_ISO8859_15=y CONFIG_NLS_UTF8=y -CONFIG_CRC_T10DIF=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_MUTEXES=y CONFIG_BOOTX_TEXT=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m diff --git a/arch/powerpc/configs/gamecube_defconfig b/arch/powerpc/configs/gamecube_defconfig index d77eeb525366..cdd99657b71b 100644 --- a/arch/powerpc/configs/gamecube_defconfig +++ b/arch/powerpc/configs/gamecube_defconfig @@ -82,7 +82,6 @@ CONFIG_ROOT_NFS=y CONFIG_CIFS=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_CCITT=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_SPINLOCK=y CONFIG_DEBUG_MUTEXES=y diff --git a/arch/powerpc/configs/linkstation_defconfig b/arch/powerpc/configs/linkstation_defconfig index fa707de761be..b564f9e33a0d 100644 --- a/arch/powerpc/configs/linkstation_defconfig +++ b/arch/powerpc/configs/linkstation_defconfig @@ -125,8 +125,6 @@ CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_932=m CONFIG_NLS_ISO8859_1=m CONFIG_NLS_UTF8=m -CONFIG_CRC_CCITT=m -CONFIG_CRC_T10DIF=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y diff --git a/arch/powerpc/configs/mpc83xx_defconfig b/arch/powerpc/configs/mpc83xx_defconfig index 83c4710017e9..a815d9e5e3e8 100644 --- a/arch/powerpc/configs/mpc83xx_defconfig +++ b/arch/powerpc/configs/mpc83xx_defconfig @@ -97,7 +97,6 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=y CONFIG_NFS_V4=y CONFIG_ROOT_NFS=y -CONFIG_CRC_T10DIF=y CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_SHA512=y diff --git a/arch/powerpc/configs/mpc866_ads_defconfig b/arch/powerpc/configs/mpc866_ads_defconfig index 6f449411abf7..dfbdd5e8e108 100644 --- a/arch/powerpc/configs/mpc866_ads_defconfig +++ b/arch/powerpc/configs/mpc866_ads_defconfig @@ -38,5 +38,3 @@ CONFIG_TMPFS=y CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y -CONFIG_CRC_CCITT=y -CONFIG_CRC32_SLICEBY4=y diff --git a/arch/powerpc/configs/mpc885_ads_defconfig b/arch/powerpc/configs/mpc885_ads_defconfig index 77306be62e9e..9bc2758a6a9a 100644 --- a/arch/powerpc/configs/mpc885_ads_defconfig +++ b/arch/powerpc/configs/mpc885_ads_defconfig @@ -70,7 +70,6 @@ CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_CRYPTO=y CONFIG_CRYPTO_DEV_TALITOS=y -CONFIG_CRC32_SLICEBY4=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_FS=y @@ -78,4 +77,4 @@ CONFIG_DEBUG_VM_PGTABLE=y CONFIG_DETECT_HUNG_TASK=y CONFIG_BDI_SWITCH=y CONFIG_PPC_EARLY_DEBUG=y -CONFIG_GENERIC_PTDUMP=y +CONFIG_PTDUMP_DEBUGFS=y diff --git a/arch/powerpc/configs/mvme5100_defconfig b/arch/powerpc/configs/mvme5100_defconfig index d1c7fd5bf34b..fa2b3b9c5945 100644 --- a/arch/powerpc/configs/mvme5100_defconfig +++ b/arch/powerpc/configs/mvme5100_defconfig @@ -107,8 +107,6 @@ CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_CODEPAGE_932=m CONFIG_NLS_ISO8859_1=m CONFIG_NLS_UTF8=m -CONFIG_CRC_CCITT=m -CONFIG_CRC_T10DIF=y CONFIG_XZ_DEC=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y diff --git a/arch/powerpc/configs/pasemi_defconfig b/arch/powerpc/configs/pasemi_defconfig index 61993944db40..8bbf51b38480 100644 --- a/arch/powerpc/configs/pasemi_defconfig +++ b/arch/powerpc/configs/pasemi_defconfig @@ -159,7 +159,6 @@ CONFIG_NFSD=y CONFIG_NFSD_V4=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_CCITT=y CONFIG_PRINTK_TIME=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y diff --git a/arch/powerpc/configs/pmac32_defconfig b/arch/powerpc/configs/pmac32_defconfig index e8b3f67bf3f5..ae45f70b29f0 100644 --- a/arch/powerpc/configs/pmac32_defconfig +++ b/arch/powerpc/configs/pmac32_defconfig @@ -87,7 +87,6 @@ CONFIG_IP_NF_RAW=m CONFIG_IP_NF_ARPTABLES=m CONFIG_IP_NF_ARPFILTER=m CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_IP_DCCP=m CONFIG_BT=m CONFIG_BT_RFCOMM=m CONFIG_BT_RFCOMM_TTY=y @@ -276,7 +275,6 @@ CONFIG_NFSD_V3_ACL=y CONFIG_NFSD_V4=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_ISO8859_1=m -CONFIG_CRC_T10DIF=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DETECT_HUNG_TASK=y diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index ee84ade7a033..d06388b0f66e 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -46,7 +46,7 @@ CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y CONFIG_CPU_IDLE=y -CONFIG_HZ_100=y +CONFIG_HZ_1000=y CONFIG_BINFMT_MISC=m CONFIG_PPC_TRANSACTIONAL_MEM=y CONFIG_PPC_UV=y @@ -317,14 +317,11 @@ CONFIG_CODE_PATCHING_SELFTEST=y CONFIG_FTR_FIXUP_SELFTEST=y CONFIG_MSI_BITMAP_SELFTEST=y CONFIG_XMON=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_CRC32C_VPMSUM=m -CONFIG_CRYPTO_CRCT10DIF_VPMSUM=m CONFIG_CRYPTO_MD5_PPC=m CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_SHA1_PPC=m CONFIG_CRYPTO_SHA256=y CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_ANUBIS=m @@ -343,3 +340,4 @@ CONFIG_KVM_BOOK3S_64_HV=m CONFIG_VHOST_NET=m CONFIG_PRINTK_TIME=y CONFIG_PRINTK_CALLER=y +CONFIG_KALLSYMS_ALL=y diff --git a/arch/powerpc/configs/ppc44x_defconfig b/arch/powerpc/configs/ppc44x_defconfig index 8b595f67068c..41c930f74ed4 100644 --- a/arch/powerpc/configs/ppc44x_defconfig +++ b/arch/powerpc/configs/ppc44x_defconfig @@ -90,7 +90,6 @@ CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_NLS_CODEPAGE_437=m CONFIG_NLS_ISO8859_1=m -CONFIG_CRC_T10DIF=m CONFIG_MAGIC_SYSRQ=y CONFIG_DETECT_HUNG_TASK=y CONFIG_CRYPTO_ECB=y diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index f39c0d000c43..ce34597e9f3e 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -51,14 +51,13 @@ CONFIG_PS3_DISK=m CONFIG_PS3_ROM=m CONFIG_PS3_FLASH=m CONFIG_PS3_LPM=m -CONFIG_PPC_IBM_CELL_BLADE=y CONFIG_RTAS_FLASH=m CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND=y CONFIG_CPU_FREQ_GOV_POWERSAVE=y CONFIG_CPU_FREQ_GOV_USERSPACE=y CONFIG_CPU_FREQ_GOV_CONSERVATIVE=y CONFIG_CPU_FREQ_PMAC64=y -CONFIG_HZ_100=y +CONFIG_HZ_1000=y CONFIG_PPC_TRANSACTIONAL_MEM=y CONFIG_KEXEC=y CONFIG_KEXEC_FILE=y @@ -228,7 +227,6 @@ CONFIG_NETXEN_NIC=m CONFIG_SUNGEM=y CONFIG_GELIC_NET=m CONFIG_GELIC_WIRELESS=y -CONFIG_SPIDER_NET=m CONFIG_BROADCOM_PHY=m CONFIG_MARVELL_PHY=y CONFIG_PPP=m @@ -379,7 +377,7 @@ CONFIG_IMA_WRITE_POLICY=y CONFIG_IMA_APPRAISE=y CONFIG_IMA_ARCH_POLICY=y CONFIG_IMA_APPRAISE_MODSIG=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_SERPENT=m @@ -389,11 +387,7 @@ CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_SHA256=y CONFIG_CRYPTO_WP512=m CONFIG_CRYPTO_LZO=m -CONFIG_CRYPTO_CRC32C_VPMSUM=m -CONFIG_CRYPTO_CRCT10DIF_VPMSUM=m -CONFIG_CRYPTO_VPMSUM_TESTER=m CONFIG_CRYPTO_MD5_PPC=m -CONFIG_CRYPTO_SHA1_PPC=m CONFIG_CRYPTO_AES_GCM_P10=m CONFIG_CRYPTO_DEV_NX=y CONFIG_CRYPTO_DEV_NX_ENCRYPT=m @@ -471,3 +465,4 @@ CONFIG_TEST_MEMCAT_P=m CONFIG_TEST_MEMINIT=m CONFIG_TEST_FREE_PAGES=m CONFIG_MEMTEST=y +CONFIG_KALLSYMS_ALL=y diff --git a/arch/powerpc/configs/ppc64e_defconfig b/arch/powerpc/configs/ppc64e_defconfig index 4c05f4e4d505..90247b2a0ab0 100644 --- a/arch/powerpc/configs/ppc64e_defconfig +++ b/arch/powerpc/configs/ppc64e_defconfig @@ -207,7 +207,6 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y -CONFIG_CRC_T10DIF=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_STACK_USAGE=y @@ -221,7 +220,7 @@ CONFIG_CODE_PATCHING_SELFTEST=y CONFIG_FTR_FIXUP_SELFTEST=y CONFIG_MSI_BITMAP_SELFTEST=y CONFIG_XMON=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index ca0c90e95837..bb359643ddc1 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -225,7 +225,6 @@ CONFIG_BRIDGE_EBT_REDIRECT=m CONFIG_BRIDGE_EBT_SNAT=m CONFIG_BRIDGE_EBT_LOG=m CONFIG_BRIDGE_EBT_NFLOG=m -CONFIG_IP_DCCP=m CONFIG_TIPC=m CONFIG_ATM=m CONFIG_ATM_CLIP=m @@ -253,7 +252,6 @@ CONFIG_NET_SCH_DSMARK=m CONFIG_NET_SCH_NETEM=m CONFIG_NET_SCH_INGRESS=m CONFIG_NET_CLS_BASIC=m -CONFIG_NET_CLS_TCINDEX=m CONFIG_NET_CLS_ROUTE4=m CONFIG_NET_CLS_FW=m CONFIG_NET_CLS_U32=m @@ -485,7 +483,6 @@ CONFIG_VIA_VELOCITY=m CONFIG_PCMCIA_XIRC2PS=m CONFIG_FDDI=y CONFIG_SKFP=m -CONFIG_NET_SB1000=m CONFIG_BROADCOM_PHY=m CONFIG_CICADA_PHY=m CONFIG_DAVICOM_PHY=m @@ -986,7 +983,6 @@ CONFIG_MINIX_FS=m CONFIG_OMFS_FS=m CONFIG_QNX4FS_FS=m CONFIG_ROMFS_FS=m -CONFIG_SYSV_FS=m CONFIG_UFS_FS=m CONFIG_NFS_FS=m CONFIG_NFS_V3_ACL=y @@ -1075,7 +1071,7 @@ CONFIG_SECURITY_NETWORK_XFRM=y CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_SELINUX_BOOTPARAM=y CONFIG_SECURITY_SELINUX_DISABLE=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_CTS=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/ps3_defconfig b/arch/powerpc/configs/ps3_defconfig index 2b175ddf82f0..0b48d2b776c4 100644 --- a/arch/powerpc/configs/ps3_defconfig +++ b/arch/powerpc/configs/ps3_defconfig @@ -148,8 +148,6 @@ CONFIG_NLS_ISO8859_1=y CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_LZO=m -CONFIG_CRC_CCITT=m -CONFIG_CRC_T10DIF=y CONFIG_PRINTK_TIME=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/powerpc/configs/skiroot_defconfig b/arch/powerpc/configs/skiroot_defconfig index 9d44e6630908..2b71a6dc399e 100644 --- a/arch/powerpc/configs/skiroot_defconfig +++ b/arch/powerpc/configs/skiroot_defconfig @@ -78,7 +78,6 @@ CONFIG_VIRTIO_BLK=m CONFIG_BLK_DEV_NVME=m CONFIG_NVME_MULTIPATH=y CONFIG_EEPROM_AT24=m -# CONFIG_CXL is not set # CONFIG_OCXL is not set CONFIG_BLK_DEV_SD=m CONFIG_BLK_DEV_SR=m @@ -279,9 +278,6 @@ CONFIG_LOCK_DOWN_KERNEL_FORCE_INTEGRITY=y # CONFIG_INTEGRITY is not set CONFIG_LSM="yama,loadpin,safesetid,integrity" # CONFIG_CRYPTO_HW is not set -CONFIG_CRC16=y -CONFIG_CRC_ITU_T=y -CONFIG_LIBCRC32C=y # CONFIG_XZ_DEC_X86 is not set # CONFIG_XZ_DEC_IA64 is not set # CONFIG_XZ_DEC_ARM is not set diff --git a/arch/powerpc/configs/storcenter_defconfig b/arch/powerpc/configs/storcenter_defconfig index 7a978d396991..e415222bd839 100644 --- a/arch/powerpc/configs/storcenter_defconfig +++ b/arch/powerpc/configs/storcenter_defconfig @@ -75,4 +75,3 @@ CONFIG_NLS_DEFAULT="utf8" CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y -CONFIG_CRC_T10DIF=y diff --git a/arch/powerpc/configs/tqm8xx_defconfig b/arch/powerpc/configs/tqm8xx_defconfig index 383c0966e92f..425f10837a18 100644 --- a/arch/powerpc/configs/tqm8xx_defconfig +++ b/arch/powerpc/configs/tqm8xx_defconfig @@ -54,7 +54,6 @@ CONFIG_TMPFS=y CONFIG_CRAMFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y -CONFIG_CRC32_SLICEBY4=y CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_MAGIC_SYSRQ=y CONFIG_DETECT_HUNG_TASK=y diff --git a/arch/powerpc/configs/wii_defconfig b/arch/powerpc/configs/wii_defconfig index 5017a697b67b..7c714a19221e 100644 --- a/arch/powerpc/configs/wii_defconfig +++ b/arch/powerpc/configs/wii_defconfig @@ -114,7 +114,6 @@ CONFIG_ROOT_NFS=y CONFIG_CIFS=m CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -CONFIG_CRC_CCITT=y CONFIG_PRINTK_TIME=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_SPINLOCK=y diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 951a43726461..cfe39fc221cf 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -3,89 +3,29 @@ menu "Accelerated Cryptographic Algorithms for CPU (powerpc)" config CRYPTO_CURVE25519_PPC64 - tristate "Public key crypto: Curve25519 (PowerPC64)" + tristate depends on PPC64 && CPU_LITTLE_ENDIAN + select CRYPTO_KPP select CRYPTO_LIB_CURVE25519_GENERIC select CRYPTO_ARCH_HAVE_LIB_CURVE25519 + default CRYPTO_LIB_CURVE25519_INTERNAL help Curve25519 algorithm Architecture: PowerPC64 - Little-endian -config CRYPTO_CRC32C_VPMSUM - tristate "CRC32c" - depends on PPC64 && ALTIVEC - select CRYPTO_HASH - select CRC32 - help - CRC32c CRC algorithm with the iSCSI polynomial (RFC 3385 and RFC 3720) - - Architecture: powerpc64 using - - AltiVec extensions - - Enable on POWER8 and newer processors for improved performance. - -config CRYPTO_CRCT10DIF_VPMSUM - tristate "CRC32T10DIF" - depends on PPC64 && ALTIVEC && CRC_T10DIF - select CRYPTO_HASH - help - CRC16 CRC algorithm used for the T10 (SCSI) Data Integrity Field (DIF) - - Architecture: powerpc64 using - - AltiVec extensions - - Enable on POWER8 and newer processors for improved performance. - -config CRYPTO_VPMSUM_TESTER - tristate "CRC32c and CRC32T10DIF hardware acceleration tester" - depends on CRYPTO_CRCT10DIF_VPMSUM && CRYPTO_CRC32C_VPMSUM - help - Stress test for CRC32c and CRCT10DIF algorithms implemented with - powerpc64 AltiVec extensions (POWER8 vpmsum instructions). - Unless you are testing these algorithms, you don't need this. - config CRYPTO_MD5_PPC tristate "Digests: MD5" - depends on PPC select CRYPTO_HASH help MD5 message digest algorithm (RFC1321) Architecture: powerpc -config CRYPTO_SHA1_PPC - tristate "Hash functions: SHA-1" - depends on PPC - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: powerpc - -config CRYPTO_SHA1_PPC_SPE - tristate "Hash functions: SHA-1 (SPE)" - depends on PPC && SPE - help - SHA-1 secure hash algorithm (FIPS 180) - - Architecture: powerpc using - - SPE (Signal Processing Engine) extensions - -config CRYPTO_SHA256_PPC_SPE - tristate "Hash functions: SHA-224 and SHA-256 (SPE)" - depends on PPC && SPE - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: powerpc using - - SPE (Signal Processing Engine) extensions - config CRYPTO_AES_PPC_SPE tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)" - depends on PPC && SPE + depends on SPE select CRYPTO_SKCIPHER help Block ciphers: AES cipher algorithms (FIPS-197) @@ -123,32 +63,6 @@ config CRYPTO_AES_GCM_P10 Support for cryptographic acceleration instructions on Power10 or later CPU. This module supports stitched acceleration for AES/GCM. -config CRYPTO_CHACHA20_P10 - tristate "Ciphers: ChaCha20, XChacha20, XChacha12 (P10 or later)" - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_SKCIPHER - select CRYPTO_LIB_CHACHA_GENERIC - select CRYPTO_ARCH_HAVE_LIB_CHACHA - help - Length-preserving ciphers: ChaCha20, XChaCha20, and XChaCha12 - stream cipher algorithms - - Architecture: PowerPC64 - - Power10 or later - - Little-endian - -config CRYPTO_POLY1305_P10 - tristate "Hash functions: Poly1305 (P10 or later)" - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX - select CRYPTO_HASH - select CRYPTO_LIB_POLY1305_GENERIC - help - Poly1305 authenticator algorithm (RFC7539) - - Architecture: PowerPC64 - - Power10 or later - - Little-endian - config CRYPTO_DEV_VMX bool "Support for VMX cryptographic acceleration instructions" depends on PPC64 && VSX diff --git a/arch/powerpc/crypto/Makefile b/arch/powerpc/crypto/Makefile index 59808592f0a1..bc8fd27344b8 100644 --- a/arch/powerpc/crypto/Makefile +++ b/arch/powerpc/crypto/Makefile @@ -7,28 +7,13 @@ obj-$(CONFIG_CRYPTO_AES_PPC_SPE) += aes-ppc-spe.o obj-$(CONFIG_CRYPTO_MD5_PPC) += md5-ppc.o -obj-$(CONFIG_CRYPTO_SHA1_PPC) += sha1-powerpc.o -obj-$(CONFIG_CRYPTO_SHA1_PPC_SPE) += sha1-ppc-spe.o -obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o -obj-$(CONFIG_CRYPTO_CRC32C_VPMSUM) += crc32c-vpmsum.o -obj-$(CONFIG_CRYPTO_CRCT10DIF_VPMSUM) += crct10dif-vpmsum.o -obj-$(CONFIG_CRYPTO_VPMSUM_TESTER) += crc-vpmsum_test.o obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o -obj-$(CONFIG_CRYPTO_CHACHA20_P10) += chacha-p10-crypto.o -obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o obj-$(CONFIG_CRYPTO_CURVE25519_PPC64) += curve25519-ppc64le.o aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o md5-ppc-y := md5-asm.o md5-glue.o -sha1-powerpc-y := sha1-powerpc-asm.o sha1.o -sha1-ppc-spe-y := sha1-spe-asm.o sha1-spe-glue.o -sha256-ppc-spe-y := sha256-spe-asm.o sha256-spe-glue.o -crc32c-vpmsum-y := crc32c-vpmsum_asm.o crc32c-vpmsum_glue.o -crct10dif-vpmsum-y := crct10dif-vpmsum_asm.o crct10dif-vpmsum_glue.o aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o -chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o -poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o curve25519-ppc64le-y := curve25519-ppc64le-core.o curve25519-ppc64le_asm.o @@ -56,3 +41,4 @@ $(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y OBJECT_FILES_NON_STANDARD_aesp8-ppc.o := y +OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c index f37b3d13fc53..85f4fd4b1bdc 100644 --- a/arch/powerpc/crypto/aes-gcm-p10-glue.c +++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c @@ -35,9 +35,9 @@ MODULE_ALIAS_CRYPTO("aes"); asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits, void *key); asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key); -asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, +asmlinkage void aes_p10_gcm_encrypt(const u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); -asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len, +asmlinkage void aes_p10_gcm_decrypt(const u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, @@ -214,7 +214,6 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN); u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN]; struct Hash_ctx *hash = PTR_ALIGN((void *)hashbuf, PPC_ALIGN); - struct scatter_walk assoc_sg_walk; struct skcipher_walk walk; u8 *assocmem = NULL; u8 *assoc; @@ -234,8 +233,7 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, /* Linearize assoc, if not already linear */ if (req->src->length >= assoclen && req->src->length) { - scatterwalk_start(&assoc_sg_walk, req->src); - assoc = scatterwalk_map(&assoc_sg_walk); + assoc = sg_virt(req->src); /* ppc64 is !HIGHMEM */ } else { gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; @@ -253,10 +251,7 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, gcmp10_init(gctx, iv, (unsigned char *) &ctx->enc_key, hash, assoc, assoclen); vsx_end(); - if (!assocmem) - scatterwalk_unmap(assoc); - else - kfree(assocmem); + kfree(assocmem); if (enc) ret = skcipher_walk_aead_encrypt(&walk, req, false); @@ -266,7 +261,7 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, return ret; while ((nbytes = walk.nbytes) > 0 && ret == 0) { - u8 *src = walk.src.virt.addr; + const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; u8 buf[AES_BLOCK_SIZE]; diff --git a/arch/powerpc/crypto/aes.c b/arch/powerpc/crypto/aes.c index ec06189fbf99..3f1e5e894902 100644 --- a/arch/powerpc/crypto/aes.c +++ b/arch/powerpc/crypto/aes.c @@ -7,15 +7,15 @@ * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> */ -#include <linux/types.h> -#include <linux/err.h> -#include <linux/crypto.h> -#include <linux/delay.h> #include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> #include <crypto/internal/cipher.h> #include <crypto/internal/simd.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_cbc.c b/arch/powerpc/crypto/aes_cbc.c index ed0debc7acb5..5f2a4f375eef 100644 --- a/arch/powerpc/crypto/aes_cbc.c +++ b/arch/powerpc/crypto/aes_cbc.c @@ -12,6 +12,10 @@ #include <crypto/aes.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/aes_ctr.c b/arch/powerpc/crypto/aes_ctr.c index 9a3da8cd62f3..e27c4036e711 100644 --- a/arch/powerpc/crypto/aes_ctr.c +++ b/arch/powerpc/crypto/aes_ctr.c @@ -12,6 +12,10 @@ #include <crypto/aes.h> #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" @@ -69,9 +73,9 @@ static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx, struct skcipher_walk *walk) { + const u8 *src = walk->src.virt.addr; u8 *ctrblk = walk->iv; u8 keystream[AES_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; diff --git a/arch/powerpc/crypto/aes_xts.c b/arch/powerpc/crypto/aes_xts.c index dabbccb41550..9440e771cede 100644 --- a/arch/powerpc/crypto/aes_xts.c +++ b/arch/powerpc/crypto/aes_xts.c @@ -13,6 +13,10 @@ #include <crypto/internal/simd.h> #include <crypto/internal/skcipher.h> #include <crypto/xts.h> +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/uaccess.h> #include "aesp8-ppc.h" diff --git a/arch/powerpc/crypto/chacha-p10-glue.c b/arch/powerpc/crypto/chacha-p10-glue.c deleted file mode 100644 index 7c728755852e..000000000000 --- a/arch/powerpc/crypto/chacha-p10-glue.c +++ /dev/null @@ -1,227 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * PowerPC P10 (ppc64le) accelerated ChaCha and XChaCha stream ciphers, - * including ChaCha20 (RFC7539) - * - * Copyright 2023- IBM Corp. All rights reserved. - */ - -#include <crypto/algapi.h> -#include <crypto/internal/chacha.h> -#include <crypto/internal/simd.h> -#include <crypto/internal/skcipher.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/cpufeature.h> -#include <linux/sizes.h> -#include <asm/simd.h> -#include <asm/switch_to.h> - -asmlinkage void chacha_p10le_8x(u32 *state, u8 *dst, const u8 *src, - unsigned int len, int nrounds); - -static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); - -static void vsx_begin(void) -{ - preempt_disable(); - enable_kernel_vsx(); -} - -static void vsx_end(void) -{ - disable_kernel_vsx(); - preempt_enable(); -} - -static void chacha_p10_do_8x(u32 *state, u8 *dst, const u8 *src, - unsigned int bytes, int nrounds) -{ - unsigned int l = bytes & ~0x0FF; - - if (l > 0) { - chacha_p10le_8x(state, dst, src, l, nrounds); - bytes -= l; - src += l; - dst += l; - state[12] += l / CHACHA_BLOCK_SIZE; - } - - if (bytes > 0) - chacha_crypt_generic(state, dst, src, bytes, nrounds); -} - -void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds) -{ - hchacha_block_generic(state, stream, nrounds); -} -EXPORT_SYMBOL(hchacha_block_arch); - -void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv) -{ - chacha_init_generic(state, key, iv); -} -EXPORT_SYMBOL(chacha_init_arch); - -void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes, - int nrounds) -{ - if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE || - !crypto_simd_usable()) - return chacha_crypt_generic(state, dst, src, bytes, nrounds); - - do { - unsigned int todo = min_t(unsigned int, bytes, SZ_4K); - - vsx_begin(); - chacha_p10_do_8x(state, dst, src, todo, nrounds); - vsx_end(); - - bytes -= todo; - src += todo; - dst += todo; - } while (bytes); -} -EXPORT_SYMBOL(chacha_crypt_arch); - -static int chacha_p10_stream_xor(struct skcipher_request *req, - const struct chacha_ctx *ctx, const u8 *iv) -{ - struct skcipher_walk walk; - u32 state[16]; - int err; - - err = skcipher_walk_virt(&walk, req, false); - if (err) - return err; - - chacha_init_generic(state, ctx->key, iv); - - while (walk.nbytes > 0) { - unsigned int nbytes = walk.nbytes; - - if (nbytes < walk.total) - nbytes = rounddown(nbytes, walk.stride); - - if (!crypto_simd_usable()) { - chacha_crypt_generic(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, - ctx->nrounds); - } else { - vsx_begin(); - chacha_p10_do_8x(state, walk.dst.virt.addr, - walk.src.virt.addr, nbytes, ctx->nrounds); - vsx_end(); - } - err = skcipher_walk_done(&walk, walk.nbytes - nbytes); - if (err) - break; - } - - return err; -} - -static int chacha_p10(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - - return chacha_p10_stream_xor(req, ctx, req->iv); -} - -static int xchacha_p10(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm); - struct chacha_ctx subctx; - u32 state[16]; - u8 real_iv[16]; - - chacha_init_generic(state, ctx->key, req->iv); - hchacha_block_arch(state, subctx.key, ctx->nrounds); - subctx.nrounds = ctx->nrounds; - - memcpy(&real_iv[0], req->iv + 24, 8); - memcpy(&real_iv[8], req->iv + 16, 8); - return chacha_p10_stream_xor(req, &subctx, real_iv); -} - -static struct skcipher_alg algs[] = { - { - .base.cra_name = "chacha20", - .base.cra_driver_name = "chacha20-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = CHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = chacha_p10, - .decrypt = chacha_p10, - }, { - .base.cra_name = "xchacha20", - .base.cra_driver_name = "xchacha20-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha20_setkey, - .encrypt = xchacha_p10, - .decrypt = xchacha_p10, - }, { - .base.cra_name = "xchacha12", - .base.cra_driver_name = "xchacha12-p10", - .base.cra_priority = 300, - .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct chacha_ctx), - .base.cra_module = THIS_MODULE, - - .min_keysize = CHACHA_KEY_SIZE, - .max_keysize = CHACHA_KEY_SIZE, - .ivsize = XCHACHA_IV_SIZE, - .chunksize = CHACHA_BLOCK_SIZE, - .setkey = chacha12_setkey, - .encrypt = xchacha_p10, - .decrypt = xchacha_p10, - } -}; - -static int __init chacha_p10_init(void) -{ - if (!cpu_has_feature(CPU_FTR_ARCH_31)) - return 0; - - static_branch_enable(&have_p10); - - return crypto_register_skciphers(algs, ARRAY_SIZE(algs)); -} - -static void __exit chacha_p10_exit(void) -{ - if (!static_branch_likely(&have_p10)) - return; - - crypto_unregister_skciphers(algs, ARRAY_SIZE(algs)); -} - -module_init(chacha_p10_init); -module_exit(chacha_p10_exit); - -MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (P10 accelerated)"); -MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>"); -MODULE_LICENSE("GPL v2"); -MODULE_ALIAS_CRYPTO("chacha20"); -MODULE_ALIAS_CRYPTO("chacha20-p10"); -MODULE_ALIAS_CRYPTO("xchacha20"); -MODULE_ALIAS_CRYPTO("xchacha20-p10"); -MODULE_ALIAS_CRYPTO("xchacha12"); -MODULE_ALIAS_CRYPTO("xchacha12-p10"); diff --git a/arch/powerpc/crypto/chacha-p10le-8x.S b/arch/powerpc/crypto/chacha-p10le-8x.S deleted file mode 100644 index 17bedb66b822..000000000000 --- a/arch/powerpc/crypto/chacha-p10le-8x.S +++ /dev/null @@ -1,842 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -# -# Accelerated chacha20 implementation for ppc64le. -# -# Copyright 2023- IBM Corp. All rights reserved -# -#=================================================================================== -# Written by Danny Tsen <dtsen@us.ibm.com> -# -# chacha_p10le_8x(u32 *state, byte *dst, const byte *src, -# size_t len, int nrounds); -# -# do rounds, 8 quarter rounds -# 1. a += b; d ^= a; d <<<= 16; -# 2. c += d; b ^= c; b <<<= 12; -# 3. a += b; d ^= a; d <<<= 8; -# 4. c += d; b ^= c; b <<<= 7 -# -# row1 = (row1 + row2), row4 = row1 xor row4, row4 rotate each word by 16 -# row3 = (row3 + row4), row2 = row3 xor row2, row2 rotate each word by 12 -# row1 = (row1 + row2), row4 = row1 xor row4, row4 rotate each word by 8 -# row3 = (row3 + row4), row2 = row3 xor row2, row2 rotate each word by 7 -# -# 4 blocks (a b c d) -# -# a0 b0 c0 d0 -# a1 b1 c1 d1 -# ... -# a4 b4 c4 d4 -# ... -# a8 b8 c8 d8 -# ... -# a12 b12 c12 d12 -# a13 ... -# a14 ... -# a15 b15 c15 d15 -# -# Column round (v0, v4, v8, v12, v1, v5, v9, v13, v2, v6, v10, v14, v3, v7, v11, v15) -# Diagnal round (v0, v5, v10, v15, v1, v6, v11, v12, v2, v7, v8, v13, v3, v4, v9, v14) -# - -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> -#include <asm/asm-compat.h> -#include <linux/linkage.h> - -.machine "any" -.text - -.macro SAVE_GPR GPR OFFSET FRAME - std \GPR,\OFFSET(\FRAME) -.endm - -.macro SAVE_VRS VRS OFFSET FRAME - li 16, \OFFSET - stvx \VRS, 16, \FRAME -.endm - -.macro SAVE_VSX VSX OFFSET FRAME - li 16, \OFFSET - stxvx \VSX, 16, \FRAME -.endm - -.macro RESTORE_GPR GPR OFFSET FRAME - ld \GPR,\OFFSET(\FRAME) -.endm - -.macro RESTORE_VRS VRS OFFSET FRAME - li 16, \OFFSET - lvx \VRS, 16, \FRAME -.endm - -.macro RESTORE_VSX VSX OFFSET FRAME - li 16, \OFFSET - lxvx \VSX, 16, \FRAME -.endm - -.macro SAVE_REGS - mflr 0 - std 0, 16(1) - stdu 1,-752(1) - - SAVE_GPR 14, 112, 1 - SAVE_GPR 15, 120, 1 - SAVE_GPR 16, 128, 1 - SAVE_GPR 17, 136, 1 - SAVE_GPR 18, 144, 1 - SAVE_GPR 19, 152, 1 - SAVE_GPR 20, 160, 1 - SAVE_GPR 21, 168, 1 - SAVE_GPR 22, 176, 1 - SAVE_GPR 23, 184, 1 - SAVE_GPR 24, 192, 1 - SAVE_GPR 25, 200, 1 - SAVE_GPR 26, 208, 1 - SAVE_GPR 27, 216, 1 - SAVE_GPR 28, 224, 1 - SAVE_GPR 29, 232, 1 - SAVE_GPR 30, 240, 1 - SAVE_GPR 31, 248, 1 - - addi 9, 1, 256 - SAVE_VRS 20, 0, 9 - SAVE_VRS 21, 16, 9 - SAVE_VRS 22, 32, 9 - SAVE_VRS 23, 48, 9 - SAVE_VRS 24, 64, 9 - SAVE_VRS 25, 80, 9 - SAVE_VRS 26, 96, 9 - SAVE_VRS 27, 112, 9 - SAVE_VRS 28, 128, 9 - SAVE_VRS 29, 144, 9 - SAVE_VRS 30, 160, 9 - SAVE_VRS 31, 176, 9 - - SAVE_VSX 14, 192, 9 - SAVE_VSX 15, 208, 9 - SAVE_VSX 16, 224, 9 - SAVE_VSX 17, 240, 9 - SAVE_VSX 18, 256, 9 - SAVE_VSX 19, 272, 9 - SAVE_VSX 20, 288, 9 - SAVE_VSX 21, 304, 9 - SAVE_VSX 22, 320, 9 - SAVE_VSX 23, 336, 9 - SAVE_VSX 24, 352, 9 - SAVE_VSX 25, 368, 9 - SAVE_VSX 26, 384, 9 - SAVE_VSX 27, 400, 9 - SAVE_VSX 28, 416, 9 - SAVE_VSX 29, 432, 9 - SAVE_VSX 30, 448, 9 - SAVE_VSX 31, 464, 9 -.endm # SAVE_REGS - -.macro RESTORE_REGS - addi 9, 1, 256 - RESTORE_VRS 20, 0, 9 - RESTORE_VRS 21, 16, 9 - RESTORE_VRS 22, 32, 9 - RESTORE_VRS 23, 48, 9 - RESTORE_VRS 24, 64, 9 - RESTORE_VRS 25, 80, 9 - RESTORE_VRS 26, 96, 9 - RESTORE_VRS 27, 112, 9 - RESTORE_VRS 28, 128, 9 - RESTORE_VRS 29, 144, 9 - RESTORE_VRS 30, 160, 9 - RESTORE_VRS 31, 176, 9 - - RESTORE_VSX 14, 192, 9 - RESTORE_VSX 15, 208, 9 - RESTORE_VSX 16, 224, 9 - RESTORE_VSX 17, 240, 9 - RESTORE_VSX 18, 256, 9 - RESTORE_VSX 19, 272, 9 - RESTORE_VSX 20, 288, 9 - RESTORE_VSX 21, 304, 9 - RESTORE_VSX 22, 320, 9 - RESTORE_VSX 23, 336, 9 - RESTORE_VSX 24, 352, 9 - RESTORE_VSX 25, 368, 9 - RESTORE_VSX 26, 384, 9 - RESTORE_VSX 27, 400, 9 - RESTORE_VSX 28, 416, 9 - RESTORE_VSX 29, 432, 9 - RESTORE_VSX 30, 448, 9 - RESTORE_VSX 31, 464, 9 - - RESTORE_GPR 14, 112, 1 - RESTORE_GPR 15, 120, 1 - RESTORE_GPR 16, 128, 1 - RESTORE_GPR 17, 136, 1 - RESTORE_GPR 18, 144, 1 - RESTORE_GPR 19, 152, 1 - RESTORE_GPR 20, 160, 1 - RESTORE_GPR 21, 168, 1 - RESTORE_GPR 22, 176, 1 - RESTORE_GPR 23, 184, 1 - RESTORE_GPR 24, 192, 1 - RESTORE_GPR 25, 200, 1 - RESTORE_GPR 26, 208, 1 - RESTORE_GPR 27, 216, 1 - RESTORE_GPR 28, 224, 1 - RESTORE_GPR 29, 232, 1 - RESTORE_GPR 30, 240, 1 - RESTORE_GPR 31, 248, 1 - - addi 1, 1, 752 - ld 0, 16(1) - mtlr 0 -.endm # RESTORE_REGS - -.macro QT_loop_8x - # QR(v0, v4, v8, v12, v1, v5, v9, v13, v2, v6, v10, v14, v3, v7, v11, v15) - xxlor 0, 32+25, 32+25 - xxlor 32+25, 20, 20 - vadduwm 0, 0, 4 - vadduwm 1, 1, 5 - vadduwm 2, 2, 6 - vadduwm 3, 3, 7 - vadduwm 16, 16, 20 - vadduwm 17, 17, 21 - vadduwm 18, 18, 22 - vadduwm 19, 19, 23 - - vpermxor 12, 12, 0, 25 - vpermxor 13, 13, 1, 25 - vpermxor 14, 14, 2, 25 - vpermxor 15, 15, 3, 25 - vpermxor 28, 28, 16, 25 - vpermxor 29, 29, 17, 25 - vpermxor 30, 30, 18, 25 - vpermxor 31, 31, 19, 25 - xxlor 32+25, 0, 0 - vadduwm 8, 8, 12 - vadduwm 9, 9, 13 - vadduwm 10, 10, 14 - vadduwm 11, 11, 15 - vadduwm 24, 24, 28 - vadduwm 25, 25, 29 - vadduwm 26, 26, 30 - vadduwm 27, 27, 31 - vxor 4, 4, 8 - vxor 5, 5, 9 - vxor 6, 6, 10 - vxor 7, 7, 11 - vxor 20, 20, 24 - vxor 21, 21, 25 - vxor 22, 22, 26 - vxor 23, 23, 27 - - xxlor 0, 32+25, 32+25 - xxlor 32+25, 21, 21 - vrlw 4, 4, 25 # - vrlw 5, 5, 25 - vrlw 6, 6, 25 - vrlw 7, 7, 25 - vrlw 20, 20, 25 # - vrlw 21, 21, 25 - vrlw 22, 22, 25 - vrlw 23, 23, 25 - xxlor 32+25, 0, 0 - vadduwm 0, 0, 4 - vadduwm 1, 1, 5 - vadduwm 2, 2, 6 - vadduwm 3, 3, 7 - vadduwm 16, 16, 20 - vadduwm 17, 17, 21 - vadduwm 18, 18, 22 - vadduwm 19, 19, 23 - - xxlor 0, 32+25, 32+25 - xxlor 32+25, 22, 22 - vpermxor 12, 12, 0, 25 - vpermxor 13, 13, 1, 25 - vpermxor 14, 14, 2, 25 - vpermxor 15, 15, 3, 25 - vpermxor 28, 28, 16, 25 - vpermxor 29, 29, 17, 25 - vpermxor 30, 30, 18, 25 - vpermxor 31, 31, 19, 25 - xxlor 32+25, 0, 0 - vadduwm 8, 8, 12 - vadduwm 9, 9, 13 - vadduwm 10, 10, 14 - vadduwm 11, 11, 15 - vadduwm 24, 24, 28 - vadduwm 25, 25, 29 - vadduwm 26, 26, 30 - vadduwm 27, 27, 31 - xxlor 0, 32+28, 32+28 - xxlor 32+28, 23, 23 - vxor 4, 4, 8 - vxor 5, 5, 9 - vxor 6, 6, 10 - vxor 7, 7, 11 - vxor 20, 20, 24 - vxor 21, 21, 25 - vxor 22, 22, 26 - vxor 23, 23, 27 - vrlw 4, 4, 28 # - vrlw 5, 5, 28 - vrlw 6, 6, 28 - vrlw 7, 7, 28 - vrlw 20, 20, 28 # - vrlw 21, 21, 28 - vrlw 22, 22, 28 - vrlw 23, 23, 28 - xxlor 32+28, 0, 0 - - # QR(v0, v5, v10, v15, v1, v6, v11, v12, v2, v7, v8, v13, v3, v4, v9, v14) - xxlor 0, 32+25, 32+25 - xxlor 32+25, 20, 20 - vadduwm 0, 0, 5 - vadduwm 1, 1, 6 - vadduwm 2, 2, 7 - vadduwm 3, 3, 4 - vadduwm 16, 16, 21 - vadduwm 17, 17, 22 - vadduwm 18, 18, 23 - vadduwm 19, 19, 20 - - vpermxor 15, 15, 0, 25 - vpermxor 12, 12, 1, 25 - vpermxor 13, 13, 2, 25 - vpermxor 14, 14, 3, 25 - vpermxor 31, 31, 16, 25 - vpermxor 28, 28, 17, 25 - vpermxor 29, 29, 18, 25 - vpermxor 30, 30, 19, 25 - - xxlor 32+25, 0, 0 - vadduwm 10, 10, 15 - vadduwm 11, 11, 12 - vadduwm 8, 8, 13 - vadduwm 9, 9, 14 - vadduwm 26, 26, 31 - vadduwm 27, 27, 28 - vadduwm 24, 24, 29 - vadduwm 25, 25, 30 - vxor 5, 5, 10 - vxor 6, 6, 11 - vxor 7, 7, 8 - vxor 4, 4, 9 - vxor 21, 21, 26 - vxor 22, 22, 27 - vxor 23, 23, 24 - vxor 20, 20, 25 - - xxlor 0, 32+25, 32+25 - xxlor 32+25, 21, 21 - vrlw 5, 5, 25 - vrlw 6, 6, 25 - vrlw 7, 7, 25 - vrlw 4, 4, 25 - vrlw 21, 21, 25 - vrlw 22, 22, 25 - vrlw 23, 23, 25 - vrlw 20, 20, 25 - xxlor 32+25, 0, 0 - - vadduwm 0, 0, 5 - vadduwm 1, 1, 6 - vadduwm 2, 2, 7 - vadduwm 3, 3, 4 - vadduwm 16, 16, 21 - vadduwm 17, 17, 22 - vadduwm 18, 18, 23 - vadduwm 19, 19, 20 - - xxlor 0, 32+25, 32+25 - xxlor 32+25, 22, 22 - vpermxor 15, 15, 0, 25 - vpermxor 12, 12, 1, 25 - vpermxor 13, 13, 2, 25 - vpermxor 14, 14, 3, 25 - vpermxor 31, 31, 16, 25 - vpermxor 28, 28, 17, 25 - vpermxor 29, 29, 18, 25 - vpermxor 30, 30, 19, 25 - xxlor 32+25, 0, 0 - - vadduwm 10, 10, 15 - vadduwm 11, 11, 12 - vadduwm 8, 8, 13 - vadduwm 9, 9, 14 - vadduwm 26, 26, 31 - vadduwm 27, 27, 28 - vadduwm 24, 24, 29 - vadduwm 25, 25, 30 - - xxlor 0, 32+28, 32+28 - xxlor 32+28, 23, 23 - vxor 5, 5, 10 - vxor 6, 6, 11 - vxor 7, 7, 8 - vxor 4, 4, 9 - vxor 21, 21, 26 - vxor 22, 22, 27 - vxor 23, 23, 24 - vxor 20, 20, 25 - vrlw 5, 5, 28 - vrlw 6, 6, 28 - vrlw 7, 7, 28 - vrlw 4, 4, 28 - vrlw 21, 21, 28 - vrlw 22, 22, 28 - vrlw 23, 23, 28 - vrlw 20, 20, 28 - xxlor 32+28, 0, 0 -.endm - -.macro QT_loop_4x - # QR(v0, v4, v8, v12, v1, v5, v9, v13, v2, v6, v10, v14, v3, v7, v11, v15) - vadduwm 0, 0, 4 - vadduwm 1, 1, 5 - vadduwm 2, 2, 6 - vadduwm 3, 3, 7 - vpermxor 12, 12, 0, 20 - vpermxor 13, 13, 1, 20 - vpermxor 14, 14, 2, 20 - vpermxor 15, 15, 3, 20 - vadduwm 8, 8, 12 - vadduwm 9, 9, 13 - vadduwm 10, 10, 14 - vadduwm 11, 11, 15 - vxor 4, 4, 8 - vxor 5, 5, 9 - vxor 6, 6, 10 - vxor 7, 7, 11 - vrlw 4, 4, 21 - vrlw 5, 5, 21 - vrlw 6, 6, 21 - vrlw 7, 7, 21 - vadduwm 0, 0, 4 - vadduwm 1, 1, 5 - vadduwm 2, 2, 6 - vadduwm 3, 3, 7 - vpermxor 12, 12, 0, 22 - vpermxor 13, 13, 1, 22 - vpermxor 14, 14, 2, 22 - vpermxor 15, 15, 3, 22 - vadduwm 8, 8, 12 - vadduwm 9, 9, 13 - vadduwm 10, 10, 14 - vadduwm 11, 11, 15 - vxor 4, 4, 8 - vxor 5, 5, 9 - vxor 6, 6, 10 - vxor 7, 7, 11 - vrlw 4, 4, 23 - vrlw 5, 5, 23 - vrlw 6, 6, 23 - vrlw 7, 7, 23 - - # QR(v0, v5, v10, v15, v1, v6, v11, v12, v2, v7, v8, v13, v3, v4, v9, v14) - vadduwm 0, 0, 5 - vadduwm 1, 1, 6 - vadduwm 2, 2, 7 - vadduwm 3, 3, 4 - vpermxor 15, 15, 0, 20 - vpermxor 12, 12, 1, 20 - vpermxor 13, 13, 2, 20 - vpermxor 14, 14, 3, 20 - vadduwm 10, 10, 15 - vadduwm 11, 11, 12 - vadduwm 8, 8, 13 - vadduwm 9, 9, 14 - vxor 5, 5, 10 - vxor 6, 6, 11 - vxor 7, 7, 8 - vxor 4, 4, 9 - vrlw 5, 5, 21 - vrlw 6, 6, 21 - vrlw 7, 7, 21 - vrlw 4, 4, 21 - vadduwm 0, 0, 5 - vadduwm 1, 1, 6 - vadduwm 2, 2, 7 - vadduwm 3, 3, 4 - vpermxor 15, 15, 0, 22 - vpermxor 12, 12, 1, 22 - vpermxor 13, 13, 2, 22 - vpermxor 14, 14, 3, 22 - vadduwm 10, 10, 15 - vadduwm 11, 11, 12 - vadduwm 8, 8, 13 - vadduwm 9, 9, 14 - vxor 5, 5, 10 - vxor 6, 6, 11 - vxor 7, 7, 8 - vxor 4, 4, 9 - vrlw 5, 5, 23 - vrlw 6, 6, 23 - vrlw 7, 7, 23 - vrlw 4, 4, 23 -.endm - -# Transpose -.macro TP_4x a0 a1 a2 a3 - xxmrghw 10, 32+\a0, 32+\a1 # a0, a1, b0, b1 - xxmrghw 11, 32+\a2, 32+\a3 # a2, a3, b2, b3 - xxmrglw 12, 32+\a0, 32+\a1 # c0, c1, d0, d1 - xxmrglw 13, 32+\a2, 32+\a3 # c2, c3, d2, d3 - xxpermdi 32+\a0, 10, 11, 0 # a0, a1, a2, a3 - xxpermdi 32+\a1, 10, 11, 3 # b0, b1, b2, b3 - xxpermdi 32+\a2, 12, 13, 0 # c0, c1, c2, c3 - xxpermdi 32+\a3, 12, 13, 3 # d0, d1, d2, d3 -.endm - -# key stream = working state + state -.macro Add_state S - vadduwm \S+0, \S+0, 16-\S - vadduwm \S+4, \S+4, 17-\S - vadduwm \S+8, \S+8, 18-\S - vadduwm \S+12, \S+12, 19-\S - - vadduwm \S+1, \S+1, 16-\S - vadduwm \S+5, \S+5, 17-\S - vadduwm \S+9, \S+9, 18-\S - vadduwm \S+13, \S+13, 19-\S - - vadduwm \S+2, \S+2, 16-\S - vadduwm \S+6, \S+6, 17-\S - vadduwm \S+10, \S+10, 18-\S - vadduwm \S+14, \S+14, 19-\S - - vadduwm \S+3, \S+3, 16-\S - vadduwm \S+7, \S+7, 17-\S - vadduwm \S+11, \S+11, 18-\S - vadduwm \S+15, \S+15, 19-\S -.endm - -# -# write 256 bytes -# -.macro Write_256 S - add 9, 14, 5 - add 16, 14, 4 - lxvw4x 0, 0, 9 - lxvw4x 1, 17, 9 - lxvw4x 2, 18, 9 - lxvw4x 3, 19, 9 - lxvw4x 4, 20, 9 - lxvw4x 5, 21, 9 - lxvw4x 6, 22, 9 - lxvw4x 7, 23, 9 - lxvw4x 8, 24, 9 - lxvw4x 9, 25, 9 - lxvw4x 10, 26, 9 - lxvw4x 11, 27, 9 - lxvw4x 12, 28, 9 - lxvw4x 13, 29, 9 - lxvw4x 14, 30, 9 - lxvw4x 15, 31, 9 - - xxlxor \S+32, \S+32, 0 - xxlxor \S+36, \S+36, 1 - xxlxor \S+40, \S+40, 2 - xxlxor \S+44, \S+44, 3 - xxlxor \S+33, \S+33, 4 - xxlxor \S+37, \S+37, 5 - xxlxor \S+41, \S+41, 6 - xxlxor \S+45, \S+45, 7 - xxlxor \S+34, \S+34, 8 - xxlxor \S+38, \S+38, 9 - xxlxor \S+42, \S+42, 10 - xxlxor \S+46, \S+46, 11 - xxlxor \S+35, \S+35, 12 - xxlxor \S+39, \S+39, 13 - xxlxor \S+43, \S+43, 14 - xxlxor \S+47, \S+47, 15 - - stxvw4x \S+32, 0, 16 - stxvw4x \S+36, 17, 16 - stxvw4x \S+40, 18, 16 - stxvw4x \S+44, 19, 16 - - stxvw4x \S+33, 20, 16 - stxvw4x \S+37, 21, 16 - stxvw4x \S+41, 22, 16 - stxvw4x \S+45, 23, 16 - - stxvw4x \S+34, 24, 16 - stxvw4x \S+38, 25, 16 - stxvw4x \S+42, 26, 16 - stxvw4x \S+46, 27, 16 - - stxvw4x \S+35, 28, 16 - stxvw4x \S+39, 29, 16 - stxvw4x \S+43, 30, 16 - stxvw4x \S+47, 31, 16 - -.endm - -# -# chacha20_p10le_8x(u32 *state, byte *dst, const byte *src, size_t len, int nrounds); -# -SYM_FUNC_START(chacha_p10le_8x) -.align 5 - cmpdi 6, 0 - ble Out_no_chacha - - SAVE_REGS - - # r17 - r31 mainly for Write_256 macro. - li 17, 16 - li 18, 32 - li 19, 48 - li 20, 64 - li 21, 80 - li 22, 96 - li 23, 112 - li 24, 128 - li 25, 144 - li 26, 160 - li 27, 176 - li 28, 192 - li 29, 208 - li 30, 224 - li 31, 240 - - mr 15, 6 # len - li 14, 0 # offset to inp and outp - - lxvw4x 48, 0, 3 # vr16, constants - lxvw4x 49, 17, 3 # vr17, key 1 - lxvw4x 50, 18, 3 # vr18, key 2 - lxvw4x 51, 19, 3 # vr19, counter, nonce - - # create (0, 1, 2, 3) counters - vspltisw 0, 0 - vspltisw 1, 1 - vspltisw 2, 2 - vspltisw 3, 3 - vmrghw 4, 0, 1 - vmrglw 5, 2, 3 - vsldoi 30, 4, 5, 8 # vr30 counter, 4 (0, 1, 2, 3) - - vspltisw 21, 12 - vspltisw 23, 7 - - addis 11, 2, permx@toc@ha - addi 11, 11, permx@toc@l - lxvw4x 32+20, 0, 11 - lxvw4x 32+22, 17, 11 - - sradi 8, 7, 1 - - mtctr 8 - - # save constants to vsx - xxlor 16, 48, 48 - xxlor 17, 49, 49 - xxlor 18, 50, 50 - xxlor 19, 51, 51 - - vspltisw 25, 4 - vspltisw 26, 8 - - xxlor 25, 32+26, 32+26 - xxlor 24, 32+25, 32+25 - - vadduwm 31, 30, 25 # counter = (0, 1, 2, 3) + (4, 4, 4, 4) - xxlor 30, 32+30, 32+30 - xxlor 31, 32+31, 32+31 - - xxlor 20, 32+20, 32+20 - xxlor 21, 32+21, 32+21 - xxlor 22, 32+22, 32+22 - xxlor 23, 32+23, 32+23 - - cmpdi 6, 512 - blt Loop_last - -Loop_8x: - xxspltw 32+0, 16, 0 - xxspltw 32+1, 16, 1 - xxspltw 32+2, 16, 2 - xxspltw 32+3, 16, 3 - - xxspltw 32+4, 17, 0 - xxspltw 32+5, 17, 1 - xxspltw 32+6, 17, 2 - xxspltw 32+7, 17, 3 - xxspltw 32+8, 18, 0 - xxspltw 32+9, 18, 1 - xxspltw 32+10, 18, 2 - xxspltw 32+11, 18, 3 - xxspltw 32+12, 19, 0 - xxspltw 32+13, 19, 1 - xxspltw 32+14, 19, 2 - xxspltw 32+15, 19, 3 - vadduwm 12, 12, 30 # increase counter - - xxspltw 32+16, 16, 0 - xxspltw 32+17, 16, 1 - xxspltw 32+18, 16, 2 - xxspltw 32+19, 16, 3 - - xxspltw 32+20, 17, 0 - xxspltw 32+21, 17, 1 - xxspltw 32+22, 17, 2 - xxspltw 32+23, 17, 3 - xxspltw 32+24, 18, 0 - xxspltw 32+25, 18, 1 - xxspltw 32+26, 18, 2 - xxspltw 32+27, 18, 3 - xxspltw 32+28, 19, 0 - xxspltw 32+29, 19, 1 - vadduwm 28, 28, 31 # increase counter - xxspltw 32+30, 19, 2 - xxspltw 32+31, 19, 3 - -.align 5 -quarter_loop_8x: - QT_loop_8x - - bdnz quarter_loop_8x - - xxlor 0, 32+30, 32+30 - xxlor 32+30, 30, 30 - vadduwm 12, 12, 30 - xxlor 32+30, 0, 0 - TP_4x 0, 1, 2, 3 - TP_4x 4, 5, 6, 7 - TP_4x 8, 9, 10, 11 - TP_4x 12, 13, 14, 15 - - xxlor 0, 48, 48 - xxlor 1, 49, 49 - xxlor 2, 50, 50 - xxlor 3, 51, 51 - xxlor 48, 16, 16 - xxlor 49, 17, 17 - xxlor 50, 18, 18 - xxlor 51, 19, 19 - Add_state 0 - xxlor 48, 0, 0 - xxlor 49, 1, 1 - xxlor 50, 2, 2 - xxlor 51, 3, 3 - Write_256 0 - addi 14, 14, 256 # offset +=256 - addi 15, 15, -256 # len -=256 - - xxlor 5, 32+31, 32+31 - xxlor 32+31, 31, 31 - vadduwm 28, 28, 31 - xxlor 32+31, 5, 5 - TP_4x 16+0, 16+1, 16+2, 16+3 - TP_4x 16+4, 16+5, 16+6, 16+7 - TP_4x 16+8, 16+9, 16+10, 16+11 - TP_4x 16+12, 16+13, 16+14, 16+15 - - xxlor 32, 16, 16 - xxlor 33, 17, 17 - xxlor 34, 18, 18 - xxlor 35, 19, 19 - Add_state 16 - Write_256 16 - addi 14, 14, 256 # offset +=256 - addi 15, 15, -256 # len +=256 - - xxlor 32+24, 24, 24 - xxlor 32+25, 25, 25 - xxlor 32+30, 30, 30 - vadduwm 30, 30, 25 - vadduwm 31, 30, 24 - xxlor 30, 32+30, 32+30 - xxlor 31, 32+31, 32+31 - - cmpdi 15, 0 - beq Out_loop - - cmpdi 15, 512 - blt Loop_last - - mtctr 8 - b Loop_8x - -Loop_last: - lxvw4x 48, 0, 3 # vr16, constants - lxvw4x 49, 17, 3 # vr17, key 1 - lxvw4x 50, 18, 3 # vr18, key 2 - lxvw4x 51, 19, 3 # vr19, counter, nonce - - vspltisw 21, 12 - vspltisw 23, 7 - addis 11, 2, permx@toc@ha - addi 11, 11, permx@toc@l - lxvw4x 32+20, 0, 11 - lxvw4x 32+22, 17, 11 - - sradi 8, 7, 1 - mtctr 8 - -Loop_4x: - vspltw 0, 16, 0 - vspltw 1, 16, 1 - vspltw 2, 16, 2 - vspltw 3, 16, 3 - - vspltw 4, 17, 0 - vspltw 5, 17, 1 - vspltw 6, 17, 2 - vspltw 7, 17, 3 - vspltw 8, 18, 0 - vspltw 9, 18, 1 - vspltw 10, 18, 2 - vspltw 11, 18, 3 - vspltw 12, 19, 0 - vadduwm 12, 12, 30 # increase counter - vspltw 13, 19, 1 - vspltw 14, 19, 2 - vspltw 15, 19, 3 - -.align 5 -quarter_loop: - QT_loop_4x - - bdnz quarter_loop - - vadduwm 12, 12, 30 - TP_4x 0, 1, 2, 3 - TP_4x 4, 5, 6, 7 - TP_4x 8, 9, 10, 11 - TP_4x 12, 13, 14, 15 - - Add_state 0 - Write_256 0 - addi 14, 14, 256 # offset += 256 - addi 15, 15, -256 # len += 256 - - # Update state counter - vspltisw 25, 4 - vadduwm 30, 30, 25 - - cmpdi 15, 0 - beq Out_loop - cmpdi 15, 256 - blt Out_loop - - mtctr 8 - b Loop_4x - -Out_loop: - RESTORE_REGS - blr - -Out_no_chacha: - li 3, 0 - blr -SYM_FUNC_END(chacha_p10le_8x) - -SYM_DATA_START_LOCAL(PERMX) -.align 5 -permx: -.long 0x22330011, 0x66774455, 0xaabb8899, 0xeeffccdd -.long 0x11223300, 0x55667744, 0x99aabb88, 0xddeeffcc -SYM_DATA_END(PERMX) diff --git a/arch/powerpc/crypto/crc-vpmsum_test.c b/arch/powerpc/crypto/crc-vpmsum_test.c deleted file mode 100644 index c61a874a3a5c..000000000000 --- a/arch/powerpc/crypto/crc-vpmsum_test.c +++ /dev/null @@ -1,133 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * CRC vpmsum tester - * Copyright 2017 Daniel Axtens, IBM Corporation. - */ - -#include <linux/crc-t10dif.h> -#include <linux/crc32.h> -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/random.h> -#include <linux/string.h> -#include <linux/kernel.h> -#include <linux/cpufeature.h> -#include <asm/switch_to.h> - -static unsigned long iterations = 10000; - -#define MAX_CRC_LENGTH 65535 - - -static int __init crc_test_init(void) -{ - u16 crc16 = 0, verify16 = 0; - __le32 verify32le = 0; - unsigned char *data; - u32 verify32 = 0; - unsigned long i; - __le32 crc32; - int ret; - - struct crypto_shash *crct10dif_tfm; - struct crypto_shash *crc32c_tfm; - - if (!cpu_has_feature(CPU_FTR_ARCH_207S)) - return -ENODEV; - - data = kmalloc(MAX_CRC_LENGTH, GFP_KERNEL); - if (!data) - return -ENOMEM; - - crct10dif_tfm = crypto_alloc_shash("crct10dif", 0, 0); - - if (IS_ERR(crct10dif_tfm)) { - pr_err("Error allocating crc-t10dif\n"); - goto free_buf; - } - - crc32c_tfm = crypto_alloc_shash("crc32c", 0, 0); - - if (IS_ERR(crc32c_tfm)) { - pr_err("Error allocating crc32c\n"); - goto free_16; - } - - do { - SHASH_DESC_ON_STACK(crct10dif_shash, crct10dif_tfm); - SHASH_DESC_ON_STACK(crc32c_shash, crc32c_tfm); - - crct10dif_shash->tfm = crct10dif_tfm; - ret = crypto_shash_init(crct10dif_shash); - - if (ret) { - pr_err("Error initing crc-t10dif\n"); - goto free_32; - } - - - crc32c_shash->tfm = crc32c_tfm; - ret = crypto_shash_init(crc32c_shash); - - if (ret) { - pr_err("Error initing crc32c\n"); - goto free_32; - } - - pr_info("crc-vpmsum_test begins, %lu iterations\n", iterations); - for (i=0; i<iterations; i++) { - size_t offset = get_random_u32_below(16); - size_t len = get_random_u32_below(MAX_CRC_LENGTH); - - if (len <= offset) - continue; - get_random_bytes(data, len); - len -= offset; - - crypto_shash_update(crct10dif_shash, data+offset, len); - crypto_shash_final(crct10dif_shash, (u8 *)(&crc16)); - verify16 = crc_t10dif_generic(verify16, data+offset, len); - - - if (crc16 != verify16) { - pr_err("FAILURE in CRC16: got 0x%04x expected 0x%04x (len %lu)\n", - crc16, verify16, len); - break; - } - - crypto_shash_update(crc32c_shash, data+offset, len); - crypto_shash_final(crc32c_shash, (u8 *)(&crc32)); - verify32 = le32_to_cpu(verify32le); - verify32le = ~cpu_to_le32(__crc32c_le(~verify32, data+offset, len)); - if (crc32 != verify32le) { - pr_err("FAILURE in CRC32: got 0x%08x expected 0x%08x (len %lu)\n", - crc32, verify32, len); - break; - } - cond_resched(); - } - pr_info("crc-vpmsum_test done, completed %lu iterations\n", i); - } while (0); - -free_32: - crypto_free_shash(crc32c_tfm); - -free_16: - crypto_free_shash(crct10dif_tfm); - -free_buf: - kfree(data); - - return 0; -} - -static void __exit crc_test_exit(void) {} - -module_init(crc_test_init); -module_exit(crc_test_exit); -module_param(iterations, long, 0400); - -MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>"); -MODULE_DESCRIPTION("Vector polynomial multiply-sum CRC tester"); -MODULE_LICENSE("GPL"); diff --git a/arch/powerpc/crypto/crc32-vpmsum_core.S b/arch/powerpc/crypto/crc32-vpmsum_core.S deleted file mode 100644 index b0f87f595b26..000000000000 --- a/arch/powerpc/crypto/crc32-vpmsum_core.S +++ /dev/null @@ -1,746 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Core of the accelerated CRC algorithm. - * In your file, define the constants and CRC_FUNCTION_NAME - * Then include this file. - * - * Calculate the checksum of data that is 16 byte aligned and a multiple of - * 16 bytes. - * - * The first step is to reduce it to 1024 bits. We do this in 8 parallel - * chunks in order to mask the latency of the vpmsum instructions. If we - * have more than 32 kB of data to checksum we repeat this step multiple - * times, passing in the previous 1024 bits. - * - * The next step is to reduce the 1024 bits to 64 bits. This step adds - * 32 bits of 0s to the end - this matches what a CRC does. We just - * calculate constants that land the data in this 32 bits. - * - * We then use fixed point Barrett reduction to compute a mod n over GF(2) - * for n = CRC using POWER8 instructions. We use x = 32. - * - * https://en.wikipedia.org/wiki/Barrett_reduction - * - * Copyright (C) 2015 Anton Blanchard <anton@au.ibm.com>, IBM -*/ - -#include <asm/ppc_asm.h> -#include <asm/ppc-opcode.h> - -#define MAX_SIZE 32768 - - .text - -#if defined(__BIG_ENDIAN__) && defined(REFLECT) -#define BYTESWAP_DATA -#elif defined(__LITTLE_ENDIAN__) && !defined(REFLECT) -#define BYTESWAP_DATA -#else -#undef BYTESWAP_DATA -#endif - -#define off16 r25 -#define off32 r26 -#define off48 r27 -#define off64 r28 -#define off80 r29 -#define off96 r30 -#define off112 r31 - -#define const1 v24 -#define const2 v25 - -#define byteswap v26 -#define mask_32bit v27 -#define mask_64bit v28 -#define zeroes v29 - -#ifdef BYTESWAP_DATA -#define VPERM(A, B, C, D) vperm A, B, C, D -#else -#define VPERM(A, B, C, D) -#endif - -/* unsigned int CRC_FUNCTION_NAME(unsigned int crc, void *p, unsigned long len) */ -FUNC_START(CRC_FUNCTION_NAME) - std r31,-8(r1) - std r30,-16(r1) - std r29,-24(r1) - std r28,-32(r1) - std r27,-40(r1) - std r26,-48(r1) - std r25,-56(r1) - - li off16,16 - li off32,32 - li off48,48 - li off64,64 - li off80,80 - li off96,96 - li off112,112 - li r0,0 - - /* Enough room for saving 10 non volatile VMX registers */ - subi r6,r1,56+10*16 - subi r7,r1,56+2*16 - - stvx v20,0,r6 - stvx v21,off16,r6 - stvx v22,off32,r6 - stvx v23,off48,r6 - stvx v24,off64,r6 - stvx v25,off80,r6 - stvx v26,off96,r6 - stvx v27,off112,r6 - stvx v28,0,r7 - stvx v29,off16,r7 - - mr r10,r3 - - vxor zeroes,zeroes,zeroes - vspltisw v0,-1 - - vsldoi mask_32bit,zeroes,v0,4 - vsldoi mask_64bit,zeroes,v0,8 - - /* Get the initial value into v8 */ - vxor v8,v8,v8 - MTVRD(v8, R3) -#ifdef REFLECT - vsldoi v8,zeroes,v8,8 /* shift into bottom 32 bits */ -#else - vsldoi v8,v8,zeroes,4 /* shift into top 32 bits */ -#endif - -#ifdef BYTESWAP_DATA - LOAD_REG_ADDR(r3, .byteswap_constant) - lvx byteswap,0,r3 - addi r3,r3,16 -#endif - - cmpdi r5,256 - blt .Lshort - - rldicr r6,r5,0,56 - - /* Checksum in blocks of MAX_SIZE */ -1: lis r7,MAX_SIZE@h - ori r7,r7,MAX_SIZE@l - mr r9,r7 - cmpd r6,r7 - bgt 2f - mr r7,r6 -2: subf r6,r7,r6 - - /* our main loop does 128 bytes at a time */ - srdi r7,r7,7 - - /* - * Work out the offset into the constants table to start at. Each - * constant is 16 bytes, and it is used against 128 bytes of input - * data - 128 / 16 = 8 - */ - sldi r8,r7,4 - srdi r9,r9,3 - subf r8,r8,r9 - - /* We reduce our final 128 bytes in a separate step */ - addi r7,r7,-1 - mtctr r7 - - LOAD_REG_ADDR(r3, .constants) - - /* Find the start of our constants */ - add r3,r3,r8 - - /* zero v0-v7 which will contain our checksums */ - vxor v0,v0,v0 - vxor v1,v1,v1 - vxor v2,v2,v2 - vxor v3,v3,v3 - vxor v4,v4,v4 - vxor v5,v5,v5 - vxor v6,v6,v6 - vxor v7,v7,v7 - - lvx const1,0,r3 - - /* - * If we are looping back to consume more data we use the values - * already in v16-v23. - */ - cmpdi r0,1 - beq 2f - - /* First warm up pass */ - lvx v16,0,r4 - lvx v17,off16,r4 - VPERM(v16,v16,v16,byteswap) - VPERM(v17,v17,v17,byteswap) - lvx v18,off32,r4 - lvx v19,off48,r4 - VPERM(v18,v18,v18,byteswap) - VPERM(v19,v19,v19,byteswap) - lvx v20,off64,r4 - lvx v21,off80,r4 - VPERM(v20,v20,v20,byteswap) - VPERM(v21,v21,v21,byteswap) - lvx v22,off96,r4 - lvx v23,off112,r4 - VPERM(v22,v22,v22,byteswap) - VPERM(v23,v23,v23,byteswap) - addi r4,r4,8*16 - - /* xor in initial value */ - vxor v16,v16,v8 - -2: bdz .Lfirst_warm_up_done - - addi r3,r3,16 - lvx const2,0,r3 - - /* Second warm up pass */ - VPMSUMD(v8,v16,const1) - lvx v16,0,r4 - VPERM(v16,v16,v16,byteswap) - ori r2,r2,0 - - VPMSUMD(v9,v17,const1) - lvx v17,off16,r4 - VPERM(v17,v17,v17,byteswap) - ori r2,r2,0 - - VPMSUMD(v10,v18,const1) - lvx v18,off32,r4 - VPERM(v18,v18,v18,byteswap) - ori r2,r2,0 - - VPMSUMD(v11,v19,const1) - lvx v19,off48,r4 - VPERM(v19,v19,v19,byteswap) - ori r2,r2,0 - - VPMSUMD(v12,v20,const1) - lvx v20,off64,r4 - VPERM(v20,v20,v20,byteswap) - ori r2,r2,0 - - VPMSUMD(v13,v21,const1) - lvx v21,off80,r4 - VPERM(v21,v21,v21,byteswap) - ori r2,r2,0 - - VPMSUMD(v14,v22,const1) - lvx v22,off96,r4 - VPERM(v22,v22,v22,byteswap) - ori r2,r2,0 - - VPMSUMD(v15,v23,const1) - lvx v23,off112,r4 - VPERM(v23,v23,v23,byteswap) - - addi r4,r4,8*16 - - bdz .Lfirst_cool_down - - /* - * main loop. We modulo schedule it such that it takes three iterations - * to complete - first iteration load, second iteration vpmsum, third - * iteration xor. - */ - .balign 16 -4: lvx const1,0,r3 - addi r3,r3,16 - ori r2,r2,0 - - vxor v0,v0,v8 - VPMSUMD(v8,v16,const2) - lvx v16,0,r4 - VPERM(v16,v16,v16,byteswap) - ori r2,r2,0 - - vxor v1,v1,v9 - VPMSUMD(v9,v17,const2) - lvx v17,off16,r4 - VPERM(v17,v17,v17,byteswap) - ori r2,r2,0 - - vxor v2,v2,v10 - VPMSUMD(v10,v18,const2) - lvx v18,off32,r4 - VPERM(v18,v18,v18,byteswap) - ori r2,r2,0 - - vxor v3,v3,v11 - VPMSUMD(v11,v19,const2) - lvx v19,off48,r4 - VPERM(v19,v19,v19,byteswap) - lvx const2,0,r3 - ori r2,r2,0 - - vxor v4,v4,v12 - VPMSUMD(v12,v20,const1) - lvx v20,off64,r4 - VPERM(v20,v20,v20,byteswap) - ori r2,r2,0 - - vxor v5,v5,v13 - VPMSUMD(v13,v21,const1) - lvx v21,off80,r4 - VPERM(v21,v21,v21,byteswap) - ori r2,r2,0 - - vxor v6,v6,v14 - VPMSUMD(v14,v22,const1) - lvx v22,off96,r4 - VPERM(v22,v22,v22,byteswap) - ori r2,r2,0 - - vxor v7,v7,v15 - VPMSUMD(v15,v23,const1) - lvx v23,off112,r4 - VPERM(v23,v23,v23,byteswap) - - addi r4,r4,8*16 - - bdnz 4b - -.Lfirst_cool_down: - /* First cool down pass */ - lvx const1,0,r3 - addi r3,r3,16 - - vxor v0,v0,v8 - VPMSUMD(v8,v16,const1) - ori r2,r2,0 - - vxor v1,v1,v9 - VPMSUMD(v9,v17,const1) - ori r2,r2,0 - - vxor v2,v2,v10 - VPMSUMD(v10,v18,const1) - ori r2,r2,0 - - vxor v3,v3,v11 - VPMSUMD(v11,v19,const1) - ori r2,r2,0 - - vxor v4,v4,v12 - VPMSUMD(v12,v20,const1) - ori r2,r2,0 - - vxor v5,v5,v13 - VPMSUMD(v13,v21,const1) - ori r2,r2,0 - - vxor v6,v6,v14 - VPMSUMD(v14,v22,const1) - ori r2,r2,0 - - vxor v7,v7,v15 - VPMSUMD(v15,v23,const1) - ori r2,r2,0 - -.Lsecond_cool_down: - /* Second cool down pass */ - vxor v0,v0,v8 - vxor v1,v1,v9 - vxor v2,v2,v10 - vxor v3,v3,v11 - vxor v4,v4,v12 - vxor v5,v5,v13 - vxor v6,v6,v14 - vxor v7,v7,v15 - -#ifdef REFLECT - /* - * vpmsumd produces a 96 bit result in the least significant bits - * of the register. Since we are bit reflected we have to shift it - * left 32 bits so it occupies the least significant bits in the - * bit reflected domain. - */ - vsldoi v0,v0,zeroes,4 - vsldoi v1,v1,zeroes,4 - vsldoi v2,v2,zeroes,4 - vsldoi v3,v3,zeroes,4 - vsldoi v4,v4,zeroes,4 - vsldoi v5,v5,zeroes,4 - vsldoi v6,v6,zeroes,4 - vsldoi v7,v7,zeroes,4 -#endif - - /* xor with last 1024 bits */ - lvx v8,0,r4 - lvx v9,off16,r4 - VPERM(v8,v8,v8,byteswap) - VPERM(v9,v9,v9,byteswap) - lvx v10,off32,r4 - lvx v11,off48,r4 - VPERM(v10,v10,v10,byteswap) - VPERM(v11,v11,v11,byteswap) - lvx v12,off64,r4 - lvx v13,off80,r4 - VPERM(v12,v12,v12,byteswap) - VPERM(v13,v13,v13,byteswap) - lvx v14,off96,r4 - lvx v15,off112,r4 - VPERM(v14,v14,v14,byteswap) - VPERM(v15,v15,v15,byteswap) - - addi r4,r4,8*16 - - vxor v16,v0,v8 - vxor v17,v1,v9 - vxor v18,v2,v10 - vxor v19,v3,v11 - vxor v20,v4,v12 - vxor v21,v5,v13 - vxor v22,v6,v14 - vxor v23,v7,v15 - - li r0,1 - cmpdi r6,0 - addi r6,r6,128 - bne 1b - - /* Work out how many bytes we have left */ - andi. r5,r5,127 - - /* Calculate where in the constant table we need to start */ - subfic r6,r5,128 - add r3,r3,r6 - - /* How many 16 byte chunks are in the tail */ - srdi r7,r5,4 - mtctr r7 - - /* - * Reduce the previously calculated 1024 bits to 64 bits, shifting - * 32 bits to include the trailing 32 bits of zeros - */ - lvx v0,0,r3 - lvx v1,off16,r3 - lvx v2,off32,r3 - lvx v3,off48,r3 - lvx v4,off64,r3 - lvx v5,off80,r3 - lvx v6,off96,r3 - lvx v7,off112,r3 - addi r3,r3,8*16 - - VPMSUMW(v0,v16,v0) - VPMSUMW(v1,v17,v1) - VPMSUMW(v2,v18,v2) - VPMSUMW(v3,v19,v3) - VPMSUMW(v4,v20,v4) - VPMSUMW(v5,v21,v5) - VPMSUMW(v6,v22,v6) - VPMSUMW(v7,v23,v7) - - /* Now reduce the tail (0 - 112 bytes) */ - cmpdi r7,0 - beq 1f - - lvx v16,0,r4 - lvx v17,0,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off16,r4 - lvx v17,off16,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off32,r4 - lvx v17,off32,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off48,r4 - lvx v17,off48,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off64,r4 - lvx v17,off64,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off80,r4 - lvx v17,off80,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - bdz 1f - - lvx v16,off96,r4 - lvx v17,off96,r3 - VPERM(v16,v16,v16,byteswap) - VPMSUMW(v16,v16,v17) - vxor v0,v0,v16 - - /* Now xor all the parallel chunks together */ -1: vxor v0,v0,v1 - vxor v2,v2,v3 - vxor v4,v4,v5 - vxor v6,v6,v7 - - vxor v0,v0,v2 - vxor v4,v4,v6 - - vxor v0,v0,v4 - -.Lbarrett_reduction: - /* Barrett constants */ - LOAD_REG_ADDR(r3, .barrett_constants) - - lvx const1,0,r3 - lvx const2,off16,r3 - - vsldoi v1,v0,v0,8 - vxor v0,v0,v1 /* xor two 64 bit results together */ - -#ifdef REFLECT - /* shift left one bit */ - vspltisb v1,1 - vsl v0,v0,v1 -#endif - - vand v0,v0,mask_64bit -#ifndef REFLECT - /* - * Now for the Barrett reduction algorithm. The idea is to calculate q, - * the multiple of our polynomial that we need to subtract. By - * doing the computation 2x bits higher (ie 64 bits) and shifting the - * result back down 2x bits, we round down to the nearest multiple. - */ - VPMSUMD(v1,v0,const1) /* ma */ - vsldoi v1,zeroes,v1,8 /* q = floor(ma/(2^64)) */ - VPMSUMD(v1,v1,const2) /* qn */ - vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ - - /* - * Get the result into r3. We need to shift it left 8 bytes: - * V0 [ 0 1 2 X ] - * V0 [ 0 X 2 3 ] - */ - vsldoi v0,v0,zeroes,8 /* shift result into top 64 bits */ -#else - /* - * The reflected version of Barrett reduction. Instead of bit - * reflecting our data (which is expensive to do), we bit reflect our - * constants and our algorithm, which means the intermediate data in - * our vector registers goes from 0-63 instead of 63-0. We can reflect - * the algorithm because we don't carry in mod 2 arithmetic. - */ - vand v1,v0,mask_32bit /* bottom 32 bits of a */ - VPMSUMD(v1,v1,const1) /* ma */ - vand v1,v1,mask_32bit /* bottom 32bits of ma */ - VPMSUMD(v1,v1,const2) /* qn */ - vxor v0,v0,v1 /* a - qn, subtraction is xor in GF(2) */ - - /* - * Since we are bit reflected, the result (ie the low 32 bits) is in - * the high 32 bits. We just need to shift it left 4 bytes - * V0 [ 0 1 X 3 ] - * V0 [ 0 X 2 3 ] - */ - vsldoi v0,v0,zeroes,4 /* shift result into top 64 bits of */ -#endif - - /* Get it into r3 */ - MFVRD(R3, v0) - -.Lout: - subi r6,r1,56+10*16 - subi r7,r1,56+2*16 - - lvx v20,0,r6 - lvx v21,off16,r6 - lvx v22,off32,r6 - lvx v23,off48,r6 - lvx v24,off64,r6 - lvx v25,off80,r6 - lvx v26,off96,r6 - lvx v27,off112,r6 - lvx v28,0,r7 - lvx v29,off16,r7 - - ld r31,-8(r1) - ld r30,-16(r1) - ld r29,-24(r1) - ld r28,-32(r1) - ld r27,-40(r1) - ld r26,-48(r1) - ld r25,-56(r1) - - blr - -.Lfirst_warm_up_done: - lvx const1,0,r3 - addi r3,r3,16 - - VPMSUMD(v8,v16,const1) - VPMSUMD(v9,v17,const1) - VPMSUMD(v10,v18,const1) - VPMSUMD(v11,v19,const1) - VPMSUMD(v12,v20,const1) - VPMSUMD(v13,v21,const1) - VPMSUMD(v14,v22,const1) - VPMSUMD(v15,v23,const1) - - b .Lsecond_cool_down - -.Lshort: - cmpdi r5,0 - beq .Lzero - - LOAD_REG_ADDR(r3, .short_constants) - - /* Calculate where in the constant table we need to start */ - subfic r6,r5,256 - add r3,r3,r6 - - /* How many 16 byte chunks? */ - srdi r7,r5,4 - mtctr r7 - - vxor v19,v19,v19 - vxor v20,v20,v20 - - lvx v0,0,r4 - lvx v16,0,r3 - VPERM(v0,v0,v16,byteswap) - vxor v0,v0,v8 /* xor in initial value */ - VPMSUMW(v0,v0,v16) - bdz .Lv0 - - lvx v1,off16,r4 - lvx v17,off16,r3 - VPERM(v1,v1,v17,byteswap) - VPMSUMW(v1,v1,v17) - bdz .Lv1 - - lvx v2,off32,r4 - lvx v16,off32,r3 - VPERM(v2,v2,v16,byteswap) - VPMSUMW(v2,v2,v16) - bdz .Lv2 - - lvx v3,off48,r4 - lvx v17,off48,r3 - VPERM(v3,v3,v17,byteswap) - VPMSUMW(v3,v3,v17) - bdz .Lv3 - - lvx v4,off64,r4 - lvx v16,off64,r3 - VPERM(v4,v4,v16,byteswap) - VPMSUMW(v4,v4,v16) - bdz .Lv4 - - lvx v5,off80,r4 - lvx v17,off80,r3 - VPERM(v5,v5,v17,byteswap) - VPMSUMW(v5,v5,v17) - bdz .Lv5 - - lvx v6,off96,r4 - lvx v16,off96,r3 - VPERM(v6,v6,v16,byteswap) - VPMSUMW(v6,v6,v16) - bdz .Lv6 - - lvx v7,off112,r4 - lvx v17,off112,r3 - VPERM(v7,v7,v17,byteswap) - VPMSUMW(v7,v7,v17) - bdz .Lv7 - - addi r3,r3,128 - addi r4,r4,128 - - lvx v8,0,r4 - lvx v16,0,r3 - VPERM(v8,v8,v16,byteswap) - VPMSUMW(v8,v8,v16) - bdz .Lv8 - - lvx v9,off16,r4 - lvx v17,off16,r3 - VPERM(v9,v9,v17,byteswap) - VPMSUMW(v9,v9,v17) - bdz .Lv9 - - lvx v10,off32,r4 - lvx v16,off32,r3 - VPERM(v10,v10,v16,byteswap) - VPMSUMW(v10,v10,v16) - bdz .Lv10 - - lvx v11,off48,r4 - lvx v17,off48,r3 - VPERM(v11,v11,v17,byteswap) - VPMSUMW(v11,v11,v17) - bdz .Lv11 - - lvx v12,off64,r4 - lvx v16,off64,r3 - VPERM(v12,v12,v16,byteswap) - VPMSUMW(v12,v12,v16) - bdz .Lv12 - - lvx v13,off80,r4 - lvx v17,off80,r3 - VPERM(v13,v13,v17,byteswap) - VPMSUMW(v13,v13,v17) - bdz .Lv13 - - lvx v14,off96,r4 - lvx v16,off96,r3 - VPERM(v14,v14,v16,byteswap) - VPMSUMW(v14,v14,v16) - bdz .Lv14 - - lvx v15,off112,r4 - lvx v17,off112,r3 - VPERM(v15,v15,v17,byteswap) - VPMSUMW(v15,v15,v17) - -.Lv15: vxor v19,v19,v15 -.Lv14: vxor v20,v20,v14 -.Lv13: vxor v19,v19,v13 -.Lv12: vxor v20,v20,v12 -.Lv11: vxor v19,v19,v11 -.Lv10: vxor v20,v20,v10 -.Lv9: vxor v19,v19,v9 -.Lv8: vxor v20,v20,v8 -.Lv7: vxor v19,v19,v7 -.Lv6: vxor v20,v20,v6 -.Lv5: vxor v19,v19,v5 -.Lv4: vxor v20,v20,v4 -.Lv3: vxor v19,v19,v3 -.Lv2: vxor v20,v20,v2 -.Lv1: vxor v19,v19,v1 -.Lv0: vxor v20,v20,v0 - - vxor v0,v19,v20 - - b .Lbarrett_reduction - -.Lzero: - mr r3,r10 - b .Lout - -FUNC_END(CRC_FUNCTION_NAME) diff --git a/arch/powerpc/crypto/crc32c-vpmsum_asm.S b/arch/powerpc/crypto/crc32c-vpmsum_asm.S deleted file mode 100644 index bf442004ea1f..000000000000 --- a/arch/powerpc/crypto/crc32c-vpmsum_asm.S +++ /dev/null @@ -1,842 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Calculate a crc32c with vpmsum acceleration - * - * Copyright (C) 2015 Anton Blanchard <anton@au.ibm.com>, IBM - */ - .section .rodata -.balign 16 - -.byteswap_constant: - /* byte reverse permute constant */ - .octa 0x0F0E0D0C0B0A09080706050403020100 - -.constants: - - /* Reduce 262144 kbits to 1024 bits */ - /* x^261120 mod p(x)` << 1, x^261184 mod p(x)` << 1 */ - .octa 0x00000000b6ca9e20000000009c37c408 - - /* x^260096 mod p(x)` << 1, x^260160 mod p(x)` << 1 */ - .octa 0x00000000350249a800000001b51df26c - - /* x^259072 mod p(x)` << 1, x^259136 mod p(x)` << 1 */ - .octa 0x00000001862dac54000000000724b9d0 - - /* x^258048 mod p(x)` << 1, x^258112 mod p(x)` << 1 */ - .octa 0x00000001d87fb48c00000001c00532fe - - /* x^257024 mod p(x)` << 1, x^257088 mod p(x)` << 1 */ - .octa 0x00000001f39b699e00000000f05a9362 - - /* x^256000 mod p(x)` << 1, x^256064 mod p(x)` << 1 */ - .octa 0x0000000101da11b400000001e1007970 - - /* x^254976 mod p(x)` << 1, x^255040 mod p(x)` << 1 */ - .octa 0x00000001cab571e000000000a57366ee - - /* x^253952 mod p(x)` << 1, x^254016 mod p(x)` << 1 */ - .octa 0x00000000c7020cfe0000000192011284 - - /* x^252928 mod p(x)` << 1, x^252992 mod p(x)` << 1 */ - .octa 0x00000000cdaed1ae0000000162716d9a - - /* x^251904 mod p(x)` << 1, x^251968 mod p(x)` << 1 */ - .octa 0x00000001e804effc00000000cd97ecde - - /* x^250880 mod p(x)` << 1, x^250944 mod p(x)` << 1 */ - .octa 0x0000000077c3ea3a0000000058812bc0 - - /* x^249856 mod p(x)` << 1, x^249920 mod p(x)` << 1 */ - .octa 0x0000000068df31b40000000088b8c12e - - /* x^248832 mod p(x)` << 1, x^248896 mod p(x)` << 1 */ - .octa 0x00000000b059b6c200000001230b234c - - /* x^247808 mod p(x)` << 1, x^247872 mod p(x)` << 1 */ - .octa 0x0000000145fb8ed800000001120b416e - - /* x^246784 mod p(x)` << 1, x^246848 mod p(x)` << 1 */ - .octa 0x00000000cbc0916800000001974aecb0 - - /* x^245760 mod p(x)` << 1, x^245824 mod p(x)` << 1 */ - .octa 0x000000005ceeedc2000000008ee3f226 - - /* x^244736 mod p(x)` << 1, x^244800 mod p(x)` << 1 */ - .octa 0x0000000047d74e8600000001089aba9a - - /* x^243712 mod p(x)` << 1, x^243776 mod p(x)` << 1 */ - .octa 0x00000001407e9e220000000065113872 - - /* x^242688 mod p(x)` << 1, x^242752 mod p(x)` << 1 */ - .octa 0x00000001da967bda000000005c07ec10 - - /* x^241664 mod p(x)` << 1, x^241728 mod p(x)` << 1 */ - .octa 0x000000006c8983680000000187590924 - - /* x^240640 mod p(x)` << 1, x^240704 mod p(x)` << 1 */ - .octa 0x00000000f2d14c9800000000e35da7c6 - - /* x^239616 mod p(x)` << 1, x^239680 mod p(x)` << 1 */ - .octa 0x00000001993c6ad4000000000415855a - - /* x^238592 mod p(x)` << 1, x^238656 mod p(x)` << 1 */ - .octa 0x000000014683d1ac0000000073617758 - - /* x^237568 mod p(x)` << 1, x^237632 mod p(x)` << 1 */ - .octa 0x00000001a7c93e6c0000000176021d28 - - /* x^236544 mod p(x)` << 1, x^236608 mod p(x)` << 1 */ - .octa 0x000000010211e90a00000001c358fd0a - - /* x^235520 mod p(x)` << 1, x^235584 mod p(x)` << 1 */ - .octa 0x000000001119403e00000001ff7a2c18 - - /* x^234496 mod p(x)` << 1, x^234560 mod p(x)` << 1 */ - .octa 0x000000001c3261aa00000000f2d9f7e4 - - /* x^233472 mod p(x)` << 1, x^233536 mod p(x)` << 1 */ - .octa 0x000000014e37a634000000016cf1f9c8 - - /* x^232448 mod p(x)` << 1, x^232512 mod p(x)` << 1 */ - .octa 0x0000000073786c0c000000010af9279a - - /* x^231424 mod p(x)` << 1, x^231488 mod p(x)` << 1 */ - .octa 0x000000011dc037f80000000004f101e8 - - /* x^230400 mod p(x)` << 1, x^230464 mod p(x)` << 1 */ - .octa 0x0000000031433dfc0000000070bcf184 - - /* x^229376 mod p(x)` << 1, x^229440 mod p(x)` << 1 */ - .octa 0x000000009cde8348000000000a8de642 - - /* x^228352 mod p(x)` << 1, x^228416 mod p(x)` << 1 */ - .octa 0x0000000038d3c2a60000000062ea130c - - /* x^227328 mod p(x)` << 1, x^227392 mod p(x)` << 1 */ - .octa 0x000000011b25f26000000001eb31cbb2 - - /* x^226304 mod p(x)` << 1, x^226368 mod p(x)` << 1 */ - .octa 0x000000001629e6f00000000170783448 - - /* x^225280 mod p(x)` << 1, x^225344 mod p(x)` << 1 */ - .octa 0x0000000160838b4c00000001a684b4c6 - - /* x^224256 mod p(x)` << 1, x^224320 mod p(x)` << 1 */ - .octa 0x000000007a44011c00000000253ca5b4 - - /* x^223232 mod p(x)` << 1, x^223296 mod p(x)` << 1 */ - .octa 0x00000000226f417a0000000057b4b1e2 - - /* x^222208 mod p(x)` << 1, x^222272 mod p(x)` << 1 */ - .octa 0x0000000045eb2eb400000000b6bd084c - - /* x^221184 mod p(x)` << 1, x^221248 mod p(x)` << 1 */ - .octa 0x000000014459d70c0000000123c2d592 - - /* x^220160 mod p(x)` << 1, x^220224 mod p(x)` << 1 */ - .octa 0x00000001d406ed8200000000159dafce - - /* x^219136 mod p(x)` << 1, x^219200 mod p(x)` << 1 */ - .octa 0x0000000160c8e1a80000000127e1a64e - - /* x^218112 mod p(x)` << 1, x^218176 mod p(x)` << 1 */ - .octa 0x0000000027ba80980000000056860754 - - /* x^217088 mod p(x)` << 1, x^217152 mod p(x)` << 1 */ - .octa 0x000000006d92d01800000001e661aae8 - - /* x^216064 mod p(x)` << 1, x^216128 mod p(x)` << 1 */ - .octa 0x000000012ed7e3f200000000f82c6166 - - /* x^215040 mod p(x)` << 1, x^215104 mod p(x)` << 1 */ - .octa 0x000000002dc8778800000000c4f9c7ae - - /* x^214016 mod p(x)` << 1, x^214080 mod p(x)` << 1 */ - .octa 0x0000000018240bb80000000074203d20 - - /* x^212992 mod p(x)` << 1, x^213056 mod p(x)` << 1 */ - .octa 0x000000001ad381580000000198173052 - - /* x^211968 mod p(x)` << 1, x^212032 mod p(x)` << 1 */ - .octa 0x00000001396b78f200000001ce8aba54 - - /* x^210944 mod p(x)` << 1, x^211008 mod p(x)` << 1 */ - .octa 0x000000011a68133400000001850d5d94 - - /* x^209920 mod p(x)` << 1, x^209984 mod p(x)` << 1 */ - .octa 0x000000012104732e00000001d609239c - - /* x^208896 mod p(x)` << 1, x^208960 mod p(x)` << 1 */ - .octa 0x00000000a140d90c000000001595f048 - - /* x^207872 mod p(x)` << 1, x^207936 mod p(x)` << 1 */ - .octa 0x00000001b7215eda0000000042ccee08 - - /* x^206848 mod p(x)` << 1, x^206912 mod p(x)` << 1 */ - .octa 0x00000001aaf1df3c000000010a389d74 - - /* x^205824 mod p(x)` << 1, x^205888 mod p(x)` << 1 */ - .octa 0x0000000029d15b8a000000012a840da6 - - /* x^204800 mod p(x)` << 1, x^204864 mod p(x)` << 1 */ - .octa 0x00000000f1a96922000000001d181c0c - - /* x^203776 mod p(x)` << 1, x^203840 mod p(x)` << 1 */ - .octa 0x00000001ac80d03c0000000068b7d1f6 - - /* x^202752 mod p(x)` << 1, x^202816 mod p(x)` << 1 */ - .octa 0x000000000f11d56a000000005b0f14fc - - /* x^201728 mod p(x)` << 1, x^201792 mod p(x)` << 1 */ - .octa 0x00000001f1c022a20000000179e9e730 - - /* x^200704 mod p(x)` << 1, x^200768 mod p(x)` << 1 */ - .octa 0x0000000173d00ae200000001ce1368d6 - - /* x^199680 mod p(x)` << 1, x^199744 mod p(x)` << 1 */ - .octa 0x00000001d4ffe4ac0000000112c3a84c - - /* x^198656 mod p(x)` << 1, x^198720 mod p(x)` << 1 */ - .octa 0x000000016edc5ae400000000de940fee - - /* x^197632 mod p(x)` << 1, x^197696 mod p(x)` << 1 */ - .octa 0x00000001f1a0214000000000fe896b7e - - /* x^196608 mod p(x)` << 1, x^196672 mod p(x)` << 1 */ - .octa 0x00000000ca0b28a000000001f797431c - - /* x^195584 mod p(x)` << 1, x^195648 mod p(x)` << 1 */ - .octa 0x00000001928e30a20000000053e989ba - - /* x^194560 mod p(x)` << 1, x^194624 mod p(x)` << 1 */ - .octa 0x0000000097b1b002000000003920cd16 - - /* x^193536 mod p(x)` << 1, x^193600 mod p(x)` << 1 */ - .octa 0x00000000b15bf90600000001e6f579b8 - - /* x^192512 mod p(x)` << 1, x^192576 mod p(x)` << 1 */ - .octa 0x00000000411c5d52000000007493cb0a - - /* x^191488 mod p(x)` << 1, x^191552 mod p(x)` << 1 */ - .octa 0x00000001c36f330000000001bdd376d8 - - /* x^190464 mod p(x)` << 1, x^190528 mod p(x)` << 1 */ - .octa 0x00000001119227e0000000016badfee6 - - /* x^189440 mod p(x)` << 1, x^189504 mod p(x)` << 1 */ - .octa 0x00000000114d47020000000071de5c58 - - /* x^188416 mod p(x)` << 1, x^188480 mod p(x)` << 1 */ - .octa 0x00000000458b5b9800000000453f317c - - /* x^187392 mod p(x)` << 1, x^187456 mod p(x)` << 1 */ - .octa 0x000000012e31fb8e0000000121675cce - - /* x^186368 mod p(x)` << 1, x^186432 mod p(x)` << 1 */ - .octa 0x000000005cf619d800000001f409ee92 - - /* x^185344 mod p(x)` << 1, x^185408 mod p(x)` << 1 */ - .octa 0x0000000063f4d8b200000000f36b9c88 - - /* x^184320 mod p(x)` << 1, x^184384 mod p(x)` << 1 */ - .octa 0x000000004138dc8a0000000036b398f4 - - /* x^183296 mod p(x)` << 1, x^183360 mod p(x)` << 1 */ - .octa 0x00000001d29ee8e000000001748f9adc - - /* x^182272 mod p(x)` << 1, x^182336 mod p(x)` << 1 */ - .octa 0x000000006a08ace800000001be94ec00 - - /* x^181248 mod p(x)` << 1, x^181312 mod p(x)` << 1 */ - .octa 0x0000000127d4201000000000b74370d6 - - /* x^180224 mod p(x)` << 1, x^180288 mod p(x)` << 1 */ - .octa 0x0000000019d76b6200000001174d0b98 - - /* x^179200 mod p(x)` << 1, x^179264 mod p(x)` << 1 */ - .octa 0x00000001b1471f6e00000000befc06a4 - - /* x^178176 mod p(x)` << 1, x^178240 mod p(x)` << 1 */ - .octa 0x00000001f64c19cc00000001ae125288 - - /* x^177152 mod p(x)` << 1, x^177216 mod p(x)` << 1 */ - .octa 0x00000000003c0ea00000000095c19b34 - - /* x^176128 mod p(x)` << 1, x^176192 mod p(x)` << 1 */ - .octa 0x000000014d73abf600000001a78496f2 - - /* x^175104 mod p(x)` << 1, x^175168 mod p(x)` << 1 */ - .octa 0x00000001620eb84400000001ac5390a0 - - /* x^174080 mod p(x)` << 1, x^174144 mod p(x)` << 1 */ - .octa 0x0000000147655048000000002a80ed6e - - /* x^173056 mod p(x)` << 1, x^173120 mod p(x)` << 1 */ - .octa 0x0000000067b5077e00000001fa9b0128 - - /* x^172032 mod p(x)` << 1, x^172096 mod p(x)` << 1 */ - .octa 0x0000000010ffe20600000001ea94929e - - /* x^171008 mod p(x)` << 1, x^171072 mod p(x)` << 1 */ - .octa 0x000000000fee8f1e0000000125f4305c - - /* x^169984 mod p(x)` << 1, x^170048 mod p(x)` << 1 */ - .octa 0x00000001da26fbae00000001471e2002 - - /* x^168960 mod p(x)` << 1, x^169024 mod p(x)` << 1 */ - .octa 0x00000001b3a8bd880000000132d2253a - - /* x^167936 mod p(x)` << 1, x^168000 mod p(x)` << 1 */ - .octa 0x00000000e8f3898e00000000f26b3592 - - /* x^166912 mod p(x)` << 1, x^166976 mod p(x)` << 1 */ - .octa 0x00000000b0d0d28c00000000bc8b67b0 - - /* x^165888 mod p(x)` << 1, x^165952 mod p(x)` << 1 */ - .octa 0x0000000030f2a798000000013a826ef2 - - /* x^164864 mod p(x)` << 1, x^164928 mod p(x)` << 1 */ - .octa 0x000000000fba10020000000081482c84 - - /* x^163840 mod p(x)` << 1, x^163904 mod p(x)` << 1 */ - .octa 0x00000000bdb9bd7200000000e77307c2 - - /* x^162816 mod p(x)` << 1, x^162880 mod p(x)` << 1 */ - .octa 0x0000000075d3bf5a00000000d4a07ec8 - - /* x^161792 mod p(x)` << 1, x^161856 mod p(x)` << 1 */ - .octa 0x00000000ef1f98a00000000017102100 - - /* x^160768 mod p(x)` << 1, x^160832 mod p(x)` << 1 */ - .octa 0x00000000689c760200000000db406486 - - /* x^159744 mod p(x)` << 1, x^159808 mod p(x)` << 1 */ - .octa 0x000000016d5fa5fe0000000192db7f88 - - /* x^158720 mod p(x)` << 1, x^158784 mod p(x)` << 1 */ - .octa 0x00000001d0d2b9ca000000018bf67b1e - - /* x^157696 mod p(x)` << 1, x^157760 mod p(x)` << 1 */ - .octa 0x0000000041e7b470000000007c09163e - - /* x^156672 mod p(x)` << 1, x^156736 mod p(x)` << 1 */ - .octa 0x00000001cbb6495e000000000adac060 - - /* x^155648 mod p(x)` << 1, x^155712 mod p(x)` << 1 */ - .octa 0x000000010052a0b000000000bd8316ae - - /* x^154624 mod p(x)` << 1, x^154688 mod p(x)` << 1 */ - .octa 0x00000001d8effb5c000000019f09ab54 - - /* x^153600 mod p(x)` << 1, x^153664 mod p(x)` << 1 */ - .octa 0x00000001d969853c0000000125155542 - - /* x^152576 mod p(x)` << 1, x^152640 mod p(x)` << 1 */ - .octa 0x00000000523ccce2000000018fdb5882 - - /* x^151552 mod p(x)` << 1, x^151616 mod p(x)` << 1 */ - .octa 0x000000001e2436bc00000000e794b3f4 - - /* x^150528 mod p(x)` << 1, x^150592 mod p(x)` << 1 */ - .octa 0x00000000ddd1c3a2000000016f9bb022 - - /* x^149504 mod p(x)` << 1, x^149568 mod p(x)` << 1 */ - .octa 0x0000000019fcfe3800000000290c9978 - - /* x^148480 mod p(x)` << 1, x^148544 mod p(x)` << 1 */ - .octa 0x00000001ce95db640000000083c0f350 - - /* x^147456 mod p(x)` << 1, x^147520 mod p(x)` << 1 */ - .octa 0x00000000af5828060000000173ea6628 - - /* x^146432 mod p(x)` << 1, x^146496 mod p(x)` << 1 */ - .octa 0x00000001006388f600000001c8b4e00a - - /* x^145408 mod p(x)` << 1, x^145472 mod p(x)` << 1 */ - .octa 0x0000000179eca00a00000000de95d6aa - - /* x^144384 mod p(x)` << 1, x^144448 mod p(x)` << 1 */ - .octa 0x0000000122410a6a000000010b7f7248 - - /* x^143360 mod p(x)` << 1, x^143424 mod p(x)` << 1 */ - .octa 0x000000004288e87c00000001326e3a06 - - /* x^142336 mod p(x)` << 1, x^142400 mod p(x)` << 1 */ - .octa 0x000000016c5490da00000000bb62c2e6 - - /* x^141312 mod p(x)` << 1, x^141376 mod p(x)` << 1 */ - .octa 0x00000000d1c71f6e0000000156a4b2c2 - - /* x^140288 mod p(x)` << 1, x^140352 mod p(x)` << 1 */ - .octa 0x00000001b4ce08a6000000011dfe763a - - /* x^139264 mod p(x)` << 1, x^139328 mod p(x)` << 1 */ - .octa 0x00000001466ba60c000000007bcca8e2 - - /* x^138240 mod p(x)` << 1, x^138304 mod p(x)` << 1 */ - .octa 0x00000001f6c488a40000000186118faa - - /* x^137216 mod p(x)` << 1, x^137280 mod p(x)` << 1 */ - .octa 0x000000013bfb06820000000111a65a88 - - /* x^136192 mod p(x)` << 1, x^136256 mod p(x)` << 1 */ - .octa 0x00000000690e9e54000000003565e1c4 - - /* x^135168 mod p(x)` << 1, x^135232 mod p(x)` << 1 */ - .octa 0x00000000281346b6000000012ed02a82 - - /* x^134144 mod p(x)` << 1, x^134208 mod p(x)` << 1 */ - .octa 0x000000015646402400000000c486ecfc - - /* x^133120 mod p(x)` << 1, x^133184 mod p(x)` << 1 */ - .octa 0x000000016063a8dc0000000001b951b2 - - /* x^132096 mod p(x)` << 1, x^132160 mod p(x)` << 1 */ - .octa 0x0000000116a663620000000048143916 - - /* x^131072 mod p(x)` << 1, x^131136 mod p(x)` << 1 */ - .octa 0x000000017e8aa4d200000001dc2ae124 - - /* x^130048 mod p(x)` << 1, x^130112 mod p(x)` << 1 */ - .octa 0x00000001728eb10c00000001416c58d6 - - /* x^129024 mod p(x)` << 1, x^129088 mod p(x)` << 1 */ - .octa 0x00000001b08fd7fa00000000a479744a - - /* x^128000 mod p(x)` << 1, x^128064 mod p(x)` << 1 */ - .octa 0x00000001092a16e80000000096ca3a26 - - /* x^126976 mod p(x)` << 1, x^127040 mod p(x)` << 1 */ - .octa 0x00000000a505637c00000000ff223d4e - - /* x^125952 mod p(x)` << 1, x^126016 mod p(x)` << 1 */ - .octa 0x00000000d94869b2000000010e84da42 - - /* x^124928 mod p(x)` << 1, x^124992 mod p(x)` << 1 */ - .octa 0x00000001c8b203ae00000001b61ba3d0 - - /* x^123904 mod p(x)` << 1, x^123968 mod p(x)` << 1 */ - .octa 0x000000005704aea000000000680f2de8 - - /* x^122880 mod p(x)` << 1, x^122944 mod p(x)` << 1 */ - .octa 0x000000012e295fa2000000008772a9a8 - - /* x^121856 mod p(x)` << 1, x^121920 mod p(x)` << 1 */ - .octa 0x000000011d0908bc0000000155f295bc - - /* x^120832 mod p(x)` << 1, x^120896 mod p(x)` << 1 */ - .octa 0x0000000193ed97ea00000000595f9282 - - /* x^119808 mod p(x)` << 1, x^119872 mod p(x)` << 1 */ - .octa 0x000000013a0f1c520000000164b1c25a - - /* x^118784 mod p(x)` << 1, x^118848 mod p(x)` << 1 */ - .octa 0x000000010c2c40c000000000fbd67c50 - - /* x^117760 mod p(x)` << 1, x^117824 mod p(x)` << 1 */ - .octa 0x00000000ff6fac3e0000000096076268 - - /* x^116736 mod p(x)` << 1, x^116800 mod p(x)` << 1 */ - .octa 0x000000017b3609c000000001d288e4cc - - /* x^115712 mod p(x)` << 1, x^115776 mod p(x)` << 1 */ - .octa 0x0000000088c8c92200000001eaac1bdc - - /* x^114688 mod p(x)` << 1, x^114752 mod p(x)` << 1 */ - .octa 0x00000001751baae600000001f1ea39e2 - - /* x^113664 mod p(x)` << 1, x^113728 mod p(x)` << 1 */ - .octa 0x000000010795297200000001eb6506fc - - /* x^112640 mod p(x)` << 1, x^112704 mod p(x)` << 1 */ - .octa 0x0000000162b00abe000000010f806ffe - - /* x^111616 mod p(x)` << 1, x^111680 mod p(x)` << 1 */ - .octa 0x000000000d7b404c000000010408481e - - /* x^110592 mod p(x)` << 1, x^110656 mod p(x)` << 1 */ - .octa 0x00000000763b13d40000000188260534 - - /* x^109568 mod p(x)` << 1, x^109632 mod p(x)` << 1 */ - .octa 0x00000000f6dc22d80000000058fc73e0 - - /* x^108544 mod p(x)` << 1, x^108608 mod p(x)` << 1 */ - .octa 0x000000007daae06000000000391c59b8 - - /* x^107520 mod p(x)` << 1, x^107584 mod p(x)` << 1 */ - .octa 0x000000013359ab7c000000018b638400 - - /* x^106496 mod p(x)` << 1, x^106560 mod p(x)` << 1 */ - .octa 0x000000008add438a000000011738f5c4 - - /* x^105472 mod p(x)` << 1, x^105536 mod p(x)` << 1 */ - .octa 0x00000001edbefdea000000008cf7c6da - - /* x^104448 mod p(x)` << 1, x^104512 mod p(x)` << 1 */ - .octa 0x000000004104e0f800000001ef97fb16 - - /* x^103424 mod p(x)` << 1, x^103488 mod p(x)` << 1 */ - .octa 0x00000000b48a82220000000102130e20 - - /* x^102400 mod p(x)` << 1, x^102464 mod p(x)` << 1 */ - .octa 0x00000001bcb4684400000000db968898 - - /* x^101376 mod p(x)` << 1, x^101440 mod p(x)` << 1 */ - .octa 0x000000013293ce0a00000000b5047b5e - - /* x^100352 mod p(x)` << 1, x^100416 mod p(x)` << 1 */ - .octa 0x00000001710d0844000000010b90fdb2 - - /* x^99328 mod p(x)` << 1, x^99392 mod p(x)` << 1 */ - .octa 0x0000000117907f6e000000004834a32e - - /* x^98304 mod p(x)` << 1, x^98368 mod p(x)` << 1 */ - .octa 0x0000000087ddf93e0000000059c8f2b0 - - /* x^97280 mod p(x)` << 1, x^97344 mod p(x)` << 1 */ - .octa 0x000000005970e9b00000000122cec508 - - /* x^96256 mod p(x)` << 1, x^96320 mod p(x)` << 1 */ - .octa 0x0000000185b2b7d0000000000a330cda - - /* x^95232 mod p(x)` << 1, x^95296 mod p(x)` << 1 */ - .octa 0x00000001dcee0efc000000014a47148c - - /* x^94208 mod p(x)` << 1, x^94272 mod p(x)` << 1 */ - .octa 0x0000000030da27220000000042c61cb8 - - /* x^93184 mod p(x)` << 1, x^93248 mod p(x)` << 1 */ - .octa 0x000000012f925a180000000012fe6960 - - /* x^92160 mod p(x)` << 1, x^92224 mod p(x)` << 1 */ - .octa 0x00000000dd2e357c00000000dbda2c20 - - /* x^91136 mod p(x)` << 1, x^91200 mod p(x)` << 1 */ - .octa 0x00000000071c80de000000011122410c - - /* x^90112 mod p(x)` << 1, x^90176 mod p(x)` << 1 */ - .octa 0x000000011513140a00000000977b2070 - - /* x^89088 mod p(x)` << 1, x^89152 mod p(x)` << 1 */ - .octa 0x00000001df876e8e000000014050438e - - /* x^88064 mod p(x)` << 1, x^88128 mod p(x)` << 1 */ - .octa 0x000000015f81d6ce0000000147c840e8 - - /* x^87040 mod p(x)` << 1, x^87104 mod p(x)` << 1 */ - .octa 0x000000019dd94dbe00000001cc7c88ce - - /* x^86016 mod p(x)` << 1, x^86080 mod p(x)` << 1 */ - .octa 0x00000001373d206e00000001476b35a4 - - /* x^84992 mod p(x)` << 1, x^85056 mod p(x)` << 1 */ - .octa 0x00000000668ccade000000013d52d508 - - /* x^83968 mod p(x)` << 1, x^84032 mod p(x)` << 1 */ - .octa 0x00000001b192d268000000008e4be32e - - /* x^82944 mod p(x)` << 1, x^83008 mod p(x)` << 1 */ - .octa 0x00000000e30f3a7800000000024120fe - - /* x^81920 mod p(x)` << 1, x^81984 mod p(x)` << 1 */ - .octa 0x000000010ef1f7bc00000000ddecddb4 - - /* x^80896 mod p(x)` << 1, x^80960 mod p(x)` << 1 */ - .octa 0x00000001f5ac738000000000d4d403bc - - /* x^79872 mod p(x)` << 1, x^79936 mod p(x)` << 1 */ - .octa 0x000000011822ea7000000001734b89aa - - /* x^78848 mod p(x)` << 1, x^78912 mod p(x)` << 1 */ - .octa 0x00000000c3a33848000000010e7a58d6 - - /* x^77824 mod p(x)` << 1, x^77888 mod p(x)` << 1 */ - .octa 0x00000001bd151c2400000001f9f04e9c - - /* x^76800 mod p(x)` << 1, x^76864 mod p(x)` << 1 */ - .octa 0x0000000056002d7600000000b692225e - - /* x^75776 mod p(x)` << 1, x^75840 mod p(x)` << 1 */ - .octa 0x000000014657c4f4000000019b8d3f3e - - /* x^74752 mod p(x)` << 1, x^74816 mod p(x)` << 1 */ - .octa 0x0000000113742d7c00000001a874f11e - - /* x^73728 mod p(x)` << 1, x^73792 mod p(x)` << 1 */ - .octa 0x000000019c5920ba000000010d5a4254 - - /* x^72704 mod p(x)` << 1, x^72768 mod p(x)` << 1 */ - .octa 0x000000005216d2d600000000bbb2f5d6 - - /* x^71680 mod p(x)` << 1, x^71744 mod p(x)` << 1 */ - .octa 0x0000000136f5ad8a0000000179cc0e36 - - /* x^70656 mod p(x)` << 1, x^70720 mod p(x)` << 1 */ - .octa 0x000000018b07beb600000001dca1da4a - - /* x^69632 mod p(x)` << 1, x^69696 mod p(x)` << 1 */ - .octa 0x00000000db1e93b000000000feb1a192 - - /* x^68608 mod p(x)` << 1, x^68672 mod p(x)` << 1 */ - .octa 0x000000000b96fa3a00000000d1eeedd6 - - /* x^67584 mod p(x)` << 1, x^67648 mod p(x)` << 1 */ - .octa 0x00000001d9968af0000000008fad9bb4 - - /* x^66560 mod p(x)` << 1, x^66624 mod p(x)` << 1 */ - .octa 0x000000000e4a77a200000001884938e4 - - /* x^65536 mod p(x)` << 1, x^65600 mod p(x)` << 1 */ - .octa 0x00000000508c2ac800000001bc2e9bc0 - - /* x^64512 mod p(x)` << 1, x^64576 mod p(x)` << 1 */ - .octa 0x0000000021572a8000000001f9658a68 - - /* x^63488 mod p(x)` << 1, x^63552 mod p(x)` << 1 */ - .octa 0x00000001b859daf2000000001b9224fc - - /* x^62464 mod p(x)` << 1, x^62528 mod p(x)` << 1 */ - .octa 0x000000016f7884740000000055b2fb84 - - /* x^61440 mod p(x)` << 1, x^61504 mod p(x)` << 1 */ - .octa 0x00000001b438810e000000018b090348 - - /* x^60416 mod p(x)` << 1, x^60480 mod p(x)` << 1 */ - .octa 0x0000000095ddc6f2000000011ccbd5ea - - /* x^59392 mod p(x)` << 1, x^59456 mod p(x)` << 1 */ - .octa 0x00000001d977c20c0000000007ae47f8 - - /* x^58368 mod p(x)` << 1, x^58432 mod p(x)` << 1 */ - .octa 0x00000000ebedb99a0000000172acbec0 - - /* x^57344 mod p(x)` << 1, x^57408 mod p(x)` << 1 */ - .octa 0x00000001df9e9e9200000001c6e3ff20 - - /* x^56320 mod p(x)` << 1, x^56384 mod p(x)` << 1 */ - .octa 0x00000001a4a3f95200000000e1b38744 - - /* x^55296 mod p(x)` << 1, x^55360 mod p(x)` << 1 */ - .octa 0x00000000e2f5122000000000791585b2 - - /* x^54272 mod p(x)` << 1, x^54336 mod p(x)` << 1 */ - .octa 0x000000004aa01f3e00000000ac53b894 - - /* x^53248 mod p(x)` << 1, x^53312 mod p(x)` << 1 */ - .octa 0x00000000b3e90a5800000001ed5f2cf4 - - /* x^52224 mod p(x)` << 1, x^52288 mod p(x)` << 1 */ - .octa 0x000000000c9ca2aa00000001df48b2e0 - - /* x^51200 mod p(x)` << 1, x^51264 mod p(x)` << 1 */ - .octa 0x000000015168231600000000049c1c62 - - /* x^50176 mod p(x)` << 1, x^50240 mod p(x)` << 1 */ - .octa 0x0000000036fce78c000000017c460c12 - - /* x^49152 mod p(x)` << 1, x^49216 mod p(x)` << 1 */ - .octa 0x000000009037dc10000000015be4da7e - - /* x^48128 mod p(x)` << 1, x^48192 mod p(x)` << 1 */ - .octa 0x00000000d3298582000000010f38f668 - - /* x^47104 mod p(x)` << 1, x^47168 mod p(x)` << 1 */ - .octa 0x00000001b42e8ad60000000039f40a00 - - /* x^46080 mod p(x)` << 1, x^46144 mod p(x)` << 1 */ - .octa 0x00000000142a983800000000bd4c10c4 - - /* x^45056 mod p(x)` << 1, x^45120 mod p(x)` << 1 */ - .octa 0x0000000109c7f1900000000042db1d98 - - /* x^44032 mod p(x)` << 1, x^44096 mod p(x)` << 1 */ - .octa 0x0000000056ff931000000001c905bae6 - - /* x^43008 mod p(x)` << 1, x^43072 mod p(x)` << 1 */ - .octa 0x00000001594513aa00000000069d40ea - - /* x^41984 mod p(x)` << 1, x^42048 mod p(x)` << 1 */ - .octa 0x00000001e3b5b1e8000000008e4fbad0 - - /* x^40960 mod p(x)` << 1, x^41024 mod p(x)` << 1 */ - .octa 0x000000011dd5fc080000000047bedd46 - - /* x^39936 mod p(x)` << 1, x^40000 mod p(x)` << 1 */ - .octa 0x00000001675f0cc20000000026396bf8 - - /* x^38912 mod p(x)` << 1, x^38976 mod p(x)` << 1 */ - .octa 0x00000000d1c8dd4400000000379beb92 - - /* x^37888 mod p(x)` << 1, x^37952 mod p(x)` << 1 */ - .octa 0x0000000115ebd3d8000000000abae54a - - /* x^36864 mod p(x)` << 1, x^36928 mod p(x)` << 1 */ - .octa 0x00000001ecbd0dac0000000007e6a128 - - /* x^35840 mod p(x)` << 1, x^35904 mod p(x)` << 1 */ - .octa 0x00000000cdf67af2000000000ade29d2 - - /* x^34816 mod p(x)` << 1, x^34880 mod p(x)` << 1 */ - .octa 0x000000004c01ff4c00000000f974c45c - - /* x^33792 mod p(x)` << 1, x^33856 mod p(x)` << 1 */ - .octa 0x00000000f2d8657e00000000e77ac60a - - /* x^32768 mod p(x)` << 1, x^32832 mod p(x)` << 1 */ - .octa 0x000000006bae74c40000000145895816 - - /* x^31744 mod p(x)` << 1, x^31808 mod p(x)` << 1 */ - .octa 0x0000000152af8aa00000000038e362be - - /* x^30720 mod p(x)` << 1, x^30784 mod p(x)` << 1 */ - .octa 0x0000000004663802000000007f991a64 - - /* x^29696 mod p(x)` << 1, x^29760 mod p(x)` << 1 */ - .octa 0x00000001ab2f5afc00000000fa366d3a - - /* x^28672 mod p(x)` << 1, x^28736 mod p(x)` << 1 */ - .octa 0x0000000074a4ebd400000001a2bb34f0 - - /* x^27648 mod p(x)` << 1, x^27712 mod p(x)` << 1 */ - .octa 0x00000001d7ab3a4c0000000028a9981e - - /* x^26624 mod p(x)` << 1, x^26688 mod p(x)` << 1 */ - .octa 0x00000001a8da60c600000001dbc672be - - /* x^25600 mod p(x)` << 1, x^25664 mod p(x)` << 1 */ - .octa 0x000000013cf6382000000000b04d77f6 - - /* x^24576 mod p(x)` << 1, x^24640 mod p(x)` << 1 */ - .octa 0x00000000bec12e1e0000000124400d96 - - /* x^23552 mod p(x)` << 1, x^23616 mod p(x)` << 1 */ - .octa 0x00000001c6368010000000014ca4b414 - - /* x^22528 mod p(x)` << 1, x^22592 mod p(x)` << 1 */ - .octa 0x00000001e6e78758000000012fe2c938 - - /* x^21504 mod p(x)` << 1, x^21568 mod p(x)` << 1 */ - .octa 0x000000008d7f2b3c00000001faed01e6 - - /* x^20480 mod p(x)` << 1, x^20544 mod p(x)` << 1 */ - .octa 0x000000016b4a156e000000007e80ecfe - - /* x^19456 mod p(x)` << 1, x^19520 mod p(x)` << 1 */ - .octa 0x00000001c63cfeb60000000098daee94 - - /* x^18432 mod p(x)` << 1, x^18496 mod p(x)` << 1 */ - .octa 0x000000015f902670000000010a04edea - - /* x^17408 mod p(x)` << 1, x^17472 mod p(x)` << 1 */ - .octa 0x00000001cd5de11e00000001c00b4524 - - /* x^16384 mod p(x)` << 1, x^16448 mod p(x)` << 1 */ - .octa 0x000000001acaec540000000170296550 - - /* x^15360 mod p(x)` << 1, x^15424 mod p(x)` << 1 */ - .octa 0x000000002bd0ca780000000181afaa48 - - /* x^14336 mod p(x)` << 1, x^14400 mod p(x)` << 1 */ - .octa 0x0000000032d63d5c0000000185a31ffa - - /* x^13312 mod p(x)` << 1, x^13376 mod p(x)` << 1 */ - .octa 0x000000001c6d4e4c000000002469f608 - - /* x^12288 mod p(x)` << 1, x^12352 mod p(x)` << 1 */ - .octa 0x0000000106a60b92000000006980102a - - /* x^11264 mod p(x)` << 1, x^11328 mod p(x)` << 1 */ - .octa 0x00000000d3855e120000000111ea9ca8 - - /* x^10240 mod p(x)` << 1, x^10304 mod p(x)` << 1 */ - .octa 0x00000000e312563600000001bd1d29ce - - /* x^9216 mod p(x)` << 1, x^9280 mod p(x)` << 1 */ - .octa 0x000000009e8f7ea400000001b34b9580 - - /* x^8192 mod p(x)` << 1, x^8256 mod p(x)` << 1 */ - .octa 0x00000001c82e562c000000003076054e - - /* x^7168 mod p(x)` << 1, x^7232 mod p(x)` << 1 */ - .octa 0x00000000ca9f09ce000000012a608ea4 - - /* x^6144 mod p(x)` << 1, x^6208 mod p(x)` << 1 */ - .octa 0x00000000c63764e600000000784d05fe - - /* x^5120 mod p(x)` << 1, x^5184 mod p(x)` << 1 */ - .octa 0x0000000168d2e49e000000016ef0d82a - - /* x^4096 mod p(x)` << 1, x^4160 mod p(x)` << 1 */ - .octa 0x00000000e986c1480000000075bda454 - - /* x^3072 mod p(x)` << 1, x^3136 mod p(x)` << 1 */ - .octa 0x00000000cfb65894000000003dc0a1c4 - - /* x^2048 mod p(x)` << 1, x^2112 mod p(x)` << 1 */ - .octa 0x0000000111cadee400000000e9a5d8be - - /* x^1024 mod p(x)` << 1, x^1088 mod p(x)` << 1 */ - .octa 0x0000000171fb63ce00000001609bc4b4 - -.short_constants: - - /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ - /* x^1952 mod p(x)`, x^1984 mod p(x)`, x^2016 mod p(x)`, x^2048 mod p(x)` */ - .octa 0x7fec2963e5bf80485cf015c388e56f72 - - /* x^1824 mod p(x)`, x^1856 mod p(x)`, x^1888 mod p(x)`, x^1920 mod p(x)` */ - .octa 0x38e888d4844752a9963a18920246e2e6 - - /* x^1696 mod p(x)`, x^1728 mod p(x)`, x^1760 mod p(x)`, x^1792 mod p(x)` */ - .octa 0x42316c00730206ad419a441956993a31 - - /* x^1568 mod p(x)`, x^1600 mod p(x)`, x^1632 mod p(x)`, x^1664 mod p(x)` */ - .octa 0x543d5c543e65ddf9924752ba2b830011 - - /* x^1440 mod p(x)`, x^1472 mod p(x)`, x^1504 mod p(x)`, x^1536 mod p(x)` */ - .octa 0x78e87aaf56767c9255bd7f9518e4a304 - - /* x^1312 mod p(x)`, x^1344 mod p(x)`, x^1376 mod p(x)`, x^1408 mod p(x)` */ - .octa 0x8f68fcec1903da7f6d76739fe0553f1e - - /* x^1184 mod p(x)`, x^1216 mod p(x)`, x^1248 mod p(x)`, x^1280 mod p(x)` */ - .octa 0x3f4840246791d588c133722b1fe0b5c3 - - /* x^1056 mod p(x)`, x^1088 mod p(x)`, x^1120 mod p(x)`, x^1152 mod p(x)` */ - .octa 0x34c96751b04de25a64b67ee0e55ef1f3 - - /* x^928 mod p(x)`, x^960 mod p(x)`, x^992 mod p(x)`, x^1024 mod p(x)` */ - .octa 0x156c8e180b4a395b069db049b8fdb1e7 - - /* x^800 mod p(x)`, x^832 mod p(x)`, x^864 mod p(x)`, x^896 mod p(x)` */ - .octa 0xe0b99ccbe661f7bea11bfaf3c9e90b9e - - /* x^672 mod p(x)`, x^704 mod p(x)`, x^736 mod p(x)`, x^768 mod p(x)` */ - .octa 0x041d37768cd75659817cdc5119b29a35 - - /* x^544 mod p(x)`, x^576 mod p(x)`, x^608 mod p(x)`, x^640 mod p(x)` */ - .octa 0x3a0777818cfaa9651ce9d94b36c41f1c - - /* x^416 mod p(x)`, x^448 mod p(x)`, x^480 mod p(x)`, x^512 mod p(x)` */ - .octa 0x0e148e8252377a554f256efcb82be955 - - /* x^288 mod p(x)`, x^320 mod p(x)`, x^352 mod p(x)`, x^384 mod p(x)` */ - .octa 0x9c25531d19e65ddeec1631edb2dea967 - - /* x^160 mod p(x)`, x^192 mod p(x)`, x^224 mod p(x)`, x^256 mod p(x)` */ - .octa 0x790606ff9957c0a65d27e147510ac59a - - /* x^32 mod p(x)`, x^64 mod p(x)`, x^96 mod p(x)`, x^128 mod p(x)` */ - .octa 0x82f63b786ea2d55ca66805eb18b8ea18 - - -.barrett_constants: - /* 33 bit reflected Barrett constant m - (4^32)/n */ - .octa 0x000000000000000000000000dea713f1 /* x^64 div p(x)` */ - /* 33 bit reflected Barrett constant n */ - .octa 0x00000000000000000000000105ec76f1 - -#define CRC_FUNCTION_NAME __crc32c_vpmsum -#define REFLECT -#include "crc32-vpmsum_core.S" diff --git a/arch/powerpc/crypto/crc32c-vpmsum_glue.c b/arch/powerpc/crypto/crc32c-vpmsum_glue.c deleted file mode 100644 index 63760b7dbb76..000000000000 --- a/arch/powerpc/crypto/crc32c-vpmsum_glue.c +++ /dev/null @@ -1,173 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include <linux/crc32.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/kernel.h> -#include <linux/cpufeature.h> -#include <asm/simd.h> -#include <asm/switch_to.h> - -#define CHKSUM_BLOCK_SIZE 1 -#define CHKSUM_DIGEST_SIZE 4 - -#define VMX_ALIGN 16 -#define VMX_ALIGN_MASK (VMX_ALIGN-1) - -#define VECTOR_BREAKPOINT 512 - -u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len); - -static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len) -{ - unsigned int prealign; - unsigned int tail; - - if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable()) - return __crc32c_le(crc, p, len); - - if ((unsigned long)p & VMX_ALIGN_MASK) { - prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); - crc = __crc32c_le(crc, p, prealign); - len -= prealign; - p += prealign; - } - - if (len & ~VMX_ALIGN_MASK) { - preempt_disable(); - pagefault_disable(); - enable_kernel_altivec(); - crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); - disable_kernel_altivec(); - pagefault_enable(); - preempt_enable(); - } - - tail = len & VMX_ALIGN_MASK; - if (tail) { - p += len & ~VMX_ALIGN_MASK; - crc = __crc32c_le(crc, p, tail); - } - - return crc; -} - -static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm) -{ - u32 *key = crypto_tfm_ctx(tfm); - - *key = ~0; - - return 0; -} - -/* - * Setting the seed allows arbitrary accumulators and flexible XOR policy - * If your algorithm starts with ~0, then XOR with ~0 before you set - * the seed. - */ -static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key, - unsigned int keylen) -{ - u32 *mctx = crypto_shash_ctx(hash); - - if (keylen != sizeof(u32)) - return -EINVAL; - *mctx = le32_to_cpup((__le32 *)key); - return 0; -} - -static int crc32c_vpmsum_init(struct shash_desc *desc) -{ - u32 *mctx = crypto_shash_ctx(desc->tfm); - u32 *crcp = shash_desc_ctx(desc); - - *crcp = *mctx; - - return 0; -} - -static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - u32 *crcp = shash_desc_ctx(desc); - - *crcp = crc32c_vpmsum(*crcp, data, len); - - return 0; -} - -static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len, - u8 *out) -{ - *(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len)); - - return 0; -} - -static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out); -} - -static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out) -{ - u32 *crcp = shash_desc_ctx(desc); - - *(__le32 *)out = ~cpu_to_le32p(crcp); - - return 0; -} - -static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len, - out); -} - -static struct shash_alg alg = { - .setkey = crc32c_vpmsum_setkey, - .init = crc32c_vpmsum_init, - .update = crc32c_vpmsum_update, - .final = crc32c_vpmsum_final, - .finup = crc32c_vpmsum_finup, - .digest = crc32c_vpmsum_digest, - .descsize = sizeof(u32), - .digestsize = CHKSUM_DIGEST_SIZE, - .base = { - .cra_name = "crc32c", - .cra_driver_name = "crc32c-vpmsum", - .cra_priority = 200, - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .cra_blocksize = CHKSUM_BLOCK_SIZE, - .cra_ctxsize = sizeof(u32), - .cra_module = THIS_MODULE, - .cra_init = crc32c_vpmsum_cra_init, - } -}; - -static int __init crc32c_vpmsum_mod_init(void) -{ - if (!cpu_has_feature(CPU_FTR_ARCH_207S)) - return -ENODEV; - - return crypto_register_shash(&alg); -} - -static void __exit crc32c_vpmsum_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init); -module_exit(crc32c_vpmsum_mod_fini); - -MODULE_AUTHOR("Anton Blanchard <anton@samba.org>"); -MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("crc32c"); -MODULE_ALIAS_CRYPTO("crc32c-vpmsum"); diff --git a/arch/powerpc/crypto/crct10dif-vpmsum_asm.S b/arch/powerpc/crypto/crct10dif-vpmsum_asm.S deleted file mode 100644 index f0b93a0fe168..000000000000 --- a/arch/powerpc/crypto/crct10dif-vpmsum_asm.S +++ /dev/null @@ -1,845 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Calculate a CRC T10DIF with vpmsum acceleration - * - * Constants generated by crc32-vpmsum, available at - * https://github.com/antonblanchard/crc32-vpmsum - * - * crc32-vpmsum is - * Copyright (C) 2015 Anton Blanchard <anton@au.ibm.com>, IBM - */ - .section .rodata -.balign 16 - -.byteswap_constant: - /* byte reverse permute constant */ - .octa 0x0F0E0D0C0B0A09080706050403020100 - -.constants: - - /* Reduce 262144 kbits to 1024 bits */ - /* x^261184 mod p(x), x^261120 mod p(x) */ - .octa 0x0000000056d300000000000052550000 - - /* x^260160 mod p(x), x^260096 mod p(x) */ - .octa 0x00000000ee67000000000000a1e40000 - - /* x^259136 mod p(x), x^259072 mod p(x) */ - .octa 0x0000000060830000000000004ad10000 - - /* x^258112 mod p(x), x^258048 mod p(x) */ - .octa 0x000000008cfe0000000000009ab40000 - - /* x^257088 mod p(x), x^257024 mod p(x) */ - .octa 0x000000003e93000000000000fdb50000 - - /* x^256064 mod p(x), x^256000 mod p(x) */ - .octa 0x000000003c2000000000000045480000 - - /* x^255040 mod p(x), x^254976 mod p(x) */ - .octa 0x00000000b1fc0000000000008d690000 - - /* x^254016 mod p(x), x^253952 mod p(x) */ - .octa 0x00000000f82b00000000000024ad0000 - - /* x^252992 mod p(x), x^252928 mod p(x) */ - .octa 0x0000000044420000000000009f1a0000 - - /* x^251968 mod p(x), x^251904 mod p(x) */ - .octa 0x00000000e88c00000000000066ec0000 - - /* x^250944 mod p(x), x^250880 mod p(x) */ - .octa 0x00000000385c000000000000c87d0000 - - /* x^249920 mod p(x), x^249856 mod p(x) */ - .octa 0x000000003227000000000000c8ff0000 - - /* x^248896 mod p(x), x^248832 mod p(x) */ - .octa 0x00000000a9a900000000000033440000 - - /* x^247872 mod p(x), x^247808 mod p(x) */ - .octa 0x00000000abaa00000000000066eb0000 - - /* x^246848 mod p(x), x^246784 mod p(x) */ - .octa 0x000000001ac3000000000000c4ef0000 - - /* x^245824 mod p(x), x^245760 mod p(x) */ - .octa 0x0000000063f000000000000056f30000 - - /* x^244800 mod p(x), x^244736 mod p(x) */ - .octa 0x0000000032cc00000000000002050000 - - /* x^243776 mod p(x), x^243712 mod p(x) */ - .octa 0x00000000f8b5000000000000568e0000 - - /* x^242752 mod p(x), x^242688 mod p(x) */ - .octa 0x000000008db100000000000064290000 - - /* x^241728 mod p(x), x^241664 mod p(x) */ - .octa 0x0000000059ca0000000000006b660000 - - /* x^240704 mod p(x), x^240640 mod p(x) */ - .octa 0x000000005f5c00000000000018f80000 - - /* x^239680 mod p(x), x^239616 mod p(x) */ - .octa 0x0000000061af000000000000b6090000 - - /* x^238656 mod p(x), x^238592 mod p(x) */ - .octa 0x00000000e29e000000000000099a0000 - - /* x^237632 mod p(x), x^237568 mod p(x) */ - .octa 0x000000000975000000000000a8360000 - - /* x^236608 mod p(x), x^236544 mod p(x) */ - .octa 0x0000000043900000000000004f570000 - - /* x^235584 mod p(x), x^235520 mod p(x) */ - .octa 0x00000000f9cd000000000000134c0000 - - /* x^234560 mod p(x), x^234496 mod p(x) */ - .octa 0x000000007c29000000000000ec380000 - - /* x^233536 mod p(x), x^233472 mod p(x) */ - .octa 0x000000004c6a000000000000b0d10000 - - /* x^232512 mod p(x), x^232448 mod p(x) */ - .octa 0x00000000e7290000000000007d3e0000 - - /* x^231488 mod p(x), x^231424 mod p(x) */ - .octa 0x00000000f1ab000000000000f0b20000 - - /* x^230464 mod p(x), x^230400 mod p(x) */ - .octa 0x0000000039db0000000000009c270000 - - /* x^229440 mod p(x), x^229376 mod p(x) */ - .octa 0x000000005e2800000000000092890000 - - /* x^228416 mod p(x), x^228352 mod p(x) */ - .octa 0x00000000d44e000000000000d5ee0000 - - /* x^227392 mod p(x), x^227328 mod p(x) */ - .octa 0x00000000cd0a00000000000041f50000 - - /* x^226368 mod p(x), x^226304 mod p(x) */ - .octa 0x00000000c5b400000000000010520000 - - /* x^225344 mod p(x), x^225280 mod p(x) */ - .octa 0x00000000fd2100000000000042170000 - - /* x^224320 mod p(x), x^224256 mod p(x) */ - .octa 0x000000002f2500000000000095c20000 - - /* x^223296 mod p(x), x^223232 mod p(x) */ - .octa 0x000000001b0100000000000001ce0000 - - /* x^222272 mod p(x), x^222208 mod p(x) */ - .octa 0x000000000d430000000000002aca0000 - - /* x^221248 mod p(x), x^221184 mod p(x) */ - .octa 0x0000000030a6000000000000385e0000 - - /* x^220224 mod p(x), x^220160 mod p(x) */ - .octa 0x00000000e37b0000000000006f7a0000 - - /* x^219200 mod p(x), x^219136 mod p(x) */ - .octa 0x00000000873600000000000024320000 - - /* x^218176 mod p(x), x^218112 mod p(x) */ - .octa 0x00000000e9fb000000000000bd9c0000 - - /* x^217152 mod p(x), x^217088 mod p(x) */ - .octa 0x000000003b9500000000000054bc0000 - - /* x^216128 mod p(x), x^216064 mod p(x) */ - .octa 0x00000000133e000000000000a4660000 - - /* x^215104 mod p(x), x^215040 mod p(x) */ - .octa 0x00000000784500000000000079930000 - - /* x^214080 mod p(x), x^214016 mod p(x) */ - .octa 0x00000000b9800000000000001bb80000 - - /* x^213056 mod p(x), x^212992 mod p(x) */ - .octa 0x00000000687600000000000024400000 - - /* x^212032 mod p(x), x^211968 mod p(x) */ - .octa 0x00000000aff300000000000029e10000 - - /* x^211008 mod p(x), x^210944 mod p(x) */ - .octa 0x0000000024b50000000000005ded0000 - - /* x^209984 mod p(x), x^209920 mod p(x) */ - .octa 0x0000000017e8000000000000b12e0000 - - /* x^208960 mod p(x), x^208896 mod p(x) */ - .octa 0x00000000128400000000000026d20000 - - /* x^207936 mod p(x), x^207872 mod p(x) */ - .octa 0x000000002115000000000000a32a0000 - - /* x^206912 mod p(x), x^206848 mod p(x) */ - .octa 0x000000009595000000000000a1210000 - - /* x^205888 mod p(x), x^205824 mod p(x) */ - .octa 0x00000000281e000000000000ee8b0000 - - /* x^204864 mod p(x), x^204800 mod p(x) */ - .octa 0x0000000006010000000000003d0d0000 - - /* x^203840 mod p(x), x^203776 mod p(x) */ - .octa 0x00000000e2b600000000000034e90000 - - /* x^202816 mod p(x), x^202752 mod p(x) */ - .octa 0x000000001bd40000000000004cdb0000 - - /* x^201792 mod p(x), x^201728 mod p(x) */ - .octa 0x00000000df2800000000000030e90000 - - /* x^200768 mod p(x), x^200704 mod p(x) */ - .octa 0x0000000049c200000000000042590000 - - /* x^199744 mod p(x), x^199680 mod p(x) */ - .octa 0x000000009b97000000000000df950000 - - /* x^198720 mod p(x), x^198656 mod p(x) */ - .octa 0x000000006184000000000000da7b0000 - - /* x^197696 mod p(x), x^197632 mod p(x) */ - .octa 0x00000000461700000000000012510000 - - /* x^196672 mod p(x), x^196608 mod p(x) */ - .octa 0x000000009b40000000000000f37e0000 - - /* x^195648 mod p(x), x^195584 mod p(x) */ - .octa 0x00000000eeb2000000000000ecf10000 - - /* x^194624 mod p(x), x^194560 mod p(x) */ - .octa 0x00000000b2e800000000000050f20000 - - /* x^193600 mod p(x), x^193536 mod p(x) */ - .octa 0x00000000f59a000000000000e0b30000 - - /* x^192576 mod p(x), x^192512 mod p(x) */ - .octa 0x00000000467f0000000000004d5a0000 - - /* x^191552 mod p(x), x^191488 mod p(x) */ - .octa 0x00000000da92000000000000bb010000 - - /* x^190528 mod p(x), x^190464 mod p(x) */ - .octa 0x000000001e1000000000000022a40000 - - /* x^189504 mod p(x), x^189440 mod p(x) */ - .octa 0x0000000058fe000000000000836f0000 - - /* x^188480 mod p(x), x^188416 mod p(x) */ - .octa 0x00000000b9ce000000000000d78d0000 - - /* x^187456 mod p(x), x^187392 mod p(x) */ - .octa 0x0000000022210000000000004f8d0000 - - /* x^186432 mod p(x), x^186368 mod p(x) */ - .octa 0x00000000744600000000000033760000 - - /* x^185408 mod p(x), x^185344 mod p(x) */ - .octa 0x000000001c2e000000000000a1e50000 - - /* x^184384 mod p(x), x^184320 mod p(x) */ - .octa 0x00000000dcc8000000000000a1a40000 - - /* x^183360 mod p(x), x^183296 mod p(x) */ - .octa 0x00000000910f00000000000019a20000 - - /* x^182336 mod p(x), x^182272 mod p(x) */ - .octa 0x0000000055d5000000000000f6ae0000 - - /* x^181312 mod p(x), x^181248 mod p(x) */ - .octa 0x00000000c8ba000000000000a7ac0000 - - /* x^180288 mod p(x), x^180224 mod p(x) */ - .octa 0x0000000031f8000000000000eea20000 - - /* x^179264 mod p(x), x^179200 mod p(x) */ - .octa 0x000000001966000000000000c4d90000 - - /* x^178240 mod p(x), x^178176 mod p(x) */ - .octa 0x00000000b9810000000000002b470000 - - /* x^177216 mod p(x), x^177152 mod p(x) */ - .octa 0x000000008303000000000000f7cf0000 - - /* x^176192 mod p(x), x^176128 mod p(x) */ - .octa 0x000000002ce500000000000035b30000 - - /* x^175168 mod p(x), x^175104 mod p(x) */ - .octa 0x000000002fae0000000000000c7c0000 - - /* x^174144 mod p(x), x^174080 mod p(x) */ - .octa 0x00000000f50c0000000000009edf0000 - - /* x^173120 mod p(x), x^173056 mod p(x) */ - .octa 0x00000000714f00000000000004cd0000 - - /* x^172096 mod p(x), x^172032 mod p(x) */ - .octa 0x00000000c161000000000000541b0000 - - /* x^171072 mod p(x), x^171008 mod p(x) */ - .octa 0x0000000021c8000000000000e2700000 - - /* x^170048 mod p(x), x^169984 mod p(x) */ - .octa 0x00000000b93d00000000000009a60000 - - /* x^169024 mod p(x), x^168960 mod p(x) */ - .octa 0x00000000fbcf000000000000761c0000 - - /* x^168000 mod p(x), x^167936 mod p(x) */ - .octa 0x0000000026350000000000009db30000 - - /* x^166976 mod p(x), x^166912 mod p(x) */ - .octa 0x00000000b64f0000000000003e9f0000 - - /* x^165952 mod p(x), x^165888 mod p(x) */ - .octa 0x00000000bd0e00000000000078590000 - - /* x^164928 mod p(x), x^164864 mod p(x) */ - .octa 0x00000000d9360000000000008bc80000 - - /* x^163904 mod p(x), x^163840 mod p(x) */ - .octa 0x000000002f140000000000008c9f0000 - - /* x^162880 mod p(x), x^162816 mod p(x) */ - .octa 0x000000006a270000000000006af70000 - - /* x^161856 mod p(x), x^161792 mod p(x) */ - .octa 0x000000006685000000000000e5210000 - - /* x^160832 mod p(x), x^160768 mod p(x) */ - .octa 0x0000000062da00000000000008290000 - - /* x^159808 mod p(x), x^159744 mod p(x) */ - .octa 0x00000000bb4b000000000000e4d00000 - - /* x^158784 mod p(x), x^158720 mod p(x) */ - .octa 0x00000000d2490000000000004ae10000 - - /* x^157760 mod p(x), x^157696 mod p(x) */ - .octa 0x00000000c85b00000000000000e70000 - - /* x^156736 mod p(x), x^156672 mod p(x) */ - .octa 0x00000000c37a00000000000015650000 - - /* x^155712 mod p(x), x^155648 mod p(x) */ - .octa 0x0000000018530000000000001c2f0000 - - /* x^154688 mod p(x), x^154624 mod p(x) */ - .octa 0x00000000b46600000000000037bd0000 - - /* x^153664 mod p(x), x^153600 mod p(x) */ - .octa 0x00000000439b00000000000012190000 - - /* x^152640 mod p(x), x^152576 mod p(x) */ - .octa 0x00000000b1260000000000005ece0000 - - /* x^151616 mod p(x), x^151552 mod p(x) */ - .octa 0x00000000d8110000000000002a5e0000 - - /* x^150592 mod p(x), x^150528 mod p(x) */ - .octa 0x00000000099f00000000000052330000 - - /* x^149568 mod p(x), x^149504 mod p(x) */ - .octa 0x00000000f9f9000000000000f9120000 - - /* x^148544 mod p(x), x^148480 mod p(x) */ - .octa 0x000000005cc00000000000000ddc0000 - - /* x^147520 mod p(x), x^147456 mod p(x) */ - .octa 0x00000000343b00000000000012200000 - - /* x^146496 mod p(x), x^146432 mod p(x) */ - .octa 0x000000009222000000000000d12b0000 - - /* x^145472 mod p(x), x^145408 mod p(x) */ - .octa 0x00000000d781000000000000eb2d0000 - - /* x^144448 mod p(x), x^144384 mod p(x) */ - .octa 0x000000000bf400000000000058970000 - - /* x^143424 mod p(x), x^143360 mod p(x) */ - .octa 0x00000000094200000000000013690000 - - /* x^142400 mod p(x), x^142336 mod p(x) */ - .octa 0x00000000d55100000000000051950000 - - /* x^141376 mod p(x), x^141312 mod p(x) */ - .octa 0x000000008f11000000000000954b0000 - - /* x^140352 mod p(x), x^140288 mod p(x) */ - .octa 0x00000000140f000000000000b29e0000 - - /* x^139328 mod p(x), x^139264 mod p(x) */ - .octa 0x00000000c6db000000000000db5d0000 - - /* x^138304 mod p(x), x^138240 mod p(x) */ - .octa 0x00000000715b000000000000dfaf0000 - - /* x^137280 mod p(x), x^137216 mod p(x) */ - .octa 0x000000000dea000000000000e3b60000 - - /* x^136256 mod p(x), x^136192 mod p(x) */ - .octa 0x000000006f94000000000000ddaf0000 - - /* x^135232 mod p(x), x^135168 mod p(x) */ - .octa 0x0000000024e1000000000000e4f70000 - - /* x^134208 mod p(x), x^134144 mod p(x) */ - .octa 0x000000008810000000000000aa110000 - - /* x^133184 mod p(x), x^133120 mod p(x) */ - .octa 0x0000000030c2000000000000a8e60000 - - /* x^132160 mod p(x), x^132096 mod p(x) */ - .octa 0x00000000e6d0000000000000ccf30000 - - /* x^131136 mod p(x), x^131072 mod p(x) */ - .octa 0x000000004da000000000000079bf0000 - - /* x^130112 mod p(x), x^130048 mod p(x) */ - .octa 0x000000007759000000000000b3a30000 - - /* x^129088 mod p(x), x^129024 mod p(x) */ - .octa 0x00000000597400000000000028790000 - - /* x^128064 mod p(x), x^128000 mod p(x) */ - .octa 0x000000007acd000000000000b5820000 - - /* x^127040 mod p(x), x^126976 mod p(x) */ - .octa 0x00000000e6e400000000000026ad0000 - - /* x^126016 mod p(x), x^125952 mod p(x) */ - .octa 0x000000006d49000000000000985b0000 - - /* x^124992 mod p(x), x^124928 mod p(x) */ - .octa 0x000000000f0800000000000011520000 - - /* x^123968 mod p(x), x^123904 mod p(x) */ - .octa 0x000000002c7f000000000000846c0000 - - /* x^122944 mod p(x), x^122880 mod p(x) */ - .octa 0x000000005ce7000000000000ae1d0000 - - /* x^121920 mod p(x), x^121856 mod p(x) */ - .octa 0x00000000d4cb000000000000e21d0000 - - /* x^120896 mod p(x), x^120832 mod p(x) */ - .octa 0x000000003a2300000000000019bb0000 - - /* x^119872 mod p(x), x^119808 mod p(x) */ - .octa 0x000000000e1700000000000095290000 - - /* x^118848 mod p(x), x^118784 mod p(x) */ - .octa 0x000000006e6400000000000050d20000 - - /* x^117824 mod p(x), x^117760 mod p(x) */ - .octa 0x000000008d5c0000000000000cd10000 - - /* x^116800 mod p(x), x^116736 mod p(x) */ - .octa 0x00000000ef310000000000007b570000 - - /* x^115776 mod p(x), x^115712 mod p(x) */ - .octa 0x00000000645d00000000000053d60000 - - /* x^114752 mod p(x), x^114688 mod p(x) */ - .octa 0x0000000018fc00000000000077510000 - - /* x^113728 mod p(x), x^113664 mod p(x) */ - .octa 0x000000000cb3000000000000a7b70000 - - /* x^112704 mod p(x), x^112640 mod p(x) */ - .octa 0x00000000991b000000000000d0780000 - - /* x^111680 mod p(x), x^111616 mod p(x) */ - .octa 0x00000000845a000000000000be3c0000 - - /* x^110656 mod p(x), x^110592 mod p(x) */ - .octa 0x00000000d3a9000000000000df020000 - - /* x^109632 mod p(x), x^109568 mod p(x) */ - .octa 0x0000000017d7000000000000063e0000 - - /* x^108608 mod p(x), x^108544 mod p(x) */ - .octa 0x000000007a860000000000008ab40000 - - /* x^107584 mod p(x), x^107520 mod p(x) */ - .octa 0x00000000fd7c000000000000c7bd0000 - - /* x^106560 mod p(x), x^106496 mod p(x) */ - .octa 0x00000000a56b000000000000efd60000 - - /* x^105536 mod p(x), x^105472 mod p(x) */ - .octa 0x0000000010e400000000000071380000 - - /* x^104512 mod p(x), x^104448 mod p(x) */ - .octa 0x00000000994500000000000004d30000 - - /* x^103488 mod p(x), x^103424 mod p(x) */ - .octa 0x00000000b83c0000000000003b0e0000 - - /* x^102464 mod p(x), x^102400 mod p(x) */ - .octa 0x00000000d6c10000000000008b020000 - - /* x^101440 mod p(x), x^101376 mod p(x) */ - .octa 0x000000009efc000000000000da940000 - - /* x^100416 mod p(x), x^100352 mod p(x) */ - .octa 0x000000005e87000000000000f9f70000 - - /* x^99392 mod p(x), x^99328 mod p(x) */ - .octa 0x000000006c9b00000000000045e40000 - - /* x^98368 mod p(x), x^98304 mod p(x) */ - .octa 0x00000000178a00000000000083940000 - - /* x^97344 mod p(x), x^97280 mod p(x) */ - .octa 0x00000000f0c8000000000000f0a00000 - - /* x^96320 mod p(x), x^96256 mod p(x) */ - .octa 0x00000000f699000000000000b74b0000 - - /* x^95296 mod p(x), x^95232 mod p(x) */ - .octa 0x00000000316d000000000000c1cf0000 - - /* x^94272 mod p(x), x^94208 mod p(x) */ - .octa 0x00000000987e00000000000072680000 - - /* x^93248 mod p(x), x^93184 mod p(x) */ - .octa 0x00000000acff000000000000e0ab0000 - - /* x^92224 mod p(x), x^92160 mod p(x) */ - .octa 0x00000000a1f6000000000000c5a80000 - - /* x^91200 mod p(x), x^91136 mod p(x) */ - .octa 0x0000000061bd000000000000cf690000 - - /* x^90176 mod p(x), x^90112 mod p(x) */ - .octa 0x00000000c9f2000000000000cbcc0000 - - /* x^89152 mod p(x), x^89088 mod p(x) */ - .octa 0x000000005a33000000000000de050000 - - /* x^88128 mod p(x), x^88064 mod p(x) */ - .octa 0x00000000e416000000000000ccd70000 - - /* x^87104 mod p(x), x^87040 mod p(x) */ - .octa 0x0000000058930000000000002f670000 - - /* x^86080 mod p(x), x^86016 mod p(x) */ - .octa 0x00000000a9d3000000000000152f0000 - - /* x^85056 mod p(x), x^84992 mod p(x) */ - .octa 0x00000000c114000000000000ecc20000 - - /* x^84032 mod p(x), x^83968 mod p(x) */ - .octa 0x00000000b9270000000000007c890000 - - /* x^83008 mod p(x), x^82944 mod p(x) */ - .octa 0x000000002e6000000000000006ee0000 - - /* x^81984 mod p(x), x^81920 mod p(x) */ - .octa 0x00000000dfc600000000000009100000 - - /* x^80960 mod p(x), x^80896 mod p(x) */ - .octa 0x000000004911000000000000ad4e0000 - - /* x^79936 mod p(x), x^79872 mod p(x) */ - .octa 0x00000000ae1b000000000000b04d0000 - - /* x^78912 mod p(x), x^78848 mod p(x) */ - .octa 0x0000000005fa000000000000e9900000 - - /* x^77888 mod p(x), x^77824 mod p(x) */ - .octa 0x0000000004a1000000000000cc6f0000 - - /* x^76864 mod p(x), x^76800 mod p(x) */ - .octa 0x00000000af73000000000000ed110000 - - /* x^75840 mod p(x), x^75776 mod p(x) */ - .octa 0x0000000082530000000000008f7e0000 - - /* x^74816 mod p(x), x^74752 mod p(x) */ - .octa 0x00000000cfdc000000000000594f0000 - - /* x^73792 mod p(x), x^73728 mod p(x) */ - .octa 0x00000000a6b6000000000000a8750000 - - /* x^72768 mod p(x), x^72704 mod p(x) */ - .octa 0x00000000fd76000000000000aa0c0000 - - /* x^71744 mod p(x), x^71680 mod p(x) */ - .octa 0x0000000006f500000000000071db0000 - - /* x^70720 mod p(x), x^70656 mod p(x) */ - .octa 0x0000000037ca000000000000ab0c0000 - - /* x^69696 mod p(x), x^69632 mod p(x) */ - .octa 0x00000000d7ab000000000000b7a00000 - - /* x^68672 mod p(x), x^68608 mod p(x) */ - .octa 0x00000000440800000000000090d30000 - - /* x^67648 mod p(x), x^67584 mod p(x) */ - .octa 0x00000000186100000000000054730000 - - /* x^66624 mod p(x), x^66560 mod p(x) */ - .octa 0x000000007368000000000000a3a20000 - - /* x^65600 mod p(x), x^65536 mod p(x) */ - .octa 0x0000000026d0000000000000f9040000 - - /* x^64576 mod p(x), x^64512 mod p(x) */ - .octa 0x00000000fe770000000000009c0a0000 - - /* x^63552 mod p(x), x^63488 mod p(x) */ - .octa 0x000000002cba000000000000d1e70000 - - /* x^62528 mod p(x), x^62464 mod p(x) */ - .octa 0x00000000f8bd0000000000005ac10000 - - /* x^61504 mod p(x), x^61440 mod p(x) */ - .octa 0x000000007372000000000000d68d0000 - - /* x^60480 mod p(x), x^60416 mod p(x) */ - .octa 0x00000000f37f00000000000089f60000 - - /* x^59456 mod p(x), x^59392 mod p(x) */ - .octa 0x00000000078400000000000008a90000 - - /* x^58432 mod p(x), x^58368 mod p(x) */ - .octa 0x00000000d3e400000000000042360000 - - /* x^57408 mod p(x), x^57344 mod p(x) */ - .octa 0x00000000eba800000000000092d50000 - - /* x^56384 mod p(x), x^56320 mod p(x) */ - .octa 0x00000000afbe000000000000b4d50000 - - /* x^55360 mod p(x), x^55296 mod p(x) */ - .octa 0x00000000d8ca000000000000c9060000 - - /* x^54336 mod p(x), x^54272 mod p(x) */ - .octa 0x00000000c2d00000000000008f4f0000 - - /* x^53312 mod p(x), x^53248 mod p(x) */ - .octa 0x00000000373200000000000028690000 - - /* x^52288 mod p(x), x^52224 mod p(x) */ - .octa 0x0000000046ae000000000000c3b30000 - - /* x^51264 mod p(x), x^51200 mod p(x) */ - .octa 0x00000000b243000000000000f8700000 - - /* x^50240 mod p(x), x^50176 mod p(x) */ - .octa 0x00000000f7f500000000000029eb0000 - - /* x^49216 mod p(x), x^49152 mod p(x) */ - .octa 0x000000000c7e000000000000fe730000 - - /* x^48192 mod p(x), x^48128 mod p(x) */ - .octa 0x00000000c38200000000000096000000 - - /* x^47168 mod p(x), x^47104 mod p(x) */ - .octa 0x000000008956000000000000683c0000 - - /* x^46144 mod p(x), x^46080 mod p(x) */ - .octa 0x00000000422d0000000000005f1e0000 - - /* x^45120 mod p(x), x^45056 mod p(x) */ - .octa 0x00000000ac0f0000000000006f810000 - - /* x^44096 mod p(x), x^44032 mod p(x) */ - .octa 0x00000000ce30000000000000031f0000 - - /* x^43072 mod p(x), x^43008 mod p(x) */ - .octa 0x000000003d43000000000000455a0000 - - /* x^42048 mod p(x), x^41984 mod p(x) */ - .octa 0x000000007ebe000000000000a6050000 - - /* x^41024 mod p(x), x^40960 mod p(x) */ - .octa 0x00000000976e00000000000077eb0000 - - /* x^40000 mod p(x), x^39936 mod p(x) */ - .octa 0x000000000872000000000000389c0000 - - /* x^38976 mod p(x), x^38912 mod p(x) */ - .octa 0x000000008979000000000000c7b20000 - - /* x^37952 mod p(x), x^37888 mod p(x) */ - .octa 0x000000005c1e0000000000001d870000 - - /* x^36928 mod p(x), x^36864 mod p(x) */ - .octa 0x00000000aebb00000000000045810000 - - /* x^35904 mod p(x), x^35840 mod p(x) */ - .octa 0x000000004f7e0000000000006d4a0000 - - /* x^34880 mod p(x), x^34816 mod p(x) */ - .octa 0x00000000ea98000000000000b9200000 - - /* x^33856 mod p(x), x^33792 mod p(x) */ - .octa 0x00000000f39600000000000022f20000 - - /* x^32832 mod p(x), x^32768 mod p(x) */ - .octa 0x000000000bc500000000000041ca0000 - - /* x^31808 mod p(x), x^31744 mod p(x) */ - .octa 0x00000000786400000000000078500000 - - /* x^30784 mod p(x), x^30720 mod p(x) */ - .octa 0x00000000be970000000000009e7e0000 - - /* x^29760 mod p(x), x^29696 mod p(x) */ - .octa 0x00000000dd6d000000000000a53c0000 - - /* x^28736 mod p(x), x^28672 mod p(x) */ - .octa 0x000000004c3f00000000000039340000 - - /* x^27712 mod p(x), x^27648 mod p(x) */ - .octa 0x0000000093a4000000000000b58e0000 - - /* x^26688 mod p(x), x^26624 mod p(x) */ - .octa 0x0000000050fb00000000000062d40000 - - /* x^25664 mod p(x), x^25600 mod p(x) */ - .octa 0x00000000f505000000000000a26f0000 - - /* x^24640 mod p(x), x^24576 mod p(x) */ - .octa 0x0000000064f900000000000065e60000 - - /* x^23616 mod p(x), x^23552 mod p(x) */ - .octa 0x00000000e8c2000000000000aad90000 - - /* x^22592 mod p(x), x^22528 mod p(x) */ - .octa 0x00000000720b000000000000a3b00000 - - /* x^21568 mod p(x), x^21504 mod p(x) */ - .octa 0x00000000e992000000000000d2680000 - - /* x^20544 mod p(x), x^20480 mod p(x) */ - .octa 0x000000009132000000000000cf4c0000 - - /* x^19520 mod p(x), x^19456 mod p(x) */ - .octa 0x00000000608a00000000000076610000 - - /* x^18496 mod p(x), x^18432 mod p(x) */ - .octa 0x000000009948000000000000fb9f0000 - - /* x^17472 mod p(x), x^17408 mod p(x) */ - .octa 0x00000000173000000000000003770000 - - /* x^16448 mod p(x), x^16384 mod p(x) */ - .octa 0x000000006fe300000000000004880000 - - /* x^15424 mod p(x), x^15360 mod p(x) */ - .octa 0x00000000e15300000000000056a70000 - - /* x^14400 mod p(x), x^14336 mod p(x) */ - .octa 0x0000000092d60000000000009dfd0000 - - /* x^13376 mod p(x), x^13312 mod p(x) */ - .octa 0x0000000002fd00000000000074c80000 - - /* x^12352 mod p(x), x^12288 mod p(x) */ - .octa 0x00000000c78b000000000000a3ec0000 - - /* x^11328 mod p(x), x^11264 mod p(x) */ - .octa 0x000000009262000000000000b3530000 - - /* x^10304 mod p(x), x^10240 mod p(x) */ - .octa 0x0000000084f200000000000047bf0000 - - /* x^9280 mod p(x), x^9216 mod p(x) */ - .octa 0x0000000067ee000000000000e97c0000 - - /* x^8256 mod p(x), x^8192 mod p(x) */ - .octa 0x00000000535b00000000000091e10000 - - /* x^7232 mod p(x), x^7168 mod p(x) */ - .octa 0x000000007ebb00000000000055060000 - - /* x^6208 mod p(x), x^6144 mod p(x) */ - .octa 0x00000000c6a1000000000000fd360000 - - /* x^5184 mod p(x), x^5120 mod p(x) */ - .octa 0x000000001be500000000000055860000 - - /* x^4160 mod p(x), x^4096 mod p(x) */ - .octa 0x00000000ae0e0000000000005bd00000 - - /* x^3136 mod p(x), x^3072 mod p(x) */ - .octa 0x0000000022040000000000008db20000 - - /* x^2112 mod p(x), x^2048 mod p(x) */ - .octa 0x00000000c9eb000000000000efe20000 - - /* x^1088 mod p(x), x^1024 mod p(x) */ - .octa 0x0000000039b400000000000051d10000 - -.short_constants: - - /* Reduce final 1024-2048 bits to 64 bits, shifting 32 bits to include the trailing 32 bits of zeros */ - /* x^2048 mod p(x), x^2016 mod p(x), x^1984 mod p(x), x^1952 mod p(x) */ - .octa 0xefe20000dccf00009440000033590000 - - /* x^1920 mod p(x), x^1888 mod p(x), x^1856 mod p(x), x^1824 mod p(x) */ - .octa 0xee6300002f3f000062180000e0ed0000 - - /* x^1792 mod p(x), x^1760 mod p(x), x^1728 mod p(x), x^1696 mod p(x) */ - .octa 0xcf5f000017ef0000ccbe000023d30000 - - /* x^1664 mod p(x), x^1632 mod p(x), x^1600 mod p(x), x^1568 mod p(x) */ - .octa 0x6d0c0000a30e00000920000042630000 - - /* x^1536 mod p(x), x^1504 mod p(x), x^1472 mod p(x), x^1440 mod p(x) */ - .octa 0x21d30000932b0000a7a00000efcc0000 - - /* x^1408 mod p(x), x^1376 mod p(x), x^1344 mod p(x), x^1312 mod p(x) */ - .octa 0x10be00000b310000666f00000d1c0000 - - /* x^1280 mod p(x), x^1248 mod p(x), x^1216 mod p(x), x^1184 mod p(x) */ - .octa 0x1f240000ce9e0000caad0000589e0000 - - /* x^1152 mod p(x), x^1120 mod p(x), x^1088 mod p(x), x^1056 mod p(x) */ - .octa 0x29610000d02b000039b400007cf50000 - - /* x^1024 mod p(x), x^992 mod p(x), x^960 mod p(x), x^928 mod p(x) */ - .octa 0x51d100009d9d00003c0e0000bfd60000 - - /* x^896 mod p(x), x^864 mod p(x), x^832 mod p(x), x^800 mod p(x) */ - .octa 0xda390000ceae000013830000713c0000 - - /* x^768 mod p(x), x^736 mod p(x), x^704 mod p(x), x^672 mod p(x) */ - .octa 0xb67800001e16000085c0000080a60000 - - /* x^640 mod p(x), x^608 mod p(x), x^576 mod p(x), x^544 mod p(x) */ - .octa 0x0db40000f7f90000371d0000e6580000 - - /* x^512 mod p(x), x^480 mod p(x), x^448 mod p(x), x^416 mod p(x) */ - .octa 0x87e70000044c0000aadb0000a4970000 - - /* x^384 mod p(x), x^352 mod p(x), x^320 mod p(x), x^288 mod p(x) */ - .octa 0x1f990000ad180000d8b30000e7b50000 - - /* x^256 mod p(x), x^224 mod p(x), x^192 mod p(x), x^160 mod p(x) */ - .octa 0xbe6c00006ee300004c1a000006df0000 - - /* x^128 mod p(x), x^96 mod p(x), x^64 mod p(x), x^32 mod p(x) */ - .octa 0xfb0b00002d560000136800008bb70000 - - -.barrett_constants: - /* Barrett constant m - (4^32)/n */ - .octa 0x000000000000000000000001f65a57f8 /* x^64 div p(x) */ - /* Barrett constant n */ - .octa 0x0000000000000000000000018bb70000 - -#define CRC_FUNCTION_NAME __crct10dif_vpmsum -#include "crc32-vpmsum_core.S" diff --git a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c b/arch/powerpc/crypto/crct10dif-vpmsum_glue.c deleted file mode 100644 index 1dc8b6915178..000000000000 --- a/arch/powerpc/crypto/crct10dif-vpmsum_glue.c +++ /dev/null @@ -1,126 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Calculate a CRC T10-DIF with vpmsum acceleration - * - * Copyright 2017, Daniel Axtens, IBM Corporation. - * [based on crc32c-vpmsum_glue.c] - */ - -#include <linux/crc-t10dif.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/simd.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/string.h> -#include <linux/kernel.h> -#include <linux/cpufeature.h> -#include <asm/simd.h> -#include <asm/switch_to.h> - -#define VMX_ALIGN 16 -#define VMX_ALIGN_MASK (VMX_ALIGN-1) - -#define VECTOR_BREAKPOINT 64 - -u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len); - -static u16 crct10dif_vpmsum(u16 crci, unsigned char const *p, size_t len) -{ - unsigned int prealign; - unsigned int tail; - u32 crc = crci; - - if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable()) - return crc_t10dif_generic(crc, p, len); - - if ((unsigned long)p & VMX_ALIGN_MASK) { - prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); - crc = crc_t10dif_generic(crc, p, prealign); - len -= prealign; - p += prealign; - } - - if (len & ~VMX_ALIGN_MASK) { - crc <<= 16; - preempt_disable(); - pagefault_disable(); - enable_kernel_altivec(); - crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); - disable_kernel_altivec(); - pagefault_enable(); - preempt_enable(); - crc >>= 16; - } - - tail = len & VMX_ALIGN_MASK; - if (tail) { - p += len & ~VMX_ALIGN_MASK; - crc = crc_t10dif_generic(crc, p, tail); - } - - return crc & 0xffff; -} - -static int crct10dif_vpmsum_init(struct shash_desc *desc) -{ - u16 *crc = shash_desc_ctx(desc); - - *crc = 0; - return 0; -} - -static int crct10dif_vpmsum_update(struct shash_desc *desc, const u8 *data, - unsigned int length) -{ - u16 *crc = shash_desc_ctx(desc); - - *crc = crct10dif_vpmsum(*crc, data, length); - - return 0; -} - - -static int crct10dif_vpmsum_final(struct shash_desc *desc, u8 *out) -{ - u16 *crcp = shash_desc_ctx(desc); - - *(u16 *)out = *crcp; - return 0; -} - -static struct shash_alg alg = { - .init = crct10dif_vpmsum_init, - .update = crct10dif_vpmsum_update, - .final = crct10dif_vpmsum_final, - .descsize = CRC_T10DIF_DIGEST_SIZE, - .digestsize = CRC_T10DIF_DIGEST_SIZE, - .base = { - .cra_name = "crct10dif", - .cra_driver_name = "crct10dif-vpmsum", - .cra_priority = 200, - .cra_blocksize = CRC_T10DIF_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init crct10dif_vpmsum_mod_init(void) -{ - if (!cpu_has_feature(CPU_FTR_ARCH_207S)) - return -ENODEV; - - return crypto_register_shash(&alg); -} - -static void __exit crct10dif_vpmsum_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crct10dif_vpmsum_mod_init); -module_exit(crct10dif_vpmsum_mod_fini); - -MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>"); -MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions"); -MODULE_LICENSE("GPL"); -MODULE_ALIAS_CRYPTO("crct10dif"); -MODULE_ALIAS_CRYPTO("crct10dif-vpmsum"); diff --git a/arch/powerpc/crypto/ghash.c b/arch/powerpc/crypto/ghash.c index 77eca20bc7ac..7308735bdb33 100644 --- a/arch/powerpc/crypto/ghash.c +++ b/arch/powerpc/crypto/ghash.c @@ -11,19 +11,18 @@ * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> */ -#include <linux/types.h> -#include <linux/err.h> -#include <linux/crypto.h> -#include <linux/delay.h> -#include <asm/simd.h> +#include "aesp8-ppc.h" #include <asm/switch_to.h> #include <crypto/aes.h> +#include <crypto/gf128mul.h> #include <crypto/ghash.h> -#include <crypto/scatterwalk.h> #include <crypto/internal/hash.h> #include <crypto/internal/simd.h> -#include <crypto/b128ops.h> -#include "aesp8-ppc.h" +#include <linux/err.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/uaccess.h> void gcm_init_p8(u128 htable[16], const u64 Xi[2]); void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); @@ -39,15 +38,12 @@ struct p8_ghash_ctx { struct p8_ghash_desc_ctx { u64 shash[2]; - u8 buffer[GHASH_DIGEST_SIZE]; - int bytes; }; static int p8_ghash_init(struct shash_desc *desc) { struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - dctx->bytes = 0; memset(dctx->shash, 0, GHASH_DIGEST_SIZE); return 0; } @@ -74,27 +70,30 @@ static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, } static inline void __ghash_block(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx) + struct p8_ghash_desc_ctx *dctx, + const u8 *src) { if (crypto_simd_usable()) { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); - gcm_ghash_p8(dctx->shash, ctx->htable, - dctx->buffer, GHASH_DIGEST_SIZE); + gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); } else { - crypto_xor((u8 *)dctx->shash, dctx->buffer, GHASH_BLOCK_SIZE); + crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); gf128mul_lle((be128 *)dctx->shash, &ctx->key); } } -static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, - struct p8_ghash_desc_ctx *dctx, - const u8 *src, unsigned int srclen) +static inline int __ghash_blocks(struct p8_ghash_ctx *ctx, + struct p8_ghash_desc_ctx *dctx, + const u8 *src, unsigned int srclen) { + int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE); + + srclen -= remain; if (crypto_simd_usable()) { preempt_disable(); pagefault_disable(); @@ -105,62 +104,38 @@ static inline void __ghash_blocks(struct p8_ghash_ctx *ctx, pagefault_enable(); preempt_enable(); } else { - while (srclen >= GHASH_BLOCK_SIZE) { + do { crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); gf128mul_lle((be128 *)dctx->shash, &ctx->key); srclen -= GHASH_BLOCK_SIZE; src += GHASH_BLOCK_SIZE; - } + } while (srclen); } + + return remain; } static int p8_ghash_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { - unsigned int len; struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (dctx->bytes) { - if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) { - memcpy(dctx->buffer + dctx->bytes, src, - srclen); - dctx->bytes += srclen; - return 0; - } - memcpy(dctx->buffer + dctx->bytes, src, - GHASH_DIGEST_SIZE - dctx->bytes); - - __ghash_block(ctx, dctx); - - src += GHASH_DIGEST_SIZE - dctx->bytes; - srclen -= GHASH_DIGEST_SIZE - dctx->bytes; - dctx->bytes = 0; - } - len = srclen & ~(GHASH_DIGEST_SIZE - 1); - if (len) { - __ghash_blocks(ctx, dctx, src, len); - src += len; - srclen -= len; - } - if (srclen) { - memcpy(dctx->buffer, src, srclen); - dctx->bytes = srclen; - } - return 0; + return __ghash_blocks(ctx, dctx, src, srclen); } -static int p8_ghash_final(struct shash_desc *desc, u8 *out) +static int p8_ghash_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { - int i; struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); - if (dctx->bytes) { - for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++) - dctx->buffer[i] = 0; - __ghash_block(ctx, dctx); - dctx->bytes = 0; + if (len) { + u8 buf[GHASH_BLOCK_SIZE] = {}; + + memcpy(buf, src, len); + __ghash_block(ctx, dctx, buf); + memzero_explicit(buf, sizeof(buf)); } memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); return 0; @@ -170,14 +145,14 @@ struct shash_alg p8_ghash_alg = { .digestsize = GHASH_DIGEST_SIZE, .init = p8_ghash_init, .update = p8_ghash_update, - .final = p8_ghash_final, + .finup = p8_ghash_finup, .setkey = p8_ghash_setkey, - .descsize = sizeof(struct p8_ghash_desc_ctx) - + sizeof(struct ghash_desc_ctx), + .descsize = sizeof(struct p8_ghash_desc_ctx), .base = { .cra_name = "ghash", .cra_driver_name = "p8_ghash", .cra_priority = 1000, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = GHASH_BLOCK_SIZE, .cra_ctxsize = sizeof(struct p8_ghash_ctx), .cra_module = THIS_MODULE, diff --git a/arch/powerpc/crypto/md5-glue.c b/arch/powerpc/crypto/md5-glue.c index c24f605033bd..204440a90cd8 100644 --- a/arch/powerpc/crypto/md5-glue.c +++ b/arch/powerpc/crypto/md5-glue.c @@ -8,25 +8,13 @@ */ #include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> #include <crypto/md5.h> -#include <asm/byteorder.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/string.h> extern void ppc_md5_transform(u32 *state, const u8 *src, u32 blocks); -static inline void ppc_md5_clear_context(struct md5_state *sctx) -{ - int count = sizeof(struct md5_state) >> 2; - u32 *ptr = (u32 *)sctx; - - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct md5_state) % 4); - do { *ptr++ = 0; } while (--count); -} - static int ppc_md5_init(struct shash_desc *desc) { struct md5_state *sctx = shash_desc_ctx(desc); @@ -44,79 +32,34 @@ static int ppc_md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct md5_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->byte_count & 0x3f; - unsigned int avail = 64 - offset; - const u8 *src = data; - sctx->byte_count += len; - - if (avail > len) { - memcpy((char *)sctx->block + offset, src, len); - return 0; - } - - if (offset) { - memcpy((char *)sctx->block + offset, src, avail); - ppc_md5_transform(sctx->hash, (const u8 *)sctx->block, 1); - len -= avail; - src += avail; - } - - if (len > 63) { - ppc_md5_transform(sctx->hash, src, len >> 6); - src += len & ~0x3f; - len &= 0x3f; - } - - memcpy((char *)sctx->block, src, len); - return 0; + sctx->byte_count += round_down(len, MD5_HMAC_BLOCK_SIZE); + ppc_md5_transform(sctx->hash, data, len >> 6); + return len - round_down(len, MD5_HMAC_BLOCK_SIZE); } -static int ppc_md5_final(struct shash_desc *desc, u8 *out) +static int ppc_md5_finup(struct shash_desc *desc, const u8 *src, + unsigned int offset, u8 *out) { struct md5_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->byte_count & 0x3f; - const u8 *src = (const u8 *)sctx->block; - u8 *p = (u8 *)src + offset; - int padlen = 55 - offset; - __le64 *pbits = (__le64 *)((char *)sctx->block + 56); + __le64 block[MD5_BLOCK_WORDS] = {}; + u8 *p = memcpy(block, src, offset); __le32 *dst = (__le32 *)out; + __le64 *pbits; + src = p; + p += offset; *p++ = 0x80; - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_md5_transform(sctx->hash, src, 1); - p = (char *)sctx->block; - padlen = 56; - } - - memset(p, 0, padlen); + sctx->byte_count += offset; + pbits = &block[(MD5_BLOCK_WORDS / (offset > 55 ? 1 : 2)) - 1]; *pbits = cpu_to_le64(sctx->byte_count << 3); - ppc_md5_transform(sctx->hash, src, 1); + ppc_md5_transform(sctx->hash, src, (pbits - block + 1) / 8); + memzero_explicit(block, sizeof(block)); dst[0] = cpu_to_le32(sctx->hash[0]); dst[1] = cpu_to_le32(sctx->hash[1]); dst[2] = cpu_to_le32(sctx->hash[2]); dst[3] = cpu_to_le32(sctx->hash[3]); - - ppc_md5_clear_context(sctx); - return 0; -} - -static int ppc_md5_export(struct shash_desc *desc, void *out) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int ppc_md5_import(struct shash_desc *desc, const void *in) -{ - struct md5_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); return 0; } @@ -124,15 +67,13 @@ static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = ppc_md5_init, .update = ppc_md5_update, - .final = ppc_md5_final, - .export = ppc_md5_export, - .import = ppc_md5_import, - .descsize = sizeof(struct md5_state), - .statesize = sizeof(struct md5_state), + .finup = ppc_md5_finup, + .descsize = MD5_STATE_SIZE, .base = { .cra_name = "md5", .cra_driver_name= "md5-ppc", .cra_priority = 200, + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = MD5_HMAC_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c deleted file mode 100644 index 369686e9370b..000000000000 --- a/arch/powerpc/crypto/poly1305-p10-glue.c +++ /dev/null @@ -1,186 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Poly1305 authenticator algorithm, RFC7539. - * - * Copyright 2023- IBM Corp. All rights reserved. - */ - -#include <crypto/algapi.h> -#include <linux/crypto.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/jump_label.h> -#include <crypto/internal/hash.h> -#include <crypto/internal/poly1305.h> -#include <crypto/internal/simd.h> -#include <linux/cpufeature.h> -#include <linux/unaligned.h> -#include <asm/simd.h> -#include <asm/switch_to.h> - -asmlinkage void poly1305_p10le_4blocks(void *h, const u8 *m, u32 mlen); -asmlinkage void poly1305_64s(void *h, const u8 *m, u32 mlen, int highbit); -asmlinkage void poly1305_emit_64(void *h, void *s, u8 *dst); - -static void vsx_begin(void) -{ - preempt_disable(); - enable_kernel_vsx(); -} - -static void vsx_end(void) -{ - disable_kernel_vsx(); - preempt_enable(); -} - -static int crypto_poly1305_p10_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - poly1305_core_init(&dctx->h); - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static unsigned int crypto_poly1305_setdctxkey(struct poly1305_desc_ctx *dctx, - const u8 *inp, unsigned int len) -{ - unsigned int acc = 0; - - if (unlikely(!dctx->sset)) { - if (!dctx->rset && len >= POLY1305_BLOCK_SIZE) { - struct poly1305_core_key *key = &dctx->core_r; - - key->key.r64[0] = get_unaligned_le64(&inp[0]); - key->key.r64[1] = get_unaligned_le64(&inp[8]); - inp += POLY1305_BLOCK_SIZE; - len -= POLY1305_BLOCK_SIZE; - acc += POLY1305_BLOCK_SIZE; - dctx->rset = 1; - } - if (len >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(&inp[0]); - dctx->s[1] = get_unaligned_le32(&inp[4]); - dctx->s[2] = get_unaligned_le32(&inp[8]); - dctx->s[3] = get_unaligned_le32(&inp[12]); - acc += POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return acc; -} - -static int crypto_poly1305_p10_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes, used; - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - if (likely(!crypto_poly1305_setdctxkey(dctx, dctx->buf, - POLY1305_BLOCK_SIZE))) { - vsx_begin(); - poly1305_64s(&dctx->h, dctx->buf, - POLY1305_BLOCK_SIZE, 1); - vsx_end(); - } - dctx->buflen = 0; - } - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - bytes = round_down(srclen, POLY1305_BLOCK_SIZE); - used = crypto_poly1305_setdctxkey(dctx, src, bytes); - if (likely(used)) { - srclen -= used; - src += used; - } - if (crypto_simd_usable() && (srclen >= POLY1305_BLOCK_SIZE*4)) { - vsx_begin(); - poly1305_p10le_4blocks(&dctx->h, src, srclen); - vsx_end(); - src += srclen - (srclen % (POLY1305_BLOCK_SIZE * 4)); - srclen %= POLY1305_BLOCK_SIZE * 4; - } - while (srclen >= POLY1305_BLOCK_SIZE) { - vsx_begin(); - poly1305_64s(&dctx->h, src, POLY1305_BLOCK_SIZE, 1); - vsx_end(); - srclen -= POLY1305_BLOCK_SIZE; - src += POLY1305_BLOCK_SIZE; - } - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } - - return 0; -} - -static int crypto_poly1305_p10_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - if ((dctx->buflen)) { - dctx->buf[dctx->buflen++] = 1; - memset(dctx->buf + dctx->buflen, 0, - POLY1305_BLOCK_SIZE - dctx->buflen); - vsx_begin(); - poly1305_64s(&dctx->h, dctx->buf, POLY1305_BLOCK_SIZE, 0); - vsx_end(); - dctx->buflen = 0; - } - - poly1305_emit_64(&dctx->h, &dctx->s, dst); - return 0; -} - -static struct shash_alg poly1305_alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_p10_init, - .update = crypto_poly1305_p10_update, - .final = crypto_poly1305_p10_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-p10", - .cra_priority = 300, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; - -static int __init poly1305_p10_init(void) -{ - return crypto_register_shash(&poly1305_alg); -} - -static void __exit poly1305_p10_exit(void) -{ - crypto_unregister_shash(&poly1305_alg); -} - -module_cpu_feature_match(PPC_MODULE_FEATURE_P10, poly1305_p10_init); -module_exit(poly1305_p10_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>"); -MODULE_DESCRIPTION("Optimized Poly1305 for P10"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-p10"); diff --git a/arch/powerpc/crypto/poly1305-p10le_64.S b/arch/powerpc/crypto/poly1305-p10le_64.S deleted file mode 100644 index a3c1987f1ecd..000000000000 --- a/arch/powerpc/crypto/poly1305-p10le_64.S +++ /dev/null @@ -1,1075 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -# -# Accelerated poly1305 implementation for ppc64le. -# -# Copyright 2023- IBM Corp. All rights reserved -# -#=================================================================================== -# Written by Danny Tsen <dtsen@us.ibm.com> -# -# Poly1305 - this version mainly using vector/VSX/Scalar -# - 26 bits limbs -# - Handle multiple 64 byte blcok. -# -# Block size 16 bytes -# key = (r, s) -# clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF -# p = 2^130 - 5 -# a += m -# a = (r + a) % p -# a += s -# -# Improve performance by breaking down polynominal to the sum of products with -# h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r -# -# 07/22/21 - this revison based on the above sum of products. Setup r^4, r^3, r^2, r and s3, s2, s1, s0 -# to 9 vectors for multiplications. -# -# setup r^4, r^3, r^2, r vectors -# vs [r^1, r^3, r^2, r^4] -# vs0 = [r0,.....] -# vs1 = [r1,.....] -# vs2 = [r2,.....] -# vs3 = [r3,.....] -# vs4 = [r4,.....] -# vs5 = [r1*5,...] -# vs6 = [r2*5,...] -# vs7 = [r2*5,...] -# vs8 = [r4*5,...] -# -# Each word in a vector consists a member of a "r/s" in [a * r/s]. -# -# r0, r4*5, r3*5, r2*5, r1*5; -# r1, r0, r4*5, r3*5, r2*5; -# r2, r1, r0, r4*5, r3*5; -# r3, r2, r1, r0, r4*5; -# r4, r3, r2, r1, r0 ; -# -# -# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) -# k = 32 bytes key -# r3 = k (r, s) -# r4 = mlen -# r5 = m -# -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> -#include <asm/asm-compat.h> -#include <linux/linkage.h> - -.machine "any" - -.text - -.macro SAVE_GPR GPR OFFSET FRAME - std \GPR,\OFFSET(\FRAME) -.endm - -.macro SAVE_VRS VRS OFFSET FRAME - li 16, \OFFSET - stvx \VRS, 16, \FRAME -.endm - -.macro SAVE_VSX VSX OFFSET FRAME - li 16, \OFFSET - stxvx \VSX, 16, \FRAME -.endm - -.macro RESTORE_GPR GPR OFFSET FRAME - ld \GPR,\OFFSET(\FRAME) -.endm - -.macro RESTORE_VRS VRS OFFSET FRAME - li 16, \OFFSET - lvx \VRS, 16, \FRAME -.endm - -.macro RESTORE_VSX VSX OFFSET FRAME - li 16, \OFFSET - lxvx \VSX, 16, \FRAME -.endm - -.macro SAVE_REGS - mflr 0 - std 0, 16(1) - stdu 1,-752(1) - - SAVE_GPR 14, 112, 1 - SAVE_GPR 15, 120, 1 - SAVE_GPR 16, 128, 1 - SAVE_GPR 17, 136, 1 - SAVE_GPR 18, 144, 1 - SAVE_GPR 19, 152, 1 - SAVE_GPR 20, 160, 1 - SAVE_GPR 21, 168, 1 - SAVE_GPR 22, 176, 1 - SAVE_GPR 23, 184, 1 - SAVE_GPR 24, 192, 1 - SAVE_GPR 25, 200, 1 - SAVE_GPR 26, 208, 1 - SAVE_GPR 27, 216, 1 - SAVE_GPR 28, 224, 1 - SAVE_GPR 29, 232, 1 - SAVE_GPR 30, 240, 1 - SAVE_GPR 31, 248, 1 - - addi 9, 1, 256 - SAVE_VRS 20, 0, 9 - SAVE_VRS 21, 16, 9 - SAVE_VRS 22, 32, 9 - SAVE_VRS 23, 48, 9 - SAVE_VRS 24, 64, 9 - SAVE_VRS 25, 80, 9 - SAVE_VRS 26, 96, 9 - SAVE_VRS 27, 112, 9 - SAVE_VRS 28, 128, 9 - SAVE_VRS 29, 144, 9 - SAVE_VRS 30, 160, 9 - SAVE_VRS 31, 176, 9 - - SAVE_VSX 14, 192, 9 - SAVE_VSX 15, 208, 9 - SAVE_VSX 16, 224, 9 - SAVE_VSX 17, 240, 9 - SAVE_VSX 18, 256, 9 - SAVE_VSX 19, 272, 9 - SAVE_VSX 20, 288, 9 - SAVE_VSX 21, 304, 9 - SAVE_VSX 22, 320, 9 - SAVE_VSX 23, 336, 9 - SAVE_VSX 24, 352, 9 - SAVE_VSX 25, 368, 9 - SAVE_VSX 26, 384, 9 - SAVE_VSX 27, 400, 9 - SAVE_VSX 28, 416, 9 - SAVE_VSX 29, 432, 9 - SAVE_VSX 30, 448, 9 - SAVE_VSX 31, 464, 9 -.endm # SAVE_REGS - -.macro RESTORE_REGS - addi 9, 1, 256 - RESTORE_VRS 20, 0, 9 - RESTORE_VRS 21, 16, 9 - RESTORE_VRS 22, 32, 9 - RESTORE_VRS 23, 48, 9 - RESTORE_VRS 24, 64, 9 - RESTORE_VRS 25, 80, 9 - RESTORE_VRS 26, 96, 9 - RESTORE_VRS 27, 112, 9 - RESTORE_VRS 28, 128, 9 - RESTORE_VRS 29, 144, 9 - RESTORE_VRS 30, 160, 9 - RESTORE_VRS 31, 176, 9 - - RESTORE_VSX 14, 192, 9 - RESTORE_VSX 15, 208, 9 - RESTORE_VSX 16, 224, 9 - RESTORE_VSX 17, 240, 9 - RESTORE_VSX 18, 256, 9 - RESTORE_VSX 19, 272, 9 - RESTORE_VSX 20, 288, 9 - RESTORE_VSX 21, 304, 9 - RESTORE_VSX 22, 320, 9 - RESTORE_VSX 23, 336, 9 - RESTORE_VSX 24, 352, 9 - RESTORE_VSX 25, 368, 9 - RESTORE_VSX 26, 384, 9 - RESTORE_VSX 27, 400, 9 - RESTORE_VSX 28, 416, 9 - RESTORE_VSX 29, 432, 9 - RESTORE_VSX 30, 448, 9 - RESTORE_VSX 31, 464, 9 - - RESTORE_GPR 14, 112, 1 - RESTORE_GPR 15, 120, 1 - RESTORE_GPR 16, 128, 1 - RESTORE_GPR 17, 136, 1 - RESTORE_GPR 18, 144, 1 - RESTORE_GPR 19, 152, 1 - RESTORE_GPR 20, 160, 1 - RESTORE_GPR 21, 168, 1 - RESTORE_GPR 22, 176, 1 - RESTORE_GPR 23, 184, 1 - RESTORE_GPR 24, 192, 1 - RESTORE_GPR 25, 200, 1 - RESTORE_GPR 26, 208, 1 - RESTORE_GPR 27, 216, 1 - RESTORE_GPR 28, 224, 1 - RESTORE_GPR 29, 232, 1 - RESTORE_GPR 30, 240, 1 - RESTORE_GPR 31, 248, 1 - - addi 1, 1, 752 - ld 0, 16(1) - mtlr 0 -.endm # RESTORE_REGS - -# -# p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5; -# p[1] = a0*r1 + a1*r0 + a2*r4*5 + a3*r3*5 + a4*r2*5; -# p[2] = a0*r2 + a1*r1 + a2*r0 + a3*r4*5 + a4*r3*5; -# p[3] = a0*r3 + a1*r2 + a2*r1 + a3*r0 + a4*r4*5; -# p[4] = a0*r4 + a1*r3 + a2*r2 + a3*r1 + a4*r0 ; -# -# [r^2, r^3, r^1, r^4] -# [m3, m2, m4, m1] -# -# multiply odd and even words -.macro mul_odd - vmulouw 14, 4, 26 - vmulouw 10, 5, 3 - vmulouw 11, 6, 2 - vmulouw 12, 7, 1 - vmulouw 13, 8, 0 - vmulouw 15, 4, 27 - vaddudm 14, 14, 10 - vaddudm 14, 14, 11 - vmulouw 10, 5, 26 - vmulouw 11, 6, 3 - vaddudm 14, 14, 12 - vaddudm 14, 14, 13 # x0 - vaddudm 15, 15, 10 - vaddudm 15, 15, 11 - vmulouw 12, 7, 2 - vmulouw 13, 8, 1 - vaddudm 15, 15, 12 - vaddudm 15, 15, 13 # x1 - vmulouw 16, 4, 28 - vmulouw 10, 5, 27 - vmulouw 11, 6, 26 - vaddudm 16, 16, 10 - vaddudm 16, 16, 11 - vmulouw 12, 7, 3 - vmulouw 13, 8, 2 - vaddudm 16, 16, 12 - vaddudm 16, 16, 13 # x2 - vmulouw 17, 4, 29 - vmulouw 10, 5, 28 - vmulouw 11, 6, 27 - vaddudm 17, 17, 10 - vaddudm 17, 17, 11 - vmulouw 12, 7, 26 - vmulouw 13, 8, 3 - vaddudm 17, 17, 12 - vaddudm 17, 17, 13 # x3 - vmulouw 18, 4, 30 - vmulouw 10, 5, 29 - vmulouw 11, 6, 28 - vaddudm 18, 18, 10 - vaddudm 18, 18, 11 - vmulouw 12, 7, 27 - vmulouw 13, 8, 26 - vaddudm 18, 18, 12 - vaddudm 18, 18, 13 # x4 -.endm - -.macro mul_even - vmuleuw 9, 4, 26 - vmuleuw 10, 5, 3 - vmuleuw 11, 6, 2 - vmuleuw 12, 7, 1 - vmuleuw 13, 8, 0 - vaddudm 14, 14, 9 - vaddudm 14, 14, 10 - vaddudm 14, 14, 11 - vaddudm 14, 14, 12 - vaddudm 14, 14, 13 # x0 - - vmuleuw 9, 4, 27 - vmuleuw 10, 5, 26 - vmuleuw 11, 6, 3 - vmuleuw 12, 7, 2 - vmuleuw 13, 8, 1 - vaddudm 15, 15, 9 - vaddudm 15, 15, 10 - vaddudm 15, 15, 11 - vaddudm 15, 15, 12 - vaddudm 15, 15, 13 # x1 - - vmuleuw 9, 4, 28 - vmuleuw 10, 5, 27 - vmuleuw 11, 6, 26 - vmuleuw 12, 7, 3 - vmuleuw 13, 8, 2 - vaddudm 16, 16, 9 - vaddudm 16, 16, 10 - vaddudm 16, 16, 11 - vaddudm 16, 16, 12 - vaddudm 16, 16, 13 # x2 - - vmuleuw 9, 4, 29 - vmuleuw 10, 5, 28 - vmuleuw 11, 6, 27 - vmuleuw 12, 7, 26 - vmuleuw 13, 8, 3 - vaddudm 17, 17, 9 - vaddudm 17, 17, 10 - vaddudm 17, 17, 11 - vaddudm 17, 17, 12 - vaddudm 17, 17, 13 # x3 - - vmuleuw 9, 4, 30 - vmuleuw 10, 5, 29 - vmuleuw 11, 6, 28 - vmuleuw 12, 7, 27 - vmuleuw 13, 8, 26 - vaddudm 18, 18, 9 - vaddudm 18, 18, 10 - vaddudm 18, 18, 11 - vaddudm 18, 18, 12 - vaddudm 18, 18, 13 # x4 -.endm - -# -# poly1305_setup_r -# -# setup r^4, r^3, r^2, r vectors -# [r, r^3, r^2, r^4] -# vs0 = [r0,...] -# vs1 = [r1,...] -# vs2 = [r2,...] -# vs3 = [r3,...] -# vs4 = [r4,...] -# vs5 = [r4*5,...] -# vs6 = [r3*5,...] -# vs7 = [r2*5,...] -# vs8 = [r1*5,...] -# -# r0, r4*5, r3*5, r2*5, r1*5; -# r1, r0, r4*5, r3*5, r2*5; -# r2, r1, r0, r4*5, r3*5; -# r3, r2, r1, r0, r4*5; -# r4, r3, r2, r1, r0 ; -# -.macro poly1305_setup_r - - # save r - xxlor 26, 58, 58 - xxlor 27, 59, 59 - xxlor 28, 60, 60 - xxlor 29, 61, 61 - xxlor 30, 62, 62 - - xxlxor 31, 31, 31 - -# [r, r^3, r^2, r^4] - # compute r^2 - vmr 4, 26 - vmr 5, 27 - vmr 6, 28 - vmr 7, 29 - vmr 8, 30 - bl do_mul # r^2 r^1 - xxpermdi 58, 58, 36, 0x3 # r0 - xxpermdi 59, 59, 37, 0x3 # r1 - xxpermdi 60, 60, 38, 0x3 # r2 - xxpermdi 61, 61, 39, 0x3 # r3 - xxpermdi 62, 62, 40, 0x3 # r4 - xxpermdi 36, 36, 36, 0x3 - xxpermdi 37, 37, 37, 0x3 - xxpermdi 38, 38, 38, 0x3 - xxpermdi 39, 39, 39, 0x3 - xxpermdi 40, 40, 40, 0x3 - vspltisb 13, 2 - vsld 9, 27, 13 - vsld 10, 28, 13 - vsld 11, 29, 13 - vsld 12, 30, 13 - vaddudm 0, 9, 27 - vaddudm 1, 10, 28 - vaddudm 2, 11, 29 - vaddudm 3, 12, 30 - - bl do_mul # r^4 r^3 - vmrgow 26, 26, 4 - vmrgow 27, 27, 5 - vmrgow 28, 28, 6 - vmrgow 29, 29, 7 - vmrgow 30, 30, 8 - vspltisb 13, 2 - vsld 9, 27, 13 - vsld 10, 28, 13 - vsld 11, 29, 13 - vsld 12, 30, 13 - vaddudm 0, 9, 27 - vaddudm 1, 10, 28 - vaddudm 2, 11, 29 - vaddudm 3, 12, 30 - - # r^2 r^4 - xxlor 0, 58, 58 - xxlor 1, 59, 59 - xxlor 2, 60, 60 - xxlor 3, 61, 61 - xxlor 4, 62, 62 - xxlor 5, 32, 32 - xxlor 6, 33, 33 - xxlor 7, 34, 34 - xxlor 8, 35, 35 - - vspltw 9, 26, 3 - vspltw 10, 26, 2 - vmrgow 26, 10, 9 - vspltw 9, 27, 3 - vspltw 10, 27, 2 - vmrgow 27, 10, 9 - vspltw 9, 28, 3 - vspltw 10, 28, 2 - vmrgow 28, 10, 9 - vspltw 9, 29, 3 - vspltw 10, 29, 2 - vmrgow 29, 10, 9 - vspltw 9, 30, 3 - vspltw 10, 30, 2 - vmrgow 30, 10, 9 - - vsld 9, 27, 13 - vsld 10, 28, 13 - vsld 11, 29, 13 - vsld 12, 30, 13 - vaddudm 0, 9, 27 - vaddudm 1, 10, 28 - vaddudm 2, 11, 29 - vaddudm 3, 12, 30 -.endm - -SYM_FUNC_START_LOCAL(do_mul) - mul_odd - - # do reduction ( h %= p ) - # carry reduction - vspltisb 9, 2 - vsrd 10, 14, 31 - vsrd 11, 17, 31 - vand 7, 17, 25 - vand 4, 14, 25 - vaddudm 18, 18, 11 - vsrd 12, 18, 31 - vaddudm 15, 15, 10 - - vsrd 11, 15, 31 - vand 8, 18, 25 - vand 5, 15, 25 - vaddudm 4, 4, 12 - vsld 10, 12, 9 - vaddudm 6, 16, 11 - - vsrd 13, 6, 31 - vand 6, 6, 25 - vaddudm 4, 4, 10 - vsrd 10, 4, 31 - vaddudm 7, 7, 13 - - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 5, 5, 10 - vaddudm 8, 8, 11 - blr -SYM_FUNC_END(do_mul) - -# -# init key -# -.macro do_poly1305_init - addis 10, 2, rmask@toc@ha - addi 10, 10, rmask@toc@l - - ld 11, 0(10) - ld 12, 8(10) - - li 14, 16 - li 15, 32 - addis 10, 2, cnum@toc@ha - addi 10, 10, cnum@toc@l - lvx 25, 0, 10 # v25 - mask - lvx 31, 14, 10 # v31 = 1a - lvx 19, 15, 10 # v19 = 1 << 24 - lxv 24, 48(10) # vs24 - lxv 25, 64(10) # vs25 - - # initialize - # load key from r3 to vectors - ld 9, 24(3) - ld 10, 32(3) - and. 9, 9, 11 - and. 10, 10, 12 - - # break 26 bits - extrdi 14, 9, 26, 38 - extrdi 15, 9, 26, 12 - extrdi 16, 9, 12, 0 - mtvsrdd 58, 0, 14 - insrdi 16, 10, 14, 38 - mtvsrdd 59, 0, 15 - extrdi 17, 10, 26, 24 - mtvsrdd 60, 0, 16 - extrdi 18, 10, 24, 0 - mtvsrdd 61, 0, 17 - mtvsrdd 62, 0, 18 - - # r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5 - li 9, 5 - mtvsrdd 36, 0, 9 - vmulouw 0, 27, 4 # v0 = rr0 - vmulouw 1, 28, 4 # v1 = rr1 - vmulouw 2, 29, 4 # v2 = rr2 - vmulouw 3, 30, 4 # v3 = rr3 -.endm - -# -# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m) -# k = 32 bytes key -# r3 = k (r, s) -# r4 = mlen -# r5 = m -# -SYM_FUNC_START(poly1305_p10le_4blocks) -.align 5 - cmpdi 5, 64 - blt Out_no_poly1305 - - SAVE_REGS - - do_poly1305_init - - li 21, 0 # counter to message - - poly1305_setup_r - - # load previous H state - # break/convert r6 to 26 bits - ld 9, 0(3) - ld 10, 8(3) - ld 19, 16(3) - sldi 19, 19, 24 - mtvsrdd 41, 0, 19 - extrdi 14, 9, 26, 38 - extrdi 15, 9, 26, 12 - extrdi 16, 9, 12, 0 - mtvsrdd 36, 0, 14 - insrdi 16, 10, 14, 38 - mtvsrdd 37, 0, 15 - extrdi 17, 10, 26, 24 - mtvsrdd 38, 0, 16 - extrdi 18, 10, 24, 0 - mtvsrdd 39, 0, 17 - mtvsrdd 40, 0, 18 - vor 8, 8, 9 - - # input m1 m2 - add 20, 4, 21 - xxlor 49, 24, 24 - xxlor 50, 25, 25 - lxvw4x 43, 0, 20 - addi 17, 20, 16 - lxvw4x 44, 0, 17 - vperm 14, 11, 12, 17 - vperm 15, 11, 12, 18 - vand 9, 14, 25 # a0 - vsrd 10, 14, 31 # >> 26 - vsrd 11, 10, 31 # 12 bits left - vand 10, 10, 25 # a1 - vspltisb 13, 12 - vand 16, 15, 25 - vsld 12, 16, 13 - vor 11, 11, 12 - vand 11, 11, 25 # a2 - vspltisb 13, 14 - vsrd 12, 15, 13 # >> 14 - vsrd 13, 12, 31 # >> 26, a4 - vand 12, 12, 25 # a3 - - vaddudm 20, 4, 9 - vaddudm 21, 5, 10 - vaddudm 22, 6, 11 - vaddudm 23, 7, 12 - vaddudm 24, 8, 13 - - # m3 m4 - addi 17, 17, 16 - lxvw4x 43, 0, 17 - addi 17, 17, 16 - lxvw4x 44, 0, 17 - vperm 14, 11, 12, 17 - vperm 15, 11, 12, 18 - vand 9, 14, 25 # a0 - vsrd 10, 14, 31 # >> 26 - vsrd 11, 10, 31 # 12 bits left - vand 10, 10, 25 # a1 - vspltisb 13, 12 - vand 16, 15, 25 - vsld 12, 16, 13 - vspltisb 13, 14 - vor 11, 11, 12 - vand 11, 11, 25 # a2 - vsrd 12, 15, 13 # >> 14 - vsrd 13, 12, 31 # >> 26, a4 - vand 12, 12, 25 # a3 - - # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] - vmrgow 4, 9, 20 - vmrgow 5, 10, 21 - vmrgow 6, 11, 22 - vmrgow 7, 12, 23 - vmrgow 8, 13, 24 - vaddudm 8, 8, 19 - - addi 5, 5, -64 # len -= 64 - addi 21, 21, 64 # offset += 64 - - li 9, 64 - divdu 31, 5, 9 - - cmpdi 31, 0 - ble Skip_block_loop - - mtctr 31 - -# h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r -# Rewrite the polynominal sum of product as follows, -# h1 = (h0 + m1) * r^2, h2 = (h0 + m2) * r^2 -# h3 = (h1 + m3) * r^2, h4 = (h2 + m4) * r^2 --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2 -# .... Repeat -# h5 = (h3 + m5) * r^2, h6 = (h4 + m6) * r^2 --> -# h7 = (h5 + m7) * r^2, h8 = (h6 + m8) * r^1 --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r -# -loop_4blocks: - - # Multiply odd words and even words - mul_odd - mul_even - # carry reduction - vspltisb 9, 2 - vsrd 10, 14, 31 - vsrd 11, 17, 31 - vand 7, 17, 25 - vand 4, 14, 25 - vaddudm 18, 18, 11 - vsrd 12, 18, 31 - vaddudm 15, 15, 10 - - vsrd 11, 15, 31 - vand 8, 18, 25 - vand 5, 15, 25 - vaddudm 4, 4, 12 - vsld 10, 12, 9 - vaddudm 6, 16, 11 - - vsrd 13, 6, 31 - vand 6, 6, 25 - vaddudm 4, 4, 10 - vsrd 10, 4, 31 - vaddudm 7, 7, 13 - - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 5, 5, 10 - vaddudm 8, 8, 11 - - # input m1 m2 m3 m4 - add 20, 4, 21 - xxlor 49, 24, 24 - xxlor 50, 25, 25 - lxvw4x 43, 0, 20 - addi 17, 20, 16 - lxvw4x 44, 0, 17 - vperm 14, 11, 12, 17 - vperm 15, 11, 12, 18 - addi 17, 17, 16 - lxvw4x 43, 0, 17 - addi 17, 17, 16 - lxvw4x 44, 0, 17 - vperm 17, 11, 12, 17 - vperm 18, 11, 12, 18 - - vand 20, 14, 25 # a0 - vand 9, 17, 25 # a0 - vsrd 21, 14, 31 # >> 26 - vsrd 22, 21, 31 # 12 bits left - vsrd 10, 17, 31 # >> 26 - vsrd 11, 10, 31 # 12 bits left - - vand 21, 21, 25 # a1 - vand 10, 10, 25 # a1 - - vspltisb 13, 12 - vand 16, 15, 25 - vsld 23, 16, 13 - vor 22, 22, 23 - vand 22, 22, 25 # a2 - vand 16, 18, 25 - vsld 12, 16, 13 - vor 11, 11, 12 - vand 11, 11, 25 # a2 - vspltisb 13, 14 - vsrd 23, 15, 13 # >> 14 - vsrd 24, 23, 31 # >> 26, a4 - vand 23, 23, 25 # a3 - vsrd 12, 18, 13 # >> 14 - vsrd 13, 12, 31 # >> 26, a4 - vand 12, 12, 25 # a3 - - vaddudm 4, 4, 20 - vaddudm 5, 5, 21 - vaddudm 6, 6, 22 - vaddudm 7, 7, 23 - vaddudm 8, 8, 24 - - # Smash 4 message blocks into 5 vectors of [m4, m2, m3, m1] - vmrgow 4, 9, 4 - vmrgow 5, 10, 5 - vmrgow 6, 11, 6 - vmrgow 7, 12, 7 - vmrgow 8, 13, 8 - vaddudm 8, 8, 19 - - addi 5, 5, -64 # len -= 64 - addi 21, 21, 64 # offset += 64 - - bdnz loop_4blocks - -Skip_block_loop: - xxlor 58, 0, 0 - xxlor 59, 1, 1 - xxlor 60, 2, 2 - xxlor 61, 3, 3 - xxlor 62, 4, 4 - xxlor 32, 5, 5 - xxlor 33, 6, 6 - xxlor 34, 7, 7 - xxlor 35, 8, 8 - - # Multiply odd words and even words - mul_odd - mul_even - - # Sum the products. - xxpermdi 41, 31, 46, 0 - xxpermdi 42, 31, 47, 0 - vaddudm 4, 14, 9 - xxpermdi 36, 31, 36, 3 - vaddudm 5, 15, 10 - xxpermdi 37, 31, 37, 3 - xxpermdi 43, 31, 48, 0 - vaddudm 6, 16, 11 - xxpermdi 38, 31, 38, 3 - xxpermdi 44, 31, 49, 0 - vaddudm 7, 17, 12 - xxpermdi 39, 31, 39, 3 - xxpermdi 45, 31, 50, 0 - vaddudm 8, 18, 13 - xxpermdi 40, 31, 40, 3 - - # carry reduction - vspltisb 9, 2 - vsrd 10, 4, 31 - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 8, 8, 11 - vsrd 12, 8, 31 - vaddudm 5, 5, 10 - - vsrd 11, 5, 31 - vand 8, 8, 25 - vand 5, 5, 25 - vaddudm 4, 4, 12 - vsld 10, 12, 9 - vaddudm 6, 6, 11 - - vsrd 13, 6, 31 - vand 6, 6, 25 - vaddudm 4, 4, 10 - vsrd 10, 4, 31 - vaddudm 7, 7, 13 - - vsrd 11, 7, 31 - vand 7, 7, 25 - vand 4, 4, 25 - vaddudm 5, 5, 10 - vsrd 10, 5, 31 - vand 5, 5, 25 - vaddudm 6, 6, 10 - vaddudm 8, 8, 11 - - b do_final_update - -do_final_update: - # combine 26 bit limbs - # v4, v5, v6, v7 and v8 are 26 bit vectors - vsld 5, 5, 31 - vor 20, 4, 5 - vspltisb 11, 12 - vsrd 12, 6, 11 - vsld 6, 6, 31 - vsld 6, 6, 31 - vor 20, 20, 6 - vspltisb 11, 14 - vsld 7, 7, 11 - vor 21, 7, 12 - mfvsrld 16, 40 # save last 2 bytes - vsld 8, 8, 11 - vsld 8, 8, 31 - vor 21, 21, 8 - mfvsrld 17, 52 - mfvsrld 19, 53 - srdi 16, 16, 24 - - std 17, 0(3) - std 19, 8(3) - stw 16, 16(3) - -Out_loop: - li 3, 0 - - RESTORE_REGS - - blr - -Out_no_poly1305: - li 3, 0 - blr -SYM_FUNC_END(poly1305_p10le_4blocks) - -# -# ======================================================================= -# The following functions implement 64 x 64 bits multiplication poly1305. -# -SYM_FUNC_START_LOCAL(Poly1305_init_64) - # mask 0x0FFFFFFC0FFFFFFC - # mask 0x0FFFFFFC0FFFFFFF - addis 10, 2, rmask@toc@ha - addi 10, 10, rmask@toc@l - ld 11, 0(10) - ld 12, 8(10) - - # initialize - # load key from r3 - ld 9, 24(3) - ld 10, 32(3) - and. 9, 9, 11 # cramp mask r0 - and. 10, 10, 12 # cramp mask r1 - - srdi 21, 10, 2 - add 19, 21, 10 # s1: r19 - (r1 >> 2) *5 - - # setup r and s - li 25, 0 - mtvsrdd 32+0, 9, 19 # r0, s1 - mtvsrdd 32+1, 10, 9 # r1, r0 - mtvsrdd 32+2, 19, 25 # s1 - mtvsrdd 32+3, 9, 25 # r0 - - blr -SYM_FUNC_END(Poly1305_init_64) - -# Poly1305_mult -# v6 = (h0, h1), v8 = h2 -# v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0 -# -# Output: v7, v10, v11 -# -SYM_FUNC_START_LOCAL(Poly1305_mult) - # - # d0 = h0 * r0 + h1 * s1 - vmsumudm 7, 6, 0, 9 # h0 * r0, h1 * s1 - - # d1 = h0 * r1 + h1 * r0 + h2 * s1 - vmsumudm 11, 6, 1, 9 # h0 * r1, h1 * r0 - vmsumudm 10, 8, 2, 11 # d1 += h2 * s1 - - # d2 = r0 - vmsumudm 11, 8, 3, 9 # d2 = h2 * r0 - blr -SYM_FUNC_END(Poly1305_mult) - -# -# carry reduction -# h %=p -# -# Input: v7, v10, v11 -# Output: r27, r28, r29 -# -SYM_FUNC_START_LOCAL(Carry_reduction) - mfvsrld 27, 32+7 - mfvsrld 28, 32+10 - mfvsrld 29, 32+11 - mfvsrd 20, 32+7 # h0.h - mfvsrd 21, 32+10 # h1.h - - addc 28, 28, 20 - adde 29, 29, 21 - srdi 22, 29, 0x2 - sldi 23, 22, 0x2 - add 23, 23, 22 # (h2 & 3) * 5 - addc 27, 27, 23 # h0 - addze 28, 28 # h1 - andi. 29, 29, 0x3 # h2 - blr -SYM_FUNC_END(Carry_reduction) - -# -# poly1305 multiplication -# h *= r, h %= p -# d0 = h0 * r0 + h1 * s1 -# d1 = h0 * r1 + h1 * r0 + h2 * s1 -# d2 = h0 * r0 -# -# -# unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit) -# - no highbit if final leftover block (highbit = 0) -# -SYM_FUNC_START(poly1305_64s) - cmpdi 5, 0 - ble Out_no_poly1305_64 - - mflr 0 - std 0, 16(1) - stdu 1,-400(1) - - SAVE_GPR 14, 112, 1 - SAVE_GPR 15, 120, 1 - SAVE_GPR 16, 128, 1 - SAVE_GPR 17, 136, 1 - SAVE_GPR 18, 144, 1 - SAVE_GPR 19, 152, 1 - SAVE_GPR 20, 160, 1 - SAVE_GPR 21, 168, 1 - SAVE_GPR 22, 176, 1 - SAVE_GPR 23, 184, 1 - SAVE_GPR 24, 192, 1 - SAVE_GPR 25, 200, 1 - SAVE_GPR 26, 208, 1 - SAVE_GPR 27, 216, 1 - SAVE_GPR 28, 224, 1 - SAVE_GPR 29, 232, 1 - SAVE_GPR 30, 240, 1 - SAVE_GPR 31, 248, 1 - - # Init poly1305 - bl Poly1305_init_64 - - li 25, 0 # offset to inp and outp - - add 11, 25, 4 - - # load h - # h0, h1, h2? - ld 27, 0(3) - ld 28, 8(3) - lwz 29, 16(3) - - li 30, 16 - divdu 31, 5, 30 - - mtctr 31 - - mr 24, 6 # highbit - -Loop_block_64: - vxor 9, 9, 9 - - ld 20, 0(11) - ld 21, 8(11) - addi 11, 11, 16 - - addc 27, 27, 20 - adde 28, 28, 21 - adde 29, 29, 24 - - li 22, 0 - mtvsrdd 32+6, 27, 28 # h0, h1 - mtvsrdd 32+8, 29, 22 # h2 - - bl Poly1305_mult - - bl Carry_reduction - - bdnz Loop_block_64 - - std 27, 0(3) - std 28, 8(3) - stw 29, 16(3) - - li 3, 0 - - RESTORE_GPR 14, 112, 1 - RESTORE_GPR 15, 120, 1 - RESTORE_GPR 16, 128, 1 - RESTORE_GPR 17, 136, 1 - RESTORE_GPR 18, 144, 1 - RESTORE_GPR 19, 152, 1 - RESTORE_GPR 20, 160, 1 - RESTORE_GPR 21, 168, 1 - RESTORE_GPR 22, 176, 1 - RESTORE_GPR 23, 184, 1 - RESTORE_GPR 24, 192, 1 - RESTORE_GPR 25, 200, 1 - RESTORE_GPR 26, 208, 1 - RESTORE_GPR 27, 216, 1 - RESTORE_GPR 28, 224, 1 - RESTORE_GPR 29, 232, 1 - RESTORE_GPR 30, 240, 1 - RESTORE_GPR 31, 248, 1 - - addi 1, 1, 400 - ld 0, 16(1) - mtlr 0 - - blr - -Out_no_poly1305_64: - li 3, 0 - blr -SYM_FUNC_END(poly1305_64s) - -# -# Input: r3 = h, r4 = s, r5 = mac -# mac = h + s -# -SYM_FUNC_START(poly1305_emit_64) - ld 10, 0(3) - ld 11, 8(3) - ld 12, 16(3) - - # compare modulus - # h + 5 + (-p) - mr 6, 10 - mr 7, 11 - mr 8, 12 - addic. 6, 6, 5 - addze 7, 7 - addze 8, 8 - srdi 9, 8, 2 # overflow? - cmpdi 9, 0 - beq Skip_h64 - mr 10, 6 - mr 11, 7 - mr 12, 8 - -Skip_h64: - ld 6, 0(4) - ld 7, 8(4) - addc 10, 10, 6 - adde 11, 11, 7 - addze 12, 12 - - std 10, 0(5) - std 11, 8(5) - blr -SYM_FUNC_END(poly1305_emit_64) - -SYM_DATA_START_LOCAL(RMASK) -.align 5 -rmask: -.byte 0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f -cnum: -.long 0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000 -.long 0x1a, 0x00, 0x1a, 0x00 -.long 0x01000000, 0x01000000, 0x01000000, 0x01000000 -.long 0x00010203, 0x04050607, 0x10111213, 0x14151617 -.long 0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f -SYM_DATA_END(RMASK) diff --git a/arch/powerpc/crypto/sha1-powerpc-asm.S b/arch/powerpc/crypto/sha1-powerpc-asm.S deleted file mode 100644 index f0d5ed557ab1..000000000000 --- a/arch/powerpc/crypto/sha1-powerpc-asm.S +++ /dev/null @@ -1,188 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * SHA-1 implementation for PowerPC. - * - * Copyright (C) 2005 Paul Mackerras <paulus@samba.org> - */ - -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> -#include <asm/asm-compat.h> - -#ifdef __BIG_ENDIAN__ -#define LWZ(rt, d, ra) \ - lwz rt,d(ra) -#else -#define LWZ(rt, d, ra) \ - li rt,d; \ - lwbrx rt,rt,ra -#endif - -/* - * We roll the registers for T, A, B, C, D, E around on each - * iteration; T on iteration t is A on iteration t+1, and so on. - * We use registers 7 - 12 for this. - */ -#define RT(t) ((((t)+5)%6)+7) -#define RA(t) ((((t)+4)%6)+7) -#define RB(t) ((((t)+3)%6)+7) -#define RC(t) ((((t)+2)%6)+7) -#define RD(t) ((((t)+1)%6)+7) -#define RE(t) ((((t)+0)%6)+7) - -/* We use registers 16 - 31 for the W values */ -#define W(t) (((t)%16)+16) - -#define LOADW(t) \ - LWZ(W(t),(t)*4,r4) - -#define STEPD0_LOAD(t) \ - andc r0,RD(t),RB(t); \ - and r6,RB(t),RC(t); \ - rotlwi RT(t),RA(t),5; \ - or r6,r6,r0; \ - add r0,RE(t),r15; \ - add RT(t),RT(t),r6; \ - add r14,r0,W(t); \ - LWZ(W((t)+4),((t)+4)*4,r4); \ - rotlwi RB(t),RB(t),30; \ - add RT(t),RT(t),r14 - -#define STEPD0_UPDATE(t) \ - and r6,RB(t),RC(t); \ - andc r0,RD(t),RB(t); \ - rotlwi RT(t),RA(t),5; \ - rotlwi RB(t),RB(t),30; \ - or r6,r6,r0; \ - add r0,RE(t),r15; \ - xor r5,W((t)+4-3),W((t)+4-8); \ - add RT(t),RT(t),r6; \ - xor W((t)+4),W((t)+4-16),W((t)+4-14); \ - add r0,r0,W(t); \ - xor W((t)+4),W((t)+4),r5; \ - add RT(t),RT(t),r0; \ - rotlwi W((t)+4),W((t)+4),1 - -#define STEPD1(t) \ - xor r6,RB(t),RC(t); \ - rotlwi RT(t),RA(t),5; \ - rotlwi RB(t),RB(t),30; \ - xor r6,r6,RD(t); \ - add r0,RE(t),r15; \ - add RT(t),RT(t),r6; \ - add r0,r0,W(t); \ - add RT(t),RT(t),r0 - -#define STEPD1_UPDATE(t) \ - xor r6,RB(t),RC(t); \ - rotlwi RT(t),RA(t),5; \ - rotlwi RB(t),RB(t),30; \ - xor r6,r6,RD(t); \ - add r0,RE(t),r15; \ - xor r5,W((t)+4-3),W((t)+4-8); \ - add RT(t),RT(t),r6; \ - xor W((t)+4),W((t)+4-16),W((t)+4-14); \ - add r0,r0,W(t); \ - xor W((t)+4),W((t)+4),r5; \ - add RT(t),RT(t),r0; \ - rotlwi W((t)+4),W((t)+4),1 - -#define STEPD2_UPDATE(t) \ - and r6,RB(t),RC(t); \ - and r0,RB(t),RD(t); \ - rotlwi RT(t),RA(t),5; \ - or r6,r6,r0; \ - rotlwi RB(t),RB(t),30; \ - and r0,RC(t),RD(t); \ - xor r5,W((t)+4-3),W((t)+4-8); \ - or r6,r6,r0; \ - xor W((t)+4),W((t)+4-16),W((t)+4-14); \ - add r0,RE(t),r15; \ - add RT(t),RT(t),r6; \ - add r0,r0,W(t); \ - xor W((t)+4),W((t)+4),r5; \ - add RT(t),RT(t),r0; \ - rotlwi W((t)+4),W((t)+4),1 - -#define STEP0LD4(t) \ - STEPD0_LOAD(t); \ - STEPD0_LOAD((t)+1); \ - STEPD0_LOAD((t)+2); \ - STEPD0_LOAD((t)+3) - -#define STEPUP4(t, fn) \ - STEP##fn##_UPDATE(t); \ - STEP##fn##_UPDATE((t)+1); \ - STEP##fn##_UPDATE((t)+2); \ - STEP##fn##_UPDATE((t)+3) - -#define STEPUP20(t, fn) \ - STEPUP4(t, fn); \ - STEPUP4((t)+4, fn); \ - STEPUP4((t)+8, fn); \ - STEPUP4((t)+12, fn); \ - STEPUP4((t)+16, fn) - -_GLOBAL(powerpc_sha_transform) - PPC_STLU r1,-INT_FRAME_SIZE(r1) - SAVE_GPRS(14, 31, r1) - - /* Load up A - E */ - lwz RA(0),0(r3) /* A */ - lwz RB(0),4(r3) /* B */ - lwz RC(0),8(r3) /* C */ - lwz RD(0),12(r3) /* D */ - lwz RE(0),16(r3) /* E */ - - LOADW(0) - LOADW(1) - LOADW(2) - LOADW(3) - - lis r15,0x5a82 /* K0-19 */ - ori r15,r15,0x7999 - STEP0LD4(0) - STEP0LD4(4) - STEP0LD4(8) - STEPUP4(12, D0) - STEPUP4(16, D0) - - lis r15,0x6ed9 /* K20-39 */ - ori r15,r15,0xeba1 - STEPUP20(20, D1) - - lis r15,0x8f1b /* K40-59 */ - ori r15,r15,0xbcdc - STEPUP20(40, D2) - - lis r15,0xca62 /* K60-79 */ - ori r15,r15,0xc1d6 - STEPUP4(60, D1) - STEPUP4(64, D1) - STEPUP4(68, D1) - STEPUP4(72, D1) - lwz r20,16(r3) - STEPD1(76) - lwz r19,12(r3) - STEPD1(77) - lwz r18,8(r3) - STEPD1(78) - lwz r17,4(r3) - STEPD1(79) - - lwz r16,0(r3) - add r20,RE(80),r20 - add RD(0),RD(80),r19 - add RC(0),RC(80),r18 - add RB(0),RB(80),r17 - add RA(0),RA(80),r16 - mr RE(0),r20 - stw RA(0),0(r3) - stw RB(0),4(r3) - stw RC(0),8(r3) - stw RD(0),12(r3) - stw RE(0),16(r3) - - REST_GPRS(14, 31, r1) - addi r1,r1,INT_FRAME_SIZE - blr diff --git a/arch/powerpc/crypto/sha1-spe-asm.S b/arch/powerpc/crypto/sha1-spe-asm.S deleted file mode 100644 index 0f447523be5e..000000000000 --- a/arch/powerpc/crypto/sha1-spe-asm.S +++ /dev/null @@ -1,294 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Fast SHA-1 implementation for SPE instruction set (PPC) - * - * This code makes use of the SPE SIMD instruction set as defined in - * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf - * Implementation is based on optimization guide notes from - * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf - * - * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> - */ - -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> - -#define rHP r3 /* pointer to hash value */ -#define rWP r4 /* pointer to input */ -#define rKP r5 /* pointer to constants */ - -#define rW0 r14 /* 64 bit round words */ -#define rW1 r15 -#define rW2 r16 -#define rW3 r17 -#define rW4 r18 -#define rW5 r19 -#define rW6 r20 -#define rW7 r21 - -#define rH0 r6 /* 32 bit hash values */ -#define rH1 r7 -#define rH2 r8 -#define rH3 r9 -#define rH4 r10 - -#define rT0 r22 /* 64 bit temporary */ -#define rT1 r0 /* 32 bit temporaries */ -#define rT2 r11 -#define rT3 r12 - -#define rK r23 /* 64 bit constant in volatile register */ - -#define LOAD_K01 - -#define LOAD_K11 \ - evlwwsplat rK,0(rKP); - -#define LOAD_K21 \ - evlwwsplat rK,4(rKP); - -#define LOAD_K31 \ - evlwwsplat rK,8(rKP); - -#define LOAD_K41 \ - evlwwsplat rK,12(rKP); - -#define INITIALIZE \ - stwu r1,-128(r1); /* create stack frame */ \ - evstdw r14,8(r1); /* We must save non volatile */ \ - evstdw r15,16(r1); /* registers. Take the chance */ \ - evstdw r16,24(r1); /* and save the SPE part too */ \ - evstdw r17,32(r1); \ - evstdw r18,40(r1); \ - evstdw r19,48(r1); \ - evstdw r20,56(r1); \ - evstdw r21,64(r1); \ - evstdw r22,72(r1); \ - evstdw r23,80(r1); - - -#define FINALIZE \ - evldw r14,8(r1); /* restore SPE registers */ \ - evldw r15,16(r1); \ - evldw r16,24(r1); \ - evldw r17,32(r1); \ - evldw r18,40(r1); \ - evldw r19,48(r1); \ - evldw r20,56(r1); \ - evldw r21,64(r1); \ - evldw r22,72(r1); \ - evldw r23,80(r1); \ - xor r0,r0,r0; \ - stw r0,8(r1); /* Delete sensitive data */ \ - stw r0,16(r1); /* that we might have pushed */ \ - stw r0,24(r1); /* from other context that runs */ \ - stw r0,32(r1); /* the same code. Assume that */ \ - stw r0,40(r1); /* the lower part of the GPRs */ \ - stw r0,48(r1); /* were already overwritten on */ \ - stw r0,56(r1); /* the way down to here */ \ - stw r0,64(r1); \ - stw r0,72(r1); \ - stw r0,80(r1); \ - addi r1,r1,128; /* cleanup stack frame */ - -#ifdef __BIG_ENDIAN__ -#define LOAD_DATA(reg, off) \ - lwz reg,off(rWP); /* load data */ -#define NEXT_BLOCK \ - addi rWP,rWP,64; /* increment per block */ -#else -#define LOAD_DATA(reg, off) \ - lwbrx reg,0,rWP; /* load data */ \ - addi rWP,rWP,4; /* increment per word */ -#define NEXT_BLOCK /* nothing to do */ -#endif - -#define R_00_15(a, b, c, d, e, w0, w1, k, off) \ - LOAD_DATA(w0, off) /* 1: W */ \ - and rT2,b,c; /* 1: F' = B and C */ \ - LOAD_K##k##1 \ - andc rT1,d,b; /* 1: F" = ~B and D */ \ - rotrwi rT0,a,27; /* 1: A' = A rotl 5 */ \ - or rT2,rT2,rT1; /* 1: F = F' or F" */ \ - add e,e,rT0; /* 1: E = E + A' */ \ - rotrwi b,b,2; /* 1: B = B rotl 30 */ \ - add e,e,w0; /* 1: E = E + W */ \ - LOAD_DATA(w1, off+4) /* 2: W */ \ - add e,e,rT2; /* 1: E = E + F */ \ - and rT1,a,b; /* 2: F' = B and C */ \ - add e,e,rK; /* 1: E = E + K */ \ - andc rT2,c,a; /* 2: F" = ~B and D */ \ - add d,d,rK; /* 2: E = E + K */ \ - or rT2,rT2,rT1; /* 2: F = F' or F" */ \ - rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ - add d,d,w1; /* 2: E = E + W */ \ - rotrwi a,a,2; /* 2: B = B rotl 30 */ \ - add d,d,rT0; /* 2: E = E + A' */ \ - evmergelo w1,w1,w0; /* mix W[0]/W[1] */ \ - add d,d,rT2 /* 2: E = E + F */ - -#define R_16_19(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ - and rT2,b,c; /* 1: F' = B and C */ \ - evmergelohi rT0,w7,w6; /* W[-3] */ \ - andc rT1,d,b; /* 1: F" = ~B and D */ \ - evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \ - or rT1,rT1,rT2; /* 1: F = F' or F" */ \ - evxor w0,w0,w4; /* W = W xor W[-8] */ \ - add e,e,rT1; /* 1: E = E + F */ \ - evxor w0,w0,w1; /* W = W xor W[-14] */ \ - rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \ - evrlwi w0,w0,1; /* W = W rotl 1 */ \ - add e,e,rT2; /* 1: E = E + A' */ \ - evaddw rT0,w0,rK; /* WK = W + K */ \ - rotrwi b,b,2; /* 1: B = B rotl 30 */ \ - LOAD_K##k##1 \ - evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \ - add e,e,rT0; /* 1: E = E + WK */ \ - add d,d,rT1; /* 2: E = E + WK */ \ - and rT2,a,b; /* 2: F' = B and C */ \ - andc rT1,c,a; /* 2: F" = ~B and D */ \ - rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ - or rT1,rT1,rT2; /* 2: F = F' or F" */ \ - add d,d,rT0; /* 2: E = E + A' */ \ - rotrwi a,a,2; /* 2: B = B rotl 30 */ \ - add d,d,rT1 /* 2: E = E + F */ - -#define R_20_39(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ - evmergelohi rT0,w7,w6; /* W[-3] */ \ - xor rT2,b,c; /* 1: F' = B xor C */ \ - evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \ - xor rT2,rT2,d; /* 1: F = F' xor D */ \ - evxor w0,w0,w4; /* W = W xor W[-8] */ \ - add e,e,rT2; /* 1: E = E + F */ \ - evxor w0,w0,w1; /* W = W xor W[-14] */ \ - rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \ - evrlwi w0,w0,1; /* W = W rotl 1 */ \ - add e,e,rT2; /* 1: E = E + A' */ \ - evaddw rT0,w0,rK; /* WK = W + K */ \ - rotrwi b,b,2; /* 1: B = B rotl 30 */ \ - LOAD_K##k##1 \ - evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \ - add e,e,rT0; /* 1: E = E + WK */ \ - xor rT2,a,b; /* 2: F' = B xor C */ \ - add d,d,rT1; /* 2: E = E + WK */ \ - xor rT2,rT2,c; /* 2: F = F' xor D */ \ - rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ - add d,d,rT2; /* 2: E = E + F */ \ - rotrwi a,a,2; /* 2: B = B rotl 30 */ \ - add d,d,rT0 /* 2: E = E + A' */ - -#define R_40_59(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ - and rT2,b,c; /* 1: F' = B and C */ \ - evmergelohi rT0,w7,w6; /* W[-3] */ \ - or rT1,b,c; /* 1: F" = B or C */ \ - evxor w0,w0,rT0; /* W = W[-16] xor W[-3] */ \ - and rT1,d,rT1; /* 1: F" = F" and D */ \ - evxor w0,w0,w4; /* W = W xor W[-8] */ \ - or rT2,rT2,rT1; /* 1: F = F' or F" */ \ - evxor w0,w0,w1; /* W = W xor W[-14] */ \ - add e,e,rT2; /* 1: E = E + F */ \ - evrlwi w0,w0,1; /* W = W rotl 1 */ \ - rotrwi rT2,a,27; /* 1: A' = A rotl 5 */ \ - evaddw rT0,w0,rK; /* WK = W + K */ \ - add e,e,rT2; /* 1: E = E + A' */ \ - LOAD_K##k##1 \ - evmergehi rT1,rT1,rT0; /* WK1/WK2 */ \ - rotrwi b,b,2; /* 1: B = B rotl 30 */ \ - add e,e,rT0; /* 1: E = E + WK */ \ - and rT2,a,b; /* 2: F' = B and C */ \ - or rT0,a,b; /* 2: F" = B or C */ \ - add d,d,rT1; /* 2: E = E + WK */ \ - and rT0,c,rT0; /* 2: F" = F" and D */ \ - rotrwi a,a,2; /* 2: B = B rotl 30 */ \ - or rT2,rT2,rT0; /* 2: F = F' or F" */ \ - rotrwi rT0,e,27; /* 2: A' = A rotl 5 */ \ - add d,d,rT2; /* 2: E = E + F */ \ - add d,d,rT0 /* 2: E = E + A' */ - -#define R_60_79(a, b, c, d, e, w0, w1, w4, w6, w7, k) \ - R_20_39(a, b, c, d, e, w0, w1, w4, w6, w7, k) - -_GLOBAL(ppc_spe_sha1_transform) - INITIALIZE - - lwz rH0,0(rHP) - lwz rH1,4(rHP) - mtctr r5 - lwz rH2,8(rHP) - lis rKP,PPC_SPE_SHA1_K@h - lwz rH3,12(rHP) - ori rKP,rKP,PPC_SPE_SHA1_K@l - lwz rH4,16(rHP) - -ppc_spe_sha1_main: - R_00_15(rH0, rH1, rH2, rH3, rH4, rW1, rW0, 1, 0) - R_00_15(rH3, rH4, rH0, rH1, rH2, rW2, rW1, 0, 8) - R_00_15(rH1, rH2, rH3, rH4, rH0, rW3, rW2, 0, 16) - R_00_15(rH4, rH0, rH1, rH2, rH3, rW4, rW3, 0, 24) - R_00_15(rH2, rH3, rH4, rH0, rH1, rW5, rW4, 0, 32) - R_00_15(rH0, rH1, rH2, rH3, rH4, rW6, rW5, 0, 40) - R_00_15(rH3, rH4, rH0, rH1, rH2, rT3, rW6, 0, 48) - R_00_15(rH1, rH2, rH3, rH4, rH0, rT3, rW7, 0, 56) - - R_16_19(rH4, rH0, rH1, rH2, rH3, rW0, rW1, rW4, rW6, rW7, 0) - R_16_19(rH2, rH3, rH4, rH0, rH1, rW1, rW2, rW5, rW7, rW0, 2) - - R_20_39(rH0, rH1, rH2, rH3, rH4, rW2, rW3, rW6, rW0, rW1, 0) - R_20_39(rH3, rH4, rH0, rH1, rH2, rW3, rW4, rW7, rW1, rW2, 0) - R_20_39(rH1, rH2, rH3, rH4, rH0, rW4, rW5, rW0, rW2, rW3, 0) - R_20_39(rH4, rH0, rH1, rH2, rH3, rW5, rW6, rW1, rW3, rW4, 0) - R_20_39(rH2, rH3, rH4, rH0, rH1, rW6, rW7, rW2, rW4, rW5, 0) - R_20_39(rH0, rH1, rH2, rH3, rH4, rW7, rW0, rW3, rW5, rW6, 0) - R_20_39(rH3, rH4, rH0, rH1, rH2, rW0, rW1, rW4, rW6, rW7, 0) - R_20_39(rH1, rH2, rH3, rH4, rH0, rW1, rW2, rW5, rW7, rW0, 0) - R_20_39(rH4, rH0, rH1, rH2, rH3, rW2, rW3, rW6, rW0, rW1, 0) - R_20_39(rH2, rH3, rH4, rH0, rH1, rW3, rW4, rW7, rW1, rW2, 3) - - R_40_59(rH0, rH1, rH2, rH3, rH4, rW4, rW5, rW0, rW2, rW3, 0) - R_40_59(rH3, rH4, rH0, rH1, rH2, rW5, rW6, rW1, rW3, rW4, 0) - R_40_59(rH1, rH2, rH3, rH4, rH0, rW6, rW7, rW2, rW4, rW5, 0) - R_40_59(rH4, rH0, rH1, rH2, rH3, rW7, rW0, rW3, rW5, rW6, 0) - R_40_59(rH2, rH3, rH4, rH0, rH1, rW0, rW1, rW4, rW6, rW7, 0) - R_40_59(rH0, rH1, rH2, rH3, rH4, rW1, rW2, rW5, rW7, rW0, 0) - R_40_59(rH3, rH4, rH0, rH1, rH2, rW2, rW3, rW6, rW0, rW1, 0) - R_40_59(rH1, rH2, rH3, rH4, rH0, rW3, rW4, rW7, rW1, rW2, 0) - R_40_59(rH4, rH0, rH1, rH2, rH3, rW4, rW5, rW0, rW2, rW3, 0) - R_40_59(rH2, rH3, rH4, rH0, rH1, rW5, rW6, rW1, rW3, rW4, 4) - - R_60_79(rH0, rH1, rH2, rH3, rH4, rW6, rW7, rW2, rW4, rW5, 0) - R_60_79(rH3, rH4, rH0, rH1, rH2, rW7, rW0, rW3, rW5, rW6, 0) - R_60_79(rH1, rH2, rH3, rH4, rH0, rW0, rW1, rW4, rW6, rW7, 0) - R_60_79(rH4, rH0, rH1, rH2, rH3, rW1, rW2, rW5, rW7, rW0, 0) - R_60_79(rH2, rH3, rH4, rH0, rH1, rW2, rW3, rW6, rW0, rW1, 0) - R_60_79(rH0, rH1, rH2, rH3, rH4, rW3, rW4, rW7, rW1, rW2, 0) - R_60_79(rH3, rH4, rH0, rH1, rH2, rW4, rW5, rW0, rW2, rW3, 0) - lwz rT3,0(rHP) - R_60_79(rH1, rH2, rH3, rH4, rH0, rW5, rW6, rW1, rW3, rW4, 0) - lwz rW1,4(rHP) - R_60_79(rH4, rH0, rH1, rH2, rH3, rW6, rW7, rW2, rW4, rW5, 0) - lwz rW2,8(rHP) - R_60_79(rH2, rH3, rH4, rH0, rH1, rW7, rW0, rW3, rW5, rW6, 0) - lwz rW3,12(rHP) - NEXT_BLOCK - lwz rW4,16(rHP) - - add rH0,rH0,rT3 - stw rH0,0(rHP) - add rH1,rH1,rW1 - stw rH1,4(rHP) - add rH2,rH2,rW2 - stw rH2,8(rHP) - add rH3,rH3,rW3 - stw rH3,12(rHP) - add rH4,rH4,rW4 - stw rH4,16(rHP) - - bdnz ppc_spe_sha1_main - - FINALIZE - blr - -.data -.align 4 -PPC_SPE_SHA1_K: - .long 0x5A827999,0x6ED9EBA1,0x8F1BBCDC,0xCA62C1D6 diff --git a/arch/powerpc/crypto/sha1-spe-glue.c b/arch/powerpc/crypto/sha1-spe-glue.c deleted file mode 100644 index 9170892a8557..000000000000 --- a/arch/powerpc/crypto/sha1-spe-glue.c +++ /dev/null @@ -1,191 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Glue code for SHA-1 implementation for SPE instructions (PPC) - * - * Based on generic implementation. - * - * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> - */ - -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> -#include <crypto/sha1.h> -#include <crypto/sha1_base.h> -#include <asm/byteorder.h> -#include <asm/switch_to.h> -#include <linux/hardirq.h> - -/* - * MAX_BYTES defines the number of bytes that are allowed to be processed - * between preempt_disable() and preempt_enable(). SHA1 takes ~1000 - * operations per 64 bytes. e500 cores can issue two arithmetic instructions - * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). - * Thus 2KB of input data will need an estimated maximum of 18,000 cycles. - * Headroom for cache misses included. Even with the low end model clocked - * at 667 MHz this equals to a critical time window of less than 27us. - * - */ -#define MAX_BYTES 2048 - -extern void ppc_spe_sha1_transform(u32 *state, const u8 *src, u32 blocks); - -static void spe_begin(void) -{ - /* We just start SPE operations and will save SPE registers later. */ - preempt_disable(); - enable_kernel_spe(); -} - -static void spe_end(void) -{ - disable_kernel_spe(); - /* reenable preemption */ - preempt_enable(); -} - -static inline void ppc_sha1_clear_context(struct sha1_state *sctx) -{ - int count = sizeof(struct sha1_state) >> 2; - u32 *ptr = (u32 *)sctx; - - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct sha1_state) % 4); - do { *ptr++ = 0; } while (--count); -} - -static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - const unsigned int avail = 64 - offset; - unsigned int bytes; - const u8 *src = data; - - if (avail > len) { - sctx->count += len; - memcpy((char *)sctx->buffer + offset, src, len); - return 0; - } - - sctx->count += len; - - if (offset) { - memcpy((char *)sctx->buffer + offset, src, avail); - - spe_begin(); - ppc_spe_sha1_transform(sctx->state, (const u8 *)sctx->buffer, 1); - spe_end(); - - len -= avail; - src += avail; - } - - while (len > 63) { - bytes = (len > MAX_BYTES) ? MAX_BYTES : len; - bytes = bytes & ~0x3f; - - spe_begin(); - ppc_spe_sha1_transform(sctx->state, src, bytes >> 6); - spe_end(); - - src += bytes; - len -= bytes; - } - - memcpy((char *)sctx->buffer, src, len); - return 0; -} - -static int ppc_spe_sha1_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - char *p = (char *)sctx->buffer + offset; - int padlen; - __be64 *pbits = (__be64 *)(((char *)&sctx->buffer) + 56); - __be32 *dst = (__be32 *)out; - - padlen = 55 - offset; - *p++ = 0x80; - - spe_begin(); - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); - p = (char *)sctx->buffer; - padlen = 56; - } - - memset(p, 0, padlen); - *pbits = cpu_to_be64(sctx->count << 3); - ppc_spe_sha1_transform(sctx->state, sctx->buffer, 1); - - spe_end(); - - dst[0] = cpu_to_be32(sctx->state[0]); - dst[1] = cpu_to_be32(sctx->state[1]); - dst[2] = cpu_to_be32(sctx->state[2]); - dst[3] = cpu_to_be32(sctx->state[3]); - dst[4] = cpu_to_be32(sctx->state[4]); - - ppc_sha1_clear_context(sctx); - return 0; -} - -static int ppc_spe_sha1_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = ppc_spe_sha1_update, - .final = ppc_spe_sha1_final, - .export = ppc_spe_sha1_export, - .import = ppc_spe_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-ppc-spe", - .cra_priority = 300, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init ppc_spe_sha1_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit ppc_spe_sha1_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(ppc_spe_sha1_mod_init); -module_exit(ppc_spe_sha1_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, SPE optimized"); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_ALIAS_CRYPTO("sha1-ppc-spe"); diff --git a/arch/powerpc/crypto/sha1.c b/arch/powerpc/crypto/sha1.c deleted file mode 100644 index f283bbd3f121..000000000000 --- a/arch/powerpc/crypto/sha1.c +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cryptographic API. - * - * powerpc implementation of the SHA1 Secure Hash Algorithm. - * - * Derived from cryptoapi implementation, adapted for in-place - * scatterlist interface. - * - * Derived from "crypto/sha1.c" - * Copyright (c) Alan Smithee. - * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> - * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> - */ -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> -#include <crypto/sha1.h> -#include <crypto/sha1_base.h> -#include <asm/byteorder.h> - -void powerpc_sha_transform(u32 *state, const u8 *src); - -static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - unsigned int partial, done; - const u8 *src; - - partial = sctx->count & 0x3f; - sctx->count += len; - done = 0; - src = data; - - if ((partial + len) > 63) { - - if (partial) { - done = -partial; - memcpy(sctx->buffer + partial, data, done + 64); - src = sctx->buffer; - } - - do { - powerpc_sha_transform(sctx->state, src); - done += 64; - src = data + done; - } while (done + 63 < len); - - partial = 0; - } - memcpy(sctx->buffer + partial, src, len - done); - - return 0; -} - - -/* Add padding and return the message digest. */ -static int powerpc_sha1_final(struct shash_desc *desc, u8 *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - __be32 *dst = (__be32 *)out; - u32 i, index, padlen; - __be64 bits; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_be64(sctx->count << 3); - - /* Pad out to 56 mod 64 */ - index = sctx->count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - powerpc_sha1_update(desc, padding, padlen); - - /* Append length */ - powerpc_sha1_update(desc, (const u8 *)&bits, sizeof(bits)); - - /* Store state in digest */ - for (i = 0; i < 5; i++) - dst[i] = cpu_to_be32(sctx->state[i]); - - /* Wipe context */ - memset(sctx, 0, sizeof *sctx); - - return 0; -} - -static int powerpc_sha1_export(struct shash_desc *desc, void *out) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int powerpc_sha1_import(struct shash_desc *desc, const void *in) -{ - struct sha1_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; -} - -static struct shash_alg alg = { - .digestsize = SHA1_DIGEST_SIZE, - .init = sha1_base_init, - .update = powerpc_sha1_update, - .final = powerpc_sha1_final, - .export = powerpc_sha1_export, - .import = powerpc_sha1_import, - .descsize = sizeof(struct sha1_state), - .statesize = sizeof(struct sha1_state), - .base = { - .cra_name = "sha1", - .cra_driver_name= "sha1-powerpc", - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static int __init sha1_powerpc_mod_init(void) -{ - return crypto_register_shash(&alg); -} - -static void __exit sha1_powerpc_mod_fini(void) -{ - crypto_unregister_shash(&alg); -} - -module_init(sha1_powerpc_mod_init); -module_exit(sha1_powerpc_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm"); - -MODULE_ALIAS_CRYPTO("sha1"); -MODULE_ALIAS_CRYPTO("sha1-powerpc"); diff --git a/arch/powerpc/crypto/sha256-spe-asm.S b/arch/powerpc/crypto/sha256-spe-asm.S deleted file mode 100644 index cd99d71dae34..000000000000 --- a/arch/powerpc/crypto/sha256-spe-asm.S +++ /dev/null @@ -1,318 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Fast SHA-256 implementation for SPE instruction set (PPC) - * - * This code makes use of the SPE SIMD instruction set as defined in - * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf - * Implementation is based on optimization guide notes from - * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf - * - * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> - */ - -#include <asm/ppc_asm.h> -#include <asm/asm-offsets.h> - -#define rHP r3 /* pointer to hash values in memory */ -#define rKP r24 /* pointer to round constants */ -#define rWP r4 /* pointer to input data */ - -#define rH0 r5 /* 8 32 bit hash values in 8 registers */ -#define rH1 r6 -#define rH2 r7 -#define rH3 r8 -#define rH4 r9 -#define rH5 r10 -#define rH6 r11 -#define rH7 r12 - -#define rW0 r14 /* 64 bit registers. 16 words in 8 registers */ -#define rW1 r15 -#define rW2 r16 -#define rW3 r17 -#define rW4 r18 -#define rW5 r19 -#define rW6 r20 -#define rW7 r21 - -#define rT0 r22 /* 64 bit temporaries */ -#define rT1 r23 -#define rT2 r0 /* 32 bit temporaries */ -#define rT3 r25 - -#define CMP_KN_LOOP -#define CMP_KC_LOOP \ - cmpwi rT1,0; - -#define INITIALIZE \ - stwu r1,-128(r1); /* create stack frame */ \ - evstdw r14,8(r1); /* We must save non volatile */ \ - evstdw r15,16(r1); /* registers. Take the chance */ \ - evstdw r16,24(r1); /* and save the SPE part too */ \ - evstdw r17,32(r1); \ - evstdw r18,40(r1); \ - evstdw r19,48(r1); \ - evstdw r20,56(r1); \ - evstdw r21,64(r1); \ - evstdw r22,72(r1); \ - evstdw r23,80(r1); \ - stw r24,88(r1); /* save normal registers */ \ - stw r25,92(r1); - - -#define FINALIZE \ - evldw r14,8(r1); /* restore SPE registers */ \ - evldw r15,16(r1); \ - evldw r16,24(r1); \ - evldw r17,32(r1); \ - evldw r18,40(r1); \ - evldw r19,48(r1); \ - evldw r20,56(r1); \ - evldw r21,64(r1); \ - evldw r22,72(r1); \ - evldw r23,80(r1); \ - lwz r24,88(r1); /* restore normal registers */ \ - lwz r25,92(r1); \ - xor r0,r0,r0; \ - stw r0,8(r1); /* Delete sensitive data */ \ - stw r0,16(r1); /* that we might have pushed */ \ - stw r0,24(r1); /* from other context that runs */ \ - stw r0,32(r1); /* the same code. Assume that */ \ - stw r0,40(r1); /* the lower part of the GPRs */ \ - stw r0,48(r1); /* was already overwritten on */ \ - stw r0,56(r1); /* the way down to here */ \ - stw r0,64(r1); \ - stw r0,72(r1); \ - stw r0,80(r1); \ - addi r1,r1,128; /* cleanup stack frame */ - -#ifdef __BIG_ENDIAN__ -#define LOAD_DATA(reg, off) \ - lwz reg,off(rWP); /* load data */ -#define NEXT_BLOCK \ - addi rWP,rWP,64; /* increment per block */ -#else -#define LOAD_DATA(reg, off) \ - lwbrx reg,0,rWP; /* load data */ \ - addi rWP,rWP,4; /* increment per word */ -#define NEXT_BLOCK /* nothing to do */ -#endif - -#define R_LOAD_W(a, b, c, d, e, f, g, h, w, off) \ - LOAD_DATA(w, off) /* 1: W */ \ - rotrwi rT0,e,6; /* 1: S1 = e rotr 6 */ \ - rotrwi rT1,e,11; /* 1: S1' = e rotr 11 */ \ - rotrwi rT2,e,25; /* 1: S1" = e rotr 25 */ \ - xor rT0,rT0,rT1; /* 1: S1 = S1 xor S1' */ \ - and rT3,e,f; /* 1: ch = e and f */ \ - xor rT0,rT0,rT2; /* 1: S1 = S1 xor S1" */ \ - andc rT1,g,e; /* 1: ch' = ~e and g */ \ - lwz rT2,off(rKP); /* 1: K */ \ - xor rT3,rT3,rT1; /* 1: ch = ch xor ch' */ \ - add h,h,rT0; /* 1: temp1 = h + S1 */ \ - add rT3,rT3,w; /* 1: temp1' = ch + w */ \ - rotrwi rT0,a,2; /* 1: S0 = a rotr 2 */ \ - add h,h,rT3; /* 1: temp1 = temp1 + temp1' */ \ - rotrwi rT1,a,13; /* 1: S0' = a rotr 13 */ \ - add h,h,rT2; /* 1: temp1 = temp1 + K */ \ - rotrwi rT3,a,22; /* 1: S0" = a rotr 22 */ \ - xor rT0,rT0,rT1; /* 1: S0 = S0 xor S0' */ \ - add d,d,h; /* 1: d = d + temp1 */ \ - xor rT3,rT0,rT3; /* 1: S0 = S0 xor S0" */ \ - evmergelo w,w,w; /* shift W */ \ - or rT2,a,b; /* 1: maj = a or b */ \ - and rT1,a,b; /* 1: maj' = a and b */ \ - and rT2,rT2,c; /* 1: maj = maj and c */ \ - LOAD_DATA(w, off+4) /* 2: W */ \ - or rT2,rT1,rT2; /* 1: maj = maj or maj' */ \ - rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ - add rT3,rT3,rT2; /* 1: temp2 = S0 + maj */ \ - rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ - add h,h,rT3; /* 1: h = temp1 + temp2 */ \ - rotrwi rT2,d,25; /* 2: S1" = e rotr 25 */ \ - xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ - and rT3,d,e; /* 2: ch = e and f */ \ - xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ - andc rT1,f,d; /* 2: ch' = ~e and g */ \ - lwz rT2,off+4(rKP); /* 2: K */ \ - xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ - add g,g,rT0; /* 2: temp1 = h + S1 */ \ - add rT3,rT3,w; /* 2: temp1' = ch + w */ \ - rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ - add g,g,rT3; /* 2: temp1 = temp1 + temp1' */ \ - rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ - add g,g,rT2; /* 2: temp1 = temp1 + K */ \ - rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ - xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ - or rT2,h,a; /* 2: maj = a or b */ \ - xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ - and rT1,h,a; /* 2: maj' = a and b */ \ - and rT2,rT2,b; /* 2: maj = maj and c */ \ - add c,c,g; /* 2: d = d + temp1 */ \ - or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ - add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ - add g,g,rT3 /* 2: h = temp1 + temp2 */ - -#define R_CALC_W(a, b, c, d, e, f, g, h, w0, w1, w4, w5, w7, k, off) \ - rotrwi rT2,e,6; /* 1: S1 = e rotr 6 */ \ - evmergelohi rT0,w0,w1; /* w[-15] */ \ - rotrwi rT3,e,11; /* 1: S1' = e rotr 11 */ \ - evsrwiu rT1,rT0,3; /* s0 = w[-15] >> 3 */ \ - xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ - evrlwi rT0,rT0,25; /* s0' = w[-15] rotr 7 */ \ - rotrwi rT3,e,25; /* 1: S1' = e rotr 25 */ \ - evxor rT1,rT1,rT0; /* s0 = s0 xor s0' */ \ - xor rT2,rT2,rT3; /* 1: S1 = S1 xor S1' */ \ - evrlwi rT0,rT0,21; /* s0' = w[-15] rotr 18 */ \ - add h,h,rT2; /* 1: temp1 = h + S1 */ \ - evxor rT0,rT0,rT1; /* s0 = s0 xor s0' */ \ - and rT2,e,f; /* 1: ch = e and f */ \ - evaddw w0,w0,rT0; /* w = w[-16] + s0 */ \ - andc rT3,g,e; /* 1: ch' = ~e and g */ \ - evsrwiu rT0,w7,10; /* s1 = w[-2] >> 10 */ \ - xor rT2,rT2,rT3; /* 1: ch = ch xor ch' */ \ - evrlwi rT1,w7,15; /* s1' = w[-2] rotr 17 */ \ - add h,h,rT2; /* 1: temp1 = temp1 + ch */ \ - evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ - rotrwi rT2,a,2; /* 1: S0 = a rotr 2 */ \ - evrlwi rT1,w7,13; /* s1' = w[-2] rotr 19 */ \ - rotrwi rT3,a,13; /* 1: S0' = a rotr 13 */ \ - evxor rT0,rT0,rT1; /* s1 = s1 xor s1' */ \ - xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ - evldw rT1,off(rKP); /* k */ \ - rotrwi rT3,a,22; /* 1: S0' = a rotr 22 */ \ - evaddw w0,w0,rT0; /* w = w + s1 */ \ - xor rT2,rT2,rT3; /* 1: S0 = S0 xor S0' */ \ - evmergelohi rT0,w4,w5; /* w[-7] */ \ - and rT3,a,b; /* 1: maj = a and b */ \ - evaddw w0,w0,rT0; /* w = w + w[-7] */ \ - CMP_K##k##_LOOP \ - add rT2,rT2,rT3; /* 1: temp2 = S0 + maj */ \ - evaddw rT1,rT1,w0; /* wk = w + k */ \ - xor rT3,a,b; /* 1: maj = a xor b */ \ - evmergehi rT0,rT1,rT1; /* wk1/wk2 */ \ - and rT3,rT3,c; /* 1: maj = maj and c */ \ - add h,h,rT0; /* 1: temp1 = temp1 + wk */ \ - add rT2,rT2,rT3; /* 1: temp2 = temp2 + maj */ \ - add g,g,rT1; /* 2: temp1 = temp1 + wk */ \ - add d,d,h; /* 1: d = d + temp1 */ \ - rotrwi rT0,d,6; /* 2: S1 = e rotr 6 */ \ - add h,h,rT2; /* 1: h = temp1 + temp2 */ \ - rotrwi rT1,d,11; /* 2: S1' = e rotr 11 */ \ - rotrwi rT2,d,25; /* 2: S" = e rotr 25 */ \ - xor rT0,rT0,rT1; /* 2: S1 = S1 xor S1' */ \ - and rT3,d,e; /* 2: ch = e and f */ \ - xor rT0,rT0,rT2; /* 2: S1 = S1 xor S1" */ \ - andc rT1,f,d; /* 2: ch' = ~e and g */ \ - add g,g,rT0; /* 2: temp1 = h + S1 */ \ - xor rT3,rT3,rT1; /* 2: ch = ch xor ch' */ \ - rotrwi rT0,h,2; /* 2: S0 = a rotr 2 */ \ - add g,g,rT3; /* 2: temp1 = temp1 + ch */ \ - rotrwi rT1,h,13; /* 2: S0' = a rotr 13 */ \ - rotrwi rT3,h,22; /* 2: S0" = a rotr 22 */ \ - xor rT0,rT0,rT1; /* 2: S0 = S0 xor S0' */ \ - or rT2,h,a; /* 2: maj = a or b */ \ - and rT1,h,a; /* 2: maj' = a and b */ \ - and rT2,rT2,b; /* 2: maj = maj and c */ \ - xor rT3,rT0,rT3; /* 2: S0 = S0 xor S0" */ \ - or rT2,rT1,rT2; /* 2: maj = maj or maj' */ \ - add c,c,g; /* 2: d = d + temp1 */ \ - add rT3,rT3,rT2; /* 2: temp2 = S0 + maj */ \ - add g,g,rT3 /* 2: h = temp1 + temp2 */ - -_GLOBAL(ppc_spe_sha256_transform) - INITIALIZE - - mtctr r5 - lwz rH0,0(rHP) - lwz rH1,4(rHP) - lwz rH2,8(rHP) - lwz rH3,12(rHP) - lwz rH4,16(rHP) - lwz rH5,20(rHP) - lwz rH6,24(rHP) - lwz rH7,28(rHP) - -ppc_spe_sha256_main: - lis rKP,PPC_SPE_SHA256_K@ha - addi rKP,rKP,PPC_SPE_SHA256_K@l - - R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW0, 0) - R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW1, 8) - R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW2, 16) - R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW3, 24) - R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW4, 32) - R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW5, 40) - R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW6, 48) - R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW7, 56) -ppc_spe_sha256_16_rounds: - addi rKP,rKP,64 - R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, - rW0, rW1, rW4, rW5, rW7, N, 0) - R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, - rW1, rW2, rW5, rW6, rW0, N, 8) - R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, - rW2, rW3, rW6, rW7, rW1, N, 16) - R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, - rW3, rW4, rW7, rW0, rW2, N, 24) - R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, - rW4, rW5, rW0, rW1, rW3, N, 32) - R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, - rW5, rW6, rW1, rW2, rW4, N, 40) - R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, - rW6, rW7, rW2, rW3, rW5, N, 48) - R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, - rW7, rW0, rW3, rW4, rW6, C, 56) - bt gt,ppc_spe_sha256_16_rounds - - lwz rW0,0(rHP) - NEXT_BLOCK - lwz rW1,4(rHP) - lwz rW2,8(rHP) - lwz rW3,12(rHP) - lwz rW4,16(rHP) - lwz rW5,20(rHP) - lwz rW6,24(rHP) - lwz rW7,28(rHP) - - add rH0,rH0,rW0 - stw rH0,0(rHP) - add rH1,rH1,rW1 - stw rH1,4(rHP) - add rH2,rH2,rW2 - stw rH2,8(rHP) - add rH3,rH3,rW3 - stw rH3,12(rHP) - add rH4,rH4,rW4 - stw rH4,16(rHP) - add rH5,rH5,rW5 - stw rH5,20(rHP) - add rH6,rH6,rW6 - stw rH6,24(rHP) - add rH7,rH7,rW7 - stw rH7,28(rHP) - - bdnz ppc_spe_sha256_main - - FINALIZE - blr - -.data -.align 5 -PPC_SPE_SHA256_K: - .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 - .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 - .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 - .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 - .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc - .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da - .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 - .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 - .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 - .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 - .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 - .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 - .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 - .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 - .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 - .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 diff --git a/arch/powerpc/crypto/sha256-spe-glue.c b/arch/powerpc/crypto/sha256-spe-glue.c deleted file mode 100644 index 2997d13236e0..000000000000 --- a/arch/powerpc/crypto/sha256-spe-glue.c +++ /dev/null @@ -1,235 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Glue code for SHA-256 implementation for SPE instructions (PPC) - * - * Based on generic implementation. The assembler module takes care - * about the SPE registers so it can run from interrupt context. - * - * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de> - */ - -#include <crypto/internal/hash.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/mm.h> -#include <linux/types.h> -#include <crypto/sha2.h> -#include <crypto/sha256_base.h> -#include <asm/byteorder.h> -#include <asm/switch_to.h> -#include <linux/hardirq.h> - -/* - * MAX_BYTES defines the number of bytes that are allowed to be processed - * between preempt_disable() and preempt_enable(). SHA256 takes ~2,000 - * operations per 64 bytes. e500 cores can issue two arithmetic instructions - * per clock cycle using one 32/64 bit unit (SU1) and one 32 bit unit (SU2). - * Thus 1KB of input data will need an estimated maximum of 18,000 cycles. - * Headroom for cache misses included. Even with the low end model clocked - * at 667 MHz this equals to a critical time window of less than 27us. - * - */ -#define MAX_BYTES 1024 - -extern void ppc_spe_sha256_transform(u32 *state, const u8 *src, u32 blocks); - -static void spe_begin(void) -{ - /* We just start SPE operations and will save SPE registers later. */ - preempt_disable(); - enable_kernel_spe(); -} - -static void spe_end(void) -{ - disable_kernel_spe(); - /* reenable preemption */ - preempt_enable(); -} - -static inline void ppc_sha256_clear_context(struct sha256_state *sctx) -{ - int count = sizeof(struct sha256_state) >> 2; - u32 *ptr = (u32 *)sctx; - - /* make sure we can clear the fast way */ - BUILD_BUG_ON(sizeof(struct sha256_state) % 4); - do { *ptr++ = 0; } while (--count); -} - -static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - const unsigned int avail = 64 - offset; - unsigned int bytes; - const u8 *src = data; - - if (avail > len) { - sctx->count += len; - memcpy((char *)sctx->buf + offset, src, len); - return 0; - } - - sctx->count += len; - - if (offset) { - memcpy((char *)sctx->buf + offset, src, avail); - - spe_begin(); - ppc_spe_sha256_transform(sctx->state, (const u8 *)sctx->buf, 1); - spe_end(); - - len -= avail; - src += avail; - } - - while (len > 63) { - /* cut input data into smaller blocks */ - bytes = (len > MAX_BYTES) ? MAX_BYTES : len; - bytes = bytes & ~0x3f; - - spe_begin(); - ppc_spe_sha256_transform(sctx->state, src, bytes >> 6); - spe_end(); - - src += bytes; - len -= bytes; - } - - memcpy((char *)sctx->buf, src, len); - return 0; -} - -static int ppc_spe_sha256_final(struct shash_desc *desc, u8 *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - const unsigned int offset = sctx->count & 0x3f; - char *p = (char *)sctx->buf + offset; - int padlen; - __be64 *pbits = (__be64 *)(((char *)&sctx->buf) + 56); - __be32 *dst = (__be32 *)out; - - padlen = 55 - offset; - *p++ = 0x80; - - spe_begin(); - - if (padlen < 0) { - memset(p, 0x00, padlen + sizeof (u64)); - ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); - p = (char *)sctx->buf; - padlen = 56; - } - - memset(p, 0, padlen); - *pbits = cpu_to_be64(sctx->count << 3); - ppc_spe_sha256_transform(sctx->state, sctx->buf, 1); - - spe_end(); - - dst[0] = cpu_to_be32(sctx->state[0]); - dst[1] = cpu_to_be32(sctx->state[1]); - dst[2] = cpu_to_be32(sctx->state[2]); - dst[3] = cpu_to_be32(sctx->state[3]); - dst[4] = cpu_to_be32(sctx->state[4]); - dst[5] = cpu_to_be32(sctx->state[5]); - dst[6] = cpu_to_be32(sctx->state[6]); - dst[7] = cpu_to_be32(sctx->state[7]); - - ppc_sha256_clear_context(sctx); - return 0; -} - -static int ppc_spe_sha224_final(struct shash_desc *desc, u8 *out) -{ - __be32 D[SHA256_DIGEST_SIZE >> 2]; - __be32 *dst = (__be32 *)out; - - ppc_spe_sha256_final(desc, (u8 *)D); - - /* avoid bytewise memcpy */ - dst[0] = D[0]; - dst[1] = D[1]; - dst[2] = D[2]; - dst[3] = D[3]; - dst[4] = D[4]; - dst[5] = D[5]; - dst[6] = D[6]; - - /* clear sensitive data */ - memzero_explicit(D, SHA256_DIGEST_SIZE); - return 0; -} - -static int ppc_spe_sha256_export(struct shash_desc *desc, void *out) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(out, sctx, sizeof(*sctx)); - return 0; -} - -static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - memcpy(sctx, in, sizeof(*sctx)); - return 0; -} - -static struct shash_alg algs[2] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = ppc_spe_sha256_update, - .final = ppc_spe_sha256_final, - .export = ppc_spe_sha256_export, - .import = ppc_spe_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "sha256-ppc-spe", - .cra_priority = 300, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = ppc_spe_sha256_update, - .final = ppc_spe_sha224_final, - .export = ppc_spe_sha256_export, - .import = ppc_spe_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "sha224-ppc-spe", - .cra_priority = 300, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int __init ppc_spe_sha256_mod_init(void) -{ - return crypto_register_shashes(algs, ARRAY_SIZE(algs)); -} - -static void __exit ppc_spe_sha256_mod_fini(void) -{ - crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); -} - -module_init(ppc_spe_sha256_mod_init); -module_exit(ppc_spe_sha256_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, SPE optimized"); - -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha224-ppc-spe"); -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha256-ppc-spe"); diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 42c3af90d1f0..92d21c6faf1e 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -365,7 +365,7 @@ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) >> 3 }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val << 3 }) -static inline int pte_swp_exclusive(pte_t pte) +static inline bool pte_swp_exclusive(pte_t pte) { return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; } diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index c3efacab4b94..7132392fa7cd 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -77,9 +77,17 @@ /* * With 4K page size the real_pte machinery is all nops. */ -#define __real_pte(e, p, o) ((real_pte_t){(e)}) +static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset) +{ + return (real_pte_t){pte}; +} + #define __rpte_to_pte(r) ((r).pte) -#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT) + +static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) +{ + return pte_val(__rpte_to_pte(rpte)) >> H_PAGE_F_GIX_SHIFT; +} #define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \ do { \ @@ -160,12 +168,6 @@ extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, extern int hash__has_transparent_hugepage(void); #endif -static inline pmd_t hash__pmd_mkdevmap(pmd_t pmd) -{ - BUG(); - return pmd; -} - #endif /* !__ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_HASH_4K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index 0bf6fd0bf42a..0fb5b7da9478 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -259,7 +259,7 @@ static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, */ static inline int hash__pmd_trans_huge(pmd_t pmd) { - return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE | _PAGE_DEVMAP)) == + return !!((pmd_val(pmd) & (_PAGE_PTE | H_PAGE_THP_HUGE)) == (_PAGE_PTE | H_PAGE_THP_HUGE)); } @@ -281,11 +281,6 @@ extern pmd_t hash__pmdp_huge_get_and_clear(struct mm_struct *mm, extern int hash__has_transparent_hugepage(void); #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline pmd_t hash__pmd_mkdevmap(pmd_t pmd) -{ - return __pmd(pmd_val(pmd) | (_PAGE_PTE | H_PAGE_THP_HUGE | _PAGE_DEVMAP)); -} - #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_BOOK3S_64_HASH_64K_H */ diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb.h b/arch/powerpc/include/asm/book3s/64/hugetlb.h index f0bba9c5f9c3..bb786694dd26 100644 --- a/arch/powerpc/include/asm/book3s/64/hugetlb.h +++ b/arch/powerpc/include/asm/book3s/64/hugetlb.h @@ -94,4 +94,10 @@ static inline int check_and_get_huge_psize(int shift) return mmu_psize; } +#define arch_has_huge_bootmem_alloc arch_has_huge_bootmem_alloc + +static inline bool arch_has_huge_bootmem_alloc(void) +{ + return (firmware_has_feature(FW_FEATURE_LPAR) && !radix_enabled()); +} #endif diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 6d98e6f08d4d..c19800365315 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -88,7 +88,6 @@ #define _PAGE_SOFT_DIRTY _RPAGE_SW3 /* software: software dirty tracking */ #define _PAGE_SPECIAL _RPAGE_SW2 /* software: special page */ -#define _PAGE_DEVMAP _RPAGE_SW1 /* software: ZONE_DEVICE page */ /* * Drivers request for cache inhibited pte mapping using _PAGE_NO_CACHE @@ -109,7 +108,7 @@ */ #define _HPAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ _PAGE_ACCESSED | H_PAGE_THP_HUGE | _PAGE_PTE | \ - _PAGE_SOFT_DIRTY | _PAGE_DEVMAP) + _PAGE_SOFT_DIRTY) /* * user access blocked by key */ @@ -123,7 +122,7 @@ */ #define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \ - _PAGE_SOFT_DIRTY | _PAGE_DEVMAP) + _PAGE_SOFT_DIRTY) /* * We define 2 sets of base prot bits, one for basic pages (ie, @@ -609,24 +608,6 @@ static inline pte_t pte_mkhuge(pte_t pte) return pte; } -static inline pte_t pte_mkdevmap(pte_t pte) -{ - return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SPECIAL | _PAGE_DEVMAP)); -} - -/* - * This is potentially called with a pmd as the argument, in which case it's not - * safe to check _PAGE_DEVMAP unless we also confirm that _PAGE_PTE is set. - * That's because the bit we use for _PAGE_DEVMAP is not reserved for software - * use in page directory entries (ie. non-ptes). - */ -static inline int pte_devmap(pte_t pte) -{ - __be64 mask = cpu_to_be64(_PAGE_DEVMAP | _PAGE_PTE); - - return (pte_raw(pte) & mask) == mask; -} - static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { /* FIXME!! check whether this need to be a conditional */ @@ -693,7 +674,7 @@ static inline pte_t pte_swp_mkexclusive(pte_t pte) return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_SWP_EXCLUSIVE)); } -static inline int pte_swp_exclusive(pte_t pte) +static inline bool pte_swp_exclusive(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_SWP_EXCLUSIVE)); } @@ -1096,7 +1077,6 @@ static inline bool pmd_access_permitted(pmd_t pmd, bool write) #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot); extern pud_t pfn_pud(unsigned long pfn, pgprot_t pgprot); -extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot); extern pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot); extern pud_t pud_modify(pud_t pud, pgprot_t newprot); extern void set_pmd_at(struct mm_struct *mm, unsigned long addr, @@ -1380,36 +1360,6 @@ static inline bool arch_needs_pgtable_deposit(void) } extern void serialize_against_pte_lookup(struct mm_struct *mm); - -static inline pmd_t pmd_mkdevmap(pmd_t pmd) -{ - if (radix_enabled()) - return radix__pmd_mkdevmap(pmd); - return hash__pmd_mkdevmap(pmd); -} - -static inline pud_t pud_mkdevmap(pud_t pud) -{ - if (radix_enabled()) - return radix__pud_mkdevmap(pud); - BUG(); - return pud; -} - -static inline int pmd_devmap(pmd_t pmd) -{ - return pte_devmap(pmd_pte(pmd)); -} - -static inline int pud_devmap(pud_t pud) -{ - return pte_devmap(pud_pte(pud)); -} - -static inline int pgd_devmap(pgd_t pgd) -{ - return 0; -} #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION diff --git a/arch/powerpc/include/asm/book3s/64/pkeys.h b/arch/powerpc/include/asm/book3s/64/pkeys.h index 5b178139f3c0..ff911b4251d9 100644 --- a/arch/powerpc/include/asm/book3s/64/pkeys.h +++ b/arch/powerpc/include/asm/book3s/64/pkeys.h @@ -5,7 +5,7 @@ #include <asm/book3s/64/hash-pkey.h> -static inline u64 vmflag_to_pte_pkey_bits(u64 vm_flags) +static inline u64 vmflag_to_pte_pkey_bits(vm_flags_t vm_flags) { if (!mmu_has_feature(MMU_FTR_PKEY)) return 0x0UL; diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h index 8f55ff74bb68..df23a8267e4d 100644 --- a/arch/powerpc/include/asm/book3s/64/radix.h +++ b/arch/powerpc/include/asm/book3s/64/radix.h @@ -264,7 +264,7 @@ static inline int radix__p4d_bad(p4d_t p4d) static inline int radix__pmd_trans_huge(pmd_t pmd) { - return (pmd_val(pmd) & (_PAGE_PTE | _PAGE_DEVMAP)) == _PAGE_PTE; + return (pmd_val(pmd) & _PAGE_PTE) == _PAGE_PTE; } static inline pmd_t radix__pmd_mkhuge(pmd_t pmd) @@ -274,7 +274,7 @@ static inline pmd_t radix__pmd_mkhuge(pmd_t pmd) static inline int radix__pud_trans_huge(pud_t pud) { - return (pud_val(pud) & (_PAGE_PTE | _PAGE_DEVMAP)) == _PAGE_PTE; + return (pud_val(pud) & _PAGE_PTE) == _PAGE_PTE; } static inline pud_t radix__pud_mkhuge(pud_t pud) @@ -315,16 +315,6 @@ static inline int radix__has_transparent_pud_hugepage(void) } #endif -static inline pmd_t radix__pmd_mkdevmap(pmd_t pmd) -{ - return __pmd(pmd_val(pmd) | (_PAGE_PTE | _PAGE_DEVMAP)); -} - -static inline pud_t radix__pud_mkdevmap(pud_t pud) -{ - return __pud(pud_val(pud) | (_PAGE_PTE | _PAGE_DEVMAP)); -} - struct vmem_altmap; struct dev_pagemap; extern int __meminit radix__vmemmap_create_mapping(unsigned long start, diff --git a/arch/powerpc/include/asm/cell-pmu.h b/arch/powerpc/include/asm/cell-pmu.h index 6a79b5d1c44f..7fbefd64b4fb 100644 --- a/arch/powerpc/include/asm/cell-pmu.h +++ b/arch/powerpc/include/asm/cell-pmu.h @@ -20,36 +20,9 @@ /* Macros for the pm_control register. */ #define CBE_PM_16BIT_CTR(ctr) (1 << (24 - ((ctr) & (NR_PHYS_CTRS - 1)))) -#define CBE_PM_ENABLE_PERF_MON 0x80000000 -#define CBE_PM_STOP_AT_MAX 0x40000000 -#define CBE_PM_TRACE_MODE_GET(pm_control) (((pm_control) >> 28) & 0x3) -#define CBE_PM_TRACE_MODE_SET(mode) (((mode) & 0x3) << 28) -#define CBE_PM_TRACE_BUF_OVFLW(bit) (((bit) & 0x1) << 17) -#define CBE_PM_COUNT_MODE_SET(count) (((count) & 0x3) << 18) -#define CBE_PM_FREEZE_ALL_CTRS 0x00100000 -#define CBE_PM_ENABLE_EXT_TRACE 0x00008000 -#define CBE_PM_SPU_ADDR_TRACE_SET(msk) (((msk) & 0x3) << 9) /* Macros for the trace_address register. */ -#define CBE_PM_TRACE_BUF_FULL 0x00000800 #define CBE_PM_TRACE_BUF_EMPTY 0x00000400 -#define CBE_PM_TRACE_BUF_DATA_COUNT(ta) ((ta) & 0x3ff) -#define CBE_PM_TRACE_BUF_MAX_COUNT 0x400 - -/* Macros for the pm07_control registers. */ -#define CBE_PM_CTR_INPUT_MUX(pm07_control) (((pm07_control) >> 26) & 0x3f) -#define CBE_PM_CTR_INPUT_CONTROL 0x02000000 -#define CBE_PM_CTR_POLARITY 0x01000000 -#define CBE_PM_CTR_COUNT_CYCLES 0x00800000 -#define CBE_PM_CTR_ENABLE 0x00400000 -#define PM07_CTR_INPUT_MUX(x) (((x) & 0x3F) << 26) -#define PM07_CTR_INPUT_CONTROL(x) (((x) & 1) << 25) -#define PM07_CTR_POLARITY(x) (((x) & 1) << 24) -#define PM07_CTR_COUNT_CYCLES(x) (((x) & 1) << 23) -#define PM07_CTR_ENABLE(x) (((x) & 1) << 22) - -/* Macros for the pm_status register. */ -#define CBE_PM_CTR_OVERFLOW_INTR(ctr) (1 << (31 - ((ctr) & 7))) enum pm_reg_name { group_control, @@ -62,33 +35,4 @@ enum pm_reg_name { pm_start_stop, }; -/* Routines for reading/writing the PMU registers. */ -extern u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr); -extern void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val); -extern u32 cbe_read_ctr(u32 cpu, u32 ctr); -extern void cbe_write_ctr(u32 cpu, u32 ctr, u32 val); - -extern u32 cbe_read_pm07_control(u32 cpu, u32 ctr); -extern void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val); -extern u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg); -extern void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val); - -extern u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr); -extern void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size); - -extern void cbe_enable_pm(u32 cpu); -extern void cbe_disable_pm(u32 cpu); - -extern void cbe_read_trace_buffer(u32 cpu, u64 *buf); - -extern void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask); -extern void cbe_disable_pm_interrupts(u32 cpu); -extern u32 cbe_get_and_clear_pm_interrupts(u32 cpu); -extern void cbe_sync_irq(int node); - -#define CBE_COUNT_SUPERVISOR_MODE 0 -#define CBE_COUNT_HYPERVISOR_MODE 1 -#define CBE_COUNT_PROBLEM_MODE 2 -#define CBE_COUNT_ALL_MODES 3 - #endif /* __ASM_CELL_PMU_H__ */ diff --git a/arch/powerpc/include/asm/cell-regs.h b/arch/powerpc/include/asm/cell-regs.h index e1c431ef30e0..20f7339a3d4a 100644 --- a/arch/powerpc/include/asm/cell-regs.h +++ b/arch/powerpc/include/asm/cell-regs.h @@ -18,293 +18,6 @@ #include <asm/cell-pmu.h> -/* - * - * Some HID register definitions - * - */ - -/* CBE specific HID0 bits */ -#define HID0_CBE_THERM_WAKEUP 0x0000020000000000ul -#define HID0_CBE_SYSERR_WAKEUP 0x0000008000000000ul -#define HID0_CBE_THERM_INT_EN 0x0000000400000000ul -#define HID0_CBE_SYSERR_INT_EN 0x0000000200000000ul - -#define MAX_CBE 2 - -/* - * - * Pervasive unit register definitions - * - */ - -union spe_reg { - u64 val; - u8 spe[8]; -}; - -union ppe_spe_reg { - u64 val; - struct { - u32 ppe; - u32 spe; - }; -}; - - -struct cbe_pmd_regs { - /* Debug Bus Control */ - u64 pad_0x0000; /* 0x0000 */ - - u64 group_control; /* 0x0008 */ - - u8 pad_0x0010_0x00a8 [0x00a8 - 0x0010]; /* 0x0010 */ - - u64 debug_bus_control; /* 0x00a8 */ - - u8 pad_0x00b0_0x0100 [0x0100 - 0x00b0]; /* 0x00b0 */ - - u64 trace_aux_data; /* 0x0100 */ - u64 trace_buffer_0_63; /* 0x0108 */ - u64 trace_buffer_64_127; /* 0x0110 */ - u64 trace_address; /* 0x0118 */ - u64 ext_tr_timer; /* 0x0120 */ - - u8 pad_0x0128_0x0400 [0x0400 - 0x0128]; /* 0x0128 */ - - /* Performance Monitor */ - u64 pm_status; /* 0x0400 */ - u64 pm_control; /* 0x0408 */ - u64 pm_interval; /* 0x0410 */ - u64 pm_ctr[4]; /* 0x0418 */ - u64 pm_start_stop; /* 0x0438 */ - u64 pm07_control[8]; /* 0x0440 */ - - u8 pad_0x0480_0x0800 [0x0800 - 0x0480]; /* 0x0480 */ - - /* Thermal Sensor Registers */ - union spe_reg ts_ctsr1; /* 0x0800 */ - u64 ts_ctsr2; /* 0x0808 */ - union spe_reg ts_mtsr1; /* 0x0810 */ - u64 ts_mtsr2; /* 0x0818 */ - union spe_reg ts_itr1; /* 0x0820 */ - u64 ts_itr2; /* 0x0828 */ - u64 ts_gitr; /* 0x0830 */ - u64 ts_isr; /* 0x0838 */ - u64 ts_imr; /* 0x0840 */ - union spe_reg tm_cr1; /* 0x0848 */ - u64 tm_cr2; /* 0x0850 */ - u64 tm_simr; /* 0x0858 */ - union ppe_spe_reg tm_tpr; /* 0x0860 */ - union spe_reg tm_str1; /* 0x0868 */ - u64 tm_str2; /* 0x0870 */ - union ppe_spe_reg tm_tsr; /* 0x0878 */ - - /* Power Management */ - u64 pmcr; /* 0x0880 */ -#define CBE_PMD_PAUSE_ZERO_CONTROL 0x10000 - u64 pmsr; /* 0x0888 */ - - /* Time Base Register */ - u64 tbr; /* 0x0890 */ - - u8 pad_0x0898_0x0c00 [0x0c00 - 0x0898]; /* 0x0898 */ - - /* Fault Isolation Registers */ - u64 checkstop_fir; /* 0x0c00 */ - u64 recoverable_fir; /* 0x0c08 */ - u64 spec_att_mchk_fir; /* 0x0c10 */ - u32 fir_mode_reg; /* 0x0c18 */ - u8 pad_0x0c1c_0x0c20 [4]; /* 0x0c1c */ -#define CBE_PMD_FIR_MODE_M8 0x00800 - u64 fir_enable_mask; /* 0x0c20 */ - - u8 pad_0x0c28_0x0ca8 [0x0ca8 - 0x0c28]; /* 0x0c28 */ - u64 ras_esc_0; /* 0x0ca8 */ - u8 pad_0x0cb0_0x1000 [0x1000 - 0x0cb0]; /* 0x0cb0 */ -}; - -extern struct cbe_pmd_regs __iomem *cbe_get_pmd_regs(struct device_node *np); -extern struct cbe_pmd_regs __iomem *cbe_get_cpu_pmd_regs(int cpu); - -/* - * PMU shadow registers - * - * Many of the registers in the performance monitoring unit are write-only, - * so we need to save a copy of what we write to those registers. - * - * The actual data counters are read/write. However, writing to the counters - * only takes effect if the PMU is enabled. Otherwise the value is stored in - * a hardware latch until the next time the PMU is enabled. So we save a copy - * of the counter values if we need to read them back while the PMU is - * disabled. The counter_value_in_latch field is a bitmap indicating which - * counters currently have a value waiting to be written. - */ - -struct cbe_pmd_shadow_regs { - u32 group_control; - u32 debug_bus_control; - u32 trace_address; - u32 ext_tr_timer; - u32 pm_status; - u32 pm_control; - u32 pm_interval; - u32 pm_start_stop; - u32 pm07_control[NR_CTRS]; - - u32 pm_ctr[NR_PHYS_CTRS]; - u32 counter_value_in_latch; -}; - -extern struct cbe_pmd_shadow_regs *cbe_get_pmd_shadow_regs(struct device_node *np); -extern struct cbe_pmd_shadow_regs *cbe_get_cpu_pmd_shadow_regs(int cpu); - -/* - * - * IIC unit register definitions - * - */ - -struct cbe_iic_pending_bits { - u32 data; - u8 flags; - u8 class; - u8 source; - u8 prio; -}; - -#define CBE_IIC_IRQ_VALID 0x80 -#define CBE_IIC_IRQ_IPI 0x40 - -struct cbe_iic_thread_regs { - struct cbe_iic_pending_bits pending; - struct cbe_iic_pending_bits pending_destr; - u64 generate; - u64 prio; -}; - -struct cbe_iic_regs { - u8 pad_0x0000_0x0400[0x0400 - 0x0000]; /* 0x0000 */ - - /* IIC interrupt registers */ - struct cbe_iic_thread_regs thread[2]; /* 0x0400 */ - - u64 iic_ir; /* 0x0440 */ -#define CBE_IIC_IR_PRIO(x) (((x) & 0xf) << 12) -#define CBE_IIC_IR_DEST_NODE(x) (((x) & 0xf) << 4) -#define CBE_IIC_IR_DEST_UNIT(x) ((x) & 0xf) -#define CBE_IIC_IR_IOC_0 0x0 -#define CBE_IIC_IR_IOC_1S 0xb -#define CBE_IIC_IR_PT_0 0xe -#define CBE_IIC_IR_PT_1 0xf - - u64 iic_is; /* 0x0448 */ -#define CBE_IIC_IS_PMI 0x2 - - u8 pad_0x0450_0x0500[0x0500 - 0x0450]; /* 0x0450 */ - - /* IOC FIR */ - u64 ioc_fir_reset; /* 0x0500 */ - u64 ioc_fir_set; /* 0x0508 */ - u64 ioc_checkstop_enable; /* 0x0510 */ - u64 ioc_fir_error_mask; /* 0x0518 */ - u64 ioc_syserr_enable; /* 0x0520 */ - u64 ioc_fir; /* 0x0528 */ - - u8 pad_0x0530_0x1000[0x1000 - 0x0530]; /* 0x0530 */ -}; - -extern struct cbe_iic_regs __iomem *cbe_get_iic_regs(struct device_node *np); -extern struct cbe_iic_regs __iomem *cbe_get_cpu_iic_regs(int cpu); - - -struct cbe_mic_tm_regs { - u8 pad_0x0000_0x0040[0x0040 - 0x0000]; /* 0x0000 */ - - u64 mic_ctl_cnfg2; /* 0x0040 */ -#define CBE_MIC_ENABLE_AUX_TRC 0x8000000000000000LL -#define CBE_MIC_DISABLE_PWR_SAV_2 0x0200000000000000LL -#define CBE_MIC_DISABLE_AUX_TRC_WRAP 0x0100000000000000LL -#define CBE_MIC_ENABLE_AUX_TRC_INT 0x0080000000000000LL - - u64 pad_0x0048; /* 0x0048 */ - - u64 mic_aux_trc_base; /* 0x0050 */ - u64 mic_aux_trc_max_addr; /* 0x0058 */ - u64 mic_aux_trc_cur_addr; /* 0x0060 */ - u64 mic_aux_trc_grf_addr; /* 0x0068 */ - u64 mic_aux_trc_grf_data; /* 0x0070 */ - - u64 pad_0x0078; /* 0x0078 */ - - u64 mic_ctl_cnfg_0; /* 0x0080 */ -#define CBE_MIC_DISABLE_PWR_SAV_0 0x8000000000000000LL - - u64 pad_0x0088; /* 0x0088 */ - - u64 slow_fast_timer_0; /* 0x0090 */ - u64 slow_next_timer_0; /* 0x0098 */ - - u8 pad_0x00a0_0x00f8[0x00f8 - 0x00a0]; /* 0x00a0 */ - u64 mic_df_ecc_address_0; /* 0x00f8 */ - - u8 pad_0x0100_0x01b8[0x01b8 - 0x0100]; /* 0x0100 */ - u64 mic_df_ecc_address_1; /* 0x01b8 */ - - u64 mic_ctl_cnfg_1; /* 0x01c0 */ -#define CBE_MIC_DISABLE_PWR_SAV_1 0x8000000000000000LL - - u64 pad_0x01c8; /* 0x01c8 */ - - u64 slow_fast_timer_1; /* 0x01d0 */ - u64 slow_next_timer_1; /* 0x01d8 */ - - u8 pad_0x01e0_0x0208[0x0208 - 0x01e0]; /* 0x01e0 */ - u64 mic_exc; /* 0x0208 */ -#define CBE_MIC_EXC_BLOCK_SCRUB 0x0800000000000000ULL -#define CBE_MIC_EXC_FAST_SCRUB 0x0100000000000000ULL - - u64 mic_mnt_cfg; /* 0x0210 */ -#define CBE_MIC_MNT_CFG_CHAN_0_POP 0x0002000000000000ULL -#define CBE_MIC_MNT_CFG_CHAN_1_POP 0x0004000000000000ULL - - u64 mic_df_config; /* 0x0218 */ -#define CBE_MIC_ECC_DISABLE_0 0x4000000000000000ULL -#define CBE_MIC_ECC_REP_SINGLE_0 0x2000000000000000ULL -#define CBE_MIC_ECC_DISABLE_1 0x0080000000000000ULL -#define CBE_MIC_ECC_REP_SINGLE_1 0x0040000000000000ULL - - u8 pad_0x0220_0x0230[0x0230 - 0x0220]; /* 0x0220 */ - u64 mic_fir; /* 0x0230 */ -#define CBE_MIC_FIR_ECC_SINGLE_0_ERR 0x0200000000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_0_ERR 0x0100000000000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_ERR 0x0080000000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_1_ERR 0x0040000000000000ULL -#define CBE_MIC_FIR_ECC_ERR_MASK 0xffff000000000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_0_CTE 0x0000020000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_0_CTE 0x0000010000000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_CTE 0x0000008000000000ULL -#define CBE_MIC_FIR_ECC_MULTI_1_CTE 0x0000004000000000ULL -#define CBE_MIC_FIR_ECC_CTE_MASK 0x0000ffff00000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_0_RESET 0x0000000002000000ULL -#define CBE_MIC_FIR_ECC_MULTI_0_RESET 0x0000000001000000ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_RESET 0x0000000000800000ULL -#define CBE_MIC_FIR_ECC_MULTI_1_RESET 0x0000000000400000ULL -#define CBE_MIC_FIR_ECC_RESET_MASK 0x00000000ffff0000ULL -#define CBE_MIC_FIR_ECC_SINGLE_0_SET 0x0000000000000200ULL -#define CBE_MIC_FIR_ECC_MULTI_0_SET 0x0000000000000100ULL -#define CBE_MIC_FIR_ECC_SINGLE_1_SET 0x0000000000000080ULL -#define CBE_MIC_FIR_ECC_MULTI_1_SET 0x0000000000000040ULL -#define CBE_MIC_FIR_ECC_SET_MASK 0x000000000000ffffULL - u64 mic_fir_debug; /* 0x0238 */ - - u8 pad_0x0240_0x1000[0x1000 - 0x0240]; /* 0x0240 */ -}; - -extern struct cbe_mic_tm_regs __iomem *cbe_get_mic_tm_regs(struct device_node *np); -extern struct cbe_mic_tm_regs __iomem *cbe_get_cpu_mic_tm_regs(int cpu); - - /* Cell page table entries */ #define CBE_IOPTE_PP_W 0x8000000000000000ul /* protection: write */ #define CBE_IOPTE_PP_R 0x4000000000000000ul /* protection: read */ @@ -315,13 +28,4 @@ extern struct cbe_mic_tm_regs __iomem *cbe_get_cpu_mic_tm_regs(int cpu); #define CBE_IOPTE_H 0x0000000000000800ul /* cache hint */ #define CBE_IOPTE_IOID_Mask 0x00000000000007fful /* ioid */ -/* some utility functions to deal with SMT */ -extern u32 cbe_get_hw_thread_id(int cpu); -extern u32 cbe_cpu_to_node(int cpu); -extern u32 cbe_node_to_cpu(int node); - -/* Init this module early */ -extern void cbe_regs_init(void); - - #endif /* CBE_REGS_H */ diff --git a/arch/powerpc/include/asm/copro.h b/arch/powerpc/include/asm/copro.h index fd2e166ea02a..81bd176203ab 100644 --- a/arch/powerpc/include/asm/copro.h +++ b/arch/powerpc/include/asm/copro.h @@ -18,10 +18,4 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea, int copro_calculate_slb(struct mm_struct *mm, u64 ea, struct copro_slb *slb); - -#ifdef CONFIG_PPC_COPRO_BASE -void copro_flush_all_slbs(struct mm_struct *mm); -#else -static inline void copro_flush_all_slbs(struct mm_struct *mm) {} -#endif #endif /* _ASM_POWERPC_COPRO_H */ diff --git a/arch/powerpc/include/asm/crash_reserve.h b/arch/powerpc/include/asm/crash_reserve.h new file mode 100644 index 000000000000..6467ce29b1fa --- /dev/null +++ b/arch/powerpc/include/asm/crash_reserve.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_CRASH_RESERVE_H +#define _ASM_POWERPC_CRASH_RESERVE_H + +/* crash kernel regions are Page size agliged */ +#define CRASH_ALIGN PAGE_SIZE + +#endif /* _ASM_POWERPC_CRASH_RESERVE_H */ diff --git a/arch/powerpc/include/asm/dcr-generic.h b/arch/powerpc/include/asm/dcr-generic.h deleted file mode 100644 index 099c28dd40b9..000000000000 --- a/arch/powerpc/include/asm/dcr-generic.h +++ /dev/null @@ -1,36 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * (c) Copyright 2006 Benjamin Herrenschmidt, IBM Corp. - * <benh@kernel.crashing.org> - */ - -#ifndef _ASM_POWERPC_DCR_GENERIC_H -#define _ASM_POWERPC_DCR_GENERIC_H -#ifdef __KERNEL__ -#ifndef __ASSEMBLY__ - -enum host_type_t {DCR_HOST_MMIO, DCR_HOST_NATIVE, DCR_HOST_INVALID}; - -typedef struct { - enum host_type_t type; - union { - dcr_host_mmio_t mmio; - dcr_host_native_t native; - } host; -} dcr_host_t; - -extern bool dcr_map_ok_generic(dcr_host_t host); - -extern dcr_host_t dcr_map_generic(struct device_node *dev, unsigned int dcr_n, - unsigned int dcr_c); -extern void dcr_unmap_generic(dcr_host_t host, unsigned int dcr_c); - -extern u32 dcr_read_generic(dcr_host_t host, unsigned int dcr_n); - -extern void dcr_write_generic(dcr_host_t host, unsigned int dcr_n, u32 value); - -#endif /* __ASSEMBLY__ */ -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_DCR_GENERIC_H */ - - diff --git a/arch/powerpc/include/asm/dcr-mmio.h b/arch/powerpc/include/asm/dcr-mmio.h deleted file mode 100644 index fc6d93ef4a13..000000000000 --- a/arch/powerpc/include/asm/dcr-mmio.h +++ /dev/null @@ -1,44 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * (c) Copyright 2006 Benjamin Herrenschmidt, IBM Corp. - * <benh@kernel.crashing.org> - */ - -#ifndef _ASM_POWERPC_DCR_MMIO_H -#define _ASM_POWERPC_DCR_MMIO_H -#ifdef __KERNEL__ - -#include <asm/io.h> - -typedef struct { - void __iomem *token; - unsigned int stride; - unsigned int base; -} dcr_host_mmio_t; - -static inline bool dcr_map_ok_mmio(dcr_host_mmio_t host) -{ - return host.token != NULL; -} - -extern dcr_host_mmio_t dcr_map_mmio(struct device_node *dev, - unsigned int dcr_n, - unsigned int dcr_c); -extern void dcr_unmap_mmio(dcr_host_mmio_t host, unsigned int dcr_c); - -static inline u32 dcr_read_mmio(dcr_host_mmio_t host, unsigned int dcr_n) -{ - return in_be32(host.token + ((host.base + dcr_n) * host.stride)); -} - -static inline void dcr_write_mmio(dcr_host_mmio_t host, - unsigned int dcr_n, - u32 value) -{ - out_be32(host.token + ((host.base + dcr_n) * host.stride), value); -} - -#endif /* __KERNEL__ */ -#endif /* _ASM_POWERPC_DCR_MMIO_H */ - - diff --git a/arch/powerpc/include/asm/dcr.h b/arch/powerpc/include/asm/dcr.h index 64030e3a1f30..180021cd0b30 100644 --- a/arch/powerpc/include/asm/dcr.h +++ b/arch/powerpc/include/asm/dcr.h @@ -10,46 +10,14 @@ #ifndef __ASSEMBLY__ #ifdef CONFIG_PPC_DCR -#ifdef CONFIG_PPC_DCR_NATIVE #include <asm/dcr-native.h> -#endif -#ifdef CONFIG_PPC_DCR_MMIO -#include <asm/dcr-mmio.h> -#endif - - -/* Indirection layer for providing both NATIVE and MMIO support. */ - -#if defined(CONFIG_PPC_DCR_NATIVE) && defined(CONFIG_PPC_DCR_MMIO) - -#include <asm/dcr-generic.h> - -#define DCR_MAP_OK(host) dcr_map_ok_generic(host) -#define dcr_map(dev, dcr_n, dcr_c) dcr_map_generic(dev, dcr_n, dcr_c) -#define dcr_unmap(host, dcr_c) dcr_unmap_generic(host, dcr_c) -#define dcr_read(host, dcr_n) dcr_read_generic(host, dcr_n) -#define dcr_write(host, dcr_n, value) dcr_write_generic(host, dcr_n, value) - -#else - -#ifdef CONFIG_PPC_DCR_NATIVE typedef dcr_host_native_t dcr_host_t; #define DCR_MAP_OK(host) dcr_map_ok_native(host) #define dcr_map(dev, dcr_n, dcr_c) dcr_map_native(dev, dcr_n, dcr_c) #define dcr_unmap(host, dcr_c) dcr_unmap_native(host, dcr_c) #define dcr_read(host, dcr_n) dcr_read_native(host, dcr_n) #define dcr_write(host, dcr_n, value) dcr_write_native(host, dcr_n, value) -#else -typedef dcr_host_mmio_t dcr_host_t; -#define DCR_MAP_OK(host) dcr_map_ok_mmio(host) -#define dcr_map(dev, dcr_n, dcr_c) dcr_map_mmio(dev, dcr_n, dcr_c) -#define dcr_unmap(host, dcr_c) dcr_unmap_mmio(host, dcr_c) -#define dcr_read(host, dcr_n) dcr_read_mmio(host, dcr_n) -#define dcr_write(host, dcr_n, value) dcr_write_mmio(host, dcr_n, value) -#endif - -#endif /* defined(CONFIG_PPC_DCR_NATIVE) && defined(CONFIG_PPC_DCR_MMIO) */ /* * additional helpers to read the DCR * base from the device-tree diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h index 47ed639f3b8f..a4dc27655b3e 100644 --- a/arch/powerpc/include/asm/device.h +++ b/arch/powerpc/include/asm/device.h @@ -38,9 +38,6 @@ struct dev_archdata { #ifdef CONFIG_FAIL_IOMMU int fail_iommu; #endif -#ifdef CONFIG_CXL_BASE - struct cxl_context *cxl_ctx; -#endif #ifdef CONFIG_PCI_IOV void *iov_data; #endif diff --git a/arch/powerpc/include/asm/floppy.h b/arch/powerpc/include/asm/floppy.h index f8ce178b43b7..34abf8bea2cc 100644 --- a/arch/powerpc/include/asm/floppy.h +++ b/arch/powerpc/include/asm/floppy.h @@ -144,9 +144,12 @@ static int hard_dma_setup(char *addr, unsigned long size, int mode, int io) bus_addr = 0; } - if (!bus_addr) /* need to map it */ + if (!bus_addr) { /* need to map it */ bus_addr = dma_map_single(&isa_bridge_pcidev->dev, addr, size, dir); + if (dma_mapping_error(&isa_bridge_pcidev->dev, bus_addr)) + return -ENOMEM; + } /* remember this one as prev */ prev_addr = addr; diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h index db481b336bca..82da7c7a1d12 100644 --- a/arch/powerpc/include/asm/ftrace.h +++ b/arch/powerpc/include/asm/ftrace.h @@ -43,6 +43,13 @@ static __always_inline struct pt_regs *arch_ftrace_get_regs(struct ftrace_regs * return arch_ftrace_regs(fregs)->regs.msr ? &arch_ftrace_regs(fregs)->regs : NULL; } +#define arch_ftrace_fill_perf_regs(fregs, _regs) do { \ + (_regs)->result = 0; \ + (_regs)->nip = arch_ftrace_regs(fregs)->regs.nip; \ + (_regs)->gpr[1] = arch_ftrace_regs(fregs)->regs.gpr[1]; \ + asm volatile("mfmsr %0" : "=r" ((_regs)->msr)); \ + } while (0) + static __always_inline void ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs, unsigned long ip) @@ -50,6 +57,12 @@ ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs, regs_set_return_ip(&arch_ftrace_regs(fregs)->regs, ip); } +static __always_inline unsigned long +ftrace_regs_get_return_address(struct ftrace_regs *fregs) +{ + return arch_ftrace_regs(fregs)->regs.link; +} + struct ftrace_ops; #define ftrace_graph_func ftrace_graph_func diff --git a/arch/powerpc/include/asm/guest-state-buffer.h b/arch/powerpc/include/asm/guest-state-buffer.h index d107abe1468f..acd61eb36d59 100644 --- a/arch/powerpc/include/asm/guest-state-buffer.h +++ b/arch/powerpc/include/asm/guest-state-buffer.h @@ -28,6 +28,21 @@ /* Process Table Info */ #define KVMPPC_GSID_PROCESS_TABLE 0x0006 +/* Guest Management Heap Size */ +#define KVMPPC_GSID_L0_GUEST_HEAP 0x0800 + +/* Guest Management Heap Max Size */ +#define KVMPPC_GSID_L0_GUEST_HEAP_MAX 0x0801 + +/* Guest Pagetable Size */ +#define KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE 0x0802 + +/* Guest Pagetable Max Size */ +#define KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX 0x0803 + +/* Guest Pagetable Reclaim in bytes */ +#define KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM 0x0804 + /* H_GUEST_RUN_VCPU input buffer Info */ #define KVMPPC_GSID_RUN_INPUT 0x0C00 /* H_GUEST_RUN_VCPU output buffer Info */ @@ -106,6 +121,11 @@ #define KVMPPC_GSE_GUESTWIDE_COUNT \ (KVMPPC_GSE_GUESTWIDE_END - KVMPPC_GSE_GUESTWIDE_START + 1) +#define KVMPPC_GSE_HOSTWIDE_START KVMPPC_GSID_L0_GUEST_HEAP +#define KVMPPC_GSE_HOSTWIDE_END KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM +#define KVMPPC_GSE_HOSTWIDE_COUNT \ + (KVMPPC_GSE_HOSTWIDE_END - KVMPPC_GSE_HOSTWIDE_START + 1) + #define KVMPPC_GSE_META_START KVMPPC_GSID_RUN_INPUT #define KVMPPC_GSE_META_END KVMPPC_GSID_VPA #define KVMPPC_GSE_META_COUNT (KVMPPC_GSE_META_END - KVMPPC_GSE_META_START + 1) @@ -130,7 +150,8 @@ (KVMPPC_GSE_INTR_REGS_END - KVMPPC_GSE_INTR_REGS_START + 1) #define KVMPPC_GSE_IDEN_COUNT \ - (KVMPPC_GSE_GUESTWIDE_COUNT + KVMPPC_GSE_META_COUNT + \ + (KVMPPC_GSE_HOSTWIDE_COUNT + \ + KVMPPC_GSE_GUESTWIDE_COUNT + KVMPPC_GSE_META_COUNT + \ KVMPPC_GSE_DW_REGS_COUNT + KVMPPC_GSE_W_REGS_COUNT + \ KVMPPC_GSE_VSRS_COUNT + KVMPPC_GSE_INTR_REGS_COUNT) @@ -139,10 +160,11 @@ */ enum { KVMPPC_GS_CLASS_GUESTWIDE = 0x01, - KVMPPC_GS_CLASS_META = 0x02, - KVMPPC_GS_CLASS_DWORD_REG = 0x04, - KVMPPC_GS_CLASS_WORD_REG = 0x08, - KVMPPC_GS_CLASS_VECTOR = 0x10, + KVMPPC_GS_CLASS_HOSTWIDE = 0x02, + KVMPPC_GS_CLASS_META = 0x04, + KVMPPC_GS_CLASS_DWORD_REG = 0x08, + KVMPPC_GS_CLASS_WORD_REG = 0x10, + KVMPPC_GS_CLASS_VECTOR = 0x18, KVMPPC_GS_CLASS_INTR = 0x20, }; @@ -164,6 +186,7 @@ enum { */ enum { KVMPPC_GS_FLAGS_WIDE = 0x01, + KVMPPC_GS_FLAGS_HOST_WIDE = 0x02, }; /** @@ -287,7 +310,7 @@ struct kvmppc_gs_msg_ops { * struct kvmppc_gs_msg - a guest state message * @bitmap: the guest state ids that should be included * @ops: modify message behavior for reading and writing to buffers - * @flags: guest wide or thread wide + * @flags: host wide, guest wide or thread wide * @data: location where buffer data will be written to or from. * * A guest state message is allows flexibility in sending in receiving data diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index 18a3028ac3b6..86326587e58d 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -15,6 +15,15 @@ extern bool hugetlb_disabled; +static inline bool hugepages_supported(void) +{ + if (hugetlb_disabled) + return false; + + return HPAGE_SHIFT != 0; +} +#define hugepages_supported hugepages_supported + void __init hugetlbpage_init_defaultsize(void); int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, @@ -36,7 +45,8 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, #define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, - unsigned long addr, pte_t *ptep) + unsigned long addr, pte_t *ptep, + unsigned long sz) { return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1)); } @@ -46,8 +56,9 @@ static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { pte_t pte; + unsigned long sz = huge_page_size(hstate_vma(vma)); - pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep); + pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep, sz); flush_hugetlb_page(vma, addr); return pte; } diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index 65d1f291393d..ea6c8dc400d2 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -270,6 +270,7 @@ #define H_QUERY_INT_STATE 0x1E4 #define H_POLL_PENDING 0x1D8 #define H_ILLAN_ATTRIBUTES 0x244 +#define H_ADD_LOGICAL_LAN_BUFFERS 0x248 #define H_MODIFY_HEA_QP 0x250 #define H_QUERY_HEA_QP 0x254 #define H_QUERY_HEA 0x258 @@ -348,6 +349,7 @@ #define H_SCM_FLUSH 0x44C #define H_GET_ENERGY_SCALE_INFO 0x450 #define H_PKS_SIGNED_UPDATE 0x454 +#define H_HTM 0x458 #define H_WATCHDOG 0x45C #define H_GUEST_GET_CAPABILITIES 0x460 #define H_GUEST_SET_CAPABILITIES 0x464 @@ -489,14 +491,48 @@ #define H_RPTI_PAGE_ALL (-1UL) /* Flags for H_GUEST_{S,G}_STATE */ -#define H_GUEST_FLAGS_WIDE (1UL<<(63-0)) +#define H_GUEST_FLAGS_WIDE (1UL << (63 - 0)) +#define H_GUEST_FLAGS_HOST_WIDE (1UL << (63 - 1)) /* Flag values used for H_{S,G}SET_GUEST_CAPABILITIES */ -#define H_GUEST_CAP_COPY_MEM (1UL<<(63-0)) -#define H_GUEST_CAP_POWER9 (1UL<<(63-1)) -#define H_GUEST_CAP_POWER10 (1UL<<(63-2)) -#define H_GUEST_CAP_POWER11 (1UL<<(63-3)) -#define H_GUEST_CAP_BITMAP2 (1UL<<(63-63)) +#define H_GUEST_CAP_COPY_MEM (1UL << (63 - 0)) +#define H_GUEST_CAP_POWER9 (1UL << (63 - 1)) +#define H_GUEST_CAP_POWER10 (1UL << (63 - 2)) +#define H_GUEST_CAP_POWER11 (1UL << (63 - 3)) +#define H_GUEST_CAP_BITMAP2 (1UL << (63 - 63)) + +/* + * Defines for H_HTM - Macros for hardware trace macro (HTM) function. + */ +#define H_HTM_FLAGS_HARDWARE_TARGET (1ul << 63) +#define H_HTM_FLAGS_LOGICAL_TARGET (1ul << 62) +#define H_HTM_FLAGS_PROCID_TARGET (1ul << 61) +#define H_HTM_FLAGS_NOWRAP (1ul << 60) + +#define H_HTM_OP_SHIFT (63-15) +#define H_HTM_OP(x) ((unsigned long)(x)<<H_HTM_OP_SHIFT) +#define H_HTM_OP_CAPABILITIES 0x01 +#define H_HTM_OP_STATUS 0x02 +#define H_HTM_OP_SETUP 0x03 +#define H_HTM_OP_CONFIGURE 0x04 +#define H_HTM_OP_START 0x05 +#define H_HTM_OP_STOP 0x06 +#define H_HTM_OP_DECONFIGURE 0x07 +#define H_HTM_OP_DUMP_DETAILS 0x08 +#define H_HTM_OP_DUMP_DATA 0x09 +#define H_HTM_OP_DUMP_SYSMEM_CONF 0x0a +#define H_HTM_OP_DUMP_SYSPROC_CONF 0x0b + +#define H_HTM_TYPE_SHIFT (63-31) +#define H_HTM_TYPE(x) ((unsigned long)(x)<<H_HTM_TYPE_SHIFT) +#define H_HTM_TYPE_NEST 0x01 +#define H_HTM_TYPE_CORE 0x02 +#define H_HTM_TYPE_LLAT 0x03 +#define H_HTM_TYPE_GLOBAL 0xff + +#define H_HTM_TARGET_NODE_INDEX(x) ((unsigned long)(x)<<(63-15)) +#define H_HTM_TARGET_NODAL_CHIP_INDEX(x) ((unsigned long)(x)<<(63-31)) +#define H_HTM_TARGET_CORE_INDEX_ON_CHIP(x) ((unsigned long)(x)<<(63-47)) #ifndef __ASSEMBLY__ #include <linux/types.h> diff --git a/arch/powerpc/include/asm/io-defs.h b/arch/powerpc/include/asm/io-defs.h index faf8617cc574..5c2be9b54a9d 100644 --- a/arch/powerpc/include/asm/io-defs.h +++ b/arch/powerpc/include/asm/io-defs.h @@ -1,61 +1,15 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* This file is meant to be include multiple times by other headers */ -/* last 2 argments are used by platforms/cell/io-workarounds.[ch] */ -DEF_PCI_AC_RET(readb, u8, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readw, u16, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readl, u32, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readw_be, u16, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readl_be, u32, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_NORET(writeb, (u8 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writew, (u16 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writel, (u32 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writew_be, (u16 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writel_be, (u32 val, PCI_IO_ADDR addr), (val, addr), mem, addr) - -#ifdef __powerpc64__ -DEF_PCI_AC_RET(readq, u64, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_RET(readq_be, u64, (const PCI_IO_ADDR addr), (addr), mem, addr) -DEF_PCI_AC_NORET(writeq, (u64 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -DEF_PCI_AC_NORET(writeq_be, (u64 val, PCI_IO_ADDR addr), (val, addr), mem, addr) -#endif /* __powerpc64__ */ - -DEF_PCI_AC_RET(inb, u8, (unsigned long port), (port), pio, port) -DEF_PCI_AC_RET(inw, u16, (unsigned long port), (port), pio, port) -DEF_PCI_AC_RET(inl, u32, (unsigned long port), (port), pio, port) -DEF_PCI_AC_NORET(outb, (u8 val, unsigned long port), (val, port), pio, port) -DEF_PCI_AC_NORET(outw, (u16 val, unsigned long port), (val, port), pio, port) -DEF_PCI_AC_NORET(outl, (u32 val, unsigned long port), (val, port), pio, port) - -DEF_PCI_AC_NORET(readsb, (const PCI_IO_ADDR a, void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(readsw, (const PCI_IO_ADDR a, void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(readsl, (const PCI_IO_ADDR a, void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(writesb, (PCI_IO_ADDR a, const void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(writesw, (PCI_IO_ADDR a, const void *b, unsigned long c), - (a, b, c), mem, a) -DEF_PCI_AC_NORET(writesl, (PCI_IO_ADDR a, const void *b, unsigned long c), - (a, b, c), mem, a) - -DEF_PCI_AC_NORET(insb, (unsigned long p, void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(insw, (unsigned long p, void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(insl, (unsigned long p, void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(outsb, (unsigned long p, const void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(outsw, (unsigned long p, const void *b, unsigned long c), - (p, b, c), pio, p) -DEF_PCI_AC_NORET(outsl, (unsigned long p, const void *b, unsigned long c), - (p, b, c), pio, p) - -DEF_PCI_AC_NORET(memset_io, (PCI_IO_ADDR a, int c, unsigned long n), - (a, c, n), mem, a) -DEF_PCI_AC_NORET(memcpy_fromio, (void *d, const PCI_IO_ADDR s, unsigned long n), - (d, s, n), mem, s) -DEF_PCI_AC_NORET(memcpy_toio, (PCI_IO_ADDR d, const void *s, unsigned long n), - (d, s, n), mem, d) +DEF_PCI_AC_RET(inb, u8, (unsigned long port), (port)) +DEF_PCI_AC_RET(inw, u16, (unsigned long port), (port)) +DEF_PCI_AC_RET(inl, u32, (unsigned long port), (port)) +DEF_PCI_AC_NORET(outb, (u8 val, unsigned long port), (val, port)) +DEF_PCI_AC_NORET(outw, (u16 val, unsigned long port), (val, port)) +DEF_PCI_AC_NORET(outl, (u32 val, unsigned long port), (val, port)) +DEF_PCI_AC_NORET(insb, (unsigned long p, void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(insw, (unsigned long p, void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(insl, (unsigned long p, void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(outsb, (unsigned long p, const void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(outsw, (unsigned long p, const void *b, unsigned long c), (p, b, c)) +DEF_PCI_AC_NORET(outsl, (unsigned long p, const void *b, unsigned long c), (p, b, c)) diff --git a/arch/powerpc/include/asm/io-workarounds.h b/arch/powerpc/include/asm/io-workarounds.h deleted file mode 100644 index 3cce499fbe27..000000000000 --- a/arch/powerpc/include/asm/io-workarounds.h +++ /dev/null @@ -1,55 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Support PCI IO workaround - * - * (C) Copyright 2007-2008 TOSHIBA CORPORATION - */ - -#ifndef _IO_WORKAROUNDS_H -#define _IO_WORKAROUNDS_H - -#ifdef CONFIG_PPC_IO_WORKAROUNDS -#include <linux/io.h> -#include <asm/pci-bridge.h> - -/* Bus info */ -struct iowa_bus { - struct pci_controller *phb; - struct ppc_pci_io *ops; - void *private; -}; - -void iowa_register_bus(struct pci_controller *, struct ppc_pci_io *, - int (*)(struct iowa_bus *, void *), void *); -struct iowa_bus *iowa_mem_find_bus(const PCI_IO_ADDR); -struct iowa_bus *iowa_pio_find_bus(unsigned long); - -extern struct ppc_pci_io spiderpci_ops; -extern int spiderpci_iowa_init(struct iowa_bus *, void *); - -#define SPIDER_PCI_REG_BASE 0xd000 -#define SPIDER_PCI_REG_SIZE 0x1000 -#define SPIDER_PCI_VCI_CNTL_STAT 0x0110 -#define SPIDER_PCI_DUMMY_READ 0x0810 -#define SPIDER_PCI_DUMMY_READ_BASE 0x0814 - -#endif - -#if defined(CONFIG_PPC_IO_WORKAROUNDS) && defined(CONFIG_PPC_INDIRECT_MMIO) -extern bool io_workaround_inited; - -static inline bool iowa_is_active(void) -{ - return unlikely(io_workaround_inited); -} -#else -static inline bool iowa_is_active(void) -{ - return false; -} -#endif - -void __iomem *iowa_ioremap(phys_addr_t addr, unsigned long size, - pgprot_t prot, void *caller); - -#endif /* _IO_WORKAROUNDS_H */ diff --git a/arch/powerpc/include/asm/io.h b/arch/powerpc/include/asm/io.h index fd92ac450169..7a89754842d6 100644 --- a/arch/powerpc/include/asm/io.h +++ b/arch/powerpc/include/asm/io.h @@ -65,8 +65,8 @@ extern resource_size_t isa_mem_base; extern bool isa_io_special; #ifdef CONFIG_PPC32 -#if defined(CONFIG_PPC_INDIRECT_PIO) || defined(CONFIG_PPC_INDIRECT_MMIO) -#error CONFIG_PPC_INDIRECT_{PIO,MMIO} are not yet supported on 32 bits +#ifdef CONFIG_PPC_INDIRECT_PIO +#error CONFIG_PPC_INDIRECT_PIO is not yet supported on 32 bits #endif #endif @@ -80,16 +80,12 @@ extern bool isa_io_special; * * in_8, in_le16, in_be16, in_le32, in_be32, in_le64, in_be64 * out_8, out_le16, out_be16, out_le32, out_be32, out_le64, out_be64 - * _insb, _insw_ns, _insl_ns, _outsb, _outsw_ns, _outsl_ns + * _insb, _insw, _insl, _outsb, _outsw, _outsl * * Those operate directly on a kernel virtual address. Note that the prototype * for the out_* accessors has the arguments in opposite order from the usual * linux PCI accessors. Unlike those, they take the address first and the value * next. - * - * Note: I might drop the _ns suffix on the stream operations soon as it is - * simply normal for stream operations to not swap in the first place. - * */ /* -mprefixed can generate offsets beyond range, fall back hack */ @@ -228,19 +224,10 @@ static inline void out_be64(volatile u64 __iomem *addr, u64 val) */ extern void _insb(const volatile u8 __iomem *addr, void *buf, long count); extern void _outsb(volatile u8 __iomem *addr,const void *buf,long count); -extern void _insw_ns(const volatile u16 __iomem *addr, void *buf, long count); -extern void _outsw_ns(volatile u16 __iomem *addr, const void *buf, long count); -extern void _insl_ns(const volatile u32 __iomem *addr, void *buf, long count); -extern void _outsl_ns(volatile u32 __iomem *addr, const void *buf, long count); - -/* The _ns naming is historical and will be removed. For now, just #define - * the non _ns equivalent names - */ -#define _insw _insw_ns -#define _insl _insl_ns -#define _outsw _outsw_ns -#define _outsl _outsl_ns - +extern void _insw(const volatile u16 __iomem *addr, void *buf, long count); +extern void _outsw(volatile u16 __iomem *addr, const void *buf, long count); +extern void _insl(const volatile u32 __iomem *addr, void *buf, long count); +extern void _outsl(volatile u32 __iomem *addr, const void *buf, long count); /* * memset_io, memcpy_toio, memcpy_fromio base implementations are out of line @@ -261,9 +248,9 @@ extern void _memcpy_toio(volatile void __iomem *dest, const void *src, * for PowerPC is as close as possible to the x86 version of these, and thus * provides fairly heavy weight barriers for the non-raw versions * - * In addition, they support a hook mechanism when CONFIG_PPC_INDIRECT_MMIO - * or CONFIG_PPC_INDIRECT_PIO are set allowing the platform to provide its - * own implementation of some or all of the accessors. + * In addition, they support a hook mechanism when CONFIG_PPC_INDIRECT_PIO + * is set allowing the platform to provide its own implementation of some + * of the accessors. */ /* @@ -274,116 +261,11 @@ extern void _memcpy_toio(volatile void __iomem *dest, const void *src, #include <asm/eeh.h> #endif -/* Shortcut to the MMIO argument pointer */ -#define PCI_IO_ADDR volatile void __iomem * - -/* Indirect IO address tokens: - * - * When CONFIG_PPC_INDIRECT_MMIO is set, the platform can provide hooks - * on all MMIOs. (Note that this is all 64 bits only for now) - * - * To help platforms who may need to differentiate MMIO addresses in - * their hooks, a bitfield is reserved for use by the platform near the - * top of MMIO addresses (not PIO, those have to cope the hard way). - * - * The highest address in the kernel virtual space are: - * - * d0003fffffffffff # with Hash MMU - * c00fffffffffffff # with Radix MMU - * - * The top 4 bits are reserved as the region ID on hash, leaving us 8 bits - * that can be used for the field. - * - * The direct IO mapping operations will then mask off those bits - * before doing the actual access, though that only happen when - * CONFIG_PPC_INDIRECT_MMIO is set, thus be careful when you use that - * mechanism - * - * For PIO, there is a separate CONFIG_PPC_INDIRECT_PIO which makes - * all PIO functions call through a hook. - */ - -#ifdef CONFIG_PPC_INDIRECT_MMIO -#define PCI_IO_IND_TOKEN_SHIFT 52 -#define PCI_IO_IND_TOKEN_MASK (0xfful << PCI_IO_IND_TOKEN_SHIFT) -#define PCI_FIX_ADDR(addr) \ - ((PCI_IO_ADDR)(((unsigned long)(addr)) & ~PCI_IO_IND_TOKEN_MASK)) -#define PCI_GET_ADDR_TOKEN(addr) \ - (((unsigned long)(addr) & PCI_IO_IND_TOKEN_MASK) >> \ - PCI_IO_IND_TOKEN_SHIFT) -#define PCI_SET_ADDR_TOKEN(addr, token) \ -do { \ - unsigned long __a = (unsigned long)(addr); \ - __a &= ~PCI_IO_IND_TOKEN_MASK; \ - __a |= ((unsigned long)(token)) << PCI_IO_IND_TOKEN_SHIFT; \ - (addr) = (void __iomem *)__a; \ -} while(0) -#else -#define PCI_FIX_ADDR(addr) (addr) -#endif - - -/* - * Non ordered and non-swapping "raw" accessors - */ - -static inline unsigned char __raw_readb(const volatile void __iomem *addr) -{ - return *(volatile unsigned char __force *)PCI_FIX_ADDR(addr); -} -#define __raw_readb __raw_readb - -static inline unsigned short __raw_readw(const volatile void __iomem *addr) -{ - return *(volatile unsigned short __force *)PCI_FIX_ADDR(addr); -} -#define __raw_readw __raw_readw - -static inline unsigned int __raw_readl(const volatile void __iomem *addr) -{ - return *(volatile unsigned int __force *)PCI_FIX_ADDR(addr); -} -#define __raw_readl __raw_readl - -static inline void __raw_writeb(unsigned char v, volatile void __iomem *addr) -{ - *(volatile unsigned char __force *)PCI_FIX_ADDR(addr) = v; -} -#define __raw_writeb __raw_writeb - -static inline void __raw_writew(unsigned short v, volatile void __iomem *addr) -{ - *(volatile unsigned short __force *)PCI_FIX_ADDR(addr) = v; -} -#define __raw_writew __raw_writew - -static inline void __raw_writel(unsigned int v, volatile void __iomem *addr) -{ - *(volatile unsigned int __force *)PCI_FIX_ADDR(addr) = v; -} -#define __raw_writel __raw_writel +#define _IO_PORT(port) ((volatile void __iomem *)(_IO_BASE + (port))) #ifdef __powerpc64__ -static inline unsigned long __raw_readq(const volatile void __iomem *addr) -{ - return *(volatile unsigned long __force *)PCI_FIX_ADDR(addr); -} -#define __raw_readq __raw_readq - -static inline void __raw_writeq(unsigned long v, volatile void __iomem *addr) -{ - *(volatile unsigned long __force *)PCI_FIX_ADDR(addr) = v; -} -#define __raw_writeq __raw_writeq - -static inline void __raw_writeq_be(unsigned long v, volatile void __iomem *addr) -{ - __raw_writeq((__force unsigned long)cpu_to_be64(v), addr); -} -#define __raw_writeq_be __raw_writeq_be - /* - * Real mode versions of the above. Those instructions are only supposed + * Real mode versions of raw accessors. Those instructions are only supposed * to be used in hypervisor real mode as per the architecture spec. */ static inline void __raw_rm_writeb(u8 val, volatile void __iomem *paddr) @@ -551,30 +433,23 @@ __do_out_asm(_rec_outl, "stwbrx") * possible to hook directly at the toplevel PIO operation if they have to * be handled differently */ -#define __do_writeb(val, addr) out_8(PCI_FIX_ADDR(addr), val) -#define __do_writew(val, addr) out_le16(PCI_FIX_ADDR(addr), val) -#define __do_writel(val, addr) out_le32(PCI_FIX_ADDR(addr), val) -#define __do_writeq(val, addr) out_le64(PCI_FIX_ADDR(addr), val) -#define __do_writew_be(val, addr) out_be16(PCI_FIX_ADDR(addr), val) -#define __do_writel_be(val, addr) out_be32(PCI_FIX_ADDR(addr), val) -#define __do_writeq_be(val, addr) out_be64(PCI_FIX_ADDR(addr), val) #ifdef CONFIG_EEH -#define __do_readb(addr) eeh_readb(PCI_FIX_ADDR(addr)) -#define __do_readw(addr) eeh_readw(PCI_FIX_ADDR(addr)) -#define __do_readl(addr) eeh_readl(PCI_FIX_ADDR(addr)) -#define __do_readq(addr) eeh_readq(PCI_FIX_ADDR(addr)) -#define __do_readw_be(addr) eeh_readw_be(PCI_FIX_ADDR(addr)) -#define __do_readl_be(addr) eeh_readl_be(PCI_FIX_ADDR(addr)) -#define __do_readq_be(addr) eeh_readq_be(PCI_FIX_ADDR(addr)) +#define __do_readb(addr) eeh_readb(addr) +#define __do_readw(addr) eeh_readw(addr) +#define __do_readl(addr) eeh_readl(addr) +#define __do_readq(addr) eeh_readq(addr) +#define __do_readw_be(addr) eeh_readw_be(addr) +#define __do_readl_be(addr) eeh_readl_be(addr) +#define __do_readq_be(addr) eeh_readq_be(addr) #else /* CONFIG_EEH */ -#define __do_readb(addr) in_8(PCI_FIX_ADDR(addr)) -#define __do_readw(addr) in_le16(PCI_FIX_ADDR(addr)) -#define __do_readl(addr) in_le32(PCI_FIX_ADDR(addr)) -#define __do_readq(addr) in_le64(PCI_FIX_ADDR(addr)) -#define __do_readw_be(addr) in_be16(PCI_FIX_ADDR(addr)) -#define __do_readl_be(addr) in_be32(PCI_FIX_ADDR(addr)) -#define __do_readq_be(addr) in_be64(PCI_FIX_ADDR(addr)) +#define __do_readb(addr) in_8(addr) +#define __do_readw(addr) in_le16(addr) +#define __do_readl(addr) in_le32(addr) +#define __do_readq(addr) in_le64(addr) +#define __do_readw_be(addr) in_be16(addr) +#define __do_readl_be(addr) in_be32(addr) +#define __do_readq_be(addr) in_be64(addr) #endif /* !defined(CONFIG_EEH) */ #ifdef CONFIG_PPC32 @@ -585,64 +460,185 @@ __do_out_asm(_rec_outl, "stwbrx") #define __do_inw(port) _rec_inw(port) #define __do_inl(port) _rec_inl(port) #else /* CONFIG_PPC32 */ -#define __do_outb(val, port) writeb(val,(PCI_IO_ADDR)(_IO_BASE+port)); -#define __do_outw(val, port) writew(val,(PCI_IO_ADDR)(_IO_BASE+port)); -#define __do_outl(val, port) writel(val,(PCI_IO_ADDR)(_IO_BASE+port)); -#define __do_inb(port) readb((PCI_IO_ADDR)(_IO_BASE + port)); -#define __do_inw(port) readw((PCI_IO_ADDR)(_IO_BASE + port)); -#define __do_inl(port) readl((PCI_IO_ADDR)(_IO_BASE + port)); +#define __do_outb(val, port) writeb(val,_IO_PORT(port)); +#define __do_outw(val, port) writew(val,_IO_PORT(port)); +#define __do_outl(val, port) writel(val,_IO_PORT(port)); +#define __do_inb(port) readb(_IO_PORT(port)); +#define __do_inw(port) readw(_IO_PORT(port)); +#define __do_inl(port) readl(_IO_PORT(port)); #endif /* !CONFIG_PPC32 */ #ifdef CONFIG_EEH -#define __do_readsb(a, b, n) eeh_readsb(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsw(a, b, n) eeh_readsw(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsl(a, b, n) eeh_readsl(PCI_FIX_ADDR(a), (b), (n)) +#define __do_readsb(a, b, n) eeh_readsb(a, (b), (n)) +#define __do_readsw(a, b, n) eeh_readsw(a, (b), (n)) +#define __do_readsl(a, b, n) eeh_readsl(a, (b), (n)) #else /* CONFIG_EEH */ -#define __do_readsb(a, b, n) _insb(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsw(a, b, n) _insw(PCI_FIX_ADDR(a), (b), (n)) -#define __do_readsl(a, b, n) _insl(PCI_FIX_ADDR(a), (b), (n)) +#define __do_readsb(a, b, n) _insb(a, (b), (n)) +#define __do_readsw(a, b, n) _insw(a, (b), (n)) +#define __do_readsl(a, b, n) _insl(a, (b), (n)) #endif /* !CONFIG_EEH */ -#define __do_writesb(a, b, n) _outsb(PCI_FIX_ADDR(a),(b),(n)) -#define __do_writesw(a, b, n) _outsw(PCI_FIX_ADDR(a),(b),(n)) -#define __do_writesl(a, b, n) _outsl(PCI_FIX_ADDR(a),(b),(n)) - -#define __do_insb(p, b, n) readsb((PCI_IO_ADDR)(_IO_BASE+(p)), (b), (n)) -#define __do_insw(p, b, n) readsw((PCI_IO_ADDR)(_IO_BASE+(p)), (b), (n)) -#define __do_insl(p, b, n) readsl((PCI_IO_ADDR)(_IO_BASE+(p)), (b), (n)) -#define __do_outsb(p, b, n) writesb((PCI_IO_ADDR)(_IO_BASE+(p)),(b),(n)) -#define __do_outsw(p, b, n) writesw((PCI_IO_ADDR)(_IO_BASE+(p)),(b),(n)) -#define __do_outsl(p, b, n) writesl((PCI_IO_ADDR)(_IO_BASE+(p)),(b),(n)) - -#define __do_memset_io(addr, c, n) \ - _memset_io(PCI_FIX_ADDR(addr), c, n) -#define __do_memcpy_toio(dst, src, n) \ - _memcpy_toio(PCI_FIX_ADDR(dst), src, n) +#define __do_writesb(a, b, n) _outsb(a, (b), (n)) +#define __do_writesw(a, b, n) _outsw(a, (b), (n)) +#define __do_writesl(a, b, n) _outsl(a, (b), (n)) + +#define __do_insb(p, b, n) readsb(_IO_PORT(p), (b), (n)) +#define __do_insw(p, b, n) readsw(_IO_PORT(p), (b), (n)) +#define __do_insl(p, b, n) readsl(_IO_PORT(p), (b), (n)) +#define __do_outsb(p, b, n) writesb(_IO_PORT(p),(b),(n)) +#define __do_outsw(p, b, n) writesw(_IO_PORT(p),(b),(n)) +#define __do_outsl(p, b, n) writesl(_IO_PORT(p),(b),(n)) #ifdef CONFIG_EEH #define __do_memcpy_fromio(dst, src, n) \ - eeh_memcpy_fromio(dst, PCI_FIX_ADDR(src), n) + eeh_memcpy_fromio(dst, src, n) #else /* CONFIG_EEH */ #define __do_memcpy_fromio(dst, src, n) \ - _memcpy_fromio(dst,PCI_FIX_ADDR(src),n) + _memcpy_fromio(dst, src, n) #endif /* !CONFIG_EEH */ -#ifdef CONFIG_PPC_INDIRECT_PIO -#define DEF_PCI_HOOK_pio(x) x -#else -#define DEF_PCI_HOOK_pio(x) NULL -#endif +static inline u8 readb(const volatile void __iomem *addr) +{ + return __do_readb(addr); +} +#define readb readb + +static inline u16 readw(const volatile void __iomem *addr) +{ + return __do_readw(addr); +} +#define readw readw + +static inline u32 readl(const volatile void __iomem *addr) +{ + return __do_readl(addr); +} +#define readl readl + +static inline u16 readw_be(const volatile void __iomem *addr) +{ + return __do_readw_be(addr); +} + +static inline u32 readl_be(const volatile void __iomem *addr) +{ + return __do_readl_be(addr); +} -#ifdef CONFIG_PPC_INDIRECT_MMIO -#define DEF_PCI_HOOK_mem(x) x +static inline void writeb(u8 val, volatile void __iomem *addr) +{ + out_8(addr, val); +} +#define writeb writeb + +static inline void writew(u16 val, volatile void __iomem *addr) +{ + out_le16(addr, val); +} +#define writew writew + +static inline void writel(u32 val, volatile void __iomem *addr) +{ + out_le32(addr, val); +} +#define writel writel + +static inline void writew_be(u16 val, volatile void __iomem *addr) +{ + out_be16(addr, val); +} + +static inline void writel_be(u32 val, volatile void __iomem *addr) +{ + out_be32(addr, val); +} + +static inline void readsb(const volatile void __iomem *a, void *b, unsigned long c) +{ + __do_readsb(a, b, c); +} +#define readsb readsb + +static inline void readsw(const volatile void __iomem *a, void *b, unsigned long c) +{ + __do_readsw(a, b, c); +} +#define readsw readsw + +static inline void readsl(const volatile void __iomem *a, void *b, unsigned long c) +{ + __do_readsl(a, b, c); +} +#define readsl readsl + +static inline void writesb(volatile void __iomem *a, const void *b, unsigned long c) +{ + __do_writesb(a, b, c); +} +#define writesb writesb + +static inline void writesw(volatile void __iomem *a, const void *b, unsigned long c) +{ + __do_writesw(a, b, c); +} +#define writesw writesw + +static inline void writesl(volatile void __iomem *a, const void *b, unsigned long c) +{ + __do_writesl(a, b, c); +} +#define writesl writesl + +static inline void memset_io(volatile void __iomem *a, int c, unsigned long n) +{ + _memset_io(a, c, n); +} +#define memset_io memset_io + +static inline void memcpy_fromio(void *d, const volatile void __iomem *s, unsigned long n) +{ + __do_memcpy_fromio(d, s, n); +} +#define memcpy_fromio memcpy_fromio + +static inline void memcpy_toio(volatile void __iomem *d, const void *s, unsigned long n) +{ + _memcpy_toio(d, s, n); +} +#define memcpy_toio memcpy_toio + +#ifdef __powerpc64__ +static inline u64 readq(const volatile void __iomem *addr) +{ + return __do_readq(addr); +} + +static inline u64 readq_be(const volatile void __iomem *addr) +{ + return __do_readq_be(addr); +} + +static inline void writeq(u64 val, volatile void __iomem *addr) +{ + out_le64(addr, val); +} + +static inline void writeq_be(u64 val, volatile void __iomem *addr) +{ + out_be64(addr, val); +} +#endif /* __powerpc64__ */ + +#ifdef CONFIG_PPC_INDIRECT_PIO +#define DEF_PCI_HOOK(x) x #else -#define DEF_PCI_HOOK_mem(x) NULL +#define DEF_PCI_HOOK(x) NULL #endif /* Structure containing all the hooks */ extern struct ppc_pci_io { -#define DEF_PCI_AC_RET(name, ret, at, al, space, aa) ret (*name) at; -#define DEF_PCI_AC_NORET(name, at, al, space, aa) void (*name) at; +#define DEF_PCI_AC_RET(name, ret, at, al) ret (*name) at; +#define DEF_PCI_AC_NORET(name, at, al) void (*name) at; #include <asm/io-defs.h> @@ -652,18 +648,18 @@ extern struct ppc_pci_io { } ppc_pci_io; /* The inline wrappers */ -#define DEF_PCI_AC_RET(name, ret, at, al, space, aa) \ +#define DEF_PCI_AC_RET(name, ret, at, al) \ static inline ret name at \ { \ - if (DEF_PCI_HOOK_##space(ppc_pci_io.name) != NULL) \ + if (DEF_PCI_HOOK(ppc_pci_io.name) != NULL) \ return ppc_pci_io.name al; \ return __do_##name al; \ } -#define DEF_PCI_AC_NORET(name, at, al, space, aa) \ +#define DEF_PCI_AC_NORET(name, at, al) \ static inline void name at \ { \ - if (DEF_PCI_HOOK_##space(ppc_pci_io.name) != NULL) \ + if (DEF_PCI_HOOK(ppc_pci_io.name) != NULL) \ ppc_pci_io.name al; \ else \ __do_##name al; \ @@ -674,21 +670,7 @@ static inline void name at \ #undef DEF_PCI_AC_RET #undef DEF_PCI_AC_NORET -/* Some drivers check for the presence of readq & writeq with - * a #ifdef, so we make them happy here. - */ -#define readb readb -#define readw readw -#define readl readl -#define writeb writeb -#define writew writew -#define writel writel -#define readsb readsb -#define readsw readsw -#define readsl readsl -#define writesb writesb -#define writesw writesw -#define writesl writesl +// Signal to asm-generic/io.h that we have implemented these. #define inb inb #define inw inw #define inl inl @@ -705,9 +687,6 @@ static inline void name at \ #define readq readq #define writeq writeq #endif -#define memset_io memset_io -#define memcpy_fromio memcpy_fromio -#define memcpy_toio memcpy_toio /* * We don't do relaxed operations yet, at least not with this semantic @@ -738,35 +717,11 @@ static inline unsigned int ioread32be(const void __iomem *addr) #define ioread32be ioread32be #ifdef __powerpc64__ -static inline u64 ioread64_lo_hi(const void __iomem *addr) -{ - return readq(addr); -} -#define ioread64_lo_hi ioread64_lo_hi - -static inline u64 ioread64_hi_lo(const void __iomem *addr) -{ - return readq(addr); -} -#define ioread64_hi_lo ioread64_hi_lo - static inline u64 ioread64be(const void __iomem *addr) { return readq_be(addr); } #define ioread64be ioread64be - -static inline u64 ioread64be_lo_hi(const void __iomem *addr) -{ - return readq_be(addr); -} -#define ioread64be_lo_hi ioread64be_lo_hi - -static inline u64 ioread64be_hi_lo(const void __iomem *addr) -{ - return readq_be(addr); -} -#define ioread64be_hi_lo ioread64be_hi_lo #endif /* __powerpc64__ */ static inline void iowrite16be(u16 val, void __iomem *addr) @@ -782,35 +737,11 @@ static inline void iowrite32be(u32 val, void __iomem *addr) #define iowrite32be iowrite32be #ifdef __powerpc64__ -static inline void iowrite64_lo_hi(u64 val, void __iomem *addr) -{ - writeq(val, addr); -} -#define iowrite64_lo_hi iowrite64_lo_hi - -static inline void iowrite64_hi_lo(u64 val, void __iomem *addr) -{ - writeq(val, addr); -} -#define iowrite64_hi_lo iowrite64_hi_lo - static inline void iowrite64be(u64 val, void __iomem *addr) { writeq_be(val, addr); } #define iowrite64be iowrite64be - -static inline void iowrite64be_lo_hi(u64 val, void __iomem *addr) -{ - writeq_be(val, addr); -} -#define iowrite64be_lo_hi iowrite64be_lo_hi - -static inline void iowrite64be_hi_lo(u64 val, void __iomem *addr) -{ - writeq_be(val, addr); -} -#define iowrite64be_hi_lo iowrite64be_hi_lo #endif /* __powerpc64__ */ struct pci_dev; @@ -895,7 +826,7 @@ void __iomem *ioremap_wt(phys_addr_t address, unsigned long size); void __iomem *ioremap_coherent(phys_addr_t address, unsigned long size); #define ioremap_cache(addr, size) \ - ioremap_prot((addr), (size), pgprot_val(PAGE_KERNEL)) + ioremap_prot((addr), (size), PAGE_KERNEL) #define iounmap iounmap @@ -1030,6 +961,14 @@ static inline void * bus_to_virt(unsigned long address) #include <asm-generic/io.h> +#ifdef __powerpc64__ +static inline void __raw_writeq_be(unsigned long v, volatile void __iomem *addr) +{ + __raw_writeq((__force unsigned long)cpu_to_be64(v), addr); +} +#define __raw_writeq_be __raw_writeq_be +#endif // __powerpc64__ + #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_IO_H */ diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index 04072b5f8962..b410021ad4c6 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -317,12 +317,6 @@ extern void iommu_flush_tce(struct iommu_table *tbl); extern enum dma_data_direction iommu_tce_direction(unsigned long tce); extern unsigned long iommu_direction_to_tce_perm(enum dma_data_direction dir); -#ifdef CONFIG_PPC_CELL_NATIVE -extern bool iommu_fixed_is_weak; -#else -#define iommu_fixed_is_weak false -#endif - extern const struct dma_map_ops dma_iommu_ops; #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 270ee93a0f7d..70f2f0517509 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -61,7 +61,6 @@ struct pt_regs; extern void kexec_smp_wait(void); /* get and clear naca physid, wait for master to copy new code to 0 */ extern void default_machine_kexec(struct kimage *image); -extern void machine_kexec_mask_interrupts(void); void relocate_new_kernel(unsigned long indirection_page, unsigned long reboot_code_buffer, unsigned long start_address) __noreturn; @@ -95,8 +94,10 @@ int arch_kexec_kernel_image_probe(struct kimage *image, void *buf, unsigned long int arch_kimage_file_post_load_cleanup(struct kimage *image); #define arch_kimage_file_post_load_cleanup arch_kimage_file_post_load_cleanup -int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf); -#define arch_kexec_locate_mem_hole arch_kexec_locate_mem_hole +int arch_check_excluded_range(struct kimage *image, unsigned long start, + unsigned long end); +#define arch_check_excluded_range arch_check_excluded_range + int load_crashdump_segments_ppc64(struct kimage *image, struct kexec_buf *kbuf); @@ -113,9 +114,9 @@ int setup_new_fdt_ppc64(const struct kimage *image, void *fdt, struct crash_mem #ifdef CONFIG_CRASH_RESERVE int __init overlaps_crashkernel(unsigned long start, unsigned long size); -extern void reserve_crashkernel(void); +extern void arch_reserve_crashkernel(void); #else -static inline void reserve_crashkernel(void) {} +static inline void arch_reserve_crashkernel(void) {} static inline int overlaps_crashkernel(unsigned long start, unsigned long size) { return 0; } #endif diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 6e1108f8fce6..2d139c807577 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -902,7 +902,6 @@ struct kvm_vcpu_arch { #define __KVM_HAVE_ARCH_WQP #define __KVM_HAVE_CREATE_DEVICE -static inline void kvm_arch_sync_events(struct kvm *kvm) {} static inline void kvm_arch_memslots_updated(struct kvm *kvm, u64 gen) {} static inline void kvm_arch_flush_shadow_all(struct kvm *kvm) {} static inline void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) {} diff --git a/arch/powerpc/include/asm/mman.h b/arch/powerpc/include/asm/mman.h index 42a51a993d94..912f78a956a1 100644 --- a/arch/powerpc/include/asm/mman.h +++ b/arch/powerpc/include/asm/mman.h @@ -14,7 +14,7 @@ #include <asm/cpu_has_feature.h> #include <asm/firmware.h> -static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, +static inline vm_flags_t arch_calc_vm_prot_bits(unsigned long prot, unsigned long pkey) { #ifdef CONFIG_PPC_MEM_KEYS diff --git a/arch/powerpc/include/asm/mmzone.h b/arch/powerpc/include/asm/mmzone.h index d99863cd6cde..049152f8d597 100644 --- a/arch/powerpc/include/asm/mmzone.h +++ b/arch/powerpc/include/asm/mmzone.h @@ -29,6 +29,7 @@ extern cpumask_var_t node_to_cpumask_map[]; #ifdef CONFIG_MEMORY_HOTPLUG extern unsigned long max_pfn; u64 memory_hotplug_max(void); +u64 hot_add_drconf_memory_max(void); #else #define memory_hotplug_max() memblock_end_of_DRAM() #endif diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 8d1f0b7062eb..7d6b9e5b286e 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -286,7 +286,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); } -static inline int pte_swp_exclusive(pte_t pte) +static inline bool pte_swp_exclusive(pte_t pte) { return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; } diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h index 2f72ad885332..93d77ad5a92f 100644 --- a/arch/powerpc/include/asm/pgtable.h +++ b/arch/powerpc/include/asm/pgtable.h @@ -53,9 +53,8 @@ void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, #define MAX_PTRS_PER_PGD PTRS_PER_PGD #endif -/* Keep these as a macros to avoid include dependency mess */ +/* Keep this as a macro to avoid include dependency mess */ #define pte_page(x) pfn_to_page(pte_pfn(x)) -#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) static inline unsigned long pte_pfn(pte_t pte) { diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 59a2c7dbc78f..28e752138996 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -30,9 +30,9 @@ extern u32 reserved_allocation_mask; /* bits set for reserved keys */ #endif -static inline u64 pkey_to_vmflag_bits(u16 pkey) +static inline vm_flags_t pkey_to_vmflag_bits(u16 pkey) { - return (((u64)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS); + return (((vm_flags_t)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS); } static inline int vma_pkey(struct vm_area_struct *vma) diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h index 71648c126970..f2b6cc4341bb 100644 --- a/arch/powerpc/include/asm/plpar_wrappers.h +++ b/arch/powerpc/include/asm/plpar_wrappers.h @@ -65,6 +65,35 @@ static inline long register_dtl(unsigned long cpu, unsigned long vpa) return vpa_call(H_VPA_REG_DTL, cpu, vpa); } +/* + * Invokes H_HTM hcall with parameters passed from htm_hcall_wrapper. + * flags: Set to hardwareTarget. + * target: Specifies target using node index, nodal chip index and core index. + * operation : action to perform ie configure, start, stop, deconfigure, trace + * based on the HTM type. + * param1, param2, param3: parameters for each action. + */ +static inline long htm_call(unsigned long flags, unsigned long target, + unsigned long operation, unsigned long param1, + unsigned long param2, unsigned long param3) +{ + return plpar_hcall_norets(H_HTM, flags, target, operation, + param1, param2, param3); +} + +static inline long htm_hcall_wrapper(unsigned long flags, unsigned long nodeindex, + unsigned long nodalchipindex, unsigned long coreindexonchip, + unsigned long type, unsigned long htm_op, unsigned long param1, unsigned long param2, + unsigned long param3) +{ + return htm_call(H_HTM_FLAGS_HARDWARE_TARGET | flags, + H_HTM_TARGET_NODE_INDEX(nodeindex) | + H_HTM_TARGET_NODAL_CHIP_INDEX(nodalchipindex) | + H_HTM_TARGET_CORE_INDEX_ON_CHIP(coreindexonchip), + H_HTM_OP(htm_op) | H_HTM_TYPE(type), + param1, param2, param3); +} + extern void vpa_init(int cpu); static inline long plpar_pte_enter(unsigned long flags, diff --git a/arch/powerpc/include/asm/pmi.h b/arch/powerpc/include/asm/pmi.h deleted file mode 100644 index 478f0a2fe7f4..000000000000 --- a/arch/powerpc/include/asm/pmi.h +++ /dev/null @@ -1,53 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -#ifndef _POWERPC_PMI_H -#define _POWERPC_PMI_H - -/* - * Definitions for talking with PMI device on PowerPC - * - * PMI (Platform Management Interrupt) is a way to communicate - * with the BMC (Baseboard Management Controller) via interrupts. - * Unlike IPMI it is bidirectional and has a low latency. - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * - * Author: Christian Krafft <krafft@de.ibm.com> - */ - -#ifdef __KERNEL__ - -#define PMI_TYPE_FREQ_CHANGE 0x01 -#define PMI_TYPE_POWER_BUTTON 0x02 -#define PMI_READ_TYPE 0 -#define PMI_READ_DATA0 1 -#define PMI_READ_DATA1 2 -#define PMI_READ_DATA2 3 -#define PMI_WRITE_TYPE 4 -#define PMI_WRITE_DATA0 5 -#define PMI_WRITE_DATA1 6 -#define PMI_WRITE_DATA2 7 - -#define PMI_ACK 0x80 - -#define PMI_TIMEOUT 100 - -typedef struct { - u8 type; - u8 data0; - u8 data1; - u8 data2; -} pmi_message_t; - -struct pmi_handler { - struct list_head node; - u8 type; - void (*handle_pmi_message) (pmi_message_t); -}; - -int pmi_register_handler(struct pmi_handler *); -void pmi_unregister_handler(struct pmi_handler *); - -int pmi_send_message(pmi_message_t); - -#endif /* __KERNEL__ */ -#endif /* _POWERPC_PMI_H */ diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h index 8afc92860dbb..7e9a479951a3 100644 --- a/arch/powerpc/include/asm/pnv-pci.h +++ b/arch/powerpc/include/asm/pnv-pci.h @@ -10,7 +10,6 @@ #include <linux/pci_hotplug.h> #include <linux/irq.h> #include <linux/of.h> -#include <misc/cxl-base.h> #include <asm/opal-api.h> #define PCI_SLOT_ID_PREFIX (1UL << 63) @@ -25,25 +24,9 @@ extern int pnv_pci_get_power_state(uint64_t id, uint8_t *state); extern int pnv_pci_set_power_state(uint64_t id, uint8_t state, struct opal_msg *msg); -extern int pnv_pci_set_tunnel_bar(struct pci_dev *dev, uint64_t addr, - int enable); -int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode); -int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq, - unsigned int virq); -int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num); -void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num); -int pnv_cxl_get_irq_count(struct pci_dev *dev); -struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev); int64_t pnv_opal_pci_msi_eoi(struct irq_data *d); bool is_pnv_opal_msi(struct irq_chip *chip); -#ifdef CONFIG_CXL_BASE -int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs, - struct pci_dev *dev, int num); -void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs, - struct pci_dev *dev); -#endif - struct pnv_php_slot { struct hotplug_slot slot; uint64_t id; diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 4312bcb913a4..8053b24afc39 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -425,6 +425,7 @@ #define PPC_RAW_SC() (0x44000002) #define PPC_RAW_SYNC() (0x7c0004ac) #define PPC_RAW_ISYNC() (0x4c00012c) +#define PPC_RAW_LWSYNC() (0x7c2004ac) /* * Define what the VSX XX1 form instructions will look like, then add diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 02897f4b0dbf..b891910fce8a 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -183,7 +183,7 @@ /* * Used to name C functions called from asm */ -#ifdef CONFIG_PPC_KERNEL_PCREL +#if defined(__powerpc64__) && defined(CONFIG_PPC_KERNEL_PCREL) #define CFUNC(name) name@notoc #else #define CFUNC(name) name diff --git a/arch/powerpc/include/asm/preempt.h b/arch/powerpc/include/asm/preempt.h new file mode 100644 index 000000000000..000e2b9681f3 --- /dev/null +++ b/arch/powerpc/include/asm/preempt.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_POWERPC_PREEMPT_H +#define __ASM_POWERPC_PREEMPT_H + +#include <asm-generic/preempt.h> + +#if defined(CONFIG_PREEMPT_DYNAMIC) +#include <linux/jump_label.h> +DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); +#define need_irq_preemption() \ + (static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched)) +#else +#define need_irq_preemption() (IS_ENABLED(CONFIG_PREEMPTION)) +#endif + +#endif /* __ASM_POWERPC_PREEMPT_H */ diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index c0107d8ddd8c..f679a11a7e7f 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -17,6 +17,8 @@ struct device_node; struct property; +#define MIN_RMA 768 /* Minimum RMA (in MB) for CAS negotiation */ + #define OF_DT_BEGIN_NODE 0x1 /* Start of node, full name */ #define OF_DT_END_NODE 0x2 /* End node */ #define OF_DT_PROP 0x3 /* Property: name off, size, diff --git a/arch/powerpc/include/asm/rtas.h b/arch/powerpc/include/asm/rtas.h index 04406162fc5a..75fa0293c508 100644 --- a/arch/powerpc/include/asm/rtas.h +++ b/arch/powerpc/include/asm/rtas.h @@ -515,6 +515,10 @@ extern char rtas_data_buf[RTAS_DATA_BUF_SIZE]; extern unsigned long rtas_rmo_buf; extern struct mutex rtas_ibm_get_vpd_lock; +extern struct mutex rtas_ibm_get_indices_lock; +extern struct mutex rtas_ibm_set_dynamic_indicator_lock; +extern struct mutex rtas_ibm_get_dynamic_sensor_state_lock; +extern struct mutex rtas_ibm_physical_attestation_lock; #define GLOBAL_INTERRUPT_QUEUE 9005 diff --git a/arch/powerpc/include/asm/spu_priv1.h b/arch/powerpc/include/asm/spu_priv1.h index 6fee411d973d..66b111fa1cd1 100644 --- a/arch/powerpc/include/asm/spu_priv1.h +++ b/arch/powerpc/include/asm/spu_priv1.h @@ -215,8 +215,6 @@ spu_disable_spu (struct spu_context *ctx) * and only intended to be used by the platform setup code. */ -extern const struct spu_priv1_ops spu_priv1_mmio_ops; - extern const struct spu_management_ops spu_management_of_ops; #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/static_call.h b/arch/powerpc/include/asm/static_call.h index de1018cc522b..e3d5d3823dac 100644 --- a/arch/powerpc/include/asm/static_call.h +++ b/arch/powerpc/include/asm/static_call.h @@ -26,4 +26,6 @@ #define ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name) __PPC_SCT(name, "blr") #define ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name) __PPC_SCT(name, "b .+20") +#define CALL_INSN_SIZE 4 + #endif /* _ASM_POWERPC_STATIC_CALL_H */ diff --git a/arch/powerpc/include/asm/syscall.h b/arch/powerpc/include/asm/syscall.h index 3dd36c5e334a..4b3c52ed6e9d 100644 --- a/arch/powerpc/include/asm/syscall.h +++ b/arch/powerpc/include/asm/syscall.h @@ -39,6 +39,16 @@ static inline int syscall_get_nr(struct task_struct *task, struct pt_regs *regs) return -1; } +static inline void syscall_set_nr(struct task_struct *task, struct pt_regs *regs, int nr) +{ + /* + * Unlike syscall_get_nr(), syscall_set_nr() can be called only when + * the target task is stopped for tracing on entering syscall, so + * there is no need to have the same check syscall_get_nr() has. + */ + regs->gpr[0] = nr; +} + static inline void syscall_rollback(struct task_struct *task, struct pt_regs *regs) { @@ -110,6 +120,16 @@ static inline void syscall_get_arguments(struct task_struct *task, } } +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + const unsigned long *args) +{ + memcpy(®s->gpr[3], args, 6 * sizeof(args[0])); + + /* Also copy the first argument into orig_gpr3 */ + regs->orig_gpr3 = args[0]; +} + static inline int syscall_get_arch(struct task_struct *task) { if (is_tsk_32bit_task(task)) diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index 6ebca2996f18..2785c7462ebf 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -103,6 +103,7 @@ void arch_setup_new_exec(void); #define TIF_PATCH_PENDING 6 /* pending live patching update */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ #define TIF_SINGLESTEP 8 /* singlestepping active */ +#define TIF_NEED_RESCHED_LAZY 9 /* Scheduler driven lazy preemption */ #define TIF_SECCOMP 10 /* secure computing */ #define TIF_RESTOREALL 11 /* Restore all regs (implies NOERROR) */ #define TIF_NOERROR 12 /* Force successful syscall return */ @@ -122,6 +123,7 @@ void arch_setup_new_exec(void); #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) #define _TIF_SIGPENDING (1<<TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) +#define _TIF_NEED_RESCHED_LAZY (1<<TIF_NEED_RESCHED_LAZY) #define _TIF_NOTIFY_SIGNAL (1<<TIF_NOTIFY_SIGNAL) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) #define _TIF_32BIT (1<<TIF_32BIT) @@ -142,9 +144,10 @@ void arch_setup_new_exec(void); _TIF_SYSCALL_EMU) #define _TIF_USER_WORK_MASK (_TIF_SIGPENDING | _TIF_NEED_RESCHED | \ - _TIF_NOTIFY_RESUME | _TIF_UPROBE | \ - _TIF_RESTORE_TM | _TIF_PATCH_PENDING | \ - _TIF_NOTIFY_SIGNAL) + _TIF_NEED_RESCHED_LAZY | _TIF_NOTIFY_RESUME | \ + _TIF_UPROBE | _TIF_RESTORE_TM | \ + _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL) + #define _TIF_PERSYSCALL_MASK (_TIF_RESTOREALL|_TIF_NOERROR) /* Bits in local_flags */ diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h index 221c8f8ff89b..f8885586efaf 100644 --- a/arch/powerpc/include/asm/time.h +++ b/arch/powerpc/include/asm/time.h @@ -86,12 +86,9 @@ static inline unsigned long tb_ticks_since(unsigned long tstamp) #define mulhdu(x,y) \ ({unsigned long z; asm ("mulhdu %0,%1,%2" : "=r" (z) : "r" (x), "r" (y)); z;}) #else -extern u64 mulhdu(u64, u64); +#define mulhdu(x, y) mul_u64_u64_shr(x, y, 64) #endif -extern void div128_by_32(u64 dividend_high, u64 dividend_low, - unsigned divisor, struct div_result *dr); - extern void secondary_cpu_time_init(void); extern void __init time_init(void); diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h index 1ca7d4c4b90d..2058e8d3e013 100644 --- a/arch/powerpc/include/asm/tlb.h +++ b/arch/powerpc/include/asm/tlb.h @@ -37,6 +37,7 @@ extern void tlb_flush(struct mmu_gather *tlb); */ #define tlb_needs_table_invalidate() radix_enabled() +#define __HAVE_ARCH_TLB_REMOVE_TABLE /* Get the generic bits... */ #include <asm-generic/tlb.h> diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h index 16bacfe8c7a2..da15b5efe807 100644 --- a/arch/powerpc/include/asm/topology.h +++ b/arch/powerpc/include/asm/topology.h @@ -152,6 +152,7 @@ static inline bool topology_is_primary_thread(unsigned int cpu) { return cpu == cpu_first_thread_sibling(cpu); } +#define topology_is_primary_thread topology_is_primary_thread static inline bool topology_smt_thread_allowed(unsigned int cpu) { diff --git a/arch/powerpc/include/asm/vdso.h b/arch/powerpc/include/asm/vdso.h index 8d972bc98b55..1ca23fbfe087 100644 --- a/arch/powerpc/include/asm/vdso.h +++ b/arch/powerpc/include/asm/vdso.h @@ -3,6 +3,7 @@ #define _ASM_POWERPC_VDSO_H #define VDSO_VERSION_STRING LINUX_2.6.15 +#define __VDSO_PAGES 4 #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/vdso/arch_data.h b/arch/powerpc/include/asm/vdso/arch_data.h new file mode 100644 index 000000000000..c240a6b87518 --- /dev/null +++ b/arch/powerpc/include/asm/vdso/arch_data.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2002 Peter Bergner <bergner@vnet.ibm.com>, IBM + * Copyright (C) 2005 Benjamin Herrenschmidy <benh@kernel.crashing.org>, + * IBM Corp. + */ +#ifndef _ASM_POWERPC_VDSO_ARCH_DATA_H +#define _ASM_POWERPC_VDSO_ARCH_DATA_H + +#include <linux/unistd.h> +#include <linux/types.h> + +#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32) + +#ifdef CONFIG_PPC64 + +struct vdso_arch_data { + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ + __u32 dcache_block_size; /* L1 d-cache block size */ + __u32 icache_block_size; /* L1 i-cache block size */ + __u32 dcache_log_block_size; /* L1 d-cache log block size */ + __u32 icache_log_block_size; /* L1 i-cache log block size */ + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ + __u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */ +}; + +#else /* CONFIG_PPC64 */ + +struct vdso_arch_data { + __u64 tb_ticks_per_sec; /* Timebase tics / sec */ + __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ + __u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */ +}; + +#endif /* CONFIG_PPC64 */ + +#endif /* _ASM_POWERPC_VDSO_ARCH_DATA_H */ diff --git a/arch/powerpc/include/asm/vdso/getrandom.h b/arch/powerpc/include/asm/vdso/getrandom.h index 80ce0709725e..067a5396aac6 100644 --- a/arch/powerpc/include/asm/vdso/getrandom.h +++ b/arch/powerpc/include/asm/vdso/getrandom.h @@ -43,20 +43,21 @@ static __always_inline ssize_t getrandom_syscall(void *buffer, size_t len, unsig (unsigned long)len, (unsigned long)flags); } -static __always_inline struct vdso_rng_data *__arch_get_vdso_rng_data(void) +static __always_inline const struct vdso_rng_data *__arch_get_vdso_u_rng_data(void) { - struct vdso_arch_data *data; + struct vdso_rng_data *data; asm ( " bcl 20, 31, .+4 ;" "0: mflr %0 ;" - " addis %0, %0, (_vdso_datapage - 0b)@ha ;" - " addi %0, %0, (_vdso_datapage - 0b)@l ;" + " addis %0, %0, (vdso_u_rng_data - 0b)@ha ;" + " addi %0, %0, (vdso_u_rng_data - 0b)@l ;" : "=r" (data) : : "lr" ); - return &data->rng_data; + return data; } +#define __arch_get_vdso_u_rng_data __arch_get_vdso_u_rng_data ssize_t __c_kernel_getrandom(void *buffer, size_t len, unsigned int flags, void *opaque_state, size_t opaque_len); diff --git a/arch/powerpc/include/asm/vdso/gettimeofday.h b/arch/powerpc/include/asm/vdso/gettimeofday.h index c6390890a60c..99c9d6f43fde 100644 --- a/arch/powerpc/include/asm/vdso/gettimeofday.h +++ b/arch/powerpc/include/asm/vdso/gettimeofday.h @@ -94,22 +94,12 @@ int clock_getres32_fallback(clockid_t _clkid, struct old_timespec32 *_ts) #endif static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return get_tb(); } -const struct vdso_data *__arch_get_vdso_data(void); - -#ifdef CONFIG_TIME_NS -static __always_inline -const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd) -{ - return (void *)vd + (1U << CONFIG_PAGE_SHIFT); -} -#endif - -static inline bool vdso_clocksource_ok(const struct vdso_data *vd) +static inline bool vdso_clocksource_ok(const struct vdso_clock *vc) { return true; } @@ -135,21 +125,22 @@ static __always_inline u64 vdso_shift_ns(u64 ns, unsigned long shift) #ifdef __powerpc64__ int __c_kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd); + const struct vdso_time_data *vd); int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res, - const struct vdso_data *vd); + const struct vdso_time_data *vd); #else int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts, - const struct vdso_data *vd); + const struct vdso_time_data *vd); int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd); + const struct vdso_time_data *vd); int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res, - const struct vdso_data *vd); + const struct vdso_time_data *vd); #endif int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz, - const struct vdso_data *vd); + const struct vdso_time_data *vd); __kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, - const struct vdso_data *vd); + const struct vdso_time_data *vd); + #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/powerpc/include/asm/vdso/vsyscall.h b/arch/powerpc/include/asm/vdso/vsyscall.h index 48560a119559..c2c9ae1b22e7 100644 --- a/arch/powerpc/include/asm/vdso/vsyscall.h +++ b/arch/powerpc/include/asm/vdso/vsyscall.h @@ -6,19 +6,6 @@ #include <asm/vdso_datapage.h> -static __always_inline -struct vdso_data *__arch_get_k_vdso_data(void) -{ - return vdso_data->data; -} -#define __arch_get_k_vdso_data __arch_get_k_vdso_data - -static __always_inline -struct vdso_rng_data *__arch_get_k_vdso_rng_data(void) -{ - return &vdso_data->rng_data; -} - /* The asm-generic header needs to be included after the definitions above */ #include <asm-generic/vdso/vsyscall.h> diff --git a/arch/powerpc/include/asm/vdso_datapage.h b/arch/powerpc/include/asm/vdso_datapage.h index a202f5b63479..95d45a50355d 100644 --- a/arch/powerpc/include/asm/vdso_datapage.h +++ b/arch/powerpc/include/asm/vdso_datapage.h @@ -11,56 +11,18 @@ #ifndef __ASSEMBLY__ -#include <linux/unistd.h> -#include <linux/time.h> #include <vdso/datapage.h> -#define SYSCALL_MAP_SIZE ((NR_syscalls + 31) / 32) - -#ifdef CONFIG_PPC64 - -struct vdso_arch_data { - __u64 tb_ticks_per_sec; /* Timebase tics / sec */ - __u32 dcache_block_size; /* L1 d-cache block size */ - __u32 icache_block_size; /* L1 i-cache block size */ - __u32 dcache_log_block_size; /* L1 d-cache log block size */ - __u32 icache_log_block_size; /* L1 i-cache log block size */ - __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ - __u32 compat_syscall_map[SYSCALL_MAP_SIZE]; /* Map of compat syscalls */ - - struct vdso_rng_data rng_data; - - struct vdso_data data[CS_BASES] __aligned(1 << CONFIG_PAGE_SHIFT); -}; - -#else /* CONFIG_PPC64 */ - -struct vdso_arch_data { - __u64 tb_ticks_per_sec; /* Timebase tics / sec */ - __u32 syscall_map[SYSCALL_MAP_SIZE]; /* Map of syscalls */ - __u32 compat_syscall_map[0]; /* No compat syscalls on PPC32 */ - struct vdso_rng_data rng_data; - - struct vdso_data data[CS_BASES] __aligned(1 << CONFIG_PAGE_SHIFT); -}; - -#endif /* CONFIG_PPC64 */ - -extern struct vdso_arch_data *vdso_data; - #else /* __ASSEMBLY__ */ -.macro get_datapage ptr offset=0 +.macro get_datapage ptr symbol bcl 20, 31, .+4 999: mflr \ptr - addis \ptr, \ptr, (_vdso_datapage - 999b + \offset)@ha - addi \ptr, \ptr, (_vdso_datapage - 999b + \offset)@l + addis \ptr, \ptr, (\symbol - 999b)@ha + addi \ptr, \ptr, (\symbol - 999b)@l .endm -#include <asm/asm-offsets.h> -#include <asm/page.h> - #endif /* __ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/xics.h b/arch/powerpc/include/asm/xics.h index 89090485bec1..60ef312dab05 100644 --- a/arch/powerpc/include/asm/xics.h +++ b/arch/powerpc/include/asm/xics.h @@ -31,7 +31,6 @@ #ifdef CONFIG_PPC_ICP_NATIVE extern int icp_native_init(void); extern void icp_native_flush_interrupt(void); -extern void icp_native_cause_ipi_rm(int cpu); #else static inline int icp_native_init(void) { return -ENODEV; } #endif diff --git a/arch/powerpc/include/asm/xmon.h b/arch/powerpc/include/asm/xmon.h index f2d44b44f46c..535cdb1e411a 100644 --- a/arch/powerpc/include/asm/xmon.h +++ b/arch/powerpc/include/asm/xmon.h @@ -12,13 +12,11 @@ #ifdef CONFIG_XMON extern void xmon_setup(void); -void __init xmon_register_spus(struct list_head *list); struct pt_regs; extern int xmon(struct pt_regs *excp); extern irqreturn_t xmon_irq(int, void *); #else static inline void xmon_setup(void) { } -static inline void xmon_register_spus(struct list_head *list) { } #endif #if defined(CONFIG_XMON) && defined(CONFIG_SMP) diff --git a/arch/powerpc/include/uapi/asm/eeh.h b/arch/powerpc/include/uapi/asm/eeh.h index 28186071fafc..3b5c47ff3fc4 100644 --- a/arch/powerpc/include/uapi/asm/eeh.h +++ b/arch/powerpc/include/uapi/asm/eeh.h @@ -1,18 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * * Copyright IBM Corp. 2015 * * Authors: Gavin Shan <gwshan@linux.vnet.ibm.com> diff --git a/arch/powerpc/include/uapi/asm/ioctls.h b/arch/powerpc/include/uapi/asm/ioctls.h index 2c145da3b774..b5211e413829 100644 --- a/arch/powerpc/include/uapi/asm/ioctls.h +++ b/arch/powerpc/include/uapi/asm/ioctls.h @@ -23,10 +23,10 @@ #define TCSETSW _IOW('t', 21, struct termios) #define TCSETSF _IOW('t', 22, struct termios) -#define TCGETA _IOR('t', 23, struct termio) -#define TCSETA _IOW('t', 24, struct termio) -#define TCSETAW _IOW('t', 25, struct termio) -#define TCSETAF _IOW('t', 28, struct termio) +#define TCGETA 0x40147417 /* _IOR('t', 23, struct termio) */ +#define TCSETA 0x80147418 /* _IOW('t', 24, struct termio) */ +#define TCSETAW 0x80147419 /* _IOW('t', 25, struct termio) */ +#define TCSETAF 0x8014741c /* _IOW('t', 28, struct termio) */ #define TCSBRK _IO('t', 29) #define TCXONC _IO('t', 30) diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h index eaeda001784e..077c5437f521 100644 --- a/arch/powerpc/include/uapi/asm/kvm.h +++ b/arch/powerpc/include/uapi/asm/kvm.h @@ -1,18 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * * Copyright IBM Corp. 2007 * * Authors: Hollis Blanchard <hollisb@us.ibm.com> diff --git a/arch/powerpc/include/uapi/asm/kvm_para.h b/arch/powerpc/include/uapi/asm/kvm_para.h index a809b1b44ddf..ac596064d4c7 100644 --- a/arch/powerpc/include/uapi/asm/kvm_para.h +++ b/arch/powerpc/include/uapi/asm/kvm_para.h @@ -1,18 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * * Copyright IBM Corp. 2008 * * Authors: Hollis Blanchard <hollisb@us.ibm.com> diff --git a/arch/powerpc/include/uapi/asm/papr-indices.h b/arch/powerpc/include/uapi/asm/papr-indices.h new file mode 100644 index 000000000000..c2999d89d52a --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-indices.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_INDICES_H_ +#define _UAPI_PAPR_INDICES_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +#define LOC_CODE_SIZE 80 +#define RTAS_GET_INDICES_BUF_SIZE SZ_4K + +struct papr_indices_io_block { + union { + struct { + __u8 is_sensor; /* 0 for indicator and 1 for sensor */ + __u32 indice_type; + } indices; + struct { + __u32 token; /* Sensor or indicator token */ + __u32 state; /* get / set state */ + /* + * PAPR+ 12.3.2.4 Converged Location Code Rules - Length + * Restrictions. 79 characters plus null. + */ + char location_code_str[LOC_CODE_SIZE]; /* location code */ + } dynamic_param; + }; +}; + +/* + * ioctls for /dev/papr-indices. + * PAPR_INDICES_IOC_GET: Returns a get-indices handle fd to read data + * PAPR_DYNAMIC_SENSOR_IOC_GET: Gets the state of the input sensor + * PAPR_DYNAMIC_INDICATOR_IOC_SET: Sets the new state for the input indicator + */ +#define PAPR_INDICES_IOC_GET _IOW(PAPR_MISCDEV_IOC_ID, 3, struct papr_indices_io_block) +#define PAPR_DYNAMIC_SENSOR_IOC_GET _IOWR(PAPR_MISCDEV_IOC_ID, 4, struct papr_indices_io_block) +#define PAPR_DYNAMIC_INDICATOR_IOC_SET _IOW(PAPR_MISCDEV_IOC_ID, 5, struct papr_indices_io_block) + + +#endif /* _UAPI_PAPR_INDICES_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-physical-attestation.h b/arch/powerpc/include/uapi/asm/papr-physical-attestation.h new file mode 100644 index 000000000000..ea746837bb9a --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-physical-attestation.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_PHYSICAL_ATTESTATION_H_ +#define _UAPI_PAPR_PHYSICAL_ATTESTATION_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +#define PAPR_PHYATTEST_MAX_INPUT 4084 /* Max 4K buffer: 4K-12 */ + +/* + * Defined in PAPR 2.13+ 21.6 Attestation Command Structures. + * User space pass this struct and the max size should be 4K. + */ +struct papr_phy_attest_io_block { + __u8 version; + __u8 command; + __u8 TCG_major_ver; + __u8 TCG_minor_ver; + __be32 length; + __be32 correlator; + __u8 payload[PAPR_PHYATTEST_MAX_INPUT]; +}; + +/* + * ioctl for /dev/papr-physical-attestation. Returns a attestation + * command fd handle + */ +#define PAPR_PHY_ATTEST_IOC_HANDLE _IOW(PAPR_MISCDEV_IOC_ID, 8, struct papr_phy_attest_io_block) + +#endif /* _UAPI_PAPR_PHYSICAL_ATTESTATION_H_ */ diff --git a/arch/powerpc/include/uapi/asm/papr-platform-dump.h b/arch/powerpc/include/uapi/asm/papr-platform-dump.h new file mode 100644 index 000000000000..8a1c060e89a9 --- /dev/null +++ b/arch/powerpc/include/uapi/asm/papr-platform-dump.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_PAPR_PLATFORM_DUMP_H_ +#define _UAPI_PAPR_PLATFORM_DUMP_H_ + +#include <linux/types.h> +#include <asm/ioctl.h> +#include <asm/papr-miscdev.h> + +/* + * ioctl for /dev/papr-platform-dump. Returns a platform-dump handle fd + * corresponding to dump tag. + */ +#define PAPR_PLATFORM_DUMP_IOC_CREATE_HANDLE _IOW(PAPR_MISCDEV_IOC_ID, 6, __u64) +#define PAPR_PLATFORM_DUMP_IOC_INVALIDATE _IOW(PAPR_MISCDEV_IOC_ID, 7, __u64) + +#endif /* _UAPI_PAPR_PLATFORM_DUMP_H_ */ diff --git a/arch/powerpc/include/uapi/asm/ps3fb.h b/arch/powerpc/include/uapi/asm/ps3fb.h index fd7e3a0d35d5..b1c6b0cd9e80 100644 --- a/arch/powerpc/include/uapi/asm/ps3fb.h +++ b/arch/powerpc/include/uapi/asm/ps3fb.h @@ -2,19 +2,6 @@ /* * Copyright (C) 2006 Sony Computer Entertainment Inc. * Copyright 2006, 2007 Sony Corporation - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published - * by the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #ifndef _ASM_POWERPC_PS3FB_H_ diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index f43c1198768c..fb2b95267022 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -70,7 +70,7 @@ obj-y := cputable.o syscalls.o switch.o \ signal.o sysfs.o cacheinfo.o time.o \ prom.o traps.o setup-common.o \ udbg.o misc.o io.o misc_$(BITS).o \ - of_platform.o prom_parse.o firmware.o \ + prom_parse.o firmware.o \ hw_breakpoint_constraints.o interrupt.o \ kdebugfs.o stacktrace.o syscall.o obj-y += ptrace/ @@ -126,7 +126,7 @@ obj-$(CONFIG_PPC_BOOK3S_32) += head_book3s_32.o obj-$(CONFIG_44x) += head_44x.o obj-$(CONFIG_PPC_8xx) += head_8xx.o obj-$(CONFIG_PPC_85xx) += head_85xx.o -extra-y += vmlinux.lds +always-$(KBUILD_BUILTIN) += vmlinux.lds obj-$(CONFIG_RELOCATABLE) += reloc_$(BITS).o @@ -152,8 +152,6 @@ obj-$(CONFIG_PCI_MSI) += msi.o obj-$(CONFIG_AUDIT) += audit.o obj64-$(CONFIG_AUDIT) += compat_audit.o -obj-$(CONFIG_PPC_IO_WORKAROUNDS) += io-workarounds.o - obj-y += trace/ ifneq ($(CONFIG_PPC_INDIRECT_PIO),y) @@ -162,9 +160,7 @@ endif obj64-$(CONFIG_PPC_TRANSACTIONAL_MEM) += tm.o -ifneq ($(CONFIG_XMON)$(CONFIG_KEXEC_CORE)(CONFIG_PPC_BOOK3S),) obj-y += ppc_save_regs.o -endif obj-$(CONFIG_EPAPR_PARAVIRT) += epapr_paravirt.o epapr_hcalls.o obj-$(CONFIG_KVM_GUEST) += kvm.o kvm_emul.o diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 7a390bd4f4af..b3048f6d3822 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -334,7 +334,6 @@ int main(void) #endif /* ! CONFIG_PPC64 */ /* datapage offsets for use by vdso */ - OFFSET(VDSO_DATA_OFFSET, vdso_arch_data, data); OFFSET(CFG_TB_TICKS_PER_SEC, vdso_arch_data, tb_ticks_per_sec); #ifdef CONFIG_PPC64 OFFSET(CFG_ICACHE_BLOCKSZ, vdso_arch_data, icache_block_size); diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index f0ae39e77e37..4d64a5db50f3 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -136,7 +136,7 @@ static bool dma_iommu_bypass_supported(struct device *dev, u64 mask) struct pci_dev *pdev = to_pci_dev(dev); struct pci_controller *phb = pci_bus_to_host(pdev->bus); - if (iommu_fixed_is_weak || !phb->controller_ops.iommu_bypass_supported) + if (!phb->controller_ops.iommu_bypass_supported) return false; return phb->controller_ops.iommu_bypass_supported(pdev, mask); } diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 1bee15c013e7..3af6c06af02f 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -1087,12 +1087,10 @@ static int __init dt_cpu_ftrs_scan_callback(unsigned long node, const char /* Count and allocate space for cpu features */ of_scan_flat_dt_subnodes(node, count_cpufeatures_subnodes, &nr_dt_cpu_features); - dt_cpu_features = memblock_alloc(sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, PAGE_SIZE); - if (!dt_cpu_features) - panic("%s: Failed to allocate %zu bytes align=0x%lx\n", - __func__, - sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, - PAGE_SIZE); + dt_cpu_features = + memblock_alloc_or_panic( + sizeof(struct dt_cpu_feature) * nr_dt_cpu_features, + PAGE_SIZE); cpufeatures_setup_start(isa); diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 83fe99861eb1..bb836f02101c 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1139,6 +1139,7 @@ int eeh_unfreeze_pe(struct eeh_pe *pe) return ret; } +EXPORT_SYMBOL_GPL(eeh_unfreeze_pe); static struct pci_device_id eeh_reset_ids[] = { @@ -1208,16 +1209,16 @@ int eeh_dev_open(struct pci_dev *pdev) struct eeh_dev *edev; int ret = -ENODEV; - mutex_lock(&eeh_dev_mutex); + guard(mutex)(&eeh_dev_mutex); /* No PCI device ? */ if (!pdev) - goto out; + return ret; /* No EEH device or PE ? */ edev = pci_dev_to_eeh_dev(pdev); if (!edev || !edev->pe) - goto out; + return ret; /* * The PE might have been put into frozen state, but we @@ -1227,16 +1228,12 @@ int eeh_dev_open(struct pci_dev *pdev) */ ret = eeh_pe_change_owner(edev->pe); if (ret) - goto out; + return ret; /* Increase PE's pass through count */ atomic_inc(&edev->pe->pass_dev_cnt); - mutex_unlock(&eeh_dev_mutex); return 0; -out: - mutex_unlock(&eeh_dev_mutex); - return ret; } EXPORT_SYMBOL_GPL(eeh_dev_open); @@ -1252,22 +1249,20 @@ void eeh_dev_release(struct pci_dev *pdev) { struct eeh_dev *edev; - mutex_lock(&eeh_dev_mutex); + guard(mutex)(&eeh_dev_mutex); /* No PCI device ? */ if (!pdev) - goto out; + return; /* No EEH device ? */ edev = pci_dev_to_eeh_dev(pdev); if (!edev || !edev->pe || !eeh_pe_passed(edev->pe)) - goto out; + return; /* Decrease PE's pass through count */ WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0); eeh_pe_change_owner(edev->pe); -out: - mutex_unlock(&eeh_dev_mutex); } EXPORT_SYMBOL(eeh_dev_release); @@ -1509,6 +1504,8 @@ int eeh_pe_configure(struct eeh_pe *pe) /* Invalid PE ? */ if (!pe) return -ENODEV; + else + ret = eeh_ops->configure_bridge(pe); return ret; } diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 7efe04c68f0f..48ad0116f359 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -257,13 +257,12 @@ static void eeh_pe_report_edev(struct eeh_dev *edev, eeh_report_fn fn, struct pci_driver *driver; enum pci_ers_result new_result; - pci_lock_rescan_remove(); pdev = edev->pdev; if (pdev) get_device(&pdev->dev); - pci_unlock_rescan_remove(); if (!pdev) { eeh_edev_info(edev, "no device"); + *result = PCI_ERS_RESULT_DISCONNECT; return; } device_lock(&pdev->dev); @@ -304,8 +303,9 @@ static void eeh_pe_report(const char *name, struct eeh_pe *root, struct eeh_dev *edev, *tmp; pr_info("EEH: Beginning: '%s'\n", name); - eeh_for_each_pe(root, pe) eeh_pe_for_each_dev(pe, edev, tmp) - eeh_pe_report_edev(edev, fn, result); + eeh_for_each_pe(root, pe) + eeh_pe_for_each_dev(pe, edev, tmp) + eeh_pe_report_edev(edev, fn, result); if (result) pr_info("EEH: Finished:'%s' with aggregate recovery state:'%s'\n", name, pci_ers_result_name(*result)); @@ -383,6 +383,8 @@ static void eeh_dev_restore_state(struct eeh_dev *edev, void *userdata) if (!edev) return; + pci_lock_rescan_remove(); + /* * The content in the config space isn't saved because * the blocked config space on some adapters. We have @@ -393,14 +395,19 @@ static void eeh_dev_restore_state(struct eeh_dev *edev, void *userdata) if (list_is_last(&edev->entry, &edev->pe->edevs)) eeh_pe_restore_bars(edev->pe); + pci_unlock_rescan_remove(); return; } pdev = eeh_dev_to_pci_dev(edev); - if (!pdev) + if (!pdev) { + pci_unlock_rescan_remove(); return; + } pci_restore_state(pdev); + + pci_unlock_rescan_remove(); } /** @@ -647,9 +654,7 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, if (any_passed || driver_eeh_aware || (pe->type & EEH_PE_VF)) { eeh_pe_dev_traverse(pe, eeh_rmv_device, rmv_data); } else { - pci_lock_rescan_remove(); pci_hp_remove_devices(bus); - pci_unlock_rescan_remove(); } /* @@ -665,8 +670,6 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, if (rc) return rc; - pci_lock_rescan_remove(); - /* Restore PE */ eeh_ops->configure_bridge(pe); eeh_pe_restore_bars(pe); @@ -674,7 +677,6 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, /* Clear frozen state */ rc = eeh_clear_pe_frozen_state(pe, false); if (rc) { - pci_unlock_rescan_remove(); return rc; } @@ -709,7 +711,6 @@ static int eeh_reset_device(struct eeh_pe *pe, struct pci_bus *bus, pe->tstamp = tstamp; pe->freeze_count = cnt; - pci_unlock_rescan_remove(); return 0; } @@ -843,10 +844,13 @@ void eeh_handle_normal_event(struct eeh_pe *pe) {LIST_HEAD_INIT(rmv_data.removed_vf_list), 0}; int devices = 0; + pci_lock_rescan_remove(); + bus = eeh_pe_bus_get(pe); if (!bus) { pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n", __func__, pe->phb->global_number, pe->addr); + pci_unlock_rescan_remove(); return; } @@ -907,7 +911,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) /* FIXME: Use the same format as dump_stack() */ pr_err("EEH: Call Trace:\n"); for (i = 0; i < pe->trace_entries; i++) - pr_err("EEH: [%pK] %pS\n", ptrs[i], ptrs[i]); + pr_err("EEH: [%p] %pS\n", ptrs[i], ptrs[i]); pe->trace_entries = 0; } @@ -1094,10 +1098,15 @@ recover_failed: eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - pci_lock_rescan_remove(); - pci_hp_remove_devices(bus); - pci_unlock_rescan_remove(); + bus = eeh_pe_bus_get(pe); + if (bus) + pci_hp_remove_devices(bus); + else + pr_err("%s: PCI bus for PHB#%x-PE#%x disappeared\n", + __func__, pe->phb->global_number, pe->addr); + /* The passed PE should no longer be used */ + pci_unlock_rescan_remove(); return; } @@ -1114,6 +1123,8 @@ out: eeh_clear_slot_attention(edev->pdev); eeh_pe_state_clear(pe, EEH_PE_RECOVERING, true); + + pci_unlock_rescan_remove(); } /** @@ -1132,6 +1143,7 @@ void eeh_handle_special_event(void) unsigned long flags; int rc; + pci_lock_rescan_remove(); do { rc = eeh_ops->next_error(&pe); @@ -1171,10 +1183,12 @@ void eeh_handle_special_event(void) break; case EEH_NEXT_ERR_NONE: + pci_unlock_rescan_remove(); return; default: pr_warn("%s: Invalid value %d from next_error()\n", __func__, rc); + pci_unlock_rescan_remove(); return; } @@ -1186,7 +1200,9 @@ void eeh_handle_special_event(void) if (rc == EEH_NEXT_ERR_FROZEN_PE || rc == EEH_NEXT_ERR_FENCED_PHB) { eeh_pe_state_mark(pe, EEH_PE_RECOVERING); + pci_unlock_rescan_remove(); eeh_handle_normal_event(pe); + pci_lock_rescan_remove(); } else { eeh_for_each_pe(pe, tmp_pe) eeh_pe_for_each_dev(tmp_pe, edev, tmp_edev) @@ -1199,7 +1215,6 @@ void eeh_handle_special_event(void) eeh_report_failure, NULL); eeh_set_channel_state(pe, pci_channel_io_perm_failure); - pci_lock_rescan_remove(); list_for_each_entry(hose, &hose_list, list_node) { phb_pe = eeh_phb_pe_get(hose); if (!phb_pe || @@ -1218,7 +1233,6 @@ void eeh_handle_special_event(void) } pci_hp_remove_devices(bus); } - pci_unlock_rescan_remove(); } /* @@ -1228,4 +1242,6 @@ void eeh_handle_special_event(void) if (rc == EEH_NEXT_ERR_DEAD_IOC) break; } while (rc != EEH_NEXT_ERR_NONE); + + pci_unlock_rescan_remove(); } diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index d283d281d28e..e740101fadf3 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -671,10 +671,12 @@ static void eeh_bridge_check_link(struct eeh_dev *edev) eeh_ops->write_config(edev, cap + PCI_EXP_LNKCTL, 2, val); /* Check link */ - if (!edev->pdev->link_active_reporting) { - eeh_edev_dbg(edev, "No link reporting capability\n"); - msleep(1000); - return; + if (edev->pdev) { + if (!edev->pdev->link_active_reporting) { + eeh_edev_dbg(edev, "No link reporting capability\n"); + msleep(1000); + return; + } } /* Wait the link is up until timeout (5s) */ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 195b075d116c..b7229430ca94 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -2537,27 +2537,8 @@ EXC_REAL_NONE(0x1000, 0x100) EXC_VIRT_NONE(0x5000, 0x100) EXC_REAL_NONE(0x1100, 0x100) EXC_VIRT_NONE(0x5100, 0x100) - -#ifdef CONFIG_CBE_RAS -INT_DEFINE_BEGIN(cbe_system_error) - IVEC=0x1200 - IHSRR=1 -INT_DEFINE_END(cbe_system_error) - -EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100) - GEN_INT_ENTRY cbe_system_error, virt=0 -EXC_REAL_END(cbe_system_error, 0x1200, 0x100) -EXC_VIRT_NONE(0x5200, 0x100) -EXC_COMMON_BEGIN(cbe_system_error_common) - GEN_COMMON cbe_system_error - addi r3,r1,STACK_INT_FRAME_REGS - bl CFUNC(cbe_system_error_exception) - b interrupt_return_hsrr - -#else /* CONFIG_CBE_RAS */ EXC_REAL_NONE(0x1200, 0x100) EXC_VIRT_NONE(0x5200, 0x100) -#endif /** * Interrupt 0x1300 - Instruction Address Breakpoint Interrupt. @@ -2708,26 +2689,8 @@ EXC_COMMON_BEGIN(denorm_exception_common) b interrupt_return_hsrr -#ifdef CONFIG_CBE_RAS -INT_DEFINE_BEGIN(cbe_maintenance) - IVEC=0x1600 - IHSRR=1 -INT_DEFINE_END(cbe_maintenance) - -EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100) - GEN_INT_ENTRY cbe_maintenance, virt=0 -EXC_REAL_END(cbe_maintenance, 0x1600, 0x100) -EXC_VIRT_NONE(0x5600, 0x100) -EXC_COMMON_BEGIN(cbe_maintenance_common) - GEN_COMMON cbe_maintenance - addi r3,r1,STACK_INT_FRAME_REGS - bl CFUNC(cbe_maintenance_exception) - b interrupt_return_hsrr - -#else /* CONFIG_CBE_RAS */ EXC_REAL_NONE(0x1600, 0x100) EXC_VIRT_NONE(0x5600, 0x100) -#endif INT_DEFINE_BEGIN(altivec_assist) @@ -2755,26 +2718,8 @@ EXC_COMMON_BEGIN(altivec_assist_common) b interrupt_return_srr -#ifdef CONFIG_CBE_RAS -INT_DEFINE_BEGIN(cbe_thermal) - IVEC=0x1800 - IHSRR=1 -INT_DEFINE_END(cbe_thermal) - -EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100) - GEN_INT_ENTRY cbe_thermal, virt=0 -EXC_REAL_END(cbe_thermal, 0x1800, 0x100) -EXC_VIRT_NONE(0x5800, 0x100) -EXC_COMMON_BEGIN(cbe_thermal_common) - GEN_COMMON cbe_thermal - addi r3,r1,STACK_INT_FRAME_REGS - bl CFUNC(cbe_thermal_exception) - b interrupt_return_hsrr - -#else /* CONFIG_CBE_RAS */ EXC_REAL_NONE(0x1800, 0x100) EXC_VIRT_NONE(0x5800, 0x100) -#endif #ifdef CONFIG_PPC_WATCHDOG diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c index 4b371c738213..5782e743fd27 100644 --- a/arch/powerpc/kernel/fadump.c +++ b/arch/powerpc/kernel/fadump.c @@ -33,6 +33,7 @@ #include <asm/fadump-internal.h> #include <asm/setup.h> #include <asm/interrupt.h> +#include <asm/prom.h> /* * The CPU who acquired the lock to trigger the fadump crash should @@ -289,10 +290,8 @@ static void __init fadump_show_config(void) if (!fw_dump.fadump_supported) return; - pr_debug("Fadump enabled : %s\n", - (fw_dump.fadump_enabled ? "yes" : "no")); - pr_debug("Dump Active : %s\n", - (fw_dump.dump_active ? "yes" : "no")); + pr_debug("Fadump enabled : %s\n", str_yes_no(fw_dump.fadump_enabled)); + pr_debug("Dump Active : %s\n", str_yes_no(fw_dump.dump_active)); pr_debug("Dump section sizes:\n"); pr_debug(" CPU state data size: %lx\n", fw_dump.cpu_state_data_size); pr_debug(" HPTE region size : %lx\n", fw_dump.hpte_region_size); @@ -334,7 +333,7 @@ static __init u64 fadump_calculate_reserve_size(void) * memory at a predefined offset. */ ret = parse_crashkernel(boot_command_line, memblock_phys_mem_size(), - &size, &base, NULL, NULL); + &size, &base, NULL, NULL, NULL); if (ret == 0 && size > 0) { unsigned long max_size; @@ -751,7 +750,7 @@ u32 *__init fadump_regs_to_elf_notes(u32 *buf, struct pt_regs *regs) * prstatus.pr_pid = ???? */ elf_core_copy_regs(&prstatus.pr_reg, regs); - buf = append_elf_note(buf, CRASH_CORE_NOTE_NAME, NT_PRSTATUS, + buf = append_elf_note(buf, NN_PRSTATUS, NT_PRSTATUS, &prstatus, sizeof(prstatus)); return buf; } @@ -1374,15 +1373,12 @@ static void fadump_free_elfcorehdr_buf(void) static void fadump_invalidate_release_mem(void) { - mutex_lock(&fadump_mutex); - if (!fw_dump.dump_active) { - mutex_unlock(&fadump_mutex); - return; + scoped_guard(mutex, &fadump_mutex) { + if (!fw_dump.dump_active) + return; + fadump_cleanup(); } - fadump_cleanup(); - mutex_unlock(&fadump_mutex); - fadump_free_elfcorehdr_buf(); fadump_release_memory(fw_dump.boot_mem_top, memblock_end_of_DRAM()); fadump_free_cpu_notes_buf(); @@ -1764,19 +1760,19 @@ void __init fadump_setup_param_area(void) range_end = memblock_end_of_DRAM(); } else { /* - * Passing additional parameters is supported for hash MMU only - * if the first memory block size is 768MB or higher. + * Memory range for passing additional parameters for HASH MMU + * must meet the following conditions: + * 1. The first memory block size must be higher than the + * minimum RMA (MIN_RMA) size. Bootloader can use memory + * upto RMA size. So it should be avoided. + * 2. The range should be between MIN_RMA and RMA size (ppc64_rma_size) + * 3. It must not overlap with the fadump reserved area. */ - if (ppc64_rma_size < 0x30000000) + if (ppc64_rma_size < MIN_RMA*1024*1024) return; - /* - * 640 MB to 768 MB is not used by PFW/bootloader. So, try reserving - * memory for passing additional parameters in this range to avoid - * being stomped on by PFW/bootloader. - */ - range_start = 0x2A000000; - range_end = range_start + 0x4000000; + range_start = MIN_RMA * 1024 * 1024; + range_end = min(ppc64_rma_size, fw_dump.boot_mem_top); } fw_dump.param_area = memblock_phys_alloc_range(COMMAND_LINE_SIZE, diff --git a/arch/powerpc/kernel/idle.c b/arch/powerpc/kernel/idle.c index 30b56c67fa61..e527cd3ef128 100644 --- a/arch/powerpc/kernel/idle.c +++ b/arch/powerpc/kernel/idle.c @@ -97,7 +97,7 @@ void power4_idle(void) /* * Register the sysctl to set/clear powersave_nap. */ -static struct ctl_table powersave_nap_ctl_table[] = { +static const struct ctl_table powersave_nap_ctl_table[] = { { .procname = "powersave-nap", .data = &powersave_nap, diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index af62ec974b97..e0c681d0b076 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -25,6 +25,10 @@ unsigned long global_dbcr0[NR_CPUS]; #endif +#if defined(CONFIG_PREEMPT_DYNAMIC) +DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); +#endif + #ifdef CONFIG_PPC_BOOK3S_64 DEFINE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant); static inline bool exit_must_hard_disable(void) @@ -185,7 +189,7 @@ again: ti_flags = read_thread_flags(); while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) { local_irq_enable(); - if (ti_flags & _TIF_NEED_RESCHED) { + if (ti_flags & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) { schedule(); } else { /* @@ -396,7 +400,7 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) /* Returning to a kernel context with local irqs enabled. */ WARN_ON_ONCE(!(regs->msr & MSR_EE)); again: - if (IS_ENABLED(CONFIG_PREEMPT)) { + if (need_irq_preemption()) { /* Return to preemptible kernel context */ if (unlikely(read_thread_flags() & _TIF_NEED_RESCHED)) { if (preempt_count() == 0) diff --git a/arch/powerpc/kernel/io-workarounds.c b/arch/powerpc/kernel/io-workarounds.c deleted file mode 100644 index c877f074d174..000000000000 --- a/arch/powerpc/kernel/io-workarounds.c +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Support PCI IO workaround - * - * Copyright (C) 2006 Benjamin Herrenschmidt <benh@kernel.crashing.org> - * IBM, Corp. - * (C) Copyright 2007-2008 TOSHIBA CORPORATION - */ -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/sched/mm.h> /* for init_mm */ -#include <linux/pgtable.h> - -#include <asm/io.h> -#include <asm/machdep.h> -#include <asm/ppc-pci.h> -#include <asm/io-workarounds.h> -#include <asm/pte-walk.h> - - -#define IOWA_MAX_BUS 8 - -static struct iowa_bus iowa_busses[IOWA_MAX_BUS]; -static unsigned int iowa_bus_count; - -static struct iowa_bus *iowa_pci_find(unsigned long vaddr, unsigned long paddr) -{ - int i, j; - struct resource *res; - unsigned long vstart, vend; - - for (i = 0; i < iowa_bus_count; i++) { - struct iowa_bus *bus = &iowa_busses[i]; - struct pci_controller *phb = bus->phb; - - if (vaddr) { - vstart = (unsigned long)phb->io_base_virt; - vend = vstart + phb->pci_io_size - 1; - if ((vaddr >= vstart) && (vaddr <= vend)) - return bus; - } - - if (paddr) - for (j = 0; j < 3; j++) { - res = &phb->mem_resources[j]; - if (paddr >= res->start && paddr <= res->end) - return bus; - } - } - - return NULL; -} - -#ifdef CONFIG_PPC_INDIRECT_MMIO -struct iowa_bus *iowa_mem_find_bus(const PCI_IO_ADDR addr) -{ - struct iowa_bus *bus; - int token; - - token = PCI_GET_ADDR_TOKEN(addr); - - if (token && token <= iowa_bus_count) - bus = &iowa_busses[token - 1]; - else { - unsigned long vaddr, paddr; - - vaddr = (unsigned long)PCI_FIX_ADDR(addr); - if (vaddr < PHB_IO_BASE || vaddr >= PHB_IO_END) - return NULL; - - paddr = ppc_find_vmap_phys(vaddr); - - bus = iowa_pci_find(vaddr, paddr); - - if (bus == NULL) - return NULL; - } - - return bus; -} -#else /* CONFIG_PPC_INDIRECT_MMIO */ -struct iowa_bus *iowa_mem_find_bus(const PCI_IO_ADDR addr) -{ - return NULL; -} -#endif /* !CONFIG_PPC_INDIRECT_MMIO */ - -#ifdef CONFIG_PPC_INDIRECT_PIO -struct iowa_bus *iowa_pio_find_bus(unsigned long port) -{ - unsigned long vaddr = (unsigned long)pci_io_base + port; - return iowa_pci_find(vaddr, 0); -} -#else -struct iowa_bus *iowa_pio_find_bus(unsigned long port) -{ - return NULL; -} -#endif - -#define DEF_PCI_AC_RET(name, ret, at, al, space, aa) \ -static ret iowa_##name at \ -{ \ - struct iowa_bus *bus; \ - bus = iowa_##space##_find_bus(aa); \ - if (bus && bus->ops && bus->ops->name) \ - return bus->ops->name al; \ - return __do_##name al; \ -} - -#define DEF_PCI_AC_NORET(name, at, al, space, aa) \ -static void iowa_##name at \ -{ \ - struct iowa_bus *bus; \ - bus = iowa_##space##_find_bus(aa); \ - if (bus && bus->ops && bus->ops->name) { \ - bus->ops->name al; \ - return; \ - } \ - __do_##name al; \ -} - -#include <asm/io-defs.h> - -#undef DEF_PCI_AC_RET -#undef DEF_PCI_AC_NORET - -static const struct ppc_pci_io iowa_pci_io = { - -#define DEF_PCI_AC_RET(name, ret, at, al, space, aa) .name = iowa_##name, -#define DEF_PCI_AC_NORET(name, at, al, space, aa) .name = iowa_##name, - -#include <asm/io-defs.h> - -#undef DEF_PCI_AC_RET -#undef DEF_PCI_AC_NORET - -}; - -#ifdef CONFIG_PPC_INDIRECT_MMIO -void __iomem *iowa_ioremap(phys_addr_t addr, unsigned long size, - pgprot_t prot, void *caller) -{ - struct iowa_bus *bus; - void __iomem *res = __ioremap_caller(addr, size, prot, caller); - int busno; - - bus = iowa_pci_find(0, (unsigned long)addr); - if (bus != NULL) { - busno = bus - iowa_busses; - PCI_SET_ADDR_TOKEN(res, busno + 1); - } - return res; -} -#endif /* !CONFIG_PPC_INDIRECT_MMIO */ - -bool io_workaround_inited; - -/* Enable IO workaround */ -static void io_workaround_init(void) -{ - if (io_workaround_inited) - return; - ppc_pci_io = iowa_pci_io; - io_workaround_inited = true; -} - -/* Register new bus to support workaround */ -void iowa_register_bus(struct pci_controller *phb, struct ppc_pci_io *ops, - int (*initfunc)(struct iowa_bus *, void *), void *data) -{ - struct iowa_bus *bus; - struct device_node *np = phb->dn; - - io_workaround_init(); - - if (iowa_bus_count >= IOWA_MAX_BUS) { - pr_err("IOWA:Too many pci bridges, " - "workarounds disabled for %pOF\n", np); - return; - } - - bus = &iowa_busses[iowa_bus_count]; - bus->phb = phb; - bus->ops = ops; - bus->private = data; - - if (initfunc) - if ((*initfunc)(bus, data)) - return; - - iowa_bus_count++; - - pr_debug("IOWA:[%d]Add bus, %pOF.\n", iowa_bus_count-1, np); -} - diff --git a/arch/powerpc/kernel/io.c b/arch/powerpc/kernel/io.c index 6af535905984..bcc201c01514 100644 --- a/arch/powerpc/kernel/io.c +++ b/arch/powerpc/kernel/io.c @@ -31,13 +31,14 @@ void _insb(const volatile u8 __iomem *port, void *buf, long count) if (unlikely(count <= 0)) return; - asm volatile("sync"); + + mb(); do { tmp = *(const volatile u8 __force *)port; eieio(); *tbuf++ = tmp; } while (--count != 0); - asm volatile("twi 0,%0,0; isync" : : "r" (tmp)); + data_barrier(tmp); } EXPORT_SYMBOL(_insb); @@ -47,75 +48,80 @@ void _outsb(volatile u8 __iomem *port, const void *buf, long count) if (unlikely(count <= 0)) return; - asm volatile("sync"); + + mb(); do { *(volatile u8 __force *)port = *tbuf++; } while (--count != 0); - asm volatile("sync"); + mb(); } EXPORT_SYMBOL(_outsb); -void _insw_ns(const volatile u16 __iomem *port, void *buf, long count) +void _insw(const volatile u16 __iomem *port, void *buf, long count) { u16 *tbuf = buf; u16 tmp; if (unlikely(count <= 0)) return; - asm volatile("sync"); + + mb(); do { tmp = *(const volatile u16 __force *)port; eieio(); *tbuf++ = tmp; } while (--count != 0); - asm volatile("twi 0,%0,0; isync" : : "r" (tmp)); + data_barrier(tmp); } -EXPORT_SYMBOL(_insw_ns); +EXPORT_SYMBOL(_insw); -void _outsw_ns(volatile u16 __iomem *port, const void *buf, long count) +void _outsw(volatile u16 __iomem *port, const void *buf, long count) { const u16 *tbuf = buf; if (unlikely(count <= 0)) return; - asm volatile("sync"); + + mb(); do { *(volatile u16 __force *)port = *tbuf++; } while (--count != 0); - asm volatile("sync"); + mb(); } -EXPORT_SYMBOL(_outsw_ns); +EXPORT_SYMBOL(_outsw); -void _insl_ns(const volatile u32 __iomem *port, void *buf, long count) +void _insl(const volatile u32 __iomem *port, void *buf, long count) { u32 *tbuf = buf; u32 tmp; if (unlikely(count <= 0)) return; - asm volatile("sync"); + + mb(); do { tmp = *(const volatile u32 __force *)port; eieio(); *tbuf++ = tmp; } while (--count != 0); - asm volatile("twi 0,%0,0; isync" : : "r" (tmp)); + data_barrier(tmp); } -EXPORT_SYMBOL(_insl_ns); +EXPORT_SYMBOL(_insl); -void _outsl_ns(volatile u32 __iomem *port, const void *buf, long count) +void _outsl(volatile u32 __iomem *port, const void *buf, long count) { const u32 *tbuf = buf; if (unlikely(count <= 0)) return; - asm volatile("sync"); + + mb(); do { *(volatile u32 __force *)port = *tbuf++; } while (--count != 0); - asm volatile("sync"); + mb(); } -EXPORT_SYMBOL(_outsl_ns); +EXPORT_SYMBOL(_outsl); #define IO_CHECK_ALIGN(v,a) ((((unsigned long)(v)) & ((a) - 1)) == 0) @@ -127,7 +133,7 @@ _memset_io(volatile void __iomem *addr, int c, unsigned long n) lc |= lc << 8; lc |= lc << 16; - __asm__ __volatile__ ("sync" : : : "memory"); + mb(); while(n && !IO_CHECK_ALIGN(p, 4)) { *((volatile u8 *)p) = c; p++; @@ -143,7 +149,7 @@ _memset_io(volatile void __iomem *addr, int c, unsigned long n) p++; n--; } - __asm__ __volatile__ ("sync" : : : "memory"); + mb(); } EXPORT_SYMBOL(_memset_io); @@ -152,7 +158,7 @@ void _memcpy_fromio(void *dest, const volatile void __iomem *src, { void *vsrc = (void __force *) src; - __asm__ __volatile__ ("sync" : : : "memory"); + mb(); while(n && (!IO_CHECK_ALIGN(vsrc, 4) || !IO_CHECK_ALIGN(dest, 4))) { *((u8 *)dest) = *((volatile u8 *)vsrc); eieio(); @@ -174,7 +180,7 @@ void _memcpy_fromio(void *dest, const volatile void __iomem *src, dest++; n--; } - __asm__ __volatile__ ("sync" : : : "memory"); + mb(); } EXPORT_SYMBOL(_memcpy_fromio); @@ -182,7 +188,7 @@ void _memcpy_toio(volatile void __iomem *dest, const void *src, unsigned long n) { void *vdest = (void __force *) dest; - __asm__ __volatile__ ("sync" : : : "memory"); + mb(); while(n && (!IO_CHECK_ALIGN(vdest, 4) || !IO_CHECK_ALIGN(src, 4))) { *((volatile u8 *)vdest) = *((u8 *)src); src++; @@ -201,6 +207,6 @@ void _memcpy_toio(volatile void __iomem *dest, const void *src, unsigned long n) vdest++; n--; } - __asm__ __volatile__ ("sync" : : : "memory"); + mb(); } EXPORT_SYMBOL(_memcpy_toio); diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 76381e14e800..244eb4857e7f 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -16,6 +16,7 @@ #include <linux/mm.h> #include <linux/spinlock.h> #include <linux/string.h> +#include <linux/string_choices.h> #include <linux/dma-mapping.h> #include <linux/bitmap.h> #include <linux/iommu-helper.h> @@ -687,7 +688,7 @@ void iommu_table_clear(struct iommu_table *tbl) void iommu_table_reserve_pages(struct iommu_table *tbl, unsigned long res_start, unsigned long res_end) { - int i; + unsigned long i; WARN_ON_ONCE(res_end < res_start); /* @@ -769,8 +770,8 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid, iommu_table_clear(tbl); if (!welcomed) { - printk(KERN_INFO "IOMMU table initialized, virtual merging %s\n", - novmerge ? "disabled" : "enabled"); + pr_info("IOMMU table initialized, virtual merging %s\n", + str_disabled_enabled(novmerge)); welcomed = 1; } diff --git a/arch/powerpc/kernel/legacy_serial.c b/arch/powerpc/kernel/legacy_serial.c index 1da2f6e7d2a1..ae1906bfe8a5 100644 --- a/arch/powerpc/kernel/legacy_serial.c +++ b/arch/powerpc/kernel/legacy_serial.c @@ -54,9 +54,10 @@ static int legacy_serial_console = -1; static const upf_t legacy_port_flags = UPF_BOOT_AUTOCONF | UPF_SKIP_TEST | UPF_SHARE_IRQ | UPF_FIXED_PORT; -static unsigned int tsi_serial_in(struct uart_port *p, int offset) +static u32 tsi_serial_in(struct uart_port *p, unsigned int offset) { - unsigned int tmp; + u32 tmp; + offset = offset << p->regshift; if (offset == UART_IIR) { tmp = readl(p->membase + (UART_IIR & ~3)); @@ -65,7 +66,7 @@ static unsigned int tsi_serial_in(struct uart_port *p, int offset) return readb(p->membase + offset); } -static void tsi_serial_out(struct uart_port *p, int offset, int value) +static void tsi_serial_out(struct uart_port *p, unsigned int offset, u32 value) { offset = offset << p->regshift; if (!((offset == UART_IER) && (value & UART_IER_UUE))) @@ -77,6 +78,8 @@ static int __init add_legacy_port(struct device_node *np, int want_index, phys_addr_t taddr, unsigned long irq, upf_t flags, int irq_check_parent) { + struct plat_serial8250_port *legacy_port; + struct legacy_serial_info *legacy_info; const __be32 *clk, *spd, *rs; u32 clock = BASE_BAUD * 16; u32 shift = 0; @@ -110,16 +113,17 @@ static int __init add_legacy_port(struct device_node *np, int want_index, if (index >= legacy_serial_count) legacy_serial_count = index + 1; + legacy_port = &legacy_serial_ports[index]; + legacy_info = &legacy_serial_infos[index]; + /* Check if there is a port who already claimed our slot */ - if (legacy_serial_infos[index].np != NULL) { + if (legacy_info->np != NULL) { /* if we still have some room, move it, else override */ if (legacy_serial_count < MAX_LEGACY_SERIAL_PORTS) { printk(KERN_DEBUG "Moved legacy port %d -> %d\n", index, legacy_serial_count); - legacy_serial_ports[legacy_serial_count] = - legacy_serial_ports[index]; - legacy_serial_infos[legacy_serial_count] = - legacy_serial_infos[index]; + legacy_serial_ports[legacy_serial_count] = *legacy_port; + legacy_serial_infos[legacy_serial_count] = *legacy_info; legacy_serial_count++; } else { printk(KERN_DEBUG "Replacing legacy port %d\n", index); @@ -127,36 +131,32 @@ static int __init add_legacy_port(struct device_node *np, int want_index, } /* Now fill the entry */ - memset(&legacy_serial_ports[index], 0, - sizeof(struct plat_serial8250_port)); + memset(legacy_port, 0, sizeof(*legacy_port)); if (iotype == UPIO_PORT) - legacy_serial_ports[index].iobase = base; + legacy_port->iobase = base; else - legacy_serial_ports[index].mapbase = base; - - legacy_serial_ports[index].iotype = iotype; - legacy_serial_ports[index].uartclk = clock; - legacy_serial_ports[index].irq = irq; - legacy_serial_ports[index].flags = flags; - legacy_serial_ports[index].regshift = shift; - legacy_serial_infos[index].taddr = taddr; - legacy_serial_infos[index].np = of_node_get(np); - legacy_serial_infos[index].clock = clock; - legacy_serial_infos[index].speed = spd ? be32_to_cpup(spd) : 0; - legacy_serial_infos[index].irq_check_parent = irq_check_parent; + legacy_port->mapbase = base; + + legacy_port->iotype = iotype; + legacy_port->uartclk = clock; + legacy_port->irq = irq; + legacy_port->flags = flags; + legacy_port->regshift = shift; + legacy_info->taddr = taddr; + legacy_info->np = of_node_get(np); + legacy_info->clock = clock; + legacy_info->speed = spd ? be32_to_cpup(spd) : 0; + legacy_info->irq_check_parent = irq_check_parent; if (iotype == UPIO_TSI) { - legacy_serial_ports[index].serial_in = tsi_serial_in; - legacy_serial_ports[index].serial_out = tsi_serial_out; + legacy_port->serial_in = tsi_serial_in; + legacy_port->serial_out = tsi_serial_out; } - printk(KERN_DEBUG "Found legacy serial port %d for %pOF\n", - index, np); - printk(KERN_DEBUG " %s=%llx, taddr=%llx, irq=%lx, clk=%d, speed=%d\n", + printk(KERN_DEBUG "Found legacy serial port %d for %pOF\n", index, np); + printk(KERN_DEBUG " %s=%pa, taddr=%pa, irq=%lx, clk=%d, speed=%d\n", (iotype == UPIO_PORT) ? "port" : "mem", - (unsigned long long)base, (unsigned long long)taddr, irq, - legacy_serial_ports[index].uartclk, - legacy_serial_infos[index].speed); + &base, &taddr, irq, legacy_port->uartclk, legacy_info->speed); return index; } diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S index 033cd00aa0fc..acb727f54e9d 100644 --- a/arch/powerpc/kernel/misc_32.S +++ b/arch/powerpc/kernel/misc_32.S @@ -28,32 +28,6 @@ .text /* - * This returns the high 64 bits of the product of two 64-bit numbers. - */ -_GLOBAL(mulhdu) - cmpwi r6,0 - cmpwi cr1,r3,0 - mr r10,r4 - mulhwu r4,r4,r5 - beq 1f - mulhwu r0,r10,r6 - mullw r7,r10,r5 - addc r7,r0,r7 - addze r4,r4 -1: beqlr cr1 /* all done if high part of A is 0 */ - mullw r9,r3,r5 - mulhwu r10,r3,r5 - beq 2f - mullw r0,r3,r6 - mulhwu r8,r3,r6 - addc r7,r0,r7 - adde r4,r4,r8 - addze r10,r10 -2: addc r4,r4,r9 - addze r3,r10 - blr - -/* * reloc_got2 runs through the .got2 section adding an offset * to each entry. */ diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c index 45dac7b46aa3..126bf3b06ab7 100644 --- a/arch/powerpc/kernel/module_64.c +++ b/arch/powerpc/kernel/module_64.c @@ -258,10 +258,6 @@ static unsigned long get_stubs_size(const Elf64_Ehdr *hdr, break; } } - if (i == hdr->e_shnum) { - pr_err("%s: doesn't contain __patchable_function_entries.\n", me->name); - return -ENOEXEC; - } #endif pr_debug("Looks like a total of %lu stubs, max\n", relocs); @@ -369,6 +365,24 @@ static void dedotify_versions(struct modversion_info *vers, } } +/* Same as normal versions, remove a leading dot if present. */ +static void dedotify_ext_version_names(char *str_seq, unsigned long size) +{ + unsigned long out = 0; + unsigned long in; + char last = '\0'; + + for (in = 0; in < size; in++) { + /* Skip one leading dot */ + if (last == '\0' && str_seq[in] == '.') + in++; + last = str_seq[in]; + str_seq[out++] = last; + } + /* Zero the trailing portion of the names table for robustness */ + memset(&str_seq[out], 0, size - out); +} + /* * Undefined symbols which refer to .funcname, hack to funcname. Make .TOC. * seem to be defined (value set later). @@ -438,10 +452,12 @@ int module_frob_arch_sections(Elf64_Ehdr *hdr, me->arch.toc_section = i; if (sechdrs[i].sh_addralign < 8) sechdrs[i].sh_addralign = 8; - } - else if (strcmp(secstrings+sechdrs[i].sh_name,"__versions")==0) + } else if (strcmp(secstrings + sechdrs[i].sh_name, "__versions") == 0) dedotify_versions((void *)hdr + sechdrs[i].sh_offset, sechdrs[i].sh_size); + else if (strcmp(secstrings + sechdrs[i].sh_name, "__version_ext_names") == 0) + dedotify_ext_version_names((void *)hdr + sechdrs[i].sh_offset, + sechdrs[i].sh_size); if (sechdrs[i].sh_type == SHT_SYMTAB) dedotify((void *)hdr + sechdrs[i].sh_offset, diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c deleted file mode 100644 index adc76fa58d1e..000000000000 --- a/arch/powerpc/kernel/of_platform.c +++ /dev/null @@ -1,102 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2006 Benjamin Herrenschmidt, IBM Corp. - * <benh@kernel.crashing.org> - * and Arnd Bergmann, IBM Corp. - */ - -#undef DEBUG - -#include <linux/string.h> -#include <linux/kernel.h> -#include <linux/init.h> -#include <linux/export.h> -#include <linux/mod_devicetable.h> -#include <linux/pci.h> -#include <linux/platform_device.h> -#include <linux/atomic.h> - -#include <asm/errno.h> -#include <asm/topology.h> -#include <asm/pci-bridge.h> -#include <asm/ppc-pci.h> -#include <asm/eeh.h> - -#ifdef CONFIG_PPC_OF_PLATFORM_PCI - -/* The probing of PCI controllers from of_platform is currently - * 64 bits only, mostly due to gratuitous differences between - * the 32 and 64 bits PCI code on PowerPC and the 32 bits one - * lacking some bits needed here. - */ - -static int of_pci_phb_probe(struct platform_device *dev) -{ - struct pci_controller *phb; - - /* Check if we can do that ... */ - if (ppc_md.pci_setup_phb == NULL) - return -ENODEV; - - pr_info("Setting up PCI bus %pOF\n", dev->dev.of_node); - - /* Alloc and setup PHB data structure */ - phb = pcibios_alloc_controller(dev->dev.of_node); - if (!phb) - return -ENODEV; - - /* Setup parent in sysfs */ - phb->parent = &dev->dev; - - /* Setup the PHB using arch provided callback */ - if (ppc_md.pci_setup_phb(phb)) { - pcibios_free_controller(phb); - return -ENODEV; - } - - /* Process "ranges" property */ - pci_process_bridge_OF_ranges(phb, dev->dev.of_node, 0); - - /* Init pci_dn data structures */ - pci_devs_phb_init_dynamic(phb); - - /* Create EEH PE for the PHB */ - eeh_phb_pe_create(phb); - - /* Scan the bus */ - pcibios_scan_phb(phb); - if (phb->bus == NULL) - return -ENXIO; - - /* Claim resources. This might need some rework as well depending - * whether we are doing probe-only or not, like assigning unassigned - * resources etc... - */ - pcibios_claim_one_bus(phb->bus); - - /* Add probed PCI devices to the device model */ - pci_bus_add_devices(phb->bus); - - return 0; -} - -static const struct of_device_id of_pci_phb_ids[] = { - { .type = "pci", }, - { .type = "pcix", }, - { .type = "pcie", }, - { .type = "pciex", }, - { .type = "ht", }, - {} -}; - -static struct platform_driver of_pci_phb_driver = { - .probe = of_pci_phb_probe, - .driver = { - .name = "of-pci", - .of_match_table = of_pci_phb_ids, - }, -}; - -builtin_platform_driver(of_pci_phb_driver); - -#endif /* CONFIG_PPC_OF_PLATFORM_PCI */ diff --git a/arch/powerpc/kernel/pci-hotplug.c b/arch/powerpc/kernel/pci-hotplug.c index 9ea74973d78d..6f444d0822d8 100644 --- a/arch/powerpc/kernel/pci-hotplug.c +++ b/arch/powerpc/kernel/pci-hotplug.c @@ -141,6 +141,9 @@ void pci_hp_add_devices(struct pci_bus *bus) struct pci_controller *phb; struct device_node *dn = pci_bus_to_OF_node(bus); + if (!dn) + return; + phb = pci_bus_to_host(bus); mode = PCI_PROBE_NORMAL; diff --git a/arch/powerpc/kernel/pci_32.c b/arch/powerpc/kernel/pci_32.c index ce0c8623e563..f8a3bd8cfae4 100644 --- a/arch/powerpc/kernel/pci_32.c +++ b/arch/powerpc/kernel/pci_32.c @@ -213,11 +213,8 @@ pci_create_OF_bus_map(void) struct property* of_prop; struct device_node *dn; - of_prop = memblock_alloc(sizeof(struct property) + 256, + of_prop = memblock_alloc_or_panic(sizeof(struct property) + 256, SMP_CACHE_BYTES); - if (!of_prop) - panic("%s: Failed to allocate %zu bytes\n", __func__, - sizeof(struct property) + 256); dn = of_find_node_by_path("/"); if (dn) { memset(of_prop, -1, sizeof(struct property) + 256); diff --git a/arch/powerpc/kernel/proc_powerpc.c b/arch/powerpc/kernel/proc_powerpc.c index 3816a2bf2b84..d083b4517065 100644 --- a/arch/powerpc/kernel/proc_powerpc.c +++ b/arch/powerpc/kernel/proc_powerpc.c @@ -9,6 +9,7 @@ #include <linux/proc_fs.h> #include <linux/kernel.h> #include <linux/of.h> +#include <linux/string.h> #include <asm/machdep.h> #include <asm/vdso_datapage.h> @@ -56,7 +57,7 @@ static int __init proc_ppc64_init(void) { struct proc_dir_entry *pde; - strcpy((char *)systemcfg->eye_catcher, "SYSTEMCFG:PPC64"); + strscpy(systemcfg->eye_catcher, "SYSTEMCFG:PPC64"); systemcfg->version.major = SYSTEMCFG_MAJOR; systemcfg->version.minor = SYSTEMCFG_MINOR; systemcfg->processor = mfspr(SPRN_PVR); diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 7b739b9a91ab..855e09886503 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -1000,7 +1000,7 @@ static inline void tm_reclaim_task(struct task_struct *tsk) WARN_ON(tm_suspend_disabled); - TM_DEBUG("--- tm_reclaim on pid %d (NIP=%lx, " + TM_DEBUG("---- tm_reclaim on pid %d (NIP=%lx, " "ccr=%lx, msr=%lx, trap=%lx)\n", tsk->pid, thr->regs->nip, thr->regs->ccr, thr->regs->msr, @@ -1008,7 +1008,7 @@ static inline void tm_reclaim_task(struct task_struct *tsk) tm_reclaim_thread(thr, TM_CAUSE_RESCHED); - TM_DEBUG("--- tm_reclaim on pid %d complete\n", + TM_DEBUG("---- tm_reclaim on pid %d complete\n", tsk->pid); out_and_saveregs: @@ -1960,8 +1960,8 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) * address of _start and the second entry is the TOC * value we need to use. */ - __get_user(entry, (unsigned long __user *)start); - __get_user(toc, (unsigned long __user *)start+1); + get_user(entry, (unsigned long __user *)start); + get_user(toc, (unsigned long __user *)start+1); /* Check whether the e_entry function descriptor entries * need to be relocated before we can use them. @@ -2367,14 +2367,14 @@ void __no_sanitize_address show_stack(struct task_struct *tsk, (sp + STACK_INT_FRAME_REGS); lr = regs->link; - printk("%s--- interrupt: %lx at %pS\n", + printk("%s---- interrupt: %lx at %pS\n", loglvl, regs->trap, (void *)regs->nip); // Detect the case of an empty pt_regs at the very base // of the stack and suppress showing it in full. if (!empty_user_regs(regs, tsk)) { __show_regs(regs); - printk("%s--- interrupt: %lx\n", loglvl, regs->trap); + printk("%s---- interrupt: %lx\n", loglvl, regs->trap); } firstframe = 1; diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index e0059842a1c6..9ed9dde7d231 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -860,7 +860,7 @@ void __init early_init_devtree(void *params) */ if (fadump_reserve_mem() == 0) #endif - reserve_crashkernel(); + arch_reserve_crashkernel(); early_reserve_mem(); if (memory_limit > memblock_phys_mem_size()) diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 8e776ba39497..827c958677f8 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -1061,7 +1061,7 @@ static const struct ibm_arch_vec ibm_architecture_vec_template __initconst = { .virt_base = cpu_to_be32(0xffffffff), .virt_size = cpu_to_be32(0xffffffff), .load_base = cpu_to_be32(0xffffffff), - .min_rma = cpu_to_be32(512), /* 512MB min RMA */ + .min_rma = cpu_to_be32(MIN_RMA), .min_load = cpu_to_be32(0xffffffff), /* full client load */ .min_rma_percent = 0, /* min RMA percentage of total RAM */ .max_pft_size = 48, /* max log_2(hash table size) */ @@ -2792,7 +2792,6 @@ static void __init flatten_device_tree(void) dt_struct_start, dt_struct_end); } -#ifdef CONFIG_PPC_CHRP /* * Pegasos and BriQ lacks the "ranges" property in the isa node * Pegasos needs decimal IRQ 14/15, not hexadecimal @@ -2843,11 +2842,7 @@ static void __init fixup_device_tree_chrp(void) } } } -#else -#define fixup_device_tree_chrp() -#endif -#if defined(CONFIG_PPC64) && defined(CONFIG_PPC_PMAC) static void __init fixup_device_tree_pmac64(void) { phandle u3, i2c, mpic; @@ -2887,22 +2882,18 @@ static void __init fixup_device_tree_pmac64(void) prom_setprop(i2c, "/u3@0,f8000000/i2c@f8001000", "interrupt-parent", &parent, sizeof(parent)); } -#else -#define fixup_device_tree_pmac64() -#endif -#ifdef CONFIG_PPC_PMAC static void __init fixup_device_tree_pmac(void) { __be32 val = 1; char type[8]; phandle node; - // Some pmacs are missing #size-cells on escc nodes + // Some pmacs are missing #size-cells on escc or i2s nodes for (node = 0; prom_next_node(&node); ) { type[0] = '\0'; prom_getprop(node, "device_type", type, sizeof(type)); - if (prom_strcmp(type, "escc")) + if (prom_strcmp(type, "escc") && prom_strcmp(type, "i2s")) continue; if (prom_getproplen(node, "#size-cells") != PROM_ERROR) @@ -2911,11 +2902,7 @@ static void __init fixup_device_tree_pmac(void) prom_setprop(node, NULL, "#size-cells", &val, sizeof(val)); } } -#else -static inline void fixup_device_tree_pmac(void) { } -#endif -#ifdef CONFIG_PPC_EFIKA /* * The MPC5200 FEC driver requires an phy-handle property to tell it how * to talk to the phy. If the phy-handle property is missing, then this @@ -3047,11 +3034,7 @@ static void __init fixup_device_tree_efika(void) /* Make sure ethernet phy-handle property exists */ fixup_device_tree_efika_add_phy(); } -#else -#define fixup_device_tree_efika() -#endif -#ifdef CONFIG_PPC_PASEMI_NEMO /* * CFE supplied on Nemo is broken in several ways, biggest * problem is that it reassigns ISA interrupts to unused mpic ints. @@ -3127,17 +3110,23 @@ static void __init fixup_device_tree_pasemi(void) prom_setprop(iob, name, "device_type", "isa", sizeof("isa")); } -#else /* !CONFIG_PPC_PASEMI_NEMO */ -static inline void fixup_device_tree_pasemi(void) { } -#endif static void __init fixup_device_tree(void) { - fixup_device_tree_chrp(); - fixup_device_tree_pmac(); - fixup_device_tree_pmac64(); - fixup_device_tree_efika(); - fixup_device_tree_pasemi(); + if (IS_ENABLED(CONFIG_PPC_CHRP)) + fixup_device_tree_chrp(); + + if (IS_ENABLED(CONFIG_PPC_PMAC)) + fixup_device_tree_pmac(); + + if (IS_ENABLED(CONFIG_PPC_PMAC) && IS_ENABLED(CONFIG_PPC64)) + fixup_device_tree_pmac64(); + + if (IS_ENABLED(CONFIG_PPC_EFIKA)) + fixup_device_tree_efika(); + + if (IS_ENABLED(CONFIG_PPC_PASEMI_NEMO)) + fixup_device_tree_pasemi(); } static void __init prom_find_boot_cpu(void) diff --git a/arch/powerpc/kernel/ptrace/ptrace-view.c b/arch/powerpc/kernel/ptrace/ptrace-view.c index c1819e0a6684..0310f9097e39 100644 --- a/arch/powerpc/kernel/ptrace/ptrace-view.c +++ b/arch/powerpc/kernel/ptrace/ptrace-view.c @@ -568,114 +568,114 @@ static int pkey_set(struct task_struct *target, const struct user_regset *regset static const struct user_regset native_regsets[] = { [REGSET_GPR] = { - .core_note_type = NT_PRSTATUS, .n = ELF_NGREG, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = ELF_NGREG, .size = sizeof(long), .align = sizeof(long), .regset_get = gpr_get, .set = gpr_set }, [REGSET_FPR] = { - .core_note_type = NT_PRFPREG, .n = ELF_NFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = ELF_NFPREG, .size = sizeof(double), .align = sizeof(double), .regset_get = fpr_get, .set = fpr_set }, #ifdef CONFIG_ALTIVEC [REGSET_VMX] = { - .core_note_type = NT_PPC_VMX, .n = 34, + USER_REGSET_NOTE_TYPE(PPC_VMX), .n = 34, .size = sizeof(vector128), .align = sizeof(vector128), .active = vr_active, .regset_get = vr_get, .set = vr_set }, #endif #ifdef CONFIG_VSX [REGSET_VSX] = { - .core_note_type = NT_PPC_VSX, .n = 32, + USER_REGSET_NOTE_TYPE(PPC_VSX), .n = 32, .size = sizeof(double), .align = sizeof(double), .active = vsr_active, .regset_get = vsr_get, .set = vsr_set }, #endif #ifdef CONFIG_SPE [REGSET_SPE] = { - .core_note_type = NT_PPC_SPE, .n = 35, + USER_REGSET_NOTE_TYPE(PPC_SPE), .n = 35, .size = sizeof(u32), .align = sizeof(u32), .active = evr_active, .regset_get = evr_get, .set = evr_set }, #endif #ifdef CONFIG_PPC_TRANSACTIONAL_MEM [REGSET_TM_CGPR] = { - .core_note_type = NT_PPC_TM_CGPR, .n = ELF_NGREG, + USER_REGSET_NOTE_TYPE(PPC_TM_CGPR), .n = ELF_NGREG, .size = sizeof(long), .align = sizeof(long), .active = tm_cgpr_active, .regset_get = tm_cgpr_get, .set = tm_cgpr_set }, [REGSET_TM_CFPR] = { - .core_note_type = NT_PPC_TM_CFPR, .n = ELF_NFPREG, + USER_REGSET_NOTE_TYPE(PPC_TM_CFPR), .n = ELF_NFPREG, .size = sizeof(double), .align = sizeof(double), .active = tm_cfpr_active, .regset_get = tm_cfpr_get, .set = tm_cfpr_set }, [REGSET_TM_CVMX] = { - .core_note_type = NT_PPC_TM_CVMX, .n = ELF_NVMX, + USER_REGSET_NOTE_TYPE(PPC_TM_CVMX), .n = ELF_NVMX, .size = sizeof(vector128), .align = sizeof(vector128), .active = tm_cvmx_active, .regset_get = tm_cvmx_get, .set = tm_cvmx_set }, [REGSET_TM_CVSX] = { - .core_note_type = NT_PPC_TM_CVSX, .n = ELF_NVSX, + USER_REGSET_NOTE_TYPE(PPC_TM_CVSX), .n = ELF_NVSX, .size = sizeof(double), .align = sizeof(double), .active = tm_cvsx_active, .regset_get = tm_cvsx_get, .set = tm_cvsx_set }, [REGSET_TM_SPR] = { - .core_note_type = NT_PPC_TM_SPR, .n = ELF_NTMSPRREG, + USER_REGSET_NOTE_TYPE(PPC_TM_SPR), .n = ELF_NTMSPRREG, .size = sizeof(u64), .align = sizeof(u64), .active = tm_spr_active, .regset_get = tm_spr_get, .set = tm_spr_set }, [REGSET_TM_CTAR] = { - .core_note_type = NT_PPC_TM_CTAR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TM_CTAR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .active = tm_tar_active, .regset_get = tm_tar_get, .set = tm_tar_set }, [REGSET_TM_CPPR] = { - .core_note_type = NT_PPC_TM_CPPR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TM_CPPR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .active = tm_ppr_active, .regset_get = tm_ppr_get, .set = tm_ppr_set }, [REGSET_TM_CDSCR] = { - .core_note_type = NT_PPC_TM_CDSCR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TM_CDSCR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .active = tm_dscr_active, .regset_get = tm_dscr_get, .set = tm_dscr_set }, #endif #ifdef CONFIG_PPC64 [REGSET_PPR] = { - .core_note_type = NT_PPC_PPR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_PPR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .regset_get = ppr_get, .set = ppr_set }, [REGSET_DSCR] = { - .core_note_type = NT_PPC_DSCR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_DSCR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .regset_get = dscr_get, .set = dscr_set }, #endif #ifdef CONFIG_PPC_BOOK3S_64 [REGSET_TAR] = { - .core_note_type = NT_PPC_TAR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TAR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .regset_get = tar_get, .set = tar_set }, [REGSET_EBB] = { - .core_note_type = NT_PPC_EBB, .n = ELF_NEBB, + USER_REGSET_NOTE_TYPE(PPC_EBB), .n = ELF_NEBB, .size = sizeof(u64), .align = sizeof(u64), .active = ebb_active, .regset_get = ebb_get, .set = ebb_set }, [REGSET_PMR] = { - .core_note_type = NT_PPC_PMU, .n = ELF_NPMU, + USER_REGSET_NOTE_TYPE(PPC_PMU), .n = ELF_NPMU, .size = sizeof(u64), .align = sizeof(u64), .active = pmu_active, .regset_get = pmu_get, .set = pmu_set }, [REGSET_DEXCR] = { - .core_note_type = NT_PPC_DEXCR, .n = ELF_NDEXCR, + USER_REGSET_NOTE_TYPE(PPC_DEXCR), .n = ELF_NDEXCR, .size = sizeof(u64), .align = sizeof(u64), .active = dexcr_active, .regset_get = dexcr_get }, #ifdef CONFIG_CHECKPOINT_RESTORE [REGSET_HASHKEYR] = { - .core_note_type = NT_PPC_HASHKEYR, .n = ELF_NHASHKEYR, + USER_REGSET_NOTE_TYPE(PPC_HASHKEYR), .n = ELF_NHASHKEYR, .size = sizeof(u64), .align = sizeof(u64), .active = hashkeyr_active, .regset_get = hashkeyr_get, .set = hashkeyr_set }, @@ -683,7 +683,7 @@ static const struct user_regset native_regsets[] = { #endif #ifdef CONFIG_PPC_MEM_KEYS [REGSET_PKEY] = { - .core_note_type = NT_PPC_PKEY, .n = ELF_NPKEY, + USER_REGSET_NOTE_TYPE(PPC_PKEY), .n = ELF_NPKEY, .size = sizeof(u64), .align = sizeof(u64), .active = pkey_active, .regset_get = pkey_get, .set = pkey_set }, @@ -843,92 +843,92 @@ static int gpr32_set(struct task_struct *target, */ static const struct user_regset compat_regsets[] = { [REGSET_GPR] = { - .core_note_type = NT_PRSTATUS, .n = ELF_NGREG, + USER_REGSET_NOTE_TYPE(PRSTATUS), .n = ELF_NGREG, .size = sizeof(compat_long_t), .align = sizeof(compat_long_t), .regset_get = gpr32_get, .set = gpr32_set }, [REGSET_FPR] = { - .core_note_type = NT_PRFPREG, .n = ELF_NFPREG, + USER_REGSET_NOTE_TYPE(PRFPREG), .n = ELF_NFPREG, .size = sizeof(double), .align = sizeof(double), .regset_get = fpr_get, .set = fpr_set }, #ifdef CONFIG_ALTIVEC [REGSET_VMX] = { - .core_note_type = NT_PPC_VMX, .n = 34, + USER_REGSET_NOTE_TYPE(PPC_VMX), .n = 34, .size = sizeof(vector128), .align = sizeof(vector128), .active = vr_active, .regset_get = vr_get, .set = vr_set }, #endif #ifdef CONFIG_SPE [REGSET_SPE] = { - .core_note_type = NT_PPC_SPE, .n = 35, + USER_REGSET_NOTE_TYPE(PPC_SPE), .n = 35, .size = sizeof(u32), .align = sizeof(u32), .active = evr_active, .regset_get = evr_get, .set = evr_set }, #endif #ifdef CONFIG_PPC_TRANSACTIONAL_MEM [REGSET_TM_CGPR] = { - .core_note_type = NT_PPC_TM_CGPR, .n = ELF_NGREG, + USER_REGSET_NOTE_TYPE(PPC_TM_CGPR), .n = ELF_NGREG, .size = sizeof(long), .align = sizeof(long), .active = tm_cgpr_active, .regset_get = tm_cgpr32_get, .set = tm_cgpr32_set }, [REGSET_TM_CFPR] = { - .core_note_type = NT_PPC_TM_CFPR, .n = ELF_NFPREG, + USER_REGSET_NOTE_TYPE(PPC_TM_CFPR), .n = ELF_NFPREG, .size = sizeof(double), .align = sizeof(double), .active = tm_cfpr_active, .regset_get = tm_cfpr_get, .set = tm_cfpr_set }, [REGSET_TM_CVMX] = { - .core_note_type = NT_PPC_TM_CVMX, .n = ELF_NVMX, + USER_REGSET_NOTE_TYPE(PPC_TM_CVMX), .n = ELF_NVMX, .size = sizeof(vector128), .align = sizeof(vector128), .active = tm_cvmx_active, .regset_get = tm_cvmx_get, .set = tm_cvmx_set }, [REGSET_TM_CVSX] = { - .core_note_type = NT_PPC_TM_CVSX, .n = ELF_NVSX, + USER_REGSET_NOTE_TYPE(PPC_TM_CVSX), .n = ELF_NVSX, .size = sizeof(double), .align = sizeof(double), .active = tm_cvsx_active, .regset_get = tm_cvsx_get, .set = tm_cvsx_set }, [REGSET_TM_SPR] = { - .core_note_type = NT_PPC_TM_SPR, .n = ELF_NTMSPRREG, + USER_REGSET_NOTE_TYPE(PPC_TM_SPR), .n = ELF_NTMSPRREG, .size = sizeof(u64), .align = sizeof(u64), .active = tm_spr_active, .regset_get = tm_spr_get, .set = tm_spr_set }, [REGSET_TM_CTAR] = { - .core_note_type = NT_PPC_TM_CTAR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TM_CTAR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .active = tm_tar_active, .regset_get = tm_tar_get, .set = tm_tar_set }, [REGSET_TM_CPPR] = { - .core_note_type = NT_PPC_TM_CPPR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TM_CPPR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .active = tm_ppr_active, .regset_get = tm_ppr_get, .set = tm_ppr_set }, [REGSET_TM_CDSCR] = { - .core_note_type = NT_PPC_TM_CDSCR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TM_CDSCR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .active = tm_dscr_active, .regset_get = tm_dscr_get, .set = tm_dscr_set }, #endif #ifdef CONFIG_PPC64 [REGSET_PPR] = { - .core_note_type = NT_PPC_PPR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_PPR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .regset_get = ppr_get, .set = ppr_set }, [REGSET_DSCR] = { - .core_note_type = NT_PPC_DSCR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_DSCR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .regset_get = dscr_get, .set = dscr_set }, #endif #ifdef CONFIG_PPC_BOOK3S_64 [REGSET_TAR] = { - .core_note_type = NT_PPC_TAR, .n = 1, + USER_REGSET_NOTE_TYPE(PPC_TAR), .n = 1, .size = sizeof(u64), .align = sizeof(u64), .regset_get = tar_get, .set = tar_set }, [REGSET_EBB] = { - .core_note_type = NT_PPC_EBB, .n = ELF_NEBB, + USER_REGSET_NOTE_TYPE(PPC_EBB), .n = ELF_NEBB, .size = sizeof(u64), .align = sizeof(u64), .active = ebb_active, .regset_get = ebb_get, .set = ebb_set }, diff --git a/arch/powerpc/kernel/ptrace/ptrace.c b/arch/powerpc/kernel/ptrace/ptrace.c index 727ed4a14545..c6997df63287 100644 --- a/arch/powerpc/kernel/ptrace/ptrace.c +++ b/arch/powerpc/kernel/ptrace/ptrace.c @@ -215,7 +215,7 @@ static int do_seccomp(struct pt_regs *regs) * have already loaded -ENOSYS into r3, or seccomp has put * something else in r3 (via SECCOMP_RET_ERRNO/TRACE). */ - if (__secure_computing(NULL)) + if (__secure_computing()) return -1; /* diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index d31c9799cab2..e61245c4468e 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -92,12 +92,12 @@ struct rtas_function { * Per-function locks for sequence-based RTAS functions. */ static DEFINE_MUTEX(rtas_ibm_activate_firmware_lock); -static DEFINE_MUTEX(rtas_ibm_get_dynamic_sensor_state_lock); -static DEFINE_MUTEX(rtas_ibm_get_indices_lock); static DEFINE_MUTEX(rtas_ibm_lpar_perftools_lock); -static DEFINE_MUTEX(rtas_ibm_physical_attestation_lock); -static DEFINE_MUTEX(rtas_ibm_set_dynamic_indicator_lock); +DEFINE_MUTEX(rtas_ibm_physical_attestation_lock); DEFINE_MUTEX(rtas_ibm_get_vpd_lock); +DEFINE_MUTEX(rtas_ibm_get_indices_lock); +DEFINE_MUTEX(rtas_ibm_set_dynamic_indicator_lock); +DEFINE_MUTEX(rtas_ibm_get_dynamic_sensor_state_lock); static struct rtas_function rtas_function_table[] __ro_after_init = { [RTAS_FNIDX__CHECK_EXCEPTION] = { @@ -798,66 +798,6 @@ void __init udbg_init_rtas_panel(void) udbg_putc = call_rtas_display_status_delay; } -#ifdef CONFIG_UDBG_RTAS_CONSOLE - -/* If you think you're dying before early_init_dt_scan_rtas() does its - * work, you can hard code the token values for your firmware here and - * hardcode rtas.base/entry etc. - */ -static unsigned int rtas_putchar_token = RTAS_UNKNOWN_SERVICE; -static unsigned int rtas_getchar_token = RTAS_UNKNOWN_SERVICE; - -static void udbg_rtascon_putc(char c) -{ - int tries; - - if (!rtas.base) - return; - - /* Add CRs before LFs */ - if (c == '\n') - udbg_rtascon_putc('\r'); - - /* if there is more than one character to be displayed, wait a bit */ - for (tries = 0; tries < 16; tries++) { - if (rtas_call(rtas_putchar_token, 1, 1, NULL, c) == 0) - break; - udelay(1000); - } -} - -static int udbg_rtascon_getc_poll(void) -{ - int c; - - if (!rtas.base) - return -1; - - if (rtas_call(rtas_getchar_token, 0, 2, &c)) - return -1; - - return c; -} - -static int udbg_rtascon_getc(void) -{ - int c; - - while ((c = udbg_rtascon_getc_poll()) == -1) - ; - - return c; -} - - -void __init udbg_init_rtas_console(void) -{ - udbg_putc = udbg_rtascon_putc; - udbg_getc = udbg_rtascon_getc; - udbg_getc_poll = udbg_rtascon_getc_poll; -} -#endif /* CONFIG_UDBG_RTAS_CONSOLE */ - void rtas_progress(char *s, unsigned short hex) { struct device_node *root; @@ -2135,21 +2075,6 @@ int __init early_init_dt_scan_rtas(unsigned long node, rtas.size = *sizep; } -#ifdef CONFIG_UDBG_RTAS_CONSOLE - basep = of_get_flat_dt_prop(node, "put-term-char", NULL); - if (basep) - rtas_putchar_token = *basep; - - basep = of_get_flat_dt_prop(node, "get-term-char", NULL); - if (basep) - rtas_getchar_token = *basep; - - if (rtas_putchar_token != RTAS_UNKNOWN_SERVICE && - rtas_getchar_token != RTAS_UNKNOWN_SERVICE) - udbg_init_rtas_console(); - -#endif - /* break now */ return 1; } diff --git a/arch/powerpc/kernel/rtas_flash.c b/arch/powerpc/kernel/rtas_flash.c index 5407024881e5..583dc16e9d3c 100644 --- a/arch/powerpc/kernel/rtas_flash.c +++ b/arch/powerpc/kernel/rtas_flash.c @@ -312,13 +312,13 @@ static ssize_t rtas_flash_write(struct file *file, const char __user *buffer, { struct rtas_update_flash_t *const uf = &rtas_update_flash_data; char *p; - int next_free, rc; + int next_free; struct flash_block_list *fl; - mutex_lock(&rtas_update_flash_mutex); + guard(mutex)(&rtas_update_flash_mutex); if (uf->status == FLASH_AUTH || count == 0) - goto out; /* discard data */ + return count; /* discard data */ /* In the case that the image is not ready for flashing, the memory * allocated for the block list will be freed upon the release of the @@ -327,7 +327,7 @@ static ssize_t rtas_flash_write(struct file *file, const char __user *buffer, if (uf->flist == NULL) { uf->flist = kmem_cache_zalloc(flash_block_cache, GFP_KERNEL); if (!uf->flist) - goto nomem; + return -ENOMEM; } fl = uf->flist; @@ -338,7 +338,7 @@ static ssize_t rtas_flash_write(struct file *file, const char __user *buffer, /* Need to allocate another block_list */ fl->next = kmem_cache_zalloc(flash_block_cache, GFP_KERNEL); if (!fl->next) - goto nomem; + return -ENOMEM; fl = fl->next; next_free = 0; } @@ -347,25 +347,17 @@ static ssize_t rtas_flash_write(struct file *file, const char __user *buffer, count = RTAS_BLK_SIZE; p = kmem_cache_zalloc(flash_block_cache, GFP_KERNEL); if (!p) - goto nomem; + return -ENOMEM; if(copy_from_user(p, buffer, count)) { kmem_cache_free(flash_block_cache, p); - rc = -EFAULT; - goto error; + return -EFAULT; } fl->blocks[next_free].data = p; fl->blocks[next_free].length = count; fl->num_blocks++; -out: - mutex_unlock(&rtas_update_flash_mutex); - return count; -nomem: - rc = -ENOMEM; -error: - mutex_unlock(&rtas_update_flash_mutex); - return rc; + return count; } /* @@ -405,19 +397,18 @@ static ssize_t manage_flash_write(struct file *file, const char __user *buf, static const char reject_str[] = "0"; static const char commit_str[] = "1"; char stkbuf[10]; - int op, rc; + int op; - mutex_lock(&rtas_manage_flash_mutex); + guard(mutex)(&rtas_manage_flash_mutex); if ((args_buf->status == MANAGE_AUTH) || (count == 0)) - goto out; + return count; op = -1; if (buf) { if (count > 9) count = 9; - rc = -EFAULT; if (copy_from_user (stkbuf, buf, count)) - goto error; + return -EFAULT; if (strncmp(stkbuf, reject_str, strlen(reject_str)) == 0) op = RTAS_REJECT_TMP_IMG; else if (strncmp(stkbuf, commit_str, strlen(commit_str)) == 0) @@ -425,18 +416,11 @@ static ssize_t manage_flash_write(struct file *file, const char __user *buf, } if (op == -1) { /* buf is empty, or contains invalid string */ - rc = -EINVAL; - goto error; + return -EINVAL; } manage_flash(args_buf, op); -out: - mutex_unlock(&rtas_manage_flash_mutex); return count; - -error: - mutex_unlock(&rtas_manage_flash_mutex); - return rc; } /* @@ -499,16 +483,14 @@ static ssize_t validate_flash_write(struct file *file, const char __user *buf, { struct rtas_validate_flash_t *const args_buf = &rtas_validate_flash_data; - int rc; - mutex_lock(&rtas_validate_flash_mutex); + guard(mutex)(&rtas_validate_flash_mutex); /* We are only interested in the first 4K of the * candidate image */ if ((*off >= VALIDATE_BUF_SIZE) || (args_buf->status == VALIDATE_AUTH)) { *off += count; - mutex_unlock(&rtas_validate_flash_mutex); return count; } @@ -519,20 +501,14 @@ static ssize_t validate_flash_write(struct file *file, const char __user *buf, args_buf->status = VALIDATE_INCOMPLETE; } - if (!access_ok(buf, count)) { - rc = -EFAULT; - goto done; - } - if (copy_from_user(args_buf->buf + *off, buf, count)) { - rc = -EFAULT; - goto done; - } + if (!access_ok(buf, count)) + return -EFAULT; + + if (copy_from_user(args_buf->buf + *off, buf, count)) + return -EFAULT; *off += count; - rc = count; -done: - mutex_unlock(&rtas_validate_flash_mutex); - return rc; + return count; } static int validate_flash_release(struct inode *inode, struct file *file) diff --git a/arch/powerpc/kernel/secvar-sysfs.c b/arch/powerpc/kernel/secvar-sysfs.c index fbeb1cbac01b..ec900bce0257 100644 --- a/arch/powerpc/kernel/secvar-sysfs.c +++ b/arch/powerpc/kernel/secvar-sysfs.c @@ -52,7 +52,7 @@ static ssize_t size_show(struct kobject *kobj, struct kobj_attribute *attr, } static ssize_t data_read(struct file *filep, struct kobject *kobj, - struct bin_attribute *attr, char *buf, loff_t off, + const struct bin_attribute *attr, char *buf, loff_t off, size_t count) { char *data; @@ -85,7 +85,7 @@ data_fail: } static ssize_t update_write(struct file *filep, struct kobject *kobj, - struct bin_attribute *attr, char *buf, loff_t off, + const struct bin_attribute *attr, char *buf, loff_t off, size_t count) { int rc; @@ -104,11 +104,11 @@ static struct kobj_attribute format_attr = __ATTR_RO(format); static struct kobj_attribute size_attr = __ATTR_RO(size); -static struct bin_attribute data_attr = __BIN_ATTR_RO(data, 0); +static struct bin_attribute data_attr __ro_after_init = __BIN_ATTR_RO(data, 0); -static struct bin_attribute update_attr = __BIN_ATTR_WO(update, 0); +static struct bin_attribute update_attr __ro_after_init = __BIN_ATTR_WO(update, 0); -static struct bin_attribute *secvar_bin_attrs[] = { +static const struct bin_attribute *const secvar_bin_attrs[] = { &data_attr, &update_attr, NULL, @@ -130,7 +130,7 @@ static const struct kobj_type secvar_ktype = { .default_groups = secvar_attr_groups, }; -static int update_kobj_size(void) +static __init int update_kobj_size(void) { u64 varsize; @@ -145,7 +145,7 @@ static int update_kobj_size(void) return 0; } -static int secvar_sysfs_config(struct kobject *kobj) +static __init int secvar_sysfs_config(struct kobject *kobj) { struct attribute_group config_group = { .name = "config", @@ -158,7 +158,7 @@ static int secvar_sysfs_config(struct kobject *kobj) return 0; } -static int add_var(const char *name) +static __init int add_var(const char *name) { struct kobject *kobj; int rc; @@ -181,7 +181,7 @@ static int add_var(const char *name) return 0; } -static int secvar_sysfs_load(void) +static __init int secvar_sysfs_load(void) { u64 namesize = 0; char *name; @@ -209,7 +209,7 @@ static int secvar_sysfs_load(void) return rc; } -static int secvar_sysfs_load_static(void) +static __init int secvar_sysfs_load_static(void) { const char * const *name_ptr = secvar_ops->var_names; int rc; @@ -224,7 +224,7 @@ static int secvar_sysfs_load_static(void) return 0; } -static int secvar_sysfs_init(void) +static __init int secvar_sysfs_init(void) { u64 max_size; int rc; diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 6fa179448c33..68d47c53876c 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -458,11 +458,8 @@ void __init smp_setup_cpu_maps(void) DBG("smp_setup_cpu_maps()\n"); - cpu_to_phys_id = memblock_alloc(nr_cpu_ids * sizeof(u32), + cpu_to_phys_id = memblock_alloc_or_panic(nr_cpu_ids * sizeof(u32), __alignof__(u32)); - if (!cpu_to_phys_id) - panic("%s: Failed to allocate %zu bytes align=0x%zx\n", - __func__, nr_cpu_ids * sizeof(u32), __alignof__(u32)); for_each_node_by_type(dn, "cpu") { const __be32 *intserv; @@ -834,8 +831,8 @@ static int __init check_cache_coherency(void) if (devtree_coherency != KERNEL_COHERENCY) { printk(KERN_ERR "kernel coherency:%s != device tree_coherency:%s\n", - KERNEL_COHERENCY ? "on" : "off", - devtree_coherency ? "on" : "off"); + str_on_off(KERNEL_COHERENCY), + str_on_off(devtree_coherency)); BUG(); } @@ -960,8 +957,6 @@ void __init setup_arch(char **cmdline_p) /* Parse memory topology */ mem_topology_setup(); - /* Set max_mapnr before paging_init() */ - set_max_mapnr(max_pfn); high_memory = (void *)__va(max_low_pfn * PAGE_SIZE); /* diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c index 75dbf3e0d9c4..5a1bf501fbe1 100644 --- a/arch/powerpc/kernel/setup_32.c +++ b/arch/powerpc/kernel/setup_32.c @@ -140,13 +140,7 @@ arch_initcall(ppc_init); static void *__init alloc_stack(void) { - void *ptr = memblock_alloc(THREAD_SIZE, THREAD_ALIGN); - - if (!ptr) - panic("cannot allocate %d bytes for stack at %pS\n", - THREAD_SIZE, (void *)_RET_IP_); - - return ptr; + return memblock_alloc_or_panic(THREAD_SIZE, THREAD_ALIGN); } void __init irqstack_early_init(void) diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index e67f3048611f..7284c8021eeb 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -892,7 +892,7 @@ unsigned long memory_block_size_bytes(void) } #endif -#if defined(CONFIG_PPC_INDIRECT_PIO) || defined(CONFIG_PPC_INDIRECT_MMIO) +#ifdef CONFIG_PPC_INDIRECT_PIO struct ppc_pci_io ppc_pci_io; EXPORT_SYMBOL(ppc_pci_io); #endif diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 5ac7084eebc0..f59e4b9cc207 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -1700,28 +1700,23 @@ static void __init build_sched_topology(void) #ifdef CONFIG_SCHED_SMT if (has_big_cores) { pr_info("Big cores detected but using small core scheduling\n"); - powerpc_topology[i++] = (struct sched_domain_topology_level){ - smallcore_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) - }; + powerpc_topology[i++] = + SDTL_INIT(smallcore_smt_mask, powerpc_smt_flags, SMT); } else { - powerpc_topology[i++] = (struct sched_domain_topology_level){ - cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) - }; + powerpc_topology[i++] = SDTL_INIT(cpu_smt_mask, powerpc_smt_flags, SMT); } #endif if (shared_caches) { - powerpc_topology[i++] = (struct sched_domain_topology_level){ - shared_cache_mask, powerpc_shared_cache_flags, SD_INIT_NAME(CACHE) - }; + powerpc_topology[i++] = + SDTL_INIT(shared_cache_mask, powerpc_shared_cache_flags, CACHE); } + if (has_coregroup_support()) { - powerpc_topology[i++] = (struct sched_domain_topology_level){ - cpu_mc_mask, powerpc_shared_proc_flags, SD_INIT_NAME(MC) - }; + powerpc_topology[i++] = + SDTL_INIT(cpu_mc_mask, powerpc_shared_proc_flags, MC); } - powerpc_topology[i++] = (struct sched_domain_topology_level){ - cpu_cpu_mask, powerpc_shared_proc_flags, SD_INIT_NAME(PKG) - }; + + powerpc_topology[i++] = SDTL_INIT(cpu_cpu_mask, powerpc_shared_proc_flags, PKG); /* There must be one trailing NULL entry left. */ BUG_ON(i >= ARRAY_SIZE(powerpc_topology) - 1); diff --git a/arch/powerpc/kernel/static_call.c b/arch/powerpc/kernel/static_call.c index 7cfd0710e757..ec3101f95e53 100644 --- a/arch/powerpc/kernel/static_call.c +++ b/arch/powerpc/kernel/static_call.c @@ -8,26 +8,54 @@ void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) { int err; bool is_ret0 = (func == __static_call_return0); - unsigned long target = (unsigned long)(is_ret0 ? tramp + PPC_SCT_RET0 : func); - bool is_short = is_offset_in_branch_range((long)target - (long)tramp); - - if (!tramp) - return; + unsigned long _tramp = (unsigned long)tramp; + unsigned long _func = (unsigned long)func; + unsigned long _ret0 = _tramp + PPC_SCT_RET0; + bool is_short = is_offset_in_branch_range((long)func - (long)(site ? : tramp)); mutex_lock(&text_mutex); - if (func && !is_short) { - err = patch_ulong(tramp + PPC_SCT_DATA, target); - if (err) - goto out; + if (site && tail) { + if (!func) + err = patch_instruction(site, ppc_inst(PPC_RAW_BLR())); + else if (is_ret0) + err = patch_branch(site, _ret0, 0); + else if (is_short) + err = patch_branch(site, _func, 0); + else if (tramp) + err = patch_branch(site, _tramp, 0); + else + err = 0; + } else if (site) { + if (!func) + err = patch_instruction(site, ppc_inst(PPC_RAW_NOP())); + else if (is_ret0) + err = patch_instruction(site, ppc_inst(PPC_RAW_LI(_R3, 0))); + else if (is_short) + err = patch_branch(site, _func, BRANCH_SET_LINK); + else if (tramp) + err = patch_branch(site, _tramp, BRANCH_SET_LINK); + else + err = 0; + } else if (tramp) { + if (func && !is_short) { + err = patch_ulong(tramp + PPC_SCT_DATA, _func); + if (err) + goto out; + } + + if (!func) + err = patch_instruction(tramp, ppc_inst(PPC_RAW_BLR())); + else if (is_ret0) + err = patch_branch(tramp, _ret0, 0); + else if (is_short) + err = patch_branch(tramp, _func, 0); + else + err = patch_instruction(tramp, ppc_inst(PPC_RAW_NOP())); + } else { + err = 0; } - if (!func) - err = patch_instruction(tramp, ppc_inst(PPC_RAW_BLR())); - else if (is_short) - err = patch_branch(tramp, target, 0); - else - err = patch_instruction(tramp, ppc_inst(PPC_RAW_NOP())); out: mutex_unlock(&text_mutex); diff --git a/arch/powerpc/kernel/switch.S b/arch/powerpc/kernel/switch.S index 608c0ce7cec6..59e3ee99db0e 100644 --- a/arch/powerpc/kernel/switch.S +++ b/arch/powerpc/kernel/switch.S @@ -39,7 +39,6 @@ flush_branch_caches: // Flush the link stack .rept 64 - ANNOTATE_INTRA_FUNCTION_CALL bl .+4 .endr b 1f diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index d8b4ab78bef0..b453e80dfc00 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -557,3 +557,6 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr +468 common file_getattr sys_file_getattr +469 common file_setattr sys_file_setattr diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 0727332ad86f..8224381c1dba 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -901,6 +901,38 @@ void secondary_cpu_time_init(void) register_decrementer_clockevent(smp_processor_id()); } +/* + * Divide a 128-bit dividend by a 32-bit divisor, leaving a 128 bit + * result. + */ +static __init void div128_by_32(u64 dividend_high, u64 dividend_low, + unsigned int divisor, struct div_result *dr) +{ + unsigned long a, b, c, d; + unsigned long w, x, y, z; + u64 ra, rb, rc; + + a = dividend_high >> 32; + b = dividend_high & 0xffffffff; + c = dividend_low >> 32; + d = dividend_low & 0xffffffff; + + w = a / divisor; + ra = ((u64)(a - (w * divisor)) << 32) + b; + + rb = ((u64)do_div(ra, divisor) << 32) + c; + x = ra; + + rc = ((u64)do_div(rb, divisor) << 32) + d; + y = rb; + + do_div(rc, divisor); + z = rc; + + dr->result_high = ((u64)w << 32) + x; + dr->result_low = ((u64)y << 32) + z; +} + /* This function is only called on the boot processor */ void __init time_init(void) { @@ -950,7 +982,7 @@ void __init time_init(void) sys_tz.tz_dsttime = 0; } - vdso_data->tb_ticks_per_sec = tb_ticks_per_sec; + vdso_k_arch_data->tb_ticks_per_sec = tb_ticks_per_sec; #ifdef CONFIG_PPC64_PROC_SYSTEMCFG systemcfg->tb_ticks_per_sec = tb_ticks_per_sec; #endif @@ -974,39 +1006,6 @@ void __init time_init(void) enable_sched_clock_irqtime(); } -/* - * Divide a 128-bit dividend by a 32-bit divisor, leaving a 128 bit - * result. - */ -void div128_by_32(u64 dividend_high, u64 dividend_low, - unsigned divisor, struct div_result *dr) -{ - unsigned long a, b, c, d; - unsigned long w, x, y, z; - u64 ra, rb, rc; - - a = dividend_high >> 32; - b = dividend_high & 0xffffffff; - c = dividend_low >> 32; - d = dividend_low & 0xffffffff; - - w = a / divisor; - ra = ((u64)(a - (w * divisor)) << 32) + b; - - rb = ((u64) do_div(ra, divisor) << 32) + c; - x = ra; - - rc = ((u64) do_div(rb, divisor) << 32) + d; - y = rb; - - do_div(rc, divisor); - z = rc; - - dr->result_high = ((u64)w << 32) + x; - dr->result_low = ((u64)y << 32) + z; - -} - /* We don't need to calibrate delay, we use the CPU timebase for that */ void calibrate_delay(void) { diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c index 5ccd791761e8..6dca92d5a6e8 100644 --- a/arch/powerpc/kernel/trace/ftrace.c +++ b/arch/powerpc/kernel/trace/ftrace.c @@ -115,10 +115,8 @@ static unsigned long ftrace_lookup_module_stub(unsigned long ip, unsigned long a { struct module *mod = NULL; - preempt_disable(); - mod = __module_text_address(ip); - preempt_enable(); - + scoped_guard(rcu) + mod = __module_text_address(ip); if (!mod) pr_err("No module loaded at addr=%lx\n", ip); @@ -658,7 +656,6 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { unsigned long sp = arch_ftrace_regs(fregs)->regs.gpr[1]; - int bit; if (unlikely(ftrace_graph_is_dead())) goto out; @@ -666,14 +663,9 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, if (unlikely(atomic_read(¤t->tracing_graph_pause))) goto out; - bit = ftrace_test_recursion_trylock(ip, parent_ip); - if (bit < 0) - goto out; - - if (!function_graph_enter(parent_ip, ip, 0, (unsigned long *)sp)) + if (!function_graph_enter_regs(parent_ip, ip, 0, (unsigned long *)sp, fregs)) parent_ip = ppc_function_entry(return_to_handler); - ftrace_test_recursion_unlock(bit); out: arch_ftrace_regs(fregs)->regs.link = parent_ip; } diff --git a/arch/powerpc/kernel/trace/ftrace_64_pg.c b/arch/powerpc/kernel/trace/ftrace_64_pg.c index 98787376eb87..5c6e545d1708 100644 --- a/arch/powerpc/kernel/trace/ftrace_64_pg.c +++ b/arch/powerpc/kernel/trace/ftrace_64_pg.c @@ -120,10 +120,8 @@ static struct module *ftrace_lookup_module(struct dyn_ftrace *rec) { struct module *mod; - preempt_disable(); - mod = __module_text_address(rec->ip); - preempt_enable(); - + scoped_guard(rcu) + mod = __module_text_address(rec->ip); if (!mod) pr_err("No module loaded at addr=%lx\n", rec->ip); @@ -787,10 +785,10 @@ int ftrace_disable_ftrace_graph_caller(void) * in current thread info. Return the address we want to divert to. */ static unsigned long -__prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp) +__prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp, + struct ftrace_regs *fregs) { unsigned long return_hooker; - int bit; if (unlikely(ftrace_graph_is_dead())) goto out; @@ -798,16 +796,11 @@ __prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp if (unlikely(atomic_read(¤t->tracing_graph_pause))) goto out; - bit = ftrace_test_recursion_trylock(ip, parent); - if (bit < 0) - goto out; - return_hooker = ppc_function_entry(return_to_handler); - if (!function_graph_enter(parent, ip, 0, (unsigned long *)sp)) + if (!function_graph_enter_regs(parent, ip, 0, (unsigned long *)sp, fregs)) parent = return_hooker; - ftrace_test_recursion_unlock(bit); out: return parent; } @@ -816,13 +809,14 @@ out: void ftrace_graph_func(unsigned long ip, unsigned long parent_ip, struct ftrace_ops *op, struct ftrace_regs *fregs) { - arch_ftrace_regs(fregs)->regs.link = __prepare_ftrace_return(parent_ip, ip, arch_ftrace_regs(fregs)->regs.gpr[1]); + arch_ftrace_regs(fregs)->regs.link = __prepare_ftrace_return(parent_ip, ip, + arch_ftrace_regs(fregs)->regs.gpr[1], fregs); } #else unsigned long prepare_ftrace_return(unsigned long parent, unsigned long ip, unsigned long sp) { - return __prepare_ftrace_return(parent, ip, sp); + return __prepare_ftrace_return(parent, ip, sp, NULL); } #endif #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/powerpc/kernel/trace/ftrace_entry.S b/arch/powerpc/kernel/trace/ftrace_entry.S index 2c1b24100eca..3565c67fc638 100644 --- a/arch/powerpc/kernel/trace/ftrace_entry.S +++ b/arch/powerpc/kernel/trace/ftrace_entry.S @@ -212,10 +212,10 @@ bne- 1f mr r3, r15 +1: mtlr r3 .if \allregs == 0 REST_GPR(15, r1) .endif -1: mtlr r3 #endif /* Restore gprs */ diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index edf5cabe5dfd..cb8e9357383e 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -263,10 +263,9 @@ static int __die(const char *str, struct pt_regs *regs, long err) { printk("Oops: %s, sig: %ld [#%d]\n", str, err, ++die_counter); - printk("%s PAGE_SIZE=%luK%s%s%s%s%s%s %s\n", + printk("%s PAGE_SIZE=%luK%s %s%s%s%s %s\n", IS_ENABLED(CONFIG_CPU_LITTLE_ENDIAN) ? "LE" : "BE", PAGE_SIZE / 1024, get_mmu_str(), - IS_ENABLED(CONFIG_PREEMPT) ? " PREEMPT" : "", IS_ENABLED(CONFIG_SMP) ? " SMP" : "", IS_ENABLED(CONFIG_SMP) ? (" NR_CPUS=" __stringify(NR_CPUS)) : "", debug_pagealloc_enabled() ? " DEBUG_PAGEALLOC" : "", diff --git a/arch/powerpc/kernel/udbg.c b/arch/powerpc/kernel/udbg.c index 0a72a537f879..862b22b2b616 100644 --- a/arch/powerpc/kernel/udbg.c +++ b/arch/powerpc/kernel/udbg.c @@ -36,9 +36,6 @@ void __init udbg_early_init(void) #elif defined(CONFIG_PPC_EARLY_DEBUG_RTAS_PANEL) /* RTAS panel debug */ udbg_init_rtas_panel(); -#elif defined(CONFIG_PPC_EARLY_DEBUG_RTAS_CONSOLE) - /* RTAS console debug */ - udbg_init_rtas_console(); #elif defined(CONFIG_PPC_EARLY_DEBUG_PAS_REALMODE) udbg_init_pas_realmode(); #elif defined(CONFIG_PPC_EARLY_DEBUG_BOOTX) diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index 43379365ce1b..219d67bcf747 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -17,7 +17,7 @@ #include <linux/elf.h> #include <linux/security.h> #include <linux/syscalls.h> -#include <linux/time_namespace.h> +#include <linux/vdso_datastore.h> #include <vdso/datapage.h> #include <asm/syscall.h> @@ -32,6 +32,8 @@ #include <asm/vdso_datapage.h> #include <asm/setup.h> +static_assert(__VDSO_PAGES == VDSO_NR_PAGES); + /* The alignment of the vDSO */ #define VDSO_ALIGNMENT (1 << 16) @@ -40,24 +42,6 @@ extern char vdso64_start, vdso64_end; long sys_ni_syscall(void); -/* - * The vdso data page (aka. systemcfg for old ppc64 fans) is here. - * Once the early boot kernel code no longer needs to muck around - * with it, it will become dynamically allocated - */ -static union { - struct vdso_arch_data data; - u8 page[2 * PAGE_SIZE]; -} vdso_data_store __page_aligned_data; -struct vdso_arch_data *vdso_data = &vdso_data_store.data; - -enum vvar_pages { - VVAR_BASE_PAGE_OFFSET, - VVAR_TIME_PAGE_OFFSET, - VVAR_TIMENS_PAGE_OFFSET, - VVAR_NR_PAGES, -}; - static int vdso_mremap(const struct vm_special_mapping *sm, struct vm_area_struct *new_vma, unsigned long text_size) { @@ -96,14 +80,6 @@ static void vdso_close(const struct vm_special_mapping *sm, struct vm_area_struc mm->context.vdso = NULL; } -static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, - struct vm_area_struct *vma, struct vm_fault *vmf); - -static struct vm_special_mapping vvar_spec __ro_after_init = { - .name = "[vvar]", - .fault = vvar_fault, -}; - static struct vm_special_mapping vdso32_spec __ro_after_init = { .name = "[vdso]", .mremap = vdso32_mremap, @@ -116,73 +92,6 @@ static struct vm_special_mapping vdso64_spec __ro_after_init = { .close = vdso_close, }; -#ifdef CONFIG_TIME_NS -struct vdso_data *arch_get_vdso_data(void *vvar_page) -{ - return vvar_page; -} - -/* - * The vvar mapping contains data for a specific time namespace, so when a task - * changes namespace we must unmap its vvar data for the old namespace. - * Subsequent faults will map in data for the new namespace. - * - * For more details see timens_setup_vdso_data(). - */ -int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) -{ - struct mm_struct *mm = task->mm; - VMA_ITERATOR(vmi, mm, 0); - struct vm_area_struct *vma; - - mmap_read_lock(mm); - for_each_vma(vmi, vma) { - if (vma_is_special_mapping(vma, &vvar_spec)) - zap_vma_pages(vma); - } - mmap_read_unlock(mm); - - return 0; -} -#endif - -static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, - struct vm_area_struct *vma, struct vm_fault *vmf) -{ - struct page *timens_page = find_timens_vvar_page(vma); - unsigned long pfn; - - switch (vmf->pgoff) { - case VVAR_BASE_PAGE_OFFSET: - pfn = virt_to_pfn(vdso_data); - break; - case VVAR_TIME_PAGE_OFFSET: - if (timens_page) - pfn = page_to_pfn(timens_page); - else - pfn = virt_to_pfn(vdso_data->data); - break; -#ifdef CONFIG_TIME_NS - case VVAR_TIMENS_PAGE_OFFSET: - /* - * If a task belongs to a time namespace then a namespace - * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and - * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET - * offset. - * See also the comment near timens_setup_vdso_data(). - */ - if (!timens_page) - return VM_FAULT_SIGBUS; - pfn = virt_to_pfn(vdso_data->data); - break; -#endif /* CONFIG_TIME_NS */ - default: - return VM_FAULT_SIGBUS; - } - - return vmf_insert_pfn(vma, vmf->address, pfn); -} - /* * This is called from binfmt_elf, we create the special vma for the * vDSO and insert it into the mm struct tree @@ -191,7 +100,7 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int { unsigned long vdso_size, vdso_base, mappings_size; struct vm_special_mapping *vdso_spec; - unsigned long vvar_size = VVAR_NR_PAGES * PAGE_SIZE; + unsigned long vvar_size = VDSO_NR_PAGES * PAGE_SIZE; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; @@ -217,9 +126,7 @@ static int __arch_setup_additional_pages(struct linux_binprm *bprm, int uses_int /* Add required alignment. */ vdso_base = ALIGN(vdso_base, VDSO_ALIGNMENT); - vma = _install_special_mapping(mm, vdso_base, vvar_size, - VM_READ | VM_MAYREAD | VM_IO | - VM_DONTDUMP | VM_PFNMAP, &vvar_spec); + vma = vdso_install_vvar_mapping(mm, vdso_base); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -299,10 +206,10 @@ static void __init vdso_setup_syscall_map(void) for (i = 0; i < NR_syscalls; i++) { if (sys_call_table[i] != (void *)&sys_ni_syscall) - vdso_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); + vdso_k_arch_data->syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); if (IS_ENABLED(CONFIG_COMPAT) && compat_sys_call_table[i] != (void *)&sys_ni_syscall) - vdso_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); + vdso_k_arch_data->compat_syscall_map[i >> 5] |= 0x80000000UL >> (i & 0x1f); } } @@ -352,10 +259,10 @@ static struct page ** __init vdso_setup_pages(void *start, void *end) static int __init vdso_init(void) { #ifdef CONFIG_PPC64 - vdso_data->dcache_block_size = ppc64_caches.l1d.block_size; - vdso_data->icache_block_size = ppc64_caches.l1i.block_size; - vdso_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size; - vdso_data->icache_log_block_size = ppc64_caches.l1i.log_block_size; + vdso_k_arch_data->dcache_block_size = ppc64_caches.l1d.block_size; + vdso_k_arch_data->icache_block_size = ppc64_caches.l1i.block_size; + vdso_k_arch_data->dcache_log_block_size = ppc64_caches.l1d.log_block_size; + vdso_k_arch_data->icache_log_block_size = ppc64_caches.l1i.log_block_size; #endif /* CONFIG_PPC64 */ vdso_setup_syscall_map(); diff --git a/arch/powerpc/kernel/vdso/Makefile b/arch/powerpc/kernel/vdso/Makefile index 0e3ed6fb199f..8834dfe9d727 100644 --- a/arch/powerpc/kernel/vdso/Makefile +++ b/arch/powerpc/kernel/vdso/Makefile @@ -3,7 +3,7 @@ # List of files in the vdso, has to be asm only for now # Include the generic Makefile to check the built vdso. -include $(srctree)/lib/vdso/Makefile +include $(srctree)/lib/vdso/Makefile.include obj-vdso32 = sigtramp32-32.o gettimeofday-32.o datapage-32.o cacheflush-32.o note-32.o getcpu-32.o obj-vdso64 = sigtramp64-64.o gettimeofday-64.o datapage-64.o cacheflush-64.o note-64.o getcpu-64.o @@ -53,7 +53,7 @@ ldflags-$(CONFIG_LD_ORPHAN_WARN) += -Wl,--orphan-handling=$(CONFIG_LD_ORPHAN_WAR ldflags-y += $(filter-out $(CC_AUTO_VAR_INIT_ZERO_ENABLER) $(CC_FLAGS_FTRACE) -Wa$(comma)%, $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS)) CC32FLAGS := -m32 -CC32FLAGSREMOVE := -mcmodel=medium -mabi=elfv1 -mabi=elfv2 -mcall-aixdesc +CC32FLAGSREMOVE := -mcmodel=medium -mabi=elfv1 -mabi=elfv2 -mcall-aixdesc -mpcrel ifdef CONFIG_CC_IS_CLANG # This flag is supported by clang for 64-bit but not 32-bit so it will cause # an unused command line flag warning for this file. diff --git a/arch/powerpc/kernel/vdso/cacheflush.S b/arch/powerpc/kernel/vdso/cacheflush.S index 0085ae464dac..488d3ade11e6 100644 --- a/arch/powerpc/kernel/vdso/cacheflush.S +++ b/arch/powerpc/kernel/vdso/cacheflush.S @@ -30,7 +30,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) #ifdef CONFIG_PPC64 mflr r12 .cfi_register lr,r12 - get_datapage r10 + get_datapage r10 vdso_u_arch_data mtlr r12 .cfi_restore lr #endif diff --git a/arch/powerpc/kernel/vdso/datapage.S b/arch/powerpc/kernel/vdso/datapage.S index db8e167f0166..d23b2e8e2a34 100644 --- a/arch/powerpc/kernel/vdso/datapage.S +++ b/arch/powerpc/kernel/vdso/datapage.S @@ -28,7 +28,7 @@ V_FUNCTION_BEGIN(__kernel_get_syscall_map) mflr r12 .cfi_register lr,r12 mr. r4,r3 - get_datapage r3 + get_datapage r3 vdso_u_arch_data mtlr r12 #ifdef __powerpc64__ addi r3,r3,CFG_SYSCALL_MAP64 @@ -52,7 +52,7 @@ V_FUNCTION_BEGIN(__kernel_get_tbfreq) .cfi_startproc mflr r12 .cfi_register lr,r12 - get_datapage r3 + get_datapage r3 vdso_u_arch_data #ifndef __powerpc64__ lwz r4,(CFG_TB_TICKS_PER_SEC + 4)(r3) #endif diff --git a/arch/powerpc/kernel/vdso/gettimeofday.S b/arch/powerpc/kernel/vdso/gettimeofday.S index 5333848322ca..79c967212444 100644 --- a/arch/powerpc/kernel/vdso/gettimeofday.S +++ b/arch/powerpc/kernel/vdso/gettimeofday.S @@ -33,9 +33,9 @@ .cfi_rel_offset r2, PPC_MIN_STKFRM + STK_GOT #endif .ifeq \call_time - get_datapage r5 VDSO_DATA_OFFSET + get_datapage r5 vdso_u_time_data .else - get_datapage r4 VDSO_DATA_OFFSET + get_datapage r4 vdso_u_time_data .endif bl CFUNC(DOTSYM(\funct)) PPC_LL r0, PPC_MIN_STKFRM + PPC_LR_STKOFF(r1) diff --git a/arch/powerpc/kernel/vdso/vdso32.lds.S b/arch/powerpc/kernel/vdso/vdso32.lds.S index 1a1b0b6d681a..72a1012b8a20 100644 --- a/arch/powerpc/kernel/vdso/vdso32.lds.S +++ b/arch/powerpc/kernel/vdso/vdso32.lds.S @@ -6,6 +6,7 @@ #include <asm/vdso.h> #include <asm/page.h> #include <asm-generic/vmlinux.lds.h> +#include <vdso/datapage.h> #ifdef __LITTLE_ENDIAN__ OUTPUT_FORMAT("elf32-powerpcle", "elf32-powerpcle", "elf32-powerpcle") @@ -16,7 +17,8 @@ OUTPUT_ARCH(powerpc:common) SECTIONS { - PROVIDE(_vdso_datapage = . - 3 * PAGE_SIZE); + VDSO_VVAR_SYMS + . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/powerpc/kernel/vdso/vdso64.lds.S b/arch/powerpc/kernel/vdso/vdso64.lds.S index e21b5506cad6..32102a05eaa7 100644 --- a/arch/powerpc/kernel/vdso/vdso64.lds.S +++ b/arch/powerpc/kernel/vdso/vdso64.lds.S @@ -6,6 +6,7 @@ #include <asm/vdso.h> #include <asm/page.h> #include <asm-generic/vmlinux.lds.h> +#include <vdso/datapage.h> #ifdef __LITTLE_ENDIAN__ OUTPUT_FORMAT("elf64-powerpcle", "elf64-powerpcle", "elf64-powerpcle") @@ -16,7 +17,8 @@ OUTPUT_ARCH(powerpc:common64) SECTIONS { - PROVIDE(_vdso_datapage = . - 3 * PAGE_SIZE); + VDSO_VVAR_SYMS + . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/powerpc/kernel/vdso/vgettimeofday.c b/arch/powerpc/kernel/vdso/vgettimeofday.c index 55a287c9a736..6f5167d81af5 100644 --- a/arch/powerpc/kernel/vdso/vgettimeofday.c +++ b/arch/powerpc/kernel/vdso/vgettimeofday.c @@ -7,43 +7,43 @@ #ifdef __powerpc64__ int __c_kernel_clock_gettime(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_gettime_data(vd, clock, ts); } int __c_kernel_clock_getres(clockid_t clock_id, struct __kernel_timespec *res, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_getres_data(vd, clock_id, res); } #else int __c_kernel_clock_gettime(clockid_t clock, struct old_timespec32 *ts, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_gettime32_data(vd, clock, ts); } int __c_kernel_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_gettime_data(vd, clock, ts); } int __c_kernel_clock_getres(clockid_t clock_id, struct old_timespec32 *res, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_clock_getres_time32_data(vd, clock_id, res); } #endif int __c_kernel_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz, - const struct vdso_data *vd) + const struct vdso_time_data *vd) { return __cvdso_gettimeofday_data(vd, tv, tz); } -__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, const struct vdso_data *vd) +__kernel_old_time_t __c_kernel_time(__kernel_old_time_t *time, const struct vdso_time_data *vd) { return __cvdso_time_data(vd, time); } diff --git a/arch/powerpc/kernel/vdso32_wrapper.S b/arch/powerpc/kernel/vdso32_wrapper.S index 10f92f265d51..20bca3548b44 100644 --- a/arch/powerpc/kernel/vdso32_wrapper.S +++ b/arch/powerpc/kernel/vdso32_wrapper.S @@ -2,7 +2,7 @@ #include <linux/linkage.h> #include <asm/page.h> - __PAGE_ALIGNED_DATA + .section ".data..ro_after_init", "aw" .globl vdso32_start, vdso32_end .balign PAGE_SIZE diff --git a/arch/powerpc/kernel/vdso64_wrapper.S b/arch/powerpc/kernel/vdso64_wrapper.S index 839d1a61411d..1912936fa227 100644 --- a/arch/powerpc/kernel/vdso64_wrapper.S +++ b/arch/powerpc/kernel/vdso64_wrapper.S @@ -2,7 +2,7 @@ #include <linux/linkage.h> #include <asm/page.h> - __PAGE_ALIGNED_DATA + .section ".data..ro_after_init", "aw" .globl vdso64_start, vdso64_end .balign PAGE_SIZE diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index b4c9decc7a75..de6ee7d35cff 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -1,10 +1,4 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifdef CONFIG_PPC64 -#define PROVIDE32(x) PROVIDE(__unused__##x) -#else -#define PROVIDE32(x) PROVIDE(x) -#endif - #define BSS_FIRST_SECTIONS *(.bss.prominit) #define EMITS_PT_NOTE #define RO_EXCEPTION_TABLE_ALIGN 0 @@ -127,7 +121,6 @@ SECTIONS . = ALIGN(PAGE_SIZE); _etext = .; - PROVIDE32 (etext = .); /* Read-only data */ RO_DATA(PAGE_SIZE) @@ -394,7 +387,6 @@ SECTIONS . = ALIGN(PAGE_SIZE); _edata = .; - PROVIDE32 (edata = .); /* * And finally the bss @@ -404,7 +396,6 @@ SECTIONS . = ALIGN(PAGE_SIZE); _end = . ; - PROVIDE32 (end = .); DWARF_DEBUG ELF_DETAILS diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c index 8c464a5d8246..2429cb1c7baa 100644 --- a/arch/powerpc/kernel/watchdog.c +++ b/arch/powerpc/kernel/watchdog.c @@ -495,8 +495,7 @@ static void start_watchdog(void *arg) *this_cpu_ptr(&wd_timer_tb) = get_tb(); - hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hrtimer->function = watchdog_timer_fn; + hrtimer_setup(hrtimer, watchdog_timer_fn, CLOCK_MONOTONIC, HRTIMER_MODE_REL); hrtimer_start(hrtimer, ms_to_ktime(wd_timer_period_ms), HRTIMER_MODE_REL_PINNED); } diff --git a/arch/powerpc/kexec/core.c b/arch/powerpc/kexec/core.c index b8333a49ea5d..d1a2d755381c 100644 --- a/arch/powerpc/kexec/core.c +++ b/arch/powerpc/kexec/core.c @@ -22,28 +22,6 @@ #include <asm/setup.h> #include <asm/firmware.h> -void machine_kexec_mask_interrupts(void) { - unsigned int i; - struct irq_desc *desc; - - for_each_irq_desc(i, desc) { - struct irq_chip *chip; - - chip = irq_desc_get_chip(desc); - if (!chip) - continue; - - if (chip->irq_eoi && irqd_irq_inprogress(&desc->irq_data)) - chip->irq_eoi(&desc->irq_data); - - if (chip->irq_mask) - chip->irq_mask(&desc->irq_data); - - if (chip->irq_disable && !irqd_irq_disabled(&desc->irq_data)) - chip->irq_disable(&desc->irq_data); - } -} - #ifdef CONFIG_CRASH_DUMP void machine_crash_shutdown(struct pt_regs *regs) { @@ -80,38 +58,20 @@ void machine_kexec(struct kimage *image) } #ifdef CONFIG_CRASH_RESERVE -void __init reserve_crashkernel(void) -{ - unsigned long long crash_size, crash_base, total_mem_sz; - int ret; - - total_mem_sz = memory_limit ? memory_limit : memblock_phys_mem_size(); - /* use common parsing */ - ret = parse_crashkernel(boot_command_line, total_mem_sz, - &crash_size, &crash_base, NULL, NULL); - if (ret == 0 && crash_size > 0) { - crashk_res.start = crash_base; - crashk_res.end = crash_base + crash_size - 1; - } - - if (crashk_res.end == crashk_res.start) { - crashk_res.start = crashk_res.end = 0; - return; - } - /* We might have got these values via the command line or the - * device tree, either way sanitise them now. */ - - crash_size = resource_size(&crashk_res); +static unsigned long long __init get_crash_base(unsigned long long crash_base) +{ #ifndef CONFIG_NONSTATIC_KERNEL - if (crashk_res.start != KDUMP_KERNELBASE) + if (crash_base != KDUMP_KERNELBASE) printk("Crash kernel location must be 0x%x\n", KDUMP_KERNELBASE); - crashk_res.start = KDUMP_KERNELBASE; + return KDUMP_KERNELBASE; #else - if (!crashk_res.start) { + unsigned long long crash_base_align; + + if (!crash_base) { #ifdef CONFIG_PPC64 /* * On the LPAR platform place the crash kernel to mid of @@ -123,53 +83,51 @@ void __init reserve_crashkernel(void) * kernel starts at 128MB offset on other platforms. */ if (firmware_has_feature(FW_FEATURE_LPAR)) - crashk_res.start = min_t(u64, ppc64_rma_size / 2, SZ_512M); + crash_base = min_t(u64, ppc64_rma_size / 2, SZ_512M); else - crashk_res.start = min_t(u64, ppc64_rma_size / 2, SZ_128M); + crash_base = min_t(u64, ppc64_rma_size / 2, SZ_128M); #else - crashk_res.start = KDUMP_KERNELBASE; + crash_base = KDUMP_KERNELBASE; #endif } - crash_base = PAGE_ALIGN(crashk_res.start); - if (crash_base != crashk_res.start) { - printk("Crash kernel base must be aligned to 0x%lx\n", - PAGE_SIZE); - crashk_res.start = crash_base; - } + crash_base_align = PAGE_ALIGN(crash_base); + if (crash_base != crash_base_align) + pr_warn("Crash kernel base must be aligned to 0x%lx\n", PAGE_SIZE); + return crash_base_align; #endif - crash_size = PAGE_ALIGN(crash_size); - crashk_res.end = crashk_res.start + crash_size - 1; +} - /* The crash region must not overlap the current kernel */ - if (overlaps_crashkernel(__pa(_stext), _end - _stext)) { - printk(KERN_WARNING - "Crash kernel can not overlap current kernel\n"); - crashk_res.start = crashk_res.end = 0; +void __init arch_reserve_crashkernel(void) +{ + unsigned long long crash_size, crash_base, crash_end; + unsigned long long kernel_start, kernel_size; + unsigned long long total_mem_sz; + int ret; + + total_mem_sz = memory_limit ? memory_limit : memblock_phys_mem_size(); + + /* use common parsing */ + ret = parse_crashkernel(boot_command_line, total_mem_sz, &crash_size, + &crash_base, NULL, NULL, NULL); + + if (ret) return; - } - /* Crash kernel trumps memory limit */ - if (memory_limit && memory_limit <= crashk_res.end) { - memory_limit = crashk_res.end + 1; - total_mem_sz = memory_limit; - printk("Adjusted memory limit for crashkernel, now 0x%llx\n", - memory_limit); - } + crash_base = get_crash_base(crash_base); + crash_end = crash_base + crash_size - 1; - printk(KERN_INFO "Reserving %ldMB of memory at %ldMB " - "for crashkernel (System RAM: %ldMB)\n", - (unsigned long)(crash_size >> 20), - (unsigned long)(crashk_res.start >> 20), - (unsigned long)(total_mem_sz >> 20)); + kernel_start = __pa(_stext); + kernel_size = _end - _stext; - if (!memblock_is_region_memory(crashk_res.start, crash_size) || - memblock_reserve(crashk_res.start, crash_size)) { - pr_err("Failed to reserve memory for crashkernel!\n"); - crashk_res.start = crashk_res.end = 0; + /* The crash region must not overlap the current kernel */ + if ((kernel_start + kernel_size > crash_base) && (kernel_start <= crash_end)) { + pr_warn("Crash kernel can not overlap current kernel\n"); return; } + + reserve_crashkernel_generic(crash_size, crash_base, 0, false); } int __init overlaps_crashkernel(unsigned long start, unsigned long size) diff --git a/arch/powerpc/kexec/core_32.c b/arch/powerpc/kexec/core_32.c index c95f96850c9e..deb28eb44f30 100644 --- a/arch/powerpc/kexec/core_32.c +++ b/arch/powerpc/kexec/core_32.c @@ -7,6 +7,7 @@ * Copyright (C) 2005 IBM Corporation. */ +#include <linux/irq.h> #include <linux/kexec.h> #include <linux/mm.h> #include <linux/string.h> diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c index 9ac3266e4965..a325c1c02f96 100644 --- a/arch/powerpc/kexec/crash.c +++ b/arch/powerpc/kexec/crash.c @@ -359,7 +359,10 @@ void default_machine_crash_shutdown(struct pt_regs *regs) if (TRAP(regs) == INTERRUPT_SYSTEM_RESET) is_via_system_reset = 1; - crash_smp_send_stop(); + if (IS_ENABLED(CONFIG_SMP)) + crash_smp_send_stop(); + else + crash_kexec_prepare(); crash_save_cpu(regs, crashing_cpu); diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index dc65c1391157..e7ef8b2a2554 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -49,201 +49,18 @@ const struct kexec_file_ops * const kexec_file_loaders[] = { NULL }; -/** - * __locate_mem_hole_top_down - Looks top down for a large enough memory hole - * in the memory regions between buf_min & buf_max - * for the buffer. If found, sets kbuf->mem. - * @kbuf: Buffer contents and memory parameters. - * @buf_min: Minimum address for the buffer. - * @buf_max: Maximum address for the buffer. - * - * Returns 0 on success, negative errno on error. - */ -static int __locate_mem_hole_top_down(struct kexec_buf *kbuf, - u64 buf_min, u64 buf_max) -{ - int ret = -EADDRNOTAVAIL; - phys_addr_t start, end; - u64 i; - - for_each_mem_range_rev(i, &start, &end) { - /* - * memblock uses [start, end) convention while it is - * [start, end] here. Fix the off-by-one to have the - * same convention. - */ - end -= 1; - - if (start > buf_max) - continue; - - /* Memory hole not found */ - if (end < buf_min) - break; - - /* Adjust memory region based on the given range */ - if (start < buf_min) - start = buf_min; - if (end > buf_max) - end = buf_max; - - start = ALIGN(start, kbuf->buf_align); - if (start < end && (end - start + 1) >= kbuf->memsz) { - /* Suitable memory range found. Set kbuf->mem */ - kbuf->mem = ALIGN_DOWN(end - kbuf->memsz + 1, - kbuf->buf_align); - ret = 0; - break; - } - } - - return ret; -} - -/** - * locate_mem_hole_top_down_ppc64 - Skip special memory regions to find a - * suitable buffer with top down approach. - * @kbuf: Buffer contents and memory parameters. - * @buf_min: Minimum address for the buffer. - * @buf_max: Maximum address for the buffer. - * @emem: Exclude memory ranges. - * - * Returns 0 on success, negative errno on error. - */ -static int locate_mem_hole_top_down_ppc64(struct kexec_buf *kbuf, - u64 buf_min, u64 buf_max, - const struct crash_mem *emem) +int arch_check_excluded_range(struct kimage *image, unsigned long start, + unsigned long end) { - int i, ret = 0, err = -EADDRNOTAVAIL; - u64 start, end, tmin, tmax; - - tmax = buf_max; - for (i = (emem->nr_ranges - 1); i >= 0; i--) { - start = emem->ranges[i].start; - end = emem->ranges[i].end; - - if (start > tmax) - continue; - - if (end < tmax) { - tmin = (end < buf_min ? buf_min : end + 1); - ret = __locate_mem_hole_top_down(kbuf, tmin, tmax); - if (!ret) - return 0; - } - - tmax = start - 1; - - if (tmax < buf_min) { - ret = err; - break; - } - ret = 0; - } - - if (!ret) { - tmin = buf_min; - ret = __locate_mem_hole_top_down(kbuf, tmin, tmax); - } - return ret; -} - -/** - * __locate_mem_hole_bottom_up - Looks bottom up for a large enough memory hole - * in the memory regions between buf_min & buf_max - * for the buffer. If found, sets kbuf->mem. - * @kbuf: Buffer contents and memory parameters. - * @buf_min: Minimum address for the buffer. - * @buf_max: Maximum address for the buffer. - * - * Returns 0 on success, negative errno on error. - */ -static int __locate_mem_hole_bottom_up(struct kexec_buf *kbuf, - u64 buf_min, u64 buf_max) -{ - int ret = -EADDRNOTAVAIL; - phys_addr_t start, end; - u64 i; - - for_each_mem_range(i, &start, &end) { - /* - * memblock uses [start, end) convention while it is - * [start, end] here. Fix the off-by-one to have the - * same convention. - */ - end -= 1; - - if (end < buf_min) - continue; - - /* Memory hole not found */ - if (start > buf_max) - break; - - /* Adjust memory region based on the given range */ - if (start < buf_min) - start = buf_min; - if (end > buf_max) - end = buf_max; - - start = ALIGN(start, kbuf->buf_align); - if (start < end && (end - start + 1) >= kbuf->memsz) { - /* Suitable memory range found. Set kbuf->mem */ - kbuf->mem = start; - ret = 0; - break; - } - } - - return ret; -} - -/** - * locate_mem_hole_bottom_up_ppc64 - Skip special memory regions to find a - * suitable buffer with bottom up approach. - * @kbuf: Buffer contents and memory parameters. - * @buf_min: Minimum address for the buffer. - * @buf_max: Maximum address for the buffer. - * @emem: Exclude memory ranges. - * - * Returns 0 on success, negative errno on error. - */ -static int locate_mem_hole_bottom_up_ppc64(struct kexec_buf *kbuf, - u64 buf_min, u64 buf_max, - const struct crash_mem *emem) -{ - int i, ret = 0, err = -EADDRNOTAVAIL; - u64 start, end, tmin, tmax; - - tmin = buf_min; - for (i = 0; i < emem->nr_ranges; i++) { - start = emem->ranges[i].start; - end = emem->ranges[i].end; - - if (end < tmin) - continue; - - if (start > tmin) { - tmax = (start > buf_max ? buf_max : start - 1); - ret = __locate_mem_hole_bottom_up(kbuf, tmin, tmax); - if (!ret) - return 0; - } - - tmin = end + 1; + struct crash_mem *emem; + int i; - if (tmin > buf_max) { - ret = err; - break; - } - ret = 0; - } + emem = image->arch.exclude_ranges; + for (i = 0; i < emem->nr_ranges; i++) + if (start < emem->ranges[i].end && end > emem->ranges[i].start) + return 1; - if (!ret) { - tmax = buf_max; - ret = __locate_mem_hole_bottom_up(kbuf, tmin, tmax); - } - return ret; + return 0; } #ifdef CONFIG_CRASH_DUMP @@ -1005,64 +822,6 @@ out: } /** - * arch_kexec_locate_mem_hole - Skip special memory regions like rtas, opal, - * tce-table, reserved-ranges & such (exclude - * memory ranges) as they can't be used for kexec - * segment buffer. Sets kbuf->mem when a suitable - * memory hole is found. - * @kbuf: Buffer contents and memory parameters. - * - * Assumes minimum of PAGE_SIZE alignment for kbuf->memsz & kbuf->buf_align. - * - * Returns 0 on success, negative errno on error. - */ -int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf) -{ - struct crash_mem **emem; - u64 buf_min, buf_max; - int ret; - - /* Look up the exclude ranges list while locating the memory hole */ - emem = &(kbuf->image->arch.exclude_ranges); - if (!(*emem) || ((*emem)->nr_ranges == 0)) { - pr_warn("No exclude range list. Using the default locate mem hole method\n"); - return kexec_locate_mem_hole(kbuf); - } - - buf_min = kbuf->buf_min; - buf_max = kbuf->buf_max; - /* Segments for kdump kernel should be within crashkernel region */ - if (IS_ENABLED(CONFIG_CRASH_DUMP) && kbuf->image->type == KEXEC_TYPE_CRASH) { - buf_min = (buf_min < crashk_res.start ? - crashk_res.start : buf_min); - buf_max = (buf_max > crashk_res.end ? - crashk_res.end : buf_max); - } - - if (buf_min > buf_max) { - pr_err("Invalid buffer min and/or max values\n"); - return -EINVAL; - } - - if (kbuf->top_down) - ret = locate_mem_hole_top_down_ppc64(kbuf, buf_min, buf_max, - *emem); - else - ret = locate_mem_hole_bottom_up_ppc64(kbuf, buf_min, buf_max, - *emem); - - /* Add the buffer allocated to the exclude list for the next lookup */ - if (!ret) { - add_mem_range(emem, kbuf->mem, kbuf->memsz); - sort_memory_ranges(*emem, true); - } else { - pr_err("Failed to locate memory buffer of size %lu\n", - kbuf->memsz); - } - return ret; -} - -/** * arch_kexec_kernel_image_probe - Does additional handling needed to setup * kexec segments. * @image: kexec image being loaded. diff --git a/arch/powerpc/kexec/relocate_32.S b/arch/powerpc/kexec/relocate_32.S index 104c9911f406..dd86e338307d 100644 --- a/arch/powerpc/kexec/relocate_32.S +++ b/arch/powerpc/kexec/relocate_32.S @@ -348,16 +348,13 @@ write_utlb: rlwinm r10, r24, 0, 22, 27 cmpwi r10, PPC47x_TLB0_4K - bne 0f li r10, 0x1000 /* r10 = 4k */ - ANNOTATE_INTRA_FUNCTION_CALL - bl 1f + beq 0f -0: /* Defaults to 256M */ lis r10, 0x1000 - bcl 20,31,$+4 +0: bcl 20,31,$+4 1: mflr r4 addi r4, r4, (2f-1b) /* virtual address of 2f */ diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index dbfdc126bf14..2f2702c867f7 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -83,6 +83,7 @@ config KVM_BOOK3S_64_HV depends on KVM_BOOK3S_64 && PPC_POWERNV select KVM_BOOK3S_HV_POSSIBLE select KVM_GENERIC_MMU_NOTIFIER + select KVM_BOOK3S_HV_PMU select CMA help Support running unmodified book3s_64 guest kernels in @@ -171,6 +172,18 @@ config KVM_BOOK3S_HV_NESTED_PMU_WORKAROUND those buggy L1s which saves the L2 state, at the cost of performance in all nested-capable guest entry/exit. +config KVM_BOOK3S_HV_PMU + tristate "Hypervisor Perf events for KVM Book3s-HV" + depends on KVM_BOOK3S_64_HV + help + Enable Book3s-HV Hypervisor Perf events PMU named 'kvm-hv'. These + Perf events give an overview of hypervisor performance overall + instead of a specific guests. Currently the PMU reports + L0-Hypervisor stats on a kvm-hv enabled PSeries LPAR like: + * Total/Used Guest-Heap + * Total/Used Guest Page-table Memory + * Total amount of Guest Page-table Memory reclaimed + config KVM_BOOKE_HV bool diff --git a/arch/powerpc/kvm/book3s_32_mmu_host.c b/arch/powerpc/kvm/book3s_32_mmu_host.c index 5b7212edbb13..c7e4b62642ea 100644 --- a/arch/powerpc/kvm/book3s_32_mmu_host.c +++ b/arch/powerpc/kvm/book3s_32_mmu_host.c @@ -125,8 +125,6 @@ static u32 *kvmppc_mmu_get_pteg(struct kvm_vcpu *vcpu, u32 vsid, u32 eaddr, return (u32*)pteg; } -extern char etext[]; - int kvmppc_mmu_map_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *orig_pte, bool iswrite) { diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 25429905ae90..7667563fb9ff 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -4957,7 +4957,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, * states are synchronized from L0 to L1. L1 needs to inform L0 about * MER=1 only when there are pending external interrupts. * In the above if check, MER bit is set if there are pending - * external interrupts. Hence, explicity mask off MER bit + * external interrupts. Hence, explicitly mask off MER bit * here as otherwise it may generate spurious interrupts in L2 KVM * causing an endless loop, which results in L2 guest getting hung. */ @@ -6041,7 +6041,7 @@ static int kvmppc_set_passthru_irq(struct kvm *kvm, int host_irq, int guest_gsi) * the underlying calls, which will EOI the interrupt in real * mode, need an HW IRQ number mapped in the XICS IRQ domain. */ - host_data = irq_domain_get_irq_data(irq_get_default_host(), host_irq); + host_data = irq_domain_get_irq_data(irq_get_default_domain(), host_irq); irq_map->r_hwirq = (unsigned int)irqd_to_hwirq(host_data); if (i == pimap->n_mapped) @@ -6541,10 +6541,6 @@ static struct kvmppc_ops kvm_ops_hv = { .fast_vcpu_kick = kvmppc_fast_vcpu_kick_hv, .arch_vm_ioctl = kvm_arch_vm_ioctl_hv, .hcall_implemented = kvmppc_hcall_impl_hv, -#ifdef CONFIG_KVM_XICS - .irq_bypass_add_producer = kvmppc_irq_bypass_add_producer_hv, - .irq_bypass_del_producer = kvmppc_irq_bypass_del_producer_hv, -#endif .configure_mmu = kvmhv_configure_mmu, .get_rmmu_info = kvmhv_get_rmmu_info, .set_smt_mode = kvmhv_set_smt_mode, @@ -6662,6 +6658,22 @@ static int kvmppc_book3s_init_hv(void) return r; } +#if defined(CONFIG_KVM_XICS) + /* + * IRQ bypass is supported only for interrupts whose EOI operations are + * handled via OPAL calls. Therefore, register IRQ bypass handlers + * exclusively for PowerNV KVM when booted with 'xive=off', indicating + * the use of the emulated XICS interrupt controller. + */ + if (!kvmhv_on_pseries()) { + pr_info("KVM-HV: Enabling IRQ bypass\n"); + kvm_ops_hv.irq_bypass_add_producer = + kvmppc_irq_bypass_add_producer_hv; + kvm_ops_hv.irq_bypass_del_producer = + kvmppc_irq_bypass_del_producer_hv; + } +#endif + kvm_ops_hv.owner = THIS_MODULE; kvmppc_hv_ops = &kvm_ops_hv; diff --git a/arch/powerpc/kvm/book3s_hv_nestedv2.c b/arch/powerpc/kvm/book3s_hv_nestedv2.c index e5c7ce1fb761..87691cf86cae 100644 --- a/arch/powerpc/kvm/book3s_hv_nestedv2.c +++ b/arch/powerpc/kvm/book3s_hv_nestedv2.c @@ -123,6 +123,12 @@ static size_t gs_msg_ops_vcpu_get_size(struct kvmppc_gs_msg *gsm) case KVMPPC_GSID_PROCESS_TABLE: case KVMPPC_GSID_RUN_INPUT: case KVMPPC_GSID_RUN_OUTPUT: + /* Host wide counters */ + case KVMPPC_GSID_L0_GUEST_HEAP: + case KVMPPC_GSID_L0_GUEST_HEAP_MAX: + case KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE: + case KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX: + case KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM: break; default: size += kvmppc_gse_total_size(kvmppc_gsid_size(iden)); diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index ea7ad200b330..83f7504349d2 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -1524,14 +1524,12 @@ kvm_flush_link_stack: /* Flush the link stack. On Power8 it's up to 32 entries in size. */ .rept 32 - ANNOTATE_INTRA_FUNCTION_CALL bl .+4 .endr /* And on Power9 it's up to 64. */ BEGIN_FTR_SECTION .rept 32 - ANNOTATE_INTRA_FUNCTION_CALL bl .+4 .endr END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index 3a6592a31a10..03f8c34fa0a2 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -393,7 +393,7 @@ static int kvmppc_memslot_page_merge(struct kvm *kvm, { unsigned long gfn = memslot->base_gfn; unsigned long end, start = gfn_to_hva(kvm, gfn); - unsigned long vm_flags; + vm_flags_t vm_flags; int ret = 0; struct vm_area_struct *vma; int merge_flag = (merge) ? MADV_MERGEABLE : MADV_UNMERGEABLE; diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index 1362c672387e..1302b5ac5672 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -1555,7 +1555,7 @@ int kvmppc_xive_set_mapped(struct kvm *kvm, unsigned long guest_irq, struct kvmppc_xive_src_block *sb; struct kvmppc_xive_irq_state *state; struct irq_data *host_data = - irq_domain_get_irq_data(irq_get_default_host(), host_irq); + irq_domain_get_irq_data(irq_get_default_domain(), host_irq); unsigned int hw_irq = (unsigned int)irqd_to_hwirq(host_data); u16 idx; u8 prio; diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 6a5be025a8af..3401b96be475 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -572,7 +572,7 @@ static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu, /* * Return the number of jiffies until the next timeout. If the timeout is - * longer than the NEXT_TIMER_MAX_DELTA, then return NEXT_TIMER_MAX_DELTA + * longer than the TIMER_NEXT_MAX_DELTA, then return TIMER_NEXT_MAX_DELTA * because the larger value can break the timer APIs. */ static unsigned long watchdog_next_timeout(struct kvm_vcpu *vcpu) @@ -598,7 +598,7 @@ static unsigned long watchdog_next_timeout(struct kvm_vcpu *vcpu) if (do_div(nr_jiffies, tb_ticks_per_jiffy)) nr_jiffies++; - return min_t(unsigned long long, nr_jiffies, NEXT_TIMER_MAX_DELTA); + return min_t(unsigned long long, nr_jiffies, TIMER_NEXT_MAX_DELTA); } static void arm_next_watchdog(struct kvm_vcpu *vcpu) @@ -616,19 +616,19 @@ static void arm_next_watchdog(struct kvm_vcpu *vcpu) spin_lock_irqsave(&vcpu->arch.wdt_lock, flags); nr_jiffies = watchdog_next_timeout(vcpu); /* - * If the number of jiffies of watchdog timer >= NEXT_TIMER_MAX_DELTA + * If the number of jiffies of watchdog timer >= TIMER_NEXT_MAX_DELTA * then do not run the watchdog timer as this can break timer APIs. */ - if (nr_jiffies < NEXT_TIMER_MAX_DELTA) + if (nr_jiffies < TIMER_NEXT_MAX_DELTA) mod_timer(&vcpu->arch.wdt_timer, jiffies + nr_jiffies); else - del_timer(&vcpu->arch.wdt_timer); + timer_delete(&vcpu->arch.wdt_timer); spin_unlock_irqrestore(&vcpu->arch.wdt_lock, flags); } static void kvmppc_watchdog_func(struct timer_list *t) { - struct kvm_vcpu *vcpu = from_timer(vcpu, t, arch.wdt_timer); + struct kvm_vcpu *vcpu = timer_container_of(vcpu, t, arch.wdt_timer); u32 tsr, new_tsr; int final; @@ -1441,7 +1441,7 @@ int kvmppc_subarch_vcpu_init(struct kvm_vcpu *vcpu) void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu) { - del_timer_sync(&vcpu->arch.wdt_timer); + timer_delete_sync(&vcpu->arch.wdt_timer); } int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) diff --git a/arch/powerpc/kvm/e500.h b/arch/powerpc/kvm/e500.h index 6d0d329cbb35..f9acf866c709 100644 --- a/arch/powerpc/kvm/e500.h +++ b/arch/powerpc/kvm/e500.h @@ -34,6 +34,8 @@ enum vcpu_ftr { #define E500_TLB_BITMAP (1 << 30) /* TLB1 entry is mapped by host TLB0 */ #define E500_TLB_TLB0 (1 << 29) +/* entry is writable on the host */ +#define E500_TLB_WRITABLE (1 << 28) /* bits [6-5] MAS2_X1 and MAS2_X0 and [4-0] bits for WIMGE */ #define E500_TLB_MAS2_ATTR (0x7f) diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c index e5a145b578a4..06caf8bbbe2b 100644 --- a/arch/powerpc/kvm/e500_mmu_host.c +++ b/arch/powerpc/kvm/e500_mmu_host.c @@ -45,11 +45,14 @@ static inline unsigned int tlb1_max_shadow_size(void) return host_tlb_params[1].entries - tlbcam_index - 1; } -static inline u32 e500_shadow_mas3_attrib(u32 mas3, int usermode) +static inline u32 e500_shadow_mas3_attrib(u32 mas3, bool writable, int usermode) { /* Mask off reserved bits. */ mas3 &= MAS3_ATTRIB_MASK; + if (!writable) + mas3 &= ~(MAS3_UW|MAS3_SW); + #ifndef CONFIG_KVM_BOOKE_HV if (!usermode) { /* Guest is in supervisor mode, @@ -242,17 +245,18 @@ static inline int tlbe_is_writable(struct kvm_book3e_206_tlb_entry *tlbe) return tlbe->mas7_3 & (MAS3_SW|MAS3_UW); } -static inline bool kvmppc_e500_ref_setup(struct tlbe_ref *ref, +static inline void kvmppc_e500_ref_setup(struct tlbe_ref *ref, struct kvm_book3e_206_tlb_entry *gtlbe, - kvm_pfn_t pfn, unsigned int wimg) + kvm_pfn_t pfn, unsigned int wimg, + bool writable) { ref->pfn = pfn; ref->flags = E500_TLB_VALID; + if (writable) + ref->flags |= E500_TLB_WRITABLE; /* Use guest supplied MAS2_G and MAS2_E */ ref->flags |= (gtlbe->mas2 & MAS2_ATTRIB_MASK) | wimg; - - return tlbe_is_writable(gtlbe); } static inline void kvmppc_e500_ref_release(struct tlbe_ref *ref) @@ -305,6 +309,7 @@ static void kvmppc_e500_setup_stlbe( { kvm_pfn_t pfn = ref->pfn; u32 pr = vcpu->arch.shared->msr & MSR_PR; + bool writable = !!(ref->flags & E500_TLB_WRITABLE); BUG_ON(!(ref->flags & E500_TLB_VALID)); @@ -312,7 +317,7 @@ static void kvmppc_e500_setup_stlbe( stlbe->mas1 = MAS1_TSIZE(tsize) | get_tlb_sts(gtlbe) | MAS1_VALID; stlbe->mas2 = (gvaddr & MAS2_EPN) | (ref->flags & E500_TLB_MAS2_ATTR); stlbe->mas7_3 = ((u64)pfn << PAGE_SHIFT) | - e500_shadow_mas3_attrib(gtlbe->mas7_3, pr); + e500_shadow_mas3_attrib(gtlbe->mas7_3, writable, pr); } static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, @@ -321,15 +326,14 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, struct tlbe_ref *ref) { struct kvm_memory_slot *slot; - unsigned long pfn = 0; /* silence GCC warning */ + unsigned int psize; + unsigned long pfn; struct page *page = NULL; unsigned long hva; - int pfnmap = 0; int tsize = BOOK3E_PAGESZ_4K; int ret = 0; unsigned long mmu_seq; struct kvm *kvm = vcpu_e500->vcpu.kvm; - unsigned long tsize_pages = 0; pte_t *ptep; unsigned int wimg = 0; pgd_t *pgdir; @@ -351,110 +355,12 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, slot = gfn_to_memslot(vcpu_e500->vcpu.kvm, gfn); hva = gfn_to_hva_memslot(slot, gfn); - if (tlbsel == 1) { - struct vm_area_struct *vma; - mmap_read_lock(kvm->mm); - - vma = find_vma(kvm->mm, hva); - if (vma && hva >= vma->vm_start && - (vma->vm_flags & VM_PFNMAP)) { - /* - * This VMA is a physically contiguous region (e.g. - * /dev/mem) that bypasses normal Linux page - * management. Find the overlap between the - * vma and the memslot. - */ - - unsigned long start, end; - unsigned long slot_start, slot_end; - - pfnmap = 1; - - start = vma->vm_pgoff; - end = start + - vma_pages(vma); - - pfn = start + ((hva - vma->vm_start) >> PAGE_SHIFT); - - slot_start = pfn - (gfn - slot->base_gfn); - slot_end = slot_start + slot->npages; - - if (start < slot_start) - start = slot_start; - if (end > slot_end) - end = slot_end; - - tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >> - MAS1_TSIZE_SHIFT; - - /* - * e500 doesn't implement the lowest tsize bit, - * or 1K pages. - */ - tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1); - - /* - * Now find the largest tsize (up to what the guest - * requested) that will cover gfn, stay within the - * range, and for which gfn and pfn are mutually - * aligned. - */ - - for (; tsize > BOOK3E_PAGESZ_4K; tsize -= 2) { - unsigned long gfn_start, gfn_end; - tsize_pages = 1UL << (tsize - 2); - - gfn_start = gfn & ~(tsize_pages - 1); - gfn_end = gfn_start + tsize_pages; - - if (gfn_start + pfn - gfn < start) - continue; - if (gfn_end + pfn - gfn > end) - continue; - if ((gfn & (tsize_pages - 1)) != - (pfn & (tsize_pages - 1))) - continue; - - gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1); - pfn &= ~(tsize_pages - 1); - break; - } - } else if (vma && hva >= vma->vm_start && - is_vm_hugetlb_page(vma)) { - unsigned long psize = vma_kernel_pagesize(vma); - - tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >> - MAS1_TSIZE_SHIFT; - - /* - * Take the largest page size that satisfies both host - * and guest mapping - */ - tsize = min(__ilog2(psize) - 10, tsize); - - /* - * e500 doesn't implement the lowest tsize bit, - * or 1K pages. - */ - tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1); - } - - mmap_read_unlock(kvm->mm); - } - - if (likely(!pfnmap)) { - tsize_pages = 1UL << (tsize + 10 - PAGE_SHIFT); - pfn = __kvm_faultin_pfn(slot, gfn, FOLL_WRITE, NULL, &page); - if (is_error_noslot_pfn(pfn)) { - if (printk_ratelimit()) - pr_err("%s: real page not found for gfn %lx\n", - __func__, (long)gfn); - return -EINVAL; - } - - /* Align guest and physical address to page map boundaries */ - pfn &= ~(tsize_pages - 1); - gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1); + pfn = __kvm_faultin_pfn(slot, gfn, FOLL_WRITE, &writable, &page); + if (is_error_noslot_pfn(pfn)) { + if (printk_ratelimit()) + pr_err("%s: real page not found for gfn %lx\n", + __func__, (long)gfn); + return -EINVAL; } spin_lock(&kvm->mmu_lock); @@ -472,14 +378,13 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, * can't run hence pfn won't change. */ local_irq_save(flags); - ptep = find_linux_pte(pgdir, hva, NULL, NULL); + ptep = find_linux_pte(pgdir, hva, NULL, &psize); if (ptep) { pte_t pte = READ_ONCE(*ptep); if (pte_present(pte)) { wimg = (pte_val(pte) >> PTE_WIMGE_SHIFT) & MAS2_WIMGE_MASK; - local_irq_restore(flags); } else { local_irq_restore(flags); pr_err_ratelimited("%s: pte not present: gfn %lx,pfn %lx\n", @@ -488,10 +393,72 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500, goto out; } } - writable = kvmppc_e500_ref_setup(ref, gtlbe, pfn, wimg); + local_irq_restore(flags); + + if (psize && tlbsel == 1) { + unsigned long psize_pages, tsize_pages; + unsigned long start, end; + unsigned long slot_start, slot_end; + + psize_pages = 1UL << (psize - PAGE_SHIFT); + start = pfn & ~(psize_pages - 1); + end = start + psize_pages; + + slot_start = pfn - (gfn - slot->base_gfn); + slot_end = slot_start + slot->npages; + + if (start < slot_start) + start = slot_start; + if (end > slot_end) + end = slot_end; + + tsize = (gtlbe->mas1 & MAS1_TSIZE_MASK) >> + MAS1_TSIZE_SHIFT; + + /* + * Any page size that doesn't satisfy the host mapping + * will fail the start and end tests. + */ + tsize = min(psize - PAGE_SHIFT + BOOK3E_PAGESZ_4K, tsize); + + /* + * e500 doesn't implement the lowest tsize bit, + * or 1K pages. + */ + tsize = max(BOOK3E_PAGESZ_4K, tsize & ~1); + + /* + * Now find the largest tsize (up to what the guest + * requested) that will cover gfn, stay within the + * range, and for which gfn and pfn are mutually + * aligned. + */ + + for (; tsize > BOOK3E_PAGESZ_4K; tsize -= 2) { + unsigned long gfn_start, gfn_end; + tsize_pages = 1UL << (tsize - 2); + + gfn_start = gfn & ~(tsize_pages - 1); + gfn_end = gfn_start + tsize_pages; + + if (gfn_start + pfn - gfn < start) + continue; + if (gfn_end + pfn - gfn > end) + continue; + if ((gfn & (tsize_pages - 1)) != + (pfn & (tsize_pages - 1))) + continue; + + gvaddr &= ~((tsize_pages << PAGE_SHIFT) - 1); + pfn &= ~(tsize_pages - 1); + break; + } + } + kvmppc_e500_ref_setup(ref, gtlbe, pfn, wimg, writable); kvmppc_e500_setup_stlbe(&vcpu_e500->vcpu, gtlbe, tsize, ref, gvaddr, stlbe); + writable = tlbe_is_writable(stlbe); /* Clear i-cache for new pages */ kvmppc_mmu_flush_icache(pfn); diff --git a/arch/powerpc/kvm/guest-state-buffer.c b/arch/powerpc/kvm/guest-state-buffer.c index b80dbc58621f..871cf60ddeb6 100644 --- a/arch/powerpc/kvm/guest-state-buffer.c +++ b/arch/powerpc/kvm/guest-state-buffer.c @@ -92,6 +92,10 @@ static int kvmppc_gsid_class(u16 iden) (iden <= KVMPPC_GSE_GUESTWIDE_END)) return KVMPPC_GS_CLASS_GUESTWIDE; + if ((iden >= KVMPPC_GSE_HOSTWIDE_START) && + (iden <= KVMPPC_GSE_HOSTWIDE_END)) + return KVMPPC_GS_CLASS_HOSTWIDE; + if ((iden >= KVMPPC_GSE_META_START) && (iden <= KVMPPC_GSE_META_END)) return KVMPPC_GS_CLASS_META; @@ -118,6 +122,21 @@ static int kvmppc_gsid_type(u16 iden) int type = -1; switch (kvmppc_gsid_class(iden)) { + case KVMPPC_GS_CLASS_HOSTWIDE: + switch (iden) { + case KVMPPC_GSID_L0_GUEST_HEAP: + fallthrough; + case KVMPPC_GSID_L0_GUEST_HEAP_MAX: + fallthrough; + case KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE: + fallthrough; + case KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX: + fallthrough; + case KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM: + type = KVMPPC_GSE_BE64; + break; + } + break; case KVMPPC_GS_CLASS_GUESTWIDE: switch (iden) { case KVMPPC_GSID_HOST_STATE_SIZE: @@ -187,6 +206,9 @@ unsigned long kvmppc_gsid_flags(u16 iden) case KVMPPC_GS_CLASS_GUESTWIDE: flags = KVMPPC_GS_FLAGS_WIDE; break; + case KVMPPC_GS_CLASS_HOSTWIDE: + flags = KVMPPC_GS_FLAGS_HOST_WIDE; + break; case KVMPPC_GS_CLASS_META: case KVMPPC_GS_CLASS_DWORD_REG: case KVMPPC_GS_CLASS_WORD_REG: @@ -310,6 +332,13 @@ static inline int kvmppc_gse_flatten_iden(u16 iden) bit += KVMPPC_GSE_GUESTWIDE_COUNT; + if (class == KVMPPC_GS_CLASS_HOSTWIDE) { + bit += iden - KVMPPC_GSE_HOSTWIDE_START; + return bit; + } + + bit += KVMPPC_GSE_HOSTWIDE_COUNT; + if (class == KVMPPC_GS_CLASS_META) { bit += iden - KVMPPC_GSE_META_START; return bit; @@ -356,6 +385,12 @@ static inline u16 kvmppc_gse_unflatten_iden(int bit) } bit -= KVMPPC_GSE_GUESTWIDE_COUNT; + if (bit < KVMPPC_GSE_HOSTWIDE_COUNT) { + iden = KVMPPC_GSE_HOSTWIDE_START + bit; + return iden; + } + bit -= KVMPPC_GSE_HOSTWIDE_COUNT; + if (bit < KVMPPC_GSE_META_COUNT) { iden = KVMPPC_GSE_META_START + bit; return iden; @@ -588,6 +623,8 @@ int kvmppc_gsb_send(struct kvmppc_gs_buff *gsb, unsigned long flags) if (flags & KVMPPC_GS_FLAGS_WIDE) hflags |= H_GUEST_FLAGS_WIDE; + if (flags & KVMPPC_GS_FLAGS_HOST_WIDE) + hflags |= H_GUEST_FLAGS_HOST_WIDE; rc = plpar_guest_set_state(hflags, gsb->guest_id, gsb->vcpu_id, __pa(gsb->hdr), gsb->capacity, &i); @@ -613,6 +650,8 @@ int kvmppc_gsb_recv(struct kvmppc_gs_buff *gsb, unsigned long flags) if (flags & KVMPPC_GS_FLAGS_WIDE) hflags |= H_GUEST_FLAGS_WIDE; + if (flags & KVMPPC_GS_FLAGS_HOST_WIDE) + hflags |= H_GUEST_FLAGS_HOST_WIDE; rc = plpar_guest_get_state(hflags, gsb->guest_id, gsb->vcpu_id, __pa(gsb->hdr), gsb->capacity, &i); diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index ce1d91eed231..153587741864 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -550,12 +550,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) #ifdef CONFIG_PPC_BOOK3S_64 case KVM_CAP_SPAPR_TCE: + fallthrough; case KVM_CAP_SPAPR_TCE_64: - r = 1; - break; case KVM_CAP_SPAPR_TCE_VFIO: - r = !!cpu_has_feature(CPU_FTR_HVMODE); - break; case KVM_CAP_PPC_RTAS: case KVM_CAP_PPC_FIXUP_HCALL: case KVM_CAP_PPC_ENABLE_HCALL: @@ -766,8 +763,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) { int err; - hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS); - vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup; + hrtimer_setup(&vcpu->arch.dec_timer, kvmppc_decrementer_wakeup, CLOCK_REALTIME, + HRTIMER_MODE_ABS); #ifdef CONFIG_KVM_EXIT_TIMING mutex_init(&vcpu->arch.exit_timing_lock); diff --git a/arch/powerpc/kvm/test-guest-state-buffer.c b/arch/powerpc/kvm/test-guest-state-buffer.c index bfd225329a18..5ccca306997a 100644 --- a/arch/powerpc/kvm/test-guest-state-buffer.c +++ b/arch/powerpc/kvm/test-guest-state-buffer.c @@ -5,6 +5,7 @@ #include <kunit/test.h> #include <asm/guest-state-buffer.h> +#include <asm/kvm_ppc.h> static void test_creating_buffer(struct kunit *test) { @@ -141,6 +142,16 @@ static void test_gs_bitmap(struct kunit *test) i++; } + for (u16 iden = KVMPPC_GSID_L0_GUEST_HEAP; + iden <= KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM; iden++) { + kvmppc_gsbm_set(&gsbm, iden); + kvmppc_gsbm_set(&gsbm1, iden); + KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden)); + kvmppc_gsbm_clear(&gsbm, iden); + KUNIT_EXPECT_FALSE(test, kvmppc_gsbm_test(&gsbm, iden)); + i++; + } + for (u16 iden = KVMPPC_GSID_RUN_INPUT; iden <= KVMPPC_GSID_VPA; iden++) { kvmppc_gsbm_set(&gsbm, iden); @@ -309,12 +320,215 @@ static void test_gs_msg(struct kunit *test) kvmppc_gsm_free(gsm); } +/* Test data struct for hostwide/L0 counters */ +struct kvmppc_gs_msg_test_hostwide_data { + u64 guest_heap; + u64 guest_heap_max; + u64 guest_pgtable_size; + u64 guest_pgtable_size_max; + u64 guest_pgtable_reclaim; +}; + +static size_t test_hostwide_get_size(struct kvmppc_gs_msg *gsm) + +{ + size_t size = 0; + u16 ids[] = { + KVMPPC_GSID_L0_GUEST_HEAP, + KVMPPC_GSID_L0_GUEST_HEAP_MAX, + KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE, + KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX, + KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM + }; + + for (int i = 0; i < ARRAY_SIZE(ids); i++) + size += kvmppc_gse_total_size(kvmppc_gsid_size(ids[i])); + return size; +} + +static int test_hostwide_fill_info(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + struct kvmppc_gs_msg_test_hostwide_data *data = gsm->data; + + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_HEAP)) + kvmppc_gse_put_u64(gsb, KVMPPC_GSID_L0_GUEST_HEAP, + data->guest_heap); + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_HEAP_MAX)) + kvmppc_gse_put_u64(gsb, KVMPPC_GSID_L0_GUEST_HEAP_MAX, + data->guest_heap_max); + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE)) + kvmppc_gse_put_u64(gsb, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE, + data->guest_pgtable_size); + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX)) + kvmppc_gse_put_u64(gsb, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX, + data->guest_pgtable_size_max); + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM)) + kvmppc_gse_put_u64(gsb, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM, + data->guest_pgtable_reclaim); + + return 0; +} + +static int test_hostwide_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb) +{ + struct kvmppc_gs_parser gsp = { 0 }; + struct kvmppc_gs_msg_test_hostwide_data *data = gsm->data; + struct kvmppc_gs_elem *gse; + int rc; + + rc = kvmppc_gse_parse(&gsp, gsb); + if (rc < 0) + return rc; + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_HEAP); + if (gse) + data->guest_heap = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_HEAP_MAX); + if (gse) + data->guest_heap_max = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE); + if (gse) + data->guest_pgtable_size = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX); + if (gse) + data->guest_pgtable_size_max = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM); + if (gse) + data->guest_pgtable_reclaim = kvmppc_gse_get_u64(gse); + + return 0; +} + +static struct kvmppc_gs_msg_ops gs_msg_test_hostwide_ops = { + .get_size = test_hostwide_get_size, + .fill_info = test_hostwide_fill_info, + .refresh_info = test_hostwide_refresh_info, +}; + +static void test_gs_hostwide_msg(struct kunit *test) +{ + struct kvmppc_gs_msg_test_hostwide_data test_data = { + .guest_heap = 0xdeadbeef, + .guest_heap_max = ~0ULL, + .guest_pgtable_size = 0xff, + .guest_pgtable_size_max = 0xffffff, + .guest_pgtable_reclaim = 0xdeadbeef, + }; + struct kvmppc_gs_msg *gsm; + struct kvmppc_gs_buff *gsb; + + gsm = kvmppc_gsm_new(&gs_msg_test_hostwide_ops, &test_data, GSM_SEND, + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsm); + + gsb = kvmppc_gsb_new(kvmppc_gsm_size(gsm), 0, 0, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsb); + + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_HEAP); + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_HEAP_MAX); + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE); + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX); + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM); + + kvmppc_gsm_fill_info(gsm, gsb); + + memset(&test_data, 0, sizeof(test_data)); + + kvmppc_gsm_refresh_info(gsm, gsb); + KUNIT_EXPECT_EQ(test, test_data.guest_heap, 0xdeadbeef); + KUNIT_EXPECT_EQ(test, test_data.guest_heap_max, ~0ULL); + KUNIT_EXPECT_EQ(test, test_data.guest_pgtable_size, 0xff); + KUNIT_EXPECT_EQ(test, test_data.guest_pgtable_size_max, 0xffffff); + KUNIT_EXPECT_EQ(test, test_data.guest_pgtable_reclaim, 0xdeadbeef); + + kvmppc_gsm_free(gsm); +} + +/* Test if the H_GUEST_GET_STATE for hostwide counters works */ +static void test_gs_hostwide_counters(struct kunit *test) +{ + struct kvmppc_gs_msg_test_hostwide_data test_data; + struct kvmppc_gs_parser gsp = { 0 }; + + struct kvmppc_gs_msg *gsm; + struct kvmppc_gs_buff *gsb; + struct kvmppc_gs_elem *gse; + int rc; + + if (!kvmhv_on_pseries()) + kunit_skip(test, "This test need a kmv-hv guest"); + + gsm = kvmppc_gsm_new(&gs_msg_test_hostwide_ops, &test_data, GSM_SEND, + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsm); + + gsb = kvmppc_gsb_new(kvmppc_gsm_size(gsm), 0, 0, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, gsb); + + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_HEAP); + + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_HEAP_MAX); + + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE); + + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX); + + kvmppc_gsm_include(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM); + + kvmppc_gsm_fill_info(gsm, gsb); + + /* With HOST_WIDE flags guestid and vcpuid will be ignored */ + rc = kvmppc_gsb_recv(gsb, KVMPPC_GS_FLAGS_HOST_WIDE); + KUNIT_ASSERT_EQ(test, rc, 0); + + /* Parse the guest state buffer is successful */ + rc = kvmppc_gse_parse(&gsp, gsb); + KUNIT_ASSERT_EQ(test, rc, 0); + + /* Parse the GSB and get the counters */ + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_HEAP); + KUNIT_ASSERT_NOT_NULL_MSG(test, gse, "L0 Heap counter missing"); + kunit_info(test, "Guest Heap Size=%llu bytes", + kvmppc_gse_get_u64(gse)); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_HEAP_MAX); + KUNIT_ASSERT_NOT_NULL_MSG(test, gse, "L0 Heap counter max missing"); + kunit_info(test, "Guest Heap Size Max=%llu bytes", + kvmppc_gse_get_u64(gse)); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE); + KUNIT_ASSERT_NOT_NULL_MSG(test, gse, "L0 page-table size missing"); + kunit_info(test, "Guest Page-table Size=%llu bytes", + kvmppc_gse_get_u64(gse)); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX); + KUNIT_ASSERT_NOT_NULL_MSG(test, gse, "L0 page-table size-max missing"); + kunit_info(test, "Guest Page-table Size Max=%llu bytes", + kvmppc_gse_get_u64(gse)); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM); + KUNIT_ASSERT_NOT_NULL_MSG(test, gse, "L0 page-table reclaim size missing"); + kunit_info(test, "Guest Page-table Reclaim Size=%llu bytes", + kvmppc_gse_get_u64(gse)); + + kvmppc_gsm_free(gsm); + kvmppc_gsb_free(gsb); +} + static struct kunit_case guest_state_buffer_testcases[] = { KUNIT_CASE(test_creating_buffer), KUNIT_CASE(test_adding_element), KUNIT_CASE(test_gs_bitmap), KUNIT_CASE(test_gs_parsing), KUNIT_CASE(test_gs_msg), + KUNIT_CASE(test_gs_hostwide_msg), + KUNIT_CASE(test_gs_hostwide_counters), {} }; diff --git a/arch/powerpc/kvm/timing.h b/arch/powerpc/kvm/timing.h index 45817ab82bb4..14b0e23f601f 100644 --- a/arch/powerpc/kvm/timing.h +++ b/arch/powerpc/kvm/timing.h @@ -38,11 +38,7 @@ static inline void kvmppc_set_exit_type(struct kvm_vcpu *vcpu, int type) {} static inline void kvmppc_account_exit_stat(struct kvm_vcpu *vcpu, int type) { /* type has to be known at build time for optimization */ - - /* The BUILD_BUG_ON below breaks in funny ways, commented out - * for now ... -BenH BUILD_BUG_ON(!__builtin_constant_p(type)); - */ switch (type) { case EXT_INTR_EXITS: vcpu->stat.ext_intr_exits++; diff --git a/arch/powerpc/kvm/trace_book3s.h b/arch/powerpc/kvm/trace_book3s.h index 372a82fa2de3..9260ddbd557f 100644 --- a/arch/powerpc/kvm/trace_book3s.h +++ b/arch/powerpc/kvm/trace_book3s.h @@ -25,6 +25,7 @@ {0xe00, "H_DATA_STORAGE"}, \ {0xe20, "H_INST_STORAGE"}, \ {0xe40, "H_EMUL_ASSIST"}, \ + {0xea0, "H_VIRT"}, \ {0xf00, "PERFMON"}, \ {0xf20, "ALTIVEC"}, \ {0xf40, "VSX"} diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index af97fbb3c257..f84e0337cc02 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -108,7 +108,7 @@ static int text_area_cpu_up(unsigned int cpu) unsigned long addr; int err; - area = get_vm_area(PAGE_SIZE, VM_ALLOC); + area = get_vm_area(PAGE_SIZE, 0); if (!area) { WARN_ONCE(1, "Failed to create text area for cpu %d\n", cpu); @@ -493,7 +493,9 @@ static int __do_patch_instructions_mm(u32 *addr, u32 *code, size_t len, bool rep orig_mm = start_using_temp_mm(patching_mm); + kasan_disable_current(); err = __patch_instructions(patch_addr, code, len, repeat_instr); + kasan_enable_current(); /* context synchronisation performed by __patch_instructions */ stop_using_temp_mm(patching_mm, orig_mm); diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c index d491da8d1838..54340912398f 100644 --- a/arch/powerpc/lib/vmx-helper.c +++ b/arch/powerpc/lib/vmx-helper.c @@ -45,7 +45,7 @@ int exit_vmx_usercopy(void) * set and we are preemptible. The hack here is to schedule a * decrementer to fire here and reschedule for us if necessary. */ - if (IS_ENABLED(CONFIG_PREEMPT) && need_resched()) + if (need_irq_preemption() && need_resched()) set_dec(1); return 0; } diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index 0fe2f085c05a..8c1582b2987d 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile @@ -15,5 +15,5 @@ obj-$(CONFIG_NUMA) += numa.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_NOT_COHERENT_CACHE) += dma-noncoherent.o obj-$(CONFIG_PPC_COPRO_BASE) += copro_fault.o -obj-$(CONFIG_PTDUMP_CORE) += ptdump/ +obj-$(CONFIG_PTDUMP) += ptdump/ obj-$(CONFIG_KASAN) += kasan/ diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index 6978344edcb4..be9c4106e22f 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -377,10 +377,7 @@ void __init MMU_init_hw(void) * Find some memory for the hash table. */ if ( ppc_md.progress ) ppc_md.progress("hash:find piece", 0x322); - Hash = memblock_alloc(Hash_size, Hash_size); - if (!Hash) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, Hash_size, Hash_size); + Hash = memblock_alloc_or_panic(Hash_size, Hash_size); _SDR1 = __pa(Hash) | SDR1_LOW_BITS; pr_info("Total memory = %lldMB; using %ldkB for hash table\n", diff --git a/arch/powerpc/mm/book3s64/hash_hugepage.c b/arch/powerpc/mm/book3s64/hash_hugepage.c index 15d6f3ea7178..cdfd4fe75edb 100644 --- a/arch/powerpc/mm/book3s64/hash_hugepage.c +++ b/arch/powerpc/mm/book3s64/hash_hugepage.c @@ -54,7 +54,7 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, /* * Make sure this is thp or devmap entry */ - if (!(old_pmd & (H_PAGE_THP_HUGE | _PAGE_DEVMAP))) + if (!(old_pmd & H_PAGE_THP_HUGE)) return 0; rflags = htab_convert_pte_flags(new_pmd, flags); diff --git a/arch/powerpc/mm/book3s64/hash_native.c b/arch/powerpc/mm/book3s64/hash_native.c index 430d1d935a7c..e9e2dd70c060 100644 --- a/arch/powerpc/mm/book3s64/hash_native.c +++ b/arch/powerpc/mm/book3s64/hash_native.c @@ -27,8 +27,6 @@ #include <asm/ppc-opcode.h> #include <asm/feature-fixups.h> -#include <misc/cxl-base.h> - #ifdef DEBUG_LOW #define DBG_LOW(fmt...) udbg_printf(fmt) #else @@ -217,11 +215,9 @@ static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize) static inline void tlbie(unsigned long vpn, int psize, int apsize, int ssize, int local) { - unsigned int use_local; + unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL); int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); - use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && !cxl_ctx_in_use(); - if (use_local) use_local = mmu_psize_defs[psize].tlbiel; if (lock_tlbie && !use_local) @@ -789,10 +785,6 @@ static void native_flush_hash_range(unsigned long number, int local) unsigned long psize = batch->psize; int ssize = batch->ssize; int i; - unsigned int use_local; - - use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && - mmu_psize_defs[psize].tlbiel && !cxl_ctx_in_use(); local_irq_save(flags); @@ -827,7 +819,8 @@ static void native_flush_hash_range(unsigned long number, int local) } pte_iterate_hashed_end(); } - if (use_local) { + if (mmu_has_feature(MMU_FTR_TLBIEL) && + mmu_psize_defs[psize].tlbiel && local) { asm volatile("ptesync":::"memory"); for (i = 0; i < number; i++) { vpn = batch->vpn[i]; diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c b/arch/powerpc/mm/book3s64/hash_pgtable.c index 988948d69bc1..82d31177630b 100644 --- a/arch/powerpc/mm/book3s64/hash_pgtable.c +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c @@ -195,7 +195,7 @@ unsigned long hash__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr unsigned long old; #ifdef CONFIG_DEBUG_VM - WARN_ON(!hash__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); + WARN_ON(!hash__pmd_trans_huge(*pmdp)); assert_spin_locked(pmd_lockptr(mm, pmdp)); #endif @@ -227,7 +227,6 @@ pmd_t hash__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addres VM_BUG_ON(address & ~HPAGE_PMD_MASK); VM_BUG_ON(pmd_trans_huge(*pmdp)); - VM_BUG_ON(pmd_devmap(*pmdp)); pmd = *pmdp; pmd_clear(pmdp); diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index c8b4fa71d4a7..4693c464fc5a 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -56,7 +56,7 @@ #include <asm/cacheflush.h> #include <asm/cputable.h> #include <asm/sections.h> -#include <asm/copro.h> +#include <asm/spu.h> #include <asm/udbg.h> #include <asm/text-patching.h> #include <asm/fadump.h> @@ -343,7 +343,7 @@ static inline bool hash_supports_debug_pagealloc(void) static u8 *linear_map_hash_slots; static unsigned long linear_map_hash_count; static DEFINE_RAW_SPINLOCK(linear_map_hash_lock); -static void hash_debug_pagealloc_alloc_slots(void) +static __init void hash_debug_pagealloc_alloc_slots(void) { if (!hash_supports_debug_pagealloc()) return; @@ -409,7 +409,7 @@ static DEFINE_RAW_SPINLOCK(linear_map_kf_hash_lock); static phys_addr_t kfence_pool; -static inline void hash_kfence_alloc_pool(void) +static __init void hash_kfence_alloc_pool(void) { if (!kfence_early_init_enabled()) goto err; @@ -445,7 +445,7 @@ err: disable_kfence(); } -static inline void hash_kfence_map_pool(void) +static __init void hash_kfence_map_pool(void) { unsigned long kfence_pool_start, kfence_pool_end; unsigned long prot = pgprot_val(PAGE_KERNEL); @@ -1358,18 +1358,6 @@ static void __init htab_initialize(void) } else { unsigned long limit = MEMBLOCK_ALLOC_ANYWHERE; -#ifdef CONFIG_PPC_CELL - /* - * Cell may require the hash table down low when using the - * Axon IOMMU in order to fit the dynamic region over it, see - * comments in cell/iommu.c - */ - if (fdt_subnode_offset(initial_boot_params, 0, "axon") > 0) { - limit = 0x80000000; - pr_info("Hash table forced below 2G for Axon IOMMU\n"); - } -#endif /* CONFIG_PPC_CELL */ - table = memblock_phys_alloc_range(htab_size_bytes, htab_size_bytes, 0, limit); @@ -1612,7 +1600,9 @@ void demote_segment_4k(struct mm_struct *mm, unsigned long addr) if (get_slice_psize(mm, addr) == MMU_PAGE_4K) return; slice_set_range_psize(mm, addr, 1, MMU_PAGE_4K); - copro_flush_all_slbs(mm); +#ifdef CONFIG_SPU_BASE + spu_flush_all_slbs(mm); +#endif if ((get_paca_psize(addr) != MMU_PAGE_4K) && (current->mm == mm)) { copy_mm_to_paca(mm); @@ -1881,7 +1871,9 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea, "to 4kB pages because of " "non-cacheable mapping\n"); psize = mmu_vmalloc_psize = MMU_PAGE_4K; - copro_flush_all_slbs(mm); +#ifdef CONFIG_SPU_BASE + spu_flush_all_slbs(mm); +#endif } } diff --git a/arch/powerpc/mm/book3s64/hugetlbpage.c b/arch/powerpc/mm/book3s64/hugetlbpage.c index 83c3361b358b..2bcbbf9d85ac 100644 --- a/arch/powerpc/mm/book3s64/hugetlbpage.c +++ b/arch/powerpc/mm/book3s64/hugetlbpage.c @@ -74,7 +74,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, } while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); /* Make sure this is a hugetlb entry */ - if (old_pte & (H_PAGE_THP_HUGE | _PAGE_DEVMAP)) + if (old_pte & H_PAGE_THP_HUGE) return 0; rflags = htab_convert_pte_flags(new_pte, flags); diff --git a/arch/powerpc/mm/book3s64/mmu_context.c b/arch/powerpc/mm/book3s64/mmu_context.c index 1715b07c630c..4e1e45420bd4 100644 --- a/arch/powerpc/mm/book3s64/mmu_context.c +++ b/arch/powerpc/mm/book3s64/mmu_context.c @@ -253,7 +253,7 @@ static void pmd_frag_destroy(void *pmd_frag) count = ((unsigned long)pmd_frag & ~PAGE_MASK) >> PMD_FRAG_SIZE_SHIFT; /* We allow PTE_FRAG_NR fragments from a PTE page */ if (atomic_sub_and_test(PMD_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { - pagetable_pmd_dtor(ptdesc); + pagetable_dtor(ptdesc); pagetable_free(ptdesc); } } diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c index 374542528080..c9431ae7f78a 100644 --- a/arch/powerpc/mm/book3s64/pgtable.c +++ b/arch/powerpc/mm/book3s64/pgtable.c @@ -10,7 +10,6 @@ #include <linux/pkeys.h> #include <linux/debugfs.h> #include <linux/proc_fs.h> -#include <misc/cxl-base.h> #include <asm/pgalloc.h> #include <asm/tlb.h> @@ -63,7 +62,7 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, { int changed; #ifdef CONFIG_DEBUG_VM - WARN_ON(!pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); + WARN_ON(!pmd_trans_huge(*pmdp)); assert_spin_locked(pmd_lockptr(vma->vm_mm, pmdp)); #endif changed = !pmd_same(*(pmdp), entry); @@ -83,7 +82,6 @@ int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, { int changed; #ifdef CONFIG_DEBUG_VM - WARN_ON(!pud_devmap(*pudp)); assert_spin_locked(pud_lockptr(vma->vm_mm, pudp)); #endif changed = !pud_same(*(pudp), entry); @@ -205,8 +203,8 @@ pmd_t pmdp_huge_get_and_clear_full(struct vm_area_struct *vma, { pmd_t pmd; VM_BUG_ON(addr & ~HPAGE_PMD_MASK); - VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp) && - !pmd_devmap(*pmdp)) || !pmd_present(*pmdp)); + VM_BUG_ON((pmd_present(*pmdp) && !pmd_trans_huge(*pmdp)) || + !pmd_present(*pmdp)); pmd = pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp); /* * if it not a fullmm flush, then we can possibly end up converting @@ -224,8 +222,7 @@ pud_t pudp_huge_get_and_clear_full(struct vm_area_struct *vma, pud_t pud; VM_BUG_ON(addr & ~HPAGE_PMD_MASK); - VM_BUG_ON((pud_present(*pudp) && !pud_devmap(*pudp)) || - !pud_present(*pudp)); + VM_BUG_ON(!pud_present(*pudp)); pud = pudp_huge_get_and_clear(vma->vm_mm, addr, pudp); /* * if it not a fullmm flush, then we can possibly end up converting @@ -270,11 +267,6 @@ pud_t pfn_pud(unsigned long pfn, pgprot_t pgprot) return __pud_mkhuge(pud_set_protbits(__pud(pudv), pgprot)); } -pmd_t mk_pmd(struct page *page, pgprot_t pgprot) -{ - return pfn_pmd(page_to_pfn(page), pgprot); -} - pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { unsigned long pmdv; @@ -330,11 +322,7 @@ void __init mmu_partition_table_init(void) unsigned long ptcr; /* Initialize the Partition Table with no entries */ - partition_tb = memblock_alloc(patb_size, patb_size); - if (!partition_tb) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, patb_size, patb_size); - + partition_tb = memblock_alloc_or_panic(patb_size, patb_size); ptcr = __pa(partition_tb) | (PATB_SIZE_SHIFT - 12); set_ptcr_when_no_uv(ptcr); powernv_set_nmmu_ptcr(ptcr); @@ -427,7 +415,7 @@ static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm) ptdesc = pagetable_alloc(gfp, 0); if (!ptdesc) return NULL; - if (!pagetable_pmd_ctor(ptdesc)) { + if (!pagetable_pmd_ctor(mm, ptdesc)) { pagetable_free(ptdesc); return NULL; } @@ -477,7 +465,7 @@ void pmd_fragment_free(unsigned long *pmd) BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0); if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) { - pagetable_pmd_dtor(ptdesc); + pagetable_dtor(ptdesc); pagetable_free(ptdesc); } } @@ -591,7 +579,7 @@ int pmd_move_must_withdraw(struct spinlock *new_pmd_ptl, /* * Does the CPU support tlbie? */ -bool tlbie_capable __read_mostly = true; +bool tlbie_capable __read_mostly = IS_ENABLED(CONFIG_PPC_RADIX_BROADCAST_TLBIE); EXPORT_SYMBOL(tlbie_capable); /* @@ -599,7 +587,7 @@ EXPORT_SYMBOL(tlbie_capable); * address spaces? tlbie may still be used for nMMU accelerators, and for KVM * guest address spaces. */ -bool tlbie_enabled __read_mostly = true; +bool tlbie_enabled __read_mostly = IS_ENABLED(CONFIG_PPC_RADIX_BROADCAST_TLBIE); static int __init setup_disable_tlbie(char *str) { @@ -654,7 +642,7 @@ unsigned long memremap_compat_align(void) EXPORT_SYMBOL_GPL(memremap_compat_align); #endif -pgprot_t vm_get_page_prot(unsigned long vm_flags) +pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { unsigned long prot; diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 311e2112d782..be523e5fe9c5 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -363,7 +363,7 @@ static int __meminit create_physical_mapping(unsigned long start, } #ifdef CONFIG_KFENCE -static inline phys_addr_t alloc_kfence_pool(void) +static __init phys_addr_t alloc_kfence_pool(void) { phys_addr_t kfence_pool; @@ -393,7 +393,7 @@ no_kfence: return 0; } -static inline void map_kfence_pool(phys_addr_t kfence_pool) +static __init void map_kfence_pool(phys_addr_t kfence_pool) { if (!kfence_pool) return; @@ -976,7 +976,7 @@ int __meminit radix__vmemmap_create_mapping(unsigned long start, return 0; } - +#ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { if (radix_enabled()) @@ -984,6 +984,7 @@ bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) return false; } +#endif int __meminit vmemmap_check_pmd(pmd_t *pmdp, int node, unsigned long addr, unsigned long next) @@ -1120,6 +1121,26 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in pmd_t *pmd; pte_t *pte; + /* + * If altmap is present, Make sure we align the start vmemmap addr + * to PAGE_SIZE so that we calculate the correct start_pfn in + * altmap boundary check to decide whether we should use altmap or + * RAM based backing memory allocation. Also the address need to be + * aligned for set_pte operation. If the start addr is already + * PMD_SIZE aligned and with in the altmap boundary then we will + * try to use a pmd size altmap mapping else we go for page size + * mapping. + * + * If altmap is not present, align the vmemmap addr to PMD_SIZE and + * always allocate a PMD size page for vmemmap backing. + * + */ + + if (altmap) + start = ALIGN_DOWN(start, PAGE_SIZE); + else + start = ALIGN_DOWN(start, PMD_SIZE); + for (addr = start; addr < end; addr = next) { next = pmd_addr_end(addr, end); @@ -1146,7 +1167,7 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in * we fallback to RAM for vmemmap allocation. */ if (altmap && (!IS_ALIGNED(addr, PMD_SIZE) || - altmap_cross_boundary(altmap, addr, PMD_SIZE))) { + altmap_cross_boundary(altmap, addr, PMD_SIZE))) { /* * make sure we don't create altmap mappings * covering things outside the device. @@ -1159,7 +1180,7 @@ int __meminit radix__vmemmap_populate(unsigned long start, unsigned long end, in vmemmap_set_pmd(pmd, p, node, addr, next); pr_debug("PMD_SIZE vmemmap mapping\n"); continue; - } else if (altmap) { + } else { /* * A vmemmap block allocation can fail due to * alignment requirements and we trying to align @@ -1412,7 +1433,7 @@ unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long add unsigned long old; #ifdef CONFIG_DEBUG_VM - WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp)); + WARN_ON(!radix__pmd_trans_huge(*pmdp)); assert_spin_locked(pmd_lockptr(mm, pmdp)); #endif @@ -1429,7 +1450,7 @@ unsigned long radix__pud_hugepage_update(struct mm_struct *mm, unsigned long add unsigned long old; #ifdef CONFIG_DEBUG_VM - WARN_ON(!pud_devmap(*pudp)); + WARN_ON(!pud_trans_huge(*pudp)); assert_spin_locked(pud_lockptr(mm, pudp)); #endif @@ -1447,7 +1468,6 @@ pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addre VM_BUG_ON(address & ~HPAGE_PMD_MASK); VM_BUG_ON(radix__pmd_trans_huge(*pmdp)); - VM_BUG_ON(pmd_devmap(*pmdp)); /* * khugepaged calls this for normal pmd */ diff --git a/arch/powerpc/mm/book3s64/slice.c b/arch/powerpc/mm/book3s64/slice.c index bc9a39821d1c..28bec5bc7879 100644 --- a/arch/powerpc/mm/book3s64/slice.c +++ b/arch/powerpc/mm/book3s64/slice.c @@ -22,7 +22,7 @@ #include <linux/security.h> #include <asm/mman.h> #include <asm/mmu.h> -#include <asm/copro.h> +#include <asm/spu.h> #include <asm/hugetlb.h> #include <asm/mmu_context.h> @@ -248,7 +248,9 @@ static void slice_convert(struct mm_struct *mm, spin_unlock_irqrestore(&slice_convert_lock, flags); - copro_flush_all_slbs(mm); +#ifdef CONFIG_SPU_BASE + spu_flush_all_slbs(mm); +#endif } /* diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c index f49fd873df8d..f5f8692e2c69 100644 --- a/arch/powerpc/mm/copro_fault.c +++ b/arch/powerpc/mm/copro_fault.c @@ -12,8 +12,6 @@ #include <linux/export.h> #include <asm/reg.h> #include <asm/copro.h> -#include <asm/spu.h> -#include <misc/cxl-base.h> /* * This ought to be kept in sync with the powerpc specific do_page_fault @@ -135,13 +133,4 @@ int copro_calculate_slb(struct mm_struct *mm, u64 ea, struct copro_slb *slb) return 0; } EXPORT_SYMBOL_GPL(copro_calculate_slb); - -void copro_flush_all_slbs(struct mm_struct *mm) -{ -#ifdef CONFIG_SPU_BASE - spu_flush_all_slbs(mm); -#endif - cxl_slbia(mm); -} -EXPORT_SYMBOL_GPL(copro_flush_all_slbs); #endif diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index c156fe0d53c3..806c74e0d5ab 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -17,6 +17,7 @@ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/string.h> +#include <linux/string_choices.h> #include <linux/types.h> #include <linux/pagemap.h> #include <linux/ptrace.h> @@ -218,7 +219,7 @@ static bool bad_kernel_fault(struct pt_regs *regs, unsigned long error_code, // Read/write fault blocked by KUAP is bad, it can never succeed. if (bad_kuap_fault(regs, address, is_write)) { pr_crit_ratelimited("Kernel attempted to %s user page (%lx) - exploit attempt? (uid: %d)\n", - is_write ? "write" : "read", address, + str_write_read(is_write), address, from_kuid(&init_user_ns, current_uid())); // Fault on user outside of certain regions (eg. copy_tofrom_user()) is bad @@ -625,7 +626,7 @@ static void __bad_page_fault(struct pt_regs *regs, int sig) case INTERRUPT_DATA_STORAGE: case INTERRUPT_H_DATA_STORAGE: pr_alert("BUG: %s on %s at 0x%08lx\n", msg, - is_write ? "write" : "read", regs->dar); + str_write_read(is_write), regs->dar); break; case INTERRUPT_DATA_SEGMENT: pr_alert("BUG: %s at 0x%08lx\n", msg, regs->dar); diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 6b043180220a..d3c1b749dcfc 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -113,6 +113,7 @@ static int __init pseries_alloc_bootmem_huge_page(struct hstate *hstate) gpage_freearray[nr_gpages] = 0; list_add(&m->list, &huge_boot_pages[0]); m->hstate = hstate; + m->flags = 0; return 1; } diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index d96bbc001e73..b6f3ae03ca9e 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c @@ -41,6 +41,7 @@ #include <linux/libfdt.h> #include <linux/memremap.h> #include <linux/memory.h> +#include <linux/bootmem_info.h> #include <asm/pgalloc.h> #include <asm/page.h> @@ -386,10 +387,13 @@ void __ref vmemmap_free(unsigned long start, unsigned long end, } #endif + +#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE void register_page_bootmem_memmap(unsigned long section_nr, struct page *start_page, unsigned long size) { } +#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ #endif /* CONFIG_SPARSEMEM_VMEMMAP */ diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c index 7b0afcabd89f..4b4feba9873b 100644 --- a/arch/powerpc/mm/ioremap.c +++ b/arch/powerpc/mm/ioremap.c @@ -4,7 +4,6 @@ #include <linux/slab.h> #include <linux/mmzone.h> #include <linux/vmalloc.h> -#include <asm/io-workarounds.h> unsigned long ioremap_bot; EXPORT_SYMBOL(ioremap_bot); @@ -14,8 +13,6 @@ void __iomem *ioremap(phys_addr_t addr, unsigned long size) pgprot_t prot = pgprot_noncached(PAGE_KERNEL); void *caller = __builtin_return_address(0); - if (iowa_is_active()) - return iowa_ioremap(addr, size, prot, caller); return __ioremap_caller(addr, size, prot, caller); } EXPORT_SYMBOL(ioremap); @@ -25,8 +22,6 @@ void __iomem *ioremap_wc(phys_addr_t addr, unsigned long size) pgprot_t prot = pgprot_noncached_wc(PAGE_KERNEL); void *caller = __builtin_return_address(0); - if (iowa_is_active()) - return iowa_ioremap(addr, size, prot, caller); return __ioremap_caller(addr, size, prot, caller); } EXPORT_SYMBOL(ioremap_wc); @@ -36,22 +31,18 @@ void __iomem *ioremap_coherent(phys_addr_t addr, unsigned long size) pgprot_t prot = pgprot_cached(PAGE_KERNEL); void *caller = __builtin_return_address(0); - if (iowa_is_active()) - return iowa_ioremap(addr, size, prot, caller); return __ioremap_caller(addr, size, prot, caller); } -void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long flags) +void __iomem *ioremap_prot(phys_addr_t addr, size_t size, pgprot_t prot) { - pte_t pte = __pte(flags); + pte_t pte = __pte(pgprot_val(prot)); void *caller = __builtin_return_address(0); /* writeable implies dirty for kernel addresses */ if (pte_write(pte)) pte = pte_mkdirty(pte); - if (iowa_is_active()) - return iowa_ioremap(addr, size, pte_pgprot(pte), caller); return __ioremap_caller(addr, size, pte_pgprot(pte), caller); } EXPORT_SYMBOL(ioremap_prot); diff --git a/arch/powerpc/mm/ioremap_64.c b/arch/powerpc/mm/ioremap_64.c index d24e5f166723..fb8b55bd2cd5 100644 --- a/arch/powerpc/mm/ioremap_64.c +++ b/arch/powerpc/mm/ioremap_64.c @@ -52,6 +52,6 @@ void iounmap(volatile void __iomem *token) if (!slab_is_available()) return; - generic_iounmap(PCI_FIX_ADDR(token)); + generic_iounmap(token); } EXPORT_SYMBOL(iounmap); diff --git a/arch/powerpc/mm/kasan/init_book3e_64.c b/arch/powerpc/mm/kasan/init_book3e_64.c index 43c03b84ff32..60c78aac0f63 100644 --- a/arch/powerpc/mm/kasan/init_book3e_64.c +++ b/arch/powerpc/mm/kasan/init_book3e_64.c @@ -40,19 +40,19 @@ static int __init kasan_map_kernel_page(unsigned long ea, unsigned long pa, pgpr pgdp = pgd_offset_k(ea); p4dp = p4d_offset(pgdp, ea); if (kasan_pud_table(*p4dp)) { - pudp = memblock_alloc(PUD_TABLE_SIZE, PUD_TABLE_SIZE); + pudp = memblock_alloc_or_panic(PUD_TABLE_SIZE, PUD_TABLE_SIZE); memcpy(pudp, kasan_early_shadow_pud, PUD_TABLE_SIZE); p4d_populate(&init_mm, p4dp, pudp); } pudp = pud_offset(p4dp, ea); if (kasan_pmd_table(*pudp)) { - pmdp = memblock_alloc(PMD_TABLE_SIZE, PMD_TABLE_SIZE); + pmdp = memblock_alloc_or_panic(PMD_TABLE_SIZE, PMD_TABLE_SIZE); memcpy(pmdp, kasan_early_shadow_pmd, PMD_TABLE_SIZE); pud_populate(&init_mm, pudp, pmdp); } pmdp = pmd_offset(pudp, ea); if (kasan_pte_table(*pmdp)) { - ptep = memblock_alloc(PTE_TABLE_SIZE, PTE_TABLE_SIZE); + ptep = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE); memcpy(ptep, kasan_early_shadow_pte, PTE_TABLE_SIZE); pmd_populate_kernel(&init_mm, pmdp, ptep); } @@ -74,7 +74,7 @@ static void __init kasan_init_phys_region(void *start, void *end) k_start = ALIGN_DOWN((unsigned long)kasan_mem_to_shadow(start), PAGE_SIZE); k_end = ALIGN((unsigned long)kasan_mem_to_shadow(end), PAGE_SIZE); - va = memblock_alloc(k_end - k_start, PAGE_SIZE); + va = memblock_alloc_or_panic(k_end - k_start, PAGE_SIZE); for (k_cur = k_start; k_cur < k_end; k_cur += PAGE_SIZE, va += PAGE_SIZE) kasan_map_kernel_page(k_cur, __pa(va), PAGE_KERNEL); } diff --git a/arch/powerpc/mm/kasan/init_book3s_64.c b/arch/powerpc/mm/kasan/init_book3s_64.c index 3fb5ce4f48f4..7d959544c077 100644 --- a/arch/powerpc/mm/kasan/init_book3s_64.c +++ b/arch/powerpc/mm/kasan/init_book3s_64.c @@ -32,7 +32,7 @@ static void __init kasan_init_phys_region(void *start, void *end) k_start = ALIGN_DOWN((unsigned long)kasan_mem_to_shadow(start), PAGE_SIZE); k_end = ALIGN((unsigned long)kasan_mem_to_shadow(end), PAGE_SIZE); - va = memblock_alloc(k_end - k_start, PAGE_SIZE); + va = memblock_alloc_or_panic(k_end - k_start, PAGE_SIZE); for (k_cur = k_start; k_cur < k_end; k_cur += PAGE_SIZE, va += PAGE_SIZE) map_kernel_page(k_cur, __pa(va), PAGE_KERNEL); } diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index c7708c8fad29..3ddbfdbfa941 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -273,7 +273,7 @@ void __init paging_init(void) mark_nonram_nosave(); } -void __init mem_init(void) +void __init arch_mm_preinit(void) { /* * book3s is limited to 16 page sizes due to encoding this in @@ -295,22 +295,6 @@ void __init mem_init(void) kasan_late_init(); - memblock_free_all(); - -#ifdef CONFIG_HIGHMEM - { - unsigned long pfn, highmem_mapnr; - - highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT; - for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) { - phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT; - struct page *page = pfn_to_page(pfn); - if (memblock_is_memory(paddr) && !memblock_is_reserved(paddr)) - free_highmem_page(page); - } - } -#endif /* CONFIG_HIGHMEM */ - #if defined(CONFIG_PPC_E500) && !defined(CONFIG_SMP) /* * If smp is enabled, next_tlbcam_idx is initialized in the cpu up @@ -319,28 +303,6 @@ void __init mem_init(void) per_cpu(next_tlbcam_idx, smp_processor_id()) = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1; #endif - -#ifdef CONFIG_PPC32 - pr_info("Kernel virtual memory layout:\n"); -#ifdef CONFIG_KASAN - pr_info(" * 0x%08lx..0x%08lx : kasan shadow mem\n", - KASAN_SHADOW_START, KASAN_SHADOW_END); -#endif - pr_info(" * 0x%08lx..0x%08lx : fixmap\n", FIXADDR_START, FIXADDR_TOP); -#ifdef CONFIG_HIGHMEM - pr_info(" * 0x%08lx..0x%08lx : highmem PTEs\n", - PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP)); -#endif /* CONFIG_HIGHMEM */ - if (ioremap_bot != IOREMAP_TOP) - pr_info(" * 0x%08lx..0x%08lx : early ioremap\n", - ioremap_bot, IOREMAP_TOP); - pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n", - VMALLOC_START, VMALLOC_END); -#ifdef MODULES_VADDR - pr_info(" * 0x%08lx..0x%08lx : modules\n", - MODULES_VADDR, MODULES_END); -#endif -#endif /* CONFIG_PPC32 */ } void free_initmem(void) @@ -376,7 +338,7 @@ static int __init add_system_ram_resources(void) */ res->end = end - 1; res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; - WARN_ON(request_resource(&iomem_resource, res) < 0); + WARN_ON(insert_resource(&iomem_resource, res) < 0); } } diff --git a/arch/powerpc/mm/nohash/8xx.c b/arch/powerpc/mm/nohash/8xx.c index 8b54f12d1889..ab1505cf42bf 100644 --- a/arch/powerpc/mm/nohash/8xx.c +++ b/arch/powerpc/mm/nohash/8xx.c @@ -54,20 +54,13 @@ static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa, { pmd_t *pmdp = pmd_off_k(va); pte_t *ptep; - - if (WARN_ON(psize != MMU_PAGE_512K && psize != MMU_PAGE_8M)) - return -EINVAL; + unsigned int shift = mmu_psize_to_shift(psize); if (new) { if (WARN_ON(slab_is_available())) return -EINVAL; - if (psize == MMU_PAGE_512K) { - ptep = early_pte_alloc_kernel(pmdp, va); - /* The PTE should never be already present */ - if (WARN_ON(pte_present(*ptep) && pgprot_val(prot))) - return -EINVAL; - } else { + if (psize == MMU_PAGE_8M) { if (WARN_ON(!pmd_none(*pmdp) || !pmd_none(*(pmdp + 1)))) return -EINVAL; @@ -78,20 +71,25 @@ static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa, pmd_populate_kernel(&init_mm, pmdp + 1, ptep); ptep = (pte_t *)pmdp; + } else { + ptep = early_pte_alloc_kernel(pmdp, va); + /* The PTE should never be already present */ + if (WARN_ON(pte_present(*ptep) && pgprot_val(prot))) + return -EINVAL; } } else { - if (psize == MMU_PAGE_512K) - ptep = pte_offset_kernel(pmdp, va); - else + if (psize == MMU_PAGE_8M) ptep = (pte_t *)pmdp; + else + ptep = pte_offset_kernel(pmdp, va); } if (WARN_ON(!ptep)) return -ENOMEM; set_huge_pte_at(&init_mm, va, ptep, - pte_mkhuge(pfn_pte(pa >> PAGE_SHIFT, prot)), - 1UL << mmu_psize_to_shift(psize)); + arch_make_huge_pte(pfn_pte(pa >> PAGE_SHIFT, prot), shift, 0), + 1UL << shift); return 0; } @@ -123,14 +121,18 @@ static int mmu_mapin_ram_chunk(unsigned long offset, unsigned long top, unsigned long p = offset; int err = 0; - WARN_ON(!IS_ALIGNED(offset, SZ_512K) || !IS_ALIGNED(top, SZ_512K)); + WARN_ON(!IS_ALIGNED(offset, SZ_16K) || !IS_ALIGNED(top, SZ_16K)); + for (; p < ALIGN(p, SZ_512K) && p < top && !err; p += SZ_16K, v += SZ_16K) + err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_16K, new); for (; p < ALIGN(p, SZ_8M) && p < top && !err; p += SZ_512K, v += SZ_512K) err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_512K, new); for (; p < ALIGN_DOWN(top, SZ_8M) && p < top && !err; p += SZ_8M, v += SZ_8M) err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_8M, new); for (; p < ALIGN_DOWN(top, SZ_512K) && p < top && !err; p += SZ_512K, v += SZ_512K) err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_512K, new); + for (; p < ALIGN_DOWN(top, SZ_16K) && p < top && !err; p += SZ_16K, v += SZ_16K) + err = __early_map_kernel_hugepage(v, p, prot, MMU_PAGE_16K, new); if (!new) flush_tlb_kernel_range(PAGE_OFFSET + v, PAGE_OFFSET + top); diff --git a/arch/powerpc/mm/nohash/kaslr_booke.c b/arch/powerpc/mm/nohash/kaslr_booke.c index 5c8d1bb98b3e..5e4897daaaea 100644 --- a/arch/powerpc/mm/nohash/kaslr_booke.c +++ b/arch/powerpc/mm/nohash/kaslr_booke.c @@ -178,7 +178,7 @@ static void __init get_crash_kernel(void *fdt, unsigned long size) int ret; ret = parse_crashkernel(boot_command_line, size, &crash_size, - &crash_base, NULL, NULL); + &crash_base, NULL, NULL, NULL); if (ret != 0 || crash_size == 0) return; if (crash_base == 0) diff --git a/arch/powerpc/mm/nohash/mmu_context.c b/arch/powerpc/mm/nohash/mmu_context.c index 0b181da40ddb..a1a4e697251a 100644 --- a/arch/powerpc/mm/nohash/mmu_context.c +++ b/arch/powerpc/mm/nohash/mmu_context.c @@ -385,21 +385,11 @@ void __init mmu_context_init(void) /* * Allocate the maps used by context management */ - context_map = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); - if (!context_map) - panic("%s: Failed to allocate %zu bytes\n", __func__, - CTX_MAP_SIZE); - context_mm = memblock_alloc(sizeof(void *) * (LAST_CONTEXT + 1), + context_map = memblock_alloc_or_panic(CTX_MAP_SIZE, SMP_CACHE_BYTES); + context_mm = memblock_alloc_or_panic(sizeof(void *) * (LAST_CONTEXT + 1), SMP_CACHE_BYTES); - if (!context_mm) - panic("%s: Failed to allocate %zu bytes\n", __func__, - sizeof(void *) * (LAST_CONTEXT + 1)); if (IS_ENABLED(CONFIG_SMP)) { - stale_map[boot_cpuid] = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); - if (!stale_map[boot_cpuid]) - panic("%s: Failed to allocate %zu bytes\n", __func__, - CTX_MAP_SIZE); - + stale_map[boot_cpuid] = memblock_alloc_or_panic(CTX_MAP_SIZE, SMP_CACHE_BYTES); cpuhp_setup_state_nocalls(CPUHP_POWERPC_MMU_CTX_PREPARE, "powerpc/mmu/ctx:prepare", mmu_ctx_cpu_prepare, mmu_ctx_cpu_dead); diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 3c1da08304d0..603a0f652ba6 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -1336,7 +1336,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr) return nid; } -static u64 hot_add_drconf_memory_max(void) +u64 hot_add_drconf_memory_max(void) { struct device_node *memory = NULL; struct device_node *dn = NULL; diff --git a/arch/powerpc/mm/pgtable-frag.c b/arch/powerpc/mm/pgtable-frag.c index e89f64a0f24a..77e55eac16e4 100644 --- a/arch/powerpc/mm/pgtable-frag.c +++ b/arch/powerpc/mm/pgtable-frag.c @@ -25,7 +25,7 @@ void pte_frag_destroy(void *pte_frag) count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT; /* We allow PTE_FRAG_NR fragments from a PTE page */ if (atomic_sub_and_test(PTE_FRAG_NR - count, &ptdesc->pt_frag_refcount)) { - pagetable_pte_dtor(ptdesc); + pagetable_dtor(ptdesc); pagetable_free(ptdesc); } } @@ -56,19 +56,17 @@ static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel) { void *ret = NULL; struct ptdesc *ptdesc; + gfp_t gfp = PGALLOC_GFP; - if (!kernel) { - ptdesc = pagetable_alloc(PGALLOC_GFP | __GFP_ACCOUNT, 0); - if (!ptdesc) - return NULL; - if (!pagetable_pte_ctor(ptdesc)) { - pagetable_free(ptdesc); - return NULL; - } - } else { - ptdesc = pagetable_alloc(PGALLOC_GFP, 0); - if (!ptdesc) - return NULL; + if (!kernel) + gfp |= __GFP_ACCOUNT; + + ptdesc = pagetable_alloc(gfp, 0); + if (!ptdesc) + return NULL; + if (!pagetable_pte_ctor(mm, ptdesc)) { + pagetable_free(ptdesc); + return NULL; } atomic_set(&ptdesc->pt_frag_refcount, 1); @@ -111,7 +109,7 @@ static void pte_free_now(struct rcu_head *head) struct ptdesc *ptdesc; ptdesc = container_of(head, struct ptdesc, pt_rcu_head); - pagetable_pte_dtor(ptdesc); + pagetable_dtor(ptdesc); pagetable_free(ptdesc); } @@ -124,12 +122,10 @@ void pte_fragment_free(unsigned long *table, int kernel) BUG_ON(atomic_read(&ptdesc->pt_frag_refcount) <= 0); if (atomic_dec_and_test(&ptdesc->pt_frag_refcount)) { - if (kernel) - pagetable_free(ptdesc); - else if (folio_test_clear_active(ptdesc_folio(ptdesc))) - call_rcu(&ptdesc->pt_rcu_head, pte_free_now); - else + if (kernel || !folio_test_clear_active(ptdesc_folio(ptdesc))) pte_free_now(&ptdesc->pt_rcu_head); + else + call_rcu(&ptdesc->pt_rcu_head, pte_free_now); } } diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index 61df5aed7989..dfaa9fd86f7e 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -509,7 +509,7 @@ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, return NULL; #endif - if (pmd_trans_huge(pmd) || pmd_devmap(pmd)) { + if (pmd_trans_huge(pmd)) { if (is_thp) *is_thp = true; ret_pte = (pte_t *)pmdp; diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index 787b22206386..15276068f657 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -50,13 +50,8 @@ notrace void __init early_ioremap_init(void) void __init *early_alloc_pgtable(unsigned long size) { - void *ptr = memblock_alloc(size, size); + return memblock_alloc_or_panic(size, size); - if (!ptr) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, size, size); - - return ptr; } pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va) diff --git a/arch/powerpc/mm/ptdump/ptdump.c b/arch/powerpc/mm/ptdump/ptdump.c index 9dc239967b77..b2358d794855 100644 --- a/arch/powerpc/mm/ptdump/ptdump.c +++ b/arch/powerpc/mm/ptdump/ptdump.c @@ -298,6 +298,38 @@ static void populate_markers(void) #endif } +static void note_page_pte(struct ptdump_state *pt_st, unsigned long addr, pte_t pte) +{ + note_page(pt_st, addr, 4, pte_val(pte)); +} + +static void note_page_pmd(struct ptdump_state *pt_st, unsigned long addr, pmd_t pmd) +{ + note_page(pt_st, addr, 3, pmd_val(pmd)); +} + +static void note_page_pud(struct ptdump_state *pt_st, unsigned long addr, pud_t pud) +{ + note_page(pt_st, addr, 2, pud_val(pud)); +} + +static void note_page_p4d(struct ptdump_state *pt_st, unsigned long addr, p4d_t p4d) +{ + note_page(pt_st, addr, 1, p4d_val(p4d)); +} + +static void note_page_pgd(struct ptdump_state *pt_st, unsigned long addr, pgd_t pgd) +{ + note_page(pt_st, addr, 0, pgd_val(pgd)); +} + +static void note_page_flush(struct ptdump_state *pt_st) +{ + pte_t pte_zero = {0}; + + note_page(pt_st, 0, -1, pte_val(pte_zero)); +} + static int ptdump_show(struct seq_file *m, void *v) { struct pg_state st = { @@ -305,7 +337,12 @@ static int ptdump_show(struct seq_file *m, void *v) .marker = address_markers, .level = -1, .ptdump = { - .note_page = note_page, + .note_page_pte = note_page_pte, + .note_page_pmd = note_page_pmd, + .note_page_pud = note_page_pud, + .note_page_p4d = note_page_p4d, + .note_page_pgd = note_page_pgd, + .note_page_flush = note_page_flush, .range = ptdump_range, } }; @@ -338,7 +375,12 @@ bool ptdump_check_wx(void) .level = -1, .check_wx = true, .ptdump = { - .note_page = note_page, + .note_page_pte = note_page_pte, + .note_page_pmd = note_page_pmd, + .note_page_pud = note_page_pud, + .note_page_p4d = note_page_p4d, + .note_page_pgd = note_page_pgd, + .note_page_flush = note_page_flush, .range = ptdump_range, } }; diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 6beacaec63d3..4c26912c2e3c 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -51,8 +51,16 @@ EMIT(PPC_INST_BRANCH_COND | (((cond) & 0x3ff) << 16) | (offset & 0xfffc)); \ } while (0) -/* Sign-extended 32-bit immediate load */ +/* + * Sign-extended 32-bit immediate load + * + * If this is a dummy pass (!image), account for + * maximum possible instructions. + */ #define PPC_LI32(d, i) do { \ + if (!image) \ + ctx->idx += 2; \ + else { \ if ((int)(uintptr_t)(i) >= -32768 && \ (int)(uintptr_t)(i) < 32768) \ EMIT(PPC_RAW_LI(d, i)); \ @@ -60,10 +68,15 @@ EMIT(PPC_RAW_LIS(d, IMM_H(i))); \ if (IMM_L(i)) \ EMIT(PPC_RAW_ORI(d, d, IMM_L(i))); \ - } } while(0) + } \ + } } while (0) #ifdef CONFIG_PPC64 +/* If dummy pass (!image), account for maximum possible instructions */ #define PPC_LI64(d, i) do { \ + if (!image) \ + ctx->idx += 5; \ + else { \ if ((long)(i) >= -2147483648 && \ (long)(i) < 2147483648) \ PPC_LI32(d, i); \ @@ -84,7 +97,8 @@ if ((uintptr_t)(i) & 0x000000000000ffffULL) \ EMIT(PPC_RAW_ORI(d, d, (uintptr_t)(i) & \ 0xffff)); \ - } } while (0) + } \ + } } while (0) #define PPC_LI_ADDR PPC_LI64 #ifndef CONFIG_PPC_KERNEL_PCREL diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 2991bb171a9b..c0684733e9d6 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -504,10 +504,11 @@ static int invoke_bpf_prog(u32 *image, u32 *ro_image, struct codegen_context *ct EMIT(PPC_RAW_ADDI(_R3, _R1, regs_off)); if (!p->jited) PPC_LI_ADDR(_R4, (unsigned long)p->insnsi); - if (!create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx], (unsigned long)p->bpf_func, - BRANCH_SET_LINK)) { - if (image) - image[ctx->idx] = ppc_inst_val(branch_insn); + /* Account for max possible instructions during dummy pass for size calculation */ + if (image && !create_branch(&branch_insn, (u32 *)&ro_image[ctx->idx], + (unsigned long)p->bpf_func, + BRANCH_SET_LINK)) { + image[ctx->idx] = ppc_inst_val(branch_insn); ctx->idx++; } else { EMIT(PPC_RAW_LL(_R12, _R25, offsetof(struct bpf_prog, bpf_func))); @@ -889,7 +890,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im bpf_trampoline_restore_tail_call_cnt(image, ctx, func_frame_offset, r4_off); /* Reserve space to patch branch instruction to skip fexit progs */ - im->ip_after_call = &((u32 *)ro_image)[ctx->idx]; + if (ro_image) /* image is NULL for dummy pass */ + im->ip_after_call = &((u32 *)ro_image)[ctx->idx]; EMIT(PPC_RAW_NOP()); } @@ -912,7 +914,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } if (flags & BPF_TRAMP_F_CALL_ORIG) { - im->ip_epilogue = &((u32 *)ro_image)[ctx->idx]; + if (ro_image) /* image is NULL for dummy pass */ + im->ip_epilogue = &((u32 *)ro_image)[ctx->idx]; PPC_LI_ADDR(_R3, im); ret = bpf_jit_emit_func_call_rel(image, ro_image, ctx, (unsigned long)__bpf_tramp_exit); @@ -973,25 +976,9 @@ int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, void *func_addr) { struct bpf_tramp_image im; - void *image; int ret; - /* - * Allocate a temporary buffer for __arch_prepare_bpf_trampoline(). - * This will NOT cause fragmentation in direct map, as we do not - * call set_memory_*() on this buffer. - * - * We cannot use kvmalloc here, because we need image to be in - * module memory range. - */ - image = bpf_jit_alloc_exec(PAGE_SIZE); - if (!image) - return -ENOMEM; - - ret = __arch_prepare_bpf_trampoline(&im, image, image + PAGE_SIZE, image, - m, flags, tlinks, func_addr); - bpf_jit_free_exec(image); - + ret = __arch_prepare_bpf_trampoline(&im, NULL, NULL, NULL, m, flags, tlinks, func_addr); return ret; } diff --git a/arch/powerpc/net/bpf_jit_comp32.c b/arch/powerpc/net/bpf_jit_comp32.c index c4db278dae36..0aace304dfe1 100644 --- a/arch/powerpc/net/bpf_jit_comp32.c +++ b/arch/powerpc/net/bpf_jit_comp32.c @@ -313,7 +313,6 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code u64 func_addr; u32 true_cond; u32 tmp_idx; - int j; if (i && (BPF_CLASS(code) == BPF_ALU64 || BPF_CLASS(code) == BPF_ALU) && (BPF_CLASS(prevcode) == BPF_ALU64 || BPF_CLASS(prevcode) == BPF_ALU) && @@ -1099,13 +1098,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code * 16 byte instruction that uses two 'struct bpf_insn' */ case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ - tmp_idx = ctx->idx; PPC_LI32(dst_reg_h, (u32)insn[i + 1].imm); PPC_LI32(dst_reg, (u32)insn[i].imm); - /* padding to allow full 4 instructions for later patching */ - if (!image) - for (j = ctx->idx - tmp_idx; j < 4; j++) - EMIT(PPC_RAW_NOP()); /* Adjust for two bpf instructions */ addrs[++i] = ctx->idx * 4; break; diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 233703b06d7c..025524378443 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -227,7 +227,14 @@ int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context * #ifdef CONFIG_PPC_KERNEL_PCREL reladdr = func_addr - local_paca->kernelbase; - if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) { + /* + * If fimage is NULL (the initial pass to find image size), + * account for the maximum no. of instructions possible. + */ + if (!fimage) { + ctx->idx += 7; + return 0; + } else if (reladdr < (long)SZ_8G && reladdr >= -(long)SZ_8G) { EMIT(PPC_RAW_LD(_R12, _R13, offsetof(struct paca_struct, kernelbase))); /* Align for subsequent prefix instruction */ if (!IS_ALIGNED((unsigned long)fimage + CTX_NIA(ctx), 8)) @@ -363,6 +370,23 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o return 0; } +bool bpf_jit_bypass_spec_v1(void) +{ +#if defined(CONFIG_PPC_E500) || defined(CONFIG_PPC_BOOK3S_64) + return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && + security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR)); +#else + return true; +#endif +} + +bool bpf_jit_bypass_spec_v4(void) +{ + return !(security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && + security_ftr_enabled(SEC_FTR_STF_BARRIER) && + stf_barrier_type_get() != STF_BARRIER_NONE); +} + /* * We spill into the redzone always, even if the bpf program has its own stackframe. * Offsets hardcoded based on BPF_PPC_STACK_SAVE -- see bpf_jit_stack_local() @@ -385,11 +409,77 @@ asm ( " blr ;" ); +static int emit_atomic_ld_st(const struct bpf_insn insn, struct codegen_context *ctx, u32 *image) +{ + u32 code = insn.code; + u32 dst_reg = bpf_to_ppc(insn.dst_reg); + u32 src_reg = bpf_to_ppc(insn.src_reg); + u32 size = BPF_SIZE(code); + u32 tmp1_reg = bpf_to_ppc(TMP_REG_1); + u32 tmp2_reg = bpf_to_ppc(TMP_REG_2); + s16 off = insn.off; + s32 imm = insn.imm; + + switch (imm) { + case BPF_LOAD_ACQ: + switch (size) { + case BPF_B: + EMIT(PPC_RAW_LBZ(dst_reg, src_reg, off)); + break; + case BPF_H: + EMIT(PPC_RAW_LHZ(dst_reg, src_reg, off)); + break; + case BPF_W: + EMIT(PPC_RAW_LWZ(dst_reg, src_reg, off)); + break; + case BPF_DW: + if (off % 4) { + EMIT(PPC_RAW_LI(tmp1_reg, off)); + EMIT(PPC_RAW_LDX(dst_reg, src_reg, tmp1_reg)); + } else { + EMIT(PPC_RAW_LD(dst_reg, src_reg, off)); + } + break; + } + EMIT(PPC_RAW_LWSYNC()); + break; + case BPF_STORE_REL: + EMIT(PPC_RAW_LWSYNC()); + switch (size) { + case BPF_B: + EMIT(PPC_RAW_STB(src_reg, dst_reg, off)); + break; + case BPF_H: + EMIT(PPC_RAW_STH(src_reg, dst_reg, off)); + break; + case BPF_W: + EMIT(PPC_RAW_STW(src_reg, dst_reg, off)); + break; + case BPF_DW: + if (off % 4) { + EMIT(PPC_RAW_LI(tmp2_reg, off)); + EMIT(PPC_RAW_STDX(src_reg, dst_reg, tmp2_reg)); + } else { + EMIT(PPC_RAW_STD(src_reg, dst_reg, off)); + } + break; + } + break; + default: + pr_err_ratelimited("unexpected atomic load/store op code %02x\n", + imm); + return -EINVAL; + } + + return 0; +} + /* Assemble the body code between the prologue & epilogue */ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct codegen_context *ctx, u32 *addrs, int pass, bool extra_pass) { enum stf_barrier_type stf_barrier = stf_barrier_type_get(); + bool sync_emitted, ori31_emitted; const struct bpf_insn *insn = fp->insnsi; int flen = fp->len; int i, ret; @@ -412,7 +502,6 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code u64 imm64; u32 true_cond; u32 tmp_idx; - int j; /* * addrs[] maps a BPF bytecode address into a real offset from @@ -783,30 +872,51 @@ emit_clear: /* * BPF_ST NOSPEC (speculation barrier) + * + * The following must act as a barrier against both Spectre v1 + * and v4 if we requested both mitigations. Therefore, also emit + * 'isync; sync' on E500 or 'ori31' on BOOK3S_64 in addition to + * the insns needed for a Spectre v4 barrier. + * + * If we requested only !bypass_spec_v1 OR only !bypass_spec_v4, + * we can skip the respective other barrier type as an + * optimization. */ case BPF_ST | BPF_NOSPEC: - if (!security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) || - !security_ftr_enabled(SEC_FTR_STF_BARRIER)) - break; - - switch (stf_barrier) { - case STF_BARRIER_EIEIO: - EMIT(PPC_RAW_EIEIO() | 0x02000000); - break; - case STF_BARRIER_SYNC_ORI: + sync_emitted = false; + ori31_emitted = false; + if (IS_ENABLED(CONFIG_PPC_E500) && + !bpf_jit_bypass_spec_v1()) { + EMIT(PPC_RAW_ISYNC()); EMIT(PPC_RAW_SYNC()); - EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0)); - EMIT(PPC_RAW_ORI(_R31, _R31, 0)); - break; - case STF_BARRIER_FALLBACK: - ctx->seen |= SEEN_FUNC; - PPC_LI64(_R12, dereference_kernel_function_descriptor(bpf_stf_barrier)); - EMIT(PPC_RAW_MTCTR(_R12)); - EMIT(PPC_RAW_BCTRL()); - break; - case STF_BARRIER_NONE: - break; + sync_emitted = true; } + if (!bpf_jit_bypass_spec_v4()) { + switch (stf_barrier) { + case STF_BARRIER_EIEIO: + EMIT(PPC_RAW_EIEIO() | 0x02000000); + break; + case STF_BARRIER_SYNC_ORI: + if (!sync_emitted) + EMIT(PPC_RAW_SYNC()); + EMIT(PPC_RAW_LD(tmp1_reg, _R13, 0)); + EMIT(PPC_RAW_ORI(_R31, _R31, 0)); + ori31_emitted = true; + break; + case STF_BARRIER_FALLBACK: + ctx->seen |= SEEN_FUNC; + PPC_LI64(_R12, dereference_kernel_function_descriptor(bpf_stf_barrier)); + EMIT(PPC_RAW_MTCTR(_R12)); + EMIT(PPC_RAW_BCTRL()); + break; + case STF_BARRIER_NONE: + break; + } + } + if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && + !bpf_jit_bypass_spec_v1() && + !ori31_emitted) + EMIT(PPC_RAW_ORI(_R31, _R31, 0)); break; /* @@ -853,8 +963,25 @@ emit_clear: /* * BPF_STX ATOMIC (atomic ops) */ + case BPF_STX | BPF_ATOMIC | BPF_B: + case BPF_STX | BPF_ATOMIC | BPF_H: case BPF_STX | BPF_ATOMIC | BPF_W: case BPF_STX | BPF_ATOMIC | BPF_DW: + if (bpf_atomic_is_load_store(&insn[i])) { + ret = emit_atomic_ld_st(insn[i], ctx, image); + if (ret) + return ret; + + if (size != BPF_DW && insn_is_zext(&insn[i + 1])) + addrs[++i] = ctx->idx * 4; + break; + } else if (size == BPF_B || size == BPF_H) { + pr_err_ratelimited( + "eBPF filter atomic op code %02x (@%d) unsupported\n", + code, i); + return -EOPNOTSUPP; + } + save_reg = tmp2_reg; ret_reg = src_reg; @@ -1046,12 +1173,7 @@ emit_clear: case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ imm64 = ((u64)(u32) insn[i].imm) | (((u64)(u32) insn[i+1].imm) << 32); - tmp_idx = ctx->idx; PPC_LI64(dst_reg, imm64); - /* padding to allow full 5 instructions for later patching */ - if (!image) - for (j = ctx->idx - tmp_idx; j < 5; j++) - EMIT(PPC_RAW_NOP()); /* Adjust for two bpf instructions */ addrs[++i] = ctx->idx * 4; break; diff --git a/arch/powerpc/perf/Makefile b/arch/powerpc/perf/Makefile index ac2cf58d62db..7f53fcb7495a 100644 --- a/arch/powerpc/perf/Makefile +++ b/arch/powerpc/perf/Makefile @@ -18,6 +18,8 @@ obj-$(CONFIG_HV_PERF_CTRS) += hv-24x7.o hv-gpci.o hv-common.o obj-$(CONFIG_VPA_PMU) += vpa-pmu.o +obj-$(CONFIG_KVM_BOOK3S_HV_PMU) += kvm-hv-pmu.o + obj-$(CONFIG_PPC_8xx) += 8xx-pmu.o obj-$(CONFIG_PPC64) += $(obj64-y) diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 2b79171ee185..8b0081441f85 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -132,7 +132,10 @@ static unsigned long ebb_switch_in(bool ebb, struct cpu_hw_events *cpuhw) static inline void power_pmu_bhrb_enable(struct perf_event *event) {} static inline void power_pmu_bhrb_disable(struct perf_event *event) {} -static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) {} +static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, + struct task_struct *task, bool sched_in) +{ +} static inline void power_pmu_bhrb_read(struct perf_event *event, struct cpu_hw_events *cpuhw) {} static void pmao_restore_workaround(bool ebb) { } #endif /* CONFIG_PPC32 */ @@ -444,7 +447,8 @@ static void power_pmu_bhrb_disable(struct perf_event *event) /* Called from ctxsw to prevent one process's branch entries to * mingle with the other process's entries during context switch. */ -static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, bool sched_in) +static void power_pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, + struct task_struct *task, bool sched_in) { if (!ppmu->bhrb_nr) return; @@ -2222,6 +2226,10 @@ static struct pmu power_pmu = { #define PERF_SAMPLE_ADDR_TYPE (PERF_SAMPLE_ADDR | \ PERF_SAMPLE_PHYS_ADDR | \ PERF_SAMPLE_DATA_PAGE_SIZE) + +#define SIER_TYPE_SHIFT 15 +#define SIER_TYPE_MASK (0x7ull << SIER_TYPE_SHIFT) + /* * A counter has overflowed; update its count and record * things if requested. Note that interrupts are hard-disabled @@ -2231,6 +2239,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, struct pt_regs *regs) { u64 period = event->hw.sample_period; + const u64 last_period = event->hw.last_period; s64 prev, delta, left; int record = 0; @@ -2291,12 +2300,28 @@ static void record_and_restart(struct perf_event *event, unsigned long val, record = 0; /* + * SIER[46-48] presents instruction type of the sampled instruction. + * In ISA v3.0 and before values "0" and "7" are considered reserved. + * In ISA v3.1, value "7" has been used to indicate "larx/stcx". + * Drop the sample if "type" has reserved values for this field with a + * ISA version check. + */ + if (event->attr.sample_type & PERF_SAMPLE_DATA_SRC && + ppmu->get_mem_data_src) { + val = (regs->dar & SIER_TYPE_MASK) >> SIER_TYPE_SHIFT; + if (val == 0 || (val == 7 && !cpu_has_feature(CPU_FTR_ARCH_31))) { + record = 0; + atomic64_inc(&event->lost_samples); + } + } + + /* * Finally record data if requested. */ if (record) { struct perf_sample_data data; - perf_sample_data_init(&data, ~0ULL, event->hw.last_period); + perf_sample_data_init(&data, ~0ULL, last_period); if (event->attr.sample_type & PERF_SAMPLE_ADDR_TYPE) perf_get_data_addr(event, regs, &data.addr); @@ -2319,12 +2344,10 @@ static void record_and_restart(struct perf_event *event, unsigned long val, ppmu->get_mem_weight(&data.weight.full, event->attr.sample_type); data.sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; } - if (perf_event_overflow(event, &data, regs)) - power_pmu_stop(event, 0); + perf_event_overflow(event, &data, regs); } else if (period) { /* Account for interrupt in case of invalid SIAR */ - if (perf_event_account_interrupt(event)) - power_pmu_stop(event, 0); + perf_event_account_interrupt(event); } } diff --git a/arch/powerpc/perf/core-fsl-emb.c b/arch/powerpc/perf/core-fsl-emb.c index 1a53ab08447c..7120ab20cbfe 100644 --- a/arch/powerpc/perf/core-fsl-emb.c +++ b/arch/powerpc/perf/core-fsl-emb.c @@ -590,6 +590,7 @@ static void record_and_restart(struct perf_event *event, unsigned long val, struct pt_regs *regs) { u64 period = event->hw.sample_period; + const u64 last_period = event->hw.last_period; s64 prev, delta, left; int record = 0; @@ -632,10 +633,9 @@ static void record_and_restart(struct perf_event *event, unsigned long val, if (record) { struct perf_sample_data data; - perf_sample_data_init(&data, 0, event->hw.last_period); + perf_sample_data_init(&data, 0, last_period); - if (perf_event_overflow(event, &data, regs)) - fsl_emb_pmu_stop(event, 0); + perf_event_overflow(event, &data, regs); } } diff --git a/arch/powerpc/perf/hv-24x7.c b/arch/powerpc/perf/hv-24x7.c index d400fa391c27..e42677cc254a 100644 --- a/arch/powerpc/perf/hv-24x7.c +++ b/arch/powerpc/perf/hv-24x7.c @@ -713,12 +713,12 @@ static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, ev_len = be16_to_cpu(event->length); if (ev_len % 16) - pr_info("event %zu has length %zu not divisible by 16: event=%pK\n", + pr_info("event %zu has length %zu not divisible by 16: event=%p\n", event_idx, ev_len, event); ev_end = (__u8 *)event + ev_len; if (ev_end > end) { - pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%pK > end=%pK, offset=%zu\n", + pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%p > end=%p, offset=%zu\n", event_idx, ev_len, ev_end, end, offset); return -1; @@ -726,14 +726,14 @@ static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event, calc_ev_end = event_end(event, end); if (!calc_ev_end) { - pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%pK end=%pK, offset=%zu\n", + pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%p end=%p, offset=%zu\n", event_idx, event_data_bytes, event, end, offset); return -1; } if (calc_ev_end > ev_end) { - pr_warn("event %zu exceeds its own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n", + pr_warn("event %zu exceeds its own length: event=%p, end=%p, offset=%zu, calc_ev_end=%p\n", event_idx, event, ev_end, offset, calc_ev_end); return -1; } @@ -998,7 +998,7 @@ e_out: } static ssize_t catalog_read(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, char *buf, + const struct bin_attribute *bin_attr, char *buf, loff_t offset, size_t count) { long hret; @@ -1108,14 +1108,14 @@ PAGE_0_ATTR(catalog_version, "%lld\n", (unsigned long long)be64_to_cpu(page_0->version)); PAGE_0_ATTR(catalog_len, "%lld\n", (unsigned long long)be32_to_cpu(page_0->length) * 4096); -static BIN_ATTR_RO(catalog, 0/* real length varies */); +static const BIN_ATTR_RO(catalog, 0/* real length varies */); static DEVICE_ATTR_RO(domains); static DEVICE_ATTR_RO(sockets); static DEVICE_ATTR_RO(chipspersocket); static DEVICE_ATTR_RO(coresperchip); static DEVICE_ATTR_RO(cpumask); -static struct bin_attribute *if_bin_attrs[] = { +static const struct bin_attribute *const if_bin_attrs[] = { &bin_attr_catalog, NULL, }; diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index 56301b2bc8ae..2b3547fdba4a 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -319,10 +319,18 @@ void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags, return; } - sier = mfspr(SPRN_SIER); + /* + * Use regs-dar for SPRN_SIER which is saved + * during perf_read_regs at the beginning + * of the PMU interrupt handler to avoid multiple + * reads of SPRN_SIER + */ + sier = regs->dar; val = (sier & ISA207_SIER_TYPE_MASK) >> ISA207_SIER_TYPE_SHIFT; - if (val != 1 && val != 2 && !(val == 7 && cpu_has_feature(CPU_FTR_ARCH_31))) + if (val != 1 && val != 2 && !(val == 7 && cpu_has_feature(CPU_FTR_ARCH_31))) { + dsrc->val = 0; return; + } idx = (sier & ISA207_SIER_LDST_MASK) >> ISA207_SIER_LDST_SHIFT; sub_idx = (sier & ISA207_SIER_DATA_SRC_MASK) >> ISA207_SIER_DATA_SRC_SHIFT; @@ -338,8 +346,12 @@ void isa207_get_mem_data_src(union perf_mem_data_src *dsrc, u32 flags, * to determine the exact instruction type. If the sampling * criteria is neither load or store, set the type as default * to NA. + * + * Use regs->dsisr for MMCRA which is saved during perf_read_regs + * at the beginning of the PMU interrupt handler to avoid + * multiple reads of SPRN_MMCRA */ - mmcra = mfspr(SPRN_MMCRA); + mmcra = regs->dsisr; op_type = (mmcra >> MMCRA_SAMP_ELIG_SHIFT) & MMCRA_SAMP_ELIG_MASK; switch (op_type) { diff --git a/arch/powerpc/perf/kvm-hv-pmu.c b/arch/powerpc/perf/kvm-hv-pmu.c new file mode 100644 index 000000000000..ae264c9080ef --- /dev/null +++ b/arch/powerpc/perf/kvm-hv-pmu.c @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Description: PMUs specific to running nested KVM-HV guests + * on Book3S processors (specifically POWER9 and later). + */ + +#define pr_fmt(fmt) "kvmppc-pmu: " fmt + +#include "asm-generic/local64.h" +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/ratelimit.h> +#include <linux/kvm_host.h> +#include <linux/gfp_types.h> +#include <linux/pgtable.h> +#include <linux/perf_event.h> +#include <linux/spinlock_types.h> +#include <linux/spinlock.h> + +#include <asm/types.h> +#include <asm/kvm_ppc.h> +#include <asm/kvm_book3s.h> +#include <asm/mmu.h> +#include <asm/pgalloc.h> +#include <asm/pte-walk.h> +#include <asm/reg.h> +#include <asm/plpar_wrappers.h> +#include <asm/firmware.h> + +#include "asm/guest-state-buffer.h" + +enum kvmppc_pmu_eventid { + KVMPPC_EVENT_HOST_HEAP, + KVMPPC_EVENT_HOST_HEAP_MAX, + KVMPPC_EVENT_HOST_PGTABLE, + KVMPPC_EVENT_HOST_PGTABLE_MAX, + KVMPPC_EVENT_HOST_PGTABLE_RECLAIM, + KVMPPC_EVENT_MAX, +}; + +#define KVMPPC_PMU_EVENT_ATTR(_name, _id) \ + PMU_EVENT_ATTR_ID(_name, kvmppc_events_sysfs_show, _id) + +static ssize_t kvmppc_events_sysfs_show(struct device *dev, + struct device_attribute *attr, + char *page) +{ + struct perf_pmu_events_attr *pmu_attr; + + pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); + return sprintf(page, "event=0x%02llx\n", pmu_attr->id); +} + +/* Holds the hostwide stats */ +static struct kvmppc_hostwide_stats { + u64 guest_heap; + u64 guest_heap_max; + u64 guest_pgtable_size; + u64 guest_pgtable_size_max; + u64 guest_pgtable_reclaim; +} l0_stats; + +/* Protect access to l0_stats */ +static DEFINE_SPINLOCK(lock_l0_stats); + +/* GSB related structs needed to talk to L0 */ +static struct kvmppc_gs_msg *gsm_l0_stats; +static struct kvmppc_gs_buff *gsb_l0_stats; +static struct kvmppc_gs_parser gsp_l0_stats; + +static struct attribute *kvmppc_pmu_events_attr[] = { + KVMPPC_PMU_EVENT_ATTR(host_heap, KVMPPC_EVENT_HOST_HEAP), + KVMPPC_PMU_EVENT_ATTR(host_heap_max, KVMPPC_EVENT_HOST_HEAP_MAX), + KVMPPC_PMU_EVENT_ATTR(host_pagetable, KVMPPC_EVENT_HOST_PGTABLE), + KVMPPC_PMU_EVENT_ATTR(host_pagetable_max, KVMPPC_EVENT_HOST_PGTABLE_MAX), + KVMPPC_PMU_EVENT_ATTR(host_pagetable_reclaim, KVMPPC_EVENT_HOST_PGTABLE_RECLAIM), + NULL, +}; + +static const struct attribute_group kvmppc_pmu_events_group = { + .name = "events", + .attrs = kvmppc_pmu_events_attr, +}; + +PMU_FORMAT_ATTR(event, "config:0-5"); +static struct attribute *kvmppc_pmu_format_attr[] = { + &format_attr_event.attr, + NULL, +}; + +static struct attribute_group kvmppc_pmu_format_group = { + .name = "format", + .attrs = kvmppc_pmu_format_attr, +}; + +static const struct attribute_group *kvmppc_pmu_attr_groups[] = { + &kvmppc_pmu_events_group, + &kvmppc_pmu_format_group, + NULL, +}; + +/* + * Issue the hcall to get the L0-host stats. + * Should be called with l0-stat lock held + */ +static int kvmppc_update_l0_stats(void) +{ + int rc; + + /* With HOST_WIDE flags guestid and vcpuid will be ignored */ + rc = kvmppc_gsb_recv(gsb_l0_stats, KVMPPC_GS_FLAGS_HOST_WIDE); + if (rc) + goto out; + + /* Parse the guest state buffer is successful */ + rc = kvmppc_gse_parse(&gsp_l0_stats, gsb_l0_stats); + if (rc) + goto out; + + /* Update the l0 returned stats*/ + memset(&l0_stats, 0, sizeof(l0_stats)); + rc = kvmppc_gsm_refresh_info(gsm_l0_stats, gsb_l0_stats); + +out: + return rc; +} + +/* Update the value of the given perf_event */ +static int kvmppc_pmu_event_update(struct perf_event *event) +{ + int rc; + u64 curr_val, prev_val; + unsigned long flags; + unsigned int config = event->attr.config; + + /* Ensure no one else is modifying the l0_stats */ + spin_lock_irqsave(&lock_l0_stats, flags); + + rc = kvmppc_update_l0_stats(); + if (!rc) { + switch (config) { + case KVMPPC_EVENT_HOST_HEAP: + curr_val = l0_stats.guest_heap; + break; + case KVMPPC_EVENT_HOST_HEAP_MAX: + curr_val = l0_stats.guest_heap_max; + break; + case KVMPPC_EVENT_HOST_PGTABLE: + curr_val = l0_stats.guest_pgtable_size; + break; + case KVMPPC_EVENT_HOST_PGTABLE_MAX: + curr_val = l0_stats.guest_pgtable_size_max; + break; + case KVMPPC_EVENT_HOST_PGTABLE_RECLAIM: + curr_val = l0_stats.guest_pgtable_reclaim; + break; + default: + rc = -ENOENT; + break; + } + } + + spin_unlock_irqrestore(&lock_l0_stats, flags); + + /* If no error than update the perf event */ + if (!rc) { + prev_val = local64_xchg(&event->hw.prev_count, curr_val); + if (curr_val > prev_val) + local64_add(curr_val - prev_val, &event->count); + } + + return rc; +} + +static int kvmppc_pmu_event_init(struct perf_event *event) +{ + unsigned int config = event->attr.config; + + pr_debug("%s: Event(%p) id=%llu cpu=%x on_cpu=%x config=%u", + __func__, event, event->id, event->cpu, + event->oncpu, config); + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + if (config >= KVMPPC_EVENT_MAX) + return -EINVAL; + + local64_set(&event->hw.prev_count, 0); + local64_set(&event->count, 0); + + return 0; +} + +static void kvmppc_pmu_del(struct perf_event *event, int flags) +{ + kvmppc_pmu_event_update(event); +} + +static int kvmppc_pmu_add(struct perf_event *event, int flags) +{ + if (flags & PERF_EF_START) + return kvmppc_pmu_event_update(event); + return 0; +} + +static void kvmppc_pmu_read(struct perf_event *event) +{ + kvmppc_pmu_event_update(event); +} + +/* Return the size of the needed guest state buffer */ +static size_t hostwide_get_size(struct kvmppc_gs_msg *gsm) + +{ + size_t size = 0; + const u16 ids[] = { + KVMPPC_GSID_L0_GUEST_HEAP, + KVMPPC_GSID_L0_GUEST_HEAP_MAX, + KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE, + KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX, + KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM + }; + + for (int i = 0; i < ARRAY_SIZE(ids); i++) + size += kvmppc_gse_total_size(kvmppc_gsid_size(ids[i])); + return size; +} + +/* Populate the request guest state buffer */ +static int hostwide_fill_info(struct kvmppc_gs_buff *gsb, + struct kvmppc_gs_msg *gsm) +{ + int rc = 0; + struct kvmppc_hostwide_stats *stats = gsm->data; + + /* + * It doesn't matter what values are put into request buffer as + * they are going to be overwritten anyways. But for the sake of + * testcode and symmetry contents of existing stats are put + * populated into the request guest state buffer. + */ + if (kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_HEAP)) + rc = kvmppc_gse_put_u64(gsb, + KVMPPC_GSID_L0_GUEST_HEAP, + stats->guest_heap); + + if (!rc && kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_HEAP_MAX)) + rc = kvmppc_gse_put_u64(gsb, + KVMPPC_GSID_L0_GUEST_HEAP_MAX, + stats->guest_heap_max); + + if (!rc && kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE)) + rc = kvmppc_gse_put_u64(gsb, + KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE, + stats->guest_pgtable_size); + if (!rc && + kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX)) + rc = kvmppc_gse_put_u64(gsb, + KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX, + stats->guest_pgtable_size_max); + if (!rc && + kvmppc_gsm_includes(gsm, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM)) + rc = kvmppc_gse_put_u64(gsb, + KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM, + stats->guest_pgtable_reclaim); + + return rc; +} + +/* Parse and update the host wide stats from returned gsb */ +static int hostwide_refresh_info(struct kvmppc_gs_msg *gsm, + struct kvmppc_gs_buff *gsb) +{ + struct kvmppc_gs_parser gsp = { 0 }; + struct kvmppc_hostwide_stats *stats = gsm->data; + struct kvmppc_gs_elem *gse; + int rc; + + rc = kvmppc_gse_parse(&gsp, gsb); + if (rc < 0) + return rc; + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_HEAP); + if (gse) + stats->guest_heap = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_HEAP_MAX); + if (gse) + stats->guest_heap_max = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE); + if (gse) + stats->guest_pgtable_size = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX); + if (gse) + stats->guest_pgtable_size_max = kvmppc_gse_get_u64(gse); + + gse = kvmppc_gsp_lookup(&gsp, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM); + if (gse) + stats->guest_pgtable_reclaim = kvmppc_gse_get_u64(gse); + + return 0; +} + +/* gsb-message ops for setting up/parsing */ +static struct kvmppc_gs_msg_ops gsb_ops_l0_stats = { + .get_size = hostwide_get_size, + .fill_info = hostwide_fill_info, + .refresh_info = hostwide_refresh_info, +}; + +static int kvmppc_init_hostwide(void) +{ + int rc = 0; + unsigned long flags; + + spin_lock_irqsave(&lock_l0_stats, flags); + + /* already registered ? */ + if (gsm_l0_stats) { + rc = 0; + goto out; + } + + /* setup the Guest state message/buffer to talk to L0 */ + gsm_l0_stats = kvmppc_gsm_new(&gsb_ops_l0_stats, &l0_stats, + GSM_SEND, GFP_KERNEL); + if (!gsm_l0_stats) { + rc = -ENOMEM; + goto out; + } + + /* Populate the Idents */ + kvmppc_gsm_include(gsm_l0_stats, KVMPPC_GSID_L0_GUEST_HEAP); + kvmppc_gsm_include(gsm_l0_stats, KVMPPC_GSID_L0_GUEST_HEAP_MAX); + kvmppc_gsm_include(gsm_l0_stats, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE); + kvmppc_gsm_include(gsm_l0_stats, KVMPPC_GSID_L0_GUEST_PGTABLE_SIZE_MAX); + kvmppc_gsm_include(gsm_l0_stats, KVMPPC_GSID_L0_GUEST_PGTABLE_RECLAIM); + + /* allocate GSB. Guest/Vcpu Id is ignored */ + gsb_l0_stats = kvmppc_gsb_new(kvmppc_gsm_size(gsm_l0_stats), 0, 0, + GFP_KERNEL); + if (!gsb_l0_stats) { + rc = -ENOMEM; + goto out; + } + + /* ask the ops to fill in the info */ + rc = kvmppc_gsm_fill_info(gsm_l0_stats, gsb_l0_stats); + +out: + if (rc) { + if (gsm_l0_stats) + kvmppc_gsm_free(gsm_l0_stats); + if (gsb_l0_stats) + kvmppc_gsb_free(gsb_l0_stats); + gsm_l0_stats = NULL; + gsb_l0_stats = NULL; + } + spin_unlock_irqrestore(&lock_l0_stats, flags); + return rc; +} + +static void kvmppc_cleanup_hostwide(void) +{ + unsigned long flags; + + spin_lock_irqsave(&lock_l0_stats, flags); + + if (gsm_l0_stats) + kvmppc_gsm_free(gsm_l0_stats); + if (gsb_l0_stats) + kvmppc_gsb_free(gsb_l0_stats); + gsm_l0_stats = NULL; + gsb_l0_stats = NULL; + + spin_unlock_irqrestore(&lock_l0_stats, flags); +} + +/* L1 wide counters PMU */ +static struct pmu kvmppc_pmu = { + .module = THIS_MODULE, + .task_ctx_nr = perf_sw_context, + .name = "kvm-hv", + .event_init = kvmppc_pmu_event_init, + .add = kvmppc_pmu_add, + .del = kvmppc_pmu_del, + .read = kvmppc_pmu_read, + .attr_groups = kvmppc_pmu_attr_groups, + .type = -1, + .scope = PERF_PMU_SCOPE_SYS_WIDE, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE | PERF_PMU_CAP_NO_INTERRUPT, +}; + +static int __init kvmppc_register_pmu(void) +{ + int rc = -EOPNOTSUPP; + + /* only support events for nestedv2 right now */ + if (kvmhv_is_nestedv2()) { + rc = kvmppc_init_hostwide(); + if (rc) + goto out; + + /* Register the pmu */ + rc = perf_pmu_register(&kvmppc_pmu, kvmppc_pmu.name, -1); + if (rc) + goto out; + + pr_info("Registered kvm-hv pmu"); + } + +out: + return rc; +} + +static void __exit kvmppc_unregister_pmu(void) +{ + if (kvmhv_is_nestedv2()) { + kvmppc_cleanup_hostwide(); + + if (kvmppc_pmu.type != -1) + perf_pmu_unregister(&kvmppc_pmu); + + pr_info("kvmhv_pmu unregistered.\n"); + } +} + +module_init(kvmppc_register_pmu); +module_exit(kvmppc_unregister_pmu); +MODULE_DESCRIPTION("KVM PPC Book3s-hv PMU"); +MODULE_AUTHOR("Vaibhav Jain <vaibhav@linux.ibm.com>"); +MODULE_LICENSE("GPL"); diff --git a/arch/powerpc/perf/vpa-pmu.c b/arch/powerpc/perf/vpa-pmu.c index 6a5bfd2a13b5..840733468959 100644 --- a/arch/powerpc/perf/vpa-pmu.c +++ b/arch/powerpc/perf/vpa-pmu.c @@ -156,6 +156,7 @@ static void vpa_pmu_del(struct perf_event *event, int flags) } static struct pmu vpa_pmu = { + .module = THIS_MODULE, .task_ctx_nr = perf_sw_context, .name = "vpa_pmu", .event_init = vpa_pmu_event_init, diff --git a/arch/powerpc/platforms/44x/gpio.c b/arch/powerpc/platforms/44x/gpio.c index e5f2319e5cbe..08ab76582568 100644 --- a/arch/powerpc/platforms/44x/gpio.c +++ b/arch/powerpc/platforms/44x/gpio.c @@ -75,8 +75,7 @@ __ppc4xx_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) clrbits32(®s->or, GPIO_MASK(gpio)); } -static void -ppc4xx_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) +static int ppc4xx_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) { struct ppc4xx_gpio_chip *chip = gpiochip_get_data(gc); unsigned long flags; @@ -88,6 +87,8 @@ ppc4xx_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) spin_unlock_irqrestore(&chip->lock, flags); pr_debug("%s: gpio: %d val: %d\n", __func__, gpio, val); + + return 0; } static int ppc4xx_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio) diff --git a/arch/powerpc/platforms/44x/uic.c b/arch/powerpc/platforms/44x/uic.c index e3e148b9dd18..85daf841fd3f 100644 --- a/arch/powerpc/platforms/44x/uic.c +++ b/arch/powerpc/platforms/44x/uic.c @@ -37,7 +37,7 @@ #define UIC_VR 0x7 #define UIC_VCR 0x8 -struct uic *primary_uic; +static struct uic *primary_uic; struct uic { int index; @@ -254,8 +254,9 @@ static struct uic * __init uic_init_one(struct device_node *node) } uic->dcrbase = *dcrreg; - uic->irqhost = irq_domain_add_linear(node, NR_UIC_INTS, &uic_host_ops, - uic); + uic->irqhost = irq_domain_create_linear(of_fwnode_handle(node), + NR_UIC_INTS, &uic_host_ops, + uic); if (! uic->irqhost) return NULL; /* FIXME: panic? */ @@ -291,7 +292,7 @@ void __init uic_init_tree(void) if (!primary_uic) panic("Unable to initialize primary UIC %pOF\n", np); - irq_set_default_host(primary_uic->irqhost); + irq_set_default_domain(primary_uic->irqhost); of_node_put(np); /* The scan again for cascaded UICs */ @@ -327,5 +328,5 @@ unsigned int uic_get_irq(void) msr = mfdcr(primary_uic->dcrbase + UIC_MSR); src = 32 - ffs(msr); - return irq_linear_revmap(primary_uic->irqhost, src); + return irq_find_mapping(primary_uic->irqhost, src); } diff --git a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c index e995eb30bf09..2cf3c6237337 100644 --- a/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c +++ b/arch/powerpc/platforms/512x/mpc5121_ads_cpld.c @@ -188,7 +188,8 @@ mpc5121_ads_cpld_pic_init(void) cpld_pic_node = of_node_get(np); - cpld_pic_host = irq_domain_add_linear(np, 16, &cpld_pic_host_ops, NULL); + cpld_pic_host = irq_domain_create_linear(of_fwnode_handle(np), 16, + &cpld_pic_host_ops, NULL); if (!cpld_pic_host) { printk(KERN_ERR "CPLD PIC: failed to allocate irq host!\n"); goto end; diff --git a/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c b/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c index 9668b052cd4b..f251e0f68262 100644 --- a/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c +++ b/arch/powerpc/platforms/512x/mpc512x_lpbfifo.c @@ -240,10 +240,8 @@ static int mpc512x_lpbfifo_kick(void) dma_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; /* Make DMA channel work with LPB FIFO data register */ - if (dma_dev->device_config(lpbfifo.chan, &dma_conf)) { - ret = -EINVAL; - goto err_dma_prep; - } + if (dma_dev->device_config(lpbfifo.chan, &dma_conf)) + return -EINVAL; sg_init_table(&sg, 1); diff --git a/arch/powerpc/platforms/52xx/media5200.c b/arch/powerpc/platforms/52xx/media5200.c index 19626cd42406..bc7f83cfec1d 100644 --- a/arch/powerpc/platforms/52xx/media5200.c +++ b/arch/powerpc/platforms/52xx/media5200.c @@ -168,7 +168,7 @@ static void __init media5200_init_irq(void) spin_lock_init(&media5200_irq.lock); - media5200_irq.irqhost = irq_domain_add_linear(fpga_np, + media5200_irq.irqhost = irq_domain_create_linear(of_fwnode_handle(fpga_np), MEDIA5200_NUM_IRQS, &media5200_irq_ops, &media5200_irq); if (!media5200_irq.irqhost) goto out; diff --git a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c index 1ea591ec6083..7748b6641a3c 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_gpt.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_gpt.c @@ -247,9 +247,9 @@ mpc52xx_gpt_irq_setup(struct mpc52xx_gpt_priv *gpt, struct device_node *node) if (!cascade_virq) return; - gpt->irqhost = irq_domain_add_linear(node, 1, &mpc52xx_gpt_irq_ops, gpt); + gpt->irqhost = irq_domain_create_linear(of_fwnode_handle(node), 1, &mpc52xx_gpt_irq_ops, gpt); if (!gpt->irqhost) { - dev_err(gpt->dev, "irq_domain_add_linear() failed\n"); + dev_err(gpt->dev, "irq_domain_create_linear() failed\n"); return; } @@ -280,7 +280,7 @@ static int mpc52xx_gpt_gpio_get(struct gpio_chip *gc, unsigned int gpio) return (in_be32(&gpt->regs->status) >> 8) & 1; } -static void +static int mpc52xx_gpt_gpio_set(struct gpio_chip *gc, unsigned int gpio, int v) { struct mpc52xx_gpt_priv *gpt = gpiochip_get_data(gc); @@ -293,6 +293,8 @@ mpc52xx_gpt_gpio_set(struct gpio_chip *gc, unsigned int gpio, int v) raw_spin_lock_irqsave(&gpt->lock, flags); clrsetbits_be32(&gpt->regs->mode, MPC52xx_GPT_MODE_GPIO_MASK, r); raw_spin_unlock_irqrestore(&gpt->lock, flags); + + return 0; } static int mpc52xx_gpt_gpio_dir_in(struct gpio_chip *gc, unsigned int gpio) @@ -369,7 +371,7 @@ struct mpc52xx_gpt_priv *mpc52xx_gpt_from_irq(int irq) mutex_lock(&mpc52xx_gpt_list_mutex); list_for_each(pos, &mpc52xx_gpt_list) { gpt = container_of(pos, struct mpc52xx_gpt_priv, list); - if (gpt->irqhost && irq == irq_linear_revmap(gpt->irqhost, 0)) { + if (gpt->irqhost && irq == irq_find_mapping(gpt->irqhost, 0)) { mutex_unlock(&mpc52xx_gpt_list_mutex); return gpt; } diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pic.c b/arch/powerpc/platforms/52xx/mpc52xx_pic.c index 1e0a5e9644dc..eb6a4e745c08 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_pic.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_pic.c @@ -446,14 +446,14 @@ void __init mpc52xx_init_irq(void) * As last step, add an irq host to translate the real * hw irq information provided by the ofw to linux virq */ - mpc52xx_irqhost = irq_domain_add_linear(picnode, + mpc52xx_irqhost = irq_domain_create_linear(of_fwnode_handle(picnode), MPC52xx_IRQ_HIGHTESTHWIRQ, &mpc52xx_irqhost_ops, NULL); if (!mpc52xx_irqhost) panic(__FILE__ ": Cannot allocate the IRQ host\n"); - irq_set_default_host(mpc52xx_irqhost); + irq_set_default_domain(mpc52xx_irqhost); pr_info("MPC52xx PIC is up and running!\n"); } @@ -515,5 +515,5 @@ unsigned int mpc52xx_get_irq(void) return 0; } - return irq_linear_revmap(mpc52xx_irqhost, irq); + return irq_find_mapping(mpc52xx_irqhost, irq); } diff --git a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c index 4d8fa9ed1a67..cb7b9498f291 100644 --- a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c +++ b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c @@ -92,10 +92,11 @@ static void mcu_power_off(void) mutex_unlock(&mcu->lock); } -static void mcu_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) +static int mcu_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) { struct mcu *mcu = gpiochip_get_data(gc); u8 bit = 1 << (4 + gpio); + int ret; mutex_lock(&mcu->lock); if (val) @@ -103,14 +104,16 @@ static void mcu_gpio_set(struct gpio_chip *gc, unsigned int gpio, int val) else mcu->reg_ctrl |= bit; - i2c_smbus_write_byte_data(mcu->client, MCU_REG_CTRL, mcu->reg_ctrl); + ret = i2c_smbus_write_byte_data(mcu->client, MCU_REG_CTRL, + mcu->reg_ctrl); mutex_unlock(&mcu->lock); + + return ret; } static int mcu_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) { - mcu_gpio_set(gc, gpio, val); - return 0; + return mcu_gpio_set(gc, gpio, val); } static int mcu_gpiochip_add(struct mcu *mcu) diff --git a/arch/powerpc/platforms/85xx/socrates_fpga_pic.c b/arch/powerpc/platforms/85xx/socrates_fpga_pic.c index 60e0b8947ce6..4b69fb321a68 100644 --- a/arch/powerpc/platforms/85xx/socrates_fpga_pic.c +++ b/arch/powerpc/platforms/85xx/socrates_fpga_pic.c @@ -83,7 +83,7 @@ static inline unsigned int socrates_fpga_pic_get_irq(unsigned int irq) if (cause >> (i + 16)) break; } - return irq_linear_revmap(socrates_fpga_pic_irq_host, + return irq_find_mapping(socrates_fpga_pic_irq_host, (irq_hw_number_t)i); } @@ -278,7 +278,7 @@ void __init socrates_fpga_pic_init(struct device_node *pic) int i; /* Setup an irq_domain structure */ - socrates_fpga_pic_irq_host = irq_domain_add_linear(pic, + socrates_fpga_pic_irq_host = irq_domain_create_linear(of_fwnode_handle(pic), SOCRATES_FPGA_NUM_IRQS, &socrates_fpga_pic_host_ops, NULL); if (socrates_fpga_pic_irq_host == NULL) { pr_err("FPGA PIC: Unable to allocate host\n"); diff --git a/arch/powerpc/platforms/8xx/cpm1-ic.c b/arch/powerpc/platforms/8xx/cpm1-ic.c index a18fc7c99f83..a49d4a9ab3bc 100644 --- a/arch/powerpc/platforms/8xx/cpm1-ic.c +++ b/arch/powerpc/platforms/8xx/cpm1-ic.c @@ -59,7 +59,7 @@ static int cpm_get_irq(struct irq_desc *desc) cpm_vec = in_be16(&data->reg->cpic_civr); cpm_vec >>= 11; - return irq_linear_revmap(data->host, cpm_vec); + return irq_find_mapping(data->host, cpm_vec); } static void cpm_cascade(struct irq_desc *desc) @@ -110,7 +110,8 @@ static int cpm_pic_probe(struct platform_device *pdev) out_be32(&data->reg->cpic_cimr, 0); - data->host = irq_domain_add_linear(dev->of_node, 64, &cpm_pic_host_ops, data); + data->host = irq_domain_create_linear(of_fwnode_handle(dev->of_node), + 64, &cpm_pic_host_ops, data); if (!data->host) return -ENODEV; diff --git a/arch/powerpc/platforms/8xx/cpm1.c b/arch/powerpc/platforms/8xx/cpm1.c index b24d4102fbf6..7433be7d66ee 100644 --- a/arch/powerpc/platforms/8xx/cpm1.c +++ b/arch/powerpc/platforms/8xx/cpm1.c @@ -45,7 +45,7 @@ #include <sysdev/fsl_soc.h> #ifdef CONFIG_8xx_GPIO -#include <linux/gpio/legacy-of-mm-gpiochip.h> +#include <linux/gpio/driver.h> #endif #define CPM_MAP_SIZE (0x4000) @@ -376,7 +376,8 @@ int __init cpm1_clk_setup(enum cpm_clk_target target, int clock, int mode) #ifdef CONFIG_8xx_GPIO struct cpm1_gpio16_chip { - struct of_mm_gpio_chip mm_gc; + struct gpio_chip gc; + void __iomem *regs; spinlock_t lock; /* shadowed data register to clear/set bits safely */ @@ -386,19 +387,17 @@ struct cpm1_gpio16_chip { int irq[16]; }; -static void cpm1_gpio16_save_regs(struct of_mm_gpio_chip *mm_gc) +static void cpm1_gpio16_save_regs(struct cpm1_gpio16_chip *cpm1_gc) { - struct cpm1_gpio16_chip *cpm1_gc = - container_of(mm_gc, struct cpm1_gpio16_chip, mm_gc); - struct cpm_ioport16 __iomem *iop = mm_gc->regs; + struct cpm_ioport16 __iomem *iop = cpm1_gc->regs; cpm1_gc->cpdata = in_be16(&iop->dat); } static int cpm1_gpio16_get(struct gpio_chip *gc, unsigned int gpio) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm_ioport16 __iomem *iop = mm_gc->regs; + struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport16 __iomem *iop = cpm1_gc->regs; u16 pin_mask; pin_mask = 1 << (15 - gpio); @@ -406,11 +405,9 @@ static int cpm1_gpio16_get(struct gpio_chip *gc, unsigned int gpio) return !!(in_be16(&iop->dat) & pin_mask); } -static void __cpm1_gpio16_set(struct of_mm_gpio_chip *mm_gc, u16 pin_mask, - int value) +static void __cpm1_gpio16_set(struct cpm1_gpio16_chip *cpm1_gc, u16 pin_mask, int value) { - struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); - struct cpm_ioport16 __iomem *iop = mm_gc->regs; + struct cpm_ioport16 __iomem *iop = cpm1_gc->regs; if (value) cpm1_gc->cpdata |= pin_mask; @@ -420,40 +417,39 @@ static void __cpm1_gpio16_set(struct of_mm_gpio_chip *mm_gc, u16 pin_mask, out_be16(&iop->dat, cpm1_gc->cpdata); } -static void cpm1_gpio16_set(struct gpio_chip *gc, unsigned int gpio, int value) +static int cpm1_gpio16_set(struct gpio_chip *gc, unsigned int gpio, int value) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); + struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(gc); unsigned long flags; u16 pin_mask = 1 << (15 - gpio); spin_lock_irqsave(&cpm1_gc->lock, flags); - __cpm1_gpio16_set(mm_gc, pin_mask, value); + __cpm1_gpio16_set(cpm1_gc, pin_mask, value); spin_unlock_irqrestore(&cpm1_gc->lock, flags); + + return 0; } static int cpm1_gpio16_to_irq(struct gpio_chip *gc, unsigned int gpio) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); + struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(gc); return cpm1_gc->irq[gpio] ? : -ENXIO; } static int cpm1_gpio16_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); - struct cpm_ioport16 __iomem *iop = mm_gc->regs; + struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport16 __iomem *iop = cpm1_gc->regs; unsigned long flags; u16 pin_mask = 1 << (15 - gpio); spin_lock_irqsave(&cpm1_gc->lock, flags); setbits16(&iop->dir, pin_mask); - __cpm1_gpio16_set(mm_gc, pin_mask, val); + __cpm1_gpio16_set(cpm1_gc, pin_mask, val); spin_unlock_irqrestore(&cpm1_gc->lock, flags); @@ -462,9 +458,8 @@ static int cpm1_gpio16_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) static int cpm1_gpio16_dir_in(struct gpio_chip *gc, unsigned int gpio) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); - struct cpm_ioport16 __iomem *iop = mm_gc->regs; + struct cpm1_gpio16_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport16 __iomem *iop = cpm1_gc->regs; unsigned long flags; u16 pin_mask = 1 << (15 - gpio); @@ -481,11 +476,10 @@ int cpm1_gpiochip_add16(struct device *dev) { struct device_node *np = dev->of_node; struct cpm1_gpio16_chip *cpm1_gc; - struct of_mm_gpio_chip *mm_gc; struct gpio_chip *gc; u16 mask; - cpm1_gc = kzalloc(sizeof(*cpm1_gc), GFP_KERNEL); + cpm1_gc = devm_kzalloc(dev, sizeof(*cpm1_gc), GFP_KERNEL); if (!cpm1_gc) return -ENOMEM; @@ -499,10 +493,8 @@ int cpm1_gpiochip_add16(struct device *dev) cpm1_gc->irq[i] = irq_of_parse_and_map(np, j++); } - mm_gc = &cpm1_gc->mm_gc; - gc = &mm_gc->gc; - - mm_gc->save_regs = cpm1_gpio16_save_regs; + gc = &cpm1_gc->gc; + gc->base = -1; gc->ngpio = 16; gc->direction_input = cpm1_gpio16_dir_in; gc->direction_output = cpm1_gpio16_dir_out; @@ -512,30 +504,39 @@ int cpm1_gpiochip_add16(struct device *dev) gc->parent = dev; gc->owner = THIS_MODULE; - return of_mm_gpiochip_add_data(np, mm_gc, cpm1_gc); + gc->label = devm_kasprintf(dev, GFP_KERNEL, "%pOF", np); + if (!gc->label) + return -ENOMEM; + + cpm1_gc->regs = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(cpm1_gc->regs)) + return PTR_ERR(cpm1_gc->regs); + + cpm1_gpio16_save_regs(cpm1_gc); + + return devm_gpiochip_add_data(dev, gc, cpm1_gc); } struct cpm1_gpio32_chip { - struct of_mm_gpio_chip mm_gc; + struct gpio_chip gc; + void __iomem *regs; spinlock_t lock; /* shadowed data register to clear/set bits safely */ u32 cpdata; }; -static void cpm1_gpio32_save_regs(struct of_mm_gpio_chip *mm_gc) +static void cpm1_gpio32_save_regs(struct cpm1_gpio32_chip *cpm1_gc) { - struct cpm1_gpio32_chip *cpm1_gc = - container_of(mm_gc, struct cpm1_gpio32_chip, mm_gc); - struct cpm_ioport32b __iomem *iop = mm_gc->regs; + struct cpm_ioport32b __iomem *iop = cpm1_gc->regs; cpm1_gc->cpdata = in_be32(&iop->dat); } static int cpm1_gpio32_get(struct gpio_chip *gc, unsigned int gpio) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm_ioport32b __iomem *iop = mm_gc->regs; + struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport32b __iomem *iop = cpm1_gc->regs; u32 pin_mask; pin_mask = 1 << (31 - gpio); @@ -543,11 +544,9 @@ static int cpm1_gpio32_get(struct gpio_chip *gc, unsigned int gpio) return !!(in_be32(&iop->dat) & pin_mask); } -static void __cpm1_gpio32_set(struct of_mm_gpio_chip *mm_gc, u32 pin_mask, - int value) +static void __cpm1_gpio32_set(struct cpm1_gpio32_chip *cpm1_gc, u32 pin_mask, int value) { - struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); - struct cpm_ioport32b __iomem *iop = mm_gc->regs; + struct cpm_ioport32b __iomem *iop = cpm1_gc->regs; if (value) cpm1_gc->cpdata |= pin_mask; @@ -557,32 +556,32 @@ static void __cpm1_gpio32_set(struct of_mm_gpio_chip *mm_gc, u32 pin_mask, out_be32(&iop->dat, cpm1_gc->cpdata); } -static void cpm1_gpio32_set(struct gpio_chip *gc, unsigned int gpio, int value) +static int cpm1_gpio32_set(struct gpio_chip *gc, unsigned int gpio, int value) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); + struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(gc); unsigned long flags; u32 pin_mask = 1 << (31 - gpio); spin_lock_irqsave(&cpm1_gc->lock, flags); - __cpm1_gpio32_set(mm_gc, pin_mask, value); + __cpm1_gpio32_set(cpm1_gc, pin_mask, value); spin_unlock_irqrestore(&cpm1_gc->lock, flags); + + return 0; } static int cpm1_gpio32_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); - struct cpm_ioport32b __iomem *iop = mm_gc->regs; + struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport32b __iomem *iop = cpm1_gc->regs; unsigned long flags; u32 pin_mask = 1 << (31 - gpio); spin_lock_irqsave(&cpm1_gc->lock, flags); setbits32(&iop->dir, pin_mask); - __cpm1_gpio32_set(mm_gc, pin_mask, val); + __cpm1_gpio32_set(cpm1_gc, pin_mask, val); spin_unlock_irqrestore(&cpm1_gc->lock, flags); @@ -591,9 +590,8 @@ static int cpm1_gpio32_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) static int cpm1_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio) { - struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); - struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(&mm_gc->gc); - struct cpm_ioport32b __iomem *iop = mm_gc->regs; + struct cpm1_gpio32_chip *cpm1_gc = gpiochip_get_data(gc); + struct cpm_ioport32b __iomem *iop = cpm1_gc->regs; unsigned long flags; u32 pin_mask = 1 << (31 - gpio); @@ -610,19 +608,16 @@ int cpm1_gpiochip_add32(struct device *dev) { struct device_node *np = dev->of_node; struct cpm1_gpio32_chip *cpm1_gc; - struct of_mm_gpio_chip *mm_gc; struct gpio_chip *gc; - cpm1_gc = kzalloc(sizeof(*cpm1_gc), GFP_KERNEL); + cpm1_gc = devm_kzalloc(dev, sizeof(*cpm1_gc), GFP_KERNEL); if (!cpm1_gc) return -ENOMEM; spin_lock_init(&cpm1_gc->lock); - mm_gc = &cpm1_gc->mm_gc; - gc = &mm_gc->gc; - - mm_gc->save_regs = cpm1_gpio32_save_regs; + gc = &cpm1_gc->gc; + gc->base = -1; gc->ngpio = 32; gc->direction_input = cpm1_gpio32_dir_in; gc->direction_output = cpm1_gpio32_dir_out; @@ -631,7 +626,17 @@ int cpm1_gpiochip_add32(struct device *dev) gc->parent = dev; gc->owner = THIS_MODULE; - return of_mm_gpiochip_add_data(np, mm_gc, cpm1_gc); + gc->label = devm_kasprintf(dev, GFP_KERNEL, "%pOF", np); + if (!gc->label) + return -ENOMEM; + + cpm1_gc->regs = devm_of_iomap(dev, np, 0, NULL); + if (IS_ERR(cpm1_gc->regs)) + return PTR_ERR(cpm1_gc->regs); + + cpm1_gpio32_save_regs(cpm1_gc); + + return devm_gpiochip_add_data(dev, gc, cpm1_gc); } #endif /* CONFIG_8xx_GPIO */ diff --git a/arch/powerpc/platforms/8xx/pic.c b/arch/powerpc/platforms/8xx/pic.c index ea6b0e523c60..933d6ab7f512 100644 --- a/arch/powerpc/platforms/8xx/pic.c +++ b/arch/powerpc/platforms/8xx/pic.c @@ -80,7 +80,7 @@ unsigned int mpc8xx_get_irq(void) if (irq == PIC_VEC_SPURRIOUS) return 0; - return irq_linear_revmap(mpc8xx_pic_host, irq); + return irq_find_mapping(mpc8xx_pic_host, irq); } @@ -146,7 +146,8 @@ void __init mpc8xx_pic_init(void) if (!siu_reg) goto out; - mpc8xx_pic_host = irq_domain_add_linear(np, 64, &mpc8xx_pic_host_ops, NULL); + mpc8xx_pic_host = irq_domain_create_linear(of_fwnode_handle(np), 64, + &mpc8xx_pic_host_ops, NULL); if (!mpc8xx_pic_host) printk(KERN_ERR "MPC8xx PIC: failed to allocate irq host!\n"); diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig index a454149ae02f..fea3766eac0f 100644 --- a/arch/powerpc/platforms/Kconfig +++ b/arch/powerpc/platforms/Kconfig @@ -70,10 +70,6 @@ config PPC_DT_CPU_FTRS firmware provides this binding. If you're not sure say Y. -config UDBG_RTAS_CONSOLE - bool "RTAS based debug console" - depends on PPC_RTAS - config PPC_SMP_MUXED_IPI bool help @@ -186,12 +182,6 @@ config PPC_INDIRECT_PIO bool select GENERIC_IOMAP -config PPC_INDIRECT_MMIO - bool - -config PPC_IO_WORKAROUNDS - bool - source "drivers/cpufreq/Kconfig" menu "CPUIdle driver" diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index 1453ccc900c4..613b383ed8b3 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -449,6 +449,19 @@ config PPC_RADIX_MMU_DEFAULT If you're unsure, say Y. +config PPC_RADIX_BROADCAST_TLBIE + bool + depends on PPC_RADIX_MMU + help + Power ISA v3.0 and later implementations in the Linux Compliancy Subset + and lower are not required to implement broadcast TLBIE instructions. + Platforms with CPUs that do implement TLBIE broadcast, that is, where + a TLB invalidation instruction performed on one CPU operates on the + TLBs of all CPUs in the system, should select this option. If this + option is selected, the disable_tlbie kernel command line option can + be used to cause global TLB invalidations to be done via IPIs; without + it, IPIs will be used unconditionally. + config PPC_KERNEL_PREFIXED depends on PPC_HAVE_PREFIXED_SUPPORT depends on CC_HAS_PREFIXED diff --git a/arch/powerpc/platforms/amigaone/setup.c b/arch/powerpc/platforms/amigaone/setup.c index 2c8dc0886912..33f852a7625f 100644 --- a/arch/powerpc/platforms/amigaone/setup.c +++ b/arch/powerpc/platforms/amigaone/setup.c @@ -109,7 +109,7 @@ static void __init amigaone_init_IRQ(void) i8259_init(pic, int_ack); ppc_md.get_irq = i8259_irq; - irq_set_default_host(i8259_get_host()); + irq_set_default_domain(i8259_get_host()); } static int __init request_isa_regions(void) diff --git a/arch/powerpc/platforms/book3s/vas-api.c b/arch/powerpc/platforms/book3s/vas-api.c index 0b6365d85d11..49b15e7a8265 100644 --- a/arch/powerpc/platforms/book3s/vas-api.c +++ b/arch/powerpc/platforms/book3s/vas-api.c @@ -425,23 +425,22 @@ static vm_fault_t vas_mmap_fault(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } - mutex_lock(&txwin->task_ref.mmap_mutex); /* * The window may be inactive due to lost credit (Ex: core * removal with DLPAR). If the window is active again when * the credit is available, map the new paste address at the * window virtual address. */ - if (txwin->status == VAS_WIN_ACTIVE) { - paste_addr = cp_inst->coproc->vops->paste_addr(txwin); - if (paste_addr) { - fault = vmf_insert_pfn(vma, vma->vm_start, - (paste_addr >> PAGE_SHIFT)); - mutex_unlock(&txwin->task_ref.mmap_mutex); - return fault; + scoped_guard(mutex, &txwin->task_ref.mmap_mutex) { + if (txwin->status == VAS_WIN_ACTIVE) { + paste_addr = cp_inst->coproc->vops->paste_addr(txwin); + if (paste_addr) { + fault = vmf_insert_pfn(vma, vma->vm_start, + (paste_addr >> PAGE_SHIFT)); + return fault; + } } } - mutex_unlock(&txwin->task_ref.mmap_mutex); /* * Received this fault due to closing the actual window. @@ -494,9 +493,8 @@ static void vas_mmap_close(struct vm_area_struct *vma) return; } - mutex_lock(&txwin->task_ref.mmap_mutex); - txwin->task_ref.vma = NULL; - mutex_unlock(&txwin->task_ref.mmap_mutex); + scoped_guard(mutex, &txwin->task_ref.mmap_mutex) + txwin->task_ref.vma = NULL; } static const struct vm_operations_struct vas_vm_ops = { @@ -521,6 +519,15 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) return -EINVAL; } + /* + * Map complete page to the paste address. So the user + * space should pass 0ULL to the offset parameter. + */ + if (vma->vm_pgoff) { + pr_debug("Page offset unsupported to map paste address\n"); + return -EINVAL; + } + /* Ensure instance has an open send window */ if (!txwin) { pr_err("No send window open?\n"); @@ -543,18 +550,16 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) * close/open event and allows mmap() only when the window is * active. */ - mutex_lock(&txwin->task_ref.mmap_mutex); + guard(mutex)(&txwin->task_ref.mmap_mutex); if (txwin->status != VAS_WIN_ACTIVE) { pr_err("Window is not active\n"); - rc = -EACCES; - goto out; + return -EACCES; } paste_addr = cp_inst->coproc->vops->paste_addr(txwin); if (!paste_addr) { pr_err("Window paste address failed\n"); - rc = -EINVAL; - goto out; + return -EINVAL; } pfn = paste_addr >> PAGE_SHIFT; @@ -574,8 +579,6 @@ static int coproc_mmap(struct file *fp, struct vm_area_struct *vma) txwin->task_ref.vma = vma; vma->vm_ops = &vas_vm_ops; -out: - mutex_unlock(&txwin->task_ref.mmap_mutex); return rc; } diff --git a/arch/powerpc/platforms/cell/Kconfig b/arch/powerpc/platforms/cell/Kconfig index 34669b060f36..db65bfcd1e74 100644 --- a/arch/powerpc/platforms/cell/Kconfig +++ b/arch/powerpc/platforms/cell/Kconfig @@ -3,42 +3,6 @@ config PPC_CELL select PPC_64S_HASH_MMU if PPC64 bool -config PPC_CELL_COMMON - bool - select PPC_CELL - select PPC_DCR_MMIO - select PPC_INDIRECT_PIO - select PPC_INDIRECT_MMIO - select PPC_HASH_MMU_NATIVE - select PPC_RTAS - select IRQ_EDGE_EOI_HANDLER - -config PPC_CELL_NATIVE - bool - select PPC_CELL_COMMON - select MPIC - select PPC_IO_WORKAROUNDS - select IBM_EMAC_EMAC4 if IBM_EMAC - select IBM_EMAC_RGMII if IBM_EMAC - select IBM_EMAC_ZMII if IBM_EMAC #test only - select IBM_EMAC_TAH if IBM_EMAC #test only - -config PPC_IBM_CELL_BLADE - bool "IBM Cell Blade" - depends on PPC64 && PPC_BOOK3S && CPU_BIG_ENDIAN - select PPC_CELL_NATIVE - select PPC_OF_PLATFORM_PCI - select FORCE_PCI - select MMIO_NVRAM - select PPC_UDBG_16550 - select UDBG_RTAS_CONSOLE - -config AXON_MSI - bool - depends on PPC_IBM_CELL_BLADE && PCI_MSI - select IRQ_DOMAIN_NOMAP - default y - menu "Cell Broadband Engine options" depends on PPC_CELL @@ -57,48 +21,4 @@ config SPU_BASE bool select PPC_COPRO_BASE -config CBE_RAS - bool "RAS features for bare metal Cell BE" - depends on PPC_CELL_NATIVE - default y - -config PPC_IBM_CELL_RESETBUTTON - bool "IBM Cell Blade Pinhole reset button" - depends on CBE_RAS && PPC_IBM_CELL_BLADE - default y - help - Support Pinhole Resetbutton on IBM Cell blades. - This adds a method to trigger system reset via front panel pinhole button. - -config PPC_IBM_CELL_POWERBUTTON - tristate "IBM Cell Blade power button" - depends on PPC_IBM_CELL_BLADE && INPUT_EVDEV - default y - help - Support Powerbutton on IBM Cell blades. - This will enable the powerbutton as an input device. - -config CBE_THERM - tristate "CBE thermal support" - default m - depends on CBE_RAS && SPU_BASE - -config PPC_PMI - tristate - default y - depends on CPU_FREQ_CBE_PMI || PPC_IBM_CELL_POWERBUTTON - help - PMI (Platform Management Interrupt) is a way to - communicate with the BMC (Baseboard Management Controller). - It is used in some IBM Cell blades. - -config CBE_CPUFREQ_SPU_GOVERNOR - tristate "CBE frequency scaling based on SPU usage" - depends on SPU_FS && CPU_FREQ - default m - help - This governor checks for spu usage to adjust the cpu frequency. - If no spu is running on a given cpu, that cpu will be throttled to - the minimal possible frequency. - endmenu diff --git a/arch/powerpc/platforms/cell/Makefile b/arch/powerpc/platforms/cell/Makefile index 7ea6692f67e2..7e5ff239c376 100644 --- a/arch/powerpc/platforms/cell/Makefile +++ b/arch/powerpc/platforms/cell/Makefile @@ -1,27 +1,4 @@ # SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_PPC_CELL_COMMON) += cbe_regs.o interrupt.o pervasive.o - -obj-$(CONFIG_PPC_CELL_NATIVE) += iommu.o setup.o spider-pic.o \ - pmu.o spider-pci.o -obj-$(CONFIG_CBE_RAS) += ras.o - -obj-$(CONFIG_CBE_THERM) += cbe_thermal.o -obj-$(CONFIG_CBE_CPUFREQ_SPU_GOVERNOR) += cpufreq_spudemand.o - -obj-$(CONFIG_PPC_IBM_CELL_POWERBUTTON) += cbe_powerbutton.o - -ifdef CONFIG_SMP -obj-$(CONFIG_PPC_CELL_NATIVE) += smp.o -endif - -# needed only when building loadable spufs.ko -spu-priv1-$(CONFIG_PPC_CELL_COMMON) += spu_priv1_mmio.o -spu-manage-$(CONFIG_PPC_CELL_COMMON) += spu_manage.o - obj-$(CONFIG_SPU_BASE) += spu_callbacks.o spu_base.o \ spu_syscalls.o \ - $(spu-priv1-y) \ - $(spu-manage-y) \ spufs/ - -obj-$(CONFIG_AXON_MSI) += axon_msi.o diff --git a/arch/powerpc/platforms/cell/axon_msi.c b/arch/powerpc/platforms/cell/axon_msi.c deleted file mode 100644 index d243f7fd8982..000000000000 --- a/arch/powerpc/platforms/cell/axon_msi.c +++ /dev/null @@ -1,481 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright 2007, Michael Ellerman, IBM Corporation. - */ - - -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/kernel.h> -#include <linux/pci.h> -#include <linux/msi.h> -#include <linux/export.h> -#include <linux/slab.h> -#include <linux/debugfs.h> -#include <linux/of.h> -#include <linux/of_irq.h> -#include <linux/platform_device.h> - -#include <asm/dcr.h> -#include <asm/machdep.h> - -#include "cell.h" - -/* - * MSIC registers, specified as offsets from dcr_base - */ -#define MSIC_CTRL_REG 0x0 - -/* Base Address registers specify FIFO location in BE memory */ -#define MSIC_BASE_ADDR_HI_REG 0x3 -#define MSIC_BASE_ADDR_LO_REG 0x4 - -/* Hold the read/write offsets into the FIFO */ -#define MSIC_READ_OFFSET_REG 0x5 -#define MSIC_WRITE_OFFSET_REG 0x6 - - -/* MSIC control register flags */ -#define MSIC_CTRL_ENABLE 0x0001 -#define MSIC_CTRL_FIFO_FULL_ENABLE 0x0002 -#define MSIC_CTRL_IRQ_ENABLE 0x0008 -#define MSIC_CTRL_FULL_STOP_ENABLE 0x0010 - -/* - * The MSIC can be configured to use a FIFO of 32KB, 64KB, 128KB or 256KB. - * Currently we're using a 64KB FIFO size. - */ -#define MSIC_FIFO_SIZE_SHIFT 16 -#define MSIC_FIFO_SIZE_BYTES (1 << MSIC_FIFO_SIZE_SHIFT) - -/* - * To configure the FIFO size as (1 << n) bytes, we write (n - 15) into bits - * 8-9 of the MSIC control reg. - */ -#define MSIC_CTRL_FIFO_SIZE (((MSIC_FIFO_SIZE_SHIFT - 15) << 8) & 0x300) - -/* - * We need to mask the read/write offsets to make sure they stay within - * the bounds of the FIFO. Also they should always be 16-byte aligned. - */ -#define MSIC_FIFO_SIZE_MASK ((MSIC_FIFO_SIZE_BYTES - 1) & ~0xFu) - -/* Each entry in the FIFO is 16 bytes, the first 4 bytes hold the irq # */ -#define MSIC_FIFO_ENTRY_SIZE 0x10 - - -struct axon_msic { - struct irq_domain *irq_domain; - __le32 *fifo_virt; - dma_addr_t fifo_phys; - dcr_host_t dcr_host; - u32 read_offset; -#ifdef DEBUG - u32 __iomem *trigger; -#endif -}; - -#ifdef DEBUG -void axon_msi_debug_setup(struct device_node *dn, struct axon_msic *msic); -#else -static inline void axon_msi_debug_setup(struct device_node *dn, - struct axon_msic *msic) { } -#endif - - -static void msic_dcr_write(struct axon_msic *msic, unsigned int dcr_n, u32 val) -{ - pr_devel("axon_msi: dcr_write(0x%x, 0x%x)\n", val, dcr_n); - - dcr_write(msic->dcr_host, dcr_n, val); -} - -static void axon_msi_cascade(struct irq_desc *desc) -{ - struct irq_chip *chip = irq_desc_get_chip(desc); - struct axon_msic *msic = irq_desc_get_handler_data(desc); - u32 write_offset, msi; - int idx; - int retry = 0; - - write_offset = dcr_read(msic->dcr_host, MSIC_WRITE_OFFSET_REG); - pr_devel("axon_msi: original write_offset 0x%x\n", write_offset); - - /* write_offset doesn't wrap properly, so we have to mask it */ - write_offset &= MSIC_FIFO_SIZE_MASK; - - while (msic->read_offset != write_offset && retry < 100) { - idx = msic->read_offset / sizeof(__le32); - msi = le32_to_cpu(msic->fifo_virt[idx]); - msi &= 0xFFFF; - - pr_devel("axon_msi: woff %x roff %x msi %x\n", - write_offset, msic->read_offset, msi); - - if (msi < irq_get_nr_irqs() && irq_get_chip_data(msi) == msic) { - generic_handle_irq(msi); - msic->fifo_virt[idx] = cpu_to_le32(0xffffffff); - } else { - /* - * Reading the MSIC_WRITE_OFFSET_REG does not - * reliably flush the outstanding DMA to the - * FIFO buffer. Here we were reading stale - * data, so we need to retry. - */ - udelay(1); - retry++; - pr_devel("axon_msi: invalid irq 0x%x!\n", msi); - continue; - } - - if (retry) { - pr_devel("axon_msi: late irq 0x%x, retry %d\n", - msi, retry); - retry = 0; - } - - msic->read_offset += MSIC_FIFO_ENTRY_SIZE; - msic->read_offset &= MSIC_FIFO_SIZE_MASK; - } - - if (retry) { - printk(KERN_WARNING "axon_msi: irq timed out\n"); - - msic->read_offset += MSIC_FIFO_ENTRY_SIZE; - msic->read_offset &= MSIC_FIFO_SIZE_MASK; - } - - chip->irq_eoi(&desc->irq_data); -} - -static struct axon_msic *find_msi_translator(struct pci_dev *dev) -{ - struct irq_domain *irq_domain; - struct device_node *dn, *tmp; - const phandle *ph; - struct axon_msic *msic = NULL; - - dn = of_node_get(pci_device_to_OF_node(dev)); - if (!dn) { - dev_dbg(&dev->dev, "axon_msi: no pci_dn found\n"); - return NULL; - } - - for (; dn; dn = of_get_next_parent(dn)) { - ph = of_get_property(dn, "msi-translator", NULL); - if (ph) - break; - } - - if (!ph) { - dev_dbg(&dev->dev, - "axon_msi: no msi-translator property found\n"); - goto out_error; - } - - tmp = dn; - dn = of_find_node_by_phandle(*ph); - of_node_put(tmp); - if (!dn) { - dev_dbg(&dev->dev, - "axon_msi: msi-translator doesn't point to a node\n"); - goto out_error; - } - - irq_domain = irq_find_host(dn); - if (!irq_domain) { - dev_dbg(&dev->dev, "axon_msi: no irq_domain found for node %pOF\n", - dn); - goto out_error; - } - - msic = irq_domain->host_data; - -out_error: - of_node_put(dn); - - return msic; -} - -static int setup_msi_msg_address(struct pci_dev *dev, struct msi_msg *msg) -{ - struct device_node *dn; - int len; - const u32 *prop; - - dn = of_node_get(pci_device_to_OF_node(dev)); - if (!dn) { - dev_dbg(&dev->dev, "axon_msi: no pci_dn found\n"); - return -ENODEV; - } - - for (; dn; dn = of_get_next_parent(dn)) { - if (!dev->no_64bit_msi) { - prop = of_get_property(dn, "msi-address-64", &len); - if (prop) - break; - } - - prop = of_get_property(dn, "msi-address-32", &len); - if (prop) - break; - } - - if (!prop) { - dev_dbg(&dev->dev, - "axon_msi: no msi-address-(32|64) properties found\n"); - of_node_put(dn); - return -ENOENT; - } - - switch (len) { - case 8: - msg->address_hi = prop[0]; - msg->address_lo = prop[1]; - break; - case 4: - msg->address_hi = 0; - msg->address_lo = prop[0]; - break; - default: - dev_dbg(&dev->dev, - "axon_msi: malformed msi-address-(32|64) property\n"); - of_node_put(dn); - return -EINVAL; - } - - of_node_put(dn); - - return 0; -} - -static int axon_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) -{ - unsigned int virq, rc; - struct msi_desc *entry; - struct msi_msg msg; - struct axon_msic *msic; - - msic = find_msi_translator(dev); - if (!msic) - return -ENODEV; - - rc = setup_msi_msg_address(dev, &msg); - if (rc) - return rc; - - msi_for_each_desc(entry, &dev->dev, MSI_DESC_NOTASSOCIATED) { - virq = irq_create_direct_mapping(msic->irq_domain); - if (!virq) { - dev_warn(&dev->dev, - "axon_msi: virq allocation failed!\n"); - return -1; - } - dev_dbg(&dev->dev, "axon_msi: allocated virq 0x%x\n", virq); - - irq_set_msi_desc(virq, entry); - msg.data = virq; - pci_write_msi_msg(virq, &msg); - } - - return 0; -} - -static void axon_msi_teardown_msi_irqs(struct pci_dev *dev) -{ - struct msi_desc *entry; - - dev_dbg(&dev->dev, "axon_msi: tearing down msi irqs\n"); - - msi_for_each_desc(entry, &dev->dev, MSI_DESC_ASSOCIATED) { - irq_set_msi_desc(entry->irq, NULL); - irq_dispose_mapping(entry->irq); - entry->irq = 0; - } -} - -static struct irq_chip msic_irq_chip = { - .irq_mask = pci_msi_mask_irq, - .irq_unmask = pci_msi_unmask_irq, - .irq_shutdown = pci_msi_mask_irq, - .name = "AXON-MSI", -}; - -static int msic_host_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - irq_set_chip_data(virq, h->host_data); - irq_set_chip_and_handler(virq, &msic_irq_chip, handle_simple_irq); - - return 0; -} - -static const struct irq_domain_ops msic_host_ops = { - .map = msic_host_map, -}; - -static void axon_msi_shutdown(struct platform_device *device) -{ - struct axon_msic *msic = dev_get_drvdata(&device->dev); - u32 tmp; - - pr_devel("axon_msi: disabling %pOF\n", - irq_domain_get_of_node(msic->irq_domain)); - tmp = dcr_read(msic->dcr_host, MSIC_CTRL_REG); - tmp &= ~MSIC_CTRL_ENABLE & ~MSIC_CTRL_IRQ_ENABLE; - msic_dcr_write(msic, MSIC_CTRL_REG, tmp); -} - -static int axon_msi_probe(struct platform_device *device) -{ - struct device_node *dn = device->dev.of_node; - struct axon_msic *msic; - unsigned int virq; - int dcr_base, dcr_len; - - pr_devel("axon_msi: setting up dn %pOF\n", dn); - - msic = kzalloc(sizeof(*msic), GFP_KERNEL); - if (!msic) { - printk(KERN_ERR "axon_msi: couldn't allocate msic for %pOF\n", - dn); - goto out; - } - - dcr_base = dcr_resource_start(dn, 0); - dcr_len = dcr_resource_len(dn, 0); - - if (dcr_base == 0 || dcr_len == 0) { - printk(KERN_ERR - "axon_msi: couldn't parse dcr properties on %pOF\n", - dn); - goto out_free_msic; - } - - msic->dcr_host = dcr_map(dn, dcr_base, dcr_len); - if (!DCR_MAP_OK(msic->dcr_host)) { - printk(KERN_ERR "axon_msi: dcr_map failed for %pOF\n", - dn); - goto out_free_msic; - } - - msic->fifo_virt = dma_alloc_coherent(&device->dev, MSIC_FIFO_SIZE_BYTES, - &msic->fifo_phys, GFP_KERNEL); - if (!msic->fifo_virt) { - printk(KERN_ERR "axon_msi: couldn't allocate fifo for %pOF\n", - dn); - goto out_free_msic; - } - - virq = irq_of_parse_and_map(dn, 0); - if (!virq) { - printk(KERN_ERR "axon_msi: irq parse and map failed for %pOF\n", - dn); - goto out_free_fifo; - } - memset(msic->fifo_virt, 0xff, MSIC_FIFO_SIZE_BYTES); - - /* We rely on being able to stash a virq in a u16, so limit irqs to < 65536 */ - msic->irq_domain = irq_domain_add_nomap(dn, 65536, &msic_host_ops, msic); - if (!msic->irq_domain) { - printk(KERN_ERR "axon_msi: couldn't allocate irq_domain for %pOF\n", - dn); - goto out_free_fifo; - } - - irq_set_handler_data(virq, msic); - irq_set_chained_handler(virq, axon_msi_cascade); - pr_devel("axon_msi: irq 0x%x setup for axon_msi\n", virq); - - /* Enable the MSIC hardware */ - msic_dcr_write(msic, MSIC_BASE_ADDR_HI_REG, msic->fifo_phys >> 32); - msic_dcr_write(msic, MSIC_BASE_ADDR_LO_REG, - msic->fifo_phys & 0xFFFFFFFF); - msic_dcr_write(msic, MSIC_CTRL_REG, - MSIC_CTRL_IRQ_ENABLE | MSIC_CTRL_ENABLE | - MSIC_CTRL_FIFO_SIZE); - - msic->read_offset = dcr_read(msic->dcr_host, MSIC_WRITE_OFFSET_REG) - & MSIC_FIFO_SIZE_MASK; - - dev_set_drvdata(&device->dev, msic); - - cell_pci_controller_ops.setup_msi_irqs = axon_msi_setup_msi_irqs; - cell_pci_controller_ops.teardown_msi_irqs = axon_msi_teardown_msi_irqs; - - axon_msi_debug_setup(dn, msic); - - printk(KERN_DEBUG "axon_msi: setup MSIC on %pOF\n", dn); - - return 0; - -out_free_fifo: - dma_free_coherent(&device->dev, MSIC_FIFO_SIZE_BYTES, msic->fifo_virt, - msic->fifo_phys); -out_free_msic: - kfree(msic); -out: - - return -1; -} - -static const struct of_device_id axon_msi_device_id[] = { - { - .compatible = "ibm,axon-msic" - }, - {} -}; - -static struct platform_driver axon_msi_driver = { - .probe = axon_msi_probe, - .shutdown = axon_msi_shutdown, - .driver = { - .name = "axon-msi", - .of_match_table = axon_msi_device_id, - }, -}; - -static int __init axon_msi_init(void) -{ - return platform_driver_register(&axon_msi_driver); -} -subsys_initcall(axon_msi_init); - - -#ifdef DEBUG -static int msic_set(void *data, u64 val) -{ - struct axon_msic *msic = data; - out_le32(msic->trigger, val); - return 0; -} - -static int msic_get(void *data, u64 *val) -{ - *val = 0; - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(fops_msic, msic_get, msic_set, "%llu\n"); - -void axon_msi_debug_setup(struct device_node *dn, struct axon_msic *msic) -{ - char name[8]; - struct resource res; - - if (of_address_to_resource(dn, 0, &res)) { - pr_devel("axon_msi: couldn't get reg property\n"); - return; - } - - msic->trigger = ioremap(res.start, 0x4); - if (!msic->trigger) { - pr_devel("axon_msi: ioremap failed\n"); - return; - } - - snprintf(name, sizeof(name), "msic_%d", of_node_to_nid(dn)); - - debugfs_create_file(name, 0600, arch_debugfs_dir, msic, &fops_msic); -} -#endif /* DEBUG */ diff --git a/arch/powerpc/platforms/cell/cbe_powerbutton.c b/arch/powerpc/platforms/cell/cbe_powerbutton.c deleted file mode 100644 index 3d121acdf69b..000000000000 --- a/arch/powerpc/platforms/cell/cbe_powerbutton.c +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * driver for powerbutton on IBM cell blades - * - * (C) Copyright IBM Corp. 2005-2008 - * - * Author: Christian Krafft <krafft@de.ibm.com> - */ - -#include <linux/input.h> -#include <linux/module.h> -#include <linux/of.h> -#include <linux/platform_device.h> -#include <asm/pmi.h> - -static struct input_dev *button_dev; -static struct platform_device *button_pdev; - -static void cbe_powerbutton_handle_pmi(pmi_message_t pmi_msg) -{ - BUG_ON(pmi_msg.type != PMI_TYPE_POWER_BUTTON); - - input_report_key(button_dev, KEY_POWER, 1); - input_sync(button_dev); - input_report_key(button_dev, KEY_POWER, 0); - input_sync(button_dev); -} - -static struct pmi_handler cbe_pmi_handler = { - .type = PMI_TYPE_POWER_BUTTON, - .handle_pmi_message = cbe_powerbutton_handle_pmi, -}; - -static int __init cbe_powerbutton_init(void) -{ - int ret = 0; - struct input_dev *dev; - - if (!of_machine_is_compatible("IBM,CBPLUS-1.0")) { - printk(KERN_ERR "%s: Not a cell blade.\n", __func__); - ret = -ENODEV; - goto out; - } - - dev = input_allocate_device(); - if (!dev) { - ret = -ENOMEM; - printk(KERN_ERR "%s: Not enough memory.\n", __func__); - goto out; - } - - set_bit(EV_KEY, dev->evbit); - set_bit(KEY_POWER, dev->keybit); - - dev->name = "Power Button"; - dev->id.bustype = BUS_HOST; - - /* this makes the button look like an acpi power button - * no clue whether anyone relies on that though */ - dev->id.product = 0x02; - dev->phys = "LNXPWRBN/button/input0"; - - button_pdev = platform_device_register_simple("power_button", 0, NULL, 0); - if (IS_ERR(button_pdev)) { - ret = PTR_ERR(button_pdev); - goto out_free_input; - } - - dev->dev.parent = &button_pdev->dev; - ret = input_register_device(dev); - if (ret) { - printk(KERN_ERR "%s: Failed to register device\n", __func__); - goto out_free_pdev; - } - - button_dev = dev; - - ret = pmi_register_handler(&cbe_pmi_handler); - if (ret) { - printk(KERN_ERR "%s: Failed to register with pmi.\n", __func__); - goto out_free_pdev; - } - - goto out; - -out_free_pdev: - platform_device_unregister(button_pdev); -out_free_input: - input_free_device(dev); -out: - return ret; -} - -static void __exit cbe_powerbutton_exit(void) -{ - pmi_unregister_handler(&cbe_pmi_handler); - platform_device_unregister(button_pdev); - input_free_device(button_dev); -} - -module_init(cbe_powerbutton_init); -module_exit(cbe_powerbutton_exit); - -MODULE_DESCRIPTION("Driver for powerbutton on IBM cell blades"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Christian Krafft <krafft@de.ibm.com>"); diff --git a/arch/powerpc/platforms/cell/cbe_regs.c b/arch/powerpc/platforms/cell/cbe_regs.c deleted file mode 100644 index 99b3558753e9..000000000000 --- a/arch/powerpc/platforms/cell/cbe_regs.c +++ /dev/null @@ -1,298 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * cbe_regs.c - * - * Accessor routines for the various MMIO register blocks of the CBE - * - * (c) 2006 Benjamin Herrenschmidt <benh@kernel.crashing.org>, IBM Corp. - */ - -#include <linux/percpu.h> -#include <linux/types.h> -#include <linux/export.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/pgtable.h> - -#include <asm/io.h> -#include <asm/ptrace.h> -#include <asm/cell-regs.h> - -/* - * Current implementation uses "cpu" nodes. We build our own mapping - * array of cpu numbers to cpu nodes locally for now to allow interrupt - * time code to have a fast path rather than call of_get_cpu_node(). If - * we implement cpu hotplug, we'll have to install an appropriate notifier - * in order to release references to the cpu going away - */ -static struct cbe_regs_map -{ - struct device_node *cpu_node; - struct device_node *be_node; - struct cbe_pmd_regs __iomem *pmd_regs; - struct cbe_iic_regs __iomem *iic_regs; - struct cbe_mic_tm_regs __iomem *mic_tm_regs; - struct cbe_pmd_shadow_regs pmd_shadow_regs; -} cbe_regs_maps[MAX_CBE]; -static int cbe_regs_map_count; - -static struct cbe_thread_map -{ - struct device_node *cpu_node; - struct device_node *be_node; - struct cbe_regs_map *regs; - unsigned int thread_id; - unsigned int cbe_id; -} cbe_thread_map[NR_CPUS]; - -static cpumask_t cbe_local_mask[MAX_CBE] = { [0 ... MAX_CBE-1] = {CPU_BITS_NONE} }; -static cpumask_t cbe_first_online_cpu = { CPU_BITS_NONE }; - -static struct cbe_regs_map *cbe_find_map(struct device_node *np) -{ - int i; - struct device_node *tmp_np; - - if (!of_node_is_type(np, "spe")) { - for (i = 0; i < cbe_regs_map_count; i++) - if (cbe_regs_maps[i].cpu_node == np || - cbe_regs_maps[i].be_node == np) - return &cbe_regs_maps[i]; - return NULL; - } - - if (np->data) - return np->data; - - /* walk up path until cpu or be node was found */ - tmp_np = np; - do { - tmp_np = tmp_np->parent; - /* on a correct devicetree we wont get up to root */ - BUG_ON(!tmp_np); - } while (!of_node_is_type(tmp_np, "cpu") || - !of_node_is_type(tmp_np, "be")); - - np->data = cbe_find_map(tmp_np); - - return np->data; -} - -struct cbe_pmd_regs __iomem *cbe_get_pmd_regs(struct device_node *np) -{ - struct cbe_regs_map *map = cbe_find_map(np); - if (map == NULL) - return NULL; - return map->pmd_regs; -} -EXPORT_SYMBOL_GPL(cbe_get_pmd_regs); - -struct cbe_pmd_regs __iomem *cbe_get_cpu_pmd_regs(int cpu) -{ - struct cbe_regs_map *map = cbe_thread_map[cpu].regs; - if (map == NULL) - return NULL; - return map->pmd_regs; -} -EXPORT_SYMBOL_GPL(cbe_get_cpu_pmd_regs); - -struct cbe_pmd_shadow_regs *cbe_get_pmd_shadow_regs(struct device_node *np) -{ - struct cbe_regs_map *map = cbe_find_map(np); - if (map == NULL) - return NULL; - return &map->pmd_shadow_regs; -} - -struct cbe_pmd_shadow_regs *cbe_get_cpu_pmd_shadow_regs(int cpu) -{ - struct cbe_regs_map *map = cbe_thread_map[cpu].regs; - if (map == NULL) - return NULL; - return &map->pmd_shadow_regs; -} - -struct cbe_iic_regs __iomem *cbe_get_iic_regs(struct device_node *np) -{ - struct cbe_regs_map *map = cbe_find_map(np); - if (map == NULL) - return NULL; - return map->iic_regs; -} - -struct cbe_iic_regs __iomem *cbe_get_cpu_iic_regs(int cpu) -{ - struct cbe_regs_map *map = cbe_thread_map[cpu].regs; - if (map == NULL) - return NULL; - return map->iic_regs; -} - -struct cbe_mic_tm_regs __iomem *cbe_get_mic_tm_regs(struct device_node *np) -{ - struct cbe_regs_map *map = cbe_find_map(np); - if (map == NULL) - return NULL; - return map->mic_tm_regs; -} - -struct cbe_mic_tm_regs __iomem *cbe_get_cpu_mic_tm_regs(int cpu) -{ - struct cbe_regs_map *map = cbe_thread_map[cpu].regs; - if (map == NULL) - return NULL; - return map->mic_tm_regs; -} -EXPORT_SYMBOL_GPL(cbe_get_cpu_mic_tm_regs); - -u32 cbe_get_hw_thread_id(int cpu) -{ - return cbe_thread_map[cpu].thread_id; -} -EXPORT_SYMBOL_GPL(cbe_get_hw_thread_id); - -u32 cbe_cpu_to_node(int cpu) -{ - return cbe_thread_map[cpu].cbe_id; -} -EXPORT_SYMBOL_GPL(cbe_cpu_to_node); - -u32 cbe_node_to_cpu(int node) -{ - return cpumask_first(&cbe_local_mask[node]); - -} -EXPORT_SYMBOL_GPL(cbe_node_to_cpu); - -static struct device_node *__init cbe_get_be_node(int cpu_id) -{ - struct device_node *np; - - for_each_node_by_type (np, "be") { - int len,i; - const phandle *cpu_handle; - - cpu_handle = of_get_property(np, "cpus", &len); - - /* - * the CAB SLOF tree is non compliant, so we just assume - * there is only one node - */ - if (WARN_ON_ONCE(!cpu_handle)) - return np; - - for (i = 0; i < len; i++) { - struct device_node *ch_np = of_find_node_by_phandle(cpu_handle[i]); - struct device_node *ci_np = of_get_cpu_node(cpu_id, NULL); - - of_node_put(ch_np); - of_node_put(ci_np); - - if (ch_np == ci_np) - return np; - } - } - - return NULL; -} - -static void __init cbe_fill_regs_map(struct cbe_regs_map *map) -{ - if(map->be_node) { - struct device_node *be, *np, *parent_np; - - be = map->be_node; - - for_each_node_by_type(np, "pervasive") { - parent_np = of_get_parent(np); - if (parent_np == be) - map->pmd_regs = of_iomap(np, 0); - of_node_put(parent_np); - } - - for_each_node_by_type(np, "CBEA-Internal-Interrupt-Controller") { - parent_np = of_get_parent(np); - if (parent_np == be) - map->iic_regs = of_iomap(np, 2); - of_node_put(parent_np); - } - - for_each_node_by_type(np, "mic-tm") { - parent_np = of_get_parent(np); - if (parent_np == be) - map->mic_tm_regs = of_iomap(np, 0); - of_node_put(parent_np); - } - } else { - struct device_node *cpu; - /* That hack must die die die ! */ - const struct address_prop { - unsigned long address; - unsigned int len; - } __attribute__((packed)) *prop; - - cpu = map->cpu_node; - - prop = of_get_property(cpu, "pervasive", NULL); - if (prop != NULL) - map->pmd_regs = ioremap(prop->address, prop->len); - - prop = of_get_property(cpu, "iic", NULL); - if (prop != NULL) - map->iic_regs = ioremap(prop->address, prop->len); - - prop = of_get_property(cpu, "mic-tm", NULL); - if (prop != NULL) - map->mic_tm_regs = ioremap(prop->address, prop->len); - } -} - - -void __init cbe_regs_init(void) -{ - int i; - unsigned int thread_id; - struct device_node *cpu; - - /* Build local fast map of CPUs */ - for_each_possible_cpu(i) { - cbe_thread_map[i].cpu_node = of_get_cpu_node(i, &thread_id); - cbe_thread_map[i].be_node = cbe_get_be_node(i); - cbe_thread_map[i].thread_id = thread_id; - } - - /* Find maps for each device tree CPU */ - for_each_node_by_type(cpu, "cpu") { - struct cbe_regs_map *map; - unsigned int cbe_id; - - cbe_id = cbe_regs_map_count++; - map = &cbe_regs_maps[cbe_id]; - - if (cbe_regs_map_count > MAX_CBE) { - printk(KERN_ERR "cbe_regs: More BE chips than supported" - "!\n"); - cbe_regs_map_count--; - of_node_put(cpu); - return; - } - of_node_put(map->cpu_node); - map->cpu_node = of_node_get(cpu); - - for_each_possible_cpu(i) { - struct cbe_thread_map *thread = &cbe_thread_map[i]; - - if (thread->cpu_node == cpu) { - thread->regs = map; - thread->cbe_id = cbe_id; - map->be_node = thread->be_node; - cpumask_set_cpu(i, &cbe_local_mask[cbe_id]); - if(thread->thread_id == 0) - cpumask_set_cpu(i, &cbe_first_online_cpu); - } - } - - cbe_fill_regs_map(map); - } -} - diff --git a/arch/powerpc/platforms/cell/cbe_thermal.c b/arch/powerpc/platforms/cell/cbe_thermal.c deleted file mode 100644 index c295c6714f9b..000000000000 --- a/arch/powerpc/platforms/cell/cbe_thermal.c +++ /dev/null @@ -1,387 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * thermal support for the cell processor - * - * This module adds some sysfs attributes to cpu and spu nodes. - * Base for measurements are the digital thermal sensors (DTS) - * located on the chip. - * The accuracy is 2 degrees, starting from 65 up to 125 degrees celsius - * The attributes can be found under - * /sys/devices/system/cpu/cpuX/thermal - * /sys/devices/system/spu/spuX/thermal - * - * The following attributes are added for each node: - * temperature: - * contains the current temperature measured by the DTS - * throttle_begin: - * throttling begins when temperature is greater or equal to - * throttle_begin. Setting this value to 125 prevents throttling. - * throttle_end: - * throttling is being ceased, if the temperature is lower than - * throttle_end. Due to a delay between applying throttling and - * a reduced temperature this value should be less than throttle_begin. - * A value equal to throttle_begin provides only a very little hysteresis. - * throttle_full_stop: - * If the temperatrue is greater or equal to throttle_full_stop, - * full throttling is applied to the cpu or spu. This value should be - * greater than throttle_begin and throttle_end. Setting this value to - * 65 prevents the unit from running code at all. - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * - * Author: Christian Krafft <krafft@de.ibm.com> - */ - -#include <linux/module.h> -#include <linux/device.h> -#include <linux/kernel.h> -#include <linux/cpu.h> -#include <linux/stringify.h> -#include <asm/spu.h> -#include <asm/io.h> -#include <asm/cell-regs.h> - -#include "spu_priv1_mmio.h" - -#define TEMP_MIN 65 -#define TEMP_MAX 125 - -#define DEVICE_PREFIX_ATTR(_prefix,_name,_mode) \ -struct device_attribute attr_ ## _prefix ## _ ## _name = { \ - .attr = { .name = __stringify(_name), .mode = _mode }, \ - .show = _prefix ## _show_ ## _name, \ - .store = _prefix ## _store_ ## _name, \ -}; - -static inline u8 reg_to_temp(u8 reg_value) -{ - return ((reg_value & 0x3f) << 1) + TEMP_MIN; -} - -static inline u8 temp_to_reg(u8 temp) -{ - return ((temp - TEMP_MIN) >> 1) & 0x3f; -} - -static struct cbe_pmd_regs __iomem *get_pmd_regs(struct device *dev) -{ - struct spu *spu; - - spu = container_of(dev, struct spu, dev); - - return cbe_get_pmd_regs(spu_devnode(spu)); -} - -/* returns the value for a given spu in a given register */ -static u8 spu_read_register_value(struct device *dev, union spe_reg __iomem *reg) -{ - union spe_reg value; - struct spu *spu; - - spu = container_of(dev, struct spu, dev); - value.val = in_be64(®->val); - - return value.spe[spu->spe_id]; -} - -static ssize_t spu_show_temp(struct device *dev, struct device_attribute *attr, - char *buf) -{ - u8 value; - struct cbe_pmd_regs __iomem *pmd_regs; - - pmd_regs = get_pmd_regs(dev); - - value = spu_read_register_value(dev, &pmd_regs->ts_ctsr1); - - return sprintf(buf, "%d\n", reg_to_temp(value)); -} - -static ssize_t show_throttle(struct cbe_pmd_regs __iomem *pmd_regs, char *buf, int pos) -{ - u64 value; - - value = in_be64(&pmd_regs->tm_tpr.val); - /* access the corresponding byte */ - value >>= pos; - value &= 0x3F; - - return sprintf(buf, "%d\n", reg_to_temp(value)); -} - -static ssize_t store_throttle(struct cbe_pmd_regs __iomem *pmd_regs, const char *buf, size_t size, int pos) -{ - u64 reg_value; - unsigned int temp; - u64 new_value; - int ret; - - ret = sscanf(buf, "%u", &temp); - - if (ret != 1 || temp < TEMP_MIN || temp > TEMP_MAX) - return -EINVAL; - - new_value = temp_to_reg(temp); - - reg_value = in_be64(&pmd_regs->tm_tpr.val); - - /* zero out bits for new value */ - reg_value &= ~(0xffull << pos); - /* set bits to new value */ - reg_value |= new_value << pos; - - out_be64(&pmd_regs->tm_tpr.val, reg_value); - return size; -} - -static ssize_t spu_show_throttle_end(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return show_throttle(get_pmd_regs(dev), buf, 0); -} - -static ssize_t spu_show_throttle_begin(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return show_throttle(get_pmd_regs(dev), buf, 8); -} - -static ssize_t spu_show_throttle_full_stop(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return show_throttle(get_pmd_regs(dev), buf, 16); -} - -static ssize_t spu_store_throttle_end(struct device *dev, - struct device_attribute *attr, const char *buf, size_t size) -{ - return store_throttle(get_pmd_regs(dev), buf, size, 0); -} - -static ssize_t spu_store_throttle_begin(struct device *dev, - struct device_attribute *attr, const char *buf, size_t size) -{ - return store_throttle(get_pmd_regs(dev), buf, size, 8); -} - -static ssize_t spu_store_throttle_full_stop(struct device *dev, - struct device_attribute *attr, const char *buf, size_t size) -{ - return store_throttle(get_pmd_regs(dev), buf, size, 16); -} - -static ssize_t ppe_show_temp(struct device *dev, char *buf, int pos) -{ - struct cbe_pmd_regs __iomem *pmd_regs; - u64 value; - - pmd_regs = cbe_get_cpu_pmd_regs(dev->id); - value = in_be64(&pmd_regs->ts_ctsr2); - - value = (value >> pos) & 0x3f; - - return sprintf(buf, "%d\n", reg_to_temp(value)); -} - - -/* shows the temperature of the DTS on the PPE, - * located near the linear thermal sensor */ -static ssize_t ppe_show_temp0(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return ppe_show_temp(dev, buf, 32); -} - -/* shows the temperature of the second DTS on the PPE */ -static ssize_t ppe_show_temp1(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return ppe_show_temp(dev, buf, 0); -} - -static ssize_t ppe_show_throttle_end(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return show_throttle(cbe_get_cpu_pmd_regs(dev->id), buf, 32); -} - -static ssize_t ppe_show_throttle_begin(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return show_throttle(cbe_get_cpu_pmd_regs(dev->id), buf, 40); -} - -static ssize_t ppe_show_throttle_full_stop(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return show_throttle(cbe_get_cpu_pmd_regs(dev->id), buf, 48); -} - -static ssize_t ppe_store_throttle_end(struct device *dev, - struct device_attribute *attr, const char *buf, size_t size) -{ - return store_throttle(cbe_get_cpu_pmd_regs(dev->id), buf, size, 32); -} - -static ssize_t ppe_store_throttle_begin(struct device *dev, - struct device_attribute *attr, const char *buf, size_t size) -{ - return store_throttle(cbe_get_cpu_pmd_regs(dev->id), buf, size, 40); -} - -static ssize_t ppe_store_throttle_full_stop(struct device *dev, - struct device_attribute *attr, const char *buf, size_t size) -{ - return store_throttle(cbe_get_cpu_pmd_regs(dev->id), buf, size, 48); -} - - -static struct device_attribute attr_spu_temperature = { - .attr = {.name = "temperature", .mode = 0400 }, - .show = spu_show_temp, -}; - -static DEVICE_PREFIX_ATTR(spu, throttle_end, 0600); -static DEVICE_PREFIX_ATTR(spu, throttle_begin, 0600); -static DEVICE_PREFIX_ATTR(spu, throttle_full_stop, 0600); - - -static struct attribute *spu_attributes[] = { - &attr_spu_temperature.attr, - &attr_spu_throttle_end.attr, - &attr_spu_throttle_begin.attr, - &attr_spu_throttle_full_stop.attr, - NULL, -}; - -static const struct attribute_group spu_attribute_group = { - .name = "thermal", - .attrs = spu_attributes, -}; - -static struct device_attribute attr_ppe_temperature0 = { - .attr = {.name = "temperature0", .mode = 0400 }, - .show = ppe_show_temp0, -}; - -static struct device_attribute attr_ppe_temperature1 = { - .attr = {.name = "temperature1", .mode = 0400 }, - .show = ppe_show_temp1, -}; - -static DEVICE_PREFIX_ATTR(ppe, throttle_end, 0600); -static DEVICE_PREFIX_ATTR(ppe, throttle_begin, 0600); -static DEVICE_PREFIX_ATTR(ppe, throttle_full_stop, 0600); - -static struct attribute *ppe_attributes[] = { - &attr_ppe_temperature0.attr, - &attr_ppe_temperature1.attr, - &attr_ppe_throttle_end.attr, - &attr_ppe_throttle_begin.attr, - &attr_ppe_throttle_full_stop.attr, - NULL, -}; - -static struct attribute_group ppe_attribute_group = { - .name = "thermal", - .attrs = ppe_attributes, -}; - -/* - * initialize throttling with default values - */ -static int __init init_default_values(void) -{ - int cpu; - struct cbe_pmd_regs __iomem *pmd_regs; - struct device *dev; - union ppe_spe_reg tpr; - union spe_reg str1; - u64 str2; - union spe_reg cr1; - u64 cr2; - - /* TPR defaults */ - /* ppe - * 1F - no full stop - * 08 - dynamic throttling starts if over 80 degrees - * 03 - dynamic throttling ceases if below 70 degrees */ - tpr.ppe = 0x1F0803; - /* spe - * 10 - full stopped when over 96 degrees - * 08 - dynamic throttling starts if over 80 degrees - * 03 - dynamic throttling ceases if below 70 degrees - */ - tpr.spe = 0x100803; - - /* STR defaults */ - /* str1 - * 10 - stop 16 of 32 cycles - */ - str1.val = 0x1010101010101010ull; - /* str2 - * 10 - stop 16 of 32 cycles - */ - str2 = 0x10; - - /* CR defaults */ - /* cr1 - * 4 - normal operation - */ - cr1.val = 0x0404040404040404ull; - /* cr2 - * 4 - normal operation - */ - cr2 = 0x04; - - for_each_possible_cpu (cpu) { - pr_debug("processing cpu %d\n", cpu); - dev = get_cpu_device(cpu); - - if (!dev) { - pr_info("invalid dev pointer for cbe_thermal\n"); - return -EINVAL; - } - - pmd_regs = cbe_get_cpu_pmd_regs(dev->id); - - if (!pmd_regs) { - pr_info("invalid CBE regs pointer for cbe_thermal\n"); - return -EINVAL; - } - - out_be64(&pmd_regs->tm_str2, str2); - out_be64(&pmd_regs->tm_str1.val, str1.val); - out_be64(&pmd_regs->tm_tpr.val, tpr.val); - out_be64(&pmd_regs->tm_cr1.val, cr1.val); - out_be64(&pmd_regs->tm_cr2, cr2); - } - - return 0; -} - - -static int __init thermal_init(void) -{ - int rc = init_default_values(); - - if (rc == 0) { - spu_add_dev_attr_group(&spu_attribute_group); - cpu_add_dev_attr_group(&ppe_attribute_group); - } - - return rc; -} -module_init(thermal_init); - -static void __exit thermal_exit(void) -{ - spu_remove_dev_attr_group(&spu_attribute_group); - cpu_remove_dev_attr_group(&ppe_attribute_group); -} -module_exit(thermal_exit); - -MODULE_DESCRIPTION("Cell processor thermal driver"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Christian Krafft <krafft@de.ibm.com>"); - diff --git a/arch/powerpc/platforms/cell/cell.h b/arch/powerpc/platforms/cell/cell.h deleted file mode 100644 index d5142e905ab3..000000000000 --- a/arch/powerpc/platforms/cell/cell.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Cell Platform common data structures - * - * Copyright 2015, Daniel Axtens, IBM Corporation - */ - -#ifndef CELL_H -#define CELL_H - -#include <asm/pci-bridge.h> - -extern struct pci_controller_ops cell_pci_controller_ops; - -#endif diff --git a/arch/powerpc/platforms/cell/cpufreq_spudemand.c b/arch/powerpc/platforms/cell/cpufreq_spudemand.c deleted file mode 100644 index 79172ba36eca..000000000000 --- a/arch/powerpc/platforms/cell/cpufreq_spudemand.c +++ /dev/null @@ -1,134 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * spu aware cpufreq governor for the cell processor - * - * © Copyright IBM Corporation 2006-2008 - * - * Author: Christian Krafft <krafft@de.ibm.com> - */ - -#include <linux/cpufreq.h> -#include <linux/sched.h> -#include <linux/sched/loadavg.h> -#include <linux/module.h> -#include <linux/timer.h> -#include <linux/workqueue.h> -#include <linux/atomic.h> -#include <asm/machdep.h> -#include <asm/spu.h> - -#define POLL_TIME 100000 /* in µs */ -#define EXP 753 /* exp(-1) in fixed-point */ - -struct spu_gov_info_struct { - unsigned long busy_spus; /* fixed-point */ - struct cpufreq_policy *policy; - struct delayed_work work; - unsigned int poll_int; /* µs */ -}; -static DEFINE_PER_CPU(struct spu_gov_info_struct, spu_gov_info); - -static int calc_freq(struct spu_gov_info_struct *info) -{ - int cpu; - int busy_spus; - - cpu = info->policy->cpu; - busy_spus = atomic_read(&cbe_spu_info[cpu_to_node(cpu)].busy_spus); - - info->busy_spus = calc_load(info->busy_spus, EXP, busy_spus * FIXED_1); - pr_debug("cpu %d: busy_spus=%d, info->busy_spus=%ld\n", - cpu, busy_spus, info->busy_spus); - - return info->policy->max * info->busy_spus / FIXED_1; -} - -static void spu_gov_work(struct work_struct *work) -{ - struct spu_gov_info_struct *info; - int delay; - unsigned long target_freq; - - info = container_of(work, struct spu_gov_info_struct, work.work); - - /* after cancel_delayed_work_sync we unset info->policy */ - BUG_ON(info->policy == NULL); - - target_freq = calc_freq(info); - __cpufreq_driver_target(info->policy, target_freq, CPUFREQ_RELATION_H); - - delay = usecs_to_jiffies(info->poll_int); - schedule_delayed_work_on(info->policy->cpu, &info->work, delay); -} - -static void spu_gov_init_work(struct spu_gov_info_struct *info) -{ - int delay = usecs_to_jiffies(info->poll_int); - INIT_DEFERRABLE_WORK(&info->work, spu_gov_work); - schedule_delayed_work_on(info->policy->cpu, &info->work, delay); -} - -static void spu_gov_cancel_work(struct spu_gov_info_struct *info) -{ - cancel_delayed_work_sync(&info->work); -} - -static int spu_gov_start(struct cpufreq_policy *policy) -{ - unsigned int cpu = policy->cpu; - struct spu_gov_info_struct *info = &per_cpu(spu_gov_info, cpu); - struct spu_gov_info_struct *affected_info; - int i; - - if (!cpu_online(cpu)) { - printk(KERN_ERR "cpu %d is not online\n", cpu); - return -EINVAL; - } - - if (!policy->cur) { - printk(KERN_ERR "no cpu specified in policy\n"); - return -EINVAL; - } - - /* initialize spu_gov_info for all affected cpus */ - for_each_cpu(i, policy->cpus) { - affected_info = &per_cpu(spu_gov_info, i); - affected_info->policy = policy; - } - - info->poll_int = POLL_TIME; - - /* setup timer */ - spu_gov_init_work(info); - - return 0; -} - -static void spu_gov_stop(struct cpufreq_policy *policy) -{ - unsigned int cpu = policy->cpu; - struct spu_gov_info_struct *info = &per_cpu(spu_gov_info, cpu); - int i; - - /* cancel timer */ - spu_gov_cancel_work(info); - - /* clean spu_gov_info for all affected cpus */ - for_each_cpu (i, policy->cpus) { - info = &per_cpu(spu_gov_info, i); - info->policy = NULL; - } -} - -static struct cpufreq_governor spu_governor = { - .name = "spudemand", - .start = spu_gov_start, - .stop = spu_gov_stop, - .owner = THIS_MODULE, -}; -cpufreq_governor_init(spu_governor); -cpufreq_governor_exit(spu_governor); - -MODULE_DESCRIPTION("SPU-aware cpufreq governor for the cell processor"); -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Christian Krafft <krafft@de.ibm.com>"); diff --git a/arch/powerpc/platforms/cell/interrupt.c b/arch/powerpc/platforms/cell/interrupt.c deleted file mode 100644 index 03ee8152ee97..000000000000 --- a/arch/powerpc/platforms/cell/interrupt.c +++ /dev/null @@ -1,390 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cell Internal Interrupt Controller - * - * Copyright (C) 2006 Benjamin Herrenschmidt (benh@kernel.crashing.org) - * IBM, Corp. - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * - * Author: Arnd Bergmann <arndb@de.ibm.com> - * - * TODO: - * - Fix various assumptions related to HW CPU numbers vs. linux CPU numbers - * vs node numbers in the setup code - * - Implement proper handling of maxcpus=1/2 (that is, routing of irqs from - * a non-active node to the active node) - */ - -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/irqdomain.h> -#include <linux/export.h> -#include <linux/percpu.h> -#include <linux/types.h> -#include <linux/ioport.h> -#include <linux/kernel_stat.h> -#include <linux/pgtable.h> -#include <linux/of_address.h> - -#include <asm/io.h> -#include <asm/ptrace.h> -#include <asm/machdep.h> -#include <asm/cell-regs.h> - -#include "interrupt.h" - -struct iic { - struct cbe_iic_thread_regs __iomem *regs; - u8 target_id; - u8 eoi_stack[16]; - int eoi_ptr; - struct device_node *node; -}; - -static DEFINE_PER_CPU(struct iic, cpu_iic); -#define IIC_NODE_COUNT 2 -static struct irq_domain *iic_host; - -/* Convert between "pending" bits and hw irq number */ -static irq_hw_number_t iic_pending_to_hwnum(struct cbe_iic_pending_bits bits) -{ - unsigned char unit = bits.source & 0xf; - unsigned char node = bits.source >> 4; - unsigned char class = bits.class & 3; - - /* Decode IPIs */ - if (bits.flags & CBE_IIC_IRQ_IPI) - return IIC_IRQ_TYPE_IPI | (bits.prio >> 4); - else - return (node << IIC_IRQ_NODE_SHIFT) | (class << 4) | unit; -} - -static void iic_mask(struct irq_data *d) -{ -} - -static void iic_unmask(struct irq_data *d) -{ -} - -static void iic_eoi(struct irq_data *d) -{ - struct iic *iic = this_cpu_ptr(&cpu_iic); - out_be64(&iic->regs->prio, iic->eoi_stack[--iic->eoi_ptr]); - BUG_ON(iic->eoi_ptr < 0); -} - -static struct irq_chip iic_chip = { - .name = "CELL-IIC", - .irq_mask = iic_mask, - .irq_unmask = iic_unmask, - .irq_eoi = iic_eoi, -}; - - -static void iic_ioexc_eoi(struct irq_data *d) -{ -} - -static void iic_ioexc_cascade(struct irq_desc *desc) -{ - struct irq_chip *chip = irq_desc_get_chip(desc); - struct cbe_iic_regs __iomem *node_iic = - (void __iomem *)irq_desc_get_handler_data(desc); - unsigned int irq = irq_desc_get_irq(desc); - unsigned int base = (irq & 0xffffff00) | IIC_IRQ_TYPE_IOEXC; - unsigned long bits, ack; - int cascade; - - for (;;) { - bits = in_be64(&node_iic->iic_is); - if (bits == 0) - break; - /* pre-ack edge interrupts */ - ack = bits & IIC_ISR_EDGE_MASK; - if (ack) - out_be64(&node_iic->iic_is, ack); - /* handle them */ - for (cascade = 63; cascade >= 0; cascade--) - if (bits & (0x8000000000000000UL >> cascade)) - generic_handle_domain_irq(iic_host, - base | cascade); - /* post-ack level interrupts */ - ack = bits & ~IIC_ISR_EDGE_MASK; - if (ack) - out_be64(&node_iic->iic_is, ack); - } - chip->irq_eoi(&desc->irq_data); -} - - -static struct irq_chip iic_ioexc_chip = { - .name = "CELL-IOEX", - .irq_mask = iic_mask, - .irq_unmask = iic_unmask, - .irq_eoi = iic_ioexc_eoi, -}; - -/* Get an IRQ number from the pending state register of the IIC */ -static unsigned int iic_get_irq(void) -{ - struct cbe_iic_pending_bits pending; - struct iic *iic; - unsigned int virq; - - iic = this_cpu_ptr(&cpu_iic); - *(unsigned long *) &pending = - in_be64((u64 __iomem *) &iic->regs->pending_destr); - if (!(pending.flags & CBE_IIC_IRQ_VALID)) - return 0; - virq = irq_linear_revmap(iic_host, iic_pending_to_hwnum(pending)); - if (!virq) - return 0; - iic->eoi_stack[++iic->eoi_ptr] = pending.prio; - BUG_ON(iic->eoi_ptr > 15); - return virq; -} - -void iic_setup_cpu(void) -{ - out_be64(&this_cpu_ptr(&cpu_iic)->regs->prio, 0xff); -} - -u8 iic_get_target_id(int cpu) -{ - return per_cpu(cpu_iic, cpu).target_id; -} - -EXPORT_SYMBOL_GPL(iic_get_target_id); - -#ifdef CONFIG_SMP - -/* Use the highest interrupt priorities for IPI */ -static inline int iic_msg_to_irq(int msg) -{ - return IIC_IRQ_TYPE_IPI + 0xf - msg; -} - -void iic_message_pass(int cpu, int msg) -{ - out_be64(&per_cpu(cpu_iic, cpu).regs->generate, (0xf - msg) << 4); -} - -static void iic_request_ipi(int msg) -{ - int virq; - - virq = irq_create_mapping(iic_host, iic_msg_to_irq(msg)); - if (!virq) { - printk(KERN_ERR - "iic: failed to map IPI %s\n", smp_ipi_name[msg]); - return; - } - - /* - * If smp_request_message_ipi encounters an error it will notify - * the error. If a message is not needed it will return non-zero. - */ - if (smp_request_message_ipi(virq, msg)) - irq_dispose_mapping(virq); -} - -void iic_request_IPIs(void) -{ - iic_request_ipi(PPC_MSG_CALL_FUNCTION); - iic_request_ipi(PPC_MSG_RESCHEDULE); - iic_request_ipi(PPC_MSG_TICK_BROADCAST); - iic_request_ipi(PPC_MSG_NMI_IPI); -} - -#endif /* CONFIG_SMP */ - - -static int iic_host_match(struct irq_domain *h, struct device_node *node, - enum irq_domain_bus_token bus_token) -{ - return of_device_is_compatible(node, - "IBM,CBEA-Internal-Interrupt-Controller"); -} - -static int iic_host_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - switch (hw & IIC_IRQ_TYPE_MASK) { - case IIC_IRQ_TYPE_IPI: - irq_set_chip_and_handler(virq, &iic_chip, handle_percpu_irq); - break; - case IIC_IRQ_TYPE_IOEXC: - irq_set_chip_and_handler(virq, &iic_ioexc_chip, - handle_edge_eoi_irq); - break; - default: - irq_set_chip_and_handler(virq, &iic_chip, handle_edge_eoi_irq); - } - return 0; -} - -static int iic_host_xlate(struct irq_domain *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_flags) - -{ - unsigned int node, ext, unit, class; - const u32 *val; - - if (!of_device_is_compatible(ct, - "IBM,CBEA-Internal-Interrupt-Controller")) - return -ENODEV; - if (intsize != 1) - return -ENODEV; - val = of_get_property(ct, "#interrupt-cells", NULL); - if (val == NULL || *val != 1) - return -ENODEV; - - node = intspec[0] >> 24; - ext = (intspec[0] >> 16) & 0xff; - class = (intspec[0] >> 8) & 0xff; - unit = intspec[0] & 0xff; - - /* Check if node is in supported range */ - if (node > 1) - return -EINVAL; - - /* Build up interrupt number, special case for IO exceptions */ - *out_hwirq = (node << IIC_IRQ_NODE_SHIFT); - if (unit == IIC_UNIT_IIC && class == 1) - *out_hwirq |= IIC_IRQ_TYPE_IOEXC | ext; - else - *out_hwirq |= IIC_IRQ_TYPE_NORMAL | - (class << IIC_IRQ_CLASS_SHIFT) | unit; - - /* Dummy flags, ignored by iic code */ - *out_flags = IRQ_TYPE_EDGE_RISING; - - return 0; -} - -static const struct irq_domain_ops iic_host_ops = { - .match = iic_host_match, - .map = iic_host_map, - .xlate = iic_host_xlate, -}; - -static void __init init_one_iic(unsigned int hw_cpu, unsigned long addr, - struct device_node *node) -{ - /* XXX FIXME: should locate the linux CPU number from the HW cpu - * number properly. We are lucky for now - */ - struct iic *iic = &per_cpu(cpu_iic, hw_cpu); - - iic->regs = ioremap(addr, sizeof(struct cbe_iic_thread_regs)); - BUG_ON(iic->regs == NULL); - - iic->target_id = ((hw_cpu & 2) << 3) | ((hw_cpu & 1) ? 0xf : 0xe); - iic->eoi_stack[0] = 0xff; - iic->node = of_node_get(node); - out_be64(&iic->regs->prio, 0); - - printk(KERN_INFO "IIC for CPU %d target id 0x%x : %pOF\n", - hw_cpu, iic->target_id, node); -} - -static int __init setup_iic(void) -{ - struct device_node *dn; - struct resource r0, r1; - unsigned int node, cascade, found = 0; - struct cbe_iic_regs __iomem *node_iic; - const u32 *np; - - for_each_node_by_name(dn, "interrupt-controller") { - if (!of_device_is_compatible(dn, - "IBM,CBEA-Internal-Interrupt-Controller")) - continue; - np = of_get_property(dn, "ibm,interrupt-server-ranges", NULL); - if (np == NULL) { - printk(KERN_WARNING "IIC: CPU association not found\n"); - of_node_put(dn); - return -ENODEV; - } - if (of_address_to_resource(dn, 0, &r0) || - of_address_to_resource(dn, 1, &r1)) { - printk(KERN_WARNING "IIC: Can't resolve addresses\n"); - of_node_put(dn); - return -ENODEV; - } - found++; - init_one_iic(np[0], r0.start, dn); - init_one_iic(np[1], r1.start, dn); - - /* Setup cascade for IO exceptions. XXX cleanup tricks to get - * node vs CPU etc... - * Note that we configure the IIC_IRR here with a hard coded - * priority of 1. We might want to improve that later. - */ - node = np[0] >> 1; - node_iic = cbe_get_cpu_iic_regs(np[0]); - cascade = node << IIC_IRQ_NODE_SHIFT; - cascade |= 1 << IIC_IRQ_CLASS_SHIFT; - cascade |= IIC_UNIT_IIC; - cascade = irq_create_mapping(iic_host, cascade); - if (!cascade) - continue; - /* - * irq_data is a generic pointer that gets passed back - * to us later, so the forced cast is fine. - */ - irq_set_handler_data(cascade, (void __force *)node_iic); - irq_set_chained_handler(cascade, iic_ioexc_cascade); - out_be64(&node_iic->iic_ir, - (1 << 12) /* priority */ | - (node << 4) /* dest node */ | - IIC_UNIT_THREAD_0 /* route them to thread 0 */); - /* Flush pending (make sure it triggers if there is - * anything pending - */ - out_be64(&node_iic->iic_is, 0xfffffffffffffffful); - } - - if (found) - return 0; - else - return -ENODEV; -} - -void __init iic_init_IRQ(void) -{ - /* Setup an irq host data structure */ - iic_host = irq_domain_add_linear(NULL, IIC_SOURCE_COUNT, &iic_host_ops, - NULL); - BUG_ON(iic_host == NULL); - irq_set_default_host(iic_host); - - /* Discover and initialize iics */ - if (setup_iic() < 0) - panic("IIC: Failed to initialize !\n"); - - /* Set master interrupt handling function */ - ppc_md.get_irq = iic_get_irq; - - /* Enable on current CPU */ - iic_setup_cpu(); -} - -void iic_set_interrupt_routing(int cpu, int thread, int priority) -{ - struct cbe_iic_regs __iomem *iic_regs = cbe_get_cpu_iic_regs(cpu); - u64 iic_ir = 0; - int node = cpu >> 1; - - /* Set which node and thread will handle the next interrupt */ - iic_ir |= CBE_IIC_IR_PRIO(priority) | - CBE_IIC_IR_DEST_NODE(node); - if (thread == 0) - iic_ir |= CBE_IIC_IR_DEST_UNIT(CBE_IIC_IR_PT_0); - else - iic_ir |= CBE_IIC_IR_DEST_UNIT(CBE_IIC_IR_PT_1); - out_be64(&iic_regs->iic_ir, iic_ir); -} diff --git a/arch/powerpc/platforms/cell/interrupt.h b/arch/powerpc/platforms/cell/interrupt.h deleted file mode 100644 index a47902248541..000000000000 --- a/arch/powerpc/platforms/cell/interrupt.h +++ /dev/null @@ -1,90 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ASM_CELL_PIC_H -#define ASM_CELL_PIC_H -#ifdef __KERNEL__ -/* - * Mapping of IIC pending bits into per-node interrupt numbers. - * - * Interrupt numbers are in the range 0...0x1ff where the top bit - * (0x100) represent the source node. Only 2 nodes are supported with - * the current code though it's trivial to extend that if necessary using - * higher level bits - * - * The bottom 8 bits are split into 2 type bits and 6 data bits that - * depend on the type: - * - * 00 (0x00 | data) : normal interrupt. data is (class << 4) | source - * 01 (0x40 | data) : IO exception. data is the exception number as - * defined by bit numbers in IIC_SR - * 10 (0x80 | data) : IPI. data is the IPI number (obtained from the priority) - * and node is always 0 (IPIs are per-cpu, their source is - * not relevant) - * 11 (0xc0 | data) : reserved - * - * In addition, interrupt number 0x80000000 is defined as always invalid - * (that is the node field is expected to never extend to move than 23 bits) - * - */ - -enum { - IIC_IRQ_INVALID = 0x80000000u, - IIC_IRQ_NODE_MASK = 0x100, - IIC_IRQ_NODE_SHIFT = 8, - IIC_IRQ_MAX = 0x1ff, - IIC_IRQ_TYPE_MASK = 0xc0, - IIC_IRQ_TYPE_NORMAL = 0x00, - IIC_IRQ_TYPE_IOEXC = 0x40, - IIC_IRQ_TYPE_IPI = 0x80, - IIC_IRQ_CLASS_SHIFT = 4, - IIC_IRQ_CLASS_0 = 0x00, - IIC_IRQ_CLASS_1 = 0x10, - IIC_IRQ_CLASS_2 = 0x20, - IIC_SOURCE_COUNT = 0x200, - - /* Here are defined the various source/dest units. Avoid using those - * definitions if you can, they are mostly here for reference - */ - IIC_UNIT_SPU_0 = 0x4, - IIC_UNIT_SPU_1 = 0x7, - IIC_UNIT_SPU_2 = 0x3, - IIC_UNIT_SPU_3 = 0x8, - IIC_UNIT_SPU_4 = 0x2, - IIC_UNIT_SPU_5 = 0x9, - IIC_UNIT_SPU_6 = 0x1, - IIC_UNIT_SPU_7 = 0xa, - IIC_UNIT_IOC_0 = 0x0, - IIC_UNIT_IOC_1 = 0xb, - IIC_UNIT_THREAD_0 = 0xe, /* target only */ - IIC_UNIT_THREAD_1 = 0xf, /* target only */ - IIC_UNIT_IIC = 0xe, /* source only (IO exceptions) */ - - /* Base numbers for the external interrupts */ - IIC_IRQ_EXT_IOIF0 = - IIC_IRQ_TYPE_NORMAL | IIC_IRQ_CLASS_2 | IIC_UNIT_IOC_0, - IIC_IRQ_EXT_IOIF1 = - IIC_IRQ_TYPE_NORMAL | IIC_IRQ_CLASS_2 | IIC_UNIT_IOC_1, - - /* Base numbers for the IIC_ISR interrupts */ - IIC_IRQ_IOEX_TMI = IIC_IRQ_TYPE_IOEXC | IIC_IRQ_CLASS_1 | 63, - IIC_IRQ_IOEX_PMI = IIC_IRQ_TYPE_IOEXC | IIC_IRQ_CLASS_1 | 62, - IIC_IRQ_IOEX_ATI = IIC_IRQ_TYPE_IOEXC | IIC_IRQ_CLASS_1 | 61, - IIC_IRQ_IOEX_MATBFI = IIC_IRQ_TYPE_IOEXC | IIC_IRQ_CLASS_1 | 60, - IIC_IRQ_IOEX_ELDI = IIC_IRQ_TYPE_IOEXC | IIC_IRQ_CLASS_1 | 59, - - /* Which bits in IIC_ISR are edge sensitive */ - IIC_ISR_EDGE_MASK = 0x4ul, -}; - -extern void iic_init_IRQ(void); -extern void iic_message_pass(int cpu, int msg); -extern void iic_request_IPIs(void); -extern void iic_setup_cpu(void); - -extern u8 iic_get_target_id(int cpu); - -extern void spider_init_IRQ(void); - -extern void iic_set_interrupt_routing(int cpu, int thread, int priority); - -#endif -#endif /* ASM_CELL_PIC_H */ diff --git a/arch/powerpc/platforms/cell/iommu.c b/arch/powerpc/platforms/cell/iommu.c deleted file mode 100644 index 62c9679b8ca3..000000000000 --- a/arch/powerpc/platforms/cell/iommu.c +++ /dev/null @@ -1,1060 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * IOMMU implementation for Cell Broadband Processor Architecture - * - * (C) Copyright IBM Corporation 2006-2008 - * - * Author: Jeremy Kerr <jk@ozlabs.org> - */ - -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/irqdomain.h> -#include <linux/notifier.h> -#include <linux/of.h> -#include <linux/of_address.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/memblock.h> - -#include <asm/prom.h> -#include <asm/iommu.h> -#include <asm/machdep.h> -#include <asm/pci-bridge.h> -#include <asm/udbg.h> -#include <asm/firmware.h> -#include <asm/cell-regs.h> - -#include "cell.h" -#include "interrupt.h" - -/* Define CELL_IOMMU_REAL_UNMAP to actually unmap non-used pages - * instead of leaving them mapped to some dummy page. This can be - * enabled once the appropriate workarounds for spider bugs have - * been enabled - */ -#define CELL_IOMMU_REAL_UNMAP - -/* Define CELL_IOMMU_STRICT_PROTECTION to enforce protection of - * IO PTEs based on the transfer direction. That can be enabled - * once spider-net has been fixed to pass the correct direction - * to the DMA mapping functions - */ -#define CELL_IOMMU_STRICT_PROTECTION - - -#define NR_IOMMUS 2 - -/* IOC mmap registers */ -#define IOC_Reg_Size 0x2000 - -#define IOC_IOPT_CacheInvd 0x908 -#define IOC_IOPT_CacheInvd_NE_Mask 0xffe0000000000000ul -#define IOC_IOPT_CacheInvd_IOPTE_Mask 0x000003fffffffff8ul -#define IOC_IOPT_CacheInvd_Busy 0x0000000000000001ul - -#define IOC_IOST_Origin 0x918 -#define IOC_IOST_Origin_E 0x8000000000000000ul -#define IOC_IOST_Origin_HW 0x0000000000000800ul -#define IOC_IOST_Origin_HL 0x0000000000000400ul - -#define IOC_IO_ExcpStat 0x920 -#define IOC_IO_ExcpStat_V 0x8000000000000000ul -#define IOC_IO_ExcpStat_SPF_Mask 0x6000000000000000ul -#define IOC_IO_ExcpStat_SPF_S 0x6000000000000000ul -#define IOC_IO_ExcpStat_SPF_P 0x2000000000000000ul -#define IOC_IO_ExcpStat_ADDR_Mask 0x00000007fffff000ul -#define IOC_IO_ExcpStat_RW_Mask 0x0000000000000800ul -#define IOC_IO_ExcpStat_IOID_Mask 0x00000000000007fful - -#define IOC_IO_ExcpMask 0x928 -#define IOC_IO_ExcpMask_SFE 0x4000000000000000ul -#define IOC_IO_ExcpMask_PFE 0x2000000000000000ul - -#define IOC_IOCmd_Offset 0x1000 - -#define IOC_IOCmd_Cfg 0xc00 -#define IOC_IOCmd_Cfg_TE 0x0000800000000000ul - - -/* Segment table entries */ -#define IOSTE_V 0x8000000000000000ul /* valid */ -#define IOSTE_H 0x4000000000000000ul /* cache hint */ -#define IOSTE_PT_Base_RPN_Mask 0x3ffffffffffff000ul /* base RPN of IOPT */ -#define IOSTE_NPPT_Mask 0x0000000000000fe0ul /* no. pages in IOPT */ -#define IOSTE_PS_Mask 0x0000000000000007ul /* page size */ -#define IOSTE_PS_4K 0x0000000000000001ul /* - 4kB */ -#define IOSTE_PS_64K 0x0000000000000003ul /* - 64kB */ -#define IOSTE_PS_1M 0x0000000000000005ul /* - 1MB */ -#define IOSTE_PS_16M 0x0000000000000007ul /* - 16MB */ - - -/* IOMMU sizing */ -#define IO_SEGMENT_SHIFT 28 -#define IO_PAGENO_BITS(shift) (IO_SEGMENT_SHIFT - (shift)) - -/* The high bit needs to be set on every DMA address */ -#define SPIDER_DMA_OFFSET 0x80000000ul - -struct iommu_window { - struct list_head list; - struct cbe_iommu *iommu; - unsigned long offset; - unsigned long size; - unsigned int ioid; - struct iommu_table table; -}; - -#define NAMESIZE 8 -struct cbe_iommu { - int nid; - char name[NAMESIZE]; - void __iomem *xlate_regs; - void __iomem *cmd_regs; - unsigned long *stab; - unsigned long *ptab; - void *pad_page; - struct list_head windows; -}; - -/* Static array of iommus, one per node - * each contains a list of windows, keyed from dma_window property - * - on bus setup, look for a matching window, or create one - * - on dev setup, assign iommu_table ptr - */ -static struct cbe_iommu iommus[NR_IOMMUS]; -static int cbe_nr_iommus; - -static void invalidate_tce_cache(struct cbe_iommu *iommu, unsigned long *pte, - long n_ptes) -{ - u64 __iomem *reg; - u64 val; - long n; - - reg = iommu->xlate_regs + IOC_IOPT_CacheInvd; - - while (n_ptes > 0) { - /* we can invalidate up to 1 << 11 PTEs at once */ - n = min(n_ptes, 1l << 11); - val = (((n /*- 1*/) << 53) & IOC_IOPT_CacheInvd_NE_Mask) - | (__pa(pte) & IOC_IOPT_CacheInvd_IOPTE_Mask) - | IOC_IOPT_CacheInvd_Busy; - - out_be64(reg, val); - while (in_be64(reg) & IOC_IOPT_CacheInvd_Busy) - ; - - n_ptes -= n; - pte += n; - } -} - -static int tce_build_cell(struct iommu_table *tbl, long index, long npages, - unsigned long uaddr, enum dma_data_direction direction, - unsigned long attrs) -{ - int i; - unsigned long *io_pte, base_pte; - struct iommu_window *window = - container_of(tbl, struct iommu_window, table); - - /* implementing proper protection causes problems with the spidernet - * driver - check mapping directions later, but allow read & write by - * default for now.*/ -#ifdef CELL_IOMMU_STRICT_PROTECTION - /* to avoid referencing a global, we use a trick here to setup the - * protection bit. "prot" is setup to be 3 fields of 4 bits appended - * together for each of the 3 supported direction values. It is then - * shifted left so that the fields matching the desired direction - * lands on the appropriate bits, and other bits are masked out. - */ - const unsigned long prot = 0xc48; - base_pte = - ((prot << (52 + 4 * direction)) & - (CBE_IOPTE_PP_W | CBE_IOPTE_PP_R)) | - CBE_IOPTE_M | CBE_IOPTE_SO_RW | - (window->ioid & CBE_IOPTE_IOID_Mask); -#else - base_pte = CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_M | - CBE_IOPTE_SO_RW | (window->ioid & CBE_IOPTE_IOID_Mask); -#endif - if (unlikely(attrs & DMA_ATTR_WEAK_ORDERING)) - base_pte &= ~CBE_IOPTE_SO_RW; - - io_pte = (unsigned long *)tbl->it_base + (index - tbl->it_offset); - - for (i = 0; i < npages; i++, uaddr += (1 << tbl->it_page_shift)) - io_pte[i] = base_pte | (__pa(uaddr) & CBE_IOPTE_RPN_Mask); - - mb(); - - invalidate_tce_cache(window->iommu, io_pte, npages); - - pr_debug("tce_build_cell(index=%lx,n=%lx,dir=%d,base_pte=%lx)\n", - index, npages, direction, base_pte); - return 0; -} - -static void tce_free_cell(struct iommu_table *tbl, long index, long npages) -{ - - int i; - unsigned long *io_pte, pte; - struct iommu_window *window = - container_of(tbl, struct iommu_window, table); - - pr_debug("tce_free_cell(index=%lx,n=%lx)\n", index, npages); - -#ifdef CELL_IOMMU_REAL_UNMAP - pte = 0; -#else - /* spider bridge does PCI reads after freeing - insert a mapping - * to a scratch page instead of an invalid entry */ - pte = CBE_IOPTE_PP_R | CBE_IOPTE_M | CBE_IOPTE_SO_RW | - __pa(window->iommu->pad_page) | - (window->ioid & CBE_IOPTE_IOID_Mask); -#endif - - io_pte = (unsigned long *)tbl->it_base + (index - tbl->it_offset); - - for (i = 0; i < npages; i++) - io_pte[i] = pte; - - mb(); - - invalidate_tce_cache(window->iommu, io_pte, npages); -} - -static irqreturn_t ioc_interrupt(int irq, void *data) -{ - unsigned long stat, spf; - struct cbe_iommu *iommu = data; - - stat = in_be64(iommu->xlate_regs + IOC_IO_ExcpStat); - spf = stat & IOC_IO_ExcpStat_SPF_Mask; - - /* Might want to rate limit it */ - printk(KERN_ERR "iommu: DMA exception 0x%016lx\n", stat); - printk(KERN_ERR " V=%d, SPF=[%c%c], RW=%s, IOID=0x%04x\n", - !!(stat & IOC_IO_ExcpStat_V), - (spf == IOC_IO_ExcpStat_SPF_S) ? 'S' : ' ', - (spf == IOC_IO_ExcpStat_SPF_P) ? 'P' : ' ', - (stat & IOC_IO_ExcpStat_RW_Mask) ? "Read" : "Write", - (unsigned int)(stat & IOC_IO_ExcpStat_IOID_Mask)); - printk(KERN_ERR " page=0x%016lx\n", - stat & IOC_IO_ExcpStat_ADDR_Mask); - - /* clear interrupt */ - stat &= ~IOC_IO_ExcpStat_V; - out_be64(iommu->xlate_regs + IOC_IO_ExcpStat, stat); - - return IRQ_HANDLED; -} - -static int __init cell_iommu_find_ioc(int nid, unsigned long *base) -{ - struct device_node *np; - struct resource r; - - *base = 0; - - /* First look for new style /be nodes */ - for_each_node_by_name(np, "ioc") { - if (of_node_to_nid(np) != nid) - continue; - if (of_address_to_resource(np, 0, &r)) { - printk(KERN_ERR "iommu: can't get address for %pOF\n", - np); - continue; - } - *base = r.start; - of_node_put(np); - return 0; - } - - /* Ok, let's try the old way */ - for_each_node_by_type(np, "cpu") { - const unsigned int *nidp; - const unsigned long *tmp; - - nidp = of_get_property(np, "node-id", NULL); - if (nidp && *nidp == nid) { - tmp = of_get_property(np, "ioc-translation", NULL); - if (tmp) { - *base = *tmp; - of_node_put(np); - return 0; - } - } - } - - return -ENODEV; -} - -static void __init cell_iommu_setup_stab(struct cbe_iommu *iommu, - unsigned long dbase, unsigned long dsize, - unsigned long fbase, unsigned long fsize) -{ - struct page *page; - unsigned long segments, stab_size; - - segments = max(dbase + dsize, fbase + fsize) >> IO_SEGMENT_SHIFT; - - pr_debug("%s: iommu[%d]: segments: %lu\n", - __func__, iommu->nid, segments); - - /* set up the segment table */ - stab_size = segments * sizeof(unsigned long); - page = alloc_pages_node(iommu->nid, GFP_KERNEL, get_order(stab_size)); - BUG_ON(!page); - iommu->stab = page_address(page); - memset(iommu->stab, 0, stab_size); -} - -static unsigned long *__init cell_iommu_alloc_ptab(struct cbe_iommu *iommu, - unsigned long base, unsigned long size, unsigned long gap_base, - unsigned long gap_size, unsigned long page_shift) -{ - struct page *page; - int i; - unsigned long reg, segments, pages_per_segment, ptab_size, - n_pte_pages, start_seg, *ptab; - - start_seg = base >> IO_SEGMENT_SHIFT; - segments = size >> IO_SEGMENT_SHIFT; - pages_per_segment = 1ull << IO_PAGENO_BITS(page_shift); - /* PTEs for each segment must start on a 4K boundary */ - pages_per_segment = max(pages_per_segment, - (1 << 12) / sizeof(unsigned long)); - - ptab_size = segments * pages_per_segment * sizeof(unsigned long); - pr_debug("%s: iommu[%d]: ptab_size: %lu, order: %d\n", __func__, - iommu->nid, ptab_size, get_order(ptab_size)); - page = alloc_pages_node(iommu->nid, GFP_KERNEL, get_order(ptab_size)); - BUG_ON(!page); - - ptab = page_address(page); - memset(ptab, 0, ptab_size); - - /* number of 4K pages needed for a page table */ - n_pte_pages = (pages_per_segment * sizeof(unsigned long)) >> 12; - - pr_debug("%s: iommu[%d]: stab at %p, ptab at %p, n_pte_pages: %lu\n", - __func__, iommu->nid, iommu->stab, ptab, - n_pte_pages); - - /* initialise the STEs */ - reg = IOSTE_V | ((n_pte_pages - 1) << 5); - - switch (page_shift) { - case 12: reg |= IOSTE_PS_4K; break; - case 16: reg |= IOSTE_PS_64K; break; - case 20: reg |= IOSTE_PS_1M; break; - case 24: reg |= IOSTE_PS_16M; break; - default: BUG(); - } - - gap_base = gap_base >> IO_SEGMENT_SHIFT; - gap_size = gap_size >> IO_SEGMENT_SHIFT; - - pr_debug("Setting up IOMMU stab:\n"); - for (i = start_seg; i < (start_seg + segments); i++) { - if (i >= gap_base && i < (gap_base + gap_size)) { - pr_debug("\toverlap at %d, skipping\n", i); - continue; - } - iommu->stab[i] = reg | (__pa(ptab) + (n_pte_pages << 12) * - (i - start_seg)); - pr_debug("\t[%d] 0x%016lx\n", i, iommu->stab[i]); - } - - return ptab; -} - -static void __init cell_iommu_enable_hardware(struct cbe_iommu *iommu) -{ - int ret; - unsigned long reg, xlate_base; - unsigned int virq; - - if (cell_iommu_find_ioc(iommu->nid, &xlate_base)) - panic("%s: missing IOC register mappings for node %d\n", - __func__, iommu->nid); - - iommu->xlate_regs = ioremap(xlate_base, IOC_Reg_Size); - iommu->cmd_regs = iommu->xlate_regs + IOC_IOCmd_Offset; - - /* ensure that the STEs have updated */ - mb(); - - /* setup interrupts for the iommu. */ - reg = in_be64(iommu->xlate_regs + IOC_IO_ExcpStat); - out_be64(iommu->xlate_regs + IOC_IO_ExcpStat, - reg & ~IOC_IO_ExcpStat_V); - out_be64(iommu->xlate_regs + IOC_IO_ExcpMask, - IOC_IO_ExcpMask_PFE | IOC_IO_ExcpMask_SFE); - - virq = irq_create_mapping(NULL, - IIC_IRQ_IOEX_ATI | (iommu->nid << IIC_IRQ_NODE_SHIFT)); - BUG_ON(!virq); - - ret = request_irq(virq, ioc_interrupt, 0, iommu->name, iommu); - BUG_ON(ret); - - /* set the IOC segment table origin register (and turn on the iommu) */ - reg = IOC_IOST_Origin_E | __pa(iommu->stab) | IOC_IOST_Origin_HW; - out_be64(iommu->xlate_regs + IOC_IOST_Origin, reg); - in_be64(iommu->xlate_regs + IOC_IOST_Origin); - - /* turn on IO translation */ - reg = in_be64(iommu->cmd_regs + IOC_IOCmd_Cfg) | IOC_IOCmd_Cfg_TE; - out_be64(iommu->cmd_regs + IOC_IOCmd_Cfg, reg); -} - -static void __init cell_iommu_setup_hardware(struct cbe_iommu *iommu, - unsigned long base, unsigned long size) -{ - cell_iommu_setup_stab(iommu, base, size, 0, 0); - iommu->ptab = cell_iommu_alloc_ptab(iommu, base, size, 0, 0, - IOMMU_PAGE_SHIFT_4K); - cell_iommu_enable_hardware(iommu); -} - -static inline u32 cell_iommu_get_ioid(struct device_node *np) -{ - const u32 *ioid; - - ioid = of_get_property(np, "ioid", NULL); - if (ioid == NULL) { - printk(KERN_WARNING "iommu: missing ioid for %pOF using 0\n", - np); - return 0; - } - - return *ioid; -} - -static struct iommu_table_ops cell_iommu_ops = { - .set = tce_build_cell, - .clear = tce_free_cell -}; - -static struct iommu_window * __init -cell_iommu_setup_window(struct cbe_iommu *iommu, struct device_node *np, - unsigned long offset, unsigned long size, - unsigned long pte_offset) -{ - struct iommu_window *window; - struct page *page; - u32 ioid; - - ioid = cell_iommu_get_ioid(np); - - window = kzalloc_node(sizeof(*window), GFP_KERNEL, iommu->nid); - BUG_ON(window == NULL); - - window->offset = offset; - window->size = size; - window->ioid = ioid; - window->iommu = iommu; - - window->table.it_blocksize = 16; - window->table.it_base = (unsigned long)iommu->ptab; - window->table.it_index = iommu->nid; - window->table.it_page_shift = IOMMU_PAGE_SHIFT_4K; - window->table.it_offset = - (offset >> window->table.it_page_shift) + pte_offset; - window->table.it_size = size >> window->table.it_page_shift; - window->table.it_ops = &cell_iommu_ops; - - if (!iommu_init_table(&window->table, iommu->nid, 0, 0)) - panic("Failed to initialize iommu table"); - - pr_debug("\tioid %d\n", window->ioid); - pr_debug("\tblocksize %ld\n", window->table.it_blocksize); - pr_debug("\tbase 0x%016lx\n", window->table.it_base); - pr_debug("\toffset 0x%lx\n", window->table.it_offset); - pr_debug("\tsize %ld\n", window->table.it_size); - - list_add(&window->list, &iommu->windows); - - if (offset != 0) - return window; - - /* We need to map and reserve the first IOMMU page since it's used - * by the spider workaround. In theory, we only need to do that when - * running on spider but it doesn't really matter. - * - * This code also assumes that we have a window that starts at 0, - * which is the case on all spider based blades. - */ - page = alloc_pages_node(iommu->nid, GFP_KERNEL, 0); - BUG_ON(!page); - iommu->pad_page = page_address(page); - clear_page(iommu->pad_page); - - __set_bit(0, window->table.it_map); - tce_build_cell(&window->table, window->table.it_offset, 1, - (unsigned long)iommu->pad_page, DMA_TO_DEVICE, 0); - - return window; -} - -static struct cbe_iommu *cell_iommu_for_node(int nid) -{ - int i; - - for (i = 0; i < cbe_nr_iommus; i++) - if (iommus[i].nid == nid) - return &iommus[i]; - return NULL; -} - -static unsigned long cell_dma_nommu_offset; - -static unsigned long dma_iommu_fixed_base; -static bool cell_iommu_enabled; - -/* iommu_fixed_is_weak is set if booted with iommu_fixed=weak */ -bool iommu_fixed_is_weak; - -static struct iommu_table *cell_get_iommu_table(struct device *dev) -{ - struct iommu_window *window; - struct cbe_iommu *iommu; - - /* Current implementation uses the first window available in that - * node's iommu. We -might- do something smarter later though it may - * never be necessary - */ - iommu = cell_iommu_for_node(dev_to_node(dev)); - if (iommu == NULL || list_empty(&iommu->windows)) { - dev_err(dev, "iommu: missing iommu for %pOF (node %d)\n", - dev->of_node, dev_to_node(dev)); - return NULL; - } - window = list_entry(iommu->windows.next, struct iommu_window, list); - - return &window->table; -} - -static u64 cell_iommu_get_fixed_address(struct device *dev); - -static void cell_dma_dev_setup(struct device *dev) -{ - if (cell_iommu_enabled) { - u64 addr = cell_iommu_get_fixed_address(dev); - - if (addr != OF_BAD_ADDR) - dev->archdata.dma_offset = addr + dma_iommu_fixed_base; - set_iommu_table_base(dev, cell_get_iommu_table(dev)); - } else { - dev->archdata.dma_offset = cell_dma_nommu_offset; - } -} - -static void cell_pci_dma_dev_setup(struct pci_dev *dev) -{ - cell_dma_dev_setup(&dev->dev); -} - -static int cell_of_bus_notify(struct notifier_block *nb, unsigned long action, - void *data) -{ - struct device *dev = data; - - /* We are only interested in device addition */ - if (action != BUS_NOTIFY_ADD_DEVICE) - return 0; - - if (cell_iommu_enabled) - dev->dma_ops = &dma_iommu_ops; - cell_dma_dev_setup(dev); - return 0; -} - -static struct notifier_block cell_of_bus_notifier = { - .notifier_call = cell_of_bus_notify -}; - -static int __init cell_iommu_get_window(struct device_node *np, - unsigned long *base, - unsigned long *size) -{ - const __be32 *dma_window; - unsigned long index; - - /* Use ibm,dma-window if available, else, hard code ! */ - dma_window = of_get_property(np, "ibm,dma-window", NULL); - if (dma_window == NULL) { - *base = 0; - *size = 0x80000000u; - return -ENODEV; - } - - of_parse_dma_window(np, dma_window, &index, base, size); - return 0; -} - -static struct cbe_iommu * __init cell_iommu_alloc(struct device_node *np) -{ - struct cbe_iommu *iommu; - int nid, i; - - /* Get node ID */ - nid = of_node_to_nid(np); - if (nid < 0) { - printk(KERN_ERR "iommu: failed to get node for %pOF\n", - np); - return NULL; - } - pr_debug("iommu: setting up iommu for node %d (%pOF)\n", - nid, np); - - /* XXX todo: If we can have multiple windows on the same IOMMU, which - * isn't the case today, we probably want here to check whether the - * iommu for that node is already setup. - * However, there might be issue with getting the size right so let's - * ignore that for now. We might want to completely get rid of the - * multiple window support since the cell iommu supports per-page ioids - */ - - if (cbe_nr_iommus >= NR_IOMMUS) { - printk(KERN_ERR "iommu: too many IOMMUs detected ! (%pOF)\n", - np); - return NULL; - } - - /* Init base fields */ - i = cbe_nr_iommus++; - iommu = &iommus[i]; - iommu->stab = NULL; - iommu->nid = nid; - snprintf(iommu->name, sizeof(iommu->name), "iommu%d", i); - INIT_LIST_HEAD(&iommu->windows); - - return iommu; -} - -static void __init cell_iommu_init_one(struct device_node *np, - unsigned long offset) -{ - struct cbe_iommu *iommu; - unsigned long base, size; - - iommu = cell_iommu_alloc(np); - if (!iommu) - return; - - /* Obtain a window for it */ - cell_iommu_get_window(np, &base, &size); - - pr_debug("\ttranslating window 0x%lx...0x%lx\n", - base, base + size - 1); - - /* Initialize the hardware */ - cell_iommu_setup_hardware(iommu, base, size); - - /* Setup the iommu_table */ - cell_iommu_setup_window(iommu, np, base, size, - offset >> IOMMU_PAGE_SHIFT_4K); -} - -static void __init cell_disable_iommus(void) -{ - int node; - unsigned long base, val; - void __iomem *xregs, *cregs; - - /* Make sure IOC translation is disabled on all nodes */ - for_each_online_node(node) { - if (cell_iommu_find_ioc(node, &base)) - continue; - xregs = ioremap(base, IOC_Reg_Size); - if (xregs == NULL) - continue; - cregs = xregs + IOC_IOCmd_Offset; - - pr_debug("iommu: cleaning up iommu on node %d\n", node); - - out_be64(xregs + IOC_IOST_Origin, 0); - (void)in_be64(xregs + IOC_IOST_Origin); - val = in_be64(cregs + IOC_IOCmd_Cfg); - val &= ~IOC_IOCmd_Cfg_TE; - out_be64(cregs + IOC_IOCmd_Cfg, val); - (void)in_be64(cregs + IOC_IOCmd_Cfg); - - iounmap(xregs); - } -} - -static int __init cell_iommu_init_disabled(void) -{ - struct device_node *np = NULL; - unsigned long base = 0, size; - - /* When no iommu is present, we use direct DMA ops */ - - /* First make sure all IOC translation is turned off */ - cell_disable_iommus(); - - /* If we have no Axon, we set up the spider DMA magic offset */ - np = of_find_node_by_name(NULL, "axon"); - if (!np) - cell_dma_nommu_offset = SPIDER_DMA_OFFSET; - of_node_put(np); - - /* Now we need to check to see where the memory is mapped - * in PCI space. We assume that all busses use the same dma - * window which is always the case so far on Cell, thus we - * pick up the first pci-internal node we can find and check - * the DMA window from there. - */ - for_each_node_by_name(np, "axon") { - if (np->parent == NULL || np->parent->parent != NULL) - continue; - if (cell_iommu_get_window(np, &base, &size) == 0) - break; - } - if (np == NULL) { - for_each_node_by_name(np, "pci-internal") { - if (np->parent == NULL || np->parent->parent != NULL) - continue; - if (cell_iommu_get_window(np, &base, &size) == 0) - break; - } - } - of_node_put(np); - - /* If we found a DMA window, we check if it's big enough to enclose - * all of physical memory. If not, we force enable IOMMU - */ - if (np && size < memblock_end_of_DRAM()) { - printk(KERN_WARNING "iommu: force-enabled, dma window" - " (%ldMB) smaller than total memory (%lldMB)\n", - size >> 20, memblock_end_of_DRAM() >> 20); - return -ENODEV; - } - - cell_dma_nommu_offset += base; - - if (cell_dma_nommu_offset != 0) - cell_pci_controller_ops.dma_dev_setup = cell_pci_dma_dev_setup; - - printk("iommu: disabled, direct DMA offset is 0x%lx\n", - cell_dma_nommu_offset); - - return 0; -} - -/* - * Fixed IOMMU mapping support - * - * This code adds support for setting up a fixed IOMMU mapping on certain - * cell machines. For 64-bit devices this avoids the performance overhead of - * mapping and unmapping pages at runtime. 32-bit devices are unable to use - * the fixed mapping. - * - * The fixed mapping is established at boot, and maps all of physical memory - * 1:1 into device space at some offset. On machines with < 30 GB of memory - * we setup the fixed mapping immediately above the normal IOMMU window. - * - * For example a machine with 4GB of memory would end up with the normal - * IOMMU window from 0-2GB and the fixed mapping window from 2GB to 6GB. In - * this case a 64-bit device wishing to DMA to 1GB would be told to DMA to - * 3GB, plus any offset required by firmware. The firmware offset is encoded - * in the "dma-ranges" property. - * - * On machines with 30GB or more of memory, we are unable to place the fixed - * mapping above the normal IOMMU window as we would run out of address space. - * Instead we move the normal IOMMU window to coincide with the hash page - * table, this region does not need to be part of the fixed mapping as no - * device should ever be DMA'ing to it. We then setup the fixed mapping - * from 0 to 32GB. - */ - -static u64 cell_iommu_get_fixed_address(struct device *dev) -{ - u64 best_size, dev_addr = OF_BAD_ADDR; - struct device_node *np; - struct of_range_parser parser; - struct of_range range; - - /* We can be called for platform devices that have no of_node */ - np = of_node_get(dev->of_node); - if (!np) - goto out; - - while ((np = of_get_next_parent(np))) { - if (of_pci_dma_range_parser_init(&parser, np)) - continue; - - if (of_range_count(&parser)) - break; - } - - if (!np) { - dev_dbg(dev, "iommu: no dma-ranges found\n"); - goto out; - } - - best_size = 0; - for_each_of_range(&parser, &range) { - if (!range.cpu_addr) - continue; - - if (range.size > best_size) { - best_size = range.size; - dev_addr = range.bus_addr; - } - } - - if (!best_size) - dev_dbg(dev, "iommu: no suitable range found!\n"); - -out: - of_node_put(np); - - return dev_addr; -} - -static bool cell_pci_iommu_bypass_supported(struct pci_dev *pdev, u64 mask) -{ - return mask == DMA_BIT_MASK(64) && - cell_iommu_get_fixed_address(&pdev->dev) != OF_BAD_ADDR; -} - -static void __init insert_16M_pte(unsigned long addr, unsigned long *ptab, - unsigned long base_pte) -{ - unsigned long segment, offset; - - segment = addr >> IO_SEGMENT_SHIFT; - offset = (addr >> 24) - (segment << IO_PAGENO_BITS(24)); - ptab = ptab + (segment * (1 << 12) / sizeof(unsigned long)); - - pr_debug("iommu: addr %lx ptab %p segment %lx offset %lx\n", - addr, ptab, segment, offset); - - ptab[offset] = base_pte | (__pa(addr) & CBE_IOPTE_RPN_Mask); -} - -static void __init cell_iommu_setup_fixed_ptab(struct cbe_iommu *iommu, - struct device_node *np, unsigned long dbase, unsigned long dsize, - unsigned long fbase, unsigned long fsize) -{ - unsigned long base_pte, uaddr, ioaddr, *ptab; - - ptab = cell_iommu_alloc_ptab(iommu, fbase, fsize, dbase, dsize, 24); - - dma_iommu_fixed_base = fbase; - - pr_debug("iommu: mapping 0x%lx pages from 0x%lx\n", fsize, fbase); - - base_pte = CBE_IOPTE_PP_W | CBE_IOPTE_PP_R | CBE_IOPTE_M | - (cell_iommu_get_ioid(np) & CBE_IOPTE_IOID_Mask); - - if (iommu_fixed_is_weak) - pr_info("IOMMU: Using weak ordering for fixed mapping\n"); - else { - pr_info("IOMMU: Using strong ordering for fixed mapping\n"); - base_pte |= CBE_IOPTE_SO_RW; - } - - for (uaddr = 0; uaddr < fsize; uaddr += (1 << 24)) { - /* Don't touch the dynamic region */ - ioaddr = uaddr + fbase; - if (ioaddr >= dbase && ioaddr < (dbase + dsize)) { - pr_debug("iommu: fixed/dynamic overlap, skipping\n"); - continue; - } - - insert_16M_pte(uaddr, ptab, base_pte); - } - - mb(); -} - -static int __init cell_iommu_fixed_mapping_init(void) -{ - unsigned long dbase, dsize, fbase, fsize, hbase, hend; - struct cbe_iommu *iommu; - struct device_node *np; - - /* The fixed mapping is only supported on axon machines */ - np = of_find_node_by_name(NULL, "axon"); - of_node_put(np); - - if (!np) { - pr_debug("iommu: fixed mapping disabled, no axons found\n"); - return -1; - } - - /* We must have dma-ranges properties for fixed mapping to work */ - np = of_find_node_with_property(NULL, "dma-ranges"); - of_node_put(np); - - if (!np) { - pr_debug("iommu: no dma-ranges found, no fixed mapping\n"); - return -1; - } - - /* The default setup is to have the fixed mapping sit after the - * dynamic region, so find the top of the largest IOMMU window - * on any axon, then add the size of RAM and that's our max value. - * If that is > 32GB we have to do other shennanigans. - */ - fbase = 0; - for_each_node_by_name(np, "axon") { - cell_iommu_get_window(np, &dbase, &dsize); - fbase = max(fbase, dbase + dsize); - } - - fbase = ALIGN(fbase, 1 << IO_SEGMENT_SHIFT); - fsize = memblock_phys_mem_size(); - - if ((fbase + fsize) <= 0x800000000ul) - hbase = 0; /* use the device tree window */ - else { - /* If we're over 32 GB we need to cheat. We can't map all of - * RAM with the fixed mapping, and also fit the dynamic - * region. So try to place the dynamic region where the hash - * table sits, drivers never need to DMA to it, we don't - * need a fixed mapping for that area. - */ - if (!htab_address) { - pr_debug("iommu: htab is NULL, on LPAR? Huh?\n"); - return -1; - } - hbase = __pa(htab_address); - hend = hbase + htab_size_bytes; - - /* The window must start and end on a segment boundary */ - if ((hbase != ALIGN(hbase, 1 << IO_SEGMENT_SHIFT)) || - (hend != ALIGN(hend, 1 << IO_SEGMENT_SHIFT))) { - pr_debug("iommu: hash window not segment aligned\n"); - return -1; - } - - /* Check the hash window fits inside the real DMA window */ - for_each_node_by_name(np, "axon") { - cell_iommu_get_window(np, &dbase, &dsize); - - if (hbase < dbase || (hend > (dbase + dsize))) { - pr_debug("iommu: hash window doesn't fit in" - "real DMA window\n"); - of_node_put(np); - return -1; - } - } - - fbase = 0; - } - - /* Setup the dynamic regions */ - for_each_node_by_name(np, "axon") { - iommu = cell_iommu_alloc(np); - BUG_ON(!iommu); - - if (hbase == 0) - cell_iommu_get_window(np, &dbase, &dsize); - else { - dbase = hbase; - dsize = htab_size_bytes; - } - - printk(KERN_DEBUG "iommu: node %d, dynamic window 0x%lx-0x%lx " - "fixed window 0x%lx-0x%lx\n", iommu->nid, dbase, - dbase + dsize, fbase, fbase + fsize); - - cell_iommu_setup_stab(iommu, dbase, dsize, fbase, fsize); - iommu->ptab = cell_iommu_alloc_ptab(iommu, dbase, dsize, 0, 0, - IOMMU_PAGE_SHIFT_4K); - cell_iommu_setup_fixed_ptab(iommu, np, dbase, dsize, - fbase, fsize); - cell_iommu_enable_hardware(iommu); - cell_iommu_setup_window(iommu, np, dbase, dsize, 0); - } - - cell_pci_controller_ops.iommu_bypass_supported = - cell_pci_iommu_bypass_supported; - return 0; -} - -static int iommu_fixed_disabled; - -static int __init setup_iommu_fixed(char *str) -{ - struct device_node *pciep; - - if (strcmp(str, "off") == 0) - iommu_fixed_disabled = 1; - - /* If we can find a pcie-endpoint in the device tree assume that - * we're on a triblade or a CAB so by default the fixed mapping - * should be set to be weakly ordered; but only if the boot - * option WASN'T set for strong ordering - */ - pciep = of_find_node_by_type(NULL, "pcie-endpoint"); - - if (strcmp(str, "weak") == 0 || (pciep && strcmp(str, "strong") != 0)) - iommu_fixed_is_weak = true; - - of_node_put(pciep); - - return 1; -} -__setup("iommu_fixed=", setup_iommu_fixed); - -static int __init cell_iommu_init(void) -{ - struct device_node *np; - - /* If IOMMU is disabled or we have little enough RAM to not need - * to enable it, we setup a direct mapping. - * - * Note: should we make sure we have the IOMMU actually disabled ? - */ - if (iommu_is_off || - (!iommu_force_on && memblock_end_of_DRAM() <= 0x80000000ull)) - if (cell_iommu_init_disabled() == 0) - goto bail; - - /* Setup various callbacks */ - cell_pci_controller_ops.dma_dev_setup = cell_pci_dma_dev_setup; - - if (!iommu_fixed_disabled && cell_iommu_fixed_mapping_init() == 0) - goto done; - - /* Create an iommu for each /axon node. */ - for_each_node_by_name(np, "axon") { - if (np->parent == NULL || np->parent->parent != NULL) - continue; - cell_iommu_init_one(np, 0); - } - - /* Create an iommu for each toplevel /pci-internal node for - * old hardware/firmware - */ - for_each_node_by_name(np, "pci-internal") { - if (np->parent == NULL || np->parent->parent != NULL) - continue; - cell_iommu_init_one(np, SPIDER_DMA_OFFSET); - } - done: - /* Setup default PCI iommu ops */ - set_pci_dma_ops(&dma_iommu_ops); - cell_iommu_enabled = true; - bail: - /* Register callbacks on OF platform device addition/removal - * to handle linking them to the right DMA operations - */ - bus_register_notifier(&platform_bus_type, &cell_of_bus_notifier); - - return 0; -} -machine_arch_initcall(cell, cell_iommu_init); diff --git a/arch/powerpc/platforms/cell/pervasive.c b/arch/powerpc/platforms/cell/pervasive.c deleted file mode 100644 index 58d967ee38b3..000000000000 --- a/arch/powerpc/platforms/cell/pervasive.c +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * CBE Pervasive Monitor and Debug - * - * (C) Copyright IBM Corporation 2005 - * - * Authors: Maximino Aguilar (maguilar@us.ibm.com) - * Michael N. Day (mnday@us.ibm.com) - */ - -#undef DEBUG - -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/percpu.h> -#include <linux/types.h> -#include <linux/kallsyms.h> -#include <linux/pgtable.h> - -#include <asm/io.h> -#include <asm/machdep.h> -#include <asm/reg.h> -#include <asm/cell-regs.h> -#include <asm/cpu_has_feature.h> - -#include "pervasive.h" -#include "ras.h" - -static void cbe_power_save(void) -{ - unsigned long ctrl, thread_switch_control; - - /* Ensure our interrupt state is properly tracked */ - if (!prep_irq_for_idle()) - return; - - ctrl = mfspr(SPRN_CTRLF); - - /* Enable DEC and EE interrupt request */ - thread_switch_control = mfspr(SPRN_TSC_CELL); - thread_switch_control |= TSC_CELL_EE_ENABLE | TSC_CELL_EE_BOOST; - - switch (ctrl & CTRL_CT) { - case CTRL_CT0: - thread_switch_control |= TSC_CELL_DEC_ENABLE_0; - break; - case CTRL_CT1: - thread_switch_control |= TSC_CELL_DEC_ENABLE_1; - break; - default: - printk(KERN_WARNING "%s: unknown configuration\n", - __func__); - break; - } - mtspr(SPRN_TSC_CELL, thread_switch_control); - - /* - * go into low thread priority, medium priority will be - * restored for us after wake-up. - */ - HMT_low(); - - /* - * atomically disable thread execution and runlatch. - * External and Decrementer exceptions are still handled when the - * thread is disabled but now enter in cbe_system_reset_exception() - */ - ctrl &= ~(CTRL_RUNLATCH | CTRL_TE); - mtspr(SPRN_CTRLT, ctrl); - - /* Re-enable interrupts in MSR */ - __hard_irq_enable(); -} - -static int cbe_system_reset_exception(struct pt_regs *regs) -{ - switch (regs->msr & SRR1_WAKEMASK) { - case SRR1_WAKEDEC: - set_dec(1); - break; - case SRR1_WAKEEE: - /* - * Handle these when interrupts get re-enabled and we take - * them as regular exceptions. We are in an NMI context - * and can't handle these here. - */ - break; - case SRR1_WAKEMT: - return cbe_sysreset_hack(); -#ifdef CONFIG_CBE_RAS - case SRR1_WAKESYSERR: - cbe_system_error_exception(regs); - break; - case SRR1_WAKETHERM: - cbe_thermal_exception(regs); - break; -#endif /* CONFIG_CBE_RAS */ - default: - /* do system reset */ - return 0; - } - /* everything handled */ - return 1; -} - -void __init cbe_pervasive_init(void) -{ - int cpu; - - if (!cpu_has_feature(CPU_FTR_PAUSE_ZERO)) - return; - - for_each_possible_cpu(cpu) { - struct cbe_pmd_regs __iomem *regs = cbe_get_cpu_pmd_regs(cpu); - if (!regs) - continue; - - /* Enable Pause(0) control bit */ - out_be64(®s->pmcr, in_be64(®s->pmcr) | - CBE_PMD_PAUSE_ZERO_CONTROL); - } - - ppc_md.power_save = cbe_power_save; - ppc_md.system_reset_exception = cbe_system_reset_exception; -} diff --git a/arch/powerpc/platforms/cell/pervasive.h b/arch/powerpc/platforms/cell/pervasive.h deleted file mode 100644 index 0da74ab10716..000000000000 --- a/arch/powerpc/platforms/cell/pervasive.h +++ /dev/null @@ -1,26 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Cell Pervasive Monitor and Debug interface and HW structures - * - * (C) Copyright IBM Corporation 2005 - * - * Authors: Maximino Aguilar (maguilar@us.ibm.com) - * David J. Erb (djerb@us.ibm.com) - */ - - -#ifndef PERVASIVE_H -#define PERVASIVE_H - -extern void cbe_pervasive_init(void); - -#ifdef CONFIG_PPC_IBM_CELL_RESETBUTTON -extern int cbe_sysreset_hack(void); -#else -static inline int cbe_sysreset_hack(void) -{ - return 1; -} -#endif /* CONFIG_PPC_IBM_CELL_RESETBUTTON */ - -#endif diff --git a/arch/powerpc/platforms/cell/pmu.c b/arch/powerpc/platforms/cell/pmu.c deleted file mode 100644 index b207a7f99be5..000000000000 --- a/arch/powerpc/platforms/cell/pmu.c +++ /dev/null @@ -1,412 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Cell Broadband Engine Performance Monitor - * - * (C) Copyright IBM Corporation 2001,2006 - * - * Author: - * David Erb (djerb@us.ibm.com) - * Kevin Corry (kevcorry@us.ibm.com) - */ - -#include <linux/interrupt.h> -#include <linux/irqdomain.h> -#include <linux/types.h> -#include <linux/export.h> -#include <asm/io.h> -#include <asm/irq_regs.h> -#include <asm/machdep.h> -#include <asm/pmc.h> -#include <asm/reg.h> -#include <asm/spu.h> -#include <asm/cell-regs.h> - -#include "interrupt.h" - -/* - * When writing to write-only mmio addresses, save a shadow copy. All of the - * registers are 32-bit, but stored in the upper-half of a 64-bit field in - * pmd_regs. - */ - -#define WRITE_WO_MMIO(reg, x) \ - do { \ - u32 _x = (x); \ - struct cbe_pmd_regs __iomem *pmd_regs; \ - struct cbe_pmd_shadow_regs *shadow_regs; \ - pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ - shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ - out_be64(&(pmd_regs->reg), (((u64)_x) << 32)); \ - shadow_regs->reg = _x; \ - } while (0) - -#define READ_SHADOW_REG(val, reg) \ - do { \ - struct cbe_pmd_shadow_regs *shadow_regs; \ - shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \ - (val) = shadow_regs->reg; \ - } while (0) - -#define READ_MMIO_UPPER32(val, reg) \ - do { \ - struct cbe_pmd_regs __iomem *pmd_regs; \ - pmd_regs = cbe_get_cpu_pmd_regs(cpu); \ - (val) = (u32)(in_be64(&pmd_regs->reg) >> 32); \ - } while (0) - -/* - * Physical counter registers. - * Each physical counter can act as one 32-bit counter or two 16-bit counters. - */ - -u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr) -{ - u32 val_in_latch, val = 0; - - if (phys_ctr < NR_PHYS_CTRS) { - READ_SHADOW_REG(val_in_latch, counter_value_in_latch); - - /* Read the latch or the actual counter, whichever is newer. */ - if (val_in_latch & (1 << phys_ctr)) { - READ_SHADOW_REG(val, pm_ctr[phys_ctr]); - } else { - READ_MMIO_UPPER32(val, pm_ctr[phys_ctr]); - } - } - - return val; -} -EXPORT_SYMBOL_GPL(cbe_read_phys_ctr); - -void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val) -{ - struct cbe_pmd_shadow_regs *shadow_regs; - u32 pm_ctrl; - - if (phys_ctr < NR_PHYS_CTRS) { - /* Writing to a counter only writes to a hardware latch. - * The new value is not propagated to the actual counter - * until the performance monitor is enabled. - */ - WRITE_WO_MMIO(pm_ctr[phys_ctr], val); - - pm_ctrl = cbe_read_pm(cpu, pm_control); - if (pm_ctrl & CBE_PM_ENABLE_PERF_MON) { - /* The counters are already active, so we need to - * rewrite the pm_control register to "re-enable" - * the PMU. - */ - cbe_write_pm(cpu, pm_control, pm_ctrl); - } else { - shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); - shadow_regs->counter_value_in_latch |= (1 << phys_ctr); - } - } -} -EXPORT_SYMBOL_GPL(cbe_write_phys_ctr); - -/* - * "Logical" counter registers. - * These will read/write 16-bits or 32-bits depending on the - * current size of the counter. Counters 4 - 7 are always 16-bit. - */ - -u32 cbe_read_ctr(u32 cpu, u32 ctr) -{ - u32 val; - u32 phys_ctr = ctr & (NR_PHYS_CTRS - 1); - - val = cbe_read_phys_ctr(cpu, phys_ctr); - - if (cbe_get_ctr_size(cpu, phys_ctr) == 16) - val = (ctr < NR_PHYS_CTRS) ? (val >> 16) : (val & 0xffff); - - return val; -} -EXPORT_SYMBOL_GPL(cbe_read_ctr); - -void cbe_write_ctr(u32 cpu, u32 ctr, u32 val) -{ - u32 phys_ctr; - u32 phys_val; - - phys_ctr = ctr & (NR_PHYS_CTRS - 1); - - if (cbe_get_ctr_size(cpu, phys_ctr) == 16) { - phys_val = cbe_read_phys_ctr(cpu, phys_ctr); - - if (ctr < NR_PHYS_CTRS) - val = (val << 16) | (phys_val & 0xffff); - else - val = (val & 0xffff) | (phys_val & 0xffff0000); - } - - cbe_write_phys_ctr(cpu, phys_ctr, val); -} -EXPORT_SYMBOL_GPL(cbe_write_ctr); - -/* - * Counter-control registers. - * Each "logical" counter has a corresponding control register. - */ - -u32 cbe_read_pm07_control(u32 cpu, u32 ctr) -{ - u32 pm07_control = 0; - - if (ctr < NR_CTRS) - READ_SHADOW_REG(pm07_control, pm07_control[ctr]); - - return pm07_control; -} -EXPORT_SYMBOL_GPL(cbe_read_pm07_control); - -void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val) -{ - if (ctr < NR_CTRS) - WRITE_WO_MMIO(pm07_control[ctr], val); -} -EXPORT_SYMBOL_GPL(cbe_write_pm07_control); - -/* - * Other PMU control registers. Most of these are write-only. - */ - -u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg) -{ - u32 val = 0; - - switch (reg) { - case group_control: - READ_SHADOW_REG(val, group_control); - break; - - case debug_bus_control: - READ_SHADOW_REG(val, debug_bus_control); - break; - - case trace_address: - READ_MMIO_UPPER32(val, trace_address); - break; - - case ext_tr_timer: - READ_SHADOW_REG(val, ext_tr_timer); - break; - - case pm_status: - READ_MMIO_UPPER32(val, pm_status); - break; - - case pm_control: - READ_SHADOW_REG(val, pm_control); - break; - - case pm_interval: - READ_MMIO_UPPER32(val, pm_interval); - break; - - case pm_start_stop: - READ_SHADOW_REG(val, pm_start_stop); - break; - } - - return val; -} -EXPORT_SYMBOL_GPL(cbe_read_pm); - -void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val) -{ - switch (reg) { - case group_control: - WRITE_WO_MMIO(group_control, val); - break; - - case debug_bus_control: - WRITE_WO_MMIO(debug_bus_control, val); - break; - - case trace_address: - WRITE_WO_MMIO(trace_address, val); - break; - - case ext_tr_timer: - WRITE_WO_MMIO(ext_tr_timer, val); - break; - - case pm_status: - WRITE_WO_MMIO(pm_status, val); - break; - - case pm_control: - WRITE_WO_MMIO(pm_control, val); - break; - - case pm_interval: - WRITE_WO_MMIO(pm_interval, val); - break; - - case pm_start_stop: - WRITE_WO_MMIO(pm_start_stop, val); - break; - } -} -EXPORT_SYMBOL_GPL(cbe_write_pm); - -/* - * Get/set the size of a physical counter to either 16 or 32 bits. - */ - -u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr) -{ - u32 pm_ctrl, size = 0; - - if (phys_ctr < NR_PHYS_CTRS) { - pm_ctrl = cbe_read_pm(cpu, pm_control); - size = (pm_ctrl & CBE_PM_16BIT_CTR(phys_ctr)) ? 16 : 32; - } - - return size; -} -EXPORT_SYMBOL_GPL(cbe_get_ctr_size); - -void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size) -{ - u32 pm_ctrl; - - if (phys_ctr < NR_PHYS_CTRS) { - pm_ctrl = cbe_read_pm(cpu, pm_control); - switch (ctr_size) { - case 16: - pm_ctrl |= CBE_PM_16BIT_CTR(phys_ctr); - break; - - case 32: - pm_ctrl &= ~CBE_PM_16BIT_CTR(phys_ctr); - break; - } - cbe_write_pm(cpu, pm_control, pm_ctrl); - } -} -EXPORT_SYMBOL_GPL(cbe_set_ctr_size); - -/* - * Enable/disable the entire performance monitoring unit. - * When we enable the PMU, all pending writes to counters get committed. - */ - -void cbe_enable_pm(u32 cpu) -{ - struct cbe_pmd_shadow_regs *shadow_regs; - u32 pm_ctrl; - - shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); - shadow_regs->counter_value_in_latch = 0; - - pm_ctrl = cbe_read_pm(cpu, pm_control) | CBE_PM_ENABLE_PERF_MON; - cbe_write_pm(cpu, pm_control, pm_ctrl); -} -EXPORT_SYMBOL_GPL(cbe_enable_pm); - -void cbe_disable_pm(u32 cpu) -{ - u32 pm_ctrl; - pm_ctrl = cbe_read_pm(cpu, pm_control) & ~CBE_PM_ENABLE_PERF_MON; - cbe_write_pm(cpu, pm_control, pm_ctrl); -} -EXPORT_SYMBOL_GPL(cbe_disable_pm); - -/* - * Reading from the trace_buffer. - * The trace buffer is two 64-bit registers. Reading from - * the second half automatically increments the trace_address. - */ - -void cbe_read_trace_buffer(u32 cpu, u64 *buf) -{ - struct cbe_pmd_regs __iomem *pmd_regs = cbe_get_cpu_pmd_regs(cpu); - - *buf++ = in_be64(&pmd_regs->trace_buffer_0_63); - *buf++ = in_be64(&pmd_regs->trace_buffer_64_127); -} -EXPORT_SYMBOL_GPL(cbe_read_trace_buffer); - -/* - * Enabling/disabling interrupts for the entire performance monitoring unit. - */ - -u32 cbe_get_and_clear_pm_interrupts(u32 cpu) -{ - /* Reading pm_status clears the interrupt bits. */ - return cbe_read_pm(cpu, pm_status); -} -EXPORT_SYMBOL_GPL(cbe_get_and_clear_pm_interrupts); - -void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask) -{ - /* Set which node and thread will handle the next interrupt. */ - iic_set_interrupt_routing(cpu, thread, 0); - - /* Enable the interrupt bits in the pm_status register. */ - if (mask) - cbe_write_pm(cpu, pm_status, mask); -} -EXPORT_SYMBOL_GPL(cbe_enable_pm_interrupts); - -void cbe_disable_pm_interrupts(u32 cpu) -{ - cbe_get_and_clear_pm_interrupts(cpu); - cbe_write_pm(cpu, pm_status, 0); -} -EXPORT_SYMBOL_GPL(cbe_disable_pm_interrupts); - -static irqreturn_t cbe_pm_irq(int irq, void *dev_id) -{ - perf_irq(get_irq_regs()); - return IRQ_HANDLED; -} - -static int __init cbe_init_pm_irq(void) -{ - unsigned int irq; - int rc, node; - - for_each_online_node(node) { - irq = irq_create_mapping(NULL, IIC_IRQ_IOEX_PMI | - (node << IIC_IRQ_NODE_SHIFT)); - if (!irq) { - printk("ERROR: Unable to allocate irq for node %d\n", - node); - return -EINVAL; - } - - rc = request_irq(irq, cbe_pm_irq, - 0, "cbe-pmu-0", NULL); - if (rc) { - printk("ERROR: Request for irq on node %d failed\n", - node); - return rc; - } - } - - return 0; -} -machine_arch_initcall(cell, cbe_init_pm_irq); - -void cbe_sync_irq(int node) -{ - unsigned int irq; - - irq = irq_find_mapping(NULL, - IIC_IRQ_IOEX_PMI - | (node << IIC_IRQ_NODE_SHIFT)); - - if (!irq) { - printk(KERN_WARNING "ERROR, unable to get existing irq %d " \ - "for node %d\n", irq, node); - return; - } - - synchronize_irq(irq); -} -EXPORT_SYMBOL_GPL(cbe_sync_irq); - diff --git a/arch/powerpc/platforms/cell/ras.c b/arch/powerpc/platforms/cell/ras.c deleted file mode 100644 index f6b87926530c..000000000000 --- a/arch/powerpc/platforms/cell/ras.c +++ /dev/null @@ -1,352 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright 2006-2008, IBM Corporation. - */ - -#undef DEBUG - -#include <linux/types.h> -#include <linux/kernel.h> -#include <linux/slab.h> -#include <linux/smp.h> -#include <linux/reboot.h> -#include <linux/kexec.h> -#include <linux/crash_dump.h> -#include <linux/of.h> - -#include <asm/kexec.h> -#include <asm/reg.h> -#include <asm/io.h> -#include <asm/machdep.h> -#include <asm/rtas.h> -#include <asm/cell-regs.h> - -#include "ras.h" -#include "pervasive.h" - -static void dump_fir(int cpu) -{ - struct cbe_pmd_regs __iomem *pregs = cbe_get_cpu_pmd_regs(cpu); - struct cbe_iic_regs __iomem *iregs = cbe_get_cpu_iic_regs(cpu); - - if (pregs == NULL) - return; - - /* Todo: do some nicer parsing of bits and based on them go down - * to other sub-units FIRs and not only IIC - */ - printk(KERN_ERR "Global Checkstop FIR : 0x%016llx\n", - in_be64(&pregs->checkstop_fir)); - printk(KERN_ERR "Global Recoverable FIR : 0x%016llx\n", - in_be64(&pregs->checkstop_fir)); - printk(KERN_ERR "Global MachineCheck FIR : 0x%016llx\n", - in_be64(&pregs->spec_att_mchk_fir)); - - if (iregs == NULL) - return; - printk(KERN_ERR "IOC FIR : 0x%016llx\n", - in_be64(&iregs->ioc_fir)); - -} - -DEFINE_INTERRUPT_HANDLER(cbe_system_error_exception) -{ - int cpu = smp_processor_id(); - - printk(KERN_ERR "System Error Interrupt on CPU %d !\n", cpu); - dump_fir(cpu); - dump_stack(); -} - -DEFINE_INTERRUPT_HANDLER(cbe_maintenance_exception) -{ - int cpu = smp_processor_id(); - - /* - * Nothing implemented for the maintenance interrupt at this point - */ - - printk(KERN_ERR "Unhandled Maintenance interrupt on CPU %d !\n", cpu); - dump_stack(); -} - -DEFINE_INTERRUPT_HANDLER(cbe_thermal_exception) -{ - int cpu = smp_processor_id(); - - /* - * Nothing implemented for the thermal interrupt at this point - */ - - printk(KERN_ERR "Unhandled Thermal interrupt on CPU %d !\n", cpu); - dump_stack(); -} - -static int cbe_machine_check_handler(struct pt_regs *regs) -{ - int cpu = smp_processor_id(); - - printk(KERN_ERR "Machine Check Interrupt on CPU %d !\n", cpu); - dump_fir(cpu); - - /* No recovery from this code now, lets continue */ - return 0; -} - -struct ptcal_area { - struct list_head list; - int nid; - int order; - struct page *pages; -}; - -static LIST_HEAD(ptcal_list); - -static int ptcal_start_tok, ptcal_stop_tok; - -static int __init cbe_ptcal_enable_on_node(int nid, int order) -{ - struct ptcal_area *area; - int ret = -ENOMEM; - unsigned long addr; - - if (is_kdump_kernel()) - rtas_call(ptcal_stop_tok, 1, 1, NULL, nid); - - area = kmalloc(sizeof(*area), GFP_KERNEL); - if (!area) - goto out_err; - - area->nid = nid; - area->order = order; - area->pages = __alloc_pages_node(area->nid, - GFP_KERNEL|__GFP_THISNODE, - area->order); - - if (!area->pages) { - printk(KERN_WARNING "%s: no page on node %d\n", - __func__, area->nid); - goto out_free_area; - } - - /* - * We move the ptcal area to the middle of the allocated - * page, in order to avoid prefetches in memcpy and similar - * functions stepping on it. - */ - addr = __pa(page_address(area->pages)) + (PAGE_SIZE >> 1); - printk(KERN_DEBUG "%s: enabling PTCAL on node %d address=0x%016lx\n", - __func__, area->nid, addr); - - ret = -EIO; - if (rtas_call(ptcal_start_tok, 3, 1, NULL, area->nid, - (unsigned int)(addr >> 32), - (unsigned int)(addr & 0xffffffff))) { - printk(KERN_ERR "%s: error enabling PTCAL on node %d!\n", - __func__, nid); - goto out_free_pages; - } - - list_add(&area->list, &ptcal_list); - - return 0; - -out_free_pages: - __free_pages(area->pages, area->order); -out_free_area: - kfree(area); -out_err: - return ret; -} - -static int __init cbe_ptcal_enable(void) -{ - const u32 *size; - struct device_node *np; - int order, found_mic = 0; - - np = of_find_node_by_path("/rtas"); - if (!np) - return -ENODEV; - - size = of_get_property(np, "ibm,cbe-ptcal-size", NULL); - if (!size) { - of_node_put(np); - return -ENODEV; - } - - pr_debug("%s: enabling PTCAL, size = 0x%x\n", __func__, *size); - order = get_order(*size); - of_node_put(np); - - /* support for malta device trees, with be@/mic@ nodes */ - for_each_node_by_type(np, "mic-tm") { - cbe_ptcal_enable_on_node(of_node_to_nid(np), order); - found_mic = 1; - } - - if (found_mic) - return 0; - - /* support for older device tree - use cpu nodes */ - for_each_node_by_type(np, "cpu") { - const u32 *nid = of_get_property(np, "node-id", NULL); - if (!nid) { - printk(KERN_ERR "%s: node %pOF is missing node-id?\n", - __func__, np); - continue; - } - cbe_ptcal_enable_on_node(*nid, order); - found_mic = 1; - } - - return found_mic ? 0 : -ENODEV; -} - -static int cbe_ptcal_disable(void) -{ - struct ptcal_area *area, *tmp; - int ret = 0; - - pr_debug("%s: disabling PTCAL\n", __func__); - - list_for_each_entry_safe(area, tmp, &ptcal_list, list) { - /* disable ptcal on this node */ - if (rtas_call(ptcal_stop_tok, 1, 1, NULL, area->nid)) { - printk(KERN_ERR "%s: error disabling PTCAL " - "on node %d!\n", __func__, - area->nid); - ret = -EIO; - continue; - } - - /* ensure we can access the PTCAL area */ - memset(page_address(area->pages), 0, - 1 << (area->order + PAGE_SHIFT)); - - /* clean up */ - list_del(&area->list); - __free_pages(area->pages, area->order); - kfree(area); - } - - return ret; -} - -static int cbe_ptcal_notify_reboot(struct notifier_block *nb, - unsigned long code, void *data) -{ - return cbe_ptcal_disable(); -} - -static void cbe_ptcal_crash_shutdown(void) -{ - cbe_ptcal_disable(); -} - -static struct notifier_block cbe_ptcal_reboot_notifier = { - .notifier_call = cbe_ptcal_notify_reboot -}; - -#ifdef CONFIG_PPC_IBM_CELL_RESETBUTTON -static int sysreset_hack; - -static int __init cbe_sysreset_init(void) -{ - struct cbe_pmd_regs __iomem *regs; - - sysreset_hack = of_machine_is_compatible("IBM,CBPLUS-1.0"); - if (!sysreset_hack) - return 0; - - regs = cbe_get_cpu_pmd_regs(0); - if (!regs) - return 0; - - /* Enable JTAG system-reset hack */ - out_be32(®s->fir_mode_reg, - in_be32(®s->fir_mode_reg) | - CBE_PMD_FIR_MODE_M8); - - return 0; -} -device_initcall(cbe_sysreset_init); - -int cbe_sysreset_hack(void) -{ - struct cbe_pmd_regs __iomem *regs; - - /* - * The BMC can inject user triggered system reset exceptions, - * but cannot set the system reset reason in srr1, - * so check an extra register here. - */ - if (sysreset_hack && (smp_processor_id() == 0)) { - regs = cbe_get_cpu_pmd_regs(0); - if (!regs) - return 0; - if (in_be64(®s->ras_esc_0) & 0x0000ffff) { - out_be64(®s->ras_esc_0, 0); - return 0; - } - } - return 1; -} -#endif /* CONFIG_PPC_IBM_CELL_RESETBUTTON */ - -static int __init cbe_ptcal_init(void) -{ - int ret; - ptcal_start_tok = rtas_function_token(RTAS_FN_IBM_CBE_START_PTCAL); - ptcal_stop_tok = rtas_function_token(RTAS_FN_IBM_CBE_STOP_PTCAL); - - if (ptcal_start_tok == RTAS_UNKNOWN_SERVICE - || ptcal_stop_tok == RTAS_UNKNOWN_SERVICE) - return -ENODEV; - - ret = register_reboot_notifier(&cbe_ptcal_reboot_notifier); - if (ret) - goto out1; - - ret = crash_shutdown_register(&cbe_ptcal_crash_shutdown); - if (ret) - goto out2; - - return cbe_ptcal_enable(); - -out2: - unregister_reboot_notifier(&cbe_ptcal_reboot_notifier); -out1: - printk(KERN_ERR "Can't disable PTCAL, so not enabling\n"); - return ret; -} - -arch_initcall(cbe_ptcal_init); - -void __init cbe_ras_init(void) -{ - unsigned long hid0; - - /* - * Enable System Error & thermal interrupts and wakeup conditions - */ - - hid0 = mfspr(SPRN_HID0); - hid0 |= HID0_CBE_THERM_INT_EN | HID0_CBE_THERM_WAKEUP | - HID0_CBE_SYSERR_INT_EN | HID0_CBE_SYSERR_WAKEUP; - mtspr(SPRN_HID0, hid0); - mb(); - - /* - * Install machine check handler. Leave setting of precise mode to - * what the firmware did for now - */ - ppc_md.machine_check_exception = cbe_machine_check_handler; - mb(); - - /* - * For now, we assume that IOC_FIR is already set to forward some - * error conditions to the System Error handler. If that is not true - * then it will have to be fixed up here. - */ -} diff --git a/arch/powerpc/platforms/cell/ras.h b/arch/powerpc/platforms/cell/ras.h deleted file mode 100644 index 226dbd48efad..000000000000 --- a/arch/powerpc/platforms/cell/ras.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef RAS_H -#define RAS_H - -#include <asm/interrupt.h> - -DECLARE_INTERRUPT_HANDLER(cbe_system_error_exception); -DECLARE_INTERRUPT_HANDLER(cbe_maintenance_exception); -DECLARE_INTERRUPT_HANDLER(cbe_thermal_exception); - -extern void cbe_ras_init(void); - -#endif /* RAS_H */ diff --git a/arch/powerpc/platforms/cell/setup.c b/arch/powerpc/platforms/cell/setup.c deleted file mode 100644 index f64a1ef98aa8..000000000000 --- a/arch/powerpc/platforms/cell/setup.c +++ /dev/null @@ -1,274 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * linux/arch/powerpc/platforms/cell/cell_setup.c - * - * Copyright (C) 1995 Linus Torvalds - * Adapted from 'alpha' version by Gary Thomas - * Modified by Cort Dougan (cort@cs.nmt.edu) - * Modified by PPC64 Team, IBM Corp - * Modified by Cell Team, IBM Deutschland Entwicklung GmbH - */ -#undef DEBUG - -#include <linux/sched.h> -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/stddef.h> -#include <linux/export.h> -#include <linux/unistd.h> -#include <linux/user.h> -#include <linux/reboot.h> -#include <linux/init.h> -#include <linux/delay.h> -#include <linux/irq.h> -#include <linux/seq_file.h> -#include <linux/root_dev.h> -#include <linux/console.h> -#include <linux/mutex.h> -#include <linux/memory_hotplug.h> -#include <linux/of_platform.h> -#include <linux/platform_device.h> - -#include <asm/mmu.h> -#include <asm/processor.h> -#include <asm/io.h> -#include <asm/rtas.h> -#include <asm/pci-bridge.h> -#include <asm/iommu.h> -#include <asm/dma.h> -#include <asm/machdep.h> -#include <asm/time.h> -#include <asm/nvram.h> -#include <asm/cputable.h> -#include <asm/ppc-pci.h> -#include <asm/irq.h> -#include <asm/spu.h> -#include <asm/spu_priv1.h> -#include <asm/udbg.h> -#include <asm/mpic.h> -#include <asm/cell-regs.h> -#include <asm/io-workarounds.h> - -#include "cell.h" -#include "interrupt.h" -#include "pervasive.h" -#include "ras.h" - -#ifdef DEBUG -#define DBG(fmt...) udbg_printf(fmt) -#else -#define DBG(fmt...) -#endif - -static void cell_show_cpuinfo(struct seq_file *m) -{ - struct device_node *root; - const char *model = ""; - - root = of_find_node_by_path("/"); - if (root) - model = of_get_property(root, "model", NULL); - seq_printf(m, "machine\t\t: CHRP %s\n", model); - of_node_put(root); -} - -static void cell_progress(char *s, unsigned short hex) -{ - printk("*** %04x : %s\n", hex, s ? s : ""); -} - -static void cell_fixup_pcie_rootcomplex(struct pci_dev *dev) -{ - struct pci_controller *hose; - const char *s; - int i; - - if (!machine_is(cell)) - return; - - /* We're searching for a direct child of the PHB */ - if (dev->bus->self != NULL || dev->devfn != 0) - return; - - hose = pci_bus_to_host(dev->bus); - if (hose == NULL) - return; - - /* Only on PCIE */ - if (!of_device_is_compatible(hose->dn, "pciex")) - return; - - /* And only on axon */ - s = of_get_property(hose->dn, "model", NULL); - if (!s || strcmp(s, "Axon") != 0) - return; - - for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) { - dev->resource[i].start = dev->resource[i].end = 0; - dev->resource[i].flags = 0; - } - - printk(KERN_DEBUG "PCI: Hiding resources on Axon PCIE RC %s\n", - pci_name(dev)); -} -DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, cell_fixup_pcie_rootcomplex); - -static int cell_setup_phb(struct pci_controller *phb) -{ - const char *model; - struct device_node *np; - - int rc = rtas_setup_phb(phb); - if (rc) - return rc; - - phb->controller_ops = cell_pci_controller_ops; - - np = phb->dn; - model = of_get_property(np, "model", NULL); - if (model == NULL || !of_node_name_eq(np, "pci")) - return 0; - - /* Setup workarounds for spider */ - if (strcmp(model, "Spider")) - return 0; - - iowa_register_bus(phb, &spiderpci_ops, &spiderpci_iowa_init, - (void *)SPIDER_PCI_REG_BASE); - return 0; -} - -static const struct of_device_id cell_bus_ids[] __initconst = { - { .type = "soc", }, - { .compatible = "soc", }, - { .type = "spider", }, - { .type = "axon", }, - { .type = "plb5", }, - { .type = "plb4", }, - { .type = "opb", }, - { .type = "ebc", }, - {}, -}; - -static int __init cell_publish_devices(void) -{ - struct device_node *root = of_find_node_by_path("/"); - struct device_node *np; - int node; - - /* Publish OF platform devices for southbridge IOs */ - of_platform_bus_probe(NULL, cell_bus_ids, NULL); - - /* On spider based blades, we need to manually create the OF - * platform devices for the PCI host bridges - */ - for_each_child_of_node(root, np) { - if (!of_node_is_type(np, "pci") && !of_node_is_type(np, "pciex")) - continue; - of_platform_device_create(np, NULL, NULL); - } - - of_node_put(root); - - /* There is no device for the MIC memory controller, thus we create - * a platform device for it to attach the EDAC driver to. - */ - for_each_online_node(node) { - if (cbe_get_cpu_mic_tm_regs(cbe_node_to_cpu(node)) == NULL) - continue; - platform_device_register_simple("cbe-mic", node, NULL, 0); - } - - return 0; -} -machine_subsys_initcall(cell, cell_publish_devices); - -static void __init mpic_init_IRQ(void) -{ - struct device_node *dn; - struct mpic *mpic; - - for_each_node_by_name(dn, "interrupt-controller") { - if (!of_device_is_compatible(dn, "CBEA,platform-open-pic")) - continue; - - /* The MPIC driver will get everything it needs from the - * device-tree, just pass 0 to all arguments - */ - mpic = mpic_alloc(dn, 0, MPIC_SECONDARY | MPIC_NO_RESET, - 0, 0, " MPIC "); - if (mpic == NULL) - continue; - mpic_init(mpic); - } -} - - -static void __init cell_init_irq(void) -{ - iic_init_IRQ(); - spider_init_IRQ(); - mpic_init_IRQ(); -} - -static void __init cell_set_dabrx(void) -{ - mtspr(SPRN_DABRX, DABRX_KERNEL | DABRX_USER); -} - -static void __init cell_setup_arch(void) -{ -#ifdef CONFIG_SPU_BASE - spu_priv1_ops = &spu_priv1_mmio_ops; - spu_management_ops = &spu_management_of_ops; -#endif - - cbe_regs_init(); - - cell_set_dabrx(); - -#ifdef CONFIG_CBE_RAS - cbe_ras_init(); -#endif - -#ifdef CONFIG_SMP - smp_init_cell(); -#endif - /* init to some ~sane value until calibrate_delay() runs */ - loops_per_jiffy = 50000000; - - /* Find and initialize PCI host bridges */ - init_pci_config_tokens(); - - cbe_pervasive_init(); - - mmio_nvram_init(); -} - -static int __init cell_probe(void) -{ - if (!of_machine_is_compatible("IBM,CBEA") && - !of_machine_is_compatible("IBM,CPBW-1.0")) - return 0; - - pm_power_off = rtas_power_off; - - return 1; -} - -define_machine(cell) { - .name = "Cell", - .probe = cell_probe, - .setup_arch = cell_setup_arch, - .show_cpuinfo = cell_show_cpuinfo, - .restart = rtas_restart, - .halt = rtas_halt, - .get_boot_time = rtas_get_boot_time, - .get_rtc_time = rtas_get_rtc_time, - .set_rtc_time = rtas_set_rtc_time, - .progress = cell_progress, - .init_IRQ = cell_init_irq, - .pci_setup_phb = cell_setup_phb, -}; - -struct pci_controller_ops cell_pci_controller_ops; diff --git a/arch/powerpc/platforms/cell/smp.c b/arch/powerpc/platforms/cell/smp.c deleted file mode 100644 index 0e8f20ecca08..000000000000 --- a/arch/powerpc/platforms/cell/smp.c +++ /dev/null @@ -1,162 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * SMP support for BPA machines. - * - * Dave Engebretsen, Peter Bergner, and - * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com - * - * Plus various changes from other IBM teams... - */ - -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/interrupt.h> -#include <linux/delay.h> -#include <linux/init.h> -#include <linux/spinlock.h> -#include <linux/cache.h> -#include <linux/err.h> -#include <linux/device.h> -#include <linux/cpu.h> -#include <linux/pgtable.h> - -#include <asm/ptrace.h> -#include <linux/atomic.h> -#include <asm/irq.h> -#include <asm/page.h> -#include <asm/io.h> -#include <asm/smp.h> -#include <asm/paca.h> -#include <asm/machdep.h> -#include <asm/cputable.h> -#include <asm/firmware.h> -#include <asm/rtas.h> -#include <asm/cputhreads.h> -#include <asm/text-patching.h> - -#include "interrupt.h" -#include <asm/udbg.h> - -#ifdef DEBUG -#define DBG(fmt...) udbg_printf(fmt) -#else -#define DBG(fmt...) -#endif - -/* - * The Primary thread of each non-boot processor was started from the OF client - * interface by prom_hold_cpus and is spinning on secondary_hold_spinloop. - */ -static cpumask_t of_spin_map; - -/** - * smp_startup_cpu() - start the given cpu - * @lcpu: Logical CPU ID of the CPU to be started. - * - * At boot time, there is nothing to do for primary threads which were - * started from Open Firmware. For anything else, call RTAS with the - * appropriate start location. - * - * Returns: - * 0 - failure - * 1 - success - */ -static inline int smp_startup_cpu(unsigned int lcpu) -{ - int status; - unsigned long start_here = - __pa(ppc_function_entry(generic_secondary_smp_init)); - unsigned int pcpu; - int start_cpu; - - if (cpumask_test_cpu(lcpu, &of_spin_map)) - /* Already started by OF and sitting in spin loop */ - return 1; - - pcpu = get_hard_smp_processor_id(lcpu); - - /* - * If the RTAS start-cpu token does not exist then presume the - * cpu is already spinning. - */ - start_cpu = rtas_function_token(RTAS_FN_START_CPU); - if (start_cpu == RTAS_UNKNOWN_SERVICE) - return 1; - - status = rtas_call(start_cpu, 3, 1, NULL, pcpu, start_here, lcpu); - if (status != 0) { - printk(KERN_ERR "start-cpu failed: %i\n", status); - return 0; - } - - return 1; -} - -static void smp_cell_setup_cpu(int cpu) -{ - if (cpu != boot_cpuid) - iic_setup_cpu(); - - /* - * change default DABRX to allow user watchpoints - */ - mtspr(SPRN_DABRX, DABRX_KERNEL | DABRX_USER); -} - -static int smp_cell_kick_cpu(int nr) -{ - if (nr < 0 || nr >= nr_cpu_ids) - return -EINVAL; - - if (!smp_startup_cpu(nr)) - return -ENOENT; - - /* - * The processor is currently spinning, waiting for the - * cpu_start field to become non-zero After we set cpu_start, - * the processor will continue on to secondary_start - */ - paca_ptrs[nr]->cpu_start = 1; - - return 0; -} - -static struct smp_ops_t bpa_iic_smp_ops = { - .message_pass = iic_message_pass, - .probe = iic_request_IPIs, - .kick_cpu = smp_cell_kick_cpu, - .setup_cpu = smp_cell_setup_cpu, - .cpu_bootable = smp_generic_cpu_bootable, -}; - -/* This is called very early */ -void __init smp_init_cell(void) -{ - int i; - - DBG(" -> smp_init_cell()\n"); - - smp_ops = &bpa_iic_smp_ops; - - /* Mark threads which are still spinning in hold loops. */ - if (cpu_has_feature(CPU_FTR_SMT)) { - for_each_present_cpu(i) { - if (cpu_thread_in_core(i) == 0) - cpumask_set_cpu(i, &of_spin_map); - } - } else - cpumask_copy(&of_spin_map, cpu_present_mask); - - cpumask_clear_cpu(boot_cpuid, &of_spin_map); - - /* Non-lpar has additional take/give timebase */ - if (rtas_function_token(RTAS_FN_FREEZE_TIME_BASE) != RTAS_UNKNOWN_SERVICE) { - smp_ops->give_timebase = rtas_give_timebase; - smp_ops->take_timebase = rtas_take_timebase; - } - - DBG(" <- smp_init_cell()\n"); -} diff --git a/arch/powerpc/platforms/cell/spider-pci.c b/arch/powerpc/platforms/cell/spider-pci.c deleted file mode 100644 index 68439445b1c3..000000000000 --- a/arch/powerpc/platforms/cell/spider-pci.c +++ /dev/null @@ -1,170 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * IO workarounds for PCI on Celleb/Cell platform - * - * (C) Copyright 2006-2007 TOSHIBA CORPORATION - */ - -#undef DEBUG - -#include <linux/kernel.h> -#include <linux/of_address.h> -#include <linux/slab.h> -#include <linux/io.h> - -#include <asm/ppc-pci.h> -#include <asm/pci-bridge.h> -#include <asm/io-workarounds.h> - -#define SPIDER_PCI_DISABLE_PREFETCH - -struct spiderpci_iowa_private { - void __iomem *regs; -}; - -static void spiderpci_io_flush(struct iowa_bus *bus) -{ - struct spiderpci_iowa_private *priv; - - priv = bus->private; - in_be32(priv->regs + SPIDER_PCI_DUMMY_READ); - iosync(); -} - -#define SPIDER_PCI_MMIO_READ(name, ret) \ -static ret spiderpci_##name(const PCI_IO_ADDR addr) \ -{ \ - ret val = __do_##name(addr); \ - spiderpci_io_flush(iowa_mem_find_bus(addr)); \ - return val; \ -} - -#define SPIDER_PCI_MMIO_READ_STR(name) \ -static void spiderpci_##name(const PCI_IO_ADDR addr, void *buf, \ - unsigned long count) \ -{ \ - __do_##name(addr, buf, count); \ - spiderpci_io_flush(iowa_mem_find_bus(addr)); \ -} - -SPIDER_PCI_MMIO_READ(readb, u8) -SPIDER_PCI_MMIO_READ(readw, u16) -SPIDER_PCI_MMIO_READ(readl, u32) -SPIDER_PCI_MMIO_READ(readq, u64) -SPIDER_PCI_MMIO_READ(readw_be, u16) -SPIDER_PCI_MMIO_READ(readl_be, u32) -SPIDER_PCI_MMIO_READ(readq_be, u64) -SPIDER_PCI_MMIO_READ_STR(readsb) -SPIDER_PCI_MMIO_READ_STR(readsw) -SPIDER_PCI_MMIO_READ_STR(readsl) - -static void spiderpci_memcpy_fromio(void *dest, const PCI_IO_ADDR src, - unsigned long n) -{ - __do_memcpy_fromio(dest, src, n); - spiderpci_io_flush(iowa_mem_find_bus(src)); -} - -static int __init spiderpci_pci_setup_chip(struct pci_controller *phb, - void __iomem *regs) -{ - void *dummy_page_va; - dma_addr_t dummy_page_da; - -#ifdef SPIDER_PCI_DISABLE_PREFETCH - u32 val = in_be32(regs + SPIDER_PCI_VCI_CNTL_STAT); - pr_debug("SPIDER_IOWA:PVCI_Control_Status was 0x%08x\n", val); - out_be32(regs + SPIDER_PCI_VCI_CNTL_STAT, val | 0x8); -#endif /* SPIDER_PCI_DISABLE_PREFETCH */ - - /* setup dummy read */ - /* - * On CellBlade, we can't know that which XDR memory is used by - * kmalloc() to allocate dummy_page_va. - * In order to improve the performance, the XDR which is used to - * allocate dummy_page_va is the nearest the spider-pci. - * We have to select the CBE which is the nearest the spider-pci - * to allocate memory from the best XDR, but I don't know that - * how to do. - * - * Celleb does not have this problem, because it has only one XDR. - */ - dummy_page_va = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!dummy_page_va) { - pr_err("SPIDERPCI-IOWA:Alloc dummy_page_va failed.\n"); - return -1; - } - - dummy_page_da = dma_map_single(phb->parent, dummy_page_va, - PAGE_SIZE, DMA_FROM_DEVICE); - if (dma_mapping_error(phb->parent, dummy_page_da)) { - pr_err("SPIDER-IOWA:Map dummy page filed.\n"); - kfree(dummy_page_va); - return -1; - } - - out_be32(regs + SPIDER_PCI_DUMMY_READ_BASE, dummy_page_da); - - return 0; -} - -int __init spiderpci_iowa_init(struct iowa_bus *bus, void *data) -{ - void __iomem *regs = NULL; - struct spiderpci_iowa_private *priv; - struct device_node *np = bus->phb->dn; - struct resource r; - unsigned long offset = (unsigned long)data; - - pr_debug("SPIDERPCI-IOWA:Bus initialize for spider(%pOF)\n", - np); - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) { - pr_err("SPIDERPCI-IOWA:" - "Can't allocate struct spiderpci_iowa_private"); - return -1; - } - - if (of_address_to_resource(np, 0, &r)) { - pr_err("SPIDERPCI-IOWA:Can't get resource.\n"); - goto error; - } - - regs = ioremap(r.start + offset, SPIDER_PCI_REG_SIZE); - if (!regs) { - pr_err("SPIDERPCI-IOWA:ioremap failed.\n"); - goto error; - } - priv->regs = regs; - bus->private = priv; - - if (spiderpci_pci_setup_chip(bus->phb, regs)) - goto error; - - return 0; - -error: - kfree(priv); - bus->private = NULL; - - if (regs) - iounmap(regs); - - return -1; -} - -struct ppc_pci_io spiderpci_ops = { - .readb = spiderpci_readb, - .readw = spiderpci_readw, - .readl = spiderpci_readl, - .readq = spiderpci_readq, - .readw_be = spiderpci_readw_be, - .readl_be = spiderpci_readl_be, - .readq_be = spiderpci_readq_be, - .readsb = spiderpci_readsb, - .readsw = spiderpci_readsw, - .readsl = spiderpci_readsl, - .memcpy_fromio = spiderpci_memcpy_fromio, -}; - diff --git a/arch/powerpc/platforms/cell/spider-pic.c b/arch/powerpc/platforms/cell/spider-pic.c deleted file mode 100644 index 11df737c8c6a..000000000000 --- a/arch/powerpc/platforms/cell/spider-pic.c +++ /dev/null @@ -1,344 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * External Interrupt Controller on Spider South Bridge - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * - * Author: Arnd Bergmann <arndb@de.ibm.com> - */ - -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/ioport.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/pgtable.h> - -#include <asm/io.h> - -#include "interrupt.h" - -/* register layout taken from Spider spec, table 7.4-4 */ -enum { - TIR_DEN = 0x004, /* Detection Enable Register */ - TIR_MSK = 0x084, /* Mask Level Register */ - TIR_EDC = 0x0c0, /* Edge Detection Clear Register */ - TIR_PNDA = 0x100, /* Pending Register A */ - TIR_PNDB = 0x104, /* Pending Register B */ - TIR_CS = 0x144, /* Current Status Register */ - TIR_LCSA = 0x150, /* Level Current Status Register A */ - TIR_LCSB = 0x154, /* Level Current Status Register B */ - TIR_LCSC = 0x158, /* Level Current Status Register C */ - TIR_LCSD = 0x15c, /* Level Current Status Register D */ - TIR_CFGA = 0x200, /* Setting Register A0 */ - TIR_CFGB = 0x204, /* Setting Register B0 */ - /* 0x208 ... 0x3ff Setting Register An/Bn */ - TIR_PPNDA = 0x400, /* Packet Pending Register A */ - TIR_PPNDB = 0x404, /* Packet Pending Register B */ - TIR_PIERA = 0x408, /* Packet Output Error Register A */ - TIR_PIERB = 0x40c, /* Packet Output Error Register B */ - TIR_PIEN = 0x444, /* Packet Output Enable Register */ - TIR_PIPND = 0x454, /* Packet Output Pending Register */ - TIRDID = 0x484, /* Spider Device ID Register */ - REISTIM = 0x500, /* Reissue Command Timeout Time Setting */ - REISTIMEN = 0x504, /* Reissue Command Timeout Setting */ - REISWAITEN = 0x508, /* Reissue Wait Control*/ -}; - -#define SPIDER_CHIP_COUNT 4 -#define SPIDER_SRC_COUNT 64 -#define SPIDER_IRQ_INVALID 63 - -struct spider_pic { - struct irq_domain *host; - void __iomem *regs; - unsigned int node_id; -}; -static struct spider_pic spider_pics[SPIDER_CHIP_COUNT]; - -static struct spider_pic *spider_irq_data_to_pic(struct irq_data *d) -{ - return irq_data_get_irq_chip_data(d); -} - -static void __iomem *spider_get_irq_config(struct spider_pic *pic, - unsigned int src) -{ - return pic->regs + TIR_CFGA + 8 * src; -} - -static void spider_unmask_irq(struct irq_data *d) -{ - struct spider_pic *pic = spider_irq_data_to_pic(d); - void __iomem *cfg = spider_get_irq_config(pic, irqd_to_hwirq(d)); - - out_be32(cfg, in_be32(cfg) | 0x30000000u); -} - -static void spider_mask_irq(struct irq_data *d) -{ - struct spider_pic *pic = spider_irq_data_to_pic(d); - void __iomem *cfg = spider_get_irq_config(pic, irqd_to_hwirq(d)); - - out_be32(cfg, in_be32(cfg) & ~0x30000000u); -} - -static void spider_ack_irq(struct irq_data *d) -{ - struct spider_pic *pic = spider_irq_data_to_pic(d); - unsigned int src = irqd_to_hwirq(d); - - /* Reset edge detection logic if necessary - */ - if (irqd_is_level_type(d)) - return; - - /* Only interrupts 47 to 50 can be set to edge */ - if (src < 47 || src > 50) - return; - - /* Perform the clear of the edge logic */ - out_be32(pic->regs + TIR_EDC, 0x100 | (src & 0xf)); -} - -static int spider_set_irq_type(struct irq_data *d, unsigned int type) -{ - unsigned int sense = type & IRQ_TYPE_SENSE_MASK; - struct spider_pic *pic = spider_irq_data_to_pic(d); - unsigned int hw = irqd_to_hwirq(d); - void __iomem *cfg = spider_get_irq_config(pic, hw); - u32 old_mask; - u32 ic; - - /* Note that only level high is supported for most interrupts */ - if (sense != IRQ_TYPE_NONE && sense != IRQ_TYPE_LEVEL_HIGH && - (hw < 47 || hw > 50)) - return -EINVAL; - - /* Decode sense type */ - switch(sense) { - case IRQ_TYPE_EDGE_RISING: - ic = 0x3; - break; - case IRQ_TYPE_EDGE_FALLING: - ic = 0x2; - break; - case IRQ_TYPE_LEVEL_LOW: - ic = 0x0; - break; - case IRQ_TYPE_LEVEL_HIGH: - case IRQ_TYPE_NONE: - ic = 0x1; - break; - default: - return -EINVAL; - } - - /* Configure the source. One gross hack that was there before and - * that I've kept around is the priority to the BE which I set to - * be the same as the interrupt source number. I don't know whether - * that's supposed to make any kind of sense however, we'll have to - * decide that, but for now, I'm not changing the behaviour. - */ - old_mask = in_be32(cfg) & 0x30000000u; - out_be32(cfg, old_mask | (ic << 24) | (0x7 << 16) | - (pic->node_id << 4) | 0xe); - out_be32(cfg + 4, (0x2 << 16) | (hw & 0xff)); - - return 0; -} - -static struct irq_chip spider_pic = { - .name = "SPIDER", - .irq_unmask = spider_unmask_irq, - .irq_mask = spider_mask_irq, - .irq_ack = spider_ack_irq, - .irq_set_type = spider_set_irq_type, -}; - -static int spider_host_map(struct irq_domain *h, unsigned int virq, - irq_hw_number_t hw) -{ - irq_set_chip_data(virq, h->host_data); - irq_set_chip_and_handler(virq, &spider_pic, handle_level_irq); - - /* Set default irq type */ - irq_set_irq_type(virq, IRQ_TYPE_NONE); - - return 0; -} - -static int spider_host_xlate(struct irq_domain *h, struct device_node *ct, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_flags) - -{ - /* Spider interrupts have 2 cells, first is the interrupt source, - * second, well, I don't know for sure yet ... We mask the top bits - * because old device-trees encode a node number in there - */ - *out_hwirq = intspec[0] & 0x3f; - *out_flags = IRQ_TYPE_LEVEL_HIGH; - return 0; -} - -static const struct irq_domain_ops spider_host_ops = { - .map = spider_host_map, - .xlate = spider_host_xlate, -}; - -static void spider_irq_cascade(struct irq_desc *desc) -{ - struct irq_chip *chip = irq_desc_get_chip(desc); - struct spider_pic *pic = irq_desc_get_handler_data(desc); - unsigned int cs; - - cs = in_be32(pic->regs + TIR_CS) >> 24; - if (cs != SPIDER_IRQ_INVALID) - generic_handle_domain_irq(pic->host, cs); - - chip->irq_eoi(&desc->irq_data); -} - -/* For hooking up the cascade we have a problem. Our device-tree is - * crap and we don't know on which BE iic interrupt we are hooked on at - * least not the "standard" way. We can reconstitute it based on two - * informations though: which BE node we are connected to and whether - * we are connected to IOIF0 or IOIF1. Right now, we really only care - * about the IBM cell blade and we know that its firmware gives us an - * interrupt-map property which is pretty strange. - */ -static unsigned int __init spider_find_cascade_and_node(struct spider_pic *pic) -{ - unsigned int virq; - const u32 *imap, *tmp; - int imaplen, intsize, unit; - struct device_node *iic; - struct device_node *of_node; - - of_node = irq_domain_get_of_node(pic->host); - - /* First, we check whether we have a real "interrupts" in the device - * tree in case the device-tree is ever fixed - */ - virq = irq_of_parse_and_map(of_node, 0); - if (virq) - return virq; - - /* Now do the horrible hacks */ - tmp = of_get_property(of_node, "#interrupt-cells", NULL); - if (tmp == NULL) - return 0; - intsize = *tmp; - imap = of_get_property(of_node, "interrupt-map", &imaplen); - if (imap == NULL || imaplen < (intsize + 1)) - return 0; - iic = of_find_node_by_phandle(imap[intsize]); - if (iic == NULL) - return 0; - imap += intsize + 1; - tmp = of_get_property(iic, "#interrupt-cells", NULL); - if (tmp == NULL) { - of_node_put(iic); - return 0; - } - intsize = *tmp; - /* Assume unit is last entry of interrupt specifier */ - unit = imap[intsize - 1]; - /* Ok, we have a unit, now let's try to get the node */ - tmp = of_get_property(iic, "ibm,interrupt-server-ranges", NULL); - if (tmp == NULL) { - of_node_put(iic); - return 0; - } - /* ugly as hell but works for now */ - pic->node_id = (*tmp) >> 1; - of_node_put(iic); - - /* Ok, now let's get cracking. You may ask me why I just didn't match - * the iic host from the iic OF node, but that way I'm still compatible - * with really really old old firmwares for which we don't have a node - */ - /* Manufacture an IIC interrupt number of class 2 */ - virq = irq_create_mapping(NULL, - (pic->node_id << IIC_IRQ_NODE_SHIFT) | - (2 << IIC_IRQ_CLASS_SHIFT) | - unit); - if (!virq) - printk(KERN_ERR "spider_pic: failed to map cascade !"); - return virq; -} - - -static void __init spider_init_one(struct device_node *of_node, int chip, - unsigned long addr) -{ - struct spider_pic *pic = &spider_pics[chip]; - int i, virq; - - /* Map registers */ - pic->regs = ioremap(addr, 0x1000); - if (pic->regs == NULL) - panic("spider_pic: can't map registers !"); - - /* Allocate a host */ - pic->host = irq_domain_add_linear(of_node, SPIDER_SRC_COUNT, - &spider_host_ops, pic); - if (pic->host == NULL) - panic("spider_pic: can't allocate irq host !"); - - /* Go through all sources and disable them */ - for (i = 0; i < SPIDER_SRC_COUNT; i++) { - void __iomem *cfg = pic->regs + TIR_CFGA + 8 * i; - out_be32(cfg, in_be32(cfg) & ~0x30000000u); - } - - /* do not mask any interrupts because of level */ - out_be32(pic->regs + TIR_MSK, 0x0); - - /* enable interrupt packets to be output */ - out_be32(pic->regs + TIR_PIEN, in_be32(pic->regs + TIR_PIEN) | 0x1); - - /* Hook up the cascade interrupt to the iic and nodeid */ - virq = spider_find_cascade_and_node(pic); - if (!virq) - return; - irq_set_handler_data(virq, pic); - irq_set_chained_handler(virq, spider_irq_cascade); - - printk(KERN_INFO "spider_pic: node %d, addr: 0x%lx %pOF\n", - pic->node_id, addr, of_node); - - /* Enable the interrupt detection enable bit. Do this last! */ - out_be32(pic->regs + TIR_DEN, in_be32(pic->regs + TIR_DEN) | 0x1); -} - -void __init spider_init_IRQ(void) -{ - struct resource r; - struct device_node *dn; - int chip = 0; - - /* XXX node numbers are totally bogus. We _hope_ we get the device - * nodes in the right order here but that's definitely not guaranteed, - * we need to get the node from the device tree instead. - * There is currently no proper property for it (but our whole - * device-tree is bogus anyway) so all we can do is pray or maybe test - * the address and deduce the node-id - */ - for_each_node_by_name(dn, "interrupt-controller") { - if (of_device_is_compatible(dn, "CBEA,platform-spider-pic")) { - if (of_address_to_resource(dn, 0, &r)) { - printk(KERN_WARNING "spider-pic: Failed\n"); - continue; - } - } else if (of_device_is_compatible(dn, "sti,platform-spider-pic") - && (chip < 2)) { - static long hard_coded_pics[] = - { 0x24000008000ul, 0x34000008000ul}; - r.start = hard_coded_pics[chip]; - } else - continue; - spider_init_one(dn, chip++, r.start); - } -} diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c index dea6f0f25897..2c07387201d0 100644 --- a/arch/powerpc/platforms/cell/spu_base.c +++ b/arch/powerpc/platforms/cell/spu_base.c @@ -23,7 +23,6 @@ #include <asm/spu.h> #include <asm/spu_priv1.h> #include <asm/spu_csa.h> -#include <asm/xmon.h> #include <asm/kexec.h> const struct spu_management_ops *spu_management_ops; @@ -772,7 +771,6 @@ static int __init init_spu_base(void) fb_append_extra_logo(&logo_spe_clut224, ret); mutex_lock(&spu_full_list_mutex); - xmon_register_spus(&spu_full_list); crash_register_spus(&spu_full_list); mutex_unlock(&spu_full_list_mutex); spu_add_dev_attr(&dev_attr_stat); diff --git a/arch/powerpc/platforms/cell/spu_manage.c b/arch/powerpc/platforms/cell/spu_manage.c deleted file mode 100644 index f464a1f2e568..000000000000 --- a/arch/powerpc/platforms/cell/spu_manage.c +++ /dev/null @@ -1,530 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * spu management operations for of based platforms - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * Copyright 2006 Sony Corp. - * (C) Copyright 2007 TOSHIBA CORPORATION - */ - -#include <linux/interrupt.h> -#include <linux/list.h> -#include <linux/export.h> -#include <linux/ptrace.h> -#include <linux/wait.h> -#include <linux/mm.h> -#include <linux/io.h> -#include <linux/mutex.h> -#include <linux/device.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> - -#include <asm/spu.h> -#include <asm/spu_priv1.h> -#include <asm/firmware.h> - -#include "spufs/spufs.h" -#include "interrupt.h" -#include "spu_priv1_mmio.h" - -struct device_node *spu_devnode(struct spu *spu) -{ - return spu->devnode; -} - -EXPORT_SYMBOL_GPL(spu_devnode); - -static u64 __init find_spu_unit_number(struct device_node *spe) -{ - const unsigned int *prop; - int proplen; - - /* new device trees should provide the physical-id attribute */ - prop = of_get_property(spe, "physical-id", &proplen); - if (proplen == 4) - return (u64)*prop; - - /* celleb device tree provides the unit-id */ - prop = of_get_property(spe, "unit-id", &proplen); - if (proplen == 4) - return (u64)*prop; - - /* legacy device trees provide the id in the reg attribute */ - prop = of_get_property(spe, "reg", &proplen); - if (proplen == 4) - return (u64)*prop; - - return 0; -} - -static void spu_unmap(struct spu *spu) -{ - if (!firmware_has_feature(FW_FEATURE_LPAR)) - iounmap(spu->priv1); - iounmap(spu->priv2); - iounmap(spu->problem); - iounmap((__force u8 __iomem *)spu->local_store); -} - -static int __init spu_map_interrupts_old(struct spu *spu, - struct device_node *np) -{ - unsigned int isrc; - const u32 *tmp; - int nid; - - /* Get the interrupt source unit from the device-tree */ - tmp = of_get_property(np, "isrc", NULL); - if (!tmp) - return -ENODEV; - isrc = tmp[0]; - - tmp = of_get_property(np->parent->parent, "node-id", NULL); - if (!tmp) { - printk(KERN_WARNING "%s: can't find node-id\n", __func__); - nid = spu->node; - } else - nid = tmp[0]; - - /* Add the node number */ - isrc |= nid << IIC_IRQ_NODE_SHIFT; - - /* Now map interrupts of all 3 classes */ - spu->irqs[0] = irq_create_mapping(NULL, IIC_IRQ_CLASS_0 | isrc); - spu->irqs[1] = irq_create_mapping(NULL, IIC_IRQ_CLASS_1 | isrc); - spu->irqs[2] = irq_create_mapping(NULL, IIC_IRQ_CLASS_2 | isrc); - - /* Right now, we only fail if class 2 failed */ - if (!spu->irqs[2]) - return -EINVAL; - - return 0; -} - -static void __iomem * __init spu_map_prop_old(struct spu *spu, - struct device_node *n, - const char *name) -{ - const struct address_prop { - unsigned long address; - unsigned int len; - } __attribute__((packed)) *prop; - int proplen; - - prop = of_get_property(n, name, &proplen); - if (prop == NULL || proplen != sizeof (struct address_prop)) - return NULL; - - return ioremap(prop->address, prop->len); -} - -static int __init spu_map_device_old(struct spu *spu) -{ - struct device_node *node = spu->devnode; - const char *prop; - int ret; - - ret = -ENODEV; - spu->name = of_get_property(node, "name", NULL); - if (!spu->name) - goto out; - - prop = of_get_property(node, "local-store", NULL); - if (!prop) - goto out; - spu->local_store_phys = *(unsigned long *)prop; - - /* we use local store as ram, not io memory */ - spu->local_store = (void __force *) - spu_map_prop_old(spu, node, "local-store"); - if (!spu->local_store) - goto out; - - prop = of_get_property(node, "problem", NULL); - if (!prop) - goto out_unmap; - spu->problem_phys = *(unsigned long *)prop; - - spu->problem = spu_map_prop_old(spu, node, "problem"); - if (!spu->problem) - goto out_unmap; - - spu->priv2 = spu_map_prop_old(spu, node, "priv2"); - if (!spu->priv2) - goto out_unmap; - - if (!firmware_has_feature(FW_FEATURE_LPAR)) { - spu->priv1 = spu_map_prop_old(spu, node, "priv1"); - if (!spu->priv1) - goto out_unmap; - } - - ret = 0; - goto out; - -out_unmap: - spu_unmap(spu); -out: - return ret; -} - -static int __init spu_map_interrupts(struct spu *spu, struct device_node *np) -{ - int i; - - for (i=0; i < 3; i++) { - spu->irqs[i] = irq_of_parse_and_map(np, i); - if (!spu->irqs[i]) - goto err; - } - return 0; - -err: - pr_debug("failed to map irq %x for spu %s\n", i, spu->name); - for (; i >= 0; i--) { - if (spu->irqs[i]) - irq_dispose_mapping(spu->irqs[i]); - } - return -EINVAL; -} - -static int __init spu_map_resource(struct spu *spu, int nr, - void __iomem** virt, unsigned long *phys) -{ - struct device_node *np = spu->devnode; - struct resource resource = { }; - unsigned long len; - int ret; - - ret = of_address_to_resource(np, nr, &resource); - if (ret) - return ret; - if (phys) - *phys = resource.start; - len = resource_size(&resource); - *virt = ioremap(resource.start, len); - if (!*virt) - return -EINVAL; - return 0; -} - -static int __init spu_map_device(struct spu *spu) -{ - struct device_node *np = spu->devnode; - int ret = -ENODEV; - - spu->name = of_get_property(np, "name", NULL); - if (!spu->name) - goto out; - - ret = spu_map_resource(spu, 0, (void __iomem**)&spu->local_store, - &spu->local_store_phys); - if (ret) { - pr_debug("spu_new: failed to map %pOF resource 0\n", - np); - goto out; - } - ret = spu_map_resource(spu, 1, (void __iomem**)&spu->problem, - &spu->problem_phys); - if (ret) { - pr_debug("spu_new: failed to map %pOF resource 1\n", - np); - goto out_unmap; - } - ret = spu_map_resource(spu, 2, (void __iomem**)&spu->priv2, NULL); - if (ret) { - pr_debug("spu_new: failed to map %pOF resource 2\n", - np); - goto out_unmap; - } - if (!firmware_has_feature(FW_FEATURE_LPAR)) - ret = spu_map_resource(spu, 3, - (void __iomem**)&spu->priv1, NULL); - if (ret) { - pr_debug("spu_new: failed to map %pOF resource 3\n", - np); - goto out_unmap; - } - pr_debug("spu_new: %pOF maps:\n", np); - pr_debug(" local store : 0x%016lx -> 0x%p\n", - spu->local_store_phys, spu->local_store); - pr_debug(" problem state : 0x%016lx -> 0x%p\n", - spu->problem_phys, spu->problem); - pr_debug(" priv2 : 0x%p\n", spu->priv2); - pr_debug(" priv1 : 0x%p\n", spu->priv1); - - return 0; - -out_unmap: - spu_unmap(spu); -out: - pr_debug("failed to map spe %s: %d\n", spu->name, ret); - return ret; -} - -static int __init of_enumerate_spus(int (*fn)(void *data)) -{ - int ret; - struct device_node *node; - unsigned int n = 0; - - ret = -ENODEV; - for_each_node_by_type(node, "spe") { - ret = fn(node); - if (ret) { - printk(KERN_WARNING "%s: Error initializing %pOFn\n", - __func__, node); - of_node_put(node); - break; - } - n++; - } - return ret ? ret : n; -} - -static int __init of_create_spu(struct spu *spu, void *data) -{ - int ret; - struct device_node *spe = (struct device_node *)data; - static int legacy_map = 0, legacy_irq = 0; - - spu->devnode = of_node_get(spe); - spu->spe_id = find_spu_unit_number(spe); - - spu->node = of_node_to_nid(spe); - if (spu->node >= MAX_NUMNODES) { - printk(KERN_WARNING "SPE %pOF on node %d ignored," - " node number too big\n", spe, spu->node); - printk(KERN_WARNING "Check if CONFIG_NUMA is enabled.\n"); - ret = -ENODEV; - goto out; - } - - ret = spu_map_device(spu); - if (ret) { - if (!legacy_map) { - legacy_map = 1; - printk(KERN_WARNING "%s: Legacy device tree found, " - "trying to map old style\n", __func__); - } - ret = spu_map_device_old(spu); - if (ret) { - printk(KERN_ERR "Unable to map %s\n", - spu->name); - goto out; - } - } - - ret = spu_map_interrupts(spu, spe); - if (ret) { - if (!legacy_irq) { - legacy_irq = 1; - printk(KERN_WARNING "%s: Legacy device tree found, " - "trying old style irq\n", __func__); - } - ret = spu_map_interrupts_old(spu, spe); - if (ret) { - printk(KERN_ERR "%s: could not map interrupts\n", - spu->name); - goto out_unmap; - } - } - - pr_debug("Using SPE %s %p %p %p %p %d\n", spu->name, - spu->local_store, spu->problem, spu->priv1, - spu->priv2, spu->number); - goto out; - -out_unmap: - spu_unmap(spu); -out: - return ret; -} - -static int of_destroy_spu(struct spu *spu) -{ - spu_unmap(spu); - of_node_put(spu->devnode); - return 0; -} - -static void enable_spu_by_master_run(struct spu_context *ctx) -{ - ctx->ops->master_start(ctx); -} - -static void disable_spu_by_master_run(struct spu_context *ctx) -{ - ctx->ops->master_stop(ctx); -} - -/* Hardcoded affinity idxs for qs20 */ -#define QS20_SPES_PER_BE 8 -static int qs20_reg_idxs[QS20_SPES_PER_BE] = { 0, 2, 4, 6, 7, 5, 3, 1 }; -static int qs20_reg_memory[QS20_SPES_PER_BE] = { 1, 1, 0, 0, 0, 0, 0, 0 }; - -static struct spu *__init spu_lookup_reg(int node, u32 reg) -{ - struct spu *spu; - const u32 *spu_reg; - - list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) { - spu_reg = of_get_property(spu_devnode(spu), "reg", NULL); - if (*spu_reg == reg) - return spu; - } - return NULL; -} - -static void __init init_affinity_qs20_harcoded(void) -{ - int node, i; - struct spu *last_spu, *spu; - u32 reg; - - for (node = 0; node < MAX_NUMNODES; node++) { - last_spu = NULL; - for (i = 0; i < QS20_SPES_PER_BE; i++) { - reg = qs20_reg_idxs[i]; - spu = spu_lookup_reg(node, reg); - if (!spu) - continue; - spu->has_mem_affinity = qs20_reg_memory[reg]; - if (last_spu) - list_add_tail(&spu->aff_list, - &last_spu->aff_list); - last_spu = spu; - } - } -} - -static int __init of_has_vicinity(void) -{ - struct device_node *dn; - - for_each_node_by_type(dn, "spe") { - if (of_property_present(dn, "vicinity")) { - of_node_put(dn); - return 1; - } - } - return 0; -} - -static struct spu *__init devnode_spu(int cbe, struct device_node *dn) -{ - struct spu *spu; - - list_for_each_entry(spu, &cbe_spu_info[cbe].spus, cbe_list) - if (spu_devnode(spu) == dn) - return spu; - return NULL; -} - -static struct spu * __init -neighbour_spu(int cbe, struct device_node *target, struct device_node *avoid) -{ - struct spu *spu; - struct device_node *spu_dn; - const phandle *vic_handles; - int lenp, i; - - list_for_each_entry(spu, &cbe_spu_info[cbe].spus, cbe_list) { - spu_dn = spu_devnode(spu); - if (spu_dn == avoid) - continue; - vic_handles = of_get_property(spu_dn, "vicinity", &lenp); - for (i=0; i < (lenp / sizeof(phandle)); i++) { - if (vic_handles[i] == target->phandle) - return spu; - } - } - return NULL; -} - -static void __init init_affinity_node(int cbe) -{ - struct spu *spu, *last_spu; - struct device_node *vic_dn, *last_spu_dn; - phandle avoid_ph; - const phandle *vic_handles; - int lenp, i, added; - - last_spu = list_first_entry(&cbe_spu_info[cbe].spus, struct spu, - cbe_list); - avoid_ph = 0; - for (added = 1; added < cbe_spu_info[cbe].n_spus; added++) { - last_spu_dn = spu_devnode(last_spu); - vic_handles = of_get_property(last_spu_dn, "vicinity", &lenp); - - /* - * Walk through each phandle in vicinity property of the spu - * (typically two vicinity phandles per spe node) - */ - for (i = 0; i < (lenp / sizeof(phandle)); i++) { - if (vic_handles[i] == avoid_ph) - continue; - - vic_dn = of_find_node_by_phandle(vic_handles[i]); - if (!vic_dn) - continue; - - if (of_node_name_eq(vic_dn, "spe") ) { - spu = devnode_spu(cbe, vic_dn); - avoid_ph = last_spu_dn->phandle; - } else { - /* - * "mic-tm" and "bif0" nodes do not have - * vicinity property. So we need to find the - * spe which has vic_dn as neighbour, but - * skipping the one we came from (last_spu_dn) - */ - spu = neighbour_spu(cbe, vic_dn, last_spu_dn); - if (!spu) - continue; - if (of_node_name_eq(vic_dn, "mic-tm")) { - last_spu->has_mem_affinity = 1; - spu->has_mem_affinity = 1; - } - avoid_ph = vic_dn->phandle; - } - - of_node_put(vic_dn); - - list_add_tail(&spu->aff_list, &last_spu->aff_list); - last_spu = spu; - break; - } - } -} - -static void __init init_affinity_fw(void) -{ - int cbe; - - for (cbe = 0; cbe < MAX_NUMNODES; cbe++) - init_affinity_node(cbe); -} - -static int __init init_affinity(void) -{ - if (of_has_vicinity()) { - init_affinity_fw(); - } else { - if (of_machine_is_compatible("IBM,CPBW-1.0")) - init_affinity_qs20_harcoded(); - else - printk("No affinity configuration found\n"); - } - - return 0; -} - -const struct spu_management_ops spu_management_of_ops = { - .enumerate_spus = of_enumerate_spus, - .create_spu = of_create_spu, - .destroy_spu = of_destroy_spu, - .enable_spu = enable_spu_by_master_run, - .disable_spu = disable_spu_by_master_run, - .init_affinity = init_affinity, -}; diff --git a/arch/powerpc/platforms/cell/spu_priv1_mmio.c b/arch/powerpc/platforms/cell/spu_priv1_mmio.c deleted file mode 100644 index d150e3987304..000000000000 --- a/arch/powerpc/platforms/cell/spu_priv1_mmio.c +++ /dev/null @@ -1,167 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * spu hypervisor abstraction for direct hardware access. - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * Copyright 2006 Sony Corp. - */ - -#include <linux/interrupt.h> -#include <linux/list.h> -#include <linux/ptrace.h> -#include <linux/wait.h> -#include <linux/mm.h> -#include <linux/io.h> -#include <linux/mutex.h> -#include <linux/device.h> -#include <linux/sched.h> - -#include <asm/spu.h> -#include <asm/spu_priv1.h> -#include <asm/firmware.h> - -#include "interrupt.h" -#include "spu_priv1_mmio.h" - -static void int_mask_and(struct spu *spu, int class, u64 mask) -{ - u64 old_mask; - - old_mask = in_be64(&spu->priv1->int_mask_RW[class]); - out_be64(&spu->priv1->int_mask_RW[class], old_mask & mask); -} - -static void int_mask_or(struct spu *spu, int class, u64 mask) -{ - u64 old_mask; - - old_mask = in_be64(&spu->priv1->int_mask_RW[class]); - out_be64(&spu->priv1->int_mask_RW[class], old_mask | mask); -} - -static void int_mask_set(struct spu *spu, int class, u64 mask) -{ - out_be64(&spu->priv1->int_mask_RW[class], mask); -} - -static u64 int_mask_get(struct spu *spu, int class) -{ - return in_be64(&spu->priv1->int_mask_RW[class]); -} - -static void int_stat_clear(struct spu *spu, int class, u64 stat) -{ - out_be64(&spu->priv1->int_stat_RW[class], stat); -} - -static u64 int_stat_get(struct spu *spu, int class) -{ - return in_be64(&spu->priv1->int_stat_RW[class]); -} - -static void cpu_affinity_set(struct spu *spu, int cpu) -{ - u64 target; - u64 route; - - if (nr_cpus_node(spu->node)) { - const struct cpumask *spumask = cpumask_of_node(spu->node), - *cpumask = cpumask_of_node(cpu_to_node(cpu)); - - if (!cpumask_intersects(spumask, cpumask)) - return; - } - - target = iic_get_target_id(cpu); - route = target << 48 | target << 32 | target << 16; - out_be64(&spu->priv1->int_route_RW, route); -} - -static u64 mfc_dar_get(struct spu *spu) -{ - return in_be64(&spu->priv1->mfc_dar_RW); -} - -static u64 mfc_dsisr_get(struct spu *spu) -{ - return in_be64(&spu->priv1->mfc_dsisr_RW); -} - -static void mfc_dsisr_set(struct spu *spu, u64 dsisr) -{ - out_be64(&spu->priv1->mfc_dsisr_RW, dsisr); -} - -static void mfc_sdr_setup(struct spu *spu) -{ - out_be64(&spu->priv1->mfc_sdr_RW, mfspr(SPRN_SDR1)); -} - -static void mfc_sr1_set(struct spu *spu, u64 sr1) -{ - out_be64(&spu->priv1->mfc_sr1_RW, sr1); -} - -static u64 mfc_sr1_get(struct spu *spu) -{ - return in_be64(&spu->priv1->mfc_sr1_RW); -} - -static void mfc_tclass_id_set(struct spu *spu, u64 tclass_id) -{ - out_be64(&spu->priv1->mfc_tclass_id_RW, tclass_id); -} - -static u64 mfc_tclass_id_get(struct spu *spu) -{ - return in_be64(&spu->priv1->mfc_tclass_id_RW); -} - -static void tlb_invalidate(struct spu *spu) -{ - out_be64(&spu->priv1->tlb_invalidate_entry_W, 0ul); -} - -static void resource_allocation_groupID_set(struct spu *spu, u64 id) -{ - out_be64(&spu->priv1->resource_allocation_groupID_RW, id); -} - -static u64 resource_allocation_groupID_get(struct spu *spu) -{ - return in_be64(&spu->priv1->resource_allocation_groupID_RW); -} - -static void resource_allocation_enable_set(struct spu *spu, u64 enable) -{ - out_be64(&spu->priv1->resource_allocation_enable_RW, enable); -} - -static u64 resource_allocation_enable_get(struct spu *spu) -{ - return in_be64(&spu->priv1->resource_allocation_enable_RW); -} - -const struct spu_priv1_ops spu_priv1_mmio_ops = -{ - .int_mask_and = int_mask_and, - .int_mask_or = int_mask_or, - .int_mask_set = int_mask_set, - .int_mask_get = int_mask_get, - .int_stat_clear = int_stat_clear, - .int_stat_get = int_stat_get, - .cpu_affinity_set = cpu_affinity_set, - .mfc_dar_get = mfc_dar_get, - .mfc_dsisr_get = mfc_dsisr_get, - .mfc_dsisr_set = mfc_dsisr_set, - .mfc_sdr_setup = mfc_sdr_setup, - .mfc_sr1_set = mfc_sr1_set, - .mfc_sr1_get = mfc_sr1_get, - .mfc_tclass_id_set = mfc_tclass_id_set, - .mfc_tclass_id_get = mfc_tclass_id_get, - .tlb_invalidate = tlb_invalidate, - .resource_allocation_groupID_set = resource_allocation_groupID_set, - .resource_allocation_groupID_get = resource_allocation_groupID_get, - .resource_allocation_enable_set = resource_allocation_enable_set, - .resource_allocation_enable_get = resource_allocation_enable_get, -}; diff --git a/arch/powerpc/platforms/cell/spu_priv1_mmio.h b/arch/powerpc/platforms/cell/spu_priv1_mmio.h deleted file mode 100644 index 04f0db339dc1..000000000000 --- a/arch/powerpc/platforms/cell/spu_priv1_mmio.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * spu hypervisor abstraction for direct hardware access. - * - * Copyright (C) 2006 Sony Computer Entertainment Inc. - * Copyright 2006 Sony Corp. - */ - -#ifndef SPU_PRIV1_MMIO_H -#define SPU_PRIV1_MMIO_H - -struct device_node *spu_devnode(struct spu *spu); - -#endif /* SPU_PRIV1_MMIO_H */ diff --git a/arch/powerpc/platforms/cell/spufs/gang.c b/arch/powerpc/platforms/cell/spufs/gang.c index 827d338deaf4..2c2999de6bfa 100644 --- a/arch/powerpc/platforms/cell/spufs/gang.c +++ b/arch/powerpc/platforms/cell/spufs/gang.c @@ -25,6 +25,7 @@ struct spu_gang *alloc_spu_gang(void) mutex_init(&gang->aff_mutex); INIT_LIST_HEAD(&gang->list); INIT_LIST_HEAD(&gang->aff_list_head); + gang->alive = 1; out: return gang; diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index 70236d1df3d3..7ec60290abe6 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -143,42 +143,13 @@ spufs_evict_inode(struct inode *inode) put_spu_gang(ei->i_gang); } -static void spufs_prune_dir(struct dentry *dir) -{ - struct dentry *dentry; - struct hlist_node *n; - - inode_lock(d_inode(dir)); - hlist_for_each_entry_safe(dentry, n, &dir->d_children, d_sib) { - spin_lock(&dentry->d_lock); - if (simple_positive(dentry)) { - dget_dlock(dentry); - __d_drop(dentry); - spin_unlock(&dentry->d_lock); - simple_unlink(d_inode(dir), dentry); - /* XXX: what was dcache_lock protecting here? Other - * filesystems (IB, configfs) release dcache_lock - * before unlink */ - dput(dentry); - } else { - spin_unlock(&dentry->d_lock); - } - } - shrink_dcache_parent(dir); - inode_unlock(d_inode(dir)); -} - /* Caller must hold parent->i_mutex */ -static int spufs_rmdir(struct inode *parent, struct dentry *dir) +static void spufs_rmdir(struct inode *parent, struct dentry *dir) { - /* remove all entries */ - int res; - spufs_prune_dir(dir); - d_drop(dir); - res = simple_rmdir(parent, dir); - /* We have to give up the mm_struct */ - spu_forget(SPUFS_I(d_inode(dir))->i_ctx); - return res; + struct spu_context *ctx = SPUFS_I(d_inode(dir))->i_ctx; + + locked_recursive_removal(dir, NULL); + spu_forget(ctx); } static int spufs_fill_dir(struct dentry *dir, @@ -192,27 +163,45 @@ static int spufs_fill_dir(struct dentry *dir, return -ENOMEM; ret = spufs_new_file(dir->d_sb, dentry, files->ops, files->mode & mode, files->size, ctx); - if (ret) + if (ret) { + dput(dentry); return ret; + } files++; } return 0; } +static void unuse_gang(struct dentry *dir) +{ + struct inode *inode = dir->d_inode; + struct spu_gang *gang = SPUFS_I(inode)->i_gang; + + if (gang) { + bool dead; + + inode_lock(inode); // exclusion with spufs_create_context() + dead = !--gang->alive; + inode_unlock(inode); + + if (dead) + simple_recursive_removal(dir, NULL); + } +} + static int spufs_dir_close(struct inode *inode, struct file *file) { struct inode *parent; struct dentry *dir; - int ret; dir = file->f_path.dentry; parent = d_inode(dir->d_parent); inode_lock_nested(parent, I_MUTEX_PARENT); - ret = spufs_rmdir(parent, dir); + spufs_rmdir(parent, dir); inode_unlock(parent); - WARN_ON(ret); + unuse_gang(dir->d_parent); return dcache_dir_close(inode, file); } @@ -268,11 +257,11 @@ spufs_mkdir(struct inode *dir, struct dentry *dentry, unsigned int flags, ret = spufs_fill_dir(dentry, spufs_dir_debug_contents, mode, ctx); + inode_unlock(inode); + if (ret) spufs_rmdir(dir, dentry); - inode_unlock(inode); - return ret; } @@ -405,7 +394,7 @@ spufs_create_context(struct inode *inode, struct dentry *dentry, { int ret; int affinity; - struct spu_gang *gang; + struct spu_gang *gang = SPUFS_I(inode)->i_gang; struct spu_context *neighbor; struct path path = {.mnt = mnt, .dentry = dentry}; @@ -420,11 +409,15 @@ spufs_create_context(struct inode *inode, struct dentry *dentry, if ((flags & SPU_CREATE_ISOLATE) && !isolated_loader) return -ENODEV; - gang = NULL; + if (gang) { + if (!gang->alive) + return -ENOENT; + gang->alive++; + } + neighbor = NULL; affinity = flags & (SPU_CREATE_AFFINITY_MEM | SPU_CREATE_AFFINITY_SPU); if (affinity) { - gang = SPUFS_I(inode)->i_gang; if (!gang) return -EINVAL; mutex_lock(&gang->aff_mutex); @@ -436,8 +429,11 @@ spufs_create_context(struct inode *inode, struct dentry *dentry, } ret = spufs_mkdir(inode, dentry, flags, mode & 0777); - if (ret) + if (ret) { + if (neighbor) + put_spu_context(neighbor); goto out_aff_unlock; + } if (affinity) { spufs_set_affinity(flags, SPUFS_I(d_inode(dentry))->i_ctx, @@ -448,11 +444,13 @@ spufs_create_context(struct inode *inode, struct dentry *dentry, ret = spufs_context_open(&path); if (ret < 0) - WARN_ON(spufs_rmdir(inode, dentry)); + spufs_rmdir(inode, dentry); out_aff_unlock: if (affinity) mutex_unlock(&gang->aff_mutex); + if (ret && gang) + gang->alive--; // can't reach 0 return ret; } @@ -482,6 +480,7 @@ spufs_mkgang(struct inode *dir, struct dentry *dentry, umode_t mode) inode->i_fop = &simple_dir_operations; d_instantiate(dentry, inode); + dget(dentry); inc_nlink(dir); inc_nlink(d_inode(dentry)); return ret; @@ -492,6 +491,21 @@ out: return ret; } +static int spufs_gang_close(struct inode *inode, struct file *file) +{ + unuse_gang(file->f_path.dentry); + return dcache_dir_close(inode, file); +} + +static const struct file_operations spufs_gang_fops = { + .open = dcache_dir_open, + .release = spufs_gang_close, + .llseek = dcache_dir_lseek, + .read = generic_read_dir, + .iterate_shared = dcache_readdir, + .fsync = noop_fsync, +}; + static int spufs_gang_open(const struct path *path) { int ret; @@ -511,7 +525,7 @@ static int spufs_gang_open(const struct path *path) return PTR_ERR(filp); } - filp->f_op = &simple_dir_operations; + filp->f_op = &spufs_gang_fops; fd_install(ret, filp); return ret; } @@ -526,10 +540,8 @@ static int spufs_create_gang(struct inode *inode, ret = spufs_mkgang(inode, dentry, mode & 0777); if (!ret) { ret = spufs_gang_open(&path); - if (ret < 0) { - int err = simple_rmdir(inode, dentry); - WARN_ON(err); - } + if (ret < 0) + unuse_gang(dentry); } return ret; } diff --git a/arch/powerpc/platforms/cell/spufs/sched.c b/arch/powerpc/platforms/cell/spufs/sched.c index 610ca8570682..8e7ed010bfde 100644 --- a/arch/powerpc/platforms/cell/spufs/sched.c +++ b/arch/powerpc/platforms/cell/spufs/sched.c @@ -508,7 +508,7 @@ static void __spu_del_from_rq(struct spu_context *ctx) if (!list_empty(&ctx->rq)) { if (!--spu_prio->nr_waiting) - del_timer(&spusched_timer); + timer_delete(&spusched_timer); list_del_init(&ctx->rq); if (list_empty(&spu_prio->runq[prio])) @@ -1126,8 +1126,8 @@ void spu_sched_exit(void) remove_proc_entry("spu_loadavg", NULL); - del_timer_sync(&spusched_timer); - del_timer_sync(&spuloadavg_timer); + timer_delete_sync(&spusched_timer); + timer_delete_sync(&spuloadavg_timer); kthread_stop(spusched_task); for (node = 0; node < MAX_NUMNODES; node++) { diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h index 84958487f696..d33787c57c39 100644 --- a/arch/powerpc/platforms/cell/spufs/spufs.h +++ b/arch/powerpc/platforms/cell/spufs/spufs.h @@ -151,6 +151,8 @@ struct spu_gang { int aff_flags; struct spu *aff_ref_spu; atomic_t aff_sched_count; + + int alive; }; /* Flag bits for spu_gang aff_flags */ diff --git a/arch/powerpc/platforms/chrp/setup.c b/arch/powerpc/platforms/chrp/setup.c index 36ee3a5056a1..c1bfa4c3444c 100644 --- a/arch/powerpc/platforms/chrp/setup.c +++ b/arch/powerpc/platforms/chrp/setup.c @@ -486,7 +486,7 @@ static void __init chrp_find_8259(void) i8259_init(pic, chrp_int_ack); if (ppc_md.get_irq == NULL) { ppc_md.get_irq = i8259_irq; - irq_set_default_host(i8259_get_host()); + irq_set_default_domain(i8259_get_host()); } if (chrp_mpic != NULL) { cascade_irq = irq_of_parse_and_map(pic, 0); diff --git a/arch/powerpc/platforms/embedded6xx/flipper-pic.c b/arch/powerpc/platforms/embedded6xx/flipper-pic.c index 4d9200bdba78..91a8f0a7086e 100644 --- a/arch/powerpc/platforms/embedded6xx/flipper-pic.c +++ b/arch/powerpc/platforms/embedded6xx/flipper-pic.c @@ -149,8 +149,9 @@ static struct irq_domain * __init flipper_pic_init(struct device_node *np) __flipper_quiesce(io_base); - irq_domain = irq_domain_add_linear(np, FLIPPER_NR_IRQS, - &flipper_irq_domain_ops, io_base); + irq_domain = irq_domain_create_linear(of_fwnode_handle(np), + FLIPPER_NR_IRQS, + &flipper_irq_domain_ops, io_base); if (!irq_domain) { pr_err("failed to allocate irq_domain\n"); return NULL; @@ -172,7 +173,7 @@ unsigned int flipper_pic_get_irq(void) return 0; /* no more IRQs pending */ irq = __ffs(irq_status); - return irq_linear_revmap(flipper_irq_host, irq); + return irq_find_mapping(flipper_irq_host, irq); } /* @@ -190,7 +191,7 @@ void __init flipper_pic_probe(void) flipper_irq_host = flipper_pic_init(np); BUG_ON(!flipper_irq_host); - irq_set_default_host(flipper_irq_host); + irq_set_default_domain(flipper_irq_host); of_node_put(np); } diff --git a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c index 4d2d92de30af..b57e87b0b3ce 100644 --- a/arch/powerpc/platforms/embedded6xx/hlwd-pic.c +++ b/arch/powerpc/platforms/embedded6xx/hlwd-pic.c @@ -175,8 +175,9 @@ static struct irq_domain *__init hlwd_pic_init(struct device_node *np) __hlwd_quiesce(io_base); - irq_domain = irq_domain_add_linear(np, HLWD_NR_IRQS, - &hlwd_irq_domain_ops, io_base); + irq_domain = irq_domain_create_linear(of_fwnode_handle(np), + HLWD_NR_IRQS, + &hlwd_irq_domain_ops, io_base); if (!irq_domain) { pr_err("failed to allocate irq_domain\n"); iounmap(io_base); @@ -189,7 +190,7 @@ static struct irq_domain *__init hlwd_pic_init(struct device_node *np) unsigned int hlwd_pic_get_irq(void) { unsigned int hwirq = __hlwd_pic_get_irq(hlwd_irq_host); - return hwirq ? irq_linear_revmap(hlwd_irq_host, hwirq) : 0; + return hwirq ? irq_find_mapping(hlwd_irq_host, hwirq) : 0; } /* diff --git a/arch/powerpc/platforms/microwatt/Kconfig b/arch/powerpc/platforms/microwatt/Kconfig index 6af443a1db99..cb2aff635bb0 100644 --- a/arch/powerpc/platforms/microwatt/Kconfig +++ b/arch/powerpc/platforms/microwatt/Kconfig @@ -1,11 +1,12 @@ # SPDX-License-Identifier: GPL-2.0 config PPC_MICROWATT - depends on PPC_BOOK3S_64 && !SMP + depends on PPC_BOOK3S_64 bool "Microwatt SoC platform" select PPC_XICS select PPC_ICS_NATIVE select PPC_ICP_NATIVE select PPC_UDBG_16550 + select COMMON_CLK help This option enables support for FPGA-based Microwatt implementations. diff --git a/arch/powerpc/platforms/microwatt/Makefile b/arch/powerpc/platforms/microwatt/Makefile index 116d6d3ad3f0..d973b2ab4042 100644 --- a/arch/powerpc/platforms/microwatt/Makefile +++ b/arch/powerpc/platforms/microwatt/Makefile @@ -1 +1,2 @@ obj-y += setup.o rng.o +obj-$(CONFIG_SMP) += smp.o diff --git a/arch/powerpc/platforms/microwatt/microwatt.h b/arch/powerpc/platforms/microwatt/microwatt.h index 335417e95e66..891aa2800768 100644 --- a/arch/powerpc/platforms/microwatt/microwatt.h +++ b/arch/powerpc/platforms/microwatt/microwatt.h @@ -3,5 +3,6 @@ #define _MICROWATT_H void microwatt_rng_init(void); +void microwatt_init_smp(void); #endif /* _MICROWATT_H */ diff --git a/arch/powerpc/platforms/microwatt/setup.c b/arch/powerpc/platforms/microwatt/setup.c index 5e1c0997170d..6af2ccef736c 100644 --- a/arch/powerpc/platforms/microwatt/setup.c +++ b/arch/powerpc/platforms/microwatt/setup.c @@ -29,15 +29,33 @@ static int __init microwatt_populate(void) } machine_arch_initcall(microwatt, microwatt_populate); +static int __init microwatt_probe(void) +{ + /* Main reason for having this is to start the other CPU(s) */ + if (IS_ENABLED(CONFIG_SMP)) + microwatt_init_smp(); + return 1; +} + static void __init microwatt_setup_arch(void) { microwatt_rng_init(); } +static void microwatt_idle(void) +{ + if (!prep_irq_for_idle_irqsoff()) + return; + + __asm__ __volatile__ ("wait"); +} + define_machine(microwatt) { .name = "microwatt", .compatible = "microwatt-soc", + .probe = microwatt_probe, .init_IRQ = microwatt_init_IRQ, .setup_arch = microwatt_setup_arch, .progress = udbg_progress, + .power_save = microwatt_idle, }; diff --git a/arch/powerpc/platforms/microwatt/smp.c b/arch/powerpc/platforms/microwatt/smp.c new file mode 100644 index 000000000000..7dbf2ca73d47 --- /dev/null +++ b/arch/powerpc/platforms/microwatt/smp.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +/* + * SMP support functions for Microwatt + * Copyright 2025 Paul Mackerras <paulus@ozlabs.org> + */ + +#include <linux/kernel.h> +#include <linux/smp.h> +#include <linux/io.h> +#include <asm/early_ioremap.h> +#include <asm/ppc-opcode.h> +#include <asm/reg.h> +#include <asm/smp.h> +#include <asm/xics.h> + +#include "microwatt.h" + +static void __init microwatt_smp_probe(void) +{ + xics_smp_probe(); +} + +static void microwatt_smp_setup_cpu(int cpu) +{ + if (cpu != 0) + xics_setup_cpu(); +} + +static struct smp_ops_t microwatt_smp_ops = { + .probe = microwatt_smp_probe, + .message_pass = NULL, /* Use smp_muxed_ipi_message_pass */ + .kick_cpu = smp_generic_kick_cpu, + .setup_cpu = microwatt_smp_setup_cpu, +}; + +/* XXX get from device tree */ +#define SYSCON_BASE 0xc0000000 +#define SYSCON_LENGTH 0x100 + +#define SYSCON_CPU_CTRL 0x58 + +void __init microwatt_init_smp(void) +{ + volatile unsigned char __iomem *syscon; + int ncpus; + int timeout; + + syscon = early_ioremap(SYSCON_BASE, SYSCON_LENGTH); + if (syscon == NULL) { + pr_err("Failed to map SYSCON\n"); + return; + } + ncpus = (readl(syscon + SYSCON_CPU_CTRL) >> 8) & 0xff; + if (ncpus < 2) + goto out; + + smp_ops = µwatt_smp_ops; + + /* + * Write two instructions at location 0: + * mfspr r3, PIR + * b __secondary_hold + */ + *(unsigned int *)KERNELBASE = PPC_RAW_MFSPR(3, SPRN_PIR); + *(unsigned int *)(KERNELBASE+4) = PPC_RAW_BRANCH(&__secondary_hold - (char *)(KERNELBASE+4)); + + /* enable the other CPUs, they start at location 0 */ + writel((1ul << ncpus) - 1, syscon + SYSCON_CPU_CTRL); + + timeout = 10000; + while (!__secondary_hold_acknowledge) { + if (--timeout == 0) + break; + barrier(); + } + + out: + early_iounmap((void *)syscon, SYSCON_LENGTH); +} diff --git a/arch/powerpc/platforms/pasemi/setup.c b/arch/powerpc/platforms/pasemi/setup.c index 0761d98e5be3..d03b41336901 100644 --- a/arch/powerpc/platforms/pasemi/setup.c +++ b/arch/powerpc/platforms/pasemi/setup.c @@ -228,7 +228,7 @@ static void __init nemo_init_IRQ(struct mpic *mpic) irq_set_chained_handler(gpio_virq, sb600_8259_cascade); mpic_unmask_irq(irq_get_irq_data(gpio_virq)); - irq_set_default_host(mpic->irqhost); + irq_set_default_domain(mpic->irqhost); } #else diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c index c097d591670e..02474e27df9b 100644 --- a/arch/powerpc/platforms/powermac/low_i2c.c +++ b/arch/powerpc/platforms/powermac/low_i2c.c @@ -347,7 +347,7 @@ static irqreturn_t kw_i2c_irq(int irq, void *dev_id) unsigned long flags; spin_lock_irqsave(&host->lock, flags); - del_timer(&host->timeout_timer); + timer_delete(&host->timeout_timer); kw_i2c_handle_interrupt(host, kw_read_reg(reg_isr)); if (host->state != state_idle) { host->timeout_timer.expires = jiffies + KW_POLL_TIMEOUT; @@ -359,7 +359,8 @@ static irqreturn_t kw_i2c_irq(int irq, void *dev_id) static void kw_i2c_timeout(struct timer_list *t) { - struct pmac_i2c_host_kw *host = from_timer(host, t, timeout_timer); + struct pmac_i2c_host_kw *host = timer_container_of(host, t, + timeout_timer); unsigned long flags; spin_lock_irqsave(&host->lock, flags); diff --git a/arch/powerpc/platforms/powermac/nvram.c b/arch/powerpc/platforms/powermac/nvram.c index fe2e0249cbc2..a112d26185a0 100644 --- a/arch/powerpc/platforms/powermac/nvram.c +++ b/arch/powerpc/platforms/powermac/nvram.c @@ -514,10 +514,7 @@ static int __init core99_nvram_setup(struct device_node *dp, unsigned long addr) printk(KERN_ERR "nvram: no address\n"); return -EINVAL; } - nvram_image = memblock_alloc(NVRAM_SIZE, SMP_CACHE_BYTES); - if (!nvram_image) - panic("%s: Failed to allocate %u bytes\n", __func__, - NVRAM_SIZE); + nvram_image = memblock_alloc_or_panic(NVRAM_SIZE, SMP_CACHE_BYTES); nvram_data = ioremap(addr, NVRAM_SIZE*2); nvram_naddrs = 1; /* Make sure we get the correct case */ diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c index 2202bf77c7a3..c37783a03d25 100644 --- a/arch/powerpc/platforms/powermac/pic.c +++ b/arch/powerpc/platforms/powermac/pic.c @@ -250,7 +250,7 @@ static unsigned int pmac_pic_get_irq(void) raw_spin_unlock_irqrestore(&pmac_pic_lock, flags); if (unlikely(irq < 0)) return 0; - return irq_linear_revmap(pmac_pic_host, irq); + return irq_find_mapping(pmac_pic_host, irq); } static int pmac_pic_host_match(struct irq_domain *h, struct device_node *node, @@ -327,10 +327,11 @@ static void __init pmac_pic_probe_oldstyle(void) /* * Allocate an irq host */ - pmac_pic_host = irq_domain_add_linear(master, max_irqs, - &pmac_pic_host_ops, NULL); + pmac_pic_host = irq_domain_create_linear(of_fwnode_handle(master), + max_irqs, + &pmac_pic_host_ops, NULL); BUG_ON(pmac_pic_host == NULL); - irq_set_default_host(pmac_pic_host); + irq_set_default_domain(pmac_pic_host); /* Get addresses of first controller if we have a node for it */ BUG_ON(of_address_to_resource(master, 0, &r)); diff --git a/arch/powerpc/platforms/powermac/setup.c b/arch/powerpc/platforms/powermac/setup.c index 6de1cd5d8a58..eb092f293113 100644 --- a/arch/powerpc/platforms/powermac/setup.c +++ b/arch/powerpc/platforms/powermac/setup.c @@ -28,13 +28,11 @@ #include <linux/ptrace.h> #include <linux/export.h> #include <linux/user.h> -#include <linux/tty.h> #include <linux/string.h> #include <linux/delay.h> #include <linux/ioport.h> #include <linux/major.h> #include <linux/initrd.h> -#include <linux/vt_kern.h> #include <linux/console.h> #include <linux/pci.h> #include <linux/adb.h> @@ -45,6 +43,7 @@ #include <linux/root_dev.h> #include <linux/bitops.h> #include <linux/suspend.h> +#include <linux/string_choices.h> #include <linux/of.h> #include <linux/of_platform.h> @@ -238,8 +237,7 @@ static void __init l2cr_init(void) _set_L2CR(0); _set_L2CR(*l2cr); pr_info("L2CR overridden (0x%x), backside cache is %s\n", - *l2cr, ((*l2cr) & 0x80000000) ? - "enabled" : "disabled"); + *l2cr, str_enabled_disabled((*l2cr) & 0x80000000)); } of_node_put(np); break; diff --git a/arch/powerpc/platforms/powermac/smp.c b/arch/powerpc/platforms/powermac/smp.c index 09e7fe24fac1..88e92af8acf9 100644 --- a/arch/powerpc/platforms/powermac/smp.c +++ b/arch/powerpc/platforms/powermac/smp.c @@ -190,7 +190,7 @@ static int __init psurge_secondary_ipi_init(void) { int rc = -ENOMEM; - psurge_host = irq_domain_add_nomap(NULL, ~0, &psurge_host_ops, NULL); + psurge_host = irq_domain_create_nomap(NULL, ~0, &psurge_host_ops, NULL); if (psurge_host) psurge_secondary_virq = irq_create_direct_mapping(psurge_host); diff --git a/arch/powerpc/platforms/powermac/time.c b/arch/powerpc/platforms/powermac/time.c index 8633891b7aa5..b4426a35aca3 100644 --- a/arch/powerpc/platforms/powermac/time.c +++ b/arch/powerpc/platforms/powermac/time.c @@ -15,6 +15,7 @@ #include <linux/kernel.h> #include <linux/param.h> #include <linux/string.h> +#include <linux/string_choices.h> #include <linux/mm.h> #include <linux/init.h> #include <linux/time.h> @@ -77,7 +78,7 @@ long __init pmac_time_init(void) delta |= 0xFF000000UL; dst = ((pmac_xpram_read(PMAC_XPRAM_MACHINE_LOC + 0x8) & 0x80) != 0); printk("GMT Delta read from XPRAM: %d minutes, DST: %s\n", delta/60, - dst ? "on" : "off"); + str_on_off(dst)); #endif return delta; } diff --git a/arch/powerpc/platforms/powernv/Kconfig b/arch/powerpc/platforms/powernv/Kconfig index 70a46acc70d6..95d7ba73d43d 100644 --- a/arch/powerpc/platforms/powernv/Kconfig +++ b/arch/powerpc/platforms/powernv/Kconfig @@ -17,6 +17,7 @@ config PPC_POWERNV select MMU_NOTIFIER select FORCE_SMP select ARCH_SUPPORTS_PER_VMA_LOCK + select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU default y config OPAL_PRD diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile index 19f0fc5c6f1b..9e5d0c847ee2 100644 --- a/arch/powerpc/platforms/powernv/Makefile +++ b/arch/powerpc/platforms/powernv/Makefile @@ -21,7 +21,6 @@ obj-$(CONFIG_PRESERVE_FA_DUMP) += opal-fadump.o obj-$(CONFIG_OPAL_CORE) += opal-core.o obj-$(CONFIG_PCI) += pci.o pci-ioda.o pci-ioda-tce.o obj-$(CONFIG_PCI_IOV) += pci-sriov.o -obj-$(CONFIG_CXL_BASE) += pci-cxl.o obj-$(CONFIG_EEH) += eeh-powernv.o obj-$(CONFIG_MEMORY_FAILURE) += opal-memory-errors.o obj-$(CONFIG_OPAL_PRD) += opal-prd.o diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c index 877720c64515..2ea30b343354 100644 --- a/arch/powerpc/platforms/powernv/memtrace.c +++ b/arch/powerpc/platforms/powernv/memtrace.c @@ -48,11 +48,15 @@ static ssize_t memtrace_read(struct file *filp, char __user *ubuf, static int memtrace_mmap(struct file *filp, struct vm_area_struct *vma) { struct memtrace_entry *ent = filp->private_data; + unsigned long ent_nrpages = ent->size >> PAGE_SHIFT; + unsigned long vma_nrpages = vma_pages(vma); - if (ent->size < vma->vm_end - vma->vm_start) + /* The requested page offset should be within object's page count */ + if (vma->vm_pgoff >= ent_nrpages) return -EINVAL; - if (vma->vm_pgoff << PAGE_SHIFT >= ent->size) + /* The requested mapping range should remain within the bounds */ + if (vma_nrpages > ent_nrpages - vma->vm_pgoff) return -EINVAL; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); @@ -88,26 +92,6 @@ static void flush_dcache_range_chunked(unsigned long start, unsigned long stop, } } -static void memtrace_clear_range(unsigned long start_pfn, - unsigned long nr_pages) -{ - unsigned long pfn; - - /* As HIGHMEM does not apply, use clear_page() directly. */ - for (pfn = start_pfn; pfn < start_pfn + nr_pages; pfn++) { - if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) - cond_resched(); - clear_page(__va(PFN_PHYS(pfn))); - } - /* - * Before we go ahead and use this range as cache inhibited range - * flush the cache. - */ - flush_dcache_range_chunked((unsigned long)pfn_to_kaddr(start_pfn), - (unsigned long)pfn_to_kaddr(start_pfn + nr_pages), - FLUSH_CHUNK_SIZE); -} - static u64 memtrace_alloc_node(u32 nid, u64 size) { const unsigned long nr_pages = PHYS_PFN(size); @@ -119,17 +103,18 @@ static u64 memtrace_alloc_node(u32 nid, u64 size) * by alloc_contig_pages(). */ page = alloc_contig_pages(nr_pages, GFP_KERNEL | __GFP_THISNODE | - __GFP_NOWARN, nid, NULL); + __GFP_NOWARN | __GFP_ZERO, nid, NULL); if (!page) return 0; start_pfn = page_to_pfn(page); /* - * Clear the range while we still have a linear mapping. - * - * TODO: use __GFP_ZERO with alloc_contig_pages() once supported. + * Before we go ahead and use this range as cache inhibited range + * flush the cache. */ - memtrace_clear_range(start_pfn, nr_pages); + flush_dcache_range_chunked((unsigned long)pfn_to_kaddr(start_pfn), + (unsigned long)pfn_to_kaddr(start_pfn + nr_pages), + FLUSH_CHUNK_SIZE); /* * Set pages PageOffline(), to indicate that nobody (e.g., hibernation, diff --git a/arch/powerpc/platforms/powernv/ocxl.c b/arch/powerpc/platforms/powernv/ocxl.c index 64a9c7125c29..f8139948348e 100644 --- a/arch/powerpc/platforms/powernv/ocxl.c +++ b/arch/powerpc/platforms/powernv/ocxl.c @@ -172,12 +172,11 @@ static void pnv_ocxl_fixup_actag(struct pci_dev *dev) if (phb->type != PNV_PHB_NPU_OCAPI) return; - mutex_lock(&links_list_lock); + guard(mutex)(&links_list_lock); link = find_link(dev); if (!link) { dev_warn(&dev->dev, "couldn't update actag information\n"); - mutex_unlock(&links_list_lock); return; } @@ -206,7 +205,6 @@ static void pnv_ocxl_fixup_actag(struct pci_dev *dev) dev_dbg(&dev->dev, "total actags for function: %d\n", link->fn_desired_actags[PCI_FUNC(dev->devfn)]); - mutex_unlock(&links_list_lock); } DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag); @@ -253,12 +251,11 @@ int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, { struct npu_link *link; - mutex_lock(&links_list_lock); + guard(mutex)(&links_list_lock); link = find_link(dev); if (!link) { dev_err(&dev->dev, "actag information not found\n"); - mutex_unlock(&links_list_lock); return -ENODEV; } /* @@ -274,7 +271,6 @@ int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, *enabled = link->fn_actags[PCI_FUNC(dev->devfn)].count; *supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)]; - mutex_unlock(&links_list_lock); return 0; } EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag); @@ -293,12 +289,11 @@ int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count) * * We only support one AFU-carrying function for now. */ - mutex_lock(&links_list_lock); + guard(mutex)(&links_list_lock); link = find_link(dev); if (!link) { dev_err(&dev->dev, "actag information not found\n"); - mutex_unlock(&links_list_lock); return -ENODEV; } @@ -309,7 +304,6 @@ int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count) break; } - mutex_unlock(&links_list_lock); dev_dbg(&dev->dev, "%d PASIDs available for function\n", rc ? 0 : *count); return rc; diff --git a/arch/powerpc/platforms/powernv/opal-core.c b/arch/powerpc/platforms/powernv/opal-core.c index c9a9b759cc92..784602a48afb 100644 --- a/arch/powerpc/platforms/powernv/opal-core.c +++ b/arch/powerpc/platforms/powernv/opal-core.c @@ -149,7 +149,7 @@ static Elf64_Word *__init auxv_to_elf64_notes(Elf64_Word *buf, /* end of vector */ bufp[idx++] = cpu_to_be64(AT_NULL); - buf = append_elf64_note(buf, CRASH_CORE_NOTE_NAME, NT_AUXV, + buf = append_elf64_note(buf, NN_AUXV, NT_AUXV, oc_conf->auxv_buf, AUXV_DESC_SZ); return buf; } @@ -159,7 +159,7 @@ static Elf64_Word *__init auxv_to_elf64_notes(Elf64_Word *buf, * Returns number of bytes read on success, -errno on failure. */ static ssize_t read_opalcore(struct file *file, struct kobject *kobj, - struct bin_attribute *bin_attr, char *to, + const struct bin_attribute *bin_attr, char *to, loff_t pos, size_t count) { struct opalcore *m; @@ -206,7 +206,7 @@ static ssize_t read_opalcore(struct file *file, struct kobject *kobj, return (tpos - pos); } -static struct bin_attribute opal_core_attr = { +static struct bin_attribute opal_core_attr __ro_after_init = { .attr = {.name = "core", .mode = 0400}, .read = read_opalcore }; @@ -252,7 +252,7 @@ static Elf64_Word * __init opalcore_append_cpu_notes(Elf64_Word *buf) * crashing CPU's prstatus. */ first_cpu_note = buf; - buf = append_elf64_note(buf, CRASH_CORE_NOTE_NAME, NT_PRSTATUS, + buf = append_elf64_note(buf, NN_PRSTATUS, NT_PRSTATUS, &prstatus, sizeof(prstatus)); for (i = 0; i < oc_conf->num_cpus; i++, bufp += size_per_thread) { @@ -279,7 +279,7 @@ static Elf64_Word * __init opalcore_append_cpu_notes(Elf64_Word *buf) fill_prstatus(&prstatus, thread_pir, ®s); if (thread_pir != oc_conf->crashing_cpu) { - buf = append_elf64_note(buf, CRASH_CORE_NOTE_NAME, + buf = append_elf64_note(buf, NN_PRSTATUS, NT_PRSTATUS, &prstatus, sizeof(prstatus)); } else { @@ -287,7 +287,7 @@ static Elf64_Word * __init opalcore_append_cpu_notes(Elf64_Word *buf) * Add crashing CPU as the first NT_PRSTATUS note for * GDB to process the core file appropriately. */ - append_elf64_note(first_cpu_note, CRASH_CORE_NOTE_NAME, + append_elf64_note(first_cpu_note, NN_PRSTATUS, NT_PRSTATUS, &prstatus, sizeof(prstatus)); } @@ -599,7 +599,7 @@ static struct attribute *mpipl_attr[] = { NULL, }; -static struct bin_attribute *mpipl_bin_attr[] = { +static const struct bin_attribute *const mpipl_bin_attr[] = { &opal_core_attr, NULL, diff --git a/arch/powerpc/platforms/powernv/opal-dump.c b/arch/powerpc/platforms/powernv/opal-dump.c index 608e4b68c5ea..cc3cc9ddf9d1 100644 --- a/arch/powerpc/platforms/powernv/opal-dump.c +++ b/arch/powerpc/platforms/powernv/opal-dump.c @@ -286,7 +286,7 @@ out: } static ssize_t dump_attr_read(struct file *filep, struct kobject *kobj, - struct bin_attribute *bin_attr, + const struct bin_attribute *bin_attr, char *buffer, loff_t pos, size_t count) { ssize_t rc; diff --git a/arch/powerpc/platforms/powernv/opal-elog.c b/arch/powerpc/platforms/powernv/opal-elog.c index 5db1e733143b..c3fc5d258146 100644 --- a/arch/powerpc/platforms/powernv/opal-elog.c +++ b/arch/powerpc/platforms/powernv/opal-elog.c @@ -156,7 +156,7 @@ static const struct kobj_type elog_ktype = { #define OPAL_MAX_ERRLOG_SIZE 16384 static ssize_t raw_attr_read(struct file *filep, struct kobject *kobj, - struct bin_attribute *bin_attr, + const struct bin_attribute *bin_attr, char *buffer, loff_t pos, size_t count) { int opal_rc; diff --git a/arch/powerpc/platforms/powernv/opal-flash.c b/arch/powerpc/platforms/powernv/opal-flash.c index d5ea04e8e4c5..a3f7a2928767 100644 --- a/arch/powerpc/platforms/powernv/opal-flash.c +++ b/arch/powerpc/platforms/powernv/opal-flash.c @@ -432,7 +432,7 @@ static int alloc_image_buf(char *buffer, size_t count) * and pre-allocate required memory. */ static ssize_t image_data_write(struct file *filp, struct kobject *kobj, - struct bin_attribute *bin_attr, + const struct bin_attribute *bin_attr, char *buffer, loff_t pos, size_t count) { int rc; diff --git a/arch/powerpc/platforms/powernv/opal-irqchip.c b/arch/powerpc/platforms/powernv/opal-irqchip.c index d92759c21fae..e180bd8e1400 100644 --- a/arch/powerpc/platforms/powernv/opal-irqchip.c +++ b/arch/powerpc/platforms/powernv/opal-irqchip.c @@ -191,7 +191,8 @@ int __init opal_event_init(void) * fall back to the legacy method (opal_event_request(...)) * anyway. */ dn = of_find_compatible_node(NULL, NULL, "ibm,opal-event"); - opal_event_irqchip.domain = irq_domain_add_linear(dn, MAX_NUM_EVENTS, + opal_event_irqchip.domain = irq_domain_create_linear(of_fwnode_handle(dn), + MAX_NUM_EVENTS, &opal_event_domain_ops, &opal_event_irqchip); of_node_put(dn); if (!opal_event_irqchip.domain) { diff --git a/arch/powerpc/platforms/powernv/opal-msglog.c b/arch/powerpc/platforms/powernv/opal-msglog.c index 22d6efe17b0d..992a6b379a66 100644 --- a/arch/powerpc/platforms/powernv/opal-msglog.c +++ b/arch/powerpc/platforms/powernv/opal-msglog.c @@ -94,13 +94,13 @@ ssize_t opal_msglog_copy(char *to, loff_t pos, size_t count) } static ssize_t opal_msglog_read(struct file *file, struct kobject *kobj, - struct bin_attribute *bin_attr, char *to, + const struct bin_attribute *bin_attr, char *to, loff_t pos, size_t count) { return opal_msglog_copy(to, pos, count); } -static struct bin_attribute opal_msglog_attr = { +static struct bin_attribute opal_msglog_attr __ro_after_init = { .attr = {.name = "msglog", .mode = 0400}, .read = opal_msglog_read }; diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index 5d0f35bb917e..09bd93464b4f 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -180,10 +180,7 @@ int __init early_init_dt_scan_recoverable_ranges(unsigned long node, /* * Allocate a buffer to hold the MC recoverable ranges. */ - mc_recoverable_range = memblock_alloc(size, __alignof__(u64)); - if (!mc_recoverable_range) - panic("%s: Failed to allocate %u bytes align=0x%lx\n", - __func__, size, __alignof__(u64)); + mc_recoverable_range = memblock_alloc_or_panic(size, __alignof__(u64)); for (i = 0; i < mc_recoverable_range_len; i++) { mc_recoverable_range[i].start_addr = diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c deleted file mode 100644 index 7e419de71db8..000000000000 --- a/arch/powerpc/platforms/powernv/pci-cxl.c +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright 2014-2016 IBM Corp. - */ - -#include <linux/module.h> -#include <misc/cxl-base.h> -#include <asm/pnv-pci.h> -#include <asm/opal.h> - -#include "pci.h" - -int pnv_phb_to_cxl_mode(struct pci_dev *dev, uint64_t mode) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - struct pnv_ioda_pe *pe; - int rc; - - pe = pnv_ioda_get_pe(dev); - if (!pe) - return -ENODEV; - - pe_info(pe, "Switching PHB to CXL\n"); - - rc = opal_pci_set_phb_cxl_mode(phb->opal_id, mode, pe->pe_number); - if (rc == OPAL_UNSUPPORTED) - dev_err(&dev->dev, "Required cxl mode not supported by firmware - update skiboot\n"); - else if (rc) - dev_err(&dev->dev, "opal_pci_set_phb_cxl_mode failed: %i\n", rc); - - return rc; -} -EXPORT_SYMBOL(pnv_phb_to_cxl_mode); - -/* Find PHB for cxl dev and allocate MSI hwirqs? - * Returns the absolute hardware IRQ number - */ -int pnv_cxl_alloc_hwirqs(struct pci_dev *dev, int num) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - int hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, num); - - if (hwirq < 0) { - dev_warn(&dev->dev, "Failed to find a free MSI\n"); - return -ENOSPC; - } - - return phb->msi_base + hwirq; -} -EXPORT_SYMBOL(pnv_cxl_alloc_hwirqs); - -void pnv_cxl_release_hwirqs(struct pci_dev *dev, int hwirq, int num) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - - msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq - phb->msi_base, num); -} -EXPORT_SYMBOL(pnv_cxl_release_hwirqs); - -void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs, - struct pci_dev *dev) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - int i, hwirq; - - for (i = 1; i < CXL_IRQ_RANGES; i++) { - if (!irqs->range[i]) - continue; - pr_devel("cxl release irq range 0x%x: offset: 0x%lx limit: %ld\n", - i, irqs->offset[i], - irqs->range[i]); - hwirq = irqs->offset[i] - phb->msi_base; - msi_bitmap_free_hwirqs(&phb->msi_bmp, hwirq, - irqs->range[i]); - } -} -EXPORT_SYMBOL(pnv_cxl_release_hwirq_ranges); - -int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs, - struct pci_dev *dev, int num) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - int i, hwirq, try; - - memset(irqs, 0, sizeof(struct cxl_irq_ranges)); - - /* 0 is reserved for the multiplexed PSL DSI interrupt */ - for (i = 1; i < CXL_IRQ_RANGES && num; i++) { - try = num; - while (try) { - hwirq = msi_bitmap_alloc_hwirqs(&phb->msi_bmp, try); - if (hwirq >= 0) - break; - try /= 2; - } - if (!try) - goto fail; - - irqs->offset[i] = phb->msi_base + hwirq; - irqs->range[i] = try; - pr_devel("cxl alloc irq range 0x%x: offset: 0x%lx limit: %li\n", - i, irqs->offset[i], irqs->range[i]); - num -= try; - } - if (num) - goto fail; - - return 0; -fail: - pnv_cxl_release_hwirq_ranges(irqs, dev); - return -ENOSPC; -} -EXPORT_SYMBOL(pnv_cxl_alloc_hwirq_ranges); - -int pnv_cxl_get_irq_count(struct pci_dev *dev) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - - return phb->msi_bmp.irq_count; -} -EXPORT_SYMBOL(pnv_cxl_get_irq_count); - -int pnv_cxl_ioda_msi_setup(struct pci_dev *dev, unsigned int hwirq, - unsigned int virq) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - unsigned int xive_num = hwirq - phb->msi_base; - struct pnv_ioda_pe *pe; - int rc; - - if (!(pe = pnv_ioda_get_pe(dev))) - return -ENODEV; - - /* Assign XIVE to PE */ - rc = opal_pci_set_xive_pe(phb->opal_id, pe->pe_number, xive_num); - if (rc) { - pe_warn(pe, "%s: OPAL error %d setting msi_base 0x%x " - "hwirq 0x%x XIVE 0x%x PE\n", - pci_name(dev), rc, phb->msi_base, hwirq, xive_num); - return -EIO; - } - pnv_set_msi_irq_chip(phb, virq); - - return 0; -} -EXPORT_SYMBOL(pnv_cxl_ioda_msi_setup); diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index b0a14e48175c..d8ccf2c9b98a 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -39,8 +39,6 @@ #include <asm/mmzone.h> #include <asm/xive.h> -#include <misc/cxl-base.h> - #include "powernv.h" #include "pci.h" #include "../../../../drivers/pci/pci.h" @@ -1636,47 +1634,6 @@ int64_t pnv_opal_pci_msi_eoi(struct irq_data *d) return opal_pci_msi_eoi(phb->opal_id, d->parent_data->hwirq); } -/* - * The IRQ data is mapped in the XICS domain, with OPAL HW IRQ numbers - */ -static void pnv_ioda2_msi_eoi(struct irq_data *d) -{ - int64_t rc; - unsigned int hw_irq = (unsigned int)irqd_to_hwirq(d); - struct pci_controller *hose = irq_data_get_irq_chip_data(d); - struct pnv_phb *phb = hose->private_data; - - rc = opal_pci_msi_eoi(phb->opal_id, hw_irq); - WARN_ON_ONCE(rc); - - icp_native_eoi(d); -} - -/* P8/CXL only */ -void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq) -{ - struct irq_data *idata; - struct irq_chip *ichip; - - /* The MSI EOI OPAL call is only needed on PHB3 */ - if (phb->model != PNV_PHB_MODEL_PHB3) - return; - - if (!phb->ioda.irq_chip_init) { - /* - * First time we setup an MSI IRQ, we need to setup the - * corresponding IRQ chip to route correctly. - */ - idata = irq_get_irq_data(virq); - ichip = irq_data_get_irq_chip(idata); - phb->ioda.irq_chip_init = 1; - phb->ioda.irq_chip = *ichip; - phb->ioda.irq_chip.irq_eoi = pnv_ioda2_msi_eoi; - } - irq_set_chip(virq, &phb->ioda.irq_chip); - irq_set_chip_data(virq, phb->hose); -} - static struct irq_chip pnv_pci_msi_irq_chip; /* @@ -1924,7 +1881,7 @@ static const struct irq_domain_ops pnv_irq_domain_ops = { static int __init pnv_msi_allocate_domains(struct pci_controller *hose, unsigned int count) { struct pnv_phb *phb = hose->private_data; - struct irq_domain *parent = irq_get_default_host(); + struct irq_domain *parent = irq_get_default_domain(); hose->fwnode = irq_domain_alloc_named_id_fwnode("PNV-MSI", phb->opal_id); if (!hose->fwnode) @@ -1940,7 +1897,7 @@ static int __init pnv_msi_allocate_domains(struct pci_controller *hose, unsigned return -ENOMEM; } - hose->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(hose->dn), + hose->msi_domain = pci_msi_create_irq_domain(of_fwnode_handle(hose->dn), &pnv_msi_domain_info, hose->dev_domain); if (!hose->msi_domain) { diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index 35f566aa0424..b2c1da025410 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -14,7 +14,6 @@ #include <linux/io.h> #include <linux/msi.h> #include <linux/iommu.h> -#include <linux/sched/mm.h> #include <asm/sections.h> #include <asm/io.h> @@ -33,8 +32,6 @@ #include "powernv.h" #include "pci.h" -static DEFINE_MUTEX(tunnel_mutex); - int pnv_pci_get_slot_id(struct device_node *np, uint64_t *id) { struct device_node *node = np; @@ -744,64 +741,6 @@ struct iommu_table *pnv_pci_table_alloc(int nid) return tbl; } -struct device_node *pnv_pci_get_phb_node(struct pci_dev *dev) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - - return of_node_get(hose->dn); -} -EXPORT_SYMBOL(pnv_pci_get_phb_node); - -int pnv_pci_set_tunnel_bar(struct pci_dev *dev, u64 addr, int enable) -{ - struct pnv_phb *phb = pci_bus_to_pnvhb(dev->bus); - u64 tunnel_bar; - __be64 val; - int rc; - - if (!opal_check_token(OPAL_PCI_GET_PBCQ_TUNNEL_BAR)) - return -ENXIO; - if (!opal_check_token(OPAL_PCI_SET_PBCQ_TUNNEL_BAR)) - return -ENXIO; - - mutex_lock(&tunnel_mutex); - rc = opal_pci_get_pbcq_tunnel_bar(phb->opal_id, &val); - if (rc != OPAL_SUCCESS) { - rc = -EIO; - goto out; - } - tunnel_bar = be64_to_cpu(val); - if (enable) { - /* - * Only one device per PHB can use atomics. - * Our policy is first-come, first-served. - */ - if (tunnel_bar) { - if (tunnel_bar != addr) - rc = -EBUSY; - else - rc = 0; /* Setting same address twice is ok */ - goto out; - } - } else { - /* - * The device that owns atomics and wants to release - * them must pass the same address with enable == 0. - */ - if (tunnel_bar != addr) { - rc = -EPERM; - goto out; - } - addr = 0x0ULL; - } - rc = opal_pci_set_pbcq_tunnel_bar(phb->opal_id, addr); - rc = opal_error_code(rc); -out: - mutex_unlock(&tunnel_mutex); - return rc; -} -EXPORT_SYMBOL_GPL(pnv_pci_set_tunnel_bar); - void pnv_pci_shutdown(void) { struct pci_controller *hose; diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index 93fba1f8661f..42075501663b 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -163,7 +163,6 @@ struct pnv_phb { unsigned int *io_segmap; /* IRQ chip */ - int irq_chip_init; struct irq_chip irq_chip; /* Sorted list of used PE's based @@ -281,7 +280,6 @@ extern int pnv_eeh_phb_reset(struct pci_controller *hose, int option); extern struct pnv_ioda_pe *pnv_pci_bdfn_to_pe(struct pnv_phb *phb, u16 bdfn); extern struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev); -extern void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq); extern unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift, __u64 window_size, __u32 levels); extern int pnv_eeh_post_init(void); diff --git a/arch/powerpc/platforms/powernv/ultravisor.c b/arch/powerpc/platforms/powernv/ultravisor.c index 67c8c4b2d8b1..c526871a1229 100644 --- a/arch/powerpc/platforms/powernv/ultravisor.c +++ b/arch/powerpc/platforms/powernv/ultravisor.c @@ -32,13 +32,13 @@ int __init early_init_dt_scan_ultravisor(unsigned long node, const char *uname, static struct memcons *uv_memcons; static ssize_t uv_msglog_read(struct file *file, struct kobject *kobj, - struct bin_attribute *bin_attr, char *to, + const struct bin_attribute *bin_attr, char *to, loff_t pos, size_t count) { return memcons_copy(uv_memcons, to, pos, count); } -static struct bin_attribute uv_msglog_attr = { +static struct bin_attribute uv_msglog_attr __ro_after_init = { .attr = {.name = "msglog", .mode = 0400}, .read = uv_msglog_read }; diff --git a/arch/powerpc/platforms/ps3/device-init.c b/arch/powerpc/platforms/ps3/device-init.c index 61722133eb2d..22d91ac424dd 100644 --- a/arch/powerpc/platforms/ps3/device-init.c +++ b/arch/powerpc/platforms/ps3/device-init.c @@ -14,6 +14,7 @@ #include <linux/slab.h> #include <linux/reboot.h> #include <linux/rcuwait.h> +#include <linux/string_choices.h> #include <asm/firmware.h> #include <asm/lv1call.h> @@ -724,7 +725,7 @@ static irqreturn_t ps3_notification_interrupt(int irq, void *data) static int ps3_notification_read_write(struct ps3_notification_device *dev, u64 lpar, int write) { - const char *op = write ? "write" : "read"; + const char *op = str_write_read(write); unsigned long flags; int res; diff --git a/arch/powerpc/platforms/ps3/interrupt.c b/arch/powerpc/platforms/ps3/interrupt.c index af3fe9f04f24..a4ad4b49eef7 100644 --- a/arch/powerpc/platforms/ps3/interrupt.c +++ b/arch/powerpc/platforms/ps3/interrupt.c @@ -743,8 +743,8 @@ void __init ps3_init_IRQ(void) unsigned cpu; struct irq_domain *host; - host = irq_domain_add_nomap(NULL, PS3_PLUG_MAX + 1, &ps3_host_ops, NULL); - irq_set_default_host(host); + host = irq_domain_create_nomap(NULL, PS3_PLUG_MAX + 1, &ps3_host_ops, NULL); + irq_set_default_domain(host); for_each_possible_cpu(cpu) { struct ps3_private *pd = &per_cpu(ps3_private, cpu); diff --git a/arch/powerpc/platforms/ps3/setup.c b/arch/powerpc/platforms/ps3/setup.c index 5144f11359f7..150c09b58ae8 100644 --- a/arch/powerpc/platforms/ps3/setup.c +++ b/arch/powerpc/platforms/ps3/setup.c @@ -115,10 +115,7 @@ static void __init prealloc(struct ps3_prealloc *p) if (!p->size) return; - p->address = memblock_alloc(p->size, p->align); - if (!p->address) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, p->size, p->align); + p->address = memblock_alloc_or_panic(p->size, p->align); printk(KERN_INFO "%s: %lu bytes at %p\n", p->name, p->size, p->address); diff --git a/arch/powerpc/platforms/ps3/spu.c b/arch/powerpc/platforms/ps3/spu.c index 4a2520ec6d7f..61b37c9400b2 100644 --- a/arch/powerpc/platforms/ps3/spu.c +++ b/arch/powerpc/platforms/ps3/spu.c @@ -190,10 +190,10 @@ static void spu_unmap(struct spu *spu) static int __init setup_areas(struct spu *spu) { struct table {char* name; unsigned long addr; unsigned long size;}; - unsigned long shadow_flags = pgprot_val(pgprot_noncached_wc(PAGE_KERNEL_RO)); spu_pdata(spu)->shadow = ioremap_prot(spu_pdata(spu)->shadow_addr, - sizeof(struct spe_shadow), shadow_flags); + sizeof(struct spe_shadow), + pgprot_noncached_wc(PAGE_KERNEL_RO)); if (!spu_pdata(spu)->shadow) { pr_debug("%s:%d: ioremap shadow failed\n", __func__, __LINE__); goto fail_ioremap; diff --git a/arch/powerpc/platforms/pseries/Kconfig b/arch/powerpc/platforms/pseries/Kconfig index 42fc66e97539..fa3c2fff082a 100644 --- a/arch/powerpc/platforms/pseries/Kconfig +++ b/arch/powerpc/platforms/pseries/Kconfig @@ -23,6 +23,7 @@ config PPC_PSERIES select FORCE_SMP select SWIOTLB select ARCH_SUPPORTS_PER_VMA_LOCK + select PPC_RADIX_BROADCAST_TLBIE if PPC_RADIX_MMU default y config PARAVIRT @@ -128,6 +129,15 @@ config CMM will be reused for other LPARs. The interface allows firmware to balance memory across many LPARs. +config HTMDUMP + tristate "PowerVM data dumper" + depends on PPC_PSERIES && DEBUG_FS + default m + help + Select this option, if you want to enable the kernel debugfs + interface to dump the Hardware Trace Macro (HTM) function data + in the LPAR. + config HV_PERF_CTRS bool "Hypervisor supplied PMU events (24x7 & GPCI)" default y diff --git a/arch/powerpc/platforms/pseries/Makefile b/arch/powerpc/platforms/pseries/Makefile index 7bf506f6b8c8..57222678bb3f 100644 --- a/arch/powerpc/platforms/pseries/Makefile +++ b/arch/powerpc/platforms/pseries/Makefile @@ -3,7 +3,8 @@ ccflags-$(CONFIG_PPC_PSERIES_DEBUG) += -DDEBUG obj-y := lpar.o hvCall.o nvram.o reconfig.o \ of_helpers.o rtas-work-area.o papr-sysparm.o \ - papr-vpd.o \ + papr-rtas-common.o papr-vpd.o papr-indices.o \ + papr-platform-dump.o papr-phy-attest.o \ setup.o iommu.o event_sources.o ras.o \ firmware.o power.o dlpar.o mobility.o rng.o \ pci.o pci_dlpar.o eeh_pseries.o msi.o \ @@ -19,6 +20,7 @@ obj-$(CONFIG_HVC_CONSOLE) += hvconsole.o obj-$(CONFIG_HVCS) += hvcserver.o obj-$(CONFIG_HCALL_STATS) += hvCall_inst.o obj-$(CONFIG_CMM) += cmm.o +obj-$(CONFIG_HTMDUMP) += htmdump.o obj-$(CONFIG_IO_EVENT_IRQ) += io_event_irq.o obj-$(CONFIG_LPARCFG) += lparcfg.o obj-$(CONFIG_IBMVIO) += vio.o diff --git a/arch/powerpc/platforms/pseries/cmm.c b/arch/powerpc/platforms/pseries/cmm.c index 5f4037c1d7fe..5e0a718d1be7 100644 --- a/arch/powerpc/platforms/pseries/cmm.c +++ b/arch/powerpc/platforms/pseries/cmm.c @@ -532,7 +532,6 @@ static int cmm_migratepage(struct balloon_dev_info *b_dev_info, spin_lock_irqsave(&b_dev_info->pages_lock, flags); balloon_page_insert(b_dev_info, newpage); - balloon_page_delete(page); b_dev_info->isolated_pages--; spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); @@ -542,6 +541,7 @@ static int cmm_migratepage(struct balloon_dev_info *b_dev_info, */ plpar_page_set_active(page); + balloon_page_finalize(page); /* balloon page list reference */ put_page(page); diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c index 213aa26dc8b3..979487da6522 100644 --- a/arch/powerpc/platforms/pseries/dlpar.c +++ b/arch/powerpc/platforms/pseries/dlpar.c @@ -404,6 +404,45 @@ get_device_node_with_drc_info(u32 index) return NULL; } +static struct device_node * +get_device_node_with_drc_indexes(u32 drc_index) +{ + struct device_node *np = NULL; + u32 nr_indexes, index; + int i, rc; + + for_each_node_with_property(np, "ibm,drc-indexes") { + /* + * First element in the array is the total number of + * DRC indexes returned. + */ + rc = of_property_read_u32_index(np, "ibm,drc-indexes", + 0, &nr_indexes); + if (rc) + goto out_put_np; + + /* + * Retrieve DRC index from the list and return the + * device node if matched with the specified index. + */ + for (i = 0; i < nr_indexes; i++) { + rc = of_property_read_u32_index(np, "ibm,drc-indexes", + i+1, &index); + if (rc) + goto out_put_np; + + if (drc_index == index) + return np; + } + } + + return NULL; + +out_put_np: + of_node_put(np); + return NULL; +} + static int dlpar_hp_dt_add(u32 index) { struct device_node *np, *nodes; @@ -423,10 +462,19 @@ static int dlpar_hp_dt_add(u32 index) goto out; } + /* + * Recent FW provides ibm,drc-info property. So search + * for the user specified DRC index from ibm,drc-info + * property. If this property is not available, search + * in the indexes array from ibm,drc-indexes property. + */ np = get_device_node_with_drc_info(index); - if (!np) - return -EIO; + if (!np) { + np = get_device_node_with_drc_indexes(index); + if (!np) + return -EIO; + } /* Next, configure the connector. */ nodes = dlpar_configure_connector(cpu_to_be32(index), np); diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c index 1893f66371fa..b12ef382fec7 100644 --- a/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -580,8 +580,10 @@ static int pseries_eeh_get_state(struct eeh_pe *pe, int *delay) switch(rets[0]) { case 0: - result = EEH_STATE_MMIO_ACTIVE | - EEH_STATE_DMA_ACTIVE; + result = EEH_STATE_MMIO_ACTIVE | + EEH_STATE_DMA_ACTIVE | + EEH_STATE_MMIO_ENABLED | + EEH_STATE_DMA_ENABLED; break; case 1: result = EEH_STATE_RESET_ACTIVE | diff --git a/arch/powerpc/platforms/pseries/htmdump.c b/arch/powerpc/platforms/pseries/htmdump.c new file mode 100644 index 000000000000..742ec52c9d4d --- /dev/null +++ b/arch/powerpc/platforms/pseries/htmdump.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) IBM Corporation, 2024 + */ + +#define pr_fmt(fmt) "htmdump: " fmt + +#include <linux/debugfs.h> +#include <linux/module.h> +#include <asm/io.h> +#include <asm/machdep.h> +#include <asm/plpar_wrappers.h> +#include <asm/kvm_guest.h> + +static void *htm_buf; +static void *htm_status_buf; +static void *htm_info_buf; +static void *htm_caps_buf; +static u32 nodeindex; +static u32 nodalchipindex; +static u32 coreindexonchip; +static u32 htmtype; +static u32 htmconfigure; +static u32 htmstart; +static u32 htmsetup; +static u64 htmflags; + +static struct dentry *htmdump_debugfs_dir; +#define HTM_ENABLE 1 +#define HTM_DISABLE 0 +#define HTM_NOWRAP 1 +#define HTM_WRAP 0 + +/* + * Check the return code for H_HTM hcall. + * Return non-zero value (1) if either H_PARTIAL or H_SUCCESS + * is returned. For other return codes: + * Return zero if H_NOT_AVAILABLE. + * Return -EBUSY if hcall return busy. + * Return -EINVAL if any parameter or operation is not valid. + * Return -EPERM if HTM Virtualization Engine Technology code + * is not applied. + * Return -EIO if the HTM state is not valid. + */ +static ssize_t htm_return_check(long rc) +{ + switch (rc) { + case H_SUCCESS: + /* H_PARTIAL for the case where all available data can't be + * returned due to buffer size constraint. + */ + case H_PARTIAL: + break; + /* H_NOT_AVAILABLE indicates reading from an offset outside the range, + * i.e. past end of file. + */ + case H_NOT_AVAILABLE: + return 0; + case H_BUSY: + case H_LONG_BUSY_ORDER_1_MSEC: + case H_LONG_BUSY_ORDER_10_MSEC: + case H_LONG_BUSY_ORDER_100_MSEC: + case H_LONG_BUSY_ORDER_1_SEC: + case H_LONG_BUSY_ORDER_10_SEC: + case H_LONG_BUSY_ORDER_100_SEC: + return -EBUSY; + case H_PARAMETER: + case H_P2: + case H_P3: + case H_P4: + case H_P5: + case H_P6: + return -EINVAL; + case H_STATE: + return -EIO; + case H_AUTHORITY: + return -EPERM; + } + + /* + * Return 1 for H_SUCCESS/H_PARTIAL + */ + return 1; +} + +static ssize_t htmdump_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + void *htm_buf = filp->private_data; + unsigned long page, read_size, available; + loff_t offset; + long rc, ret; + + page = ALIGN_DOWN(*ppos, PAGE_SIZE); + offset = (*ppos) % PAGE_SIZE; + + /* + * Invoke H_HTM call with: + * - operation as htm dump (H_HTM_OP_DUMP_DATA) + * - last three values are address, size and offset + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_DUMP_DATA, virt_to_phys(htm_buf), + PAGE_SIZE, page); + + ret = htm_return_check(rc); + if (ret <= 0) { + pr_debug("H_HTM hcall failed for op: H_HTM_OP_DUMP_DATA, returning %ld\n", ret); + return ret; + } + + available = PAGE_SIZE; + read_size = min(count, available); + *ppos += read_size; + return simple_read_from_buffer(ubuf, count, &offset, htm_buf, available); +} + +static const struct file_operations htmdump_fops = { + .llseek = NULL, + .read = htmdump_read, + .open = simple_open, +}; + +static int htmconfigure_set(void *data, u64 val) +{ + long rc, ret; + unsigned long param1 = -1, param2 = -1; + + /* + * value as 1 : configure HTM. + * value as 0 : deconfigure HTM. Return -EINVAL for + * other values. + */ + if (val == HTM_ENABLE) { + /* + * Invoke H_HTM call with: + * - operation as htm configure (H_HTM_OP_CONFIGURE) + * - If htmflags is set, param1 and param2 will be -1 + * which is an indicator to use default htm mode reg mask + * and htm mode reg value. + * - last three values are unused, hence set to zero + */ + if (!htmflags) { + param1 = 0; + param2 = 0; + } + + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_CONFIGURE, param1, param2, 0); + } else if (val == HTM_DISABLE) { + /* + * Invoke H_HTM call with: + * - operation as htm deconfigure (H_HTM_OP_DECONFIGURE) + * - last three values are unused, hence set to zero + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_DECONFIGURE, 0, 0, 0); + } else + return -EINVAL; + + ret = htm_return_check(rc); + if (ret <= 0) { + pr_debug("H_HTM hcall failed, returning %ld\n", ret); + return ret; + } + + /* Set htmconfigure if operation succeeds */ + htmconfigure = val; + + return 0; +} + +static int htmconfigure_get(void *data, u64 *val) +{ + *val = htmconfigure; + return 0; +} + +static int htmstart_set(void *data, u64 val) +{ + long rc, ret; + + /* + * value as 1: start HTM + * value as 0: stop HTM + * Return -EINVAL for other values. + */ + if (val == HTM_ENABLE) { + /* + * Invoke H_HTM call with: + * - operation as htm start (H_HTM_OP_START) + * - last three values are unused, hence set to zero + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_START, 0, 0, 0); + + } else if (val == HTM_DISABLE) { + /* + * Invoke H_HTM call with: + * - operation as htm stop (H_HTM_OP_STOP) + * - last three values are unused, hence set to zero + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_STOP, 0, 0, 0); + } else + return -EINVAL; + + ret = htm_return_check(rc); + if (ret <= 0) { + pr_debug("H_HTM hcall failed, returning %ld\n", ret); + return ret; + } + + /* Set htmstart if H_HTM_OP_START/H_HTM_OP_STOP operation succeeds */ + htmstart = val; + + return 0; +} + +static int htmstart_get(void *data, u64 *val) +{ + *val = htmstart; + return 0; +} + +static ssize_t htmstatus_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + void *htm_status_buf = filp->private_data; + long rc, ret; + u64 *num_entries; + u64 to_copy; + int htmstatus_flag; + + /* + * Invoke H_HTM call with: + * - operation as htm status (H_HTM_OP_STATUS) + * - last three values as addr, size and offset + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_STATUS, virt_to_phys(htm_status_buf), + PAGE_SIZE, 0); + + ret = htm_return_check(rc); + if (ret <= 0) { + pr_debug("H_HTM hcall failed for op: H_HTM_OP_STATUS, returning %ld\n", ret); + return ret; + } + + /* + * HTM status buffer, start of buffer + 0x10 gives the + * number of HTM entries in the buffer. Each nest htm status + * entry is 0x6 bytes where each core htm status entry is + * 0x8 bytes. + * So total count to copy is: + * 32 bytes (for first 7 fields) + (number of HTM entries * entry size) + */ + num_entries = htm_status_buf + 0x10; + if (htmtype == 0x2) + htmstatus_flag = 0x8; + else + htmstatus_flag = 0x6; + to_copy = 32 + (be64_to_cpu(*num_entries) * htmstatus_flag); + return simple_read_from_buffer(ubuf, count, ppos, htm_status_buf, to_copy); +} + +static const struct file_operations htmstatus_fops = { + .llseek = NULL, + .read = htmstatus_read, + .open = simple_open, +}; + +static ssize_t htminfo_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + void *htm_info_buf = filp->private_data; + long rc, ret; + u64 *num_entries; + u64 to_copy; + + /* + * Invoke H_HTM call with: + * - operation as htm status (H_HTM_OP_STATUS) + * - last three values as addr, size and offset + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_DUMP_SYSPROC_CONF, virt_to_phys(htm_info_buf), + PAGE_SIZE, 0); + + ret = htm_return_check(rc); + if (ret <= 0) { + pr_debug("H_HTM hcall failed for op: H_HTM_OP_DUMP_SYSPROC_CONF, returning %ld\n", ret); + return ret; + } + + /* + * HTM status buffer, start of buffer + 0x10 gives the + * number of HTM entries in the buffer. Each entry of processor + * is 16 bytes. + * + * So total count to copy is: + * 32 bytes (for first 5 fields) + (number of HTM entries * entry size) + */ + num_entries = htm_info_buf + 0x10; + to_copy = 32 + (be64_to_cpu(*num_entries) * 16); + return simple_read_from_buffer(ubuf, count, ppos, htm_info_buf, to_copy); +} + +static ssize_t htmcaps_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *ppos) +{ + void *htm_caps_buf = filp->private_data; + long rc, ret; + + /* + * Invoke H_HTM call with: + * - operation as htm capabilities (H_HTM_OP_CAPABILITIES) + * - last three values as addr, size (0x80 for Capabilities Output Buffer + * and zero + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_CAPABILITIES, virt_to_phys(htm_caps_buf), + 0x80, 0); + + ret = htm_return_check(rc); + if (ret <= 0) { + pr_debug("H_HTM hcall failed for op: H_HTM_OP_CAPABILITIES, returning %ld\n", ret); + return ret; + } + + return simple_read_from_buffer(ubuf, count, ppos, htm_caps_buf, 0x80); +} + +static const struct file_operations htminfo_fops = { + .llseek = NULL, + .read = htminfo_read, + .open = simple_open, +}; + +static const struct file_operations htmcaps_fops = { + .llseek = NULL, + .read = htmcaps_read, + .open = simple_open, +}; + +static int htmsetup_set(void *data, u64 val) +{ + long rc, ret; + + /* + * Input value: HTM buffer size in the power of 2 + * example: hex value 0x21 ( decimal: 33 ) is for + * 8GB + * Invoke H_HTM call with: + * - operation as htm start (H_HTM_OP_SETUP) + * - parameter 1 set to input value. + * - last two values are unused, hence set to zero + */ + rc = htm_hcall_wrapper(htmflags, nodeindex, nodalchipindex, coreindexonchip, + htmtype, H_HTM_OP_SETUP, val, 0, 0); + + ret = htm_return_check(rc); + if (ret <= 0) { + pr_debug("H_HTM hcall failed for op: H_HTM_OP_SETUP, returning %ld\n", ret); + return ret; + } + + /* Set htmsetup if H_HTM_OP_SETUP operation succeeds */ + htmsetup = val; + + return 0; +} + +static int htmsetup_get(void *data, u64 *val) +{ + *val = htmsetup; + return 0; +} + +static int htmflags_set(void *data, u64 val) +{ + /* + * Input value: + * Currently supported flag value is to enable/disable + * HTM buffer wrap. wrap is used along with "configure" + * to prevent HTM buffer from wrapping. + * Writing 1 will set noWrap while configuring HTM + */ + if (val == HTM_NOWRAP) + htmflags = H_HTM_FLAGS_NOWRAP; + else if (val == HTM_WRAP) + htmflags = 0; + else + return -EINVAL; + + return 0; +} + +static int htmflags_get(void *data, u64 *val) +{ + *val = htmflags; + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(htmconfigure_fops, htmconfigure_get, htmconfigure_set, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(htmstart_fops, htmstart_get, htmstart_set, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(htmsetup_fops, htmsetup_get, htmsetup_set, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(htmflags_fops, htmflags_get, htmflags_set, "%llu\n"); + +static int htmdump_init_debugfs(void) +{ + htm_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!htm_buf) { + pr_err("Failed to allocate htmdump buf\n"); + return -ENOMEM; + } + + htmdump_debugfs_dir = debugfs_create_dir("htmdump", + arch_debugfs_dir); + + debugfs_create_u32("nodeindex", 0600, + htmdump_debugfs_dir, &nodeindex); + debugfs_create_u32("nodalchipindex", 0600, + htmdump_debugfs_dir, &nodalchipindex); + debugfs_create_u32("coreindexonchip", 0600, + htmdump_debugfs_dir, &coreindexonchip); + debugfs_create_u32("htmtype", 0600, + htmdump_debugfs_dir, &htmtype); + debugfs_create_file("trace", 0400, htmdump_debugfs_dir, htm_buf, &htmdump_fops); + + /* + * Debugfs interface files to control HTM operations: + */ + debugfs_create_file("htmconfigure", 0600, htmdump_debugfs_dir, NULL, &htmconfigure_fops); + debugfs_create_file("htmstart", 0600, htmdump_debugfs_dir, NULL, &htmstart_fops); + debugfs_create_file("htmsetup", 0600, htmdump_debugfs_dir, NULL, &htmsetup_fops); + debugfs_create_file("htmflags", 0600, htmdump_debugfs_dir, NULL, &htmflags_fops); + + /* Debugfs interface file to present status of HTM */ + htm_status_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!htm_status_buf) { + pr_err("Failed to allocate htmstatus buf\n"); + return -ENOMEM; + } + + /* Debugfs interface file to present System Processor Configuration */ + htm_info_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!htm_info_buf) { + pr_err("Failed to allocate htm info buf\n"); + return -ENOMEM; + } + + /* Debugfs interface file to present HTM capabilities */ + htm_caps_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!htm_caps_buf) { + pr_err("Failed to allocate htm caps buf\n"); + return -ENOMEM; + } + + debugfs_create_file("htmstatus", 0400, htmdump_debugfs_dir, htm_status_buf, &htmstatus_fops); + debugfs_create_file("htminfo", 0400, htmdump_debugfs_dir, htm_info_buf, &htminfo_fops); + debugfs_create_file("htmcaps", 0400, htmdump_debugfs_dir, htm_caps_buf, &htmcaps_fops); + + return 0; +} + +static int __init htmdump_init(void) +{ + /* Disable on kvm guest */ + if (is_kvm_guest()) { + pr_info("htmdump not supported inside KVM guest\n"); + return -EOPNOTSUPP; + } + + if (htmdump_init_debugfs()) + return -ENOMEM; + + return 0; +} + +static void __exit htmdump_exit(void) +{ + debugfs_remove_recursive(htmdump_debugfs_dir); + kfree(htm_buf); +} + +module_init(htmdump_init); +module_exit(htmdump_exit); +MODULE_DESCRIPTION("PHYP Hardware Trace Macro (HTM) data dumper"); +MODULE_LICENSE("GPL"); diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index 534cd159e9ab..eec333dd2e59 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -52,7 +52,8 @@ enum { enum { DDW_EXT_SIZE = 0, DDW_EXT_RESET_DMA_WIN = 1, - DDW_EXT_QUERY_OUT_SIZE = 2 + DDW_EXT_QUERY_OUT_SIZE = 2, + DDW_EXT_LIMITED_ADDR_MODE = 3 }; static struct iommu_table *iommu_pseries_alloc_table(int node) @@ -196,7 +197,7 @@ static void tce_iommu_userspace_view_free(struct iommu_table *tbl) static void tce_free_pSeries(struct iommu_table *tbl) { - if (!tbl->it_userspace) + if (tbl->it_userspace) tce_iommu_userspace_view_free(tbl); } @@ -1284,17 +1285,13 @@ static LIST_HEAD(failed_ddw_pdn_list); static phys_addr_t ddw_memory_hotplug_max(void) { - resource_size_t max_addr = memory_hotplug_max(); - struct device_node *memory; + resource_size_t max_addr; - for_each_node_by_type(memory, "memory") { - struct resource res; - - if (of_address_to_resource(memory, 0, &res)) - continue; - - max_addr = max_t(resource_size_t, max_addr, res.end + 1); - } +#if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG) + max_addr = hot_add_drconf_memory_max(); +#else + max_addr = memblock_end_of_DRAM(); +#endif return max_addr; } @@ -1331,6 +1328,54 @@ static void reset_dma_window(struct pci_dev *dev, struct device_node *par_dn) ret); } +/* + * Platforms support placing PHB in limited address mode starting with LoPAR + * level 2.13 implement. In this mode, the DMA address returned by DDW is over + * 4GB but, less than 64-bits. This benefits IO adapters that don't support + * 64-bits for DMA addresses. + */ +static int limited_dma_window(struct pci_dev *dev, struct device_node *par_dn) +{ + int ret; + u32 cfg_addr, reset_dma_win, las_supported; + u64 buid; + struct device_node *dn; + struct pci_dn *pdn; + + ret = ddw_read_ext(par_dn, DDW_EXT_RESET_DMA_WIN, &reset_dma_win); + if (ret) + goto out; + + ret = ddw_read_ext(par_dn, DDW_EXT_LIMITED_ADDR_MODE, &las_supported); + + /* Limited Address Space extension available on the platform but DDW in + * limited addressing mode not supported + */ + if (!ret && !las_supported) + ret = -EPROTO; + + if (ret) { + dev_info(&dev->dev, "Limited Address Space for DDW not Supported, err: %d", ret); + goto out; + } + + dn = pci_device_to_OF_node(dev); + pdn = PCI_DN(dn); + buid = pdn->phb->buid; + cfg_addr = (pdn->busno << 16) | (pdn->devfn << 8); + + ret = rtas_call(reset_dma_win, 4, 1, NULL, cfg_addr, BUID_HI(buid), + BUID_LO(buid), 1); + if (ret) + dev_info(&dev->dev, + "ibm,reset-pe-dma-windows(%x) for Limited Addr Support: %x %x %x returned %d ", + reset_dma_win, cfg_addr, BUID_HI(buid), BUID_LO(buid), + ret); + +out: + return ret; +} + /* Return largest page shift based on "IO Page Sizes" output of ibm,query-pe-dma-window. */ static int iommu_get_page_shift(u32 query_page_size) { @@ -1398,7 +1443,7 @@ static struct property *ddw_property_create(const char *propname, u32 liobn, u64 * * returns true if can map all pages (direct mapping), false otherwise.. */ -static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) +static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn, u64 dma_mask) { int len = 0, ret; int max_ram_len = order_base_2(ddw_memory_hotplug_max()); @@ -1417,6 +1462,9 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) bool pmem_present; struct pci_dn *pci = PCI_DN(pdn); struct property *default_win = NULL; + bool limited_addr_req = false, limited_addr_enabled = false; + int dev_max_ddw; + int ddw_sz; dn = of_find_node_by_type(NULL, "ibm,pmemory"); pmem_present = dn != NULL; @@ -1443,7 +1491,6 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) * the ibm,ddw-applicable property holds the tokens for: * ibm,query-pe-dma-window * ibm,create-pe-dma-window - * ibm,remove-pe-dma-window * for the given node in that order. * the property is actually in the parent, not the PE */ @@ -1463,6 +1510,20 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) if (ret != 0) goto out_failed; + /* DMA Limited Addressing required? This is when the driver has + * requested to create DDW but supports mask which is less than 64-bits + */ + limited_addr_req = (dma_mask != DMA_BIT_MASK(64)); + + /* place the PHB in Limited Addressing mode */ + if (limited_addr_req) { + if (limited_dma_window(dev, pdn)) + goto out_failed; + + /* PHB is in Limited address mode */ + limited_addr_enabled = true; + } + /* * If there is no window available, remove the default DMA window, * if it's present. This will make all the resources available to the @@ -1509,6 +1570,15 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) goto out_failed; } + /* Maximum DMA window size that the device can address (in log2) */ + dev_max_ddw = fls64(dma_mask); + + /* If the device DMA mask is less than 64-bits, make sure the DMA window + * size is not bigger than what the device can access + */ + ddw_sz = min(order_base_2(query.largest_available_block << page_shift), + dev_max_ddw); + /* * The "ibm,pmemory" can appear anywhere in the address space. * Assuming it is still backed by page structs, try MAX_PHYSMEM_BITS @@ -1517,23 +1587,21 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) */ len = max_ram_len; if (pmem_present) { - if (query.largest_available_block >= - (1ULL << (MAX_PHYSMEM_BITS - page_shift))) + if (ddw_sz >= MAX_PHYSMEM_BITS) len = MAX_PHYSMEM_BITS; else dev_info(&dev->dev, "Skipping ibm,pmemory"); } /* check if the available block * number of ptes will map everything */ - if (query.largest_available_block < (1ULL << (len - page_shift))) { + if (ddw_sz < len) { dev_dbg(&dev->dev, "can't map partition max 0x%llx with %llu %llu-sized pages\n", 1ULL << len, query.largest_available_block, 1ULL << page_shift); - len = order_base_2(query.largest_available_block << page_shift); - + len = ddw_sz; dynamic_mapping = true; } else { direct_mapping = !default_win_removed || @@ -1547,8 +1615,9 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) */ if (default_win_removed && pmem_present && !direct_mapping) { /* DDW is big enough to be split */ - if ((query.largest_available_block << page_shift) >= - MIN_DDW_VPMEM_DMA_WINDOW + (1ULL << max_ram_len)) { + if ((1ULL << ddw_sz) >= + MIN_DDW_VPMEM_DMA_WINDOW + (1ULL << max_ram_len)) { + direct_mapping = true; /* offset of the Dynamic part of DDW */ @@ -1559,8 +1628,7 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) dynamic_mapping = true; /* create max size DDW possible */ - len = order_base_2(query.largest_available_block - << page_shift); + len = ddw_sz; } } @@ -1600,7 +1668,7 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) if (direct_mapping) { /* DDW maps the whole partition, so enable direct DMA mapping */ - ret = walk_system_ram_range(0, memblock_end_of_DRAM() >> PAGE_SHIFT, + ret = walk_system_ram_range(0, ddw_memory_hotplug_max() >> PAGE_SHIFT, win64->value, tce_setrange_multi_pSeriesLP_walk); if (ret) { dev_info(&dev->dev, "failed to map DMA window for %pOF: %d\n", @@ -1650,7 +1718,8 @@ static bool enable_ddw(struct pci_dev *dev, struct device_node *pdn) iommu_table_setparms_common(newtbl, pci->phb->bus->number, create.liobn, dynamic_addr, dynamic_len, page_shift, NULL, &iommu_table_lpar_multi_ops); - iommu_init_table(newtbl, pci->phb->node, start, end); + iommu_init_table(newtbl, pci->phb->node, + start >> page_shift, end >> page_shift); pci->table_group->tables[default_win_removed ? 0 : 1] = newtbl; @@ -1688,7 +1757,7 @@ out_remove_win: __remove_dma_window(pdn, ddw_avail, create.liobn); out_failed: - if (default_win_removed) + if (default_win_removed || limited_addr_enabled) reset_dma_window(dev, pdn); fpdn = kzalloc(sizeof(*fpdn), GFP_KERNEL); @@ -1707,6 +1776,9 @@ out_unlock: dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + (1ULL << max_ram_len); + dev_info(&dev->dev, "lsa_required: %x, lsa_enabled: %x, direct mapping: %x\n", + limited_addr_req, limited_addr_enabled, direct_mapping); + return direct_mapping; } @@ -1832,8 +1904,11 @@ static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask) { struct device_node *dn = pci_device_to_OF_node(pdev), *pdn; - /* only attempt to use a new window if 64-bit DMA is requested */ - if (dma_mask < DMA_BIT_MASK(64)) + /* For DDW, DMA mask should be more than 32-bits. For mask more then + * 32-bits but less then 64-bits, DMA addressing is supported in + * Limited Addressing mode. + */ + if (dma_mask <= DMA_BIT_MASK(32)) return false; dev_dbg(&pdev->dev, "node is %pOF\n", dn); @@ -1846,7 +1921,7 @@ static bool iommu_bypass_supported_pSeriesLP(struct pci_dev *pdev, u64 dma_mask) */ pdn = pci_dma_find(dn, NULL); if (pdn && PCI_DN(pdn)) - return enable_ddw(pdev, pdn); + return enable_ddw(pdev, pdn, dma_mask); return false; } @@ -2065,7 +2140,9 @@ static long spapr_tce_create_table(struct iommu_table_group *table_group, int nu offset, 1UL << window_shift, IOMMU_PAGE_SHIFT_4K, NULL, &iommu_table_lpar_multi_ops); - iommu_init_table(tbl, pci->phb->node, start, end); + iommu_init_table(tbl, pci->phb->node, + start >> IOMMU_PAGE_SHIFT_4K, + end >> IOMMU_PAGE_SHIFT_4K); table_group->tables[0] = tbl; @@ -2136,7 +2213,7 @@ static long spapr_tce_create_table(struct iommu_table_group *table_group, int nu /* New table for using DDW instead of the default DMA window */ iommu_table_setparms_common(tbl, pci->phb->bus->number, create.liobn, win_addr, 1UL << len, page_shift, NULL, &iommu_table_lpar_multi_ops); - iommu_init_table(tbl, pci->phb->node, start, end); + iommu_init_table(tbl, pci->phb->node, start >> page_shift, end >> page_shift); pci->table_group->tables[num] = tbl; set_iommu_table_base(&pdev->dev, tbl); @@ -2205,6 +2282,9 @@ static long spapr_tce_unset_window(struct iommu_table_group *table_group, int nu const char *win_name; int ret = -ENODEV; + if (!tbl) /* The table was never created OR window was never opened */ + return 0; + mutex_lock(&dma_win_init_mutex); if ((num == 0) && is_default_window_table(table_group, tbl)) @@ -2343,11 +2423,17 @@ static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, struct memory_notify *arg = data; int ret = 0; + /* This notifier can get called when onlining persistent memory as well. + * TCEs are not pre-mapped for persistent memory. Persistent memory will + * always be above ddw_memory_hotplug_max() + */ + switch (action) { case MEM_GOING_ONLINE: spin_lock(&dma_win_list_lock); list_for_each_entry(window, &dma_win_list, list) { - if (window->direct) { + if (window->direct && (arg->start_pfn << PAGE_SHIFT) < + ddw_memory_hotplug_max()) { ret |= tce_setrange_multi_pSeriesLP(arg->start_pfn, arg->nr_pages, window->prop); } @@ -2359,7 +2445,8 @@ static int iommu_mem_notifier(struct notifier_block *nb, unsigned long action, case MEM_OFFLINE: spin_lock(&dma_win_list_lock); list_for_each_entry(window, &dma_win_list, list) { - if (window->direct) { + if (window->direct && (arg->start_pfn << PAGE_SHIFT) < + ddw_memory_hotplug_max()) { ret |= tce_clearrange_multi_pSeriesLP(arg->start_pfn, arg->nr_pages, window->prop); } diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c index 1798f0f14d58..62bd8e2d5d4c 100644 --- a/arch/powerpc/platforms/pseries/mobility.c +++ b/arch/powerpc/platforms/pseries/mobility.c @@ -53,7 +53,7 @@ struct update_props_workarea { static unsigned int nmi_wd_lpm_factor = 200; #ifdef CONFIG_SYSCTL -static struct ctl_table nmi_wd_lpm_factor_ctl_table[] = { +static const struct ctl_table nmi_wd_lpm_factor_ctl_table[] = { { .procname = "nmi_wd_lpm_factor", .data = &nmi_wd_lpm_factor, diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index fdc2f7f38dc9..ee1c8c6898a3 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -525,7 +525,12 @@ static struct msi_domain_info pseries_msi_domain_info = { static void pseries_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) { - __pci_read_msi_msg(irq_data_get_msi_desc(data), msg); + struct pci_dev *dev = msi_desc_to_pci_dev(irq_data_get_msi_desc(data)); + + if (dev->current_state == PCI_D0) + __pci_read_msi_msg(irq_data_get_msi_desc(data), msg); + else + get_cached_msi_msg(data->irq, msg); } static struct irq_chip pseries_msi_irq_chip = { @@ -611,7 +616,7 @@ static const struct irq_domain_ops pseries_irq_domain_ops = { static int __pseries_msi_allocate_domains(struct pci_controller *phb, unsigned int count) { - struct irq_domain *parent = irq_get_default_host(); + struct irq_domain *parent = irq_get_default_domain(); phb->fwnode = irq_domain_alloc_named_id_fwnode("pSeries-MSI", phb->global_number); @@ -628,7 +633,7 @@ static int __pseries_msi_allocate_domains(struct pci_controller *phb, return -ENOMEM; } - phb->msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(phb->dn), + phb->msi_domain = pci_msi_create_irq_domain(of_fwnode_handle(phb->dn), &pseries_msi_domain_info, phb->dev_domain); if (!phb->msi_domain) { diff --git a/arch/powerpc/platforms/pseries/papr-indices.c b/arch/powerpc/platforms/pseries/papr-indices.c new file mode 100644 index 000000000000..3c7545591c45 --- /dev/null +++ b/arch/powerpc/platforms/pseries/papr-indices.c @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#define pr_fmt(fmt) "papr-indices: " fmt + +#include <linux/build_bug.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/lockdep.h> +#include <linux/kernel.h> +#include <linux/miscdevice.h> +#include <linux/signal.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/string_helpers.h> +#include <linux/uaccess.h> +#include <asm/machdep.h> +#include <asm/rtas-work-area.h> +#include <asm/rtas.h> +#include <uapi/asm/papr-indices.h> +#include "papr-rtas-common.h" + +/* + * Function-specific return values for ibm,set-dynamic-indicator and + * ibm,get-dynamic-sensor-state RTAS calls. + * PAPR+ v2.13 7.3.18 and 7.3.19. + */ +#define RTAS_IBM_DYNAMIC_INDICE_NO_INDICATOR -3 + +/** + * struct rtas_get_indices_params - Parameters (in and out) for + * ibm,get-indices. + * @is_sensor: In: Caller-provided whether sensor or indicator. + * @indice_type:In: Caller-provided indice (sensor or indicator) token + * @work_area: In: Caller-provided work area buffer for results. + * @next: In: Sequence number. Out: Next sequence number. + * @status: Out: RTAS call status. + */ +struct rtas_get_indices_params { + u8 is_sensor; + u32 indice_type; + struct rtas_work_area *work_area; + u32 next; + s32 status; +}; + +/* + * rtas_ibm_get_indices() - Call ibm,get-indices to fill a work area buffer. + * @params: See &struct rtas_ibm_get_indices_params. + * + * Calls ibm,get-indices until it errors or successfully deposits data + * into the supplied work area. Handles RTAS retry statuses. Maps RTAS + * error statuses to reasonable errno values. + * + * The caller is expected to invoke rtas_ibm_get_indices() multiple times + * to retrieve all indices data for the provided indice type. Only one + * sequence should be in progress at any time; starting a new sequence + * will disrupt any sequence already in progress. Serialization of + * indices retrieval sequences is the responsibility of the caller. + * + * The caller should inspect @params.status to determine whether more + * calls are needed to complete the sequence. + * + * Context: May sleep. + * Return: -ve on error, 0 otherwise. + */ +static int rtas_ibm_get_indices(struct rtas_get_indices_params *params) +{ + struct rtas_work_area *work_area = params->work_area; + const s32 token = rtas_function_token(RTAS_FN_IBM_GET_INDICES); + u32 rets; + s32 fwrc; + int ret; + + if (token == RTAS_UNKNOWN_SERVICE) + return -ENOENT; + + lockdep_assert_held(&rtas_ibm_get_indices_lock); + + do { + fwrc = rtas_call(token, 5, 2, &rets, params->is_sensor, + params->indice_type, + rtas_work_area_phys(work_area), + rtas_work_area_size(work_area), + params->next); + } while (rtas_busy_delay(fwrc)); + + switch (fwrc) { + case RTAS_HARDWARE_ERROR: + ret = -EIO; + break; + case RTAS_INVALID_PARAMETER: /* Indicator type is not supported */ + ret = -EINVAL; + break; + case RTAS_SEQ_START_OVER: + ret = -EAGAIN; + pr_info_ratelimited("Indices changed during retrieval, retrying\n"); + params->next = 1; + break; + case RTAS_SEQ_MORE_DATA: + params->next = rets; + ret = 0; + break; + case RTAS_SEQ_COMPLETE: + params->next = 0; + ret = 0; + break; + default: + ret = -EIO; + pr_err_ratelimited("unexpected ibm,get-indices status %d\n", fwrc); + break; + } + + params->status = fwrc; + return ret; +} + +/* + * Internal indices sequence APIs. A sequence is a series of calls to + * ibm,get-indices for a given location code. The sequence ends when + * an error is encountered or all indices for the input has been + * returned. + */ + +/* + * indices_sequence_begin() - Begin a indices retrieval sequence. + * + * Context: May sleep. + */ +static void indices_sequence_begin(struct papr_rtas_sequence *seq) +{ + struct rtas_get_indices_params *param; + + param = (struct rtas_get_indices_params *)seq->params; + /* + * We could allocate the work area before acquiring the + * function lock, but that would allow concurrent requests to + * exhaust the limited work area pool for no benefit. So + * allocate the work area under the lock. + */ + mutex_lock(&rtas_ibm_get_indices_lock); + param->work_area = rtas_work_area_alloc(RTAS_GET_INDICES_BUF_SIZE); + param->next = 1; + param->status = 0; +} + +/* + * indices_sequence_end() - Finalize a indices retrieval sequence. + * + * Releases resources obtained by indices_sequence_begin(). + */ +static void indices_sequence_end(struct papr_rtas_sequence *seq) +{ + struct rtas_get_indices_params *param; + + param = (struct rtas_get_indices_params *)seq->params; + rtas_work_area_free(param->work_area); + mutex_unlock(&rtas_ibm_get_indices_lock); +} + +/* + * Work function to be passed to papr_rtas_blob_generate(). + * + * ibm,get-indices RTAS call fills the work area with the certain + * format but does not return the bytes written in the buffer. So + * instead of kernel parsing this work area to determine the buffer + * length, copy the complete work area (RTAS_GET_INDICES_BUF_SIZE) + * to the blob and let the user space to obtain the data. + * Means RTAS_GET_INDICES_BUF_SIZE data will be returned for each + * read(). + */ + +static const char *indices_sequence_fill_work_area(struct papr_rtas_sequence *seq, + size_t *len) +{ + struct rtas_get_indices_params *p; + bool init_state; + + p = (struct rtas_get_indices_params *)seq->params; + init_state = (p->next == 1) ? true : false; + + if (papr_rtas_sequence_should_stop(seq, p->status, init_state)) + return NULL; + if (papr_rtas_sequence_set_err(seq, rtas_ibm_get_indices(p))) + return NULL; + + *len = RTAS_GET_INDICES_BUF_SIZE; + return rtas_work_area_raw_buf(p->work_area); +} + +/* + * papr_indices_handle_read - returns indices blob data to the user space + * + * ibm,get-indices RTAS call fills the work area with the certian + * format but does not return the bytes written in the buffer and + * copied RTAS_GET_INDICES_BUF_SIZE data to the blob for each RTAS + * call. So send RTAS_GET_INDICES_BUF_SIZE buffer to the user space + * for each read(). + */ +static ssize_t papr_indices_handle_read(struct file *file, + char __user *buf, size_t size, loff_t *off) +{ + const struct papr_rtas_blob *blob = file->private_data; + + /* we should not instantiate a handle without any data attached. */ + if (!papr_rtas_blob_has_data(blob)) { + pr_err_once("handle without data\n"); + return -EIO; + } + + if (size < RTAS_GET_INDICES_BUF_SIZE) { + pr_err_once("Invalid buffer length %ld, expect %d\n", + size, RTAS_GET_INDICES_BUF_SIZE); + return -EINVAL; + } else if (size > RTAS_GET_INDICES_BUF_SIZE) + size = RTAS_GET_INDICES_BUF_SIZE; + + return simple_read_from_buffer(buf, size, off, blob->data, blob->len); +} + +static const struct file_operations papr_indices_handle_ops = { + .read = papr_indices_handle_read, + .llseek = papr_rtas_common_handle_seek, + .release = papr_rtas_common_handle_release, +}; + +/* + * papr_indices_create_handle() - Create a fd-based handle for reading + * indices data + * @ubuf: Input parameters to RTAS call such as whether sensor or indicator + * and indice type in user memory + * + * Handler for PAPR_INDICES_IOC_GET ioctl command. Validates @ubuf + * and instantiates an immutable indices "blob" for it. The blob is + * attached to a file descriptor for reading by user space. The memory + * backing the blob is freed when the file is released. + * + * The entire requested indices is retrieved by this call and all + * necessary RTAS interactions are performed before returning the fd + * to user space. This keeps the read handler simple and ensures that + * the kernel can prevent interleaving of ibm,get-indices call sequences. + * + * Return: The installed fd number if successful, -ve errno otherwise. + */ +static long papr_indices_create_handle(struct papr_indices_io_block __user *ubuf) +{ + struct papr_rtas_sequence seq = {}; + struct rtas_get_indices_params params = {}; + int fd; + + if (get_user(params.is_sensor, &ubuf->indices.is_sensor)) + return -EFAULT; + + if (get_user(params.indice_type, &ubuf->indices.indice_type)) + return -EFAULT; + + seq = (struct papr_rtas_sequence) { + .begin = indices_sequence_begin, + .end = indices_sequence_end, + .work = indices_sequence_fill_work_area, + }; + + seq.params = ¶ms; + fd = papr_rtas_setup_file_interface(&seq, + &papr_indices_handle_ops, "[papr-indices]"); + + return fd; +} + +/* + * Create work area with the input parameters. This function is used + * for both ibm,set-dynamic-indicator and ibm,get-dynamic-sensor-state + * RTAS Calls. + */ +static struct rtas_work_area * +papr_dynamic_indice_buf_from_user(struct papr_indices_io_block __user *ubuf, + struct papr_indices_io_block *kbuf) +{ + struct rtas_work_area *work_area; + u32 length; + __be32 len_be; + + if (copy_from_user(kbuf, ubuf, sizeof(*kbuf))) + return ERR_PTR(-EFAULT); + + + if (!string_is_terminated(kbuf->dynamic_param.location_code_str, + ARRAY_SIZE(kbuf->dynamic_param.location_code_str))) + return ERR_PTR(-EINVAL); + + /* + * The input data in the work area should be as follows: + * - 32-bit integer length of the location code string, + * including NULL. + * - Location code string, NULL terminated, identifying the + * token (sensor or indicator). + * PAPR 2.13 - R1–7.3.18–5 ibm,set-dynamic-indicator + * - R1–7.3.19–5 ibm,get-dynamic-sensor-state + */ + /* + * Length that user space passed should also include NULL + * terminator. + */ + length = strlen(kbuf->dynamic_param.location_code_str) + 1; + if (length > LOC_CODE_SIZE) + return ERR_PTR(-EINVAL); + + len_be = cpu_to_be32(length); + + work_area = rtas_work_area_alloc(LOC_CODE_SIZE + sizeof(u32)); + memcpy(rtas_work_area_raw_buf(work_area), &len_be, sizeof(u32)); + memcpy((rtas_work_area_raw_buf(work_area) + sizeof(u32)), + &kbuf->dynamic_param.location_code_str, length); + + return work_area; +} + +/** + * papr_dynamic_indicator_ioc_set - ibm,set-dynamic-indicator RTAS Call + * PAPR 2.13 7.3.18 + * + * @ubuf: Input parameters to RTAS call such as indicator token and + * new state. + * + * Returns success or -errno. + */ +static long papr_dynamic_indicator_ioc_set(struct papr_indices_io_block __user *ubuf) +{ + struct papr_indices_io_block kbuf; + struct rtas_work_area *work_area; + s32 fwrc, token, ret; + + token = rtas_function_token(RTAS_FN_IBM_SET_DYNAMIC_INDICATOR); + if (token == RTAS_UNKNOWN_SERVICE) + return -ENOENT; + + mutex_lock(&rtas_ibm_set_dynamic_indicator_lock); + work_area = papr_dynamic_indice_buf_from_user(ubuf, &kbuf); + if (IS_ERR(work_area)) { + ret = PTR_ERR(work_area); + goto out; + } + + do { + fwrc = rtas_call(token, 3, 1, NULL, + kbuf.dynamic_param.token, + kbuf.dynamic_param.state, + rtas_work_area_phys(work_area)); + } while (rtas_busy_delay(fwrc)); + + rtas_work_area_free(work_area); + + switch (fwrc) { + case RTAS_SUCCESS: + ret = 0; + break; + case RTAS_IBM_DYNAMIC_INDICE_NO_INDICATOR: /* No such indicator */ + ret = -EOPNOTSUPP; + break; + default: + pr_err("unexpected ibm,set-dynamic-indicator result %d\n", + fwrc); + fallthrough; + case RTAS_HARDWARE_ERROR: /* Hardware/platform error */ + ret = -EIO; + break; + } + +out: + mutex_unlock(&rtas_ibm_set_dynamic_indicator_lock); + return ret; +} + +/** + * papr_dynamic_sensor_ioc_get - ibm,get-dynamic-sensor-state RTAS Call + * PAPR 2.13 7.3.19 + * + * @ubuf: Input parameters to RTAS call such as sensor token + * Copies the state in user space buffer. + * + * + * Returns success or -errno. + */ + +static long papr_dynamic_sensor_ioc_get(struct papr_indices_io_block __user *ubuf) +{ + struct papr_indices_io_block kbuf; + struct rtas_work_area *work_area; + s32 fwrc, token, ret; + u32 rets; + + token = rtas_function_token(RTAS_FN_IBM_GET_DYNAMIC_SENSOR_STATE); + if (token == RTAS_UNKNOWN_SERVICE) + return -ENOENT; + + mutex_lock(&rtas_ibm_get_dynamic_sensor_state_lock); + work_area = papr_dynamic_indice_buf_from_user(ubuf, &kbuf); + if (IS_ERR(work_area)) { + ret = PTR_ERR(work_area); + goto out; + } + + do { + fwrc = rtas_call(token, 2, 2, &rets, + kbuf.dynamic_param.token, + rtas_work_area_phys(work_area)); + } while (rtas_busy_delay(fwrc)); + + rtas_work_area_free(work_area); + + switch (fwrc) { + case RTAS_SUCCESS: + if (put_user(rets, &ubuf->dynamic_param.state)) + ret = -EFAULT; + else + ret = 0; + break; + case RTAS_IBM_DYNAMIC_INDICE_NO_INDICATOR: /* No such indicator */ + ret = -EOPNOTSUPP; + break; + default: + pr_err("unexpected ibm,get-dynamic-sensor result %d\n", + fwrc); + fallthrough; + case RTAS_HARDWARE_ERROR: /* Hardware/platform error */ + ret = -EIO; + break; + } + +out: + mutex_unlock(&rtas_ibm_get_dynamic_sensor_state_lock); + return ret; +} + +/* + * Top-level ioctl handler for /dev/papr-indices. + */ +static long papr_indices_dev_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + void __user *argp = (__force void __user *)arg; + long ret; + + switch (ioctl) { + case PAPR_INDICES_IOC_GET: + ret = papr_indices_create_handle(argp); + break; + case PAPR_DYNAMIC_SENSOR_IOC_GET: + ret = papr_dynamic_sensor_ioc_get(argp); + break; + case PAPR_DYNAMIC_INDICATOR_IOC_SET: + if (filp->f_mode & FMODE_WRITE) + ret = papr_dynamic_indicator_ioc_set(argp); + else + ret = -EBADF; + break; + default: + ret = -ENOIOCTLCMD; + break; + } + + return ret; +} + +static const struct file_operations papr_indices_ops = { + .unlocked_ioctl = papr_indices_dev_ioctl, +}; + +static struct miscdevice papr_indices_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "papr-indices", + .fops = &papr_indices_ops, +}; + +static __init int papr_indices_init(void) +{ + if (!rtas_function_implemented(RTAS_FN_IBM_GET_INDICES)) + return -ENODEV; + + if (!rtas_function_implemented(RTAS_FN_IBM_SET_DYNAMIC_INDICATOR)) + return -ENODEV; + + if (!rtas_function_implemented(RTAS_FN_IBM_GET_DYNAMIC_SENSOR_STATE)) + return -ENODEV; + + return misc_register(&papr_indices_dev); +} +machine_device_initcall(pseries, papr_indices_init); diff --git a/arch/powerpc/platforms/pseries/papr-phy-attest.c b/arch/powerpc/platforms/pseries/papr-phy-attest.c new file mode 100644 index 000000000000..1907f2411567 --- /dev/null +++ b/arch/powerpc/platforms/pseries/papr-phy-attest.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#define pr_fmt(fmt) "papr-phy-attest: " fmt + +#include <linux/build_bug.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/lockdep.h> +#include <linux/kernel.h> +#include <linux/miscdevice.h> +#include <linux/signal.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/string_helpers.h> +#include <linux/uaccess.h> +#include <asm/machdep.h> +#include <asm/rtas-work-area.h> +#include <asm/rtas.h> +#include <uapi/asm/papr-physical-attestation.h> +#include "papr-rtas-common.h" + +/** + * struct rtas_phy_attest_params - Parameters (in and out) for + * ibm,physical-attestation. + * + * @cmd: In: Caller-provided attestation command buffer. Must be + * RTAS-addressable. + * @work_area: In: Caller-provided work area buffer for attestation + * command structure + * Out: Caller-provided work area buffer for the response + * @cmd_len: In: Caller-provided attestation command structure + * length + * @sequence: In: Sequence number. Out: Next sequence number. + * @written: Out: Bytes written by ibm,physical-attestation to + * @work_area. + * @status: Out: RTAS call status. + */ +struct rtas_phy_attest_params { + struct papr_phy_attest_io_block cmd; + struct rtas_work_area *work_area; + u32 cmd_len; + u32 sequence; + u32 written; + s32 status; +}; + +/** + * rtas_physical_attestation() - Call ibm,physical-attestation to + * fill a work area buffer. + * @params: See &struct rtas_phy_attest_params. + * + * Calls ibm,physical-attestation until it errors or successfully + * deposits data into the supplied work area. Handles RTAS retry + * statuses. Maps RTAS error statuses to reasonable errno values. + * + * The caller is expected to invoke rtas_physical_attestation() + * multiple times to retrieve all the data for the provided + * attestation command. Only one sequence should be in progress at + * any time; starting a new sequence will disrupt any sequence + * already in progress. Serialization of attestation retrieval + * sequences is the responsibility of the caller. + * + * The caller should inspect @params.status to determine whether more + * calls are needed to complete the sequence. + * + * Context: May sleep. + * Return: -ve on error, 0 otherwise. + */ +static int rtas_physical_attestation(struct rtas_phy_attest_params *params) +{ + struct rtas_work_area *work_area; + s32 fwrc, token; + u32 rets[2]; + int ret; + + work_area = params->work_area; + token = rtas_function_token(RTAS_FN_IBM_PHYSICAL_ATTESTATION); + if (token == RTAS_UNKNOWN_SERVICE) + return -ENOENT; + + lockdep_assert_held(&rtas_ibm_physical_attestation_lock); + + do { + fwrc = rtas_call(token, 3, 3, rets, + rtas_work_area_phys(work_area), + params->cmd_len, + params->sequence); + } while (rtas_busy_delay(fwrc)); + + switch (fwrc) { + case RTAS_HARDWARE_ERROR: + ret = -EIO; + break; + case RTAS_INVALID_PARAMETER: + ret = -EINVAL; + break; + case RTAS_SEQ_MORE_DATA: + params->sequence = rets[0]; + fallthrough; + case RTAS_SEQ_COMPLETE: + params->written = rets[1]; + /* + * Kernel or firmware bug, do not continue. + */ + if (WARN(params->written > rtas_work_area_size(work_area), + "possible write beyond end of work area")) + ret = -EFAULT; + else + ret = 0; + break; + default: + ret = -EIO; + pr_err_ratelimited("unexpected ibm,get-phy_attest status %d\n", fwrc); + break; + } + + params->status = fwrc; + return ret; +} + +/* + * Internal physical-attestation sequence APIs. A physical-attestation + * sequence is a series of calls to get ibm,physical-attestation + * for a given attestation command. The sequence ends when an error + * is encountered or all data for the attestation command has been + * returned. + */ + +/** + * phy_attest_sequence_begin() - Begin a response data for attestation + * command retrieval sequence. + * @seq: user specified parameters for RTAS call from seq struct. + * + * Context: May sleep. + */ +static void phy_attest_sequence_begin(struct papr_rtas_sequence *seq) +{ + struct rtas_phy_attest_params *param; + + /* + * We could allocate the work area before acquiring the + * function lock, but that would allow concurrent requests to + * exhaust the limited work area pool for no benefit. So + * allocate the work area under the lock. + */ + mutex_lock(&rtas_ibm_physical_attestation_lock); + param = (struct rtas_phy_attest_params *)seq->params; + param->work_area = rtas_work_area_alloc(SZ_4K); + memcpy(rtas_work_area_raw_buf(param->work_area), ¶m->cmd, + param->cmd_len); + param->sequence = 1; + param->status = 0; +} + +/** + * phy_attest_sequence_end() - Finalize a attestation command + * response retrieval sequence. + * @seq: Sequence state. + * + * Releases resources obtained by phy_attest_sequence_begin(). + */ +static void phy_attest_sequence_end(struct papr_rtas_sequence *seq) +{ + struct rtas_phy_attest_params *param; + + param = (struct rtas_phy_attest_params *)seq->params; + rtas_work_area_free(param->work_area); + mutex_unlock(&rtas_ibm_physical_attestation_lock); + kfree(param); +} + +/* + * Generator function to be passed to papr_rtas_blob_generate(). + */ +static const char *phy_attest_sequence_fill_work_area(struct papr_rtas_sequence *seq, + size_t *len) +{ + struct rtas_phy_attest_params *p; + bool init_state; + + p = (struct rtas_phy_attest_params *)seq->params; + init_state = (p->written == 0) ? true : false; + + if (papr_rtas_sequence_should_stop(seq, p->status, init_state)) + return NULL; + if (papr_rtas_sequence_set_err(seq, rtas_physical_attestation(p))) + return NULL; + *len = p->written; + return rtas_work_area_raw_buf(p->work_area); +} + +static const struct file_operations papr_phy_attest_handle_ops = { + .read = papr_rtas_common_handle_read, + .llseek = papr_rtas_common_handle_seek, + .release = papr_rtas_common_handle_release, +}; + +/** + * papr_phy_attest_create_handle() - Create a fd-based handle for + * reading the response for the given attestation command. + * @ulc: Attestation command in user memory; defines the scope of + * data for the attestation command to retrieve. + * + * Handler for PAPR_PHYSICAL_ATTESTATION_IOC_CREATE_HANDLE ioctl + * command. Validates @ulc and instantiates an immutable response + * "blob" for attestation command. The blob is attached to a file + * descriptor for reading by user space. The memory backing the blob + * is freed when the file is released. + * + * The entire requested response buffer for the attestation command + * retrieved by this call and all necessary RTAS interactions are + * performed before returning the fd to user space. This keeps the + * read handler simple and ensures that kernel can prevent + * interleaving ibm,physical-attestation call sequences. + * + * Return: The installed fd number if successful, -ve errno otherwise. + */ +static long papr_phy_attest_create_handle(struct papr_phy_attest_io_block __user *ulc) +{ + struct rtas_phy_attest_params *params; + struct papr_rtas_sequence seq = {}; + int fd; + + /* + * Freed in phy_attest_sequence_end(). + */ + params = kzalloc(sizeof(*params), GFP_KERNEL_ACCOUNT); + if (!params) + return -ENOMEM; + + if (copy_from_user(¶ms->cmd, ulc, + sizeof(struct papr_phy_attest_io_block))) + return -EFAULT; + + params->cmd_len = be32_to_cpu(params->cmd.length); + seq = (struct papr_rtas_sequence) { + .begin = phy_attest_sequence_begin, + .end = phy_attest_sequence_end, + .work = phy_attest_sequence_fill_work_area, + }; + + seq.params = (void *)params; + + fd = papr_rtas_setup_file_interface(&seq, + &papr_phy_attest_handle_ops, + "[papr-physical-attestation]"); + + return fd; +} + +/* + * Top-level ioctl handler for /dev/papr-physical-attestation. + */ +static long papr_phy_attest_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + void __user *argp = (__force void __user *)arg; + long ret; + + switch (ioctl) { + case PAPR_PHY_ATTEST_IOC_HANDLE: + ret = papr_phy_attest_create_handle(argp); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + return ret; +} + +static const struct file_operations papr_phy_attest_ops = { + .unlocked_ioctl = papr_phy_attest_dev_ioctl, +}; + +static struct miscdevice papr_phy_attest_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "papr-physical-attestation", + .fops = &papr_phy_attest_ops, +}; + +static __init int papr_phy_attest_init(void) +{ + if (!rtas_function_implemented(RTAS_FN_IBM_PHYSICAL_ATTESTATION)) + return -ENODEV; + + return misc_register(&papr_phy_attest_dev); +} +machine_device_initcall(pseries, papr_phy_attest_init); diff --git a/arch/powerpc/platforms/pseries/papr-platform-dump.c b/arch/powerpc/platforms/pseries/papr-platform-dump.c new file mode 100644 index 000000000000..f8d55eccdb6b --- /dev/null +++ b/arch/powerpc/platforms/pseries/papr-platform-dump.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#define pr_fmt(fmt) "papr-platform-dump: " fmt + +#include <linux/anon_inodes.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/miscdevice.h> +#include <asm/machdep.h> +#include <asm/rtas-work-area.h> +#include <asm/rtas.h> +#include <uapi/asm/papr-platform-dump.h> + +/* + * Function-specific return values for ibm,platform-dump, derived from + * PAPR+ v2.13 7.3.3.4.1 "ibm,platform-dump RTAS Call". + */ +#define RTAS_IBM_PLATFORM_DUMP_COMPLETE 0 /* Complete dump retrieved. */ +#define RTAS_IBM_PLATFORM_DUMP_CONTINUE 1 /* Continue dump */ +#define RTAS_NOT_AUTHORIZED -9002 /* Not Authorized */ + +#define RTAS_IBM_PLATFORM_DUMP_START 2 /* Linux status to start dump */ + +/** + * struct ibm_platform_dump_params - Parameters (in and out) for + * ibm,platform-dump + * @work_area: In: work area buffer for results. + * @buf_length: In: work area buffer length in bytes + * @dump_tag_hi: In: Most-significant 32 bits of a Dump_Tag representing + * an id of the dump being processed. + * @dump_tag_lo: In: Least-significant 32 bits of a Dump_Tag representing + * an id of the dump being processed. + * @sequence_hi: In: Sequence number in most-significant 32 bits. + * Out: Next sequence number in most-significant 32 bits. + * @sequence_lo: In: Sequence number in Least-significant 32 bits + * Out: Next sequence number in Least-significant 32 bits. + * @bytes_ret_hi: Out: Bytes written in most-significant 32 bits. + * @bytes_ret_lo: Out: Bytes written in Least-significant 32 bits. + * @status: Out: RTAS call status. + * @list: Maintain the list of dumps are in progress. Can + * retrieve multiple dumps with different dump IDs at + * the same time but not with the same dump ID. This list + * is used to determine whether the dump for the same ID + * is in progress. + */ +struct ibm_platform_dump_params { + struct rtas_work_area *work_area; + u32 buf_length; + u32 dump_tag_hi; + u32 dump_tag_lo; + u32 sequence_hi; + u32 sequence_lo; + u32 bytes_ret_hi; + u32 bytes_ret_lo; + s32 status; + struct list_head list; +}; + +/* + * Multiple dumps with different dump IDs can be retrieved at the same + * time, but not with dame dump ID. platform_dump_list_mutex and + * platform_dump_list are used to prevent this behavior. + */ +static DEFINE_MUTEX(platform_dump_list_mutex); +static LIST_HEAD(platform_dump_list); + +/** + * rtas_ibm_platform_dump() - Call ibm,platform-dump to fill a work area + * buffer. + * @params: See &struct ibm_platform_dump_params. + * @buf_addr: Address of dump buffer (work_area) + * @buf_length: Length of the buffer in bytes (min. 1024) + * + * Calls ibm,platform-dump until it errors or successfully deposits data + * into the supplied work area. Handles RTAS retry statuses. Maps RTAS + * error statuses to reasonable errno values. + * + * Can request multiple dumps with different dump IDs at the same time, + * but not with the same dump ID which is prevented with the check in + * the ioctl code (papr_platform_dump_create_handle()). + * + * The caller should inspect @params.status to determine whether more + * calls are needed to complete the sequence. + * + * Context: May sleep. + * Return: -ve on error, 0 for dump complete and 1 for continue dump + */ +static int rtas_ibm_platform_dump(struct ibm_platform_dump_params *params, + phys_addr_t buf_addr, u32 buf_length) +{ + u32 rets[4]; + s32 fwrc; + int ret = 0; + + do { + fwrc = rtas_call(rtas_function_token(RTAS_FN_IBM_PLATFORM_DUMP), + 6, 5, + rets, + params->dump_tag_hi, + params->dump_tag_lo, + params->sequence_hi, + params->sequence_lo, + buf_addr, + buf_length); + } while (rtas_busy_delay(fwrc)); + + switch (fwrc) { + case RTAS_HARDWARE_ERROR: + ret = -EIO; + break; + case RTAS_NOT_AUTHORIZED: + ret = -EPERM; + break; + case RTAS_IBM_PLATFORM_DUMP_CONTINUE: + case RTAS_IBM_PLATFORM_DUMP_COMPLETE: + params->sequence_hi = rets[0]; + params->sequence_lo = rets[1]; + params->bytes_ret_hi = rets[2]; + params->bytes_ret_lo = rets[3]; + break; + default: + ret = -EIO; + pr_err_ratelimited("unexpected ibm,platform-dump status %d\n", + fwrc); + break; + } + + params->status = fwrc; + return ret; +} + +/* + * Platform dump is used with multiple RTAS calls to retrieve the + * complete dump for the provided dump ID. Once the complete dump is + * retrieved, the hypervisor returns dump complete status (0) for the + * last RTAS call and expects the caller issues one more call with + * NULL buffer to invalidate the dump so that the hypervisor can remove + * the dump. + * + * After the specific dump is invalidated in the hypervisor, expect the + * dump complete status for the new sequence - the user space initiates + * new request for the same dump ID. + */ +static ssize_t papr_platform_dump_handle_read(struct file *file, + char __user *buf, size_t size, loff_t *off) +{ + struct ibm_platform_dump_params *params = file->private_data; + u64 total_bytes; + s32 fwrc; + + /* + * Dump already completed with the previous read calls. + * In case if the user space issues further reads, returns + * -EINVAL. + */ + if (!params->buf_length) { + pr_warn_once("Platform dump completed for dump ID %llu\n", + (u64) (((u64)params->dump_tag_hi << 32) | + params->dump_tag_lo)); + return -EINVAL; + } + + /* + * The hypervisor returns status 0 if no more data available to + * download. The dump will be invalidated with ioctl (see below). + */ + if (params->status == RTAS_IBM_PLATFORM_DUMP_COMPLETE) { + params->buf_length = 0; + /* + * Returns 0 to the user space so that user + * space read stops. + */ + return 0; + } + + if (size < SZ_1K) { + pr_err_once("Buffer length should be minimum 1024 bytes\n"); + return -EINVAL; + } else if (size > params->buf_length) { + /* + * Allocate 4K work area. So if the user requests > 4K, + * resize the buffer length. + */ + size = params->buf_length; + } + + fwrc = rtas_ibm_platform_dump(params, + rtas_work_area_phys(params->work_area), + size); + if (fwrc < 0) + return fwrc; + + total_bytes = (u64) (((u64)params->bytes_ret_hi << 32) | + params->bytes_ret_lo); + + /* + * Kernel or firmware bug, do not continue. + */ + if (WARN(total_bytes > size, "possible write beyond end of work area")) + return -EFAULT; + + if (copy_to_user(buf, rtas_work_area_raw_buf(params->work_area), + total_bytes)) + return -EFAULT; + + return total_bytes; +} + +static int papr_platform_dump_handle_release(struct inode *inode, + struct file *file) +{ + struct ibm_platform_dump_params *params = file->private_data; + + if (params->work_area) + rtas_work_area_free(params->work_area); + + mutex_lock(&platform_dump_list_mutex); + list_del(¶ms->list); + mutex_unlock(&platform_dump_list_mutex); + + kfree(params); + file->private_data = NULL; + return 0; +} + +/* + * This ioctl is used to invalidate the dump assuming the user space + * issue this ioctl after obtain the complete dump. + * Issue the last RTAS call with NULL buffer to invalidate the dump + * which means dump will be freed in the hypervisor. + */ +static long papr_platform_dump_invalidate_ioctl(struct file *file, + unsigned int ioctl, unsigned long arg) +{ + struct ibm_platform_dump_params *params; + u64 __user *argp = (void __user *)arg; + u64 param_dump_tag, dump_tag; + + if (ioctl != PAPR_PLATFORM_DUMP_IOC_INVALIDATE) + return -ENOIOCTLCMD; + + if (get_user(dump_tag, argp)) + return -EFAULT; + + /* + * private_data is freeded during release(), so should not + * happen. + */ + if (!file->private_data) { + pr_err("No valid FD to invalidate dump for the ID(%llu)\n", + dump_tag); + return -EINVAL; + } + + params = file->private_data; + param_dump_tag = (u64) (((u64)params->dump_tag_hi << 32) | + params->dump_tag_lo); + if (dump_tag != param_dump_tag) { + pr_err("Invalid dump ID(%llu) to invalidate dump\n", + dump_tag); + return -EINVAL; + } + + if (params->status != RTAS_IBM_PLATFORM_DUMP_COMPLETE) { + pr_err("Platform dump is not complete, but requested " + "to invalidate dump for ID(%llu)\n", + dump_tag); + return -EINPROGRESS; + } + + return rtas_ibm_platform_dump(params, 0, 0); +} + +static const struct file_operations papr_platform_dump_handle_ops = { + .read = papr_platform_dump_handle_read, + .release = papr_platform_dump_handle_release, + .unlocked_ioctl = papr_platform_dump_invalidate_ioctl, +}; + +/** + * papr_platform_dump_create_handle() - Create a fd-based handle for + * reading platform dump + * + * Handler for PAPR_PLATFORM_DUMP_IOC_CREATE_HANDLE ioctl command + * Allocates RTAS parameter struct and work area and attached to the + * file descriptor for reading by user space with the multiple RTAS + * calls until the dump is completed. This memory allocation is freed + * when the file is released. + * + * Multiple dump requests with different IDs are allowed at the same + * time, but not with the same dump ID. So if the user space is + * already opened file descriptor for the specific dump ID, return + * -EALREADY for the next request. + * + * @dump_tag: Dump ID for the dump requested to retrieve from the + * hypervisor + * + * Return: The installed fd number if successful, -ve errno otherwise. + */ +static long papr_platform_dump_create_handle(u64 dump_tag) +{ + struct ibm_platform_dump_params *params; + u64 param_dump_tag; + struct file *file; + long err; + int fd; + + /* + * Return failure if the user space is already opened FD for + * the specific dump ID. This check will prevent multiple dump + * requests for the same dump ID at the same time. Generally + * should not expect this, but in case. + */ + list_for_each_entry(params, &platform_dump_list, list) { + param_dump_tag = (u64) (((u64)params->dump_tag_hi << 32) | + params->dump_tag_lo); + if (dump_tag == param_dump_tag) { + pr_err("Platform dump for ID(%llu) is already in progress\n", + dump_tag); + return -EALREADY; + } + } + + params = kzalloc(sizeof(struct ibm_platform_dump_params), + GFP_KERNEL_ACCOUNT); + if (!params) + return -ENOMEM; + + params->work_area = rtas_work_area_alloc(SZ_4K); + params->buf_length = SZ_4K; + params->dump_tag_hi = (u32)(dump_tag >> 32); + params->dump_tag_lo = (u32)(dump_tag & 0x00000000ffffffffULL); + params->status = RTAS_IBM_PLATFORM_DUMP_START; + + fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); + if (fd < 0) { + err = fd; + goto free_area; + } + + file = anon_inode_getfile_fmode("[papr-platform-dump]", + &papr_platform_dump_handle_ops, + (void *)params, O_RDONLY, + FMODE_LSEEK | FMODE_PREAD); + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto put_fd; + } + + fd_install(fd, file); + + list_add(¶ms->list, &platform_dump_list); + + pr_info("%s (%d) initiated platform dump for dump tag %llu\n", + current->comm, current->pid, dump_tag); + return fd; +put_fd: + put_unused_fd(fd); +free_area: + rtas_work_area_free(params->work_area); + kfree(params); + return err; +} + +/* + * Top-level ioctl handler for /dev/papr-platform-dump. + */ +static long papr_platform_dump_dev_ioctl(struct file *filp, + unsigned int ioctl, + unsigned long arg) +{ + u64 __user *argp = (void __user *)arg; + u64 dump_tag; + long ret; + + if (get_user(dump_tag, argp)) + return -EFAULT; + + switch (ioctl) { + case PAPR_PLATFORM_DUMP_IOC_CREATE_HANDLE: + mutex_lock(&platform_dump_list_mutex); + ret = papr_platform_dump_create_handle(dump_tag); + mutex_unlock(&platform_dump_list_mutex); + break; + default: + ret = -ENOIOCTLCMD; + break; + } + return ret; +} + +static const struct file_operations papr_platform_dump_ops = { + .unlocked_ioctl = papr_platform_dump_dev_ioctl, +}; + +static struct miscdevice papr_platform_dump_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "papr-platform-dump", + .fops = &papr_platform_dump_ops, +}; + +static __init int papr_platform_dump_init(void) +{ + if (!rtas_function_implemented(RTAS_FN_IBM_PLATFORM_DUMP)) + return -ENODEV; + + return misc_register(&papr_platform_dump_dev); +} +machine_device_initcall(pseries, papr_platform_dump_init); diff --git a/arch/powerpc/platforms/pseries/papr-rtas-common.c b/arch/powerpc/platforms/pseries/papr-rtas-common.c new file mode 100644 index 000000000000..33c606e3378a --- /dev/null +++ b/arch/powerpc/platforms/pseries/papr-rtas-common.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#define pr_fmt(fmt) "papr-common: " fmt + +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/signal.h> +#include <linux/slab.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/anon_inodes.h> +#include <linux/sched/signal.h> +#include "papr-rtas-common.h" + +/* + * Sequence based RTAS HCALL has to issue multiple times to retrieve + * complete data from the hypervisor. For some of these RTAS calls, + * the OS should not interleave calls with different input until the + * sequence is completed. So data is collected for these calls during + * ioctl handle and export to user space with read() handle. + * This file provides common functions needed for such sequence based + * RTAS calls Ex: ibm,get-vpd and ibm,get-indices. + */ + +bool papr_rtas_blob_has_data(const struct papr_rtas_blob *blob) +{ + return blob->data && blob->len; +} + +void papr_rtas_blob_free(const struct papr_rtas_blob *blob) +{ + if (blob) { + kvfree(blob->data); + kfree(blob); + } +} + +/** + * papr_rtas_blob_extend() - Append data to a &struct papr_rtas_blob. + * @blob: The blob to extend. + * @data: The new data to append to @blob. + * @len: The length of @data. + * + * Context: May sleep. + * Return: -ENOMEM on allocation failure, 0 otherwise. + */ +static int papr_rtas_blob_extend(struct papr_rtas_blob *blob, + const char *data, size_t len) +{ + const size_t new_len = blob->len + len; + const size_t old_len = blob->len; + const char *old_ptr = blob->data; + char *new_ptr; + + new_ptr = kvrealloc(old_ptr, new_len, GFP_KERNEL_ACCOUNT); + if (!new_ptr) + return -ENOMEM; + + memcpy(&new_ptr[old_len], data, len); + blob->data = new_ptr; + blob->len = new_len; + return 0; +} + +/** + * papr_rtas_blob_generate() - Construct a new &struct papr_rtas_blob. + * @seq: work function of the caller that is called to obtain + * data with the caller RTAS call. + * + * The @work callback is invoked until it returns NULL. @seq is + * passed to @work in its first argument on each call. When + * @work returns data, it should store the data length in its + * second argument. + * + * Context: May sleep. + * Return: A completely populated &struct papr_rtas_blob, or NULL on error. + */ +static const struct papr_rtas_blob * +papr_rtas_blob_generate(struct papr_rtas_sequence *seq) +{ + struct papr_rtas_blob *blob; + const char *buf; + size_t len; + int err = 0; + + blob = kzalloc(sizeof(*blob), GFP_KERNEL_ACCOUNT); + if (!blob) + return NULL; + + if (!seq->work) + return ERR_PTR(-EINVAL); + + + while (err == 0 && (buf = seq->work(seq, &len))) + err = papr_rtas_blob_extend(blob, buf, len); + + if (err != 0 || !papr_rtas_blob_has_data(blob)) + goto free_blob; + + return blob; +free_blob: + papr_rtas_blob_free(blob); + return NULL; +} + +int papr_rtas_sequence_set_err(struct papr_rtas_sequence *seq, int err) +{ + /* Preserve the first error recorded. */ + if (seq->error == 0) + seq->error = err; + + return seq->error; +} + +/* + * Higher-level retrieval code below. These functions use the + * papr_rtas_blob_* and sequence_* APIs defined above to create fd-based + * handles for consumption by user space. + */ + +/** + * papr_rtas_run_sequence() - Run a single retrieval sequence. + * @seq: Functions of the caller to complete the sequence + * + * Context: May sleep. Holds a mutex and an RTAS work area for its + * duration. Typically performs multiple sleepable slab + * allocations. + * + * Return: A populated &struct papr_rtas_blob on success. Encoded error + * pointer otherwise. + */ +static const struct papr_rtas_blob *papr_rtas_run_sequence(struct papr_rtas_sequence *seq) +{ + const struct papr_rtas_blob *blob; + + if (seq->begin) + seq->begin(seq); + + blob = papr_rtas_blob_generate(seq); + if (!blob) + papr_rtas_sequence_set_err(seq, -ENOMEM); + + if (seq->end) + seq->end(seq); + + + if (seq->error) { + papr_rtas_blob_free(blob); + return ERR_PTR(seq->error); + } + + return blob; +} + +/** + * papr_rtas_retrieve() - Return the data blob that is exposed to + * user space. + * @seq: RTAS call specific functions to be invoked until the + * sequence is completed. + * + * Run sequences against @param until a blob is successfully + * instantiated, or a hard error is encountered, or a fatal signal is + * pending. + * + * Context: May sleep. + * Return: A fully populated data blob when successful. Encoded error + * pointer otherwise. + */ +const struct papr_rtas_blob *papr_rtas_retrieve(struct papr_rtas_sequence *seq) +{ + const struct papr_rtas_blob *blob; + + /* + * EAGAIN means the sequence returns error with a -4 (data + * changed and need to start the sequence) status from RTAS calls + * and we should attempt a new sequence. PAPR+ (v2.13 R1–7.3.20–5 + * - ibm,get-vpd, R1–7.3.17–6 - ibm,get-indices) indicates that + * this should be a transient condition, not something that + * happens continuously. But we'll stop trying on a fatal signal. + */ + do { + blob = papr_rtas_run_sequence(seq); + if (!IS_ERR(blob)) /* Success. */ + break; + if (PTR_ERR(blob) != -EAGAIN) /* Hard error. */ + break; + cond_resched(); + } while (!fatal_signal_pending(current)); + + return blob; +} + +/** + * papr_rtas_setup_file_interface - Complete the sequence and obtain + * the data and export to user space with fd-based handles. Then the + * user spave gets the data with read() handle. + * @seq: RTAS call specific functions to get the data. + * @fops: RTAS call specific file operations such as read(). + * @name: RTAS call specific char device node. + * + * Return: FD handle for consumption by user space + */ +long papr_rtas_setup_file_interface(struct papr_rtas_sequence *seq, + const struct file_operations *fops, + char *name) +{ + const struct papr_rtas_blob *blob; + struct file *file; + long ret; + int fd; + + blob = papr_rtas_retrieve(seq); + if (IS_ERR(blob)) + return PTR_ERR(blob); + + fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); + if (fd < 0) { + ret = fd; + goto free_blob; + } + + file = anon_inode_getfile_fmode(name, fops, (void *)blob, + O_RDONLY, FMODE_LSEEK | FMODE_PREAD); + if (IS_ERR(file)) { + ret = PTR_ERR(file); + goto put_fd; + } + + fd_install(fd, file); + return fd; + +put_fd: + put_unused_fd(fd); +free_blob: + papr_rtas_blob_free(blob); + return ret; +} + +/* + * papr_rtas_sequence_should_stop() - Determine whether RTAS retrieval + * sequence should continue. + * + * Examines the sequence error state and outputs of the last call to + * the specific RTAS to determine whether the sequence in progress + * should continue or stop. + * + * Return: True if the sequence has encountered an error or if all data + * for this sequence has been retrieved. False otherwise. + */ +bool papr_rtas_sequence_should_stop(const struct papr_rtas_sequence *seq, + s32 status, bool init_state) +{ + bool done; + + if (seq->error) + return true; + + switch (status) { + case RTAS_SEQ_COMPLETE: + if (init_state) + done = false; /* Initial state. */ + else + done = true; /* All data consumed. */ + break; + case RTAS_SEQ_MORE_DATA: + done = false; /* More data available. */ + break; + default: + done = true; /* Error encountered. */ + break; + } + + return done; +} + +/* + * User space read to retrieve data for the corresponding RTAS call. + * papr_rtas_blob is filled with the data using the corresponding RTAS + * call sequence API. + */ +ssize_t papr_rtas_common_handle_read(struct file *file, + char __user *buf, size_t size, loff_t *off) +{ + const struct papr_rtas_blob *blob = file->private_data; + + /* We should not instantiate a handle without any data attached. */ + if (!papr_rtas_blob_has_data(blob)) { + pr_err_once("handle without data\n"); + return -EIO; + } + + return simple_read_from_buffer(buf, size, off, blob->data, blob->len); +} + +int papr_rtas_common_handle_release(struct inode *inode, + struct file *file) +{ + const struct papr_rtas_blob *blob = file->private_data; + + papr_rtas_blob_free(blob); + + return 0; +} + +loff_t papr_rtas_common_handle_seek(struct file *file, loff_t off, + int whence) +{ + const struct papr_rtas_blob *blob = file->private_data; + + return fixed_size_llseek(file, off, whence, blob->len); +} diff --git a/arch/powerpc/platforms/pseries/papr-rtas-common.h b/arch/powerpc/platforms/pseries/papr-rtas-common.h new file mode 100644 index 000000000000..4ceabcaf4905 --- /dev/null +++ b/arch/powerpc/platforms/pseries/papr-rtas-common.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_POWERPC_PAPR_RTAS_COMMON_H +#define _ASM_POWERPC_PAPR_RTAS_COMMON_H + +#include <linux/types.h> + +/* + * Return codes for sequence based RTAS calls. + * Not listed under PAPR+ v2.13 7.2.8: "Return Codes". + * But defined in the specific section of each RTAS call. + */ +#define RTAS_SEQ_COMPLETE 0 /* All data has been retrieved. */ +#define RTAS_SEQ_MORE_DATA 1 /* More data is available */ +#define RTAS_SEQ_START_OVER -4 /* Data changed, restart call sequence. */ + +/* + * Internal "blob" APIs for accumulating RTAS call results into + * an immutable buffer to be attached to a file descriptor. + */ +struct papr_rtas_blob { + const char *data; + size_t len; +}; + +/** + * struct papr_sequence - State for managing a sequence of RTAS calls. + * @error: Shall be zero as long as the sequence has not encountered an error, + * -ve errno otherwise. Use papr_rtas_sequence_set_err() to update. + * @params: Parameter block to pass to rtas_*() calls. + * @begin: Work area allocation and initialize the needed parameter + * values passed to RTAS call + * @end: Free the allocated work area + * @work: Obtain data with RTAS call and invoke it until the sequence is + * completed. + * + */ +struct papr_rtas_sequence { + int error; + void *params; + void (*begin)(struct papr_rtas_sequence *seq); + void (*end)(struct papr_rtas_sequence *seq); + const char *(*work)(struct papr_rtas_sequence *seq, size_t *len); +}; + +extern bool papr_rtas_blob_has_data(const struct papr_rtas_blob *blob); +extern void papr_rtas_blob_free(const struct papr_rtas_blob *blob); +extern int papr_rtas_sequence_set_err(struct papr_rtas_sequence *seq, + int err); +extern const struct papr_rtas_blob *papr_rtas_retrieve(struct papr_rtas_sequence *seq); +extern long papr_rtas_setup_file_interface(struct papr_rtas_sequence *seq, + const struct file_operations *fops, char *name); +extern bool papr_rtas_sequence_should_stop(const struct papr_rtas_sequence *seq, + s32 status, bool init_state); +extern ssize_t papr_rtas_common_handle_read(struct file *file, + char __user *buf, size_t size, loff_t *off); +extern int papr_rtas_common_handle_release(struct inode *inode, + struct file *file); +extern loff_t papr_rtas_common_handle_seek(struct file *file, loff_t off, + int whence); +#endif /* _ASM_POWERPC_PAPR_RTAS_COMMON_H */ + diff --git a/arch/powerpc/platforms/pseries/papr-vpd.c b/arch/powerpc/platforms/pseries/papr-vpd.c index 1574176e3ffc..f38c188fc4a1 100644 --- a/arch/powerpc/platforms/pseries/papr-vpd.c +++ b/arch/powerpc/platforms/pseries/papr-vpd.c @@ -2,7 +2,6 @@ #define pr_fmt(fmt) "papr-vpd: " fmt -#include <linux/anon_inodes.h> #include <linux/build_bug.h> #include <linux/file.h> #include <linux/fs.h> @@ -20,14 +19,7 @@ #include <asm/rtas-work-area.h> #include <asm/rtas.h> #include <uapi/asm/papr-vpd.h> - -/* - * Function-specific return values for ibm,get-vpd, derived from PAPR+ - * v2.13 7.3.20 "ibm,get-vpd RTAS Call". - */ -#define RTAS_IBM_GET_VPD_COMPLETE 0 /* All VPD has been retrieved. */ -#define RTAS_IBM_GET_VPD_MORE_DATA 1 /* More VPD is available. */ -#define RTAS_IBM_GET_VPD_START_OVER -4 /* VPD changed, restart call sequence. */ +#include "papr-rtas-common.h" /** * struct rtas_ibm_get_vpd_params - Parameters (in and out) for ibm,get-vpd. @@ -91,13 +83,14 @@ static int rtas_ibm_get_vpd(struct rtas_ibm_get_vpd_params *params) case RTAS_INVALID_PARAMETER: ret = -EINVAL; break; - case RTAS_IBM_GET_VPD_START_OVER: + case RTAS_SEQ_START_OVER: ret = -EAGAIN; + pr_info_ratelimited("VPD changed during retrieval, retrying\n"); break; - case RTAS_IBM_GET_VPD_MORE_DATA: + case RTAS_SEQ_MORE_DATA: params->sequence = rets[0]; fallthrough; - case RTAS_IBM_GET_VPD_COMPLETE: + case RTAS_SEQ_COMPLETE: params->written = rets[1]; /* * Kernel or firmware bug, do not continue. @@ -119,91 +112,6 @@ static int rtas_ibm_get_vpd(struct rtas_ibm_get_vpd_params *params) } /* - * Internal VPD "blob" APIs for accumulating ibm,get-vpd results into - * an immutable buffer to be attached to a file descriptor. - */ -struct vpd_blob { - const char *data; - size_t len; -}; - -static bool vpd_blob_has_data(const struct vpd_blob *blob) -{ - return blob->data && blob->len; -} - -static void vpd_blob_free(const struct vpd_blob *blob) -{ - if (blob) { - kvfree(blob->data); - kfree(blob); - } -} - -/** - * vpd_blob_extend() - Append data to a &struct vpd_blob. - * @blob: The blob to extend. - * @data: The new data to append to @blob. - * @len: The length of @data. - * - * Context: May sleep. - * Return: -ENOMEM on allocation failure, 0 otherwise. - */ -static int vpd_blob_extend(struct vpd_blob *blob, const char *data, size_t len) -{ - const size_t new_len = blob->len + len; - const size_t old_len = blob->len; - const char *old_ptr = blob->data; - char *new_ptr; - - new_ptr = kvrealloc(old_ptr, new_len, GFP_KERNEL_ACCOUNT); - if (!new_ptr) - return -ENOMEM; - - memcpy(&new_ptr[old_len], data, len); - blob->data = new_ptr; - blob->len = new_len; - return 0; -} - -/** - * vpd_blob_generate() - Construct a new &struct vpd_blob. - * @generator: Function that supplies the blob data. - * @arg: Context pointer supplied by caller, passed to @generator. - * - * The @generator callback is invoked until it returns NULL. @arg is - * passed to @generator in its first argument on each call. When - * @generator returns data, it should store the data length in its - * second argument. - * - * Context: May sleep. - * Return: A completely populated &struct vpd_blob, or NULL on error. - */ -static const struct vpd_blob * -vpd_blob_generate(const char * (*generator)(void *, size_t *), void *arg) -{ - struct vpd_blob *blob; - const char *buf; - size_t len; - int err = 0; - - blob = kzalloc(sizeof(*blob), GFP_KERNEL_ACCOUNT); - if (!blob) - return NULL; - - while (err == 0 && (buf = generator(arg, &len))) - err = vpd_blob_extend(blob, buf, len); - - if (err != 0 || !vpd_blob_has_data(blob)) - goto free_blob; - - return blob; -free_blob: - vpd_blob_free(blob); - return NULL; -} - -/* * Internal VPD sequence APIs. A VPD sequence is a series of calls to * ibm,get-vpd for a given location code. The sequence ends when an * error is encountered or all VPD for the location code has been @@ -211,30 +119,14 @@ free_blob: */ /** - * struct vpd_sequence - State for managing a VPD sequence. - * @error: Shall be zero as long as the sequence has not encountered an error, - * -ve errno otherwise. Use vpd_sequence_set_err() to update this. - * @params: Parameter block to pass to rtas_ibm_get_vpd(). - */ -struct vpd_sequence { - int error; - struct rtas_ibm_get_vpd_params params; -}; - -/** * vpd_sequence_begin() - Begin a VPD retrieval sequence. - * @seq: Uninitialized sequence state. - * @loc_code: Location code that defines the scope of the VPD to return. - * - * Initializes @seq with the resources necessary to carry out a VPD - * sequence. Callers must pass @seq to vpd_sequence_end() regardless - * of whether the sequence succeeds. + * @seq: vpd call parameters from sequence struct * * Context: May sleep. */ -static void vpd_sequence_begin(struct vpd_sequence *seq, - const struct papr_location_code *loc_code) +static void vpd_sequence_begin(struct papr_rtas_sequence *seq) { + struct rtas_ibm_get_vpd_params *vpd_params; /* * Use a static data structure for the location code passed to * RTAS to ensure it's in the RMA and avoid a separate work @@ -242,6 +134,7 @@ static void vpd_sequence_begin(struct vpd_sequence *seq, */ static struct papr_location_code static_loc_code; + vpd_params = (struct rtas_ibm_get_vpd_params *)seq->params; /* * We could allocate the work area before acquiring the * function lock, but that would allow concurrent requests to @@ -249,14 +142,12 @@ static void vpd_sequence_begin(struct vpd_sequence *seq, * allocate the work area under the lock. */ mutex_lock(&rtas_ibm_get_vpd_lock); - static_loc_code = *loc_code; - *seq = (struct vpd_sequence) { - .params = { - .work_area = rtas_work_area_alloc(SZ_4K), - .loc_code = &static_loc_code, - .sequence = 1, - }, - }; + static_loc_code = *(struct papr_location_code *)vpd_params->loc_code; + vpd_params = (struct rtas_ibm_get_vpd_params *)seq->params; + vpd_params->work_area = rtas_work_area_alloc(SZ_4K); + vpd_params->loc_code = &static_loc_code; + vpd_params->sequence = 1; + vpd_params->status = 0; } /** @@ -265,180 +156,39 @@ static void vpd_sequence_begin(struct vpd_sequence *seq, * * Releases resources obtained by vpd_sequence_begin(). */ -static void vpd_sequence_end(struct vpd_sequence *seq) -{ - rtas_work_area_free(seq->params.work_area); - mutex_unlock(&rtas_ibm_get_vpd_lock); -} - -/** - * vpd_sequence_should_stop() - Determine whether a VPD retrieval sequence - * should continue. - * @seq: VPD sequence state. - * - * Examines the sequence error state and outputs of the last call to - * ibm,get-vpd to determine whether the sequence in progress should - * continue or stop. - * - * Return: True if the sequence has encountered an error or if all VPD for - * this sequence has been retrieved. False otherwise. - */ -static bool vpd_sequence_should_stop(const struct vpd_sequence *seq) -{ - bool done; - - if (seq->error) - return true; - - switch (seq->params.status) { - case 0: - if (seq->params.written == 0) - done = false; /* Initial state. */ - else - done = true; /* All data consumed. */ - break; - case 1: - done = false; /* More data available. */ - break; - default: - done = true; /* Error encountered. */ - break; - } - - return done; -} - -static int vpd_sequence_set_err(struct vpd_sequence *seq, int err) +static void vpd_sequence_end(struct papr_rtas_sequence *seq) { - /* Preserve the first error recorded. */ - if (seq->error == 0) - seq->error = err; + struct rtas_ibm_get_vpd_params *vpd_params; - return seq->error; + vpd_params = (struct rtas_ibm_get_vpd_params *)seq->params; + rtas_work_area_free(vpd_params->work_area); + mutex_unlock(&rtas_ibm_get_vpd_lock); } /* - * Generator function to be passed to vpd_blob_generate(). + * Generator function to be passed to papr_rtas_blob_generate(). */ -static const char *vpd_sequence_fill_work_area(void *arg, size_t *len) +static const char *vpd_sequence_fill_work_area(struct papr_rtas_sequence *seq, + size_t *len) { - struct vpd_sequence *seq = arg; - struct rtas_ibm_get_vpd_params *p = &seq->params; + struct rtas_ibm_get_vpd_params *p; + bool init_state; + + p = (struct rtas_ibm_get_vpd_params *)seq->params; + init_state = (p->written == 0) ? true : false; - if (vpd_sequence_should_stop(seq)) + if (papr_rtas_sequence_should_stop(seq, p->status, init_state)) return NULL; - if (vpd_sequence_set_err(seq, rtas_ibm_get_vpd(p))) + if (papr_rtas_sequence_set_err(seq, rtas_ibm_get_vpd(p))) return NULL; *len = p->written; return rtas_work_area_raw_buf(p->work_area); } -/* - * Higher-level VPD retrieval code below. These functions use the - * vpd_blob_* and vpd_sequence_* APIs defined above to create fd-based - * VPD handles for consumption by user space. - */ - -/** - * papr_vpd_run_sequence() - Run a single VPD retrieval sequence. - * @loc_code: Location code that defines the scope of VPD to return. - * - * Context: May sleep. Holds a mutex and an RTAS work area for its - * duration. Typically performs multiple sleepable slab - * allocations. - * - * Return: A populated &struct vpd_blob on success. Encoded error - * pointer otherwise. - */ -static const struct vpd_blob *papr_vpd_run_sequence(const struct papr_location_code *loc_code) -{ - const struct vpd_blob *blob; - struct vpd_sequence seq; - - vpd_sequence_begin(&seq, loc_code); - blob = vpd_blob_generate(vpd_sequence_fill_work_area, &seq); - if (!blob) - vpd_sequence_set_err(&seq, -ENOMEM); - vpd_sequence_end(&seq); - - if (seq.error) { - vpd_blob_free(blob); - return ERR_PTR(seq.error); - } - - return blob; -} - -/** - * papr_vpd_retrieve() - Return the VPD for a location code. - * @loc_code: Location code that defines the scope of VPD to return. - * - * Run VPD sequences against @loc_code until a blob is successfully - * instantiated, or a hard error is encountered, or a fatal signal is - * pending. - * - * Context: May sleep. - * Return: A fully populated VPD blob when successful. Encoded error - * pointer otherwise. - */ -static const struct vpd_blob *papr_vpd_retrieve(const struct papr_location_code *loc_code) -{ - const struct vpd_blob *blob; - - /* - * EAGAIN means the sequence errored with a -4 (VPD changed) - * status from ibm,get-vpd, and we should attempt a new - * sequence. PAPR+ v2.13 R1–7.3.20–5 indicates that this - * should be a transient condition, not something that happens - * continuously. But we'll stop trying on a fatal signal. - */ - do { - blob = papr_vpd_run_sequence(loc_code); - if (!IS_ERR(blob)) /* Success. */ - break; - if (PTR_ERR(blob) != -EAGAIN) /* Hard error. */ - break; - pr_info_ratelimited("VPD changed during retrieval, retrying\n"); - cond_resched(); - } while (!fatal_signal_pending(current)); - - return blob; -} - -static ssize_t papr_vpd_handle_read(struct file *file, char __user *buf, size_t size, loff_t *off) -{ - const struct vpd_blob *blob = file->private_data; - - /* bug: we should not instantiate a handle without any data attached. */ - if (!vpd_blob_has_data(blob)) { - pr_err_once("handle without data\n"); - return -EIO; - } - - return simple_read_from_buffer(buf, size, off, blob->data, blob->len); -} - -static int papr_vpd_handle_release(struct inode *inode, struct file *file) -{ - const struct vpd_blob *blob = file->private_data; - - vpd_blob_free(blob); - - return 0; -} - -static loff_t papr_vpd_handle_seek(struct file *file, loff_t off, int whence) -{ - const struct vpd_blob *blob = file->private_data; - - return fixed_size_llseek(file, off, whence, blob->len); -} - - static const struct file_operations papr_vpd_handle_ops = { - .read = papr_vpd_handle_read, - .llseek = papr_vpd_handle_seek, - .release = papr_vpd_handle_release, + .read = papr_rtas_common_handle_read, + .llseek = papr_rtas_common_handle_seek, + .release = papr_rtas_common_handle_release, }; /** @@ -460,10 +210,9 @@ static const struct file_operations papr_vpd_handle_ops = { */ static long papr_vpd_create_handle(struct papr_location_code __user *ulc) { + struct rtas_ibm_get_vpd_params vpd_params = {}; + struct papr_rtas_sequence seq = {}; struct papr_location_code klc; - const struct vpd_blob *blob; - struct file *file; - long err; int fd; if (copy_from_user(&klc, ulc, sizeof(klc))) @@ -472,31 +221,19 @@ static long papr_vpd_create_handle(struct papr_location_code __user *ulc) if (!string_is_terminated(klc.str, ARRAY_SIZE(klc.str))) return -EINVAL; - blob = papr_vpd_retrieve(&klc); - if (IS_ERR(blob)) - return PTR_ERR(blob); + seq = (struct papr_rtas_sequence) { + .begin = vpd_sequence_begin, + .end = vpd_sequence_end, + .work = vpd_sequence_fill_work_area, + }; - fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); - if (fd < 0) { - err = fd; - goto free_blob; - } + vpd_params.loc_code = &klc; + seq.params = (void *)&vpd_params; - file = anon_inode_getfile("[papr-vpd]", &papr_vpd_handle_ops, - (void *)blob, O_RDONLY); - if (IS_ERR(file)) { - err = PTR_ERR(file); - goto put_fd; - } + fd = papr_rtas_setup_file_interface(&seq, &papr_vpd_handle_ops, + "[papr-vpd]"); - file->f_mode |= FMODE_LSEEK | FMODE_PREAD; - fd_install(fd, file); return fd; -put_fd: - put_unused_fd(fd); -free_blob: - vpd_blob_free(blob); - return err; } /* diff --git a/arch/powerpc/platforms/pseries/papr_scm.c b/arch/powerpc/platforms/pseries/papr_scm.c index f84ac9fbe203..f7c9271bda58 100644 --- a/arch/powerpc/platforms/pseries/papr_scm.c +++ b/arch/powerpc/platforms/pseries/papr_scm.c @@ -544,7 +544,7 @@ static int drc_pmem_query_health(struct papr_scm_priv *p) /* Jiffies offset for which the health data is assumed to be same */ cache_timeout = p->lasthealth_jiffies + - msecs_to_jiffies(MIN_HEALTH_QUERY_INTERVAL * 1000); + secs_to_jiffies(MIN_HEALTH_QUERY_INTERVAL); /* Fetch new health info is its older than MIN_HEALTH_QUERY_INTERVAL */ if (time_after(jiffies, cache_timeout)) diff --git a/arch/powerpc/platforms/pseries/pci_dlpar.c b/arch/powerpc/platforms/pseries/pci_dlpar.c index 52e2623a741d..aeb8633a3d00 100644 --- a/arch/powerpc/platforms/pseries/pci_dlpar.c +++ b/arch/powerpc/platforms/pseries/pci_dlpar.c @@ -29,7 +29,7 @@ struct pci_controller *init_phb_dynamic(struct device_node *dn) nid = of_node_to_nid(dn); if (likely((nid) >= 0)) { if (!node_online(nid)) { - if (__register_one_node(nid)) { + if (register_one_node(nid)) { pr_err("PCI: Failed to register node %d\n", nid); } else { update_numa_distance(dn); diff --git a/arch/powerpc/platforms/pseries/plpks-secvar.c b/arch/powerpc/platforms/pseries/plpks-secvar.c index 257fd1f8bc19..f9e9cc40c9d0 100644 --- a/arch/powerpc/platforms/pseries/plpks-secvar.c +++ b/arch/powerpc/platforms/pseries/plpks-secvar.c @@ -59,7 +59,14 @@ static u32 get_policy(const char *name) return PLPKS_SIGNEDUPDATE; } -static const char * const plpks_var_names[] = { +static const char * const plpks_var_names_static[] = { + "PK", + "moduledb", + "trustedcadb", + NULL, +}; + +static const char * const plpks_var_names_dynamic[] = { "PK", "KEK", "db", @@ -152,39 +159,55 @@ err: return rc; } -// PLPKS dynamic secure boot doesn't give us a format string in the same way OPAL does. -// Instead, report the format using the SB_VERSION variable in the keystore. -// The string is made up by us, and takes the form "ibm,plpks-sb-v<n>" (or "ibm,plpks-sb-unknown" -// if the SB_VERSION variable doesn't exist). Hypervisor defines the SB_VERSION variable as a -// "1 byte unsigned integer value". -static ssize_t plpks_secvar_format(char *buf, size_t bufsize) +/* + * Return the key management mode. + * + * SB_VERSION is defined as a "1 byte unsigned integer value", taking values + * starting from 1. It is owned by the Partition Firmware and its presence + * indicates that the key management mode is dynamic. Any failure in + * reading SB_VERSION defaults the key management mode to static. The error + * codes -ENOENT or -EPERM are expected in static key management mode. An + * unexpected error code will have to be investigated. Only signed variables + * have null bytes in their names, SB_VERSION does not. + * + * Return 0 to indicate that the key management mode is static. Otherwise + * return the SB_VERSION value to indicate that the key management mode is + * dynamic. + */ +static u8 plpks_get_sb_keymgmt_mode(void) { - struct plpks_var var = {0}; - ssize_t ret; - u8 version; - - var.component = NULL; - // Only the signed variables have null bytes in their names, this one doesn't - var.name = "SB_VERSION"; - var.namelen = strlen(var.name); - var.datalen = 1; - var.data = &version; - - // Unlike the other vars, SB_VERSION is owned by firmware instead of the OS - ret = plpks_read_fw_var(&var); - if (ret) { - if (ret == -ENOENT) { - ret = snprintf(buf, bufsize, "ibm,plpks-sb-unknown"); - } else { - pr_err("Error %ld reading SB_VERSION from firmware\n", ret); - ret = -EIO; - } - goto err; + u8 mode; + ssize_t rc; + struct plpks_var var = { + .component = NULL, + .name = "SB_VERSION", + .namelen = 10, + .datalen = 1, + .data = &mode, + }; + + rc = plpks_read_fw_var(&var); + if (rc) { + if (rc != -ENOENT && rc != -EPERM) + pr_info("Error %ld reading SB_VERSION from firmware\n", rc); + mode = 0; } + return mode; +} - ret = snprintf(buf, bufsize, "ibm,plpks-sb-v%hhu", version); -err: - return ret; +/* + * PLPKS dynamic secure boot doesn't give us a format string in the same way + * OPAL does. Instead, report the format using the SB_VERSION variable in the + * keystore. The string, made up by us, takes the form of either + * "ibm,plpks-sb-v<n>" or "ibm,plpks-sb-v0", based on the key management mode, + * and return the length of the secvar format property. + */ +static ssize_t plpks_secvar_format(char *buf, size_t bufsize) +{ + u8 mode; + + mode = plpks_get_sb_keymgmt_mode(); + return snprintf(buf, bufsize, "ibm,plpks-sb-v%hhu", mode); } static int plpks_max_size(u64 *max_size) @@ -197,21 +220,34 @@ static int plpks_max_size(u64 *max_size) return 0; } +static const struct secvar_operations plpks_secvar_ops_static = { + .get = plpks_get_variable, + .set = plpks_set_variable, + .format = plpks_secvar_format, + .max_size = plpks_max_size, + .config_attrs = config_attrs, + .var_names = plpks_var_names_static, +}; -static const struct secvar_operations plpks_secvar_ops = { +static const struct secvar_operations plpks_secvar_ops_dynamic = { .get = plpks_get_variable, .set = plpks_set_variable, .format = plpks_secvar_format, .max_size = plpks_max_size, .config_attrs = config_attrs, - .var_names = plpks_var_names, + .var_names = plpks_var_names_dynamic, }; static int plpks_secvar_init(void) { + u8 mode; + if (!plpks_is_available()) return -ENODEV; - return set_secvar_ops(&plpks_secvar_ops); + mode = plpks_get_sb_keymgmt_mode(); + if (mode) + return set_secvar_ops(&plpks_secvar_ops_dynamic); + return set_secvar_ops(&plpks_secvar_ops_static); } machine_device_initcall(pseries, plpks_secvar_init); diff --git a/arch/powerpc/sysdev/Makefile b/arch/powerpc/sysdev/Makefile index 24a177d164f1..0834a9a12600 100644 --- a/arch/powerpc/sysdev/Makefile +++ b/arch/powerpc/sysdev/Makefile @@ -12,7 +12,6 @@ obj-$(CONFIG_PPC_MSI_BITMAP) += msi_bitmap.o obj-$(CONFIG_PPC_MPC106) += grackle.o obj-$(CONFIG_PPC_DCR_NATIVE) += dcr-low.o -obj-$(CONFIG_PPC_PMI) += pmi.o obj-$(CONFIG_U3_DART) += dart_iommu.o obj-$(CONFIG_MMIO_NVRAM) += mmio_nvram.o obj-$(CONFIG_FSL_SOC) += fsl_soc.o fsl_mpic_err.o diff --git a/arch/powerpc/sysdev/cpm2_pic.c b/arch/powerpc/sysdev/cpm2_pic.c index e14493685fe8..4a59ed1d62ce 100644 --- a/arch/powerpc/sysdev/cpm2_pic.c +++ b/arch/powerpc/sysdev/cpm2_pic.c @@ -207,7 +207,7 @@ unsigned int cpm2_get_irq(void) if (irq == 0) return(-1); - return irq_linear_revmap(cpm2_pic_host, irq); + return irq_find_mapping(cpm2_pic_host, irq); } static int cpm2_pic_host_map(struct irq_domain *h, unsigned int virq, @@ -259,7 +259,8 @@ void cpm2_pic_init(struct device_node *node) out_be32(&cpm2_intctl->ic_scprrl, 0x05309770); /* create a legacy host */ - cpm2_pic_host = irq_domain_add_linear(node, 64, &cpm2_pic_host_ops, NULL); + cpm2_pic_host = irq_domain_create_linear(of_fwnode_handle(node), 64, + &cpm2_pic_host_ops, NULL); if (cpm2_pic_host == NULL) { printk(KERN_ERR "CPM2 PIC: failed to allocate irq host!\n"); return; diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c index 47db732981a8..f469f6a9f6e0 100644 --- a/arch/powerpc/sysdev/cpm_common.c +++ b/arch/powerpc/sysdev/cpm_common.c @@ -138,7 +138,7 @@ static void __cpm2_gpio32_set(struct of_mm_gpio_chip *mm_gc, u32 pin_mask, out_be32(&iop->dat, cpm2_gc->cpdata); } -static void cpm2_gpio32_set(struct gpio_chip *gc, unsigned int gpio, int value) +static int cpm2_gpio32_set(struct gpio_chip *gc, unsigned int gpio, int value) { struct of_mm_gpio_chip *mm_gc = to_of_mm_gpio_chip(gc); struct cpm2_gpio32_chip *cpm2_gc = gpiochip_get_data(gc); @@ -150,6 +150,8 @@ static void cpm2_gpio32_set(struct gpio_chip *gc, unsigned int gpio, int value) __cpm2_gpio32_set(mm_gc, pin_mask, value); spin_unlock_irqrestore(&cpm2_gc->lock, flags); + + return 0; } static int cpm2_gpio32_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) diff --git a/arch/powerpc/sysdev/dcr.c b/arch/powerpc/sysdev/dcr.c index 70ce66eadff1..cb44a69958e7 100644 --- a/arch/powerpc/sysdev/dcr.c +++ b/arch/powerpc/sysdev/dcr.c @@ -11,107 +11,6 @@ #include <linux/of_address.h> #include <asm/dcr.h> -#ifdef CONFIG_PPC_DCR_MMIO -static struct device_node *find_dcr_parent(struct device_node *node) -{ - struct device_node *par, *tmp; - const u32 *p; - - for (par = of_node_get(node); par;) { - if (of_property_read_bool(par, "dcr-controller")) - break; - p = of_get_property(par, "dcr-parent", NULL); - tmp = par; - if (p == NULL) - par = of_get_parent(par); - else - par = of_find_node_by_phandle(*p); - of_node_put(tmp); - } - return par; -} -#endif - -#if defined(CONFIG_PPC_DCR_NATIVE) && defined(CONFIG_PPC_DCR_MMIO) - -bool dcr_map_ok_generic(dcr_host_t host) -{ - if (host.type == DCR_HOST_NATIVE) - return dcr_map_ok_native(host.host.native); - else if (host.type == DCR_HOST_MMIO) - return dcr_map_ok_mmio(host.host.mmio); - else - return false; -} -EXPORT_SYMBOL_GPL(dcr_map_ok_generic); - -dcr_host_t dcr_map_generic(struct device_node *dev, - unsigned int dcr_n, - unsigned int dcr_c) -{ - dcr_host_t host; - struct device_node *dp; - const char *prop; - - host.type = DCR_HOST_INVALID; - - dp = find_dcr_parent(dev); - if (dp == NULL) - return host; - - prop = of_get_property(dp, "dcr-access-method", NULL); - - pr_debug("dcr_map_generic(dcr-access-method = %s)\n", prop); - - if (!strcmp(prop, "native")) { - host.type = DCR_HOST_NATIVE; - host.host.native = dcr_map_native(dev, dcr_n, dcr_c); - } else if (!strcmp(prop, "mmio")) { - host.type = DCR_HOST_MMIO; - host.host.mmio = dcr_map_mmio(dev, dcr_n, dcr_c); - } - - of_node_put(dp); - return host; -} -EXPORT_SYMBOL_GPL(dcr_map_generic); - -void dcr_unmap_generic(dcr_host_t host, unsigned int dcr_c) -{ - if (host.type == DCR_HOST_NATIVE) - dcr_unmap_native(host.host.native, dcr_c); - else if (host.type == DCR_HOST_MMIO) - dcr_unmap_mmio(host.host.mmio, dcr_c); - else /* host.type == DCR_HOST_INVALID */ - WARN_ON(true); -} -EXPORT_SYMBOL_GPL(dcr_unmap_generic); - -u32 dcr_read_generic(dcr_host_t host, unsigned int dcr_n) -{ - if (host.type == DCR_HOST_NATIVE) - return dcr_read_native(host.host.native, dcr_n); - else if (host.type == DCR_HOST_MMIO) - return dcr_read_mmio(host.host.mmio, dcr_n); - else /* host.type == DCR_HOST_INVALID */ - WARN_ON(true); - return 0; -} -EXPORT_SYMBOL_GPL(dcr_read_generic); - -void dcr_write_generic(dcr_host_t host, unsigned int dcr_n, u32 value) -{ - if (host.type == DCR_HOST_NATIVE) - dcr_write_native(host.host.native, dcr_n, value); - else if (host.type == DCR_HOST_MMIO) - dcr_write_mmio(host.host.mmio, dcr_n, value); - else /* host.type == DCR_HOST_INVALID */ - WARN_ON(true); -} -EXPORT_SYMBOL_GPL(dcr_write_generic); - -#endif /* defined(CONFIG_PPC_DCR_NATIVE) && defined(CONFIG_PPC_DCR_MMIO) */ - unsigned int dcr_resource_start(const struct device_node *np, unsigned int index) { @@ -137,86 +36,5 @@ unsigned int dcr_resource_len(const struct device_node *np, unsigned int index) } EXPORT_SYMBOL_GPL(dcr_resource_len); -#ifdef CONFIG_PPC_DCR_MMIO - -static u64 of_translate_dcr_address(struct device_node *dev, - unsigned int dcr_n, - unsigned int *out_stride) -{ - struct device_node *dp; - const u32 *p; - unsigned int stride; - u64 ret = OF_BAD_ADDR; - - dp = find_dcr_parent(dev); - if (dp == NULL) - return OF_BAD_ADDR; - - /* Stride is not properly defined yet, default to 0x10 for Axon */ - p = of_get_property(dp, "dcr-mmio-stride", NULL); - stride = (p == NULL) ? 0x10 : *p; - - /* XXX FIXME: Which property name is to use of the 2 following ? */ - p = of_get_property(dp, "dcr-mmio-range", NULL); - if (p == NULL) - p = of_get_property(dp, "dcr-mmio-space", NULL); - if (p == NULL) - goto done; - - /* Maybe could do some better range checking here */ - ret = of_translate_address(dp, p); - if (ret != OF_BAD_ADDR) - ret += (u64)(stride) * (u64)dcr_n; - if (out_stride) - *out_stride = stride; - - done: - of_node_put(dp); - return ret; -} - -dcr_host_mmio_t dcr_map_mmio(struct device_node *dev, - unsigned int dcr_n, - unsigned int dcr_c) -{ - dcr_host_mmio_t ret = { .token = NULL, .stride = 0, .base = dcr_n }; - u64 addr; - - pr_debug("dcr_map(%pOF, 0x%x, 0x%x)\n", - dev, dcr_n, dcr_c); - - addr = of_translate_dcr_address(dev, dcr_n, &ret.stride); - pr_debug("translates to addr: 0x%llx, stride: 0x%x\n", - (unsigned long long) addr, ret.stride); - if (addr == OF_BAD_ADDR) - return ret; - pr_debug("mapping 0x%x bytes\n", dcr_c * ret.stride); - ret.token = ioremap(addr, dcr_c * ret.stride); - if (ret.token == NULL) - return ret; - pr_debug("mapped at 0x%p -> base is 0x%p\n", - ret.token, ret.token - dcr_n * ret.stride); - ret.token -= dcr_n * ret.stride; - return ret; -} -EXPORT_SYMBOL_GPL(dcr_map_mmio); - -void dcr_unmap_mmio(dcr_host_mmio_t host, unsigned int dcr_c) -{ - dcr_host_mmio_t h = host; - - if (h.token == NULL) - return; - h.token += host.base * h.stride; - iounmap(h.token); - h.token = NULL; -} -EXPORT_SYMBOL_GPL(dcr_unmap_mmio); - -#endif /* defined(CONFIG_PPC_DCR_MMIO) */ - -#ifdef CONFIG_PPC_DCR_NATIVE DEFINE_SPINLOCK(dcr_ind_lock); EXPORT_SYMBOL_GPL(dcr_ind_lock); -#endif /* defined(CONFIG_PPC_DCR_NATIVE) */ - diff --git a/arch/powerpc/sysdev/ehv_pic.c b/arch/powerpc/sysdev/ehv_pic.c index 040827671d21..b6f9774038e1 100644 --- a/arch/powerpc/sysdev/ehv_pic.c +++ b/arch/powerpc/sysdev/ehv_pic.c @@ -175,7 +175,7 @@ unsigned int ehv_pic_get_irq(void) * this will also setup revmap[] in the slow path for the first * time, next calls will always use fast path by indexing revmap */ - return irq_linear_revmap(global_ehv_pic->irqhost, irq); + return irq_find_mapping(global_ehv_pic->irqhost, irq); } static int ehv_pic_host_match(struct irq_domain *h, struct device_node *node, @@ -269,8 +269,9 @@ void __init ehv_pic_init(void) return; } - ehv_pic->irqhost = irq_domain_add_linear(np, NR_EHV_PIC_INTS, - &ehv_pic_host_ops, ehv_pic); + ehv_pic->irqhost = irq_domain_create_linear(of_fwnode_handle(np), + NR_EHV_PIC_INTS, + &ehv_pic_host_ops, ehv_pic); if (!ehv_pic->irqhost) { of_node_put(np); kfree(ehv_pic); @@ -291,5 +292,5 @@ void __init ehv_pic_init(void) ehv_pic->coreint_flag = of_property_read_bool(np, "has-external-proxy"); global_ehv_pic = ehv_pic; - irq_set_default_host(global_ehv_pic->irqhost); + irq_set_default_domain(global_ehv_pic->irqhost); } diff --git a/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c b/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c index ce6c739c51e5..06d9101a5d49 100644 --- a/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c +++ b/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c @@ -75,7 +75,7 @@ static ssize_t fsl_timer_wakeup_store(struct device *dev, if (kstrtoll(buf, 0, &interval)) return -EINVAL; - mutex_lock(&sysfs_lock); + guard(mutex)(&sysfs_lock); if (fsl_wakeup->timer) { disable_irq_wake(fsl_wakeup->timer->irq); @@ -83,31 +83,23 @@ static ssize_t fsl_timer_wakeup_store(struct device *dev, fsl_wakeup->timer = NULL; } - if (!interval) { - mutex_unlock(&sysfs_lock); + if (!interval) return count; - } fsl_wakeup->timer = mpic_request_timer(fsl_mpic_timer_irq, fsl_wakeup, interval); - if (!fsl_wakeup->timer) { - mutex_unlock(&sysfs_lock); + if (!fsl_wakeup->timer) return -EINVAL; - } ret = enable_irq_wake(fsl_wakeup->timer->irq); if (ret) { mpic_free_timer(fsl_wakeup->timer); fsl_wakeup->timer = NULL; - mutex_unlock(&sysfs_lock); - return ret; } mpic_start_timer(fsl_wakeup->timer); - mutex_unlock(&sysfs_lock); - return count; } diff --git a/arch/powerpc/sysdev/fsl_msi.c b/arch/powerpc/sysdev/fsl_msi.c index 1aa0cb097c9c..4fe8a7b1b288 100644 --- a/arch/powerpc/sysdev/fsl_msi.c +++ b/arch/powerpc/sysdev/fsl_msi.c @@ -75,7 +75,7 @@ static void fsl_msi_print_chip(struct irq_data *irqd, struct seq_file *p) srs = (hwirq >> msi_data->srs_shift) & MSI_SRS_MASK; cascade_virq = msi_data->cascade_array[srs]->virq; - seq_printf(p, " fsl-msi-%d", cascade_virq); + seq_printf(p, "fsl-msi-%d", cascade_virq); } @@ -412,7 +412,7 @@ static int fsl_of_msi_probe(struct platform_device *dev) } platform_set_drvdata(dev, msi); - msi->irqhost = irq_domain_add_linear(dev->dev.of_node, + msi->irqhost = irq_domain_create_linear(of_fwnode_handle(dev->dev.of_node), NR_MSI_IRQS_MAX, &fsl_msi_host_ops, msi); if (msi->irqhost == NULL) { diff --git a/arch/powerpc/sysdev/ge/ge_pic.c b/arch/powerpc/sysdev/ge/ge_pic.c index a6c424680c37..0bc3f0b36528 100644 --- a/arch/powerpc/sysdev/ge/ge_pic.c +++ b/arch/powerpc/sysdev/ge/ge_pic.c @@ -214,8 +214,9 @@ void __init gef_pic_init(struct device_node *np) } /* Setup an irq_domain structure */ - gef_pic_irq_host = irq_domain_add_linear(np, GEF_PIC_NUM_IRQS, - &gef_pic_host_ops, NULL); + gef_pic_irq_host = irq_domain_create_linear(of_fwnode_handle(np), + GEF_PIC_NUM_IRQS, + &gef_pic_host_ops, NULL); if (gef_pic_irq_host == NULL) return; @@ -244,7 +245,7 @@ unsigned int gef_pic_get_irq(void) if (active & (0x1 << hwirq)) break; } - virq = irq_linear_revmap(gef_pic_irq_host, + virq = irq_find_mapping(gef_pic_irq_host, (irq_hw_number_t)hwirq); } diff --git a/arch/powerpc/sysdev/i8259.c b/arch/powerpc/sysdev/i8259.c index 06e391485da7..99bb2b916949 100644 --- a/arch/powerpc/sysdev/i8259.c +++ b/arch/powerpc/sysdev/i8259.c @@ -260,8 +260,8 @@ void i8259_init(struct device_node *node, unsigned long intack_addr) raw_spin_unlock_irqrestore(&i8259_lock, flags); /* create a legacy host */ - i8259_host = irq_domain_add_legacy(node, NR_IRQS_LEGACY, 0, 0, - &i8259_host_ops, NULL); + i8259_host = irq_domain_create_legacy(of_fwnode_handle(node), NR_IRQS_LEGACY, 0, 0, + &i8259_host_ops, NULL); if (i8259_host == NULL) { printk(KERN_ERR "i8259: failed to allocate irq host !\n"); return; diff --git a/arch/powerpc/sysdev/ipic.c b/arch/powerpc/sysdev/ipic.c index 5f69e2d50f26..70be2105865d 100644 --- a/arch/powerpc/sysdev/ipic.c +++ b/arch/powerpc/sysdev/ipic.c @@ -711,8 +711,9 @@ struct ipic * __init ipic_init(struct device_node *node, unsigned int flags) if (ipic == NULL) return NULL; - ipic->irqhost = irq_domain_add_linear(node, NR_IPIC_INTS, - &ipic_host_ops, ipic); + ipic->irqhost = irq_domain_create_linear(of_fwnode_handle(node), + NR_IPIC_INTS, + &ipic_host_ops, ipic); if (ipic->irqhost == NULL) { kfree(ipic); return NULL; @@ -757,13 +758,12 @@ struct ipic * __init ipic_init(struct device_node *node, unsigned int flags) ipic_write(ipic->regs, IPIC_SEMSR, temp); primary_ipic = ipic; - irq_set_default_host(primary_ipic->irqhost); + irq_set_default_domain(primary_ipic->irqhost); ipic_write(ipic->regs, IPIC_SIMSR_H, 0); ipic_write(ipic->regs, IPIC_SIMSR_L, 0); - printk ("IPIC (%d IRQ sources) at %p\n", NR_IPIC_INTS, - primary_ipic->regs); + pr_info("IPIC (%d IRQ sources) at MMIO %pa\n", NR_IPIC_INTS, &res.start); return ipic; } @@ -801,7 +801,7 @@ unsigned int ipic_get_irq(void) if (irq == 0) /* 0 --> no irq is pending */ return 0; - return irq_linear_revmap(primary_ipic->irqhost, irq); + return irq_find_mapping(primary_ipic->irqhost, irq); } #ifdef CONFIG_SUSPEND diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index d94cf36b0f65..ad7310bba00b 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -27,6 +27,7 @@ #include <linux/spinlock.h> #include <linux/pci.h> #include <linux/slab.h> +#include <linux/string_choices.h> #include <linux/syscore_ops.h> #include <linux/ratelimit.h> #include <linux/pgtable.h> @@ -474,9 +475,9 @@ static void __init mpic_scan_ht_msi(struct mpic *mpic, u8 __iomem *devbase, addr = addr | ((u64)readl(base + HT_MSI_ADDR_HI) << 32); } - printk(KERN_DEBUG "mpic: - HT:%02x.%x %s MSI mapping found @ 0x%llx\n", - PCI_SLOT(devfn), PCI_FUNC(devfn), - flags & HT_MSI_FLAGS_ENABLE ? "enabled" : "disabled", addr); + pr_debug("mpic: - HT:%02x.%x %s MSI mapping found @ 0x%llx\n", + PCI_SLOT(devfn), PCI_FUNC(devfn), + str_enabled_disabled(flags & HT_MSI_FLAGS_ENABLE), addr); if (!(flags & HT_MSI_FLAGS_ENABLE)) writeb(flags | HT_MSI_FLAGS_ENABLE, base + HT_MSI_FLAGS); @@ -1483,9 +1484,9 @@ struct mpic * __init mpic_alloc(struct device_node *node, mpic->isu_shift = 1 + __ilog2(mpic->isu_size - 1); mpic->isu_mask = (1 << mpic->isu_shift) - 1; - mpic->irqhost = irq_domain_add_linear(mpic->node, - intvec_top, - &mpic_host_ops, mpic); + mpic->irqhost = irq_domain_create_linear(of_fwnode_handle(mpic->node), + intvec_top, + &mpic_host_ops, mpic); /* * FIXME: The code leaks the MPIC object and mappings here; this @@ -1520,7 +1521,7 @@ struct mpic * __init mpic_alloc(struct device_node *node, if (!(mpic->flags & MPIC_SECONDARY)) { mpic_primary = mpic; - irq_set_default_host(mpic->irqhost); + irq_set_default_domain(mpic->irqhost); } return mpic; @@ -1785,7 +1786,7 @@ static unsigned int _mpic_get_one_irq(struct mpic *mpic, int reg) return 0; } - return irq_linear_revmap(mpic->irqhost, src); + return irq_find_mapping(mpic->irqhost, src); } unsigned int mpic_get_one_irq(struct mpic *mpic) @@ -1823,7 +1824,7 @@ unsigned int mpic_get_coreint_irq(void) return 0; } - return irq_linear_revmap(mpic->irqhost, src); + return irq_find_mapping(mpic->irqhost, src); #else return 0; #endif diff --git a/arch/powerpc/sysdev/msi_bitmap.c b/arch/powerpc/sysdev/msi_bitmap.c index 0b6e37f3ffb8..456a4f64ae0a 100644 --- a/arch/powerpc/sysdev/msi_bitmap.c +++ b/arch/powerpc/sysdev/msi_bitmap.c @@ -124,10 +124,7 @@ int __ref msi_bitmap_alloc(struct msi_bitmap *bmp, unsigned int irq_count, if (bmp->bitmap_from_slab) bmp->bitmap = kzalloc(size, GFP_KERNEL); else { - bmp->bitmap = memblock_alloc(size, SMP_CACHE_BYTES); - if (!bmp->bitmap) - panic("%s: Failed to allocate %u bytes\n", __func__, - size); + bmp->bitmap = memblock_alloc_or_panic(size, SMP_CACHE_BYTES); /* the bitmap won't be freed from memblock allocator */ kmemleak_not_leak(bmp->bitmap); } diff --git a/arch/powerpc/sysdev/pmi.c b/arch/powerpc/sysdev/pmi.c deleted file mode 100644 index 2511e586fe31..000000000000 --- a/arch/powerpc/sysdev/pmi.c +++ /dev/null @@ -1,267 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * pmi driver - * - * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 - * - * PMI (Platform Management Interrupt) is a way to communicate - * with the BMC (Baseboard Management Controller) via interrupts. - * Unlike IPMI it is bidirectional and has a low latency. - * - * Author: Christian Krafft <krafft@de.ibm.com> - */ - -#include <linux/interrupt.h> -#include <linux/slab.h> -#include <linux/completion.h> -#include <linux/spinlock.h> -#include <linux/module.h> -#include <linux/mod_devicetable.h> -#include <linux/workqueue.h> -#include <linux/of_address.h> -#include <linux/of_irq.h> -#include <linux/platform_device.h> - -#include <asm/io.h> -#include <asm/pmi.h> - -struct pmi_data { - struct list_head handler; - spinlock_t handler_spinlock; - spinlock_t pmi_spinlock; - struct mutex msg_mutex; - pmi_message_t msg; - struct completion *completion; - struct platform_device *dev; - int irq; - u8 __iomem *pmi_reg; - struct work_struct work; -}; - -static struct pmi_data *data; - -static irqreturn_t pmi_irq_handler(int irq, void *dev_id) -{ - u8 type; - int rc; - - spin_lock(&data->pmi_spinlock); - - type = ioread8(data->pmi_reg + PMI_READ_TYPE); - pr_debug("pmi: got message of type %d\n", type); - - if (type & PMI_ACK && !data->completion) { - printk(KERN_WARNING "pmi: got unexpected ACK message.\n"); - rc = -EIO; - goto unlock; - } - - if (data->completion && !(type & PMI_ACK)) { - printk(KERN_WARNING "pmi: expected ACK, but got %d\n", type); - rc = -EIO; - goto unlock; - } - - data->msg.type = type; - data->msg.data0 = ioread8(data->pmi_reg + PMI_READ_DATA0); - data->msg.data1 = ioread8(data->pmi_reg + PMI_READ_DATA1); - data->msg.data2 = ioread8(data->pmi_reg + PMI_READ_DATA2); - rc = 0; -unlock: - spin_unlock(&data->pmi_spinlock); - - if (rc == -EIO) { - rc = IRQ_HANDLED; - goto out; - } - - if (data->msg.type & PMI_ACK) { - complete(data->completion); - rc = IRQ_HANDLED; - goto out; - } - - schedule_work(&data->work); - - rc = IRQ_HANDLED; -out: - return rc; -} - - -static const struct of_device_id pmi_match[] = { - { .type = "ibm,pmi", .name = "ibm,pmi" }, - { .type = "ibm,pmi" }, - {}, -}; - -MODULE_DEVICE_TABLE(of, pmi_match); - -static void pmi_notify_handlers(struct work_struct *work) -{ - struct pmi_handler *handler; - - spin_lock(&data->handler_spinlock); - list_for_each_entry(handler, &data->handler, node) { - pr_debug("pmi: notifying handler %p\n", handler); - if (handler->type == data->msg.type) - handler->handle_pmi_message(data->msg); - } - spin_unlock(&data->handler_spinlock); -} - -static int pmi_of_probe(struct platform_device *dev) -{ - struct device_node *np = dev->dev.of_node; - int rc; - - if (data) { - printk(KERN_ERR "pmi: driver has already been initialized.\n"); - rc = -EBUSY; - goto out; - } - - data = kzalloc(sizeof(struct pmi_data), GFP_KERNEL); - if (!data) { - printk(KERN_ERR "pmi: could not allocate memory.\n"); - rc = -ENOMEM; - goto out; - } - - data->pmi_reg = of_iomap(np, 0); - if (!data->pmi_reg) { - printk(KERN_ERR "pmi: invalid register address.\n"); - rc = -EFAULT; - goto error_cleanup_data; - } - - INIT_LIST_HEAD(&data->handler); - - mutex_init(&data->msg_mutex); - spin_lock_init(&data->pmi_spinlock); - spin_lock_init(&data->handler_spinlock); - - INIT_WORK(&data->work, pmi_notify_handlers); - - data->dev = dev; - - data->irq = irq_of_parse_and_map(np, 0); - if (!data->irq) { - printk(KERN_ERR "pmi: invalid interrupt.\n"); - rc = -EFAULT; - goto error_cleanup_iomap; - } - - rc = request_irq(data->irq, pmi_irq_handler, 0, "pmi", NULL); - if (rc) { - printk(KERN_ERR "pmi: can't request IRQ %d: returned %d\n", - data->irq, rc); - goto error_cleanup_iomap; - } - - printk(KERN_INFO "pmi: found pmi device at addr %p.\n", data->pmi_reg); - - goto out; - -error_cleanup_iomap: - iounmap(data->pmi_reg); - -error_cleanup_data: - kfree(data); - -out: - return rc; -} - -static void pmi_of_remove(struct platform_device *dev) -{ - struct pmi_handler *handler, *tmp; - - free_irq(data->irq, NULL); - iounmap(data->pmi_reg); - - spin_lock(&data->handler_spinlock); - - list_for_each_entry_safe(handler, tmp, &data->handler, node) - list_del(&handler->node); - - spin_unlock(&data->handler_spinlock); - - kfree(data); - data = NULL; -} - -static struct platform_driver pmi_of_platform_driver = { - .probe = pmi_of_probe, - .remove = pmi_of_remove, - .driver = { - .name = "pmi", - .of_match_table = pmi_match, - }, -}; -module_platform_driver(pmi_of_platform_driver); - -int pmi_send_message(pmi_message_t msg) -{ - unsigned long flags; - DECLARE_COMPLETION_ONSTACK(completion); - - if (!data) - return -ENODEV; - - mutex_lock(&data->msg_mutex); - - data->msg = msg; - pr_debug("pmi_send_message: msg is %08x\n", *(u32*)&msg); - - data->completion = &completion; - - spin_lock_irqsave(&data->pmi_spinlock, flags); - iowrite8(msg.data0, data->pmi_reg + PMI_WRITE_DATA0); - iowrite8(msg.data1, data->pmi_reg + PMI_WRITE_DATA1); - iowrite8(msg.data2, data->pmi_reg + PMI_WRITE_DATA2); - iowrite8(msg.type, data->pmi_reg + PMI_WRITE_TYPE); - spin_unlock_irqrestore(&data->pmi_spinlock, flags); - - pr_debug("pmi_send_message: wait for completion\n"); - - wait_for_completion_interruptible_timeout(data->completion, - PMI_TIMEOUT); - - data->completion = NULL; - - mutex_unlock(&data->msg_mutex); - - return 0; -} -EXPORT_SYMBOL_GPL(pmi_send_message); - -int pmi_register_handler(struct pmi_handler *handler) -{ - if (!data) - return -ENODEV; - - spin_lock(&data->handler_spinlock); - list_add_tail(&handler->node, &data->handler); - spin_unlock(&data->handler_spinlock); - - return 0; -} -EXPORT_SYMBOL_GPL(pmi_register_handler); - -void pmi_unregister_handler(struct pmi_handler *handler) -{ - if (!data) - return; - - pr_debug("pmi: unregistering handler %p\n", handler); - - spin_lock(&data->handler_spinlock); - list_del(&handler->node); - spin_unlock(&data->handler_spinlock); -} -EXPORT_SYMBOL_GPL(pmi_unregister_handler); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Christian Krafft <krafft@de.ibm.com>"); -MODULE_DESCRIPTION("IBM Platform Management Interrupt driver"); diff --git a/arch/powerpc/sysdev/tsi108_pci.c b/arch/powerpc/sysdev/tsi108_pci.c index 0e42f7bad7db..07d0f6a83879 100644 --- a/arch/powerpc/sysdev/tsi108_pci.c +++ b/arch/powerpc/sysdev/tsi108_pci.c @@ -404,8 +404,8 @@ void __init tsi108_pci_int_init(struct device_node *node) { DBG("Tsi108_pci_int_init: initializing PCI interrupts\n"); - pci_irq_host = irq_domain_add_legacy(node, NR_IRQS_LEGACY, 0, 0, - &pci_irq_domain_ops, NULL); + pci_irq_host = irq_domain_create_legacy(of_fwnode_handle(node), NR_IRQS_LEGACY, 0, 0, + &pci_irq_domain_ops, NULL); if (pci_irq_host == NULL) { printk(KERN_ERR "pci_irq_host: failed to allocate irq domain!\n"); return; diff --git a/arch/powerpc/sysdev/xics/icp-native.c b/arch/powerpc/sysdev/xics/icp-native.c index 700b67476a7d..4e89158a577c 100644 --- a/arch/powerpc/sysdev/xics/icp-native.c +++ b/arch/powerpc/sysdev/xics/icp-native.c @@ -145,27 +145,6 @@ static void icp_native_cause_ipi(int cpu) icp_native_set_qirr(cpu, IPI_PRIORITY); } -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE -void icp_native_cause_ipi_rm(int cpu) -{ - /* - * Currently not used to send IPIs to another CPU - * on the same core. Only caller is KVM real mode. - * Need the physical address of the XICS to be - * previously saved in kvm_hstate in the paca. - */ - void __iomem *xics_phys; - - /* - * Just like the cause_ipi functions, it is required to - * include a full barrier before causing the IPI. - */ - xics_phys = paca_ptrs[cpu]->kvm_hstate.xics_phys; - mb(); - __raw_rm_writeb(IPI_PRIORITY, xics_phys + XICS_MFRR); -} -#endif - /* * Called when an interrupt is received on an off-line CPU to * clear the interrupt, so that the CPU can go back to nap mode. diff --git a/arch/powerpc/sysdev/xics/xics-common.c b/arch/powerpc/sysdev/xics/xics-common.c index d3a4156e8788..c3fa539a9898 100644 --- a/arch/powerpc/sysdev/xics/xics-common.c +++ b/arch/powerpc/sysdev/xics/xics-common.c @@ -472,7 +472,7 @@ static int __init xics_allocate_domain(void) return -ENOMEM; } - irq_set_default_host(xics_host); + irq_set_default_domain(xics_host); return 0; } diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index a6c388bdf5d0..f10592405024 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -1464,10 +1464,10 @@ static const struct irq_domain_ops xive_irq_domain_ops = { static void __init xive_init_host(struct device_node *np) { - xive_irq_domain = irq_domain_add_tree(np, &xive_irq_domain_ops, NULL); + xive_irq_domain = irq_domain_create_tree(of_fwnode_handle(np), &xive_irq_domain_ops, NULL); if (WARN_ON(xive_irq_domain == NULL)) return; - irq_set_default_host(xive_irq_domain); + irq_set_default_domain(xive_irq_domain); } static void xive_cleanup_cpu_queues(unsigned int cpu, struct xive_cpu *xc) diff --git a/arch/powerpc/xmon/Makefile b/arch/powerpc/xmon/Makefile index d778011060a8..d74b147126b7 100644 --- a/arch/powerpc/xmon/Makefile +++ b/arch/powerpc/xmon/Makefile @@ -16,7 +16,4 @@ ccflags-$(CONFIG_CC_IS_CLANG) += -Wframe-larger-than=4096 obj-y += xmon.o nonstdio.o spr_access.o xmon_bpts.o -ifdef CONFIG_XMON_DISASSEMBLY -obj-y += ppc-dis.o ppc-opc.o -obj-$(CONFIG_SPU_BASE) += spu-dis.o spu-opc.o -endif +obj-$(CONFIG_XMON_DISASSEMBLY) += ppc-dis.o ppc-opc.o diff --git a/arch/powerpc/xmon/spu-dis.c b/arch/powerpc/xmon/spu-dis.c deleted file mode 100644 index 4b0a4e640f08..000000000000 --- a/arch/powerpc/xmon/spu-dis.c +++ /dev/null @@ -1,237 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* Disassemble SPU instructions - - Copyright 2006 Free Software Foundation, Inc. - - This file is part of GDB, GAS, and the GNU binutils. - - */ - -#include <linux/string.h> -#include "nonstdio.h" -#include "ansidecl.h" -#include "spu.h" -#include "dis-asm.h" - -/* This file provides a disassembler function which uses - the disassembler interface defined in dis-asm.h. */ - -extern const struct spu_opcode spu_opcodes[]; -extern const int spu_num_opcodes; - -#define SPU_DISASM_TBL_SIZE (1 << 11) -static const struct spu_opcode *spu_disassemble_table[SPU_DISASM_TBL_SIZE]; - -static void -init_spu_disassemble (void) -{ - int i; - - /* If two instructions have the same opcode then we prefer the first - * one. In most cases it is just an alternate mnemonic. */ - for (i = 0; i < spu_num_opcodes; i++) - { - int o = spu_opcodes[i].opcode; - if (o >= SPU_DISASM_TBL_SIZE) - continue; /* abort (); */ - if (spu_disassemble_table[o] == 0) - spu_disassemble_table[o] = &spu_opcodes[i]; - } -} - -/* Determine the instruction from the 10 least significant bits. */ -static const struct spu_opcode * -get_index_for_opcode (unsigned int insn) -{ - const struct spu_opcode *index; - unsigned int opcode = insn >> (32-11); - - /* Init the table. This assumes that element 0/opcode 0 (currently - * NOP) is always used */ - if (spu_disassemble_table[0] == 0) - init_spu_disassemble (); - - if ((index = spu_disassemble_table[opcode & 0x780]) != 0 - && index->insn_type == RRR) - return index; - - if ((index = spu_disassemble_table[opcode & 0x7f0]) != 0 - && (index->insn_type == RI18 || index->insn_type == LBT)) - return index; - - if ((index = spu_disassemble_table[opcode & 0x7f8]) != 0 - && index->insn_type == RI10) - return index; - - if ((index = spu_disassemble_table[opcode & 0x7fc]) != 0 - && (index->insn_type == RI16)) - return index; - - if ((index = spu_disassemble_table[opcode & 0x7fe]) != 0 - && (index->insn_type == RI8)) - return index; - - if ((index = spu_disassemble_table[opcode & 0x7ff]) != 0) - return index; - - return NULL; -} - -/* Print a Spu instruction. */ - -int -print_insn_spu (unsigned long insn, unsigned long memaddr) -{ - int value; - int hex_value; - const struct spu_opcode *index; - enum spu_insns tag; - - index = get_index_for_opcode (insn); - - if (index == 0) - { - printf(".long 0x%lx", insn); - } - else - { - int i; - int paren = 0; - tag = (enum spu_insns)(index - spu_opcodes); - printf("%s", index->mnemonic); - if (tag == M_BI || tag == M_BISL || tag == M_IRET || tag == M_BISLED - || tag == M_BIHNZ || tag == M_BIHZ || tag == M_BINZ || tag == M_BIZ - || tag == M_SYNC || tag == M_HBR) - { - int fb = (insn >> (32-18)) & 0x7f; - if (fb & 0x40) - printf(tag == M_SYNC ? "c" : "p"); - if (fb & 0x20) - printf("d"); - if (fb & 0x10) - printf("e"); - } - if (index->arg[0] != 0) - printf("\t"); - hex_value = 0; - for (i = 1; i <= index->arg[0]; i++) - { - int arg = index->arg[i]; - if (arg != A_P && !paren && i > 1) - printf(","); - - switch (arg) - { - case A_T: - printf("$%lu", - DECODE_INSN_RT (insn)); - break; - case A_A: - printf("$%lu", - DECODE_INSN_RA (insn)); - break; - case A_B: - printf("$%lu", - DECODE_INSN_RB (insn)); - break; - case A_C: - printf("$%lu", - DECODE_INSN_RC (insn)); - break; - case A_S: - printf("$sp%lu", - DECODE_INSN_RA (insn)); - break; - case A_H: - printf("$ch%lu", - DECODE_INSN_RA (insn)); - break; - case A_P: - paren++; - printf("("); - break; - case A_U7A: - printf("%lu", - 173 - DECODE_INSN_U8 (insn)); - break; - case A_U7B: - printf("%lu", - 155 - DECODE_INSN_U8 (insn)); - break; - case A_S3: - case A_S6: - case A_S7: - case A_S7N: - case A_U3: - case A_U5: - case A_U6: - case A_U7: - hex_value = DECODE_INSN_I7 (insn); - printf("%d", hex_value); - break; - case A_S11: - print_address(memaddr + DECODE_INSN_I9a (insn) * 4); - break; - case A_S11I: - print_address(memaddr + DECODE_INSN_I9b (insn) * 4); - break; - case A_S10: - case A_S10B: - hex_value = DECODE_INSN_I10 (insn); - printf("%d", hex_value); - break; - case A_S14: - hex_value = DECODE_INSN_I10 (insn) * 16; - printf("%d", hex_value); - break; - case A_S16: - hex_value = DECODE_INSN_I16 (insn); - printf("%d", hex_value); - break; - case A_X16: - hex_value = DECODE_INSN_U16 (insn); - printf("%u", hex_value); - break; - case A_R18: - value = DECODE_INSN_I16 (insn) * 4; - if (value == 0) - printf("%d", value); - else - { - hex_value = memaddr + value; - print_address(hex_value & 0x3ffff); - } - break; - case A_S18: - value = DECODE_INSN_U16 (insn) * 4; - if (value == 0) - printf("%d", value); - else - print_address(value); - break; - case A_U18: - value = DECODE_INSN_U18 (insn); - if (value == 0 || 1) - { - hex_value = value; - printf("%u", value); - } - else - print_address(value); - break; - case A_U14: - hex_value = DECODE_INSN_U14 (insn); - printf("%u", hex_value); - break; - } - if (arg != A_P && paren) - { - printf(")"); - paren--; - } - } - if (hex_value > 16) - printf("\t# %x", hex_value); - } - return 4; -} diff --git a/arch/powerpc/xmon/spu-insns.h b/arch/powerpc/xmon/spu-insns.h deleted file mode 100644 index 7e1126a19909..000000000000 --- a/arch/powerpc/xmon/spu-insns.h +++ /dev/null @@ -1,399 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* SPU ELF support for BFD. - - Copyright 2006 Free Software Foundation, Inc. - - This file is part of BFD, the Binary File Descriptor library. - - */ - -/* SPU Opcode Table - --=-=-= FORMAT =-=-=- - - +----+-------+-------+-------+-------+ +------------+-------+-------+-------+ -RRR | op | RC | RB | RA | RT | RI7 | op | I7 | RA | RT | - +----+-------+-------+-------+-------+ +------------+-------+-------+-------+ - 0 3 1 1 2 3 0 1 1 2 3 - 0 7 4 1 0 7 4 1 - - +-----------+--------+-------+-------+ +---------+----------+-------+-------+ -RI8 | op | I8 | RA | RT | RI10 | op | I10 | RA | RT | - +-----------+--------+-------+-------+ +---------+----------+-------+-------+ - 0 9 1 2 3 0 7 1 2 3 - 7 4 1 7 4 1 - - +----------+-----------------+-------+ +--------+-------------------+-------+ -RI16 | op | I16 | RT | RI18 | op | I18 | RT | - +----------+-----------------+-------+ +--------+-------------------+-------+ - 0 8 2 3 0 6 2 3 - 4 1 4 1 - - +------------+-------+-------+-------+ +-------+--+-----------------+-------+ -RR | op | RB | RA | RT | LBT | op |RO| I16 | RO | - +------------+-------+-------+-------+ +-------+--+-----------------+-------+ - 0 1 1 2 3 0 6 8 2 3 - 0 7 4 1 4 1 - - +------------+----+--+-------+-------+ - LBTI | op | // |RO| RA | RO | - +------------+----+--+-------+-------+ - 0 1 1 1 2 3 - 0 5 7 4 1 - --=-=-= OPCODE =-=-=- - -OPCODE field specifies the most significant 11bit of the instruction. Some formats don't have 11bits for opcode field, and in this -case, bit field other than op are defined as 0s. For example, opcode of fma instruction which is RRR format is defined as 0x700, -since 0x700 -> 11'b11100000000, this means opcode is 4'b1110, and other 7bits are defined as 7'b0000000. - --=-=-= ASM_FORMAT =-=-=- - -RRR category RI7 category - ASM_RRR mnemonic RC, RA, RB, RT ASM_RI4 mnemonic RT, RA, I4 - ASM_RI7 mnemonic RT, RA, I7 - -RI8 category RI10 category - ASM_RUI8 mnemonic RT, RA, UI8 ASM_AI10 mnemonic RA, I10 - ASM_RI10 mnemonic RT, RA, R10 - ASM_RI10IDX mnemonic RT, I10(RA) - -RI16 category RI18 category - ASM_I16W mnemonic I16W ASM_RI18 mnemonic RT, I18 - ASM_RI16 mnemonic RT, I16 - ASM_RI16W mnemonic RT, I16W - -RR category LBT category - ASM_MFSPR mnemonic RT, SA ASM_LBT mnemonic brinst, brtarg - ASM_MTSPR mnemonic SA, RT - ASM_NOOP mnemonic LBTI category - ASM_RA mnemonic RA ASM_LBTI mnemonic brinst, RA - ASM_RAB mnemonic RA, RB - ASM_RDCH mnemonic RT, CA - ASM_RR mnemonic RT, RA, RB - ASM_RT mnemonic RT - ASM_RTA mnemonic RT, RA - ASM_WRCH mnemonic CA, RT - -Note that RRR instructions have the names for RC and RT reversed from -what's in the ISA, in order to put RT in the same position it appears -for other formats. - --=-=-= DEPENDENCY =-=-=- - -DEPENDENCY filed consists of 5 digits. This represents which register is used as source and which register is used as target. -The first(most significant) digit is always 0. Then it is followd by RC, RB, RA and RT digits. -If the digit is 0, this means the corresponding register is not used in the instruction. -If the digit is 1, this means the corresponding register is used as a source in the instruction. -If the digit is 2, this means the corresponding register is used as a target in the instruction. -If the digit is 3, this means the corresponding register is used as both source and target in the instruction. -For example, fms instruction has 00113 as the DEPENDENCY field. This means RC is not used in this operation, RB and RA are -used as sources and RT is the target. - --=-=-= PIPE =-=-=- - -This field shows which execution pipe is used for the instruction - -pipe0 execution pipelines: - FP6 SP floating pipeline - FP7 integer operations executed in SP floating pipeline - FPD DP floating pipeline - FX2 FXU pipeline - FX3 Rotate/Shift pipeline - FXB Byte pipeline - NOP No pipeline - -pipe1 execution pipelines: - BR Branch pipeline - LNOP No pipeline - LS Load/Store pipeline - SHUF Shuffle pipeline - SPR SPR/CH pipeline - -*/ - -#define _A0() {0} -#define _A1(a) {1,a} -#define _A2(a,b) {2,a,b} -#define _A3(a,b,c) {3,a,b,c} -#define _A4(a,b,c,d) {4,a,b,c,d} - -/* TAG FORMAT OPCODE MNEMONIC ASM_FORMAT DEPENDENCY PIPE COMMENT */ -/* 0[RC][RB][RA][RT] */ -/* 1:src, 2:target */ - -APUOP(M_BR, RI16, 0x190, "br", _A1(A_R18), 00000, BR) /* BRel IP<-IP+I16 */ -APUOP(M_BRSL, RI16, 0x198, "brsl", _A2(A_T,A_R18), 00002, BR) /* BRelSetLink RT,IP<-IP,IP+I16 */ -APUOP(M_BRA, RI16, 0x180, "bra", _A1(A_S18), 00000, BR) /* BRAbs IP<-I16 */ -APUOP(M_BRASL, RI16, 0x188, "brasl", _A2(A_T,A_S18), 00002, BR) /* BRAbsSetLink RT,IP<-IP,I16 */ -APUOP(M_FSMBI, RI16, 0x194, "fsmbi", _A2(A_T,A_X16), 00002, SHUF) /* FormSelMask%I RT<-fsm(I16) */ -APUOP(M_LQA, RI16, 0x184, "lqa", _A2(A_T,A_S18), 00002, LS) /* LoadQAbs RT<-M[I16] */ -APUOP(M_LQR, RI16, 0x19C, "lqr", _A2(A_T,A_R18), 00002, LS) /* LoadQRel RT<-M[IP+I16] */ -APUOP(M_STOP, RR, 0x000, "stop", _A0(), 00000, BR) /* STOP stop */ -APUOP(M_STOP2, RR, 0x000, "stop", _A1(A_U14), 00000, BR) /* STOP stop */ -APUOP(M_STOPD, RR, 0x140, "stopd", _A3(A_T,A_A,A_B), 00111, BR) /* STOPD stop (with register dependencies) */ -APUOP(M_LNOP, RR, 0x001, "lnop", _A0(), 00000, LNOP) /* LNOP no_operation */ -APUOP(M_SYNC, RR, 0x002, "sync", _A0(), 00000, BR) /* SYNC flush_pipe */ -APUOP(M_DSYNC, RR, 0x003, "dsync", _A0(), 00000, BR) /* DSYNC flush_store_queue */ -APUOP(M_MFSPR, RR, 0x00c, "mfspr", _A2(A_T,A_S), 00002, SPR) /* MFSPR RT<-SA */ -APUOP(M_RDCH, RR, 0x00d, "rdch", _A2(A_T,A_H), 00002, SPR) /* ReaDCHannel RT<-CA:data */ -APUOP(M_RCHCNT, RR, 0x00f, "rchcnt", _A2(A_T,A_H), 00002, SPR) /* ReaDCHanCouNT RT<-CA:count */ -APUOP(M_HBRA, LBT, 0x080, "hbra", _A2(A_S11,A_S18), 00000, LS) /* HBRA BTB[B9]<-M[I16] */ -APUOP(M_HBRR, LBT, 0x090, "hbrr", _A2(A_S11,A_R18), 00000, LS) /* HBRR BTB[B9]<-M[IP+I16] */ -APUOP(M_BRZ, RI16, 0x100, "brz", _A2(A_T,A_R18), 00001, BR) /* BRZ IP<-IP+I16_if(RT) */ -APUOP(M_BRNZ, RI16, 0x108, "brnz", _A2(A_T,A_R18), 00001, BR) /* BRNZ IP<-IP+I16_if(RT) */ -APUOP(M_BRHZ, RI16, 0x110, "brhz", _A2(A_T,A_R18), 00001, BR) /* BRHZ IP<-IP+I16_if(RT) */ -APUOP(M_BRHNZ, RI16, 0x118, "brhnz", _A2(A_T,A_R18), 00001, BR) /* BRHNZ IP<-IP+I16_if(RT) */ -APUOP(M_STQA, RI16, 0x104, "stqa", _A2(A_T,A_S18), 00001, LS) /* SToreQAbs M[I16]<-RT */ -APUOP(M_STQR, RI16, 0x11C, "stqr", _A2(A_T,A_R18), 00001, LS) /* SToreQRel M[IP+I16]<-RT */ -APUOP(M_MTSPR, RR, 0x10c, "mtspr", _A2(A_S,A_T), 00001, SPR) /* MTSPR SA<-RT */ -APUOP(M_WRCH, RR, 0x10d, "wrch", _A2(A_H,A_T), 00001, SPR) /* ChanWRite CA<-RT */ -APUOP(M_LQD, RI10, 0x1a0, "lqd", _A4(A_T,A_S14,A_P,A_A), 00012, LS) /* LoadQDisp RT<-M[Ra+I10] */ -APUOP(M_BI, RR, 0x1a8, "bi", _A1(A_A), 00010, BR) /* BI IP<-RA */ -APUOP(M_BISL, RR, 0x1a9, "bisl", _A2(A_T,A_A), 00012, BR) /* BISL RT,IP<-IP,RA */ -APUOP(M_IRET, RR, 0x1aa, "iret", _A1(A_A), 00010, BR) /* IRET IP<-SRR0 */ -APUOP(M_IRET2, RR, 0x1aa, "iret", _A0(), 00010, BR) /* IRET IP<-SRR0 */ -APUOP(M_BISLED, RR, 0x1ab, "bisled", _A2(A_T,A_A), 00012, BR) /* BISLED RT,IP<-IP,RA_if(ext) */ -APUOP(M_HBR, LBTI, 0x1ac, "hbr", _A2(A_S11I,A_A), 00010, LS) /* HBR BTB[B9]<-M[Ra] */ -APUOP(M_FREST, RR, 0x1b8, "frest", _A2(A_T,A_A), 00012, SHUF) /* FREST RT<-recip(RA) */ -APUOP(M_FRSQEST, RR, 0x1b9, "frsqest", _A2(A_T,A_A), 00012, SHUF) /* FRSQEST RT<-rsqrt(RA) */ -APUOP(M_FSM, RR, 0x1b4, "fsm", _A2(A_T,A_A), 00012, SHUF) /* FormSelMask% RT<-expand(Ra) */ -APUOP(M_FSMH, RR, 0x1b5, "fsmh", _A2(A_T,A_A), 00012, SHUF) /* FormSelMask% RT<-expand(Ra) */ -APUOP(M_FSMB, RR, 0x1b6, "fsmb", _A2(A_T,A_A), 00012, SHUF) /* FormSelMask% RT<-expand(Ra) */ -APUOP(M_GB, RR, 0x1b0, "gb", _A2(A_T,A_A), 00012, SHUF) /* GatherBits% RT<-gather(RA) */ -APUOP(M_GBH, RR, 0x1b1, "gbh", _A2(A_T,A_A), 00012, SHUF) /* GatherBits% RT<-gather(RA) */ -APUOP(M_GBB, RR, 0x1b2, "gbb", _A2(A_T,A_A), 00012, SHUF) /* GatherBits% RT<-gather(RA) */ -APUOP(M_CBD, RI7, 0x1f4, "cbd", _A4(A_T,A_U7,A_P,A_A), 00012, SHUF) /* genCtl%%insD RT<-sta(Ra+I4,siz) */ -APUOP(M_CHD, RI7, 0x1f5, "chd", _A4(A_T,A_U7,A_P,A_A), 00012, SHUF) /* genCtl%%insD RT<-sta(Ra+I4,siz) */ -APUOP(M_CWD, RI7, 0x1f6, "cwd", _A4(A_T,A_U7,A_P,A_A), 00012, SHUF) /* genCtl%%insD RT<-sta(Ra+I4,siz) */ -APUOP(M_CDD, RI7, 0x1f7, "cdd", _A4(A_T,A_U7,A_P,A_A), 00012, SHUF) /* genCtl%%insD RT<-sta(Ra+I4,siz) */ -APUOP(M_ROTQBII, RI7, 0x1f8, "rotqbii", _A3(A_T,A_A,A_U3), 00012, SHUF) /* ROTQBII RT<-RA<<<I7 */ -APUOP(M_ROTQBYI, RI7, 0x1fc, "rotqbyi", _A3(A_T,A_A,A_S7N), 00012, SHUF) /* ROTQBYI RT<-RA<<<(I7*8) */ -APUOP(M_ROTQMBII, RI7, 0x1f9, "rotqmbii", _A3(A_T,A_A,A_S3), 00012, SHUF) /* ROTQMBII RT<-RA<<I7 */ -APUOP(M_ROTQMBYI, RI7, 0x1fd, "rotqmbyi", _A3(A_T,A_A,A_S6), 00012, SHUF) /* ROTQMBYI RT<-RA<<I7 */ -APUOP(M_SHLQBII, RI7, 0x1fb, "shlqbii", _A3(A_T,A_A,A_U3), 00012, SHUF) /* SHLQBII RT<-RA<<I7 */ -APUOP(M_SHLQBYI, RI7, 0x1ff, "shlqbyi", _A3(A_T,A_A,A_U5), 00012, SHUF) /* SHLQBYI RT<-RA<<I7 */ -APUOP(M_STQD, RI10, 0x120, "stqd", _A4(A_T,A_S14,A_P,A_A), 00011, LS) /* SToreQDisp M[Ra+I10]<-RT */ -APUOP(M_BIHNZ, RR, 0x12b, "bihnz", _A2(A_T,A_A), 00011, BR) /* BIHNZ IP<-RA_if(RT) */ -APUOP(M_BIHZ, RR, 0x12a, "bihz", _A2(A_T,A_A), 00011, BR) /* BIHZ IP<-RA_if(RT) */ -APUOP(M_BINZ, RR, 0x129, "binz", _A2(A_T,A_A), 00011, BR) /* BINZ IP<-RA_if(RT) */ -APUOP(M_BIZ, RR, 0x128, "biz", _A2(A_T,A_A), 00011, BR) /* BIZ IP<-RA_if(RT) */ -APUOP(M_CBX, RR, 0x1d4, "cbx", _A3(A_T,A_A,A_B), 00112, SHUF) /* genCtl%%insX RT<-sta(Ra+Rb,siz) */ -APUOP(M_CHX, RR, 0x1d5, "chx", _A3(A_T,A_A,A_B), 00112, SHUF) /* genCtl%%insX RT<-sta(Ra+Rb,siz) */ -APUOP(M_CWX, RR, 0x1d6, "cwx", _A3(A_T,A_A,A_B), 00112, SHUF) /* genCtl%%insX RT<-sta(Ra+Rb,siz) */ -APUOP(M_CDX, RR, 0x1d7, "cdx", _A3(A_T,A_A,A_B), 00112, SHUF) /* genCtl%%insX RT<-sta(Ra+Rb,siz) */ -APUOP(M_LQX, RR, 0x1c4, "lqx", _A3(A_T,A_A,A_B), 00112, LS) /* LoadQindeX RT<-M[Ra+Rb] */ -APUOP(M_ROTQBI, RR, 0x1d8, "rotqbi", _A3(A_T,A_A,A_B), 00112, SHUF) /* ROTQBI RT<-RA<<<Rb */ -APUOP(M_ROTQMBI, RR, 0x1d9, "rotqmbi", _A3(A_T,A_A,A_B), 00112, SHUF) /* ROTQMBI RT<-RA<<Rb */ -APUOP(M_SHLQBI, RR, 0x1db, "shlqbi", _A3(A_T,A_A,A_B), 00112, SHUF) /* SHLQBI RT<-RA<<Rb */ -APUOP(M_ROTQBY, RR, 0x1dc, "rotqby", _A3(A_T,A_A,A_B), 00112, SHUF) /* ROTQBY RT<-RA<<<(Rb*8) */ -APUOP(M_ROTQMBY, RR, 0x1dd, "rotqmby", _A3(A_T,A_A,A_B), 00112, SHUF) /* ROTQMBY RT<-RA<<Rb */ -APUOP(M_SHLQBY, RR, 0x1df, "shlqby", _A3(A_T,A_A,A_B), 00112, SHUF) /* SHLQBY RT<-RA<<Rb */ -APUOP(M_ROTQBYBI, RR, 0x1cc, "rotqbybi", _A3(A_T,A_A,A_B), 00112, SHUF) /* ROTQBYBI RT<-RA<<Rb */ -APUOP(M_ROTQMBYBI, RR, 0x1cd, "rotqmbybi", _A3(A_T,A_A,A_B), 00112, SHUF) /* ROTQMBYBI RT<-RA<<Rb */ -APUOP(M_SHLQBYBI, RR, 0x1cf, "shlqbybi", _A3(A_T,A_A,A_B), 00112, SHUF) /* SHLQBYBI RT<-RA<<Rb */ -APUOP(M_STQX, RR, 0x144, "stqx", _A3(A_T,A_A,A_B), 00111, LS) /* SToreQindeX M[Ra+Rb]<-RT */ -APUOP(M_SHUFB, RRR, 0x580, "shufb", _A4(A_C,A_A,A_B,A_T), 02111, SHUF) /* SHUFfleBytes RC<-f(RA,RB,RT) */ -APUOP(M_IL, RI16, 0x204, "il", _A2(A_T,A_S16), 00002, FX2) /* ImmLoad RT<-sxt(I16) */ -APUOP(M_ILH, RI16, 0x20c, "ilh", _A2(A_T,A_X16), 00002, FX2) /* ImmLoadH RT<-I16 */ -APUOP(M_ILHU, RI16, 0x208, "ilhu", _A2(A_T,A_X16), 00002, FX2) /* ImmLoadHUpper RT<-I16<<16 */ -APUOP(M_ILA, RI18, 0x210, "ila", _A2(A_T,A_U18), 00002, FX2) /* ImmLoadAddr RT<-zxt(I18) */ -APUOP(M_NOP, RR, 0x201, "nop", _A1(A_T), 00000, NOP) /* XNOP no_operation */ -APUOP(M_NOP2, RR, 0x201, "nop", _A0(), 00000, NOP) /* XNOP no_operation */ -APUOP(M_IOHL, RI16, 0x304, "iohl", _A2(A_T,A_X16), 00003, FX2) /* AddImmeXt RT<-RT+sxt(I16) */ -APUOP(M_ANDBI, RI10, 0x0b0, "andbi", _A3(A_T,A_A,A_S10B), 00012, FX2) /* AND%I RT<-RA&I10 */ -APUOP(M_ANDHI, RI10, 0x0a8, "andhi", _A3(A_T,A_A,A_S10), 00012, FX2) /* AND%I RT<-RA&I10 */ -APUOP(M_ANDI, RI10, 0x0a0, "andi", _A3(A_T,A_A,A_S10), 00012, FX2) /* AND%I RT<-RA&I10 */ -APUOP(M_ORBI, RI10, 0x030, "orbi", _A3(A_T,A_A,A_S10B), 00012, FX2) /* OR%I RT<-RA|I10 */ -APUOP(M_ORHI, RI10, 0x028, "orhi", _A3(A_T,A_A,A_S10), 00012, FX2) /* OR%I RT<-RA|I10 */ -APUOP(M_ORI, RI10, 0x020, "ori", _A3(A_T,A_A,A_S10), 00012, FX2) /* OR%I RT<-RA|I10 */ -APUOP(M_ORX, RR, 0x1f0, "orx", _A2(A_T,A_A), 00012, BR) /* ORX RT<-RA.w0|RA.w1|RA.w2|RA.w3 */ -APUOP(M_XORBI, RI10, 0x230, "xorbi", _A3(A_T,A_A,A_S10B), 00012, FX2) /* XOR%I RT<-RA^I10 */ -APUOP(M_XORHI, RI10, 0x228, "xorhi", _A3(A_T,A_A,A_S10), 00012, FX2) /* XOR%I RT<-RA^I10 */ -APUOP(M_XORI, RI10, 0x220, "xori", _A3(A_T,A_A,A_S10), 00012, FX2) /* XOR%I RT<-RA^I10 */ -APUOP(M_AHI, RI10, 0x0e8, "ahi", _A3(A_T,A_A,A_S10), 00012, FX2) /* Add%Immed RT<-RA+I10 */ -APUOP(M_AI, RI10, 0x0e0, "ai", _A3(A_T,A_A,A_S10), 00012, FX2) /* Add%Immed RT<-RA+I10 */ -APUOP(M_SFHI, RI10, 0x068, "sfhi", _A3(A_T,A_A,A_S10), 00012, FX2) /* SubFrom%Imm RT<-I10-RA */ -APUOP(M_SFI, RI10, 0x060, "sfi", _A3(A_T,A_A,A_S10), 00012, FX2) /* SubFrom%Imm RT<-I10-RA */ -APUOP(M_CGTBI, RI10, 0x270, "cgtbi", _A3(A_T,A_A,A_S10B), 00012, FX2) /* CGT%I RT<-(RA>I10) */ -APUOP(M_CGTHI, RI10, 0x268, "cgthi", _A3(A_T,A_A,A_S10), 00012, FX2) /* CGT%I RT<-(RA>I10) */ -APUOP(M_CGTI, RI10, 0x260, "cgti", _A3(A_T,A_A,A_S10), 00012, FX2) /* CGT%I RT<-(RA>I10) */ -APUOP(M_CLGTBI, RI10, 0x2f0, "clgtbi", _A3(A_T,A_A,A_S10B), 00012, FX2) /* CLGT%I RT<-(RA>I10) */ -APUOP(M_CLGTHI, RI10, 0x2e8, "clgthi", _A3(A_T,A_A,A_S10), 00012, FX2) /* CLGT%I RT<-(RA>I10) */ -APUOP(M_CLGTI, RI10, 0x2e0, "clgti", _A3(A_T,A_A,A_S10), 00012, FX2) /* CLGT%I RT<-(RA>I10) */ -APUOP(M_CEQBI, RI10, 0x3f0, "ceqbi", _A3(A_T,A_A,A_S10B), 00012, FX2) /* CEQ%I RT<-(RA=I10) */ -APUOP(M_CEQHI, RI10, 0x3e8, "ceqhi", _A3(A_T,A_A,A_S10), 00012, FX2) /* CEQ%I RT<-(RA=I10) */ -APUOP(M_CEQI, RI10, 0x3e0, "ceqi", _A3(A_T,A_A,A_S10), 00012, FX2) /* CEQ%I RT<-(RA=I10) */ -APUOP(M_HGTI, RI10, 0x278, "hgti", _A3(A_T,A_A,A_S10), 00010, FX2) /* HaltGTI halt_if(RA>I10) */ -APUOP(M_HGTI2, RI10, 0x278, "hgti", _A2(A_A,A_S10), 00010, FX2) /* HaltGTI halt_if(RA>I10) */ -APUOP(M_HLGTI, RI10, 0x2f8, "hlgti", _A3(A_T,A_A,A_S10), 00010, FX2) /* HaltLGTI halt_if(RA>I10) */ -APUOP(M_HLGTI2, RI10, 0x2f8, "hlgti", _A2(A_A,A_S10), 00010, FX2) /* HaltLGTI halt_if(RA>I10) */ -APUOP(M_HEQI, RI10, 0x3f8, "heqi", _A3(A_T,A_A,A_S10), 00010, FX2) /* HaltEQImm halt_if(RA=I10) */ -APUOP(M_HEQI2, RI10, 0x3f8, "heqi", _A2(A_A,A_S10), 00010, FX2) /* HaltEQImm halt_if(RA=I10) */ -APUOP(M_MPYI, RI10, 0x3a0, "mpyi", _A3(A_T,A_A,A_S10), 00012, FP7) /* MPYI RT<-RA*I10 */ -APUOP(M_MPYUI, RI10, 0x3a8, "mpyui", _A3(A_T,A_A,A_S10), 00012, FP7) /* MPYUI RT<-RA*I10 */ -APUOP(M_CFLTS, RI8, 0x3b0, "cflts", _A3(A_T,A_A,A_U7A), 00012, FP7) /* CFLTS RT<-int(RA,I8) */ -APUOP(M_CFLTU, RI8, 0x3b2, "cfltu", _A3(A_T,A_A,A_U7A), 00012, FP7) /* CFLTU RT<-int(RA,I8) */ -APUOP(M_CSFLT, RI8, 0x3b4, "csflt", _A3(A_T,A_A,A_U7B), 00012, FP7) /* CSFLT RT<-flt(RA,I8) */ -APUOP(M_CUFLT, RI8, 0x3b6, "cuflt", _A3(A_T,A_A,A_U7B), 00012, FP7) /* CUFLT RT<-flt(RA,I8) */ -APUOP(M_FESD, RR, 0x3b8, "fesd", _A2(A_T,A_A), 00012, FPD) /* FESD RT<-double(RA) */ -APUOP(M_FRDS, RR, 0x3b9, "frds", _A2(A_T,A_A), 00012, FPD) /* FRDS RT<-single(RA) */ -APUOP(M_FSCRRD, RR, 0x398, "fscrrd", _A1(A_T), 00002, FPD) /* FSCRRD RT<-FP_status */ -APUOP(M_FSCRWR, RR, 0x3ba, "fscrwr", _A2(A_T,A_A), 00010, FP7) /* FSCRWR FP_status<-RA */ -APUOP(M_FSCRWR2, RR, 0x3ba, "fscrwr", _A1(A_A), 00010, FP7) /* FSCRWR FP_status<-RA */ -APUOP(M_CLZ, RR, 0x2a5, "clz", _A2(A_T,A_A), 00012, FX2) /* CLZ RT<-clz(RA) */ -APUOP(M_CNTB, RR, 0x2b4, "cntb", _A2(A_T,A_A), 00012, FXB) /* CNT RT<-pop(RA) */ -APUOP(M_XSBH, RR, 0x2b6, "xsbh", _A2(A_T,A_A), 00012, FX2) /* eXtSignBtoH RT<-sign_ext(RA) */ -APUOP(M_XSHW, RR, 0x2ae, "xshw", _A2(A_T,A_A), 00012, FX2) /* eXtSignHtoW RT<-sign_ext(RA) */ -APUOP(M_XSWD, RR, 0x2a6, "xswd", _A2(A_T,A_A), 00012, FX2) /* eXtSignWtoD RT<-sign_ext(RA) */ -APUOP(M_ROTI, RI7, 0x078, "roti", _A3(A_T,A_A,A_S7N), 00012, FX3) /* ROT%I RT<-RA<<<I7 */ -APUOP(M_ROTMI, RI7, 0x079, "rotmi", _A3(A_T,A_A,A_S7), 00012, FX3) /* ROT%MI RT<-RA<<I7 */ -APUOP(M_ROTMAI, RI7, 0x07a, "rotmai", _A3(A_T,A_A,A_S7), 00012, FX3) /* ROTMA%I RT<-RA<<I7 */ -APUOP(M_SHLI, RI7, 0x07b, "shli", _A3(A_T,A_A,A_U6), 00012, FX3) /* SHL%I RT<-RA<<I7 */ -APUOP(M_ROTHI, RI7, 0x07c, "rothi", _A3(A_T,A_A,A_S7N), 00012, FX3) /* ROT%I RT<-RA<<<I7 */ -APUOP(M_ROTHMI, RI7, 0x07d, "rothmi", _A3(A_T,A_A,A_S6), 00012, FX3) /* ROT%MI RT<-RA<<I7 */ -APUOP(M_ROTMAHI, RI7, 0x07e, "rotmahi", _A3(A_T,A_A,A_S6), 00012, FX3) /* ROTMA%I RT<-RA<<I7 */ -APUOP(M_SHLHI, RI7, 0x07f, "shlhi", _A3(A_T,A_A,A_U5), 00012, FX3) /* SHL%I RT<-RA<<I7 */ -APUOP(M_A, RR, 0x0c0, "a", _A3(A_T,A_A,A_B), 00112, FX2) /* Add% RT<-RA+RB */ -APUOP(M_AH, RR, 0x0c8, "ah", _A3(A_T,A_A,A_B), 00112, FX2) /* Add% RT<-RA+RB */ -APUOP(M_SF, RR, 0x040, "sf", _A3(A_T,A_A,A_B), 00112, FX2) /* SubFrom% RT<-RB-RA */ -APUOP(M_SFH, RR, 0x048, "sfh", _A3(A_T,A_A,A_B), 00112, FX2) /* SubFrom% RT<-RB-RA */ -APUOP(M_CGT, RR, 0x240, "cgt", _A3(A_T,A_A,A_B), 00112, FX2) /* CGT% RT<-(RA>RB) */ -APUOP(M_CGTB, RR, 0x250, "cgtb", _A3(A_T,A_A,A_B), 00112, FX2) /* CGT% RT<-(RA>RB) */ -APUOP(M_CGTH, RR, 0x248, "cgth", _A3(A_T,A_A,A_B), 00112, FX2) /* CGT% RT<-(RA>RB) */ -APUOP(M_CLGT, RR, 0x2c0, "clgt", _A3(A_T,A_A,A_B), 00112, FX2) /* CLGT% RT<-(RA>RB) */ -APUOP(M_CLGTB, RR, 0x2d0, "clgtb", _A3(A_T,A_A,A_B), 00112, FX2) /* CLGT% RT<-(RA>RB) */ -APUOP(M_CLGTH, RR, 0x2c8, "clgth", _A3(A_T,A_A,A_B), 00112, FX2) /* CLGT% RT<-(RA>RB) */ -APUOP(M_CEQ, RR, 0x3c0, "ceq", _A3(A_T,A_A,A_B), 00112, FX2) /* CEQ% RT<-(RA=RB) */ -APUOP(M_CEQB, RR, 0x3d0, "ceqb", _A3(A_T,A_A,A_B), 00112, FX2) /* CEQ% RT<-(RA=RB) */ -APUOP(M_CEQH, RR, 0x3c8, "ceqh", _A3(A_T,A_A,A_B), 00112, FX2) /* CEQ% RT<-(RA=RB) */ -APUOP(M_HGT, RR, 0x258, "hgt", _A3(A_T,A_A,A_B), 00110, FX2) /* HaltGT halt_if(RA>RB) */ -APUOP(M_HGT2, RR, 0x258, "hgt", _A2(A_A,A_B), 00110, FX2) /* HaltGT halt_if(RA>RB) */ -APUOP(M_HLGT, RR, 0x2d8, "hlgt", _A3(A_T,A_A,A_B), 00110, FX2) /* HaltLGT halt_if(RA>RB) */ -APUOP(M_HLGT2, RR, 0x2d8, "hlgt", _A2(A_A,A_B), 00110, FX2) /* HaltLGT halt_if(RA>RB) */ -APUOP(M_HEQ, RR, 0x3d8, "heq", _A3(A_T,A_A,A_B), 00110, FX2) /* HaltEQ halt_if(RA=RB) */ -APUOP(M_HEQ2, RR, 0x3d8, "heq", _A2(A_A,A_B), 00110, FX2) /* HaltEQ halt_if(RA=RB) */ -APUOP(M_FCEQ, RR, 0x3c2, "fceq", _A3(A_T,A_A,A_B), 00112, FX2) /* FCEQ RT<-(RA=RB) */ -APUOP(M_FCMEQ, RR, 0x3ca, "fcmeq", _A3(A_T,A_A,A_B), 00112, FX2) /* FCMEQ RT<-(|RA|=|RB|) */ -APUOP(M_FCGT, RR, 0x2c2, "fcgt", _A3(A_T,A_A,A_B), 00112, FX2) /* FCGT RT<-(RA<RB) */ -APUOP(M_FCMGT, RR, 0x2ca, "fcmgt", _A3(A_T,A_A,A_B), 00112, FX2) /* FCMGT RT<-(|RA|<|RB|) */ -APUOP(M_AND, RR, 0x0c1, "and", _A3(A_T,A_A,A_B), 00112, FX2) /* AND RT<-RA&RB */ -APUOP(M_NAND, RR, 0x0c9, "nand", _A3(A_T,A_A,A_B), 00112, FX2) /* NAND RT<-!(RA&RB) */ -APUOP(M_OR, RR, 0x041, "or", _A3(A_T,A_A,A_B), 00112, FX2) /* OR RT<-RA|RB */ -APUOP(M_NOR, RR, 0x049, "nor", _A3(A_T,A_A,A_B), 00112, FX2) /* NOR RT<-!(RA&RB) */ -APUOP(M_XOR, RR, 0x241, "xor", _A3(A_T,A_A,A_B), 00112, FX2) /* XOR RT<-RA^RB */ -APUOP(M_EQV, RR, 0x249, "eqv", _A3(A_T,A_A,A_B), 00112, FX2) /* EQuiValent RT<-!(RA^RB) */ -APUOP(M_ANDC, RR, 0x2c1, "andc", _A3(A_T,A_A,A_B), 00112, FX2) /* ANDComplement RT<-RA&!RB */ -APUOP(M_ORC, RR, 0x2c9, "orc", _A3(A_T,A_A,A_B), 00112, FX2) /* ORComplement RT<-RA|!RB */ -APUOP(M_ABSDB, RR, 0x053, "absdb", _A3(A_T,A_A,A_B), 00112, FXB) /* ABSoluteDiff RT<-|RA-RB| */ -APUOP(M_AVGB, RR, 0x0d3, "avgb", _A3(A_T,A_A,A_B), 00112, FXB) /* AVG% RT<-(RA+RB+1)/2 */ -APUOP(M_SUMB, RR, 0x253, "sumb", _A3(A_T,A_A,A_B), 00112, FXB) /* SUM% RT<-f(RA,RB) */ -APUOP(M_DFA, RR, 0x2cc, "dfa", _A3(A_T,A_A,A_B), 00112, FPD) /* DFAdd RT<-RA+RB */ -APUOP(M_DFM, RR, 0x2ce, "dfm", _A3(A_T,A_A,A_B), 00112, FPD) /* DFMul RT<-RA*RB */ -APUOP(M_DFS, RR, 0x2cd, "dfs", _A3(A_T,A_A,A_B), 00112, FPD) /* DFSub RT<-RA-RB */ -APUOP(M_FA, RR, 0x2c4, "fa", _A3(A_T,A_A,A_B), 00112, FP6) /* FAdd RT<-RA+RB */ -APUOP(M_FM, RR, 0x2c6, "fm", _A3(A_T,A_A,A_B), 00112, FP6) /* FMul RT<-RA*RB */ -APUOP(M_FS, RR, 0x2c5, "fs", _A3(A_T,A_A,A_B), 00112, FP6) /* FSub RT<-RA-RB */ -APUOP(M_MPY, RR, 0x3c4, "mpy", _A3(A_T,A_A,A_B), 00112, FP7) /* MPY RT<-RA*RB */ -APUOP(M_MPYH, RR, 0x3c5, "mpyh", _A3(A_T,A_A,A_B), 00112, FP7) /* MPYH RT<-(RAh*RB)<<16 */ -APUOP(M_MPYHH, RR, 0x3c6, "mpyhh", _A3(A_T,A_A,A_B), 00112, FP7) /* MPYHH RT<-RAh*RBh */ -APUOP(M_MPYHHU, RR, 0x3ce, "mpyhhu", _A3(A_T,A_A,A_B), 00112, FP7) /* MPYHHU RT<-RAh*RBh */ -APUOP(M_MPYS, RR, 0x3c7, "mpys", _A3(A_T,A_A,A_B), 00112, FP7) /* MPYS RT<-(RA*RB)>>16 */ -APUOP(M_MPYU, RR, 0x3cc, "mpyu", _A3(A_T,A_A,A_B), 00112, FP7) /* MPYU RT<-RA*RB */ -APUOP(M_FI, RR, 0x3d4, "fi", _A3(A_T,A_A,A_B), 00112, FP7) /* FInterpolate RT<-f(RA,RB) */ -APUOP(M_ROT, RR, 0x058, "rot", _A3(A_T,A_A,A_B), 00112, FX3) /* ROT% RT<-RA<<<RB */ -APUOP(M_ROTM, RR, 0x059, "rotm", _A3(A_T,A_A,A_B), 00112, FX3) /* ROT%M RT<-RA<<Rb */ -APUOP(M_ROTMA, RR, 0x05a, "rotma", _A3(A_T,A_A,A_B), 00112, FX3) /* ROTMA% RT<-RA<<Rb */ -APUOP(M_SHL, RR, 0x05b, "shl", _A3(A_T,A_A,A_B), 00112, FX3) /* SHL% RT<-RA<<Rb */ -APUOP(M_ROTH, RR, 0x05c, "roth", _A3(A_T,A_A,A_B), 00112, FX3) /* ROT% RT<-RA<<<RB */ -APUOP(M_ROTHM, RR, 0x05d, "rothm", _A3(A_T,A_A,A_B), 00112, FX3) /* ROT%M RT<-RA<<Rb */ -APUOP(M_ROTMAH, RR, 0x05e, "rotmah", _A3(A_T,A_A,A_B), 00112, FX3) /* ROTMA% RT<-RA<<Rb */ -APUOP(M_SHLH, RR, 0x05f, "shlh", _A3(A_T,A_A,A_B), 00112, FX3) /* SHL% RT<-RA<<Rb */ -APUOP(M_MPYHHA, RR, 0x346, "mpyhha", _A3(A_T,A_A,A_B), 00113, FP7) /* MPYHHA RT<-RAh*RBh+RT */ -APUOP(M_MPYHHAU, RR, 0x34e, "mpyhhau", _A3(A_T,A_A,A_B), 00113, FP7) /* MPYHHAU RT<-RAh*RBh+RT */ -APUOP(M_DFMA, RR, 0x35c, "dfma", _A3(A_T,A_A,A_B), 00113, FPD) /* DFMAdd RT<-RT+RA*RB */ -APUOP(M_DFMS, RR, 0x35d, "dfms", _A3(A_T,A_A,A_B), 00113, FPD) /* DFMSub RT<-RA*RB-RT */ -APUOP(M_DFNMS, RR, 0x35e, "dfnms", _A3(A_T,A_A,A_B), 00113, FPD) /* DFNMSub RT<-RT-RA*RB */ -APUOP(M_DFNMA, RR, 0x35f, "dfnma", _A3(A_T,A_A,A_B), 00113, FPD) /* DFNMAdd RT<-(-RT)-RA*RB */ -APUOP(M_FMA, RRR, 0x700, "fma", _A4(A_C,A_A,A_B,A_T), 02111, FP6) /* FMAdd RC<-RT+RA*RB */ -APUOP(M_FMS, RRR, 0x780, "fms", _A4(A_C,A_A,A_B,A_T), 02111, FP6) /* FMSub RC<-RA*RB-RT */ -APUOP(M_FNMS, RRR, 0x680, "fnms", _A4(A_C,A_A,A_B,A_T), 02111, FP6) /* FNMSub RC<-RT-RA*RB */ -APUOP(M_MPYA, RRR, 0x600, "mpya", _A4(A_C,A_A,A_B,A_T), 02111, FP7) /* MPYA RC<-RA*RB+RT */ -APUOP(M_SELB, RRR, 0x400, "selb", _A4(A_C,A_A,A_B,A_T), 02111, FX2) /* SELectBits RC<-RA&RT|RB&!RT */ -/* for system function call, this uses op-code of mtspr */ -APUOP(M_SYSCALL, RI7, 0x10c, "syscall", _A3(A_T,A_A,A_S7N), 00002, SPR) /* System Call */ -/* -pseudo instruction: -system call -value of I9 operation -0 halt -1 rt[0] = open(MEM[ra[0]], ra[1]) -2 rt[0] = close(ra[0]) -3 rt[0] = read(ra[0], MEM[ra[1]], ra[2]) -4 rt[0] = write(ra[0], MEM[ra[1]], ra[2]) -5 printf(MEM[ra[0]], ra[1], ra[2], ra[3]) -42 rt[0] = clock() -52 rt[0] = lseek(ra0, ra1, ra2) - -*/ - - -/* new multiprecision add/sub */ -APUOP(M_ADDX, RR, 0x340, "addx", _A3(A_T,A_A,A_B), 00113, FX2) /* Add_eXtended RT<-RA+RB+RT */ -APUOP(M_CG, RR, 0x0c2, "cg", _A3(A_T,A_A,A_B), 00112, FX2) /* CarryGenerate RT<-cout(RA+RB) */ -APUOP(M_CGX, RR, 0x342, "cgx", _A3(A_T,A_A,A_B), 00113, FX2) /* CarryGen_eXtd RT<-cout(RA+RB+RT) */ -APUOP(M_SFX, RR, 0x341, "sfx", _A3(A_T,A_A,A_B), 00113, FX2) /* Add_eXtended RT<-RA+RB+RT */ -APUOP(M_BG, RR, 0x042, "bg", _A3(A_T,A_A,A_B), 00112, FX2) /* CarryGenerate RT<-cout(RA+RB) */ -APUOP(M_BGX, RR, 0x343, "bgx", _A3(A_T,A_A,A_B), 00113, FX2) /* CarryGen_eXtd RT<-cout(RA+RB+RT) */ - -/* - -The following ops are a subset of above except with feature bits set. -Feature bits are bits 11-17 of the instruction: - - 11 - C & P feature bit - 12 - disable interrupts - 13 - enable interrupts - -*/ -APUOPFB(M_BID, RR, 0x1a8, 0x20, "bid", _A1(A_A), 00010, BR) /* BI IP<-RA */ -APUOPFB(M_BIE, RR, 0x1a8, 0x10, "bie", _A1(A_A), 00010, BR) /* BI IP<-RA */ -APUOPFB(M_BISLD, RR, 0x1a9, 0x20, "bisld", _A2(A_T,A_A), 00012, BR) /* BISL RT,IP<-IP,RA */ -APUOPFB(M_BISLE, RR, 0x1a9, 0x10, "bisle", _A2(A_T,A_A), 00012, BR) /* BISL RT,IP<-IP,RA */ -APUOPFB(M_IRETD, RR, 0x1aa, 0x20, "iretd", _A1(A_A), 00010, BR) /* IRET IP<-SRR0 */ -APUOPFB(M_IRETD2, RR, 0x1aa, 0x20, "iretd", _A0(), 00010, BR) /* IRET IP<-SRR0 */ -APUOPFB(M_IRETE, RR, 0x1aa, 0x10, "irete", _A1(A_A), 00010, BR) /* IRET IP<-SRR0 */ -APUOPFB(M_IRETE2, RR, 0x1aa, 0x10, "irete", _A0(), 00010, BR) /* IRET IP<-SRR0 */ -APUOPFB(M_BISLEDD, RR, 0x1ab, 0x20, "bisledd", _A2(A_T,A_A), 00012, BR) /* BISLED RT,IP<-IP,RA_if(ext) */ -APUOPFB(M_BISLEDE, RR, 0x1ab, 0x10, "bislede", _A2(A_T,A_A), 00012, BR) /* BISLED RT,IP<-IP,RA_if(ext) */ -APUOPFB(M_BIHNZD, RR, 0x12b, 0x20, "bihnzd", _A2(A_T,A_A), 00011, BR) /* BIHNZ IP<-RA_if(RT) */ -APUOPFB(M_BIHNZE, RR, 0x12b, 0x10, "bihnze", _A2(A_T,A_A), 00011, BR) /* BIHNZ IP<-RA_if(RT) */ -APUOPFB(M_BIHZD, RR, 0x12a, 0x20, "bihzd", _A2(A_T,A_A), 00011, BR) /* BIHZ IP<-RA_if(RT) */ -APUOPFB(M_BIHZE, RR, 0x12a, 0x10, "bihze", _A2(A_T,A_A), 00011, BR) /* BIHZ IP<-RA_if(RT) */ -APUOPFB(M_BINZD, RR, 0x129, 0x20, "binzd", _A2(A_T,A_A), 00011, BR) /* BINZ IP<-RA_if(RT) */ -APUOPFB(M_BINZE, RR, 0x129, 0x10, "binze", _A2(A_T,A_A), 00011, BR) /* BINZ IP<-RA_if(RT) */ -APUOPFB(M_BIZD, RR, 0x128, 0x20, "bizd", _A2(A_T,A_A), 00011, BR) /* BIZ IP<-RA_if(RT) */ -APUOPFB(M_BIZE, RR, 0x128, 0x10, "bize", _A2(A_T,A_A), 00011, BR) /* BIZ IP<-RA_if(RT) */ -APUOPFB(M_SYNCC, RR, 0x002, 0x40, "syncc", _A0(), 00000, BR) /* SYNCC flush_pipe */ -APUOPFB(M_HBRP, LBTI, 0x1ac, 0x40, "hbrp", _A0(), 00010, LS) /* HBR BTB[B9]<-M[Ra] */ - -/* Synonyms required by the AS manual. */ -APUOP(M_LR, RI10, 0x020, "lr", _A2(A_T,A_A), 00012, FX2) /* OR%I RT<-RA|I10 */ -APUOP(M_BIHT, RR, 0x12b, "biht", _A2(A_T,A_A), 00011, BR) /* BIHNZ IP<-RA_if(RT) */ -APUOP(M_BIHF, RR, 0x12a, "bihf", _A2(A_T,A_A), 00011, BR) /* BIHZ IP<-RA_if(RT) */ -APUOP(M_BIT, RR, 0x129, "bit", _A2(A_T,A_A), 00011, BR) /* BINZ IP<-RA_if(RT) */ -APUOP(M_BIF, RR, 0x128, "bif", _A2(A_T,A_A), 00011, BR) /* BIZ IP<-RA_if(RT) */ -APUOPFB(M_BIHTD, RR, 0x12b, 0x20, "bihtd", _A2(A_T,A_A), 00011, BR) /* BIHNF IP<-RA_if(RT) */ -APUOPFB(M_BIHTE, RR, 0x12b, 0x10, "bihte", _A2(A_T,A_A), 00011, BR) /* BIHNF IP<-RA_if(RT) */ -APUOPFB(M_BIHFD, RR, 0x12a, 0x20, "bihfd", _A2(A_T,A_A), 00011, BR) /* BIHZ IP<-RA_if(RT) */ -APUOPFB(M_BIHFE, RR, 0x12a, 0x10, "bihfe", _A2(A_T,A_A), 00011, BR) /* BIHZ IP<-RA_if(RT) */ -APUOPFB(M_BITD, RR, 0x129, 0x20, "bitd", _A2(A_T,A_A), 00011, BR) /* BINF IP<-RA_if(RT) */ -APUOPFB(M_BITE, RR, 0x129, 0x10, "bite", _A2(A_T,A_A), 00011, BR) /* BINF IP<-RA_if(RT) */ -APUOPFB(M_BIFD, RR, 0x128, 0x20, "bifd", _A2(A_T,A_A), 00011, BR) /* BIZ IP<-RA_if(RT) */ -APUOPFB(M_BIFE, RR, 0x128, 0x10, "bife", _A2(A_T,A_A), 00011, BR) /* BIZ IP<-RA_if(RT) */ - -#undef _A0 -#undef _A1 -#undef _A2 -#undef _A3 -#undef _A4 diff --git a/arch/powerpc/xmon/spu-opc.c b/arch/powerpc/xmon/spu-opc.c deleted file mode 100644 index 6d8197cc540b..000000000000 --- a/arch/powerpc/xmon/spu-opc.c +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* SPU opcode list - - Copyright 2006 Free Software Foundation, Inc. - - This file is part of GDB, GAS, and the GNU binutils. - - */ - -#include <linux/kernel.h> -#include <linux/bug.h> -#include "spu.h" - -/* This file holds the Spu opcode table */ - - -/* - Example contents of spu-insn.h - id_tag mode mode type opcode mnemonic asmtype dependency FPU L/S? branch? instruction - QUAD WORD (0,RC,RB,RA,RT) latency - APUOP(M_LQD, 1, 0, RI9, 0x1f8, "lqd", ASM_RI9IDX, 00012, FXU, 1, 0) Load Quadword d-form - */ - -const struct spu_opcode spu_opcodes[] = { -#define APUOP(TAG,MACFORMAT,OPCODE,MNEMONIC,ASMFORMAT,DEP,PIPE) \ - { MACFORMAT, OPCODE, MNEMONIC, ASMFORMAT }, -#define APUOPFB(TAG,MACFORMAT,OPCODE,FB,MNEMONIC,ASMFORMAT,DEP,PIPE) \ - { MACFORMAT, OPCODE, MNEMONIC, ASMFORMAT }, -#include "spu-insns.h" -#undef APUOP -#undef APUOPFB -}; - -const int spu_num_opcodes = ARRAY_SIZE(spu_opcodes); diff --git a/arch/powerpc/xmon/spu.h b/arch/powerpc/xmon/spu.h deleted file mode 100644 index 2d13b1a5fa87..000000000000 --- a/arch/powerpc/xmon/spu.h +++ /dev/null @@ -1,115 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* SPU ELF support for BFD. - - Copyright 2006 Free Software Foundation, Inc. - - This file is part of GDB, GAS, and the GNU binutils. - - */ - - -/* These two enums are from rel_apu/common/spu_asm_format.h */ -/* definition of instruction format */ -typedef enum { - RRR, - RI18, - RI16, - RI10, - RI8, - RI7, - RR, - LBT, - LBTI, - IDATA, - UNKNOWN_IFORMAT -} spu_iformat; - -/* These values describe assembly instruction arguments. They indicate - * how to encode, range checking and which relocation to use. */ -typedef enum { - A_T, /* register at pos 0 */ - A_A, /* register at pos 7 */ - A_B, /* register at pos 14 */ - A_C, /* register at pos 21 */ - A_S, /* special purpose register at pos 7 */ - A_H, /* channel register at pos 7 */ - A_P, /* parenthesis, this has to separate regs from immediates */ - A_S3, - A_S6, - A_S7N, - A_S7, - A_U7A, - A_U7B, - A_S10B, - A_S10, - A_S11, - A_S11I, - A_S14, - A_S16, - A_S18, - A_R18, - A_U3, - A_U5, - A_U6, - A_U7, - A_U14, - A_X16, - A_U18, - A_MAX -} spu_aformat; - -enum spu_insns { -#define APUOP(TAG,MACFORMAT,OPCODE,MNEMONIC,ASMFORMAT,DEP,PIPE) \ - TAG, -#define APUOPFB(TAG,MACFORMAT,OPCODE,FB,MNEMONIC,ASMFORMAT,DEP,PIPE) \ - TAG, -#include "spu-insns.h" -#undef APUOP -#undef APUOPFB - M_SPU_MAX -}; - -struct spu_opcode -{ - spu_iformat insn_type; - unsigned int opcode; - char *mnemonic; - int arg[5]; -}; - -#define SIGNED_EXTRACT(insn,size,pos) (((int)((insn) << (32-size-pos))) >> (32-size)) -#define UNSIGNED_EXTRACT(insn,size,pos) (((insn) >> pos) & ((1 << size)-1)) - -#define DECODE_INSN_RT(insn) (insn & 0x7f) -#define DECODE_INSN_RA(insn) ((insn >> 7) & 0x7f) -#define DECODE_INSN_RB(insn) ((insn >> 14) & 0x7f) -#define DECODE_INSN_RC(insn) ((insn >> 21) & 0x7f) - -#define DECODE_INSN_I10(insn) SIGNED_EXTRACT(insn,10,14) -#define DECODE_INSN_U10(insn) UNSIGNED_EXTRACT(insn,10,14) - -/* For branching, immediate loads, hbr and lqa/stqa. */ -#define DECODE_INSN_I16(insn) SIGNED_EXTRACT(insn,16,7) -#define DECODE_INSN_U16(insn) UNSIGNED_EXTRACT(insn,16,7) - -/* for stop */ -#define DECODE_INSN_U14(insn) UNSIGNED_EXTRACT(insn,14,0) - -/* For ila */ -#define DECODE_INSN_I18(insn) SIGNED_EXTRACT(insn,18,7) -#define DECODE_INSN_U18(insn) UNSIGNED_EXTRACT(insn,18,7) - -/* For rotate and shift and generate control mask */ -#define DECODE_INSN_I7(insn) SIGNED_EXTRACT(insn,7,14) -#define DECODE_INSN_U7(insn) UNSIGNED_EXTRACT(insn,7,14) - -/* For float <-> int conversion */ -#define DECODE_INSN_I8(insn) SIGNED_EXTRACT(insn,8,14) -#define DECODE_INSN_U8(insn) UNSIGNED_EXTRACT(insn,8,14) - -/* For hbr */ -#define DECODE_INSN_I9a(insn) ((SIGNED_EXTRACT(insn,2,23) << 7) | UNSIGNED_EXTRACT(insn,7,0)) -#define DECODE_INSN_I9b(insn) ((SIGNED_EXTRACT(insn,2,14) << 7) | UNSIGNED_EXTRACT(insn,7,0)) -#define DECODE_INSN_U9a(insn) ((UNSIGNED_EXTRACT(insn,2,23) << 7) | UNSIGNED_EXTRACT(insn,7,0)) -#define DECODE_INSN_U9b(insn) ((UNSIGNED_EXTRACT(insn,2,14) << 7) | UNSIGNED_EXTRACT(insn,7,0)) - diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index f4e841a36458..cb3a3244ae6f 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -41,8 +41,6 @@ #include <asm/rtas.h> #include <asm/sstep.h> #include <asm/irq_regs.h> -#include <asm/spu.h> -#include <asm/spu_priv1.h> #include <asm/setjmp.h> #include <asm/reg.h> #include <asm/debug.h> @@ -188,8 +186,6 @@ static void xmon_print_symbol(unsigned long address, const char *mid, const char *after); static const char *getvecname(unsigned long vec); -static int do_spu_cmd(void); - #ifdef CONFIG_44x static void dump_tlb_44x(void); #endif @@ -272,13 +268,6 @@ Commands:\n\ P list processes/tasks\n\ r print registers\n\ s single step\n" -#ifdef CONFIG_SPU_BASE -" ss stop execution on all spus\n\ - sr restore execution on stopped spus\n\ - sf # dump spu fields for spu # (in hex)\n\ - sd # dump spu local store for spu # (in hex)\n\ - sdi # disassemble spu local store for spu # (in hex)\n" -#endif " S print special registers\n\ Sa print all SPRs\n\ Sr # read SPR #\n\ @@ -1112,8 +1101,6 @@ cmds(struct pt_regs *excp) cacheflush(); break; case 's': - if (do_spu_cmd() == 0) - break; if (do_step(excp)) return cmd; break; @@ -1271,11 +1258,7 @@ static int xmon_batch_next_cpu(void) { unsigned long cpu; - while (!cpumask_empty(&xmon_batch_cpus)) { - cpu = cpumask_next_wrap(smp_processor_id(), &xmon_batch_cpus, - xmon_batch_start_cpu, true); - if (cpu >= nr_cpu_ids) - break; + for_each_cpu_wrap(cpu, &xmon_batch_cpus, xmon_batch_start_cpu) { if (xmon_batch_start_cpu == -1) xmon_batch_start_cpu = cpu; if (xmon_switch_cpu(cpu)) @@ -1787,7 +1770,7 @@ static void xmon_show_stack(unsigned long sp, unsigned long lr, sp + STACK_INT_FRAME_REGS); break; } - printf("--- Exception: %lx %s at ", regs.trap, + printf("---- Exception: %lx %s at ", regs.trap, getvecname(TRAP(®s))); pc = regs.nip; lr = regs.link; @@ -2623,9 +2606,9 @@ static void dump_one_paca(int cpu) printf("paca for cpu 0x%x @ %px:\n", cpu, p); - printf(" %-*s = %s\n", 25, "possible", cpu_possible(cpu) ? "yes" : "no"); - printf(" %-*s = %s\n", 25, "present", cpu_present(cpu) ? "yes" : "no"); - printf(" %-*s = %s\n", 25, "online", cpu_online(cpu) ? "yes" : "no"); + printf(" %-*s = %s\n", 25, "possible", str_yes_no(cpu_possible(cpu))); + printf(" %-*s = %s\n", 25, "present", str_yes_no(cpu_present(cpu))); + printf(" %-*s = %s\n", 25, "online", str_yes_no(cpu_online(cpu))); #define DUMP(paca, name, format) \ printf(" %-*s = "format"\t(0x%lx)\n", 25, #name, 18, paca->name, \ @@ -4107,263 +4090,3 @@ void __init xmon_setup(void) if (xmon_early) debugger(NULL); } - -#ifdef CONFIG_SPU_BASE - -struct spu_info { - struct spu *spu; - u64 saved_mfc_sr1_RW; - u32 saved_spu_runcntl_RW; - unsigned long dump_addr; - u8 stopped_ok; -}; - -#define XMON_NUM_SPUS 16 /* Enough for current hardware */ - -static struct spu_info spu_info[XMON_NUM_SPUS]; - -void __init xmon_register_spus(struct list_head *list) -{ - struct spu *spu; - - list_for_each_entry(spu, list, full_list) { - if (spu->number >= XMON_NUM_SPUS) { - WARN_ON(1); - continue; - } - - spu_info[spu->number].spu = spu; - spu_info[spu->number].stopped_ok = 0; - spu_info[spu->number].dump_addr = (unsigned long) - spu_info[spu->number].spu->local_store; - } -} - -static void stop_spus(void) -{ - struct spu *spu; - volatile int i; - u64 tmp; - - for (i = 0; i < XMON_NUM_SPUS; i++) { - if (!spu_info[i].spu) - continue; - - if (setjmp(bus_error_jmp) == 0) { - catch_memory_errors = 1; - sync(); - - spu = spu_info[i].spu; - - spu_info[i].saved_spu_runcntl_RW = - in_be32(&spu->problem->spu_runcntl_RW); - - tmp = spu_mfc_sr1_get(spu); - spu_info[i].saved_mfc_sr1_RW = tmp; - - tmp &= ~MFC_STATE1_MASTER_RUN_CONTROL_MASK; - spu_mfc_sr1_set(spu, tmp); - - sync(); - __delay(200); - - spu_info[i].stopped_ok = 1; - - printf("Stopped spu %.2d (was %s)\n", i, - spu_info[i].saved_spu_runcntl_RW ? - "running" : "stopped"); - } else { - catch_memory_errors = 0; - printf("*** Error stopping spu %.2d\n", i); - } - catch_memory_errors = 0; - } -} - -static void restart_spus(void) -{ - struct spu *spu; - volatile int i; - - for (i = 0; i < XMON_NUM_SPUS; i++) { - if (!spu_info[i].spu) - continue; - - if (!spu_info[i].stopped_ok) { - printf("*** Error, spu %d was not successfully stopped" - ", not restarting\n", i); - continue; - } - - if (setjmp(bus_error_jmp) == 0) { - catch_memory_errors = 1; - sync(); - - spu = spu_info[i].spu; - spu_mfc_sr1_set(spu, spu_info[i].saved_mfc_sr1_RW); - out_be32(&spu->problem->spu_runcntl_RW, - spu_info[i].saved_spu_runcntl_RW); - - sync(); - __delay(200); - - printf("Restarted spu %.2d\n", i); - } else { - catch_memory_errors = 0; - printf("*** Error restarting spu %.2d\n", i); - } - catch_memory_errors = 0; - } -} - -#define DUMP_WIDTH 23 -#define DUMP_VALUE(format, field, value) \ -do { \ - if (setjmp(bus_error_jmp) == 0) { \ - catch_memory_errors = 1; \ - sync(); \ - printf(" %-*s = "format"\n", DUMP_WIDTH, \ - #field, value); \ - sync(); \ - __delay(200); \ - } else { \ - catch_memory_errors = 0; \ - printf(" %-*s = *** Error reading field.\n", \ - DUMP_WIDTH, #field); \ - } \ - catch_memory_errors = 0; \ -} while (0) - -#define DUMP_FIELD(obj, format, field) \ - DUMP_VALUE(format, field, obj->field) - -static void dump_spu_fields(struct spu *spu) -{ - printf("Dumping spu fields at address %p:\n", spu); - - DUMP_FIELD(spu, "0x%x", number); - DUMP_FIELD(spu, "%s", name); - DUMP_FIELD(spu, "0x%lx", local_store_phys); - DUMP_FIELD(spu, "0x%p", local_store); - DUMP_FIELD(spu, "0x%lx", ls_size); - DUMP_FIELD(spu, "0x%x", node); - DUMP_FIELD(spu, "0x%lx", flags); - DUMP_FIELD(spu, "%llu", class_0_pending); - DUMP_FIELD(spu, "0x%llx", class_0_dar); - DUMP_FIELD(spu, "0x%llx", class_1_dar); - DUMP_FIELD(spu, "0x%llx", class_1_dsisr); - DUMP_FIELD(spu, "0x%x", irqs[0]); - DUMP_FIELD(spu, "0x%x", irqs[1]); - DUMP_FIELD(spu, "0x%x", irqs[2]); - DUMP_FIELD(spu, "0x%x", slb_replace); - DUMP_FIELD(spu, "%d", pid); - DUMP_FIELD(spu, "0x%p", mm); - DUMP_FIELD(spu, "0x%p", ctx); - DUMP_FIELD(spu, "0x%p", rq); - DUMP_FIELD(spu, "0x%llx", timestamp); - DUMP_FIELD(spu, "0x%lx", problem_phys); - DUMP_FIELD(spu, "0x%p", problem); - DUMP_VALUE("0x%x", problem->spu_runcntl_RW, - in_be32(&spu->problem->spu_runcntl_RW)); - DUMP_VALUE("0x%x", problem->spu_status_R, - in_be32(&spu->problem->spu_status_R)); - DUMP_VALUE("0x%x", problem->spu_npc_RW, - in_be32(&spu->problem->spu_npc_RW)); - DUMP_FIELD(spu, "0x%p", priv2); - DUMP_FIELD(spu, "0x%p", pdata); -} - -static int spu_inst_dump(unsigned long adr, long count, int praddr) -{ - return generic_inst_dump(adr, count, praddr, print_insn_spu); -} - -static void dump_spu_ls(unsigned long num, int subcmd) -{ - unsigned long offset, addr, ls_addr; - - if (setjmp(bus_error_jmp) == 0) { - catch_memory_errors = 1; - sync(); - ls_addr = (unsigned long)spu_info[num].spu->local_store; - sync(); - __delay(200); - } else { - catch_memory_errors = 0; - printf("*** Error: accessing spu info for spu %ld\n", num); - return; - } - catch_memory_errors = 0; - - if (scanhex(&offset)) - addr = ls_addr + offset; - else - addr = spu_info[num].dump_addr; - - if (addr >= ls_addr + LS_SIZE) { - printf("*** Error: address outside of local store\n"); - return; - } - - switch (subcmd) { - case 'i': - addr += spu_inst_dump(addr, 16, 1); - last_cmd = "sdi\n"; - break; - default: - prdump(addr, 64); - addr += 64; - last_cmd = "sd\n"; - break; - } - - spu_info[num].dump_addr = addr; -} - -static int do_spu_cmd(void) -{ - static unsigned long num = 0; - int cmd, subcmd = 0; - - cmd = inchar(); - switch (cmd) { - case 's': - stop_spus(); - break; - case 'r': - restart_spus(); - break; - case 'd': - subcmd = inchar(); - if (isxdigit(subcmd) || subcmd == '\n') - termch = subcmd; - fallthrough; - case 'f': - scanhex(&num); - if (num >= XMON_NUM_SPUS || !spu_info[num].spu) { - printf("*** Error: invalid spu number\n"); - return 0; - } - - switch (cmd) { - case 'f': - dump_spu_fields(spu_info[num].spu); - break; - default: - dump_spu_ls(num, subcmd); - break; - } - - break; - default: - return -1; - } - - return 0; -} -#else /* ! CONFIG_SPU_BASE */ -static int do_spu_cmd(void) -{ - return -1; -} -#endif |