From c806a6fde1c29e7419afcf94d761827a19c5ffe7 Mon Sep 17 00:00:00 2001 From: Mathieu Malaterre Date: Thu, 23 May 2019 12:25:20 +0200 Subject: powerpc: Remove variable ‘path’ since not used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In commit eab00a208eb6 ("powerpc: Move `path` variable inside DEBUG_PROM") DEBUG_PROM sentinels were added to silence a warning (treated as error with W=1): arch/powerpc/kernel/prom_init.c:1388:8: error: variable ‘path’ set but not used [-Werror=unused-but-set-variable] Rework the original patch and simplify the code, by removing the variable ‘path’ completely. Fix line over 90 characters. Suggested-by: Michael Ellerman Signed-off-by: Mathieu Malaterre Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/prom_init.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 00682b8df330..bab79c51ba4f 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -1566,9 +1566,6 @@ static void __init reserve_mem(u64 base, u64 size) static void __init prom_init_mem(void) { phandle node; -#ifdef DEBUG_PROM - char *path; -#endif char type[64]; unsigned int plen; cell_t *p, *endp; @@ -1590,9 +1587,6 @@ static void __init prom_init_mem(void) prom_debug("root_size_cells: %x\n", rsc); prom_debug("scanning memory:\n"); -#ifdef DEBUG_PROM - path = prom_scratch; -#endif for (node = 0; prom_next_node(&node); ) { type[0] = 0; @@ -1617,9 +1611,10 @@ static void __init prom_init_mem(void) endp = p + (plen / sizeof(cell_t)); #ifdef DEBUG_PROM - memset(path, 0, sizeof(prom_scratch)); - call_prom("package-to-path", 3, 1, node, path, sizeof(prom_scratch) - 1); - prom_debug(" node %s :\n", path); + memset(prom_scratch, 0, sizeof(prom_scratch)); + call_prom("package-to-path", 3, 1, node, prom_scratch, + sizeof(prom_scratch) - 1); + prom_debug(" node %s :\n", prom_scratch); #endif /* DEBUG_PROM */ while ((endp - p) >= (rac + rsc)) { -- cgit v1.2.3 From a3bf9fbdad600b1e4335dd90979f8d6072e4f602 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Wed, 15 May 2019 12:05:01 +0200 Subject: powerpc/pseries: Fix xive=off command line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On POWER9, if the hypervisor supports XIVE exploitation mode, the guest OS will unconditionally requests for the XIVE interrupt mode even if XIVE was deactivated with the kernel command line xive=off. Later on, when the spapr XIVE init code handles xive=off, it disables XIVE and tries to fall back on the legacy mode XICS. This discrepency causes a kernel panic because the hypervisor is configured to provide the XIVE interrupt mode to the guest : kernel BUG at arch/powerpc/sysdev/xics/xics-common.c:135! ... NIP xics_smp_probe+0x38/0x98 LR xics_smp_probe+0x2c/0x98 Call Trace: xics_smp_probe+0x2c/0x98 (unreliable) pSeries_smp_probe+0x40/0xa0 smp_prepare_cpus+0x62c/0x6ec kernel_init_freeable+0x148/0x448 kernel_init+0x2c/0x148 ret_from_kernel_thread+0x5c/0x68 Look for xive=off during prom_init and don't ask for XIVE in this case. One exception though: if the host only supports XIVE, we still want to boot so we ignore xive=off. Similarly, have the spapr XIVE init code to looking at the interrupt mode negotiated during CAS, and ignore xive=off if the hypervisor only supports XIVE. Fixes: eac1e731b59e ("powerpc/xive: guest exploitation of the XIVE interrupt controller") Cc: stable@vger.kernel.org # v4.20 Reported-by: Pavithra R. Prakash Signed-off-by: Greg Kurz Reviewed-by: Cédric Le Goater Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/prom_init.c | 16 ++++++++++++- arch/powerpc/sysdev/xive/spapr.c | 52 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index bab79c51ba4f..17f1ae7fae2c 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -172,6 +172,7 @@ static unsigned long __prombss prom_tce_alloc_end; #ifdef CONFIG_PPC_PSERIES static bool __prombss prom_radix_disable; +static bool __prombss prom_xive_disable; #endif struct platform_support { @@ -808,6 +809,12 @@ static void __init early_cmdline_parse(void) } if (prom_radix_disable) prom_debug("Radix disabled from cmdline\n"); + + opt = prom_strstr(prom_cmd_line, "xive=off"); + if (opt) { + prom_xive_disable = true; + prom_debug("XIVE disabled from cmdline\n"); + } #endif /* CONFIG_PPC_PSERIES */ } @@ -1216,10 +1223,17 @@ static void __init prom_parse_xive_model(u8 val, switch (val) { case OV5_FEAT(OV5_XIVE_EITHER): /* Either Available */ prom_debug("XIVE - either mode supported\n"); - support->xive = true; + support->xive = !prom_xive_disable; break; case OV5_FEAT(OV5_XIVE_EXPLOIT): /* Only Exploitation mode */ prom_debug("XIVE - exploitation mode supported\n"); + if (prom_xive_disable) { + /* + * If we __have__ to do XIVE, we're better off ignoring + * the command line rather than not booting. + */ + prom_printf("WARNING: Ignoring cmdline option xive=off\n"); + } support->xive = true; break; case OV5_FEAT(OV5_XIVE_LEGACY): /* Only Legacy mode */ diff --git a/arch/powerpc/sysdev/xive/spapr.c b/arch/powerpc/sysdev/xive/spapr.c index 575db3b06a6b..2e2d1b8f810f 100644 --- a/arch/powerpc/sysdev/xive/spapr.c +++ b/arch/powerpc/sysdev/xive/spapr.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -663,6 +664,55 @@ static bool xive_get_max_prio(u8 *max_prio) return true; } +static const u8 *get_vec5_feature(unsigned int index) +{ + unsigned long root, chosen; + int size; + const u8 *vec5; + + root = of_get_flat_dt_root(); + chosen = of_get_flat_dt_subnode_by_name(root, "chosen"); + if (chosen == -FDT_ERR_NOTFOUND) + return NULL; + + vec5 = of_get_flat_dt_prop(chosen, "ibm,architecture-vec-5", &size); + if (!vec5) + return NULL; + + if (size <= index) + return NULL; + + return vec5 + index; +} + +static bool xive_spapr_disabled(void) +{ + const u8 *vec5_xive; + + vec5_xive = get_vec5_feature(OV5_INDX(OV5_XIVE_SUPPORT)); + if (vec5_xive) { + u8 val; + + val = *vec5_xive & OV5_FEAT(OV5_XIVE_SUPPORT); + switch (val) { + case OV5_FEAT(OV5_XIVE_EITHER): + case OV5_FEAT(OV5_XIVE_LEGACY): + break; + case OV5_FEAT(OV5_XIVE_EXPLOIT): + /* Hypervisor only supports XIVE */ + if (xive_cmdline_disabled) + pr_warn("WARNING: Ignoring cmdline option xive=off\n"); + return false; + default: + pr_warn("%s: Unknown xive support option: 0x%x\n", + __func__, val); + break; + } + } + + return xive_cmdline_disabled; +} + bool __init xive_spapr_init(void) { struct device_node *np; @@ -675,7 +725,7 @@ bool __init xive_spapr_init(void) const __be32 *reg; int i; - if (xive_cmdline_disabled) + if (xive_spapr_disabled()) return false; pr_devel("%s()\n", __func__); -- cgit v1.2.3 From d4aa219a074a5abaf95a756b9f0d190b5c03a945 Mon Sep 17 00:00:00 2001 From: Nathan Lynch Date: Tue, 11 Jun 2019 23:45:04 -0500 Subject: powerpc/cacheinfo: add cacheinfo_teardown, cacheinfo_rebuild Allow external callers to force the cacheinfo code to release all its references to cache nodes, e.g. before processing device tree updates post-migration, and to rebuild the hierarchy afterward. CPU online/offline must be blocked by callers; enforce this. Fixes: 410bccf97881 ("powerpc/pseries: Partition migration in the kernel") Signed-off-by: Nathan Lynch Reviewed-by: Gautham R. Shenoy Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/cacheinfo.c | 21 +++++++++++++++++++++ arch/powerpc/kernel/cacheinfo.h | 4 ++++ 2 files changed, 25 insertions(+) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/cacheinfo.c b/arch/powerpc/kernel/cacheinfo.c index 862e2890bd3d..42c559efe060 100644 --- a/arch/powerpc/kernel/cacheinfo.c +++ b/arch/powerpc/kernel/cacheinfo.c @@ -896,4 +896,25 @@ void cacheinfo_cpu_offline(unsigned int cpu_id) if (cache) cache_cpu_clear(cache, cpu_id); } + +void cacheinfo_teardown(void) +{ + unsigned int cpu; + + lockdep_assert_cpus_held(); + + for_each_online_cpu(cpu) + cacheinfo_cpu_offline(cpu); +} + +void cacheinfo_rebuild(void) +{ + unsigned int cpu; + + lockdep_assert_cpus_held(); + + for_each_online_cpu(cpu) + cacheinfo_cpu_online(cpu); +} + #endif /* (CONFIG_PPC_PSERIES && CONFIG_SUSPEND) || CONFIG_HOTPLUG_CPU */ diff --git a/arch/powerpc/kernel/cacheinfo.h b/arch/powerpc/kernel/cacheinfo.h index 955f5e999f1b..52bd3fc6642d 100644 --- a/arch/powerpc/kernel/cacheinfo.h +++ b/arch/powerpc/kernel/cacheinfo.h @@ -6,4 +6,8 @@ extern void cacheinfo_cpu_online(unsigned int cpu_id); extern void cacheinfo_cpu_offline(unsigned int cpu_id); +/* Allow migration/suspend to tear down and rebuild the hierarchy. */ +extern void cacheinfo_teardown(void); +extern void cacheinfo_rebuild(void); + #endif /* _PPC_CACHEINFO_H */ -- cgit v1.2.3 From 9c4e4c90ec24652921e31e9551fcaedc26eec86d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 10 May 2019 06:31:28 +0000 Subject: powerpc/64: mark start_here_multiplatform as __ref Otherwise, the following warning is encountered: WARNING: vmlinux.o(.text+0x3dc6): Section mismatch in reference from the variable start_here_multiplatform to the function .init.text:.early_setup() The function start_here_multiplatform() references the function __init .early_setup(). This is often because start_here_multiplatform lacks a __init annotation or the annotation of .early_setup is wrong. Fixes: 56c46bba9bbf ("powerpc/64: Fix booting large kernels with STRICT_KERNEL_RWX") Cc: Russell Currey Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/head_64.S | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index 5321a11c2835..259be7f6d551 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -904,6 +904,7 @@ p_toc: .8byte __toc_start + 0x8000 - 0b /* * This is where the main kernel code starts. */ +__REF start_here_multiplatform: /* set up the TOC */ bl relative_toc @@ -979,6 +980,7 @@ start_here_multiplatform: RFI b . /* prevent speculative execution */ + .previous /* This is where all platforms converge execution */ start_here_common: -- cgit v1.2.3 From 6ecb78ef56e08d2119d337ae23cb951a640dc52d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Mon, 17 Jun 2019 21:42:14 +0000 Subject: powerpc/32s: fix suspend/resume when IBATs 4-7 are used Previously, only IBAT1 and IBAT2 were used to map kernel linear mem. Since commit 63b2bc619565 ("powerpc/mm/32s: Use BATs for STRICT_KERNEL_RWX"), we may have all 8 BATs used for mapping kernel text. But the suspend/restore functions only save/restore BATs 0 to 3, and clears BATs 4 to 7. Make suspend and restore functions respectively save and reload the 8 BATs on CPUs having MMU_FTR_USE_HIGH_BATS feature. Reported-by: Andreas Schwab Cc: stable@vger.kernel.org Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/swsusp_32.S | 73 +++++++++++++++++++++++++++++---- arch/powerpc/platforms/powermac/sleep.S | 68 +++++++++++++++++++++++++++--- 2 files changed, 128 insertions(+), 13 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/swsusp_32.S b/arch/powerpc/kernel/swsusp_32.S index 7a919e9a3400..cbdf86228eaa 100644 --- a/arch/powerpc/kernel/swsusp_32.S +++ b/arch/powerpc/kernel/swsusp_32.S @@ -25,11 +25,19 @@ #define SL_IBAT2 0x48 #define SL_DBAT3 0x50 #define SL_IBAT3 0x58 -#define SL_TB 0x60 -#define SL_R2 0x68 -#define SL_CR 0x6c -#define SL_LR 0x70 -#define SL_R12 0x74 /* r12 to r31 */ +#define SL_DBAT4 0x60 +#define SL_IBAT4 0x68 +#define SL_DBAT5 0x70 +#define SL_IBAT5 0x78 +#define SL_DBAT6 0x80 +#define SL_IBAT6 0x88 +#define SL_DBAT7 0x90 +#define SL_IBAT7 0x98 +#define SL_TB 0xa0 +#define SL_R2 0xa8 +#define SL_CR 0xac +#define SL_LR 0xb0 +#define SL_R12 0xb4 /* r12 to r31 */ #define SL_SIZE (SL_R12 + 80) .section .data @@ -114,6 +122,41 @@ _GLOBAL(swsusp_arch_suspend) mfibatl r4,3 stw r4,SL_IBAT3+4(r11) +BEGIN_MMU_FTR_SECTION + mfspr r4,SPRN_DBAT4U + stw r4,SL_DBAT4(r11) + mfspr r4,SPRN_DBAT4L + stw r4,SL_DBAT4+4(r11) + mfspr r4,SPRN_DBAT5U + stw r4,SL_DBAT5(r11) + mfspr r4,SPRN_DBAT5L + stw r4,SL_DBAT5+4(r11) + mfspr r4,SPRN_DBAT6U + stw r4,SL_DBAT6(r11) + mfspr r4,SPRN_DBAT6L + stw r4,SL_DBAT6+4(r11) + mfspr r4,SPRN_DBAT7U + stw r4,SL_DBAT7(r11) + mfspr r4,SPRN_DBAT7L + stw r4,SL_DBAT7+4(r11) + mfspr r4,SPRN_IBAT4U + stw r4,SL_IBAT4(r11) + mfspr r4,SPRN_IBAT4L + stw r4,SL_IBAT4+4(r11) + mfspr r4,SPRN_IBAT5U + stw r4,SL_IBAT5(r11) + mfspr r4,SPRN_IBAT5L + stw r4,SL_IBAT5+4(r11) + mfspr r4,SPRN_IBAT6U + stw r4,SL_IBAT6(r11) + mfspr r4,SPRN_IBAT6L + stw r4,SL_IBAT6+4(r11) + mfspr r4,SPRN_IBAT7U + stw r4,SL_IBAT7(r11) + mfspr r4,SPRN_IBAT7L + stw r4,SL_IBAT7+4(r11) +END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) + #if 0 /* Backup various CPU config stuffs */ bl __save_cpu_setup @@ -279,27 +322,41 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) mtibatu 3,r4 lwz r4,SL_IBAT3+4(r11) mtibatl 3,r4 -#endif - BEGIN_MMU_FTR_SECTION - li r4,0 + lwz r4,SL_DBAT4(r11) mtspr SPRN_DBAT4U,r4 + lwz r4,SL_DBAT4+4(r11) mtspr SPRN_DBAT4L,r4 + lwz r4,SL_DBAT5(r11) mtspr SPRN_DBAT5U,r4 + lwz r4,SL_DBAT5+4(r11) mtspr SPRN_DBAT5L,r4 + lwz r4,SL_DBAT6(r11) mtspr SPRN_DBAT6U,r4 + lwz r4,SL_DBAT6+4(r11) mtspr SPRN_DBAT6L,r4 + lwz r4,SL_DBAT7(r11) mtspr SPRN_DBAT7U,r4 + lwz r4,SL_DBAT7+4(r11) mtspr SPRN_DBAT7L,r4 + lwz r4,SL_IBAT4(r11) mtspr SPRN_IBAT4U,r4 + lwz r4,SL_IBAT4+4(r11) mtspr SPRN_IBAT4L,r4 + lwz r4,SL_IBAT5(r11) mtspr SPRN_IBAT5U,r4 + lwz r4,SL_IBAT5+4(r11) mtspr SPRN_IBAT5L,r4 + lwz r4,SL_IBAT6(r11) mtspr SPRN_IBAT6U,r4 + lwz r4,SL_IBAT6+4(r11) mtspr SPRN_IBAT6L,r4 + lwz r4,SL_IBAT7(r11) mtspr SPRN_IBAT7U,r4 + lwz r4,SL_IBAT7+4(r11) mtspr SPRN_IBAT7L,r4 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) +#endif /* Flush all TLBs */ lis r4,0x1000 diff --git a/arch/powerpc/platforms/powermac/sleep.S b/arch/powerpc/platforms/powermac/sleep.S index fb64b09cad9d..eb583bd9a75d 100644 --- a/arch/powerpc/platforms/powermac/sleep.S +++ b/arch/powerpc/platforms/powermac/sleep.S @@ -38,10 +38,18 @@ #define SL_IBAT2 0x48 #define SL_DBAT3 0x50 #define SL_IBAT3 0x58 -#define SL_TB 0x60 -#define SL_R2 0x68 -#define SL_CR 0x6c -#define SL_R12 0x70 /* r12 to r31 */ +#define SL_DBAT4 0x60 +#define SL_IBAT4 0x68 +#define SL_DBAT5 0x70 +#define SL_IBAT5 0x78 +#define SL_DBAT6 0x80 +#define SL_IBAT6 0x88 +#define SL_DBAT7 0x90 +#define SL_IBAT7 0x98 +#define SL_TB 0xa0 +#define SL_R2 0xa8 +#define SL_CR 0xac +#define SL_R12 0xb0 /* r12 to r31 */ #define SL_SIZE (SL_R12 + 80) .section .text @@ -126,6 +134,41 @@ _GLOBAL(low_sleep_handler) mfibatl r4,3 stw r4,SL_IBAT3+4(r1) +BEGIN_MMU_FTR_SECTION + mfspr r4,SPRN_DBAT4U + stw r4,SL_DBAT4(r1) + mfspr r4,SPRN_DBAT4L + stw r4,SL_DBAT4+4(r1) + mfspr r4,SPRN_DBAT5U + stw r4,SL_DBAT5(r1) + mfspr r4,SPRN_DBAT5L + stw r4,SL_DBAT5+4(r1) + mfspr r4,SPRN_DBAT6U + stw r4,SL_DBAT6(r1) + mfspr r4,SPRN_DBAT6L + stw r4,SL_DBAT6+4(r1) + mfspr r4,SPRN_DBAT7U + stw r4,SL_DBAT7(r1) + mfspr r4,SPRN_DBAT7L + stw r4,SL_DBAT7+4(r1) + mfspr r4,SPRN_IBAT4U + stw r4,SL_IBAT4(r1) + mfspr r4,SPRN_IBAT4L + stw r4,SL_IBAT4+4(r1) + mfspr r4,SPRN_IBAT5U + stw r4,SL_IBAT5(r1) + mfspr r4,SPRN_IBAT5L + stw r4,SL_IBAT5+4(r1) + mfspr r4,SPRN_IBAT6U + stw r4,SL_IBAT6(r1) + mfspr r4,SPRN_IBAT6L + stw r4,SL_IBAT6+4(r1) + mfspr r4,SPRN_IBAT7U + stw r4,SL_IBAT7(r1) + mfspr r4,SPRN_IBAT7L + stw r4,SL_IBAT7+4(r1) +END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) + /* Backup various CPU config stuffs */ bl __save_cpu_setup @@ -326,22 +369,37 @@ grackle_wake_up: mtibatl 3,r4 BEGIN_MMU_FTR_SECTION - li r4,0 + lwz r4,SL_DBAT4(r1) mtspr SPRN_DBAT4U,r4 + lwz r4,SL_DBAT4+4(r1) mtspr SPRN_DBAT4L,r4 + lwz r4,SL_DBAT5(r1) mtspr SPRN_DBAT5U,r4 + lwz r4,SL_DBAT5+4(r1) mtspr SPRN_DBAT5L,r4 + lwz r4,SL_DBAT6(r1) mtspr SPRN_DBAT6U,r4 + lwz r4,SL_DBAT6+4(r1) mtspr SPRN_DBAT6L,r4 + lwz r4,SL_DBAT7(r1) mtspr SPRN_DBAT7U,r4 + lwz r4,SL_DBAT7+4(r1) mtspr SPRN_DBAT7L,r4 + lwz r4,SL_IBAT4(r1) mtspr SPRN_IBAT4U,r4 + lwz r4,SL_IBAT4+4(r1) mtspr SPRN_IBAT4L,r4 + lwz r4,SL_IBAT5(r1) mtspr SPRN_IBAT5U,r4 + lwz r4,SL_IBAT5+4(r1) mtspr SPRN_IBAT5L,r4 + lwz r4,SL_IBAT6(r1) mtspr SPRN_IBAT6U,r4 + lwz r4,SL_IBAT6+4(r1) mtspr SPRN_IBAT6L,r4 + lwz r4,SL_IBAT7(r1) mtspr SPRN_IBAT7U,r4 + lwz r4,SL_IBAT7+4(r1) mtspr SPRN_IBAT7L,r4 END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_HIGH_BATS) -- cgit v1.2.3 From f474c28fbcbe42faca4eb415172c07d76adcb819 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Thu, 13 Jun 2019 09:00:14 +0530 Subject: powerpc/watchpoint: Restore NV GPRs while returning from exception powerpc hardware triggers watchpoint before executing the instruction. To make trigger-after-execute behavior, kernel emulates the instruction. If the instruction is 'load something into non-volatile register', exception handler should restore emulated register state while returning back, otherwise there will be register state corruption. eg, adding a watchpoint on a list can corrput the list: # cat /proc/kallsyms | grep kthread_create_list c00000000121c8b8 d kthread_create_list Add watchpoint on kthread_create_list->prev: # perf record -e mem:0xc00000000121c8c0 Run some workload such that new kthread gets invoked. eg, I just logged out from console: list_add corruption. next->prev should be prev (c000000001214e00), \ but was c00000000121c8b8. (next=c00000000121c8b8). WARNING: CPU: 59 PID: 309 at lib/list_debug.c:25 __list_add_valid+0xb4/0xc0 CPU: 59 PID: 309 Comm: kworker/59:0 Kdump: loaded Not tainted 5.1.0-rc7+ #69 ... NIP __list_add_valid+0xb4/0xc0 LR __list_add_valid+0xb0/0xc0 Call Trace: __list_add_valid+0xb0/0xc0 (unreliable) __kthread_create_on_node+0xe0/0x260 kthread_create_on_node+0x34/0x50 create_worker+0xe8/0x260 worker_thread+0x444/0x560 kthread+0x160/0x1a0 ret_from_kernel_thread+0x5c/0x70 List corruption happened because it uses 'load into non-volatile register' instruction: Snippet from __kthread_create_on_node: c000000000136be8: addis r29,r2,-19 c000000000136bec: ld r29,31424(r29) if (!__list_add_valid(new, prev, next)) c000000000136bf0: mr r3,r30 c000000000136bf4: mr r5,r28 c000000000136bf8: mr r4,r29 c000000000136bfc: bl c00000000059a2f8 <__list_add_valid+0x8> Register state from WARN_ON(): GPR00: c00000000059a3a0 c000007ff23afb50 c000000001344e00 0000000000000075 GPR04: 0000000000000000 0000000000000000 0000001852af8bc1 0000000000000000 GPR08: 0000000000000001 0000000000000007 0000000000000006 00000000000004aa GPR12: 0000000000000000 c000007ffffeb080 c000000000137038 c000005ff62aaa00 GPR16: 0000000000000000 0000000000000000 c000007fffbe7600 c000007fffbe7370 GPR20: c000007fffbe7320 c000007fffbe7300 c000000001373a00 0000000000000000 GPR24: fffffffffffffef7 c00000000012e320 c000007ff23afcb0 c000000000cb8628 GPR28: c00000000121c8b8 c000000001214e00 c000007fef5b17e8 c000007fef5b17c0 Watchpoint hit at 0xc000000000136bec. addis r29,r2,-19 => r29 = 0xc000000001344e00 + (-19 << 16) => r29 = 0xc000000001214e00 ld r29,31424(r29) => r29 = *(0xc000000001214e00 + 31424) => r29 = *(0xc00000000121c8c0) 0xc00000000121c8c0 is where we placed a watchpoint and thus this instruction was emulated by emulate_step. But because handle_dabr_fault did not restore emulated register state, r29 still contains stale value in above register state. Fixes: 5aae8a5370802 ("powerpc, hw_breakpoints: Implement hw_breakpoints for 64-bit server processors") Signed-off-by: Ravi Bangoria Cc: stable@vger.kernel.org # 2.6.36+ Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 6b86055e5251..4d4fd2ad5b7d 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1746,7 +1746,7 @@ handle_page_fault: addi r3,r1,STACK_FRAME_OVERHEAD bl do_page_fault cmpdi r3,0 - beq+ 12f + beq+ ret_from_except_lite bl save_nvgprs mr r5,r3 addi r3,r1,STACK_FRAME_OVERHEAD @@ -1761,7 +1761,12 @@ handle_dabr_fault: ld r5,_DSISR(r1) addi r3,r1,STACK_FRAME_OVERHEAD bl do_break -12: b ret_from_except_lite + /* + * do_break() may have changed the NV GPRS while handling a breakpoint. + * If so, we need to restore them with their updated values. Don't use + * ret_from_except_lite here. + */ + b ret_from_except #ifdef CONFIG_PPC_BOOK3S_64 -- cgit v1.2.3 From df5be5be8735ef2ae80d5ae1f2453cd81a035c4b Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Wed, 5 Jun 2019 13:38:14 +1000 Subject: powerpc/pci/of: Fix OF flags parsing for 64bit BARs When the firmware does PCI BAR resource allocation, it passes the assigned addresses and flags (prefetch/64bit/...) via the "reg" property of a PCI device device tree node so the kernel does not need to do resource allocation. The flags are stored in resource::flags - the lower byte stores PCI_BASE_ADDRESS_SPACE/etc bits and the other bytes are IORESOURCE_IO/etc. Some flags from PCI_BASE_ADDRESS_xxx and IORESOURCE_xxx are duplicated, such as PCI_BASE_ADDRESS_MEM_PREFETCH/PCI_BASE_ADDRESS_MEM_TYPE_64/etc. When parsing the "reg" property, we copy the prefetch flag but we skip on PCI_BASE_ADDRESS_MEM_TYPE_64 which leaves the flags out of sync. The missing IORESOURCE_MEM_64 flag comes into play under 2 conditions: 1. we remove PCI_PROBE_ONLY for pseries (by hacking pSeries_setup_arch() or by passing "/chosen/linux,pci-probe-only"); 2. we request resource alignment (by passing pci=resource_alignment= via the kernel cmd line to request PAGE_SIZE alignment or defining ppc_md.pcibios_default_alignment which returns anything but 0). Note that the alignment requests are ignored if PCI_PROBE_ONLY is enabled. With 1) and 2), the generic PCI code in the kernel unconditionally decides to: - reassign the BARs in pci_specified_resource_alignment() (works fine) - write new BARs to the device - this fails for 64bit BARs as the generic code looks at IORESOURCE_MEM_64 (not set) and writes only lower 32bits of the BAR and leaves the upper 32bit unmodified which breaks BAR mapping in the hypervisor. This fixes the issue by copying the flag. This is useful if we want to enforce certain BAR alignment per platform as handling subpage sized BARs is proven to cause problems with hotplug (SLOF already aligns BARs to 64k). Signed-off-by: Alexey Kardashevskiy Reviewed-by: Sam Bobroff Reviewed-by: Oliver O'Halloran Reviewed-by: Shawn Anastasio Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/pci_of_scan.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/pci_of_scan.c b/arch/powerpc/kernel/pci_of_scan.c index 24191ea2d9a7..64ad92016b63 100644 --- a/arch/powerpc/kernel/pci_of_scan.c +++ b/arch/powerpc/kernel/pci_of_scan.c @@ -45,6 +45,8 @@ unsigned int pci_parse_of_flags(u32 addr0, int bridge) if (addr0 & 0x02000000) { flags = IORESOURCE_MEM | PCI_BASE_ADDRESS_SPACE_MEMORY; flags |= (addr0 >> 22) & PCI_BASE_ADDRESS_MEM_TYPE_64; + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) + flags |= IORESOURCE_MEM_64; flags |= (addr0 >> 28) & PCI_BASE_ADDRESS_MEM_TYPE_1M; if (addr0 & 0x40000000) flags |= IORESOURCE_PREFETCH -- cgit v1.2.3 From 9fb603050ffd94f8127df99c699cca2f575eb6a0 Mon Sep 17 00:00:00 2001 From: Nathan Lynch Date: Fri, 21 Jun 2019 01:05:18 -0500 Subject: powerpc/rtas: retry when cpu offline races with suspend/migration The protocol for suspending or migrating an LPAR requires all present processor threads to enter H_JOIN. So if we have threads offline, we have to temporarily bring them up. This can race with administrator actions such as SMT state changes. As of dfd718a2ed1f ("powerpc/rtas: Fix a potential race between CPU-Offline & Migration"), rtas_ibm_suspend_me() accounts for this, but errors out with -EBUSY for what almost certainly is a transient condition in any reasonable scenario. Callers of rtas_ibm_suspend_me() already retry when -EAGAIN is returned, and it is typical during a migration for that to happen repeatedly for several minutes polling the H_VASI_STATE hcall result before proceeding to the next stage. So return -EAGAIN instead of -EBUSY when this race is encountered. Additionally: logging this event is still appropriate but use pr_info instead of pr_err; and remove use of unlikely() while here as this is not a hot path at all. Fixes: dfd718a2ed1f ("powerpc/rtas: Fix a potential race between CPU-Offline & Migration") Signed-off-by: Nathan Lynch Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/rtas.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c index fbc676160adf..9b4d2a2ffb4f 100644 --- a/arch/powerpc/kernel/rtas.c +++ b/arch/powerpc/kernel/rtas.c @@ -984,10 +984,9 @@ int rtas_ibm_suspend_me(u64 handle) cpu_hotplug_disable(); /* Check if we raced with a CPU-Offline Operation */ - if (unlikely(!cpumask_equal(cpu_present_mask, cpu_online_mask))) { - pr_err("%s: Raced against a concurrent CPU-Offline\n", - __func__); - atomic_set(&data.error, -EBUSY); + if (!cpumask_equal(cpu_present_mask, cpu_online_mask)) { + pr_info("%s: Raced against a concurrent CPU-Offline\n", __func__); + atomic_set(&data.error, -EAGAIN); goto out_hotplug_enable; } -- cgit v1.2.3 From 3becd11dffe5d4a7467ebd841172f3e091fbcbd0 Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Wed, 5 Jun 2019 16:46:19 -0400 Subject: powerpc/eeh_cache: fix a W=1 kernel-doc warning The opening comment mark "/**" is reserved for kernel-doc comments, so it will generate a warning with "make W=1". arch/powerpc/kernel/eeh_cache.c:37: warning: cannot understand function prototype: 'struct pci_io_addr_range Since this is not a kernel-doc for the struct below, but rather an overview of this source eeh_cache.c, just use the free-form comments kernel-doc syntax instead. Signed-off-by: Qian Cai Acked-by: Russell Currey Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/eeh_cache.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/eeh_cache.c b/arch/powerpc/kernel/eeh_cache.c index 9c68f0837385..52c57d883e4a 100644 --- a/arch/powerpc/kernel/eeh_cache.c +++ b/arch/powerpc/kernel/eeh_cache.c @@ -31,6 +31,8 @@ /** + * DOC: Overview + * * The pci address cache subsystem. This subsystem places * PCI device address resources into a red-black tree, sorted * according to the address range, so that given only an i/o @@ -47,6 +49,7 @@ * than any hash algo I could think of for this problem, even * with the penalty of slow pointer chases for d-cache misses). */ + struct pci_io_addr_range { struct rb_node rb_node; resource_size_t addr_lo; -- cgit v1.2.3 From 4508a74a63154fea5c2d36d8ad064273c5ea9608 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:11 +1000 Subject: powerpc/64s/exception: remove H concatenation for EXC_HV variants Replace all instances of this with gas macros that test the hsrr parameter and use the appropriate register names / labels. No generated code change. Signed-off-by: Nicholas Piggin [mpe: Remove extraneous 2nd check for 0xea0 in SOFTEN_TEST] Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 332 ++++++++++++++++++------------- arch/powerpc/include/asm/head-64.h | 8 +- arch/powerpc/kernel/exceptions-64s.S | 97 +++++---- 3 files changed, 253 insertions(+), 184 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index b4f8b745ba01..eddd74cf36c3 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -67,6 +67,8 @@ */ #define EX_R3 EX_DAR +#ifdef __ASSEMBLY__ + #define STF_ENTRY_BARRIER_SLOT \ STF_ENTRY_BARRIER_FIXUP_SECTION; \ nop; \ @@ -148,38 +150,6 @@ hrfid; \ b hrfi_flush_fallback -#ifdef CONFIG_RELOCATABLE -#define __EXCEPTION_PROLOG_2_RELON(label, h) \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - LOAD_HANDLER(r12,label); \ - mtctr r12; \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - li r10,MSR_RI; \ - mtmsrd r10,1; /* Set RI (EE=0) */ \ - bctr; -#else -/* If not relocatable, we can jump directly -- and save messing with LR */ -#define __EXCEPTION_PROLOG_2_RELON(label, h) \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - li r10,MSR_RI; \ - mtmsrd r10,1; /* Set RI (EE=0) */ \ - b label; -#endif -#define EXCEPTION_PROLOG_2_RELON(label, h) \ - __EXCEPTION_PROLOG_2_RELON(label, h) - -/* - * As EXCEPTION_PROLOG(), except we've already got relocation on so no need to - * rfid. Save LR in case we're CONFIG_RELOCATABLE, in which case - * EXCEPTION_PROLOG_2_RELON will be using LR. - */ -#define EXCEPTION_RELON_PROLOG(area, label, h, extra, vec) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2_RELON(label, h) - /* * We're short on space and time in the exception prolog, so we can't * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. @@ -204,9 +174,54 @@ ori reg,reg,(ABS_ADDR(label))@l; \ addis reg,reg,(ABS_ADDR(label))@h +#ifdef CONFIG_RELOCATABLE +.macro EXCEPTION_PROLOG_2_RELON label, hsrr + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + .endif + LOAD_HANDLER(r12, \label\()) + mtctr r12 + .if \hsrr + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + .else + mfspr r12,SPRN_SRR1 /* and HSRR1 */ + .endif + li r10,MSR_RI + mtmsrd r10,1 /* Set RI (EE=0) */ + bctr +.endm +#else +/* If not relocatable, we can jump directly -- and save messing with LR */ +.macro EXCEPTION_PROLOG_2_RELON label, hsrr + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + mfspr r12,SPRN_SRR1 /* and SRR1 */ + .endif + li r10,MSR_RI + mtmsrd r10,1 /* Set RI (EE=0) */ + b \label +.endm +#endif + +/* + * As EXCEPTION_PROLOG(), except we've already got relocation on so no need to + * rfid. Save LR in case we're CONFIG_RELOCATABLE, in which case + * EXCEPTION_PROLOG_2_RELON will be using LR. + */ +#define EXCEPTION_RELON_PROLOG(area, label, hsrr, extra, vec) \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0(area); \ + EXCEPTION_PROLOG_1(area, extra, vec); \ + EXCEPTION_PROLOG_2_RELON label, hsrr + /* Exception register prefixes */ -#define EXC_HV H -#define EXC_STD +#define EXC_HV 1 +#define EXC_STD 0 #if defined(CONFIG_RELOCATABLE) /* @@ -308,43 +323,57 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG_1(area, extra, vec) \ _EXCEPTION_PROLOG_1(area, extra, vec) -#define __EXCEPTION_PROLOG_2(label, h) \ - ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - LOAD_HANDLER(r12,label); \ - mtspr SPRN_##h##SRR0,r12; \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - mtspr SPRN_##h##SRR1,r10; \ - h##RFI_TO_KERNEL; \ +.macro EXCEPTION_PROLOG_2 label, hsrr + ld r10,PACAKMSR(r13) /* get MSR value for kernel */ + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + .endif + LOAD_HANDLER(r12,\label\()) + .if \hsrr + mtspr SPRN_HSRR0,r12 + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + mtspr SPRN_HSRR1,r10 + HRFI_TO_KERNEL + .else + mtspr SPRN_SRR0,r12 + mfspr r12,SPRN_SRR1 /* and SRR1 */ + mtspr SPRN_SRR1,r10 + RFI_TO_KERNEL + .endif b . /* prevent speculative execution */ -#define EXCEPTION_PROLOG_2(label, h) \ - __EXCEPTION_PROLOG_2(label, h) +.endm /* _NORI variant keeps MSR_RI clear */ -#define __EXCEPTION_PROLOG_2_NORI(label, h) \ - ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ - xori r10,r10,MSR_RI; /* Clear MSR_RI */ \ - mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ - LOAD_HANDLER(r12,label); \ - mtspr SPRN_##h##SRR0,r12; \ - mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ - mtspr SPRN_##h##SRR1,r10; \ - h##RFI_TO_KERNEL; \ +.macro EXCEPTION_PROLOG_2_NORI label, hsrr + ld r10,PACAKMSR(r13) /* get MSR value for kernel */ + xori r10,r10,MSR_RI /* Clear MSR_RI */ + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + .endif + LOAD_HANDLER(r12,\label\()) + .if \hsrr + mtspr SPRN_HSRR0,r12 + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + mtspr SPRN_HSRR1,r10 + HRFI_TO_KERNEL + .else + mtspr SPRN_SRR0,r12 + mfspr r12,SPRN_SRR1 /* and SRR1 */ + mtspr SPRN_SRR1,r10 + RFI_TO_KERNEL + .endif b . /* prevent speculative execution */ - -#define EXCEPTION_PROLOG_2_NORI(label, h) \ - __EXCEPTION_PROLOG_2_NORI(label, h) +.endm #define EXCEPTION_PROLOG(area, label, h, extra, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2(label, h) - -#define __KVMTEST(h, n) \ - lbz r10,HSTATE_IN_GUEST(r13); \ - cmpwi r10,0; \ - bne do_kvm_##h##n + EXCEPTION_PROLOG_2 label, h #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* @@ -413,52 +442,66 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG_NORI(area, label, h, extra, vec) \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2_NORI(label, h) - - -#define __KVM_HANDLER(area, h, n) \ - BEGIN_FTR_SECTION_NESTED(947) \ - ld r10,area+EX_CFAR(r13); \ - std r10,HSTATE_CFAR(r13); \ - END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947); \ - BEGIN_FTR_SECTION_NESTED(948) \ - ld r10,area+EX_PPR(r13); \ - std r10,HSTATE_PPR(r13); \ - END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948); \ - ld r10,area+EX_R10(r13); \ - std r12,HSTATE_SCRATCH0(r13); \ - sldi r12,r9,32; \ - ori r12,r12,(n); \ - /* This reloads r9 before branching to kvmppc_interrupt */ \ - __BRANCH_TO_KVM_EXIT(area, kvmppc_interrupt) - -#define __KVM_HANDLER_SKIP(area, h, n) \ - cmpwi r10,KVM_GUEST_MODE_SKIP; \ - beq 89f; \ - BEGIN_FTR_SECTION_NESTED(948) \ - ld r10,area+EX_PPR(r13); \ - std r10,HSTATE_PPR(r13); \ - END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948); \ - ld r10,area+EX_R10(r13); \ - std r12,HSTATE_SCRATCH0(r13); \ - sldi r12,r9,32; \ - ori r12,r12,(n); \ - /* This reloads r9 before branching to kvmppc_interrupt */ \ - __BRANCH_TO_KVM_EXIT(area, kvmppc_interrupt); \ -89: mtocrf 0x80,r9; \ - ld r9,area+EX_R9(r13); \ - ld r10,area+EX_R10(r13); \ - b kvmppc_skip_##h##interrupt + EXCEPTION_PROLOG_2_NORI label, h #ifdef CONFIG_KVM_BOOK3S_64_HANDLER -#define KVMTEST(h, n) __KVMTEST(h, n) -#define KVM_HANDLER(area, h, n) __KVM_HANDLER(area, h, n) -#define KVM_HANDLER_SKIP(area, h, n) __KVM_HANDLER_SKIP(area, h, n) +.macro KVMTEST hsrr, n + lbz r10,HSTATE_IN_GUEST(r13) + cmpwi r10,0 + .if \hsrr + bne do_kvm_H\n + .else + bne do_kvm_\n + .endif +.endm + +.macro KVM_HANDLER area, hsrr, n + BEGIN_FTR_SECTION_NESTED(947) + ld r10,\area+EX_CFAR(r13) + std r10,HSTATE_CFAR(r13) + END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947) + BEGIN_FTR_SECTION_NESTED(948) + ld r10,\area+EX_PPR(r13) + std r10,HSTATE_PPR(r13) + END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) + ld r10,\area+EX_R10(r13) + std r12,HSTATE_SCRATCH0(r13) + sldi r12,r9,32 + ori r12,r12,(\n) + /* This reloads r9 before branching to kvmppc_interrupt */ + __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt) +.endm + +.macro KVM_HANDLER_SKIP area, hsrr, n + cmpwi r10,KVM_GUEST_MODE_SKIP + beq 89f + BEGIN_FTR_SECTION_NESTED(948) + ld r10,\area+EX_PPR(r13) + std r10,HSTATE_PPR(r13) + END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) + ld r10,\area+EX_R10(r13) + std r12,HSTATE_SCRATCH0(r13) + sldi r12,r9,32 + ori r12,r12,(\n) + /* This reloads r9 before branching to kvmppc_interrupt */ + __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt) +89: mtocrf 0x80,r9 + ld r9,\area+EX_R9(r13) + ld r10,\area+EX_R10(r13) + .if \hsrr + b kvmppc_skip_Hinterrupt + .else + b kvmppc_skip_interrupt + .endif +.endm #else -#define KVMTEST(h, n) -#define KVM_HANDLER(area, h, n) -#define KVM_HANDLER_SKIP(area, h, n) +.macro KVMTEST hsrr, n +.endm +.macro KVM_HANDLER area, hsrr, n +.endm +.macro KVM_HANDLER_SKIP area, hsrr, n +.endm #endif #define NOTEST(n) @@ -556,14 +599,14 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define STD_EXCEPTION_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec); \ - EXCEPTION_PROLOG_2(label, EXC_STD) + EXCEPTION_PROLOG_2 label, EXC_STD #define STD_EXCEPTION_HV(loc, vec, label) \ EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec) #define STD_EXCEPTION_HV_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ - EXCEPTION_PROLOG_2(label, EXC_HV) + EXCEPTION_PROLOG_2 label, EXC_HV #define STD_RELON_EXCEPTION(loc, vec, label) \ /* No guest interrupts come through here */ \ @@ -571,88 +614,97 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define STD_RELON_EXCEPTION_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \ - EXCEPTION_PROLOG_2_RELON(label, EXC_STD) + EXCEPTION_PROLOG_2_RELON label, EXC_STD #define STD_RELON_EXCEPTION_HV(loc, vec, label) \ EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec) #define STD_RELON_EXCEPTION_HV_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ - EXCEPTION_PROLOG_2_RELON(label, EXC_HV) - -/* This associate vector numbers with bits in paca->irq_happened */ -#define SOFTEN_VALUE_0x500 PACA_IRQ_EE -#define SOFTEN_VALUE_0x900 PACA_IRQ_DEC -#define SOFTEN_VALUE_0xa00 PACA_IRQ_DBELL -#define SOFTEN_VALUE_0xe80 PACA_IRQ_DBELL -#define SOFTEN_VALUE_0xe60 PACA_IRQ_HMI -#define SOFTEN_VALUE_0xea0 PACA_IRQ_EE -#define SOFTEN_VALUE_0xf00 PACA_IRQ_PMI - -#define __SOFTEN_TEST(h, vec, bitmask) \ - lbz r10,PACAIRQSOFTMASK(r13); \ - andi. r10,r10,bitmask; \ - li r10,SOFTEN_VALUE_##vec; \ - bne masked_##h##interrupt - -#define _SOFTEN_TEST(h, vec, bitmask) __SOFTEN_TEST(h, vec, bitmask) + EXCEPTION_PROLOG_2_RELON label, EXC_HV + +.macro SOFTEN_TEST hsrr, vec, bitmask + lbz r10, PACAIRQSOFTMASK(r13) + andi. r10, r10, \bitmask + /* This associates vector numbers with bits in paca->irq_happened */ + .if \vec == 0x500 || \vec == 0xea0 + li r10, PACA_IRQ_EE + .elseif \vec == 0x900 + li r10, PACA_IRQ_DEC + .elseif \vec == 0xa00 || \vec == 0xe80 + li r10, PACA_IRQ_DBELL + .elseif \vec == 0xe60 + li r10, PACA_IRQ_HMI + .elseif \vec == 0xf00 + li r10, PACA_IRQ_PMI + .else + .abort "Bad maskable vector" + .endif + + + .if \hsrr + bne masked_Hinterrupt + .else + bne masked_interrupt + .endif +.endm #define SOFTEN_TEST_PR(vec, bitmask) \ - KVMTEST(EXC_STD, vec); \ - _SOFTEN_TEST(EXC_STD, vec, bitmask) + KVMTEST EXC_STD, vec ; \ + SOFTEN_TEST EXC_STD, vec, bitmask #define SOFTEN_TEST_HV(vec, bitmask) \ - KVMTEST(EXC_HV, vec); \ - _SOFTEN_TEST(EXC_HV, vec, bitmask) + KVMTEST EXC_HV, vec ; \ + SOFTEN_TEST EXC_HV, vec, bitmask #define KVMTEST_PR(vec) \ - KVMTEST(EXC_STD, vec) + KVMTEST EXC_STD, vec #define KVMTEST_HV(vec) \ - KVMTEST(EXC_HV, vec) + KVMTEST EXC_HV, vec -#define SOFTEN_NOTEST_PR(vec, bitmask) _SOFTEN_TEST(EXC_STD, vec, bitmask) -#define SOFTEN_NOTEST_HV(vec, bitmask) _SOFTEN_TEST(EXC_HV, vec, bitmask) +#define SOFTEN_NOTEST_PR(vec, bitmask) SOFTEN_TEST EXC_STD, vec, bitmask +#define SOFTEN_NOTEST_HV(vec, bitmask) SOFTEN_TEST EXC_HV, vec, bitmask #define __MASKABLE_EXCEPTION(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_2(label, h) + EXCEPTION_PROLOG_2 label, h #define MASKABLE_EXCEPTION(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_STD, SOFTEN_TEST_PR, bitmask) #define MASKABLE_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_2(label, EXC_STD) + EXCEPTION_PROLOG_2 label, EXC_STD #define MASKABLE_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) #define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ - EXCEPTION_PROLOG_2(label, EXC_HV) + EXCEPTION_PROLOG_2 label, EXC_HV #define __MASKABLE_RELON_EXCEPTION(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_2_RELON(label, h) + EXCEPTION_PROLOG_2_RELON label, h #define MASKABLE_RELON_EXCEPTION(vec, label, bitmask) \ __MASKABLE_RELON_EXCEPTION(vec, label, EXC_STD, SOFTEN_NOTEST_PR, bitmask) #define MASKABLE_RELON_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_2(label, EXC_STD) + EXCEPTION_PROLOG_2 label, EXC_STD #define MASKABLE_RELON_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ - EXCEPTION_PROLOG_2_RELON(label, EXC_HV) + EXCEPTION_PROLOG_2_RELON label, EXC_HV /* * Our exception common code can be passed various "additions" @@ -731,4 +783,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define FINISH_NAP #endif +#endif /* __ASSEMBLY__ */ + #endif /* _ASM_POWERPC_EXCEPTION_H */ diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index e34b3d06bf61..4767d6c7b8fa 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -387,22 +387,22 @@ name: #define TRAMP_KVM(area, n) \ TRAMP_KVM_BEGIN(do_kvm_##n); \ - KVM_HANDLER(area, EXC_STD, n); \ + KVM_HANDLER area, EXC_STD, n #define TRAMP_KVM_SKIP(area, n) \ TRAMP_KVM_BEGIN(do_kvm_##n); \ - KVM_HANDLER_SKIP(area, EXC_STD, n); \ + KVM_HANDLER_SKIP area, EXC_STD, n /* * HV variant exceptions get the 0x2 bit added to their trap number. */ #define TRAMP_KVM_HV(area, n) \ TRAMP_KVM_BEGIN(do_kvm_H##n); \ - KVM_HANDLER(area, EXC_HV, n + 0x2); \ + KVM_HANDLER area, EXC_HV, n + 0x2 #define TRAMP_KVM_HV_SKIP(area, n) \ TRAMP_KVM_BEGIN(do_kvm_H##n); \ - KVM_HANDLER_SKIP(area, EXC_HV, n + 0x2); \ + KVM_HANDLER_SKIP area, EXC_HV, n + 0x2 #define EXC_COMMON(name, realvec, hdlr) \ EXC_COMMON_BEGIN(name); \ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 6c51aa845bce..13ae6b8d5e3e 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -356,7 +356,7 @@ machine_check_pSeries_0: * nested machine check corrupts it. machine_check_common enables * MSR_RI. */ - EXCEPTION_PROLOG_2_NORI(machine_check_common, EXC_STD) + EXCEPTION_PROLOG_2_NORI machine_check_common, EXC_STD TRAMP_KVM_SKIP(PACA_EXMC, 0x200) @@ -598,7 +598,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x300) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2(data_access_common, EXC_STD) +EXCEPTION_PROLOG_2 data_access_common, EXC_STD EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) SET_SCRATCH0(r13) /* save r13 */ @@ -608,7 +608,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x300) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2_RELON(data_access_common, EXC_STD) +EXCEPTION_PROLOG_2_RELON data_access_common, EXC_STD EXC_VIRT_END(data_access, 0x4300, 0x80) TRAMP_KVM_SKIP(PACA_EXGEN, 0x300) @@ -645,7 +645,7 @@ TRAMP_REAL_BEGIN(tramp_real_data_access_slb) EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380) mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) -EXCEPTION_PROLOG_2(data_access_slb_common, EXC_STD) +EXCEPTION_PROLOG_2 data_access_slb_common, EXC_STD EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) SET_SCRATCH0(r13) /* save r13 */ @@ -653,7 +653,7 @@ EXCEPTION_PROLOG_0(PACA_EXSLB) EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380) mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) -EXCEPTION_PROLOG_2_RELON(data_access_slb_common, EXC_STD) +EXCEPTION_PROLOG_2_RELON data_access_slb_common, EXC_STD EXC_VIRT_END(data_access_slb, 0x4380, 0x80) TRAMP_KVM_SKIP(PACA_EXSLB, 0x380) @@ -774,7 +774,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x600) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2(alignment_common, EXC_STD) +EXCEPTION_PROLOG_2 alignment_common, EXC_STD EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) @@ -785,7 +785,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x600) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2_RELON(alignment_common, EXC_STD) +EXCEPTION_PROLOG_2_RELON alignment_common, EXC_STD EXC_VIRT_END(alignment, 0x4600, 0x100) TRAMP_KVM(PACA_EXGEN, 0x600) @@ -1053,7 +1053,7 @@ TRAMP_KVM_BEGIN(do_kvm_0xc00) SET_SCRATCH0(r10) std r9,PACA_EXGEN+EX_R9(r13) mfcr r9 - KVM_HANDLER(PACA_EXGEN, EXC_STD, 0xc00) + KVM_HANDLER PACA_EXGEN, EXC_STD, 0xc00 #endif @@ -1320,7 +1320,7 @@ EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) #endif KVMTEST_HV(0x1500) - EXCEPTION_PROLOG_2(denorm_common, EXC_HV) + EXCEPTION_PROLOG_2 denorm_common, EXC_HV EXC_REAL_END(denorm_exception_hv, 0x1500, 0x100) #ifdef CONFIG_PPC_DENORMALISATION @@ -1442,7 +1442,7 @@ EXC_VIRT_NONE(0x5800, 0x100) std r12,PACA_EXGEN+EX_R12(r13); \ GET_SCRATCH0(r10); \ std r10,PACA_EXGEN+EX_R13(r13); \ - EXCEPTION_PROLOG_2(soft_nmi_common, _H) + EXCEPTION_PROLOG_2 soft_nmi_common, _H /* * Branch to soft_nmi_interrupt using the emergency stack. The emergency @@ -1477,35 +1477,50 @@ EXC_COMMON_BEGIN(soft_nmi_common) * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return. * This is called with r10 containing the value to OR to the paca field. */ -#define MASKED_INTERRUPT(_H) \ -masked_##_H##interrupt: \ - std r11,PACA_EXGEN+EX_R11(r13); \ - lbz r11,PACAIRQHAPPENED(r13); \ - or r11,r11,r10; \ - stb r11,PACAIRQHAPPENED(r13); \ - cmpwi r10,PACA_IRQ_DEC; \ - bne 1f; \ - lis r10,0x7fff; \ - ori r10,r10,0xffff; \ - mtspr SPRN_DEC,r10; \ - b MASKED_DEC_HANDLER_LABEL; \ -1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK; \ - beq 2f; \ - mfspr r10,SPRN_##_H##SRR1; \ - xori r10,r10,MSR_EE; /* clear MSR_EE */ \ - mtspr SPRN_##_H##SRR1,r10; \ - ori r11,r11,PACA_IRQ_HARD_DIS; \ - stb r11,PACAIRQHAPPENED(r13); \ -2: /* done */ \ - mtcrf 0x80,r9; \ - std r1,PACAR1(r13); \ - ld r9,PACA_EXGEN+EX_R9(r13); \ - ld r10,PACA_EXGEN+EX_R10(r13); \ - ld r11,PACA_EXGEN+EX_R11(r13); \ - /* returns to kernel where r13 must be set up, so don't restore it */ \ - ##_H##RFI_TO_KERNEL; \ - b .; \ - MASKED_DEC_HANDLER(_H) +.macro MASKED_INTERRUPT hsrr + .if \hsrr +masked_Hinterrupt: + .else +masked_interrupt: + .endif + std r11,PACA_EXGEN+EX_R11(r13) + lbz r11,PACAIRQHAPPENED(r13) + or r11,r11,r10 + stb r11,PACAIRQHAPPENED(r13) + cmpwi r10,PACA_IRQ_DEC + bne 1f + lis r10,0x7fff + ori r10,r10,0xffff + mtspr SPRN_DEC,r10 + b MASKED_DEC_HANDLER_LABEL +1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK + beq 2f + .if \hsrr + mfspr r10,SPRN_HSRR1 + xori r10,r10,MSR_EE /* clear MSR_EE */ + mtspr SPRN_HSRR1,r10 + .else + mfspr r10,SPRN_SRR1 + xori r10,r10,MSR_EE /* clear MSR_EE */ + mtspr SPRN_SRR1,r10 + .endif + ori r11,r11,PACA_IRQ_HARD_DIS + stb r11,PACAIRQHAPPENED(r13) +2: /* done */ + mtcrf 0x80,r9 + std r1,PACAR1(r13) + ld r9,PACA_EXGEN+EX_R9(r13) + ld r10,PACA_EXGEN+EX_R10(r13) + ld r11,PACA_EXGEN+EX_R11(r13) + /* returns to kernel where r13 must be set up, so don't restore it */ + .if \hsrr + HRFI_TO_KERNEL + .else + RFI_TO_KERNEL + .endif + b . + MASKED_DEC_HANDLER(\hsrr\()) +.endm TRAMP_REAL_BEGIN(stf_barrier_fallback) std r9,PACA_EXRFI+EX_R9(r13) @@ -1612,8 +1627,8 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) * cannot reach these if they are put there. */ USE_FIXED_SECTION(virt_trampolines) - MASKED_INTERRUPT() - MASKED_INTERRUPT(H) + MASKED_INTERRUPT EXC_STD + MASKED_INTERRUPT EXC_HV #ifdef CONFIG_KVM_BOOK3S_64_HANDLER TRAMP_REAL_BEGIN(kvmppc_skip_interrupt) -- cgit v1.2.3 From bd7b6d1334c5fd92c1e1a9c8179154e115f427b0 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:12 +1000 Subject: powerpc/64s/exception: consolidate EXCEPTION_PROLOG_2 with _NORI variant Switch to a gas macro that conditionally expands the RI clearing instruction. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 43 ++++++++------------------------ arch/powerpc/kernel/exceptions-64s.S | 12 ++++----- 2 files changed, 17 insertions(+), 38 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index eddd74cf36c3..82a1c0576fbb 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -323,32 +323,11 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG_1(area, extra, vec) \ _EXCEPTION_PROLOG_1(area, extra, vec) -.macro EXCEPTION_PROLOG_2 label, hsrr - ld r10,PACAKMSR(r13) /* get MSR value for kernel */ - .if \hsrr - mfspr r11,SPRN_HSRR0 /* save HSRR0 */ - .else - mfspr r11,SPRN_SRR0 /* save SRR0 */ - .endif - LOAD_HANDLER(r12,\label\()) - .if \hsrr - mtspr SPRN_HSRR0,r12 - mfspr r12,SPRN_HSRR1 /* and HSRR1 */ - mtspr SPRN_HSRR1,r10 - HRFI_TO_KERNEL - .else - mtspr SPRN_SRR0,r12 - mfspr r12,SPRN_SRR1 /* and SRR1 */ - mtspr SPRN_SRR1,r10 - RFI_TO_KERNEL - .endif - b . /* prevent speculative execution */ -.endm - -/* _NORI variant keeps MSR_RI clear */ -.macro EXCEPTION_PROLOG_2_NORI label, hsrr +.macro EXCEPTION_PROLOG_2 label, hsrr, set_ri ld r10,PACAKMSR(r13) /* get MSR value for kernel */ + .if ! \set_ri xori r10,r10,MSR_RI /* Clear MSR_RI */ + .endif .if \hsrr mfspr r11,SPRN_HSRR0 /* save HSRR0 */ .else @@ -373,7 +352,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2 label, h + EXCEPTION_PROLOG_2 label, h, 1 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* @@ -442,7 +421,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG_NORI(area, label, h, extra, vec) \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2_NORI label, h + EXCEPTION_PROLOG_2 label, h, 0 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER .macro KVMTEST hsrr, n @@ -599,14 +578,14 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define STD_EXCEPTION_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec); \ - EXCEPTION_PROLOG_2 label, EXC_STD + EXCEPTION_PROLOG_2 label, EXC_STD, 1 #define STD_EXCEPTION_HV(loc, vec, label) \ EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec) #define STD_EXCEPTION_HV_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ - EXCEPTION_PROLOG_2 label, EXC_HV + EXCEPTION_PROLOG_2 label, EXC_HV, 1 #define STD_RELON_EXCEPTION(loc, vec, label) \ /* No guest interrupts come through here */ \ @@ -670,21 +649,21 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_2 label, h + EXCEPTION_PROLOG_2 label, h, 1 #define MASKABLE_EXCEPTION(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_STD, SOFTEN_TEST_PR, bitmask) #define MASKABLE_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_2 label, EXC_STD + EXCEPTION_PROLOG_2 label, EXC_STD, 1 #define MASKABLE_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) #define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ - EXCEPTION_PROLOG_2 label, EXC_HV + EXCEPTION_PROLOG_2 label, EXC_HV, 1 #define __MASKABLE_RELON_EXCEPTION(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ @@ -697,7 +676,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define MASKABLE_RELON_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_2 label, EXC_STD + EXCEPTION_PROLOG_2 label, EXC_STD, 1 #define MASKABLE_RELON_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 13ae6b8d5e3e..f562b9a83b01 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -356,7 +356,7 @@ machine_check_pSeries_0: * nested machine check corrupts it. machine_check_common enables * MSR_RI. */ - EXCEPTION_PROLOG_2_NORI machine_check_common, EXC_STD + EXCEPTION_PROLOG_2 machine_check_common, EXC_STD, 0 TRAMP_KVM_SKIP(PACA_EXMC, 0x200) @@ -598,7 +598,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x300) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2 data_access_common, EXC_STD +EXCEPTION_PROLOG_2 data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) SET_SCRATCH0(r13) /* save r13 */ @@ -645,7 +645,7 @@ TRAMP_REAL_BEGIN(tramp_real_data_access_slb) EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380) mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) -EXCEPTION_PROLOG_2 data_access_slb_common, EXC_STD +EXCEPTION_PROLOG_2 data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) SET_SCRATCH0(r13) /* save r13 */ @@ -774,7 +774,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x600) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2 alignment_common, EXC_STD +EXCEPTION_PROLOG_2 alignment_common, EXC_STD, 1 EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) @@ -1320,7 +1320,7 @@ EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) #endif KVMTEST_HV(0x1500) - EXCEPTION_PROLOG_2 denorm_common, EXC_HV + EXCEPTION_PROLOG_2 denorm_common, EXC_HV, 1 EXC_REAL_END(denorm_exception_hv, 0x1500, 0x100) #ifdef CONFIG_PPC_DENORMALISATION @@ -1442,7 +1442,7 @@ EXC_VIRT_NONE(0x5800, 0x100) std r12,PACA_EXGEN+EX_R12(r13); \ GET_SCRATCH0(r10); \ std r10,PACA_EXGEN+EX_R13(r13); \ - EXCEPTION_PROLOG_2 soft_nmi_common, _H + EXCEPTION_PROLOG_2 soft_nmi_common, _H, 1 /* * Branch to soft_nmi_interrupt using the emergency stack. The emergency -- cgit v1.2.3 From 2d046308d0747848394a0c745d12af8122061792 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:13 +1000 Subject: powerpc/64s/exception: move and tidy EXCEPTION_PROLOG_2 variants - Re-name the macros to _REAL and _VIRT suffixes rather than no and _RELON suffix. - Move the macro definitions together in the file. - Move RELOCATABLE ifdef inside the _VIRT macro. Further consolidation between variants does not buy much here. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 87 +++++++++++++++----------------- arch/powerpc/kernel/exceptions-64s.S | 18 +++---- 2 files changed, 51 insertions(+), 54 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 82a1c0576fbb..3b0e736481fc 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -174,8 +174,33 @@ ori reg,reg,(ABS_ADDR(label))@l; \ addis reg,reg,(ABS_ADDR(label))@h +.macro EXCEPTION_PROLOG_2_REAL label, hsrr, set_ri + ld r10,PACAKMSR(r13) /* get MSR value for kernel */ + .if ! \set_ri + xori r10,r10,MSR_RI /* Clear MSR_RI */ + .endif + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + .endif + LOAD_HANDLER(r12, \label\()) + .if \hsrr + mtspr SPRN_HSRR0,r12 + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + mtspr SPRN_HSRR1,r10 + HRFI_TO_KERNEL + .else + mtspr SPRN_SRR0,r12 + mfspr r12,SPRN_SRR1 /* and SRR1 */ + mtspr SPRN_SRR1,r10 + RFI_TO_KERNEL + .endif + b . /* prevent speculative execution */ +.endm + +.macro EXCEPTION_PROLOG_2_VIRT label, hsrr #ifdef CONFIG_RELOCATABLE -.macro EXCEPTION_PROLOG_2_RELON label, hsrr .if \hsrr mfspr r11,SPRN_HSRR0 /* save HSRR0 */ .else @@ -191,10 +216,7 @@ li r10,MSR_RI mtmsrd r10,1 /* Set RI (EE=0) */ bctr -.endm #else -/* If not relocatable, we can jump directly -- and save messing with LR */ -.macro EXCEPTION_PROLOG_2_RELON label, hsrr .if \hsrr mfspr r11,SPRN_HSRR0 /* save HSRR0 */ mfspr r12,SPRN_HSRR1 /* and HSRR1 */ @@ -205,19 +227,19 @@ li r10,MSR_RI mtmsrd r10,1 /* Set RI (EE=0) */ b \label -.endm #endif +.endm /* * As EXCEPTION_PROLOG(), except we've already got relocation on so no need to - * rfid. Save LR in case we're CONFIG_RELOCATABLE, in which case - * EXCEPTION_PROLOG_2_RELON will be using LR. + * rfid. Save CTR in case we're CONFIG_RELOCATABLE, in which case + * EXCEPTION_PROLOG_2_VIRT will be using CTR. */ #define EXCEPTION_RELON_PROLOG(area, label, hsrr, extra, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2_RELON label, hsrr + EXCEPTION_PROLOG_2_VIRT label, hsrr /* Exception register prefixes */ #define EXC_HV 1 @@ -323,36 +345,11 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG_1(area, extra, vec) \ _EXCEPTION_PROLOG_1(area, extra, vec) -.macro EXCEPTION_PROLOG_2 label, hsrr, set_ri - ld r10,PACAKMSR(r13) /* get MSR value for kernel */ - .if ! \set_ri - xori r10,r10,MSR_RI /* Clear MSR_RI */ - .endif - .if \hsrr - mfspr r11,SPRN_HSRR0 /* save HSRR0 */ - .else - mfspr r11,SPRN_SRR0 /* save SRR0 */ - .endif - LOAD_HANDLER(r12,\label\()) - .if \hsrr - mtspr SPRN_HSRR0,r12 - mfspr r12,SPRN_HSRR1 /* and HSRR1 */ - mtspr SPRN_HSRR1,r10 - HRFI_TO_KERNEL - .else - mtspr SPRN_SRR0,r12 - mfspr r12,SPRN_SRR1 /* and SRR1 */ - mtspr SPRN_SRR1,r10 - RFI_TO_KERNEL - .endif - b . /* prevent speculative execution */ -.endm - #define EXCEPTION_PROLOG(area, label, h, extra, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2 label, h, 1 + EXCEPTION_PROLOG_2_REAL label, h, 1 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* @@ -421,7 +418,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG_NORI(area, label, h, extra, vec) \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2 label, h, 0 + EXCEPTION_PROLOG_2_REAL label, h, 0 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER .macro KVMTEST hsrr, n @@ -578,14 +575,14 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define STD_EXCEPTION_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec); \ - EXCEPTION_PROLOG_2 label, EXC_STD, 1 + EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define STD_EXCEPTION_HV(loc, vec, label) \ EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec) #define STD_EXCEPTION_HV_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ - EXCEPTION_PROLOG_2 label, EXC_HV, 1 + EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 #define STD_RELON_EXCEPTION(loc, vec, label) \ /* No guest interrupts come through here */ \ @@ -593,14 +590,14 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define STD_RELON_EXCEPTION_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \ - EXCEPTION_PROLOG_2_RELON label, EXC_STD + EXCEPTION_PROLOG_2_VIRT label, EXC_STD #define STD_RELON_EXCEPTION_HV(loc, vec, label) \ EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec) #define STD_RELON_EXCEPTION_HV_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ - EXCEPTION_PROLOG_2_RELON label, EXC_HV + EXCEPTION_PROLOG_2_VIRT label, EXC_HV .macro SOFTEN_TEST hsrr, vec, bitmask lbz r10, PACAIRQSOFTMASK(r13) @@ -649,41 +646,41 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_2 label, h, 1 + EXCEPTION_PROLOG_2_REAL label, h, 1 #define MASKABLE_EXCEPTION(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_STD, SOFTEN_TEST_PR, bitmask) #define MASKABLE_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_2 label, EXC_STD, 1 + EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define MASKABLE_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) #define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ - EXCEPTION_PROLOG_2 label, EXC_HV, 1 + EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 #define __MASKABLE_RELON_EXCEPTION(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_2_RELON label, h + EXCEPTION_PROLOG_2_VIRT label, h #define MASKABLE_RELON_EXCEPTION(vec, label, bitmask) \ __MASKABLE_RELON_EXCEPTION(vec, label, EXC_STD, SOFTEN_NOTEST_PR, bitmask) #define MASKABLE_RELON_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_2 label, EXC_STD, 1 + EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define MASKABLE_RELON_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ - EXCEPTION_PROLOG_2_RELON label, EXC_HV + EXCEPTION_PROLOG_2_VIRT label, EXC_HV /* * Our exception common code can be passed various "additions" diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index f562b9a83b01..38e21d671c3b 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -356,7 +356,7 @@ machine_check_pSeries_0: * nested machine check corrupts it. machine_check_common enables * MSR_RI. */ - EXCEPTION_PROLOG_2 machine_check_common, EXC_STD, 0 + EXCEPTION_PROLOG_2_REAL machine_check_common, EXC_STD, 0 TRAMP_KVM_SKIP(PACA_EXMC, 0x200) @@ -598,7 +598,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x300) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2 data_access_common, EXC_STD, 1 +EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) SET_SCRATCH0(r13) /* save r13 */ @@ -608,7 +608,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x300) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2_RELON data_access_common, EXC_STD +EXCEPTION_PROLOG_2_VIRT data_access_common, EXC_STD EXC_VIRT_END(data_access, 0x4300, 0x80) TRAMP_KVM_SKIP(PACA_EXGEN, 0x300) @@ -645,7 +645,7 @@ TRAMP_REAL_BEGIN(tramp_real_data_access_slb) EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380) mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) -EXCEPTION_PROLOG_2 data_access_slb_common, EXC_STD, 1 +EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) SET_SCRATCH0(r13) /* save r13 */ @@ -653,7 +653,7 @@ EXCEPTION_PROLOG_0(PACA_EXSLB) EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380) mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) -EXCEPTION_PROLOG_2_RELON data_access_slb_common, EXC_STD +EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD EXC_VIRT_END(data_access_slb, 0x4380, 0x80) TRAMP_KVM_SKIP(PACA_EXSLB, 0x380) @@ -774,7 +774,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x600) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2 alignment_common, EXC_STD, 1 +EXCEPTION_PROLOG_2_REAL alignment_common, EXC_STD, 1 EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) @@ -785,7 +785,7 @@ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x600) mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2_RELON alignment_common, EXC_STD +EXCEPTION_PROLOG_2_VIRT alignment_common, EXC_STD EXC_VIRT_END(alignment, 0x4600, 0x100) TRAMP_KVM(PACA_EXGEN, 0x600) @@ -1320,7 +1320,7 @@ EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) #endif KVMTEST_HV(0x1500) - EXCEPTION_PROLOG_2 denorm_common, EXC_HV, 1 + EXCEPTION_PROLOG_2_REAL denorm_common, EXC_HV, 1 EXC_REAL_END(denorm_exception_hv, 0x1500, 0x100) #ifdef CONFIG_PPC_DENORMALISATION @@ -1442,7 +1442,7 @@ EXC_VIRT_NONE(0x5800, 0x100) std r12,PACA_EXGEN+EX_R12(r13); \ GET_SCRATCH0(r10); \ std r10,PACA_EXGEN+EX_R13(r13); \ - EXCEPTION_PROLOG_2 soft_nmi_common, _H, 1 + EXCEPTION_PROLOG_2_REAL soft_nmi_common, _H, 1 /* * Branch to soft_nmi_interrupt using the emergency stack. The emergency -- cgit v1.2.3 From 8f528359efbbf05a7ce657ea31b3c80b7a742269 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:14 +1000 Subject: powerpc/64s/exception: fix sreset KVM test code The sreset handler KVM test theoretically should not depend on P7. In practice KVM now only supports P7 and up so no real bug fix, but this change is made now so the quirk is not propagated through cleanup patches. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 38e21d671c3b..08096d3fd5bb 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -126,10 +126,10 @@ EXC_VIRT_NONE(0x4000, 0x100) bltlr cr1 ; /* no state loss, return to idle caller */ \ BRANCH_TO_C000(r10, system_reset_idle_common) ; \ 1: \ - KVMTEST_PR(n) ; \ - END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) + END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) ; \ + KVMTEST_PR(n) #else -#define IDLETEST NOTEST +#define IDLETEST KVMTEST_PR #endif EXC_REAL_BEGIN(system_reset, 0x100, 0x100) -- cgit v1.2.3 From a7c1ca19c295962eb8e1f714416a930daa582a90 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:15 +1000 Subject: powerpc/64s/exception: remove the "extra" macro parameter Rather than pass in the soft-masking and KVM tests via macro that is passed to another macro to expand it, switch to usig gas macros and conditionally expand the soft-masking and KVM tests. The system reset with its idle test is open coded as it is a one-off. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 158 ++++++++++++++----------------- arch/powerpc/kernel/exceptions-64s.S | 78 ++++++++------- 2 files changed, 114 insertions(+), 122 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 3b0e736481fc..0a29bf2db198 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -235,10 +235,10 @@ * rfid. Save CTR in case we're CONFIG_RELOCATABLE, in which case * EXCEPTION_PROLOG_2_VIRT will be using CTR. */ -#define EXCEPTION_RELON_PROLOG(area, label, hsrr, extra, vec) \ +#define EXCEPTION_RELON_PROLOG(area, label, hsrr, kvm, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1(area, extra, vec); \ + EXCEPTION_PROLOG_1 hsrr, area, kvm, vec ; \ EXCEPTION_PROLOG_2_VIRT label, hsrr /* Exception register prefixes */ @@ -325,31 +325,58 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) /* * This version of the EXCEPTION_PROLOG_1 will carry * addition parameter called "bitmask" to support - * checking of the interrupt maskable level in the SOFTEN_TEST. + * checking of the interrupt maskable level. * Intended to be used in MASKABLE_EXCPETION_* macros. */ -#define MASKABLE_EXCEPTION_PROLOG_1(area, extra, vec, bitmask) \ - __EXCEPTION_PROLOG_1_PRE(area); \ - extra(vec, bitmask); \ - __EXCEPTION_PROLOG_1_POST(area) +.macro MASKABLE_EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask + __EXCEPTION_PROLOG_1_PRE(\area\()) + .if \kvm + KVMTEST \hsrr \vec + .endif + + lbz r10,PACAIRQSOFTMASK(r13) + andi. r10,r10,\bitmask + /* This associates vector numbers with bits in paca->irq_happened */ + .if \vec == 0x500 || \vec == 0xea0 + li r10,PACA_IRQ_EE + .elseif \vec == 0x900 + li r10,PACA_IRQ_DEC + .elseif \vec == 0xa00 || \vec == 0xe80 + li r10,PACA_IRQ_DBELL + .elseif \vec == 0xe60 + li r10,PACA_IRQ_HMI + .elseif \vec == 0xf00 + li r10,PACA_IRQ_PMI + .else + .abort "Bad maskable vector" + .endif + + .if \hsrr + bne masked_Hinterrupt + .else + bne masked_interrupt + .endif + + __EXCEPTION_PROLOG_1_POST(\area\()) +.endm /* * This version of the EXCEPTION_PROLOG_1 is intended * to be used in STD_EXCEPTION* macros */ -#define _EXCEPTION_PROLOG_1(area, extra, vec) \ - __EXCEPTION_PROLOG_1_PRE(area); \ - extra(vec); \ - __EXCEPTION_PROLOG_1_POST(area) - -#define EXCEPTION_PROLOG_1(area, extra, vec) \ - _EXCEPTION_PROLOG_1(area, extra, vec) +.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec + __EXCEPTION_PROLOG_1_PRE(\area\()) + .if \kvm + KVMTEST \hsrr \vec + .endif + __EXCEPTION_PROLOG_1_POST(\area\()) +.endm -#define EXCEPTION_PROLOG(area, label, h, extra, vec) \ +#define EXCEPTION_PROLOG(area, label, hsrr, kvm, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2_REAL label, h, 1 + EXCEPTION_PROLOG_1 hsrr, area, kvm, vec ; \ + EXCEPTION_PROLOG_2_REAL label, hsrr, 1 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* @@ -415,10 +442,10 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #endif /* Do not enable RI */ -#define EXCEPTION_PROLOG_NORI(area, label, h, extra, vec) \ +#define EXCEPTION_PROLOG_NORI(area, label, hsrr, kvm, vec) \ EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_2_REAL label, h, 0 + EXCEPTION_PROLOG_1 hsrr, area, kvm, vec ; \ + EXCEPTION_PROLOG_2_REAL label, hsrr, 0 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER .macro KVMTEST hsrr, n @@ -480,8 +507,6 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) .endm #endif -#define NOTEST(n) - #define EXCEPTION_PROLOG_COMMON_1() \ std r9,_CCR(r1); /* save CR in stackframe */ \ std r11,_NIP(r1); /* save SRR0 in stackframe */ \ @@ -565,7 +590,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) * Exception vectors. */ #define STD_EXCEPTION(vec, label) \ - EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_STD, KVMTEST_PR, vec); + EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_STD, 1, vec); /* Version of above for when we have to branch out-of-line */ #define __OOL_EXCEPTION(vec, label, hdlr) \ @@ -574,112 +599,69 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) b hdlr #define STD_EXCEPTION_OOL(vec, label) \ - EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec); \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec ; \ EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define STD_EXCEPTION_HV(loc, vec, label) \ - EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec) + EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, 1, vec) #define STD_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec ; \ EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 #define STD_RELON_EXCEPTION(loc, vec, label) \ /* No guest interrupts come through here */ \ - EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_STD, NOTEST, vec) + EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_STD, 0, vec) #define STD_RELON_EXCEPTION_OOL(vec, label) \ - EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec ; \ EXCEPTION_PROLOG_2_VIRT label, EXC_STD -#define STD_RELON_EXCEPTION_HV(loc, vec, label) \ - EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec) +#define STD_RELON_EXCEPTION_HV(loc, vec, label) \ + EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_HV, 1, vec) #define STD_RELON_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec; \ EXCEPTION_PROLOG_2_VIRT label, EXC_HV -.macro SOFTEN_TEST hsrr, vec, bitmask - lbz r10, PACAIRQSOFTMASK(r13) - andi. r10, r10, \bitmask - /* This associates vector numbers with bits in paca->irq_happened */ - .if \vec == 0x500 || \vec == 0xea0 - li r10, PACA_IRQ_EE - .elseif \vec == 0x900 - li r10, PACA_IRQ_DEC - .elseif \vec == 0xa00 || \vec == 0xe80 - li r10, PACA_IRQ_DBELL - .elseif \vec == 0xe60 - li r10, PACA_IRQ_HMI - .elseif \vec == 0xf00 - li r10, PACA_IRQ_PMI - .else - .abort "Bad maskable vector" - .endif - - - .if \hsrr - bne masked_Hinterrupt - .else - bne masked_interrupt - .endif -.endm - -#define SOFTEN_TEST_PR(vec, bitmask) \ - KVMTEST EXC_STD, vec ; \ - SOFTEN_TEST EXC_STD, vec, bitmask - -#define SOFTEN_TEST_HV(vec, bitmask) \ - KVMTEST EXC_HV, vec ; \ - SOFTEN_TEST EXC_HV, vec, bitmask - -#define KVMTEST_PR(vec) \ - KVMTEST EXC_STD, vec - -#define KVMTEST_HV(vec) \ - KVMTEST EXC_HV, vec - -#define SOFTEN_NOTEST_PR(vec, bitmask) SOFTEN_TEST EXC_STD, vec, bitmask -#define SOFTEN_NOTEST_HV(vec, bitmask) SOFTEN_TEST EXC_HV, vec, bitmask - -#define __MASKABLE_EXCEPTION(vec, label, h, extra, bitmask) \ +#define __MASKABLE_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ - MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_2_REAL label, h, 1 + MASKABLE_EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ + EXCEPTION_PROLOG_2_REAL label, hsrr, 1 #define MASKABLE_EXCEPTION(vec, label, bitmask) \ - __MASKABLE_EXCEPTION(vec, label, EXC_STD, SOFTEN_TEST_PR, bitmask) + __MASKABLE_EXCEPTION(vec, label, EXC_STD, 1, bitmask) #define MASKABLE_EXCEPTION_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\ + MASKABLE_EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define MASKABLE_EXCEPTION_HV(vec, label, bitmask) \ - __MASKABLE_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) + __MASKABLE_EXCEPTION(vec, label, EXC_HV, 1, bitmask) #define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ + MASKABLE_EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 -#define __MASKABLE_RELON_EXCEPTION(vec, label, h, extra, bitmask) \ +#define __MASKABLE_RELON_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ - MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_2_VIRT label, h + MASKABLE_EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ + EXCEPTION_PROLOG_2_VIRT label, hsrr #define MASKABLE_RELON_EXCEPTION(vec, label, bitmask) \ - __MASKABLE_RELON_EXCEPTION(vec, label, EXC_STD, SOFTEN_NOTEST_PR, bitmask) + __MASKABLE_RELON_EXCEPTION(vec, label, EXC_STD, 0, bitmask) #define MASKABLE_RELON_EXCEPTION_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\ + MASKABLE_EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define MASKABLE_RELON_EXCEPTION_HV(vec, label, bitmask) \ - __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) + __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, 1, bitmask) #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ + MASKABLE_EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ EXCEPTION_PROLOG_2_VIRT label, EXC_HV /* diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 08096d3fd5bb..23f5c39063ed 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -107,6 +107,17 @@ __start_interrupts: EXC_VIRT_NONE(0x4000, 0x100) +EXC_REAL_BEGIN(system_reset, 0x100, 0x100) + SET_SCRATCH0(r13) + EXCEPTION_PROLOG_0(PACA_EXNMI) + + /* This is EXCEPTION_PROLOG_1 with the idle feature section added */ + OPT_SAVE_REG_TO_PACA(PACA_EXNMI+EX_PPR, r9, CPU_FTR_HAS_PPR) + OPT_SAVE_REG_TO_PACA(PACA_EXNMI+EX_CFAR, r10, CPU_FTR_CFAR) + INTERRUPT_TO_KERNEL + SAVE_CTR(r10, PACA_EXNMI) + mfcr r9 + #ifdef CONFIG_PPC_P7_NAP /* * If running native on arch 2.06 or later, check if we are waking up @@ -116,30 +127,29 @@ EXC_VIRT_NONE(0x4000, 0x100) * but we branch to the 0xc000... address so we can turn on relocation * with mtmsr. */ -#define IDLETEST(n) \ - BEGIN_FTR_SECTION ; \ - mfspr r10,SPRN_SRR1 ; \ - rlwinm. r10,r10,47-31,30,31 ; \ - beq- 1f ; \ - cmpwi cr1,r10,2 ; \ - mfspr r3,SPRN_SRR1 ; \ - bltlr cr1 ; /* no state loss, return to idle caller */ \ - BRANCH_TO_C000(r10, system_reset_idle_common) ; \ -1: \ - END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) ; \ - KVMTEST_PR(n) -#else -#define IDLETEST KVMTEST_PR + BEGIN_FTR_SECTION + mfspr r10,SPRN_SRR1 + rlwinm. r10,r10,47-31,30,31 + beq- 1f + cmpwi cr1,r10,2 + mfspr r3,SPRN_SRR1 + bltlr cr1 /* no state loss, return to idle caller */ + BRANCH_TO_C000(r10, system_reset_idle_common) +1: + END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) #endif -EXC_REAL_BEGIN(system_reset, 0x100, 0x100) - SET_SCRATCH0(r13) + KVMTEST EXC_STD 0x100 + std r11,PACA_EXNMI+EX_R11(r13) + std r12,PACA_EXNMI+EX_R12(r13) + GET_SCRATCH0(r10) + std r10,PACA_EXNMI+EX_R13(r13) + + EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0 /* * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is * being used, so a nested NMI exception would corrupt it. */ - EXCEPTION_PROLOG_NORI(PACA_EXNMI, system_reset_common, EXC_STD, - IDLETEST, 0x100) EXC_REAL_END(system_reset, 0x100, 0x100) EXC_VIRT_NONE(0x4100, 0x100) @@ -246,7 +256,7 @@ TRAMP_REAL_BEGIN(system_reset_fwnmi) SET_SCRATCH0(r13) /* save r13 */ /* See comment at system_reset exception */ EXCEPTION_PROLOG_NORI(PACA_EXNMI, system_reset_common, EXC_STD, - NOTEST, 0x100) + 0, 0x100) #endif /* CONFIG_PPC_PSERIES */ @@ -265,7 +275,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_REAL_END(machine_check, 0x200, 0x100) EXC_VIRT_NONE(0x4200, 0x100) TRAMP_REAL_BEGIN(machine_check_common_early) - EXCEPTION_PROLOG_1(PACA_EXMC, NOTEST, 0x200) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 0, 0x200 /* * Register contents: * R13 = PACA @@ -350,7 +360,7 @@ BEGIN_FTR_SECTION b machine_check_common_early END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) machine_check_pSeries_0: - EXCEPTION_PROLOG_1(PACA_EXMC, KVMTEST_PR, 0x200) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200 /* * MSR_RI is not enabled, because PACA_EXMC is being used, so a * nested machine check corrupts it. machine_check_common enables @@ -588,7 +598,7 @@ EXCEPTION_PROLOG_0(PACA_EXGEN) EXC_REAL_END(data_access, 0x300, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access) -EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x300) +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300 /* * DAR/DSISR must be read before setting MSR[RI], because * a d-side MCE will clobber those registers so is not @@ -603,7 +613,7 @@ EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXGEN) -EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x300) +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) @@ -642,7 +652,7 @@ EXCEPTION_PROLOG_0(PACA_EXSLB) EXC_REAL_END(data_access_slb, 0x380, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access_slb) -EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380) +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380 mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 @@ -650,7 +660,7 @@ EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXSLB) -EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380) +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380 mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD @@ -705,11 +715,11 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) -EXCEPTION_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, KVMTEST_PR, 0x480); +EXCEPTION_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, 1, 0x480); EXC_REAL_END(instruction_access_slb, 0x480, 0x80) EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) -EXCEPTION_RELON_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, NOTEST, 0x480); +EXCEPTION_RELON_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, 0, 0x480); EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) TRAMP_KVM(PACA_EXSLB, 0x480) @@ -757,7 +767,7 @@ hardware_interrupt_relon_hv: IRQS_DISABLED) FTR_SECTION_ELSE __MASKABLE_RELON_EXCEPTION(0x500, hardware_interrupt_common, - EXC_STD, SOFTEN_TEST_PR, IRQS_DISABLED) + EXC_STD, 1, IRQS_DISABLED) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) @@ -769,7 +779,7 @@ EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ) EXC_REAL_BEGIN(alignment, 0x600, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXGEN) -EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, 0x600) +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) @@ -780,7 +790,7 @@ EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXGEN) -EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x600) +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) @@ -946,7 +956,7 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception) GET_PACA(r13); \ std r10,PACA_EXGEN+EX_R10(r13); \ INTERRUPT_TO_KERNEL; \ - KVMTEST_PR(0xc00); /* uses r10, branch to do_kvm_0xc00_system_call */ \ + KVMTEST EXC_STD 0xc00 ; /* uses r10, branch to do_kvm_0xc00_system_call */ \ HMT_MEDIUM; \ mfctr r9; @@ -1109,7 +1119,7 @@ __TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED) EXC_VIRT_NONE(0x4e60, 0x20) TRAMP_KVM_HV(PACA_EXGEN, 0xe60) TRAMP_REAL_BEGIN(hmi_exception_early) - EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, 0xe60) + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60 mr r10,r1 /* Save r1 */ ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ @@ -1311,7 +1321,7 @@ EXC_VIRT_NONE(0x5400, 0x100) EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) mtspr SPRN_SPRG_HSCRATCH0,r13 EXCEPTION_PROLOG_0(PACA_EXGEN) - EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, 0x1500) + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500 #ifdef CONFIG_PPC_DENORMALISATION mfspr r10,SPRN_HSRR1 @@ -1319,7 +1329,7 @@ EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) bne+ denorm_assist #endif - KVMTEST_HV(0x1500) + KVMTEST EXC_HV 0x1500 EXCEPTION_PROLOG_2_REAL denorm_common, EXC_HV, 1 EXC_REAL_END(denorm_exception_hv, 0x1500, 0x100) -- cgit v1.2.3 From fa4cf6b703f4d63b9a422e24332266ef7efae7b6 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:16 +1000 Subject: powerpc/64s/exception: consolidate maskable and non-maskable prologs Conditionally expand the soft-masking test if a mask is passed in. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 112 +++++++++++++------------------ arch/powerpc/kernel/exceptions-64s.S | 20 +++--- 2 files changed, 55 insertions(+), 77 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 0a29bf2db198..4111888e04d7 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -238,7 +238,7 @@ #define EXCEPTION_RELON_PROLOG(area, label, hsrr, kvm, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1 hsrr, area, kvm, vec ; \ + EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ EXCEPTION_PROLOG_2_VIRT label, hsrr /* Exception register prefixes */ @@ -309,73 +309,51 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) std r10,area+EX_R10(r13); /* save r10 - r12 */ \ OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR) -#define __EXCEPTION_PROLOG_1_PRE(area) \ - OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR); \ - OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR); \ - INTERRUPT_TO_KERNEL; \ - SAVE_CTR(r10, area); \ +.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask + OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR) + OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR) + INTERRUPT_TO_KERNEL + SAVE_CTR(r10, \area\()) mfcr r9 - -#define __EXCEPTION_PROLOG_1_POST(area) \ - std r11,area+EX_R11(r13); \ - std r12,area+EX_R12(r13); \ - GET_SCRATCH0(r10); \ - std r10,area+EX_R13(r13) - -/* - * This version of the EXCEPTION_PROLOG_1 will carry - * addition parameter called "bitmask" to support - * checking of the interrupt maskable level. - * Intended to be used in MASKABLE_EXCPETION_* macros. - */ -.macro MASKABLE_EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask - __EXCEPTION_PROLOG_1_PRE(\area\()) .if \kvm KVMTEST \hsrr \vec .endif - lbz r10,PACAIRQSOFTMASK(r13) - andi. r10,r10,\bitmask - /* This associates vector numbers with bits in paca->irq_happened */ - .if \vec == 0x500 || \vec == 0xea0 - li r10,PACA_IRQ_EE - .elseif \vec == 0x900 - li r10,PACA_IRQ_DEC - .elseif \vec == 0xa00 || \vec == 0xe80 - li r10,PACA_IRQ_DBELL - .elseif \vec == 0xe60 - li r10,PACA_IRQ_HMI - .elseif \vec == 0xf00 - li r10,PACA_IRQ_PMI - .else - .abort "Bad maskable vector" - .endif - - .if \hsrr - bne masked_Hinterrupt - .else - bne masked_interrupt + .if \bitmask + lbz r10,PACAIRQSOFTMASK(r13) + andi. r10,r10,\bitmask + /* Associate vector numbers with bits in paca->irq_happened */ + .if \vec == 0x500 || \vec == 0xea0 + li r10,PACA_IRQ_EE + .elseif \vec == 0x900 + li r10,PACA_IRQ_DEC + .elseif \vec == 0xa00 || \vec == 0xe80 + li r10,PACA_IRQ_DBELL + .elseif \vec == 0xe60 + li r10,PACA_IRQ_HMI + .elseif \vec == 0xf00 + li r10,PACA_IRQ_PMI + .else + .abort "Bad maskable vector" + .endif + + .if \hsrr + bne masked_Hinterrupt + .else + bne masked_interrupt + .endif .endif - __EXCEPTION_PROLOG_1_POST(\area\()) -.endm - -/* - * This version of the EXCEPTION_PROLOG_1 is intended - * to be used in STD_EXCEPTION* macros - */ -.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec - __EXCEPTION_PROLOG_1_PRE(\area\()) - .if \kvm - KVMTEST \hsrr \vec - .endif - __EXCEPTION_PROLOG_1_POST(\area\()) + std r11,\area\()+EX_R11(r13) + std r12,\area\()+EX_R12(r13) + GET_SCRATCH0(r10) + std r10,\area\()+EX_R13(r13) .endm #define EXCEPTION_PROLOG(area, label, hsrr, kvm, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1 hsrr, area, kvm, vec ; \ + EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ EXCEPTION_PROLOG_2_REAL label, hsrr, 1 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE @@ -444,7 +422,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) /* Do not enable RI */ #define EXCEPTION_PROLOG_NORI(area, label, hsrr, kvm, vec) \ EXCEPTION_PROLOG_0(area); \ - EXCEPTION_PROLOG_1 hsrr, area, kvm, vec ; \ + EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ EXCEPTION_PROLOG_2_REAL label, hsrr, 0 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER @@ -599,14 +577,14 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) b hdlr #define STD_EXCEPTION_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0 ; \ EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define STD_EXCEPTION_HV(loc, vec, label) \ EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, 1, vec) #define STD_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 #define STD_RELON_EXCEPTION(loc, vec, label) \ @@ -614,54 +592,54 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_STD, 0, vec) #define STD_RELON_EXCEPTION_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0 ; \ EXCEPTION_PROLOG_2_VIRT label, EXC_STD #define STD_RELON_EXCEPTION_HV(loc, vec, label) \ EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_HV, 1, vec) #define STD_RELON_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ EXCEPTION_PROLOG_2_VIRT label, EXC_HV #define __MASKABLE_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ - MASKABLE_EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ + EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, hsrr, 1 #define MASKABLE_EXCEPTION(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_STD, 1, bitmask) #define MASKABLE_EXCEPTION_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define MASKABLE_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_EXCEPTION(vec, label, EXC_HV, 1, bitmask) #define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 #define __MASKABLE_RELON_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ - MASKABLE_EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ + EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ EXCEPTION_PROLOG_2_VIRT label, hsrr #define MASKABLE_RELON_EXCEPTION(vec, label, bitmask) \ __MASKABLE_RELON_EXCEPTION(vec, label, EXC_STD, 0, bitmask) #define MASKABLE_RELON_EXCEPTION_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 #define MASKABLE_RELON_EXCEPTION_HV(vec, label, bitmask) \ __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, 1, bitmask) #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \ - MASKABLE_EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ EXCEPTION_PROLOG_2_VIRT label, EXC_HV /* diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 23f5c39063ed..cc356a31d5cc 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -275,7 +275,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_REAL_END(machine_check, 0x200, 0x100) EXC_VIRT_NONE(0x4200, 0x100) TRAMP_REAL_BEGIN(machine_check_common_early) - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 0, 0x200 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 0, 0x200, 0 /* * Register contents: * R13 = PACA @@ -360,7 +360,7 @@ BEGIN_FTR_SECTION b machine_check_common_early END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) machine_check_pSeries_0: - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200, 0 /* * MSR_RI is not enabled, because PACA_EXMC is being used, so a * nested machine check corrupts it. machine_check_common enables @@ -598,7 +598,7 @@ EXCEPTION_PROLOG_0(PACA_EXGEN) EXC_REAL_END(data_access, 0x300, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300 +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 0 /* * DAR/DSISR must be read before setting MSR[RI], because * a d-side MCE will clobber those registers so is not @@ -613,7 +613,7 @@ EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXGEN) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300 +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) @@ -652,7 +652,7 @@ EXCEPTION_PROLOG_0(PACA_EXSLB) EXC_REAL_END(data_access_slb, 0x380, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access_slb) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380 +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380, 0 mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 @@ -660,7 +660,7 @@ EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXSLB) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380 +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 0 mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD @@ -779,7 +779,7 @@ EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ) EXC_REAL_BEGIN(alignment, 0x600, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXGEN) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600 +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) @@ -790,7 +790,7 @@ EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0(PACA_EXGEN) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600 +EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) @@ -1119,7 +1119,7 @@ __TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED) EXC_VIRT_NONE(0x4e60, 0x20) TRAMP_KVM_HV(PACA_EXGEN, 0xe60) TRAMP_REAL_BEGIN(hmi_exception_early) - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60 + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0 mr r10,r1 /* Save r1 */ ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ @@ -1321,7 +1321,7 @@ EXC_VIRT_NONE(0x5400, 0x100) EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) mtspr SPRN_SPRG_HSCRATCH0,r13 EXCEPTION_PROLOG_0(PACA_EXGEN) - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500 + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0 #ifdef CONFIG_PPC_DENORMALISATION mfspr r10,SPRN_HSRR1 -- cgit v1.2.3 From 17bdc064a1fe8638b526ec44e6593685dbfdc714 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:17 +1000 Subject: powerpc/64s/exception: merge KVM handler and skip variants Conditionally expand the skip case if it is specified. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 28 ++++++++++------------------ arch/powerpc/include/asm/head-64.h | 8 ++++---- arch/powerpc/kernel/exceptions-64s.S | 2 +- 3 files changed, 15 insertions(+), 23 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 4111888e04d7..d2a5bbeb727d 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -436,26 +436,17 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) .endif .endm -.macro KVM_HANDLER area, hsrr, n +.macro KVM_HANDLER area, hsrr, n, skip + .if \skip + cmpwi r10,KVM_GUEST_MODE_SKIP + beq 89f + .else BEGIN_FTR_SECTION_NESTED(947) ld r10,\area+EX_CFAR(r13) std r10,HSTATE_CFAR(r13) END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947) - BEGIN_FTR_SECTION_NESTED(948) - ld r10,\area+EX_PPR(r13) - std r10,HSTATE_PPR(r13) - END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) - ld r10,\area+EX_R10(r13) - std r12,HSTATE_SCRATCH0(r13) - sldi r12,r9,32 - ori r12,r12,(\n) - /* This reloads r9 before branching to kvmppc_interrupt */ - __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt) -.endm + .endif -.macro KVM_HANDLER_SKIP area, hsrr, n - cmpwi r10,KVM_GUEST_MODE_SKIP - beq 89f BEGIN_FTR_SECTION_NESTED(948) ld r10,\area+EX_PPR(r13) std r10,HSTATE_PPR(r13) @@ -466,6 +457,8 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) ori r12,r12,(\n) /* This reloads r9 before branching to kvmppc_interrupt */ __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt) + + .if \skip 89: mtocrf 0x80,r9 ld r9,\area+EX_R9(r13) ld r10,\area+EX_R10(r13) @@ -474,14 +467,13 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) .else b kvmppc_skip_interrupt .endif + .endif .endm #else .macro KVMTEST hsrr, n .endm -.macro KVM_HANDLER area, hsrr, n -.endm -.macro KVM_HANDLER_SKIP area, hsrr, n +.macro KVM_HANDLER area, hsrr, n, skip .endm #endif diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index 4767d6c7b8fa..518d9758b41e 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -387,22 +387,22 @@ name: #define TRAMP_KVM(area, n) \ TRAMP_KVM_BEGIN(do_kvm_##n); \ - KVM_HANDLER area, EXC_STD, n + KVM_HANDLER area, EXC_STD, n, 0 #define TRAMP_KVM_SKIP(area, n) \ TRAMP_KVM_BEGIN(do_kvm_##n); \ - KVM_HANDLER_SKIP area, EXC_STD, n + KVM_HANDLER area, EXC_STD, n, 1 /* * HV variant exceptions get the 0x2 bit added to their trap number. */ #define TRAMP_KVM_HV(area, n) \ TRAMP_KVM_BEGIN(do_kvm_H##n); \ - KVM_HANDLER area, EXC_HV, n + 0x2 + KVM_HANDLER area, EXC_HV, n + 0x2, 0 #define TRAMP_KVM_HV_SKIP(area, n) \ TRAMP_KVM_BEGIN(do_kvm_H##n); \ - KVM_HANDLER_SKIP area, EXC_HV, n + 0x2 + KVM_HANDLER area, EXC_HV, n + 0x2, 1 #define EXC_COMMON(name, realvec, hdlr) \ EXC_COMMON_BEGIN(name); \ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index cc356a31d5cc..d30f2b5fd206 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1063,7 +1063,7 @@ TRAMP_KVM_BEGIN(do_kvm_0xc00) SET_SCRATCH0(r10) std r9,PACA_EXGEN+EX_R9(r13) mfcr r9 - KVM_HANDLER PACA_EXGEN, EXC_STD, 0xc00 + KVM_HANDLER PACA_EXGEN, EXC_STD, 0xc00, 0 #endif -- cgit v1.2.3 From 5dba1d50ba4f44a82e7228d6fb83ccdfd1bda0fb Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:19 +1000 Subject: powerpc/64s/exception: Make EXCEPTION_PROLOG_0 a gas macro for consistency with others No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 25 +++++++++++++------------ arch/powerpc/kernel/exceptions-64s.S | 24 ++++++++++++------------ 2 files changed, 25 insertions(+), 24 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index a7b514574b80..663998e411e6 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -237,7 +237,7 @@ */ #define EXCEPTION_RELON_PROLOG(area, label, hsrr, kvm, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0(area); \ + EXCEPTION_PROLOG_0 area ; \ EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ EXCEPTION_PROLOG_2_VIRT label, hsrr @@ -301,13 +301,14 @@ BEGIN_FTR_SECTION_NESTED(943) \ std ra,offset(r13); \ END_FTR_SECTION_NESTED(ftr,ftr,943) -#define EXCEPTION_PROLOG_0(area) \ - GET_PACA(r13); \ - std r9,area+EX_R9(r13); /* save r9 */ \ - OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR); \ - HMT_MEDIUM; \ - std r10,area+EX_R10(r13); /* save r10 - r12 */ \ +.macro EXCEPTION_PROLOG_0 area + GET_PACA(r13) + std r9,\area\()+EX_R9(r13) /* save r9 */ + OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR) + HMT_MEDIUM + std r10,\area\()+EX_R10(r13) /* save r10 - r12 */ OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR) +.endm .macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR) @@ -352,7 +353,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG(area, label, hsrr, kvm, vec) \ SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0(area); \ + EXCEPTION_PROLOG_0 area ; \ EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ EXCEPTION_PROLOG_2_REAL label, hsrr, 1 @@ -421,7 +422,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) /* Do not enable RI */ #define EXCEPTION_PROLOG_NORI(area, label, hsrr, kvm, vec) \ - EXCEPTION_PROLOG_0(area); \ + EXCEPTION_PROLOG_0 area ; \ EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ EXCEPTION_PROLOG_2_REAL label, hsrr, 0 @@ -570,7 +571,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) /* Version of above for when we have to branch out-of-line */ #define __OOL_EXCEPTION(vec, label, hdlr) \ SET_SCRATCH0(r13); \ - EXCEPTION_PROLOG_0(PACA_EXGEN); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ b hdlr #define STD_EXCEPTION_OOL(vec, label) \ @@ -601,7 +602,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define __MASKABLE_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0(PACA_EXGEN); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ EXCEPTION_PROLOG_2_REAL label, hsrr, 1 @@ -621,7 +622,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define __MASKABLE_RELON_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0(PACA_EXGEN); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ EXCEPTION_PROLOG_2_VIRT label, hsrr diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index d30f2b5fd206..99541f7eabde 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -109,7 +109,7 @@ EXC_VIRT_NONE(0x4000, 0x100) EXC_REAL_BEGIN(system_reset, 0x100, 0x100) SET_SCRATCH0(r13) - EXCEPTION_PROLOG_0(PACA_EXNMI) + EXCEPTION_PROLOG_0 PACA_EXNMI /* This is EXCEPTION_PROLOG_1 with the idle feature section added */ OPT_SAVE_REG_TO_PACA(PACA_EXNMI+EX_PPR, r9, CPU_FTR_HAS_PPR) @@ -266,7 +266,7 @@ EXC_REAL_BEGIN(machine_check, 0x200, 0x100) * vector */ SET_SCRATCH0(r13) /* save r13 */ - EXCEPTION_PROLOG_0(PACA_EXMC) + EXCEPTION_PROLOG_0 PACA_EXMC BEGIN_FTR_SECTION b machine_check_common_early FTR_SECTION_ELSE @@ -355,7 +355,7 @@ TRAMP_REAL_BEGIN(machine_check_pSeries) .globl machine_check_fwnmi machine_check_fwnmi: SET_SCRATCH0(r13) /* save r13 */ - EXCEPTION_PROLOG_0(PACA_EXMC) + EXCEPTION_PROLOG_0 PACA_EXMC BEGIN_FTR_SECTION b machine_check_common_early END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) @@ -568,7 +568,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) /* Deliver the machine check to host kernel in V mode. */ MACHINE_CHECK_HANDLER_WINDUP SET_SCRATCH0(r13) /* save r13 */ - EXCEPTION_PROLOG_0(PACA_EXMC) + EXCEPTION_PROLOG_0 PACA_EXMC b machine_check_pSeries_0 EXC_COMMON_BEGIN(unrecover_mce) @@ -593,7 +593,7 @@ EXC_COMMON_BEGIN(mce_return) EXC_REAL_BEGIN(data_access, 0x300, 0x80) SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0(PACA_EXGEN) +EXCEPTION_PROLOG_0 PACA_EXGEN b tramp_real_data_access EXC_REAL_END(data_access, 0x300, 0x80) @@ -612,7 +612,7 @@ EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0(PACA_EXGEN) +EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR @@ -647,7 +647,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0(PACA_EXSLB) +EXCEPTION_PROLOG_0 PACA_EXSLB b tramp_real_data_access_slb EXC_REAL_END(data_access_slb, 0x380, 0x80) @@ -659,7 +659,7 @@ EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0(PACA_EXSLB) +EXCEPTION_PROLOG_0 PACA_EXSLB EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 0 mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) @@ -778,7 +778,7 @@ EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ) EXC_REAL_BEGIN(alignment, 0x600, 0x100) SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0(PACA_EXGEN) +EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR @@ -789,7 +789,7 @@ EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0(PACA_EXGEN) +EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR @@ -1167,7 +1167,7 @@ TRAMP_REAL_BEGIN(hmi_exception_early) .globl hmi_exception_after_realmode hmi_exception_after_realmode: SET_SCRATCH0(r13) - EXCEPTION_PROLOG_0(PACA_EXGEN) + EXCEPTION_PROLOG_0 PACA_EXGEN b tramp_real_hmi_exception EXC_COMMON_BEGIN(hmi_exception_common) @@ -1320,7 +1320,7 @@ EXC_VIRT_NONE(0x5400, 0x100) EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) mtspr SPRN_SPRG_HSCRATCH0,r13 - EXCEPTION_PROLOG_0(PACA_EXGEN) + EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0 #ifdef CONFIG_PPC_DENORMALISATION -- cgit v1.2.3 From c06075f3d3fc4d9be7f3f9d9832b4a387037cc26 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:20 +1000 Subject: powerpc/64s/exception: Move EXCEPTION_COMMON handler and return branches into callers The aim is to reduce the amount of indirection it takes to get through the exception handler macros, particularly where it provides little code sharing. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 26 +++++++++++++------------- arch/powerpc/kernel/exceptions-64s.S | 21 +++++++++++++-------- 2 files changed, 26 insertions(+), 21 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 663998e411e6..c98e274b5fba 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -663,31 +663,28 @@ BEGIN_FTR_SECTION \ beql ppc64_runlatch_on_trampoline; \ END_FTR_SECTION_IFSET(CPU_FTR_CTRL) -#define EXCEPTION_COMMON(area, trap, label, hdlr, ret, additions) \ +#define EXCEPTION_COMMON(area, trap, label, additions) \ EXCEPTION_PROLOG_COMMON(trap, area); \ /* Volatile regs are potentially clobbered here */ \ - additions; \ - addi r3,r1,STACK_FRAME_OVERHEAD; \ - bl hdlr; \ - b ret + additions /* * Exception where stack is already set in r1, r1 is saved in r10, and it * continues rather than returns. */ -#define EXCEPTION_COMMON_NORET_STACK(area, trap, label, hdlr, additions) \ +#define EXCEPTION_COMMON_NORET_STACK(area, trap, label, additions) \ EXCEPTION_PROLOG_COMMON_1(); \ kuap_save_amr_and_lock r9, r10, cr1; \ EXCEPTION_PROLOG_COMMON_2(area); \ EXCEPTION_PROLOG_COMMON_3(trap); \ /* Volatile regs are potentially clobbered here */ \ - additions; \ - addi r3,r1,STACK_FRAME_OVERHEAD; \ - bl hdlr + additions #define STD_EXCEPTION_COMMON(trap, label, hdlr) \ - EXCEPTION_COMMON(PACA_EXGEN, trap, label, hdlr, \ - ret_from_except, ADD_NVGPRS;ADD_RECONCILE) + EXCEPTION_COMMON(PACA_EXGEN, trap, label, ADD_NVGPRS;ADD_RECONCILE); \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + bl hdlr; \ + b ret_from_except /* * Like STD_EXCEPTION_COMMON, but for exceptions that can occur @@ -695,8 +692,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_CTRL) * (finish nap and runlatch) */ #define STD_EXCEPTION_COMMON_ASYNC(trap, label, hdlr) \ - EXCEPTION_COMMON(PACA_EXGEN, trap, label, hdlr, \ - ret_from_except_lite, FINISH_NAP;ADD_RECONCILE;RUNLATCH_ON) + EXCEPTION_COMMON(PACA_EXGEN, trap, label, \ + FINISH_NAP;ADD_RECONCILE;RUNLATCH_ON); \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + bl hdlr; \ + b ret_from_except_lite /* * When the idle code in power4_idle puts the CPU into NAP mode, diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 99541f7eabde..7cf0e38875c3 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -195,9 +195,10 @@ EXC_COMMON_BEGIN(system_reset_common) mr r10,r1 ld r1,PACA_NMI_EMERG_SP(r13) subi r1,r1,INT_FRAME_SIZE - EXCEPTION_COMMON_NORET_STACK(PACA_EXNMI, 0x100, - system_reset, system_reset_exception, - ADD_NVGPRS;ADD_RECONCILE_NMI) + EXCEPTION_COMMON_NORET_STACK(PACA_EXNMI, 0x100, system_reset, + ADD_NVGPRS;ADD_RECONCILE_NMI) + addi r3,r1,STACK_FRAME_OVERHEAD + bl system_reset_exception /* This (and MCE) can be simplified with mtmsrd L=1 */ /* Clear MSR_RI before setting SRR0 and SRR1. */ @@ -1171,8 +1172,11 @@ hmi_exception_after_realmode: b tramp_real_hmi_exception EXC_COMMON_BEGIN(hmi_exception_common) -EXCEPTION_COMMON(PACA_EXGEN, 0xe60, hmi_exception_common, handle_hmi_exception, - ret_from_except, FINISH_NAP;ADD_NVGPRS;ADD_RECONCILE;RUNLATCH_ON) +EXCEPTION_COMMON(PACA_EXGEN, 0xe60, hmi_exception_common, + FINISH_NAP;ADD_NVGPRS;ADD_RECONCILE;RUNLATCH_ON) + addi r3,r1,STACK_FRAME_OVERHEAD + bl handle_hmi_exception + b ret_from_except EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20, IRQS_DISABLED) EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80, IRQS_DISABLED) @@ -1467,9 +1471,10 @@ EXC_COMMON_BEGIN(soft_nmi_common) mr r10,r1 ld r1,PACAEMERGSP(r13) subi r1,r1,INT_FRAME_SIZE - EXCEPTION_COMMON_NORET_STACK(PACA_EXGEN, 0x900, - system_reset, soft_nmi_interrupt, - ADD_NVGPRS;ADD_RECONCILE) + EXCEPTION_COMMON_NORET_STACK(PACA_EXGEN, 0x900, system_reset, + ADD_NVGPRS;ADD_RECONCILE) + addi r3,r1,STACK_FRAME_OVERHEAD + bl soft_nmi_interrupt b ret_from_except #else /* CONFIG_PPC_WATCHDOG */ -- cgit v1.2.3 From 47169fba3af465c995a936e6b9c67e0746f4c583 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:21 +1000 Subject: powerpc/64s/exception: Move EXCEPTION_COMMON additions into callers More cases of code insertion via macros that does not add a great deal. All the additions have to be specified in the macro arguments, so they can just as well go after the macro. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 42 +++++++++-------------------- arch/powerpc/include/asm/head-64.h | 4 +-- arch/powerpc/kernel/exceptions-64s.S | 45 +++++++++++++++++--------------- 3 files changed, 39 insertions(+), 52 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index c98e274b5fba..127ebd8d7746 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -640,21 +640,6 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ EXCEPTION_PROLOG_2_VIRT label, EXC_HV -/* - * Our exception common code can be passed various "additions" - * to specify the behaviour of interrupts, whether to kick the - * runlatch, etc... - */ - -/* - * This addition reconciles our actual IRQ state with the various software - * flags that track it. This may call C code. - */ -#define ADD_RECONCILE RECONCILE_IRQ_STATE(r10,r11) - -#define ADD_NVGPRS \ - bl save_nvgprs - #define RUNLATCH_ON \ BEGIN_FTR_SECTION \ ld r3, PACA_THREAD_INFO(r13); \ @@ -663,25 +648,22 @@ BEGIN_FTR_SECTION \ beql ppc64_runlatch_on_trampoline; \ END_FTR_SECTION_IFSET(CPU_FTR_CTRL) -#define EXCEPTION_COMMON(area, trap, label, additions) \ +#define EXCEPTION_COMMON(area, trap) \ EXCEPTION_PROLOG_COMMON(trap, area); \ - /* Volatile regs are potentially clobbered here */ \ - additions /* - * Exception where stack is already set in r1, r1 is saved in r10, and it - * continues rather than returns. + * Exception where stack is already set in r1, r1 is saved in r10 */ -#define EXCEPTION_COMMON_NORET_STACK(area, trap, label, additions) \ +#define EXCEPTION_COMMON_STACK(area, trap) \ EXCEPTION_PROLOG_COMMON_1(); \ kuap_save_amr_and_lock r9, r10, cr1; \ EXCEPTION_PROLOG_COMMON_2(area); \ - EXCEPTION_PROLOG_COMMON_3(trap); \ - /* Volatile regs are potentially clobbered here */ \ - additions + EXCEPTION_PROLOG_COMMON_3(trap) -#define STD_EXCEPTION_COMMON(trap, label, hdlr) \ - EXCEPTION_COMMON(PACA_EXGEN, trap, label, ADD_NVGPRS;ADD_RECONCILE); \ +#define STD_EXCEPTION_COMMON(trap, hdlr) \ + EXCEPTION_COMMON(PACA_EXGEN, trap); \ + bl save_nvgprs; \ + RECONCILE_IRQ_STATE(r10, r11); \ addi r3,r1,STACK_FRAME_OVERHEAD; \ bl hdlr; \ b ret_from_except @@ -691,9 +673,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_CTRL) * in the idle task and therefore need the special idle handling * (finish nap and runlatch) */ -#define STD_EXCEPTION_COMMON_ASYNC(trap, label, hdlr) \ - EXCEPTION_COMMON(PACA_EXGEN, trap, label, \ - FINISH_NAP;ADD_RECONCILE;RUNLATCH_ON); \ +#define STD_EXCEPTION_COMMON_ASYNC(trap, hdlr) \ + EXCEPTION_COMMON(PACA_EXGEN, trap); \ + FINISH_NAP; \ + RECONCILE_IRQ_STATE(r10, r11); \ + RUNLATCH_ON; \ addi r3,r1,STACK_FRAME_OVERHEAD; \ bl hdlr; \ b ret_from_except_lite diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index bdd67a26e959..acd94fcf9f40 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -403,11 +403,11 @@ name: #define EXC_COMMON(name, realvec, hdlr) \ EXC_COMMON_BEGIN(name); \ - STD_EXCEPTION_COMMON(realvec, name, hdlr) + STD_EXCEPTION_COMMON(realvec, hdlr) #define EXC_COMMON_ASYNC(name, realvec, hdlr) \ EXC_COMMON_BEGIN(name); \ - STD_EXCEPTION_COMMON_ASYNC(realvec, name, hdlr) + STD_EXCEPTION_COMMON_ASYNC(realvec, hdlr) #endif /* __ASSEMBLY__ */ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 7cf0e38875c3..2685c81e28f2 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -164,21 +164,6 @@ EXC_COMMON_BEGIN(system_reset_idle_common) b idle_return_gpr_loss #endif -/* - * Set IRQS_ALL_DISABLED unconditionally so arch_irqs_disabled does - * the right thing. We do not want to reconcile because that goes - * through irq tracing which we don't want in NMI. - * - * Save PACAIRQHAPPENED because some code will do a hard disable - * (e.g., xmon). So we want to restore this back to where it was - * when we return. DAR is unused in the stack, so save it there. - */ -#define ADD_RECONCILE_NMI \ - li r10,IRQS_ALL_DISABLED; \ - stb r10,PACAIRQSOFTMASK(r13); \ - lbz r10,PACAIRQHAPPENED(r13); \ - std r10,_DAR(r1) - EXC_COMMON_BEGIN(system_reset_common) /* * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able @@ -195,8 +180,22 @@ EXC_COMMON_BEGIN(system_reset_common) mr r10,r1 ld r1,PACA_NMI_EMERG_SP(r13) subi r1,r1,INT_FRAME_SIZE - EXCEPTION_COMMON_NORET_STACK(PACA_EXNMI, 0x100, system_reset, - ADD_NVGPRS;ADD_RECONCILE_NMI) + EXCEPTION_COMMON_STACK(PACA_EXNMI, 0x100) + bl save_nvgprs + /* + * Set IRQS_ALL_DISABLED unconditionally so arch_irqs_disabled does + * the right thing. We do not want to reconcile because that goes + * through irq tracing which we don't want in NMI. + * + * Save PACAIRQHAPPENED because some code will do a hard disable + * (e.g., xmon). So we want to restore this back to where it was + * when we return. DAR is unused in the stack, so save it there. + */ + li r10,IRQS_ALL_DISABLED + stb r10,PACAIRQSOFTMASK(r13) + lbz r10,PACAIRQHAPPENED(r13) + std r10,_DAR(r1) + addi r3,r1,STACK_FRAME_OVERHEAD bl system_reset_exception @@ -1172,8 +1171,11 @@ hmi_exception_after_realmode: b tramp_real_hmi_exception EXC_COMMON_BEGIN(hmi_exception_common) -EXCEPTION_COMMON(PACA_EXGEN, 0xe60, hmi_exception_common, - FINISH_NAP;ADD_NVGPRS;ADD_RECONCILE;RUNLATCH_ON) + EXCEPTION_COMMON(PACA_EXGEN, 0xe60) + FINISH_NAP + bl save_nvgprs + RECONCILE_IRQ_STATE(r10, r11) + RUNLATCH_ON addi r3,r1,STACK_FRAME_OVERHEAD bl handle_hmi_exception b ret_from_except @@ -1471,8 +1473,9 @@ EXC_COMMON_BEGIN(soft_nmi_common) mr r10,r1 ld r1,PACAEMERGSP(r13) subi r1,r1,INT_FRAME_SIZE - EXCEPTION_COMMON_NORET_STACK(PACA_EXGEN, 0x900, system_reset, - ADD_NVGPRS;ADD_RECONCILE) + EXCEPTION_COMMON_STACK(PACA_EXGEN, 0x900) + bl save_nvgprs + RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD bl soft_nmi_interrupt b ret_from_except -- cgit v1.2.3 From fc557537f2ad546e5c6f217143d8a95382f2e1b5 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:22 +1000 Subject: powerpc/64s/exception: unwind exception-64s.h macros Many of these macros just specify 1-4 lines which are only called a few times each at most, and often just once. Remove this indirection. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 101 ------------------------------- arch/powerpc/include/asm/head-64.h | 76 +++++++++++++++++------ arch/powerpc/kernel/exceptions-64s.S | 44 ++++++++------ 3 files changed, 82 insertions(+), 139 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 127ebd8d7746..6efd182da254 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -230,17 +230,6 @@ #endif .endm -/* - * As EXCEPTION_PROLOG(), except we've already got relocation on so no need to - * rfid. Save CTR in case we're CONFIG_RELOCATABLE, in which case - * EXCEPTION_PROLOG_2_VIRT will be using CTR. - */ -#define EXCEPTION_RELON_PROLOG(area, label, hsrr, kvm, vec) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 area ; \ - EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ - EXCEPTION_PROLOG_2_VIRT label, hsrr - /* Exception register prefixes */ #define EXC_HV 1 #define EXC_STD 0 @@ -351,12 +340,6 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) std r10,\area\()+EX_R13(r13) .endm -#define EXCEPTION_PROLOG(area, label, hsrr, kvm, vec) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 area ; \ - EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ - EXCEPTION_PROLOG_2_REAL label, hsrr, 1 - #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* * If hv is possible, interrupts come into to the hv version @@ -420,12 +403,6 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #endif -/* Do not enable RI */ -#define EXCEPTION_PROLOG_NORI(area, label, hsrr, kvm, vec) \ - EXCEPTION_PROLOG_0 area ; \ - EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, 0 ; \ - EXCEPTION_PROLOG_2_REAL label, hsrr, 0 - #ifdef CONFIG_KVM_BOOK3S_64_HANDLER .macro KVMTEST hsrr, n lbz r10,HSTATE_IN_GUEST(r13) @@ -562,84 +539,6 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) std r10,RESULT(r1); /* clear regs->result */ \ std r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ -/* - * Exception vectors. - */ -#define STD_EXCEPTION(vec, label) \ - EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_STD, 1, vec); - -/* Version of above for when we have to branch out-of-line */ -#define __OOL_EXCEPTION(vec, label, hdlr) \ - SET_SCRATCH0(r13); \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - b hdlr - -#define STD_EXCEPTION_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0 ; \ - EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 - -#define STD_EXCEPTION_HV(loc, vec, label) \ - EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, 1, vec) - -#define STD_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ - EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 - -#define STD_RELON_EXCEPTION(loc, vec, label) \ - /* No guest interrupts come through here */ \ - EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_STD, 0, vec) - -#define STD_RELON_EXCEPTION_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0 ; \ - EXCEPTION_PROLOG_2_VIRT label, EXC_STD - -#define STD_RELON_EXCEPTION_HV(loc, vec, label) \ - EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_HV, 1, vec) - -#define STD_RELON_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ - EXCEPTION_PROLOG_2_VIRT label, EXC_HV - -#define __MASKABLE_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ - EXCEPTION_PROLOG_2_REAL label, hsrr, 1 - -#define MASKABLE_EXCEPTION(vec, label, bitmask) \ - __MASKABLE_EXCEPTION(vec, label, EXC_STD, 1, bitmask) - -#define MASKABLE_EXCEPTION_OOL(vec, label, bitmask) \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ - EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 - -#define MASKABLE_EXCEPTION_HV(vec, label, bitmask) \ - __MASKABLE_EXCEPTION(vec, label, EXC_HV, 1, bitmask) - -#define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ - EXCEPTION_PROLOG_2_REAL label, EXC_HV, 1 - -#define __MASKABLE_RELON_EXCEPTION(vec, label, hsrr, kvm, bitmask) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - EXCEPTION_PROLOG_1 hsrr, PACA_EXGEN, kvm, vec, bitmask ; \ - EXCEPTION_PROLOG_2_VIRT label, hsrr - -#define MASKABLE_RELON_EXCEPTION(vec, label, bitmask) \ - __MASKABLE_RELON_EXCEPTION(vec, label, EXC_STD, 0, bitmask) - -#define MASKABLE_RELON_EXCEPTION_OOL(vec, label, bitmask) \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, bitmask ; \ - EXCEPTION_PROLOG_2_REAL label, EXC_STD, 1 - -#define MASKABLE_RELON_EXCEPTION_HV(vec, label, bitmask) \ - __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, 1, bitmask) - -#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ - EXCEPTION_PROLOG_2_VIRT label, EXC_HV - #define RUNLATCH_ON \ BEGIN_FTR_SECTION \ ld r3, PACA_THREAD_INFO(r13); \ diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index acd94fcf9f40..54db05afb80f 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -258,44 +258,71 @@ name: FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) -#define EXC_REAL(name, start, size) \ +#define __EXC_REAL(name, start, size, area) \ EXC_REAL_BEGIN(name, start, size); \ - STD_EXCEPTION(start, name##_common); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 area ; \ + EXCEPTION_PROLOG_1 EXC_STD, area, 1, start, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ EXC_REAL_END(name, start, size) -#define EXC_VIRT(name, start, size, realvec) \ +#define EXC_REAL(name, start, size) \ + __EXC_REAL(name, start, size, PACA_EXGEN) + +#define __EXC_VIRT(name, start, size, realvec, area) \ EXC_VIRT_BEGIN(name, start, size); \ - STD_RELON_EXCEPTION(start, realvec, name##_common); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 area ; \ + EXCEPTION_PROLOG_1 EXC_STD, area, 0, realvec, 0; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ EXC_VIRT_END(name, start, size) +#define EXC_VIRT(name, start, size, realvec) \ + __EXC_VIRT(name, start, size, realvec, PACA_EXGEN) + #define EXC_REAL_MASKABLE(name, start, size, bitmask) \ EXC_REAL_BEGIN(name, start, size); \ - MASKABLE_EXCEPTION(start, name##_common, bitmask); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, start, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ EXC_REAL_END(name, start, size) #define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask) \ EXC_VIRT_BEGIN(name, start, size); \ - MASKABLE_RELON_EXCEPTION(realvec, name##_common, bitmask); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ EXC_VIRT_END(name, start, size) #define EXC_REAL_HV(name, start, size) \ EXC_REAL_BEGIN(name, start, size); \ - STD_EXCEPTION_HV(start, start, name##_common); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, start, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 ; \ EXC_REAL_END(name, start, size) #define EXC_VIRT_HV(name, start, size, realvec) \ EXC_VIRT_BEGIN(name, start, size); \ - STD_RELON_EXCEPTION_HV(start, realvec, name##_common); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV ; \ EXC_VIRT_END(name, start, size) #define __EXC_REAL_OOL(name, start, size) \ EXC_REAL_BEGIN(name, start, size); \ - __OOL_EXCEPTION(start, label, tramp_real_##name); \ + SET_SCRATCH0(r13); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + b tramp_real_##name ; \ EXC_REAL_END(name, start, size) #define __TRAMP_REAL_OOL(name, vec) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - STD_EXCEPTION_OOL(vec, name##_common) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 #define EXC_REAL_OOL(name, start, size) \ __EXC_REAL_OOL(name, start, size); \ @@ -306,7 +333,8 @@ name: #define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - MASKABLE_EXCEPTION_OOL(vec, name##_common, bitmask) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 #define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask) \ __EXC_REAL_OOL_MASKABLE(name, start, size); \ @@ -314,7 +342,9 @@ name: #define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler) \ EXC_REAL_BEGIN(name, start, size); \ - __OOL_EXCEPTION(start, label, handler); \ + SET_SCRATCH0(r13); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + b handler; \ EXC_REAL_END(name, start, size) #define __EXC_REAL_OOL_HV(name, start, size) \ @@ -322,7 +352,8 @@ name: #define __TRAMP_REAL_OOL_HV(name, vec) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - STD_EXCEPTION_HV_OOL(vec, name##_common) + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 #define EXC_REAL_OOL_HV(name, start, size) \ __EXC_REAL_OOL_HV(name, start, size); \ @@ -333,7 +364,8 @@ name: #define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - MASKABLE_EXCEPTION_HV_OOL(vec, name##_common, bitmask) + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 #define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask) \ __EXC_REAL_OOL_MASKABLE_HV(name, start, size); \ @@ -341,12 +373,15 @@ name: #define __EXC_VIRT_OOL(name, start, size) \ EXC_VIRT_BEGIN(name, start, size); \ - __OOL_EXCEPTION(start, label, tramp_virt_##name); \ + SET_SCRATCH0(r13); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + b tramp_virt_##name; \ EXC_VIRT_END(name, start, size) #define __TRAMP_VIRT_OOL(name, realvec) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - STD_RELON_EXCEPTION_OOL(realvec, name##_common) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0 ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD #define EXC_VIRT_OOL(name, start, size, realvec) \ __EXC_VIRT_OOL(name, start, size); \ @@ -357,7 +392,8 @@ name: #define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - MASKABLE_RELON_EXCEPTION_OOL(realvec, name##_common, bitmask) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 #define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask) \ __EXC_VIRT_OOL_MASKABLE(name, start, size); \ @@ -368,7 +404,8 @@ name: #define __TRAMP_VIRT_OOL_HV(name, realvec) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - STD_RELON_EXCEPTION_HV_OOL(realvec, name##_common) + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV #define EXC_VIRT_OOL_HV(name, start, size, realvec) \ __EXC_VIRT_OOL_HV(name, start, size); \ @@ -379,7 +416,8 @@ name: #define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common, bitmask) + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, bitmask ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV #define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask) \ __EXC_VIRT_OOL_MASKABLE_HV(name, start, size); \ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 2685c81e28f2..2774046e6cf4 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -254,9 +254,11 @@ EXC_COMMON_BEGIN(system_reset_common) */ TRAMP_REAL_BEGIN(system_reset_fwnmi) SET_SCRATCH0(r13) /* save r13 */ - /* See comment at system_reset exception */ - EXCEPTION_PROLOG_NORI(PACA_EXNMI, system_reset_common, EXC_STD, - 0, 0x100) + /* See comment at system_reset exception, don't turn on RI */ + EXCEPTION_PROLOG_0 PACA_EXNMI + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0 + EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0 + #endif /* CONFIG_PPC_PSERIES */ @@ -714,14 +716,8 @@ MMU_FTR_SECTION_ELSE ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) -EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80) -EXCEPTION_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, 1, 0x480); -EXC_REAL_END(instruction_access_slb, 0x480, 0x80) - -EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80) -EXCEPTION_RELON_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, 0, 0x480); -EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80) - +__EXC_REAL(instruction_access_slb, 0x480, 0x80, PACA_EXSLB) +__EXC_VIRT(instruction_access_slb, 0x4480, 0x80, 0x480, PACA_EXSLB) TRAMP_KVM(PACA_EXSLB, 0x480) EXC_COMMON_BEGIN(instruction_access_slb_common) @@ -750,24 +746,34 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) - .globl hardware_interrupt_hv; + .globl hardware_interrupt_hv hardware_interrupt_hv: BEGIN_FTR_SECTION - MASKABLE_EXCEPTION_HV(0x500, hardware_interrupt_common, IRQS_DISABLED) + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_HV, 1 FTR_SECTION_ELSE - MASKABLE_EXCEPTION(0x500, hardware_interrupt_common, IRQS_DISABLED) + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_STD, 1 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) EXC_REAL_END(hardware_interrupt, 0x500, 0x100) EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) - .globl hardware_interrupt_relon_hv; + .globl hardware_interrupt_relon_hv hardware_interrupt_relon_hv: BEGIN_FTR_SECTION - MASKABLE_RELON_EXCEPTION_HV(0x500, hardware_interrupt_common, - IRQS_DISABLED) + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_HV FTR_SECTION_ELSE - __MASKABLE_RELON_EXCEPTION(0x500, hardware_interrupt_common, - EXC_STD, 1, IRQS_DISABLED) + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_STD ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) -- cgit v1.2.3 From 80bd9177def1ab92637e8cd27c2534b1224b12b5 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:23 +1000 Subject: powerpc/64s/exception: improve 0x500 handler code After the previous cleanup, it becomes possible to consolidate some common code outside the runtime alternate patching. Also remove unused labels. This results in some code change, but unchanged runtime instruction sequence. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 2774046e6cf4..6bf77070b1ba 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -746,32 +746,24 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) - .globl hardware_interrupt_hv -hardware_interrupt_hv: + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN BEGIN_FTR_SECTION - SET_SCRATCH0(r13) /* save r13 */ - EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_HV, 1 FTR_SECTION_ELSE - SET_SCRATCH0(r13) /* save r13 */ - EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_STD, 1 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) EXC_REAL_END(hardware_interrupt, 0x500, 0x100) EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) - .globl hardware_interrupt_relon_hv -hardware_interrupt_relon_hv: + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN BEGIN_FTR_SECTION - SET_SCRATCH0(r13) /* save r13 */ - EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_HV FTR_SECTION_ELSE - SET_SCRATCH0(r13) /* save r13 */ - EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_STD ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) -- cgit v1.2.3 From 12a04809905913859106ffae93572cc312017dda Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:27 +1000 Subject: powerpc/64s/exception: move exception-64s.h code to exception-64s.S where it is used No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 431 ------------------------------- arch/powerpc/kernel/exceptions-64s.S | 431 +++++++++++++++++++++++++++++++ 2 files changed, 431 insertions(+), 431 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index a3b717e666d8..c28e2d2db926 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -150,437 +150,6 @@ hrfid; \ b hrfi_flush_fallback -/* - * We're short on space and time in the exception prolog, so we can't - * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. - * Instead we get the base of the kernel from paca->kernelbase and or in the low - * part of label. This requires that the label be within 64KB of kernelbase, and - * that kernelbase be 64K aligned. - */ -#define LOAD_HANDLER(reg, label) \ - ld reg,PACAKBASE(r13); /* get high part of &label */ \ - ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) - -#define __LOAD_HANDLER(reg, label) \ - ld reg,PACAKBASE(r13); \ - ori reg,reg,(ABS_ADDR(label))@l - -/* - * Branches from unrelocated code (e.g., interrupts) to labels outside - * head-y require >64K offsets. - */ -#define __LOAD_FAR_HANDLER(reg, label) \ - ld reg,PACAKBASE(r13); \ - ori reg,reg,(ABS_ADDR(label))@l; \ - addis reg,reg,(ABS_ADDR(label))@h - -/* Exception register prefixes */ -#define EXC_HV 1 -#define EXC_STD 0 - -#if defined(CONFIG_RELOCATABLE) -/* - * If we support interrupts with relocation on AND we're a relocatable kernel, - * we need to use CTR to get to the 2nd level handler. So, save/restore it - * when required. - */ -#define SAVE_CTR(reg, area) mfctr reg ; std reg,area+EX_CTR(r13) -#define GET_CTR(reg, area) ld reg,area+EX_CTR(r13) -#define RESTORE_CTR(reg, area) ld reg,area+EX_CTR(r13) ; mtctr reg -#else -/* ...else CTR is unused and in register. */ -#define SAVE_CTR(reg, area) -#define GET_CTR(reg, area) mfctr reg -#define RESTORE_CTR(reg, area) -#endif - -/* - * PPR save/restore macros used in exceptions_64s.S - * Used for P7 or later processors - */ -#define SAVE_PPR(area, ra) \ -BEGIN_FTR_SECTION_NESTED(940) \ - ld ra,area+EX_PPR(r13); /* Read PPR from paca */ \ - std ra,_PPR(r1); \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940) - -#define RESTORE_PPR_PACA(area, ra) \ -BEGIN_FTR_SECTION_NESTED(941) \ - ld ra,area+EX_PPR(r13); \ - mtspr SPRN_PPR,ra; \ -END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,941) - -/* - * Get an SPR into a register if the CPU has the given feature - */ -#define OPT_GET_SPR(ra, spr, ftr) \ -BEGIN_FTR_SECTION_NESTED(943) \ - mfspr ra,spr; \ -END_FTR_SECTION_NESTED(ftr,ftr,943) - -/* - * Set an SPR from a register if the CPU has the given feature - */ -#define OPT_SET_SPR(ra, spr, ftr) \ -BEGIN_FTR_SECTION_NESTED(943) \ - mtspr spr,ra; \ -END_FTR_SECTION_NESTED(ftr,ftr,943) - -/* - * Save a register to the PACA if the CPU has the given feature - */ -#define OPT_SAVE_REG_TO_PACA(offset, ra, ftr) \ -BEGIN_FTR_SECTION_NESTED(943) \ - std ra,offset(r13); \ -END_FTR_SECTION_NESTED(ftr,ftr,943) - -.macro EXCEPTION_PROLOG_0 area - GET_PACA(r13) - std r9,\area\()+EX_R9(r13) /* save r9 */ - OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR) - HMT_MEDIUM - std r10,\area\()+EX_R10(r13) /* save r10 - r12 */ - OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR) -.endm - -.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask - OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR) - OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR) - INTERRUPT_TO_KERNEL - SAVE_CTR(r10, \area\()) - mfcr r9 - .if \kvm - KVMTEST \hsrr \vec - .endif - - .if \bitmask - lbz r10,PACAIRQSOFTMASK(r13) - andi. r10,r10,\bitmask - /* Associate vector numbers with bits in paca->irq_happened */ - .if \vec == 0x500 || \vec == 0xea0 - li r10,PACA_IRQ_EE - .elseif \vec == 0x900 - li r10,PACA_IRQ_DEC - .elseif \vec == 0xa00 || \vec == 0xe80 - li r10,PACA_IRQ_DBELL - .elseif \vec == 0xe60 - li r10,PACA_IRQ_HMI - .elseif \vec == 0xf00 - li r10,PACA_IRQ_PMI - .else - .abort "Bad maskable vector" - .endif - - .if \hsrr - bne masked_Hinterrupt - .else - bne masked_interrupt - .endif - .endif - - std r11,\area\()+EX_R11(r13) - std r12,\area\()+EX_R12(r13) - GET_SCRATCH0(r10) - std r10,\area\()+EX_R13(r13) -.endm - -.macro EXCEPTION_PROLOG_2_REAL label, hsrr, set_ri - ld r10,PACAKMSR(r13) /* get MSR value for kernel */ - .if ! \set_ri - xori r10,r10,MSR_RI /* Clear MSR_RI */ - .endif - .if \hsrr - mfspr r11,SPRN_HSRR0 /* save HSRR0 */ - .else - mfspr r11,SPRN_SRR0 /* save SRR0 */ - .endif - LOAD_HANDLER(r12, \label\()) - .if \hsrr - mtspr SPRN_HSRR0,r12 - mfspr r12,SPRN_HSRR1 /* and HSRR1 */ - mtspr SPRN_HSRR1,r10 - HRFI_TO_KERNEL - .else - mtspr SPRN_SRR0,r12 - mfspr r12,SPRN_SRR1 /* and SRR1 */ - mtspr SPRN_SRR1,r10 - RFI_TO_KERNEL - .endif - b . /* prevent speculative execution */ -.endm - -.macro EXCEPTION_PROLOG_2_VIRT label, hsrr -#ifdef CONFIG_RELOCATABLE - .if \hsrr - mfspr r11,SPRN_HSRR0 /* save HSRR0 */ - .else - mfspr r11,SPRN_SRR0 /* save SRR0 */ - .endif - LOAD_HANDLER(r12, \label\()) - mtctr r12 - .if \hsrr - mfspr r12,SPRN_HSRR1 /* and HSRR1 */ - .else - mfspr r12,SPRN_SRR1 /* and HSRR1 */ - .endif - li r10,MSR_RI - mtmsrd r10,1 /* Set RI (EE=0) */ - bctr -#else - .if \hsrr - mfspr r11,SPRN_HSRR0 /* save HSRR0 */ - mfspr r12,SPRN_HSRR1 /* and HSRR1 */ - .else - mfspr r11,SPRN_SRR0 /* save SRR0 */ - mfspr r12,SPRN_SRR1 /* and SRR1 */ - .endif - li r10,MSR_RI - mtmsrd r10,1 /* Set RI (EE=0) */ - b \label -#endif -.endm - -/* - * Branch to label using its 0xC000 address. This results in instruction - * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned - * on using mtmsr rather than rfid. - * - * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than - * load KBASE for a slight optimisation. - */ -#define BRANCH_TO_C000(reg, label) \ - __LOAD_HANDLER(reg, label); \ - mtctr reg; \ - bctr - -#ifdef CONFIG_RELOCATABLE -#define BRANCH_TO_COMMON(reg, label) \ - __LOAD_HANDLER(reg, label); \ - mtctr reg; \ - bctr - -#define BRANCH_LINK_TO_FAR(label) \ - __LOAD_FAR_HANDLER(r12, label); \ - mtctr r12; \ - bctrl - -#else -#define BRANCH_TO_COMMON(reg, label) \ - b label - -#define BRANCH_LINK_TO_FAR(label) \ - bl label -#endif - -#ifdef CONFIG_KVM_BOOK3S_64_HANDLER - -#ifdef CONFIG_RELOCATABLE -/* - * KVM requires __LOAD_FAR_HANDLER. - * - * __BRANCH_TO_KVM_EXIT branches are also a special case because they - * explicitly use r9 then reload it from PACA before branching. Hence - * the double-underscore. - */ -#define __BRANCH_TO_KVM_EXIT(area, label) \ - mfctr r9; \ - std r9,HSTATE_SCRATCH1(r13); \ - __LOAD_FAR_HANDLER(r9, label); \ - mtctr r9; \ - ld r9,area+EX_R9(r13); \ - bctr - -#else -#define __BRANCH_TO_KVM_EXIT(area, label) \ - ld r9,area+EX_R9(r13); \ - b label -#endif - -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE -/* - * If hv is possible, interrupts come into to the hv version - * of the kvmppc_interrupt code, which then jumps to the PR handler, - * kvmppc_interrupt_pr, if the guest is a PR guest. - */ -#define kvmppc_interrupt kvmppc_interrupt_hv -#else -#define kvmppc_interrupt kvmppc_interrupt_pr -#endif - -.macro KVMTEST hsrr, n - lbz r10,HSTATE_IN_GUEST(r13) - cmpwi r10,0 - .if \hsrr - bne do_kvm_H\n - .else - bne do_kvm_\n - .endif -.endm - -.macro KVM_HANDLER area, hsrr, n, skip - .if \skip - cmpwi r10,KVM_GUEST_MODE_SKIP - beq 89f - .else - BEGIN_FTR_SECTION_NESTED(947) - ld r10,\area+EX_CFAR(r13) - std r10,HSTATE_CFAR(r13) - END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947) - .endif - - BEGIN_FTR_SECTION_NESTED(948) - ld r10,\area+EX_PPR(r13) - std r10,HSTATE_PPR(r13) - END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) - ld r10,\area+EX_R10(r13) - std r12,HSTATE_SCRATCH0(r13) - sldi r12,r9,32 - /* HSRR variants have the 0x2 bit added to their trap number */ - .if \hsrr - ori r12,r12,(\n + 0x2) - .else - ori r12,r12,(\n) - .endif - /* This reloads r9 before branching to kvmppc_interrupt */ - __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt) - - .if \skip -89: mtocrf 0x80,r9 - ld r9,\area+EX_R9(r13) - ld r10,\area+EX_R10(r13) - .if \hsrr - b kvmppc_skip_Hinterrupt - .else - b kvmppc_skip_interrupt - .endif - .endif -.endm - -#else -.macro KVMTEST hsrr, n -.endm -.macro KVM_HANDLER area, hsrr, n, skip -.endm -#endif - -#define EXCEPTION_PROLOG_COMMON_1() \ - std r9,_CCR(r1); /* save CR in stackframe */ \ - std r11,_NIP(r1); /* save SRR0 in stackframe */ \ - std r12,_MSR(r1); /* save SRR1 in stackframe */ \ - std r10,0(r1); /* make stack chain pointer */ \ - std r0,GPR0(r1); /* save r0 in stackframe */ \ - std r10,GPR1(r1); /* save r1 in stackframe */ \ - - -/* - * The common exception prolog is used for all except a few exceptions - * such as a segment miss on a kernel address. We have to be prepared - * to take another exception from the point where we first touch the - * kernel stack onwards. - * - * On entry r13 points to the paca, r9-r13 are saved in the paca, - * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and - * SRR1, and relocation is on. - */ -#define EXCEPTION_PROLOG_COMMON(n, area) \ - andi. r10,r12,MSR_PR; /* See if coming from user */ \ - mr r10,r1; /* Save r1 */ \ - subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \ - beq- 1f; \ - ld r1,PACAKSAVE(r13); /* kernel stack to use */ \ -1: cmpdi cr1,r1,-INT_FRAME_SIZE; /* check if r1 is in userspace */ \ - blt+ cr1,3f; /* abort if it is */ \ - li r1,(n); /* will be reloaded later */ \ - sth r1,PACA_TRAP_SAVE(r13); \ - std r3,area+EX_R3(r13); \ - addi r3,r13,area; /* r3 -> where regs are saved*/ \ - RESTORE_CTR(r1, area); \ - b bad_stack; \ -3: EXCEPTION_PROLOG_COMMON_1(); \ - kuap_save_amr_and_lock r9, r10, cr1, cr0; \ - beq 4f; /* if from kernel mode */ \ - ACCOUNT_CPU_USER_ENTRY(r13, r9, r10); \ - SAVE_PPR(area, r9); \ -4: EXCEPTION_PROLOG_COMMON_2(area) \ - EXCEPTION_PROLOG_COMMON_3(n) \ - ACCOUNT_STOLEN_TIME - -/* Save original regs values from save area to stack frame. */ -#define EXCEPTION_PROLOG_COMMON_2(area) \ - ld r9,area+EX_R9(r13); /* move r9, r10 to stackframe */ \ - ld r10,area+EX_R10(r13); \ - std r9,GPR9(r1); \ - std r10,GPR10(r1); \ - ld r9,area+EX_R11(r13); /* move r11 - r13 to stackframe */ \ - ld r10,area+EX_R12(r13); \ - ld r11,area+EX_R13(r13); \ - std r9,GPR11(r1); \ - std r10,GPR12(r1); \ - std r11,GPR13(r1); \ - BEGIN_FTR_SECTION_NESTED(66); \ - ld r10,area+EX_CFAR(r13); \ - std r10,ORIG_GPR3(r1); \ - END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ - GET_CTR(r10, area); \ - std r10,_CTR(r1); - -#define EXCEPTION_PROLOG_COMMON_3(n) \ - std r2,GPR2(r1); /* save r2 in stackframe */ \ - SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ - SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ - mflr r9; /* Get LR, later save to stack */ \ - ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \ - std r9,_LINK(r1); \ - lbz r10,PACAIRQSOFTMASK(r13); \ - mfspr r11,SPRN_XER; /* save XER in stackframe */ \ - std r10,SOFTE(r1); \ - std r11,_XER(r1); \ - li r9,(n)+1; \ - std r9,_TRAP(r1); /* set trap number */ \ - li r10,0; \ - ld r11,exception_marker@toc(r2); \ - std r10,RESULT(r1); /* clear regs->result */ \ - std r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ - -#define RUNLATCH_ON \ -BEGIN_FTR_SECTION \ - ld r3, PACA_THREAD_INFO(r13); \ - ld r4,TI_LOCAL_FLAGS(r3); \ - andi. r0,r4,_TLF_RUNLATCH; \ - beql ppc64_runlatch_on_trampoline; \ -END_FTR_SECTION_IFSET(CPU_FTR_CTRL) - -#define EXCEPTION_COMMON(area, trap) \ - EXCEPTION_PROLOG_COMMON(trap, area); \ - -/* - * Exception where stack is already set in r1, r1 is saved in r10 - */ -#define EXCEPTION_COMMON_STACK(area, trap) \ - EXCEPTION_PROLOG_COMMON_1(); \ - kuap_save_amr_and_lock r9, r10, cr1; \ - EXCEPTION_PROLOG_COMMON_2(area); \ - EXCEPTION_PROLOG_COMMON_3(trap) - -/* - * When the idle code in power4_idle puts the CPU into NAP mode, - * it has to do so in a loop, and relies on the external interrupt - * and decrementer interrupt entry code to get it out of the loop. - * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags - * to signal that it is in the loop and needs help to get out. - */ -#ifdef CONFIG_PPC_970_NAP -#define FINISH_NAP \ -BEGIN_FTR_SECTION \ - ld r11, PACA_THREAD_INFO(r13); \ - ld r9,TI_LOCAL_FLAGS(r11); \ - andi. r10,r9,_TLF_NAPPING; \ - bnel power4_fixup_nap; \ -END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) -#else -#define FINISH_NAP -#endif - #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_EXCEPTION_H */ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 6bf77070b1ba..785aaa3333a9 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -21,6 +21,437 @@ #include #include +/* + * We're short on space and time in the exception prolog, so we can't + * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. + * Instead we get the base of the kernel from paca->kernelbase and or in the low + * part of label. This requires that the label be within 64KB of kernelbase, and + * that kernelbase be 64K aligned. + */ +#define LOAD_HANDLER(reg, label) \ + ld reg,PACAKBASE(r13); /* get high part of &label */ \ + ori reg,reg,FIXED_SYMBOL_ABS_ADDR(label) + +#define __LOAD_HANDLER(reg, label) \ + ld reg,PACAKBASE(r13); \ + ori reg,reg,(ABS_ADDR(label))@l + +/* + * Branches from unrelocated code (e.g., interrupts) to labels outside + * head-y require >64K offsets. + */ +#define __LOAD_FAR_HANDLER(reg, label) \ + ld reg,PACAKBASE(r13); \ + ori reg,reg,(ABS_ADDR(label))@l; \ + addis reg,reg,(ABS_ADDR(label))@h + +/* Exception register prefixes */ +#define EXC_HV 1 +#define EXC_STD 0 + +#if defined(CONFIG_RELOCATABLE) +/* + * If we support interrupts with relocation on AND we're a relocatable kernel, + * we need to use CTR to get to the 2nd level handler. So, save/restore it + * when required. + */ +#define SAVE_CTR(reg, area) mfctr reg ; std reg,area+EX_CTR(r13) +#define GET_CTR(reg, area) ld reg,area+EX_CTR(r13) +#define RESTORE_CTR(reg, area) ld reg,area+EX_CTR(r13) ; mtctr reg +#else +/* ...else CTR is unused and in register. */ +#define SAVE_CTR(reg, area) +#define GET_CTR(reg, area) mfctr reg +#define RESTORE_CTR(reg, area) +#endif + +/* + * PPR save/restore macros used in exceptions-64s.S + * Used for P7 or later processors + */ +#define SAVE_PPR(area, ra) \ +BEGIN_FTR_SECTION_NESTED(940) \ + ld ra,area+EX_PPR(r13); /* Read PPR from paca */ \ + std ra,_PPR(r1); \ +END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940) + +#define RESTORE_PPR_PACA(area, ra) \ +BEGIN_FTR_SECTION_NESTED(941) \ + ld ra,area+EX_PPR(r13); \ + mtspr SPRN_PPR,ra; \ +END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,941) + +/* + * Get an SPR into a register if the CPU has the given feature + */ +#define OPT_GET_SPR(ra, spr, ftr) \ +BEGIN_FTR_SECTION_NESTED(943) \ + mfspr ra,spr; \ +END_FTR_SECTION_NESTED(ftr,ftr,943) + +/* + * Set an SPR from a register if the CPU has the given feature + */ +#define OPT_SET_SPR(ra, spr, ftr) \ +BEGIN_FTR_SECTION_NESTED(943) \ + mtspr spr,ra; \ +END_FTR_SECTION_NESTED(ftr,ftr,943) + +/* + * Save a register to the PACA if the CPU has the given feature + */ +#define OPT_SAVE_REG_TO_PACA(offset, ra, ftr) \ +BEGIN_FTR_SECTION_NESTED(943) \ + std ra,offset(r13); \ +END_FTR_SECTION_NESTED(ftr,ftr,943) + +.macro EXCEPTION_PROLOG_0 area + GET_PACA(r13) + std r9,\area\()+EX_R9(r13) /* save r9 */ + OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR) + HMT_MEDIUM + std r10,\area\()+EX_R10(r13) /* save r10 - r12 */ + OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR) +.endm + +.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask + OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR) + OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR) + INTERRUPT_TO_KERNEL + SAVE_CTR(r10, \area\()) + mfcr r9 + .if \kvm + KVMTEST \hsrr \vec + .endif + .if \bitmask + lbz r10,PACAIRQSOFTMASK(r13) + andi. r10,r10,\bitmask + /* Associate vector numbers with bits in paca->irq_happened */ + .if \vec == 0x500 || \vec == 0xea0 + li r10,PACA_IRQ_EE + .elseif \vec == 0x900 + li r10,PACA_IRQ_DEC + .elseif \vec == 0xa00 || \vec == 0xe80 + li r10,PACA_IRQ_DBELL + .elseif \vec == 0xe60 + li r10,PACA_IRQ_HMI + .elseif \vec == 0xf00 + li r10,PACA_IRQ_PMI + .else + .abort "Bad maskable vector" + .endif + + .if \hsrr + bne masked_Hinterrupt + .else + bne masked_interrupt + .endif + .endif + + std r11,\area\()+EX_R11(r13) + std r12,\area\()+EX_R12(r13) + GET_SCRATCH0(r10) + std r10,\area\()+EX_R13(r13) +.endm + +.macro EXCEPTION_PROLOG_2_REAL label, hsrr, set_ri + ld r10,PACAKMSR(r13) /* get MSR value for kernel */ + .if ! \set_ri + xori r10,r10,MSR_RI /* Clear MSR_RI */ + .endif + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + .endif + LOAD_HANDLER(r12, \label\()) + .if \hsrr + mtspr SPRN_HSRR0,r12 + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + mtspr SPRN_HSRR1,r10 + HRFI_TO_KERNEL + .else + mtspr SPRN_SRR0,r12 + mfspr r12,SPRN_SRR1 /* and SRR1 */ + mtspr SPRN_SRR1,r10 + RFI_TO_KERNEL + .endif + b . /* prevent speculative execution */ +.endm + +.macro EXCEPTION_PROLOG_2_VIRT label, hsrr +#ifdef CONFIG_RELOCATABLE + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + .endif + LOAD_HANDLER(r12, \label\()) + mtctr r12 + .if \hsrr + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + .else + mfspr r12,SPRN_SRR1 /* and HSRR1 */ + .endif + li r10,MSR_RI + mtmsrd r10,1 /* Set RI (EE=0) */ + bctr +#else + .if \hsrr + mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + .else + mfspr r11,SPRN_SRR0 /* save SRR0 */ + mfspr r12,SPRN_SRR1 /* and SRR1 */ + .endif + li r10,MSR_RI + mtmsrd r10,1 /* Set RI (EE=0) */ + b \label +#endif +.endm + +/* + * Branch to label using its 0xC000 address. This results in instruction + * address suitable for MSR[IR]=0 or 1, which allows relocation to be turned + * on using mtmsr rather than rfid. + * + * This could set the 0xc bits for !RELOCATABLE as an immediate, rather than + * load KBASE for a slight optimisation. + */ +#define BRANCH_TO_C000(reg, label) \ + __LOAD_HANDLER(reg, label); \ + mtctr reg; \ + bctr + +#ifdef CONFIG_RELOCATABLE +#define BRANCH_TO_COMMON(reg, label) \ + __LOAD_HANDLER(reg, label); \ + mtctr reg; \ + bctr + +#define BRANCH_LINK_TO_FAR(label) \ + __LOAD_FAR_HANDLER(r12, label); \ + mtctr r12; \ + bctrl + +#else +#define BRANCH_TO_COMMON(reg, label) \ + b label + +#define BRANCH_LINK_TO_FAR(label) \ + bl label +#endif + +#ifdef CONFIG_KVM_BOOK3S_64_HANDLER + +#ifdef CONFIG_RELOCATABLE +/* + * KVM requires __LOAD_FAR_HANDLER. + * + * __BRANCH_TO_KVM_EXIT branches are also a special case because they + * explicitly use r9 then reload it from PACA before branching. Hence + * the double-underscore. + */ +#define __BRANCH_TO_KVM_EXIT(area, label) \ + mfctr r9; \ + std r9,HSTATE_SCRATCH1(r13); \ + __LOAD_FAR_HANDLER(r9, label); \ + mtctr r9; \ + ld r9,area+EX_R9(r13); \ + bctr + +#else +#define __BRANCH_TO_KVM_EXIT(area, label) \ + ld r9,area+EX_R9(r13); \ + b label +#endif + +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +/* + * If hv is possible, interrupts come into to the hv version + * of the kvmppc_interrupt code, which then jumps to the PR handler, + * kvmppc_interrupt_pr, if the guest is a PR guest. + */ +#define kvmppc_interrupt kvmppc_interrupt_hv +#else +#define kvmppc_interrupt kvmppc_interrupt_pr +#endif + +.macro KVMTEST hsrr, n + lbz r10,HSTATE_IN_GUEST(r13) + cmpwi r10,0 + .if \hsrr + bne do_kvm_H\n + .else + bne do_kvm_\n + .endif +.endm + +.macro KVM_HANDLER area, hsrr, n, skip + .if \skip + cmpwi r10,KVM_GUEST_MODE_SKIP + beq 89f + .else + BEGIN_FTR_SECTION_NESTED(947) + ld r10,\area+EX_CFAR(r13) + std r10,HSTATE_CFAR(r13) + END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947) + .endif + + BEGIN_FTR_SECTION_NESTED(948) + ld r10,\area+EX_PPR(r13) + std r10,HSTATE_PPR(r13) + END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) + ld r10,\area+EX_R10(r13) + std r12,HSTATE_SCRATCH0(r13) + sldi r12,r9,32 + /* HSRR variants have the 0x2 bit added to their trap number */ + .if \hsrr + ori r12,r12,(\n + 0x2) + .else + ori r12,r12,(\n) + .endif + /* This reloads r9 before branching to kvmppc_interrupt */ + __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt) + + .if \skip +89: mtocrf 0x80,r9 + ld r9,\area+EX_R9(r13) + ld r10,\area+EX_R10(r13) + .if \hsrr + b kvmppc_skip_Hinterrupt + .else + b kvmppc_skip_interrupt + .endif + .endif +.endm + +#else +.macro KVMTEST hsrr, n +.endm +.macro KVM_HANDLER area, hsrr, n, skip +.endm +#endif + +#define EXCEPTION_PROLOG_COMMON_1() \ + std r9,_CCR(r1); /* save CR in stackframe */ \ + std r11,_NIP(r1); /* save SRR0 in stackframe */ \ + std r12,_MSR(r1); /* save SRR1 in stackframe */ \ + std r10,0(r1); /* make stack chain pointer */ \ + std r0,GPR0(r1); /* save r0 in stackframe */ \ + std r10,GPR1(r1); /* save r1 in stackframe */ \ + + +/* + * The common exception prolog is used for all except a few exceptions + * such as a segment miss on a kernel address. We have to be prepared + * to take another exception from the point where we first touch the + * kernel stack onwards. + * + * On entry r13 points to the paca, r9-r13 are saved in the paca, + * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and + * SRR1, and relocation is on. + */ +#define EXCEPTION_PROLOG_COMMON(n, area) \ + andi. r10,r12,MSR_PR; /* See if coming from user */ \ + mr r10,r1; /* Save r1 */ \ + subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \ + beq- 1f; \ + ld r1,PACAKSAVE(r13); /* kernel stack to use */ \ +1: cmpdi cr1,r1,-INT_FRAME_SIZE; /* check if r1 is in userspace */ \ + blt+ cr1,3f; /* abort if it is */ \ + li r1,(n); /* will be reloaded later */ \ + sth r1,PACA_TRAP_SAVE(r13); \ + std r3,area+EX_R3(r13); \ + addi r3,r13,area; /* r3 -> where regs are saved*/ \ + RESTORE_CTR(r1, area); \ + b bad_stack; \ +3: EXCEPTION_PROLOG_COMMON_1(); \ + kuap_save_amr_and_lock r9, r10, cr1, cr0; \ + beq 4f; /* if from kernel mode */ \ + ACCOUNT_CPU_USER_ENTRY(r13, r9, r10); \ + SAVE_PPR(area, r9); \ +4: EXCEPTION_PROLOG_COMMON_2(area) \ + EXCEPTION_PROLOG_COMMON_3(n) \ + ACCOUNT_STOLEN_TIME + +/* Save original regs values from save area to stack frame. */ +#define EXCEPTION_PROLOG_COMMON_2(area) \ + ld r9,area+EX_R9(r13); /* move r9, r10 to stackframe */ \ + ld r10,area+EX_R10(r13); \ + std r9,GPR9(r1); \ + std r10,GPR10(r1); \ + ld r9,area+EX_R11(r13); /* move r11 - r13 to stackframe */ \ + ld r10,area+EX_R12(r13); \ + ld r11,area+EX_R13(r13); \ + std r9,GPR11(r1); \ + std r10,GPR12(r1); \ + std r11,GPR13(r1); \ + BEGIN_FTR_SECTION_NESTED(66); \ + ld r10,area+EX_CFAR(r13); \ + std r10,ORIG_GPR3(r1); \ + END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ + GET_CTR(r10, area); \ + std r10,_CTR(r1); + +#define EXCEPTION_PROLOG_COMMON_3(n) \ + std r2,GPR2(r1); /* save r2 in stackframe */ \ + SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ + SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ + mflr r9; /* Get LR, later save to stack */ \ + ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \ + std r9,_LINK(r1); \ + lbz r10,PACAIRQSOFTMASK(r13); \ + mfspr r11,SPRN_XER; /* save XER in stackframe */ \ + std r10,SOFTE(r1); \ + std r11,_XER(r1); \ + li r9,(n)+1; \ + std r9,_TRAP(r1); /* set trap number */ \ + li r10,0; \ + ld r11,exception_marker@toc(r2); \ + std r10,RESULT(r1); /* clear regs->result */ \ + std r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ + +#define RUNLATCH_ON \ +BEGIN_FTR_SECTION \ + ld r3, PACA_THREAD_INFO(r13); \ + ld r4,TI_LOCAL_FLAGS(r3); \ + andi. r0,r4,_TLF_RUNLATCH; \ + beql ppc64_runlatch_on_trampoline; \ +END_FTR_SECTION_IFSET(CPU_FTR_CTRL) + +#define EXCEPTION_COMMON(area, trap) \ + EXCEPTION_PROLOG_COMMON(trap, area); \ + +/* + * Exception where stack is already set in r1, r1 is saved in r10 + */ +#define EXCEPTION_COMMON_STACK(area, trap) \ + EXCEPTION_PROLOG_COMMON_1(); \ + kuap_save_amr_and_lock r9, r10, cr1; \ + EXCEPTION_PROLOG_COMMON_2(area); \ + EXCEPTION_PROLOG_COMMON_3(trap) + +/* + * When the idle code in power4_idle puts the CPU into NAP mode, + * it has to do so in a loop, and relies on the external interrupt + * and decrementer interrupt entry code to get it out of the loop. + * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags + * to signal that it is in the loop and needs help to get out. + */ +#ifdef CONFIG_PPC_970_NAP +#define FINISH_NAP \ +BEGIN_FTR_SECTION \ + ld r11, PACA_THREAD_INFO(r13); \ + ld r9,TI_LOCAL_FLAGS(r11); \ + andi. r10,r9,_TLF_NAPPING; \ + bnel power4_fixup_nap; \ +END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) +#else +#define FINISH_NAP +#endif + + /* * There are a few constraints to be concerned with. * - Real mode exceptions code/data must be located at their physical location. -- cgit v1.2.3 From a0502434bb02876d6c9b41d78a8db3a0b766f682 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:28 +1000 Subject: powerpc/64s/exception: move head-64.h code to exception-64s.S where it is used No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 1 - arch/powerpc/include/asm/head-64.h | 252 ------------------------------- arch/powerpc/kernel/exceptions-64s.S | 251 ++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+), 253 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index c28e2d2db926..3585e1d7e898 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -34,7 +34,6 @@ * exception handlers (including pSeries LPAR) and iSeries LPAR * implementations as possible. */ -#include #include /* PACA save area offsets (exgen, exmc, etc) */ diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index dc1940c94a86..a466765709a9 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -169,53 +169,6 @@ name: #define ABS_ADDR(label) (label - fs_label + fs_start) -/* - * Following are the BOOK3S exception handler helper macros. - * Handlers come in a number of types, and each type has a number of varieties. - * - * EXC_REAL_* - real, unrelocated exception vectors - * EXC_VIRT_* - virt (AIL), unrelocated exception vectors - * TRAMP_REAL_* - real, unrelocated helpers (virt can call these) - * TRAMP_VIRT_* - virt, unreloc helpers (in practice, real can use) - * TRAMP_KVM - KVM handlers that get put into real, unrelocated - * EXC_COMMON - virt, relocated common handlers - * - * The EXC handlers are given a name, and branch to name_common, or the - * appropriate KVM or masking function. Vector handler verieties are as - * follows: - * - * EXC_{REAL|VIRT}_BEGIN/END - used to open-code the exception - * - * EXC_{REAL|VIRT} - standard exception - * - * EXC_{REAL|VIRT}_suffix - * where _suffix is: - * - _MASKABLE - maskable exception - * - _OOL - out of line with trampoline to common handler - * - _HV - HV exception - * - * There can be combinations, e.g., EXC_VIRT_OOL_MASKABLE_HV - * - * The one unusual case is __EXC_REAL_OOL_HV_DIRECT, which is - * an OOL vector that branches to a specified handler rather than the usual - * trampoline that goes to common. It, and other underscore macros, should - * be used with care. - * - * KVM handlers come in the following verieties: - * TRAMP_KVM - * TRAMP_KVM_SKIP - * TRAMP_KVM_HV - * TRAMP_KVM_HV_SKIP - * - * COMMON handlers come in the following verieties: - * EXC_COMMON_BEGIN/END - used to open-code the handler - * EXC_COMMON - * EXC_COMMON_ASYNC - * - * TRAMP_REAL and TRAMP_VIRT can be used with BEGIN/END. KVM - * and OOL handlers are implemented as types of TRAMP and TRAMP_VIRT handlers. - */ - #define EXC_REAL_BEGIN(name, start, size) \ FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##name, start, size) @@ -257,211 +210,6 @@ name: FIXED_SECTION_ENTRY_BEGIN_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size); \ FIXED_SECTION_ENTRY_END_LOCATION(virt_vectors, exc_virt_##start##_##unused, start, size) - -#define __EXC_REAL(name, start, size, area) \ - EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 area ; \ - EXCEPTION_PROLOG_1 EXC_STD, area, 1, start, 0 ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ - EXC_REAL_END(name, start, size) - -#define EXC_REAL(name, start, size) \ - __EXC_REAL(name, start, size, PACA_EXGEN) - -#define __EXC_VIRT(name, start, size, realvec, area) \ - EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 area ; \ - EXCEPTION_PROLOG_1 EXC_STD, area, 0, realvec, 0; \ - EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ - EXC_VIRT_END(name, start, size) - -#define EXC_VIRT(name, start, size, realvec) \ - __EXC_VIRT(name, start, size, realvec, PACA_EXGEN) - -#define EXC_REAL_MASKABLE(name, start, size, bitmask) \ - EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, start, bitmask ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ - EXC_REAL_END(name, start, size) - -#define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask) \ - EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ - EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ - EXC_VIRT_END(name, start, size) - -#define EXC_REAL_HV(name, start, size) \ - EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 PACA_EXGEN; \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, start, 0 ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 ; \ - EXC_REAL_END(name, start, size) - -#define EXC_VIRT_HV(name, start, size, realvec) \ - EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_0 PACA_EXGEN; \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ - EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV ; \ - EXC_VIRT_END(name, start, size) - -#define __EXC_REAL_OOL(name, start, size) \ - EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - b tramp_real_##name ; \ - EXC_REAL_END(name, start, size) - -#define __TRAMP_REAL_OOL(name, vec) \ - TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0 ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 - -#define EXC_REAL_OOL(name, start, size) \ - __EXC_REAL_OOL(name, start, size); \ - __TRAMP_REAL_OOL(name, start) - -#define __EXC_REAL_OOL_MASKABLE(name, start, size) \ - __EXC_REAL_OOL(name, start, size) - -#define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask) \ - TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 - -#define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask) \ - __EXC_REAL_OOL_MASKABLE(name, start, size); \ - __TRAMP_REAL_OOL_MASKABLE(name, start, bitmask) - -#define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler) \ - EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - b handler; \ - EXC_REAL_END(name, start, size) - -#define __EXC_REAL_OOL_HV(name, start, size) \ - __EXC_REAL_OOL(name, start, size) - -#define __TRAMP_REAL_OOL_HV(name, vec) \ - TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 - -#define EXC_REAL_OOL_HV(name, start, size) \ - __EXC_REAL_OOL_HV(name, start, size); \ - __TRAMP_REAL_OOL_HV(name, start) - -#define __EXC_REAL_OOL_MASKABLE_HV(name, start, size) \ - __EXC_REAL_OOL(name, start, size) - -#define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask) \ - TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 - -#define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask) \ - __EXC_REAL_OOL_MASKABLE_HV(name, start, size); \ - __TRAMP_REAL_OOL_MASKABLE_HV(name, start, bitmask) - -#define __EXC_VIRT_OOL(name, start, size) \ - EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - b tramp_virt_##name; \ - EXC_VIRT_END(name, start, size) - -#define __TRAMP_VIRT_OOL(name, realvec) \ - TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0 ; \ - EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD - -#define EXC_VIRT_OOL(name, start, size, realvec) \ - __EXC_VIRT_OOL(name, start, size); \ - __TRAMP_VIRT_OOL(name, realvec) - -#define __EXC_VIRT_OOL_MASKABLE(name, start, size) \ - __EXC_VIRT_OOL(name, start, size) - -#define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) \ - TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ - EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 - -#define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask) \ - __EXC_VIRT_OOL_MASKABLE(name, start, size); \ - __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) - -#define __EXC_VIRT_OOL_HV(name, start, size) \ - __EXC_VIRT_OOL(name, start, size) - -#define __TRAMP_VIRT_OOL_HV(name, realvec) \ - TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ - EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV - -#define EXC_VIRT_OOL_HV(name, start, size, realvec) \ - __EXC_VIRT_OOL_HV(name, start, size); \ - __TRAMP_VIRT_OOL_HV(name, realvec) - -#define __EXC_VIRT_OOL_MASKABLE_HV(name, start, size) \ - __EXC_VIRT_OOL(name, start, size) - -#define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) \ - TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, bitmask ; \ - EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV - -#define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask) \ - __EXC_VIRT_OOL_MASKABLE_HV(name, start, size); \ - __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) - -#define TRAMP_KVM(area, n) \ - TRAMP_KVM_BEGIN(do_kvm_##n); \ - KVM_HANDLER area, EXC_STD, n, 0 - -#define TRAMP_KVM_SKIP(area, n) \ - TRAMP_KVM_BEGIN(do_kvm_##n); \ - KVM_HANDLER area, EXC_STD, n, 1 - -#define TRAMP_KVM_HV(area, n) \ - TRAMP_KVM_BEGIN(do_kvm_H##n); \ - KVM_HANDLER area, EXC_HV, n, 0 - -#define TRAMP_KVM_HV_SKIP(area, n) \ - TRAMP_KVM_BEGIN(do_kvm_H##n); \ - KVM_HANDLER area, EXC_HV, n, 1 - -#define EXC_COMMON(name, realvec, hdlr) \ - EXC_COMMON_BEGIN(name); \ - EXCEPTION_COMMON(PACA_EXGEN, realvec); \ - bl save_nvgprs; \ - RECONCILE_IRQ_STATE(r10, r11); \ - addi r3,r1,STACK_FRAME_OVERHEAD; \ - bl hdlr; \ - b ret_from_except - -/* - * Like EXC_COMMON, but for exceptions that can occur in the idle task and - * therefore need the special idle handling (finish nap and runlatch) - */ -#define EXC_COMMON_ASYNC(name, realvec, hdlr) \ - EXC_COMMON_BEGIN(name); \ - EXCEPTION_COMMON(PACA_EXGEN, realvec); \ - FINISH_NAP; \ - RECONCILE_IRQ_STATE(r10, r11); \ - RUNLATCH_ON; \ - addi r3,r1,STACK_FRAME_OVERHEAD; \ - bl hdlr; \ - b ret_from_except_lite - #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_HEAD_64_H */ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 785aaa3333a9..ba2ecbd06c72 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -451,6 +451,257 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define FINISH_NAP #endif +/* + * Following are the BOOK3S exception handler helper macros. + * Handlers come in a number of types, and each type has a number of varieties. + * + * EXC_REAL_* - real, unrelocated exception vectors + * EXC_VIRT_* - virt (AIL), unrelocated exception vectors + * TRAMP_REAL_* - real, unrelocated helpers (virt can call these) + * TRAMP_VIRT_* - virt, unreloc helpers (in practice, real can use) + * TRAMP_KVM - KVM handlers that get put into real, unrelocated + * EXC_COMMON - virt, relocated common handlers + * + * The EXC handlers are given a name, and branch to name_common, or the + * appropriate KVM or masking function. Vector handler verieties are as + * follows: + * + * EXC_{REAL|VIRT}_BEGIN/END - used to open-code the exception + * + * EXC_{REAL|VIRT} - standard exception + * + * EXC_{REAL|VIRT}_suffix + * where _suffix is: + * - _MASKABLE - maskable exception + * - _OOL - out of line with trampoline to common handler + * - _HV - HV exception + * + * There can be combinations, e.g., EXC_VIRT_OOL_MASKABLE_HV + * + * The one unusual case is __EXC_REAL_OOL_HV_DIRECT, which is + * an OOL vector that branches to a specified handler rather than the usual + * trampoline that goes to common. It, and other underscore macros, should + * be used with care. + * + * KVM handlers come in the following verieties: + * TRAMP_KVM + * TRAMP_KVM_SKIP + * TRAMP_KVM_HV + * TRAMP_KVM_HV_SKIP + * + * COMMON handlers come in the following verieties: + * EXC_COMMON_BEGIN/END - used to open-code the handler + * EXC_COMMON + * EXC_COMMON_ASYNC + * + * TRAMP_REAL and TRAMP_VIRT can be used with BEGIN/END. KVM + * and OOL handlers are implemented as types of TRAMP and TRAMP_VIRT handlers. + */ + +#define __EXC_REAL(name, start, size, area) \ + EXC_REAL_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 area ; \ + EXCEPTION_PROLOG_1 EXC_STD, area, 1, start, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ + EXC_REAL_END(name, start, size) + +#define EXC_REAL(name, start, size) \ + __EXC_REAL(name, start, size, PACA_EXGEN) + +#define __EXC_VIRT(name, start, size, realvec, area) \ + EXC_VIRT_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 area ; \ + EXCEPTION_PROLOG_1 EXC_STD, area, 0, realvec, 0; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ + EXC_VIRT_END(name, start, size) + +#define EXC_VIRT(name, start, size, realvec) \ + __EXC_VIRT(name, start, size, realvec, PACA_EXGEN) + +#define EXC_REAL_MASKABLE(name, start, size, bitmask) \ + EXC_REAL_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, start, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ + EXC_REAL_END(name, start, size) + +#define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask) \ + EXC_VIRT_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ + EXC_VIRT_END(name, start, size) + +#define EXC_REAL_HV(name, start, size) \ + EXC_REAL_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, start, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 ; \ + EXC_REAL_END(name, start, size) + +#define EXC_VIRT_HV(name, start, size, realvec) \ + EXC_VIRT_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); /* save r13 */ \ + EXCEPTION_PROLOG_0 PACA_EXGEN; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV ; \ + EXC_VIRT_END(name, start, size) + +#define __EXC_REAL_OOL(name, start, size) \ + EXC_REAL_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + b tramp_real_##name ; \ + EXC_REAL_END(name, start, size) + +#define __TRAMP_REAL_OOL(name, vec) \ + TRAMP_REAL_BEGIN(tramp_real_##name); \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 + +#define EXC_REAL_OOL(name, start, size) \ + __EXC_REAL_OOL(name, start, size); \ + __TRAMP_REAL_OOL(name, start) + +#define __EXC_REAL_OOL_MASKABLE(name, start, size) \ + __EXC_REAL_OOL(name, start, size) + +#define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask) \ + TRAMP_REAL_BEGIN(tramp_real_##name); \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 + +#define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask) \ + __EXC_REAL_OOL_MASKABLE(name, start, size); \ + __TRAMP_REAL_OOL_MASKABLE(name, start, bitmask) + +#define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler) \ + EXC_REAL_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + b handler; \ + EXC_REAL_END(name, start, size) + +#define __EXC_REAL_OOL_HV(name, start, size) \ + __EXC_REAL_OOL(name, start, size) + +#define __TRAMP_REAL_OOL_HV(name, vec) \ + TRAMP_REAL_BEGIN(tramp_real_##name); \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 + +#define EXC_REAL_OOL_HV(name, start, size) \ + __EXC_REAL_OOL_HV(name, start, size); \ + __TRAMP_REAL_OOL_HV(name, start) + +#define __EXC_REAL_OOL_MASKABLE_HV(name, start, size) \ + __EXC_REAL_OOL(name, start, size) + +#define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask) \ + TRAMP_REAL_BEGIN(tramp_real_##name); \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 + +#define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask) \ + __EXC_REAL_OOL_MASKABLE_HV(name, start, size); \ + __TRAMP_REAL_OOL_MASKABLE_HV(name, start, bitmask) + +#define __EXC_VIRT_OOL(name, start, size) \ + EXC_VIRT_BEGIN(name, start, size); \ + SET_SCRATCH0(r13); \ + EXCEPTION_PROLOG_0 PACA_EXGEN ; \ + b tramp_virt_##name; \ + EXC_VIRT_END(name, start, size) + +#define __TRAMP_VIRT_OOL(name, realvec) \ + TRAMP_VIRT_BEGIN(tramp_virt_##name); \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0 ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD + +#define EXC_VIRT_OOL(name, start, size, realvec) \ + __EXC_VIRT_OOL(name, start, size); \ + __TRAMP_VIRT_OOL(name, realvec) + +#define __EXC_VIRT_OOL_MASKABLE(name, start, size) \ + __EXC_VIRT_OOL(name, start, size) + +#define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) \ + TRAMP_VIRT_BEGIN(tramp_virt_##name); \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ + EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 + +#define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask) \ + __EXC_VIRT_OOL_MASKABLE(name, start, size); \ + __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) + +#define __EXC_VIRT_OOL_HV(name, start, size) \ + __EXC_VIRT_OOL(name, start, size) + +#define __TRAMP_VIRT_OOL_HV(name, realvec) \ + TRAMP_VIRT_BEGIN(tramp_virt_##name); \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV + +#define EXC_VIRT_OOL_HV(name, start, size, realvec) \ + __EXC_VIRT_OOL_HV(name, start, size); \ + __TRAMP_VIRT_OOL_HV(name, realvec) + +#define __EXC_VIRT_OOL_MASKABLE_HV(name, start, size) \ + __EXC_VIRT_OOL(name, start, size) + +#define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) \ + TRAMP_VIRT_BEGIN(tramp_virt_##name); \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, bitmask ; \ + EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV + +#define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask) \ + __EXC_VIRT_OOL_MASKABLE_HV(name, start, size); \ + __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) + +#define TRAMP_KVM(area, n) \ + TRAMP_KVM_BEGIN(do_kvm_##n); \ + KVM_HANDLER area, EXC_STD, n, 0 + +#define TRAMP_KVM_SKIP(area, n) \ + TRAMP_KVM_BEGIN(do_kvm_##n); \ + KVM_HANDLER area, EXC_STD, n, 1 + +#define TRAMP_KVM_HV(area, n) \ + TRAMP_KVM_BEGIN(do_kvm_H##n); \ + KVM_HANDLER area, EXC_HV, n, 0 + +#define TRAMP_KVM_HV_SKIP(area, n) \ + TRAMP_KVM_BEGIN(do_kvm_H##n); \ + KVM_HANDLER area, EXC_HV, n, 1 + +#define EXC_COMMON(name, realvec, hdlr) \ + EXC_COMMON_BEGIN(name); \ + EXCEPTION_COMMON(PACA_EXGEN, realvec); \ + bl save_nvgprs; \ + RECONCILE_IRQ_STATE(r10, r11); \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + bl hdlr; \ + b ret_from_except + +/* + * Like EXC_COMMON, but for exceptions that can occur in the idle task and + * therefore need the special idle handling (finish nap and runlatch) + */ +#define EXC_COMMON_ASYNC(name, realvec, hdlr) \ + EXC_COMMON_BEGIN(name); \ + EXCEPTION_COMMON(PACA_EXGEN, realvec); \ + FINISH_NAP; \ + RECONCILE_IRQ_STATE(r10, r11); \ + RUNLATCH_ON; \ + addi r3,r1,STACK_FRAME_OVERHEAD; \ + bl hdlr; \ + b ret_from_except_lite + /* * There are a few constraints to be concerned with. -- cgit v1.2.3 From 64e413515ce1e731b87f704cf0d8d9768f0bd8c8 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:29 +1000 Subject: powerpc/64s/exception: remove __BRANCH_TO_KVM No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 43 +++++++++++++++--------------------- 1 file changed, 18 insertions(+), 25 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index ba2ecbd06c72..c354bb9606b8 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -243,29 +243,6 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #endif #ifdef CONFIG_KVM_BOOK3S_64_HANDLER - -#ifdef CONFIG_RELOCATABLE -/* - * KVM requires __LOAD_FAR_HANDLER. - * - * __BRANCH_TO_KVM_EXIT branches are also a special case because they - * explicitly use r9 then reload it from PACA before branching. Hence - * the double-underscore. - */ -#define __BRANCH_TO_KVM_EXIT(area, label) \ - mfctr r9; \ - std r9,HSTATE_SCRATCH1(r13); \ - __LOAD_FAR_HANDLER(r9, label); \ - mtctr r9; \ - ld r9,area+EX_R9(r13); \ - bctr - -#else -#define __BRANCH_TO_KVM_EXIT(area, label) \ - ld r9,area+EX_R9(r13); \ - b label -#endif - #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* * If hv is possible, interrupts come into to the hv version @@ -311,8 +288,24 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) .else ori r12,r12,(\n) .endif - /* This reloads r9 before branching to kvmppc_interrupt */ - __BRANCH_TO_KVM_EXIT(\area, kvmppc_interrupt) + +#ifdef CONFIG_RELOCATABLE + /* + * KVM requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives + * outside the head section. CONFIG_RELOCATABLE KVM expects CTR + * to be saved in HSTATE_SCRATCH1. + */ + mfctr r9 + std r9,HSTATE_SCRATCH1(r13) + __LOAD_FAR_HANDLER(r9, kvmppc_interrupt) + mtctr r9 + ld r9,\area+EX_R9(r13) + bctr +#else + ld r9,\area+EX_R9(r13) + b kvmppc_interrupt +#endif + .if \skip 89: mtocrf 0x80,r9 -- cgit v1.2.3 From f945478d5c689ccdab8bf3ee47f27085c2836690 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:30 +1000 Subject: powerpc/64s/exception: remove unused BRANCH_TO_COMMON Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 8 -------- 1 file changed, 8 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index c354bb9606b8..7d062d3f6f54 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -224,20 +224,12 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) bctr #ifdef CONFIG_RELOCATABLE -#define BRANCH_TO_COMMON(reg, label) \ - __LOAD_HANDLER(reg, label); \ - mtctr reg; \ - bctr - #define BRANCH_LINK_TO_FAR(label) \ __LOAD_FAR_HANDLER(r12, label); \ mtctr r12; \ bctrl #else -#define BRANCH_TO_COMMON(reg, label) \ - b label - #define BRANCH_LINK_TO_FAR(label) \ bl label #endif -- cgit v1.2.3 From 1b4d4a7933a7afc6ea8e973b7aba84abdfb2fd0b Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:31 +1000 Subject: powerpc/64s/exception: use a gas macro for system call handler code No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 127 +++++++++++++++-------------------- 1 file changed, 55 insertions(+), 72 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 7d062d3f6f54..73ba2e3f345b 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1607,6 +1607,7 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception) * without saving, though xer is not a good idea to use, as hardware may * interpret some bits so it may be costly to change them. */ +.macro SYSTEM_CALL virt #ifdef CONFIG_KVM_BOOK3S_64_HANDLER /* * There is a little bit of juggling to get syscall and hcall @@ -1616,95 +1617,77 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception) * Userspace syscalls have already saved the PPR, hcalls must save * it before setting HMT_MEDIUM. */ -#define SYSCALL_KVMTEST \ - mtctr r13; \ - GET_PACA(r13); \ - std r10,PACA_EXGEN+EX_R10(r13); \ - INTERRUPT_TO_KERNEL; \ - KVMTEST EXC_STD 0xc00 ; /* uses r10, branch to do_kvm_0xc00_system_call */ \ - HMT_MEDIUM; \ - mfctr r9; - + mtctr r13 + GET_PACA(r13) + std r10,PACA_EXGEN+EX_R10(r13) + INTERRUPT_TO_KERNEL + KVMTEST EXC_STD 0xc00 /* uses r10, branch to do_kvm_0xc00_system_call */ + HMT_MEDIUM + mfctr r9 #else -#define SYSCALL_KVMTEST \ - HMT_MEDIUM; \ - mr r9,r13; \ - GET_PACA(r13); \ - INTERRUPT_TO_KERNEL; + HMT_MEDIUM + mr r9,r13 + GET_PACA(r13) + INTERRUPT_TO_KERNEL #endif - -#define LOAD_SYSCALL_HANDLER(reg) \ - __LOAD_HANDLER(reg, system_call_common) - -/* - * After SYSCALL_KVMTEST, we reach here with PACA in r13, r13 in r9, - * and HMT_MEDIUM. - */ -#define SYSCALL_REAL \ - mfspr r11,SPRN_SRR0 ; \ - mfspr r12,SPRN_SRR1 ; \ - LOAD_SYSCALL_HANDLER(r10) ; \ - mtspr SPRN_SRR0,r10 ; \ - ld r10,PACAKMSR(r13) ; \ - mtspr SPRN_SRR1,r10 ; \ - RFI_TO_KERNEL ; \ - b . ; /* prevent speculative execution */ #ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH -#define SYSCALL_FASTENDIAN_TEST \ -BEGIN_FTR_SECTION \ - cmpdi r0,0x1ebe ; \ - beq- 1f ; \ -END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) \ - -#define SYSCALL_FASTENDIAN \ - /* Fast LE/BE switch system call */ \ -1: mfspr r12,SPRN_SRR1 ; \ - xori r12,r12,MSR_LE ; \ - mtspr SPRN_SRR1,r12 ; \ - mr r13,r9 ; \ - RFI_TO_USER ; /* return to userspace */ \ - b . ; /* prevent speculative execution */ -#else -#define SYSCALL_FASTENDIAN_TEST -#define SYSCALL_FASTENDIAN -#endif /* CONFIG_PPC_FAST_ENDIAN_SWITCH */ +BEGIN_FTR_SECTION + cmpdi r0,0x1ebe + beq- 1f +END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) +#endif + /* We reach here with PACA in r13, r13 in r9, and HMT_MEDIUM. */ -#if defined(CONFIG_RELOCATABLE) + .if ! \virt + mfspr r11,SPRN_SRR0 + mfspr r12,SPRN_SRR1 + __LOAD_HANDLER(r10, system_call_common) + mtspr SPRN_SRR0,r10 + ld r10,PACAKMSR(r13) + mtspr SPRN_SRR1,r10 + RFI_TO_KERNEL + b . /* prevent speculative execution */ + .else +#ifdef CONFIG_RELOCATABLE /* * We can't branch directly so we do it via the CTR which * is volatile across system calls. */ -#define SYSCALL_VIRT \ - LOAD_SYSCALL_HANDLER(r10) ; \ - mtctr r10 ; \ - mfspr r11,SPRN_SRR0 ; \ - mfspr r12,SPRN_SRR1 ; \ - li r10,MSR_RI ; \ - mtmsrd r10,1 ; \ - bctr ; + __LOAD_HANDLER(r10, system_call_common) + mtctr r10 + mfspr r11,SPRN_SRR0 + mfspr r12,SPRN_SRR1 + li r10,MSR_RI + mtmsrd r10,1 + bctr #else /* We can branch directly */ -#define SYSCALL_VIRT \ - mfspr r11,SPRN_SRR0 ; \ - mfspr r12,SPRN_SRR1 ; \ - li r10,MSR_RI ; \ - mtmsrd r10,1 ; /* Set RI (EE=0) */ \ - b system_call_common ; + mfspr r11,SPRN_SRR0 + mfspr r12,SPRN_SRR1 + li r10,MSR_RI + mtmsrd r10,1 /* Set RI (EE=0) */ + b system_call_common +#endif + .endif + +#ifdef CONFIG_PPC_FAST_ENDIAN_SWITCH + /* Fast LE/BE switch system call */ +1: mfspr r12,SPRN_SRR1 + xori r12,r12,MSR_LE + mtspr SPRN_SRR1,r12 + mr r13,r9 + RFI_TO_USER /* return to userspace */ + b . /* prevent speculative execution */ #endif +.endm EXC_REAL_BEGIN(system_call, 0xc00, 0x100) - SYSCALL_KVMTEST /* loads PACA into r13, and saves r13 to r9 */ - SYSCALL_FASTENDIAN_TEST - SYSCALL_REAL - SYSCALL_FASTENDIAN + SYSTEM_CALL 0 EXC_REAL_END(system_call, 0xc00, 0x100) EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) - SYSCALL_KVMTEST /* loads PACA into r13, and saves r13 to r9 */ - SYSCALL_FASTENDIAN_TEST - SYSCALL_VIRT - SYSCALL_FASTENDIAN + SYSTEM_CALL 1 EXC_VIRT_END(system_call, 0x4c00, 0x100) #ifdef CONFIG_KVM_BOOK3S_64_HANDLER -- cgit v1.2.3 From bf66e3c4cf00bc00cdd2df90f63161eacf9c5714 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:32 +1000 Subject: powerpc/64s/exception: fix indenting irregularities Generally, macros that result in instructions being expanded are indented by a tab, and those that don't have no indent. Fix the obvious cases that go contrary to style. No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 92 ++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 46 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 73ba2e3f345b..fcc88b5446c9 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -261,16 +261,16 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) cmpwi r10,KVM_GUEST_MODE_SKIP beq 89f .else - BEGIN_FTR_SECTION_NESTED(947) +BEGIN_FTR_SECTION_NESTED(947) ld r10,\area+EX_CFAR(r13) std r10,HSTATE_CFAR(r13) - END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947) +END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947) .endif - BEGIN_FTR_SECTION_NESTED(948) +BEGIN_FTR_SECTION_NESTED(948) ld r10,\area+EX_PPR(r13) std r10,HSTATE_PPR(r13) - END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) +END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) ld r10,\area+EX_R10(r13) std r12,HSTATE_SCRATCH0(r13) sldi r12,r9,32 @@ -372,10 +372,10 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) std r9,GPR11(r1); \ std r10,GPR12(r1); \ std r11,GPR13(r1); \ - BEGIN_FTR_SECTION_NESTED(66); \ +BEGIN_FTR_SECTION_NESTED(66); \ ld r10,area+EX_CFAR(r13); \ std r10,ORIG_GPR3(r1); \ - END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ +END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ GET_CTR(r10, area); \ std r10,_CTR(r1); @@ -794,7 +794,7 @@ EXC_REAL_BEGIN(system_reset, 0x100, 0x100) * but we branch to the 0xc000... address so we can turn on relocation * with mtmsr. */ - BEGIN_FTR_SECTION +BEGIN_FTR_SECTION mfspr r10,SPRN_SRR1 rlwinm. r10,r10,47-31,30,31 beq- 1f @@ -803,7 +803,7 @@ EXC_REAL_BEGIN(system_reset, 0x100, 0x100) bltlr cr1 /* no state loss, return to idle caller */ BRANCH_TO_C000(r10, system_reset_idle_common) 1: - END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) +END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) #endif KVMTEST EXC_STD 0x100 @@ -1151,10 +1151,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) * * Go back to nap/sleep/winkle mode again if (b) is true. */ - BEGIN_FTR_SECTION +BEGIN_FTR_SECTION rlwinm. r11,r12,47-31,30,31 bne machine_check_idle_common - END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) +END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) #endif /* @@ -1261,13 +1261,13 @@ EXC_COMMON_BEGIN(mce_return) b . EXC_REAL_BEGIN(data_access, 0x300, 0x80) -SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0 PACA_EXGEN + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN b tramp_real_data_access EXC_REAL_END(data_access, 0x300, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 0 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 0 /* * DAR/DSISR must be read before setting MSR[RI], because * a d-side MCE will clobber those registers so is not @@ -1280,9 +1280,9 @@ EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 0 EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) -SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0 PACA_EXGEN -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 0 + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) @@ -1315,24 +1315,24 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) -SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0 PACA_EXSLB + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXSLB b tramp_real_data_access_slb EXC_REAL_END(data_access_slb, 0x380, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access_slb) -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380, 0 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380, 0 mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) -EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 + EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) -SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0 PACA_EXSLB -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 0 + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXSLB + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 0 mfspr r10,SPRN_DAR std r10,PACA_EXSLB+EX_DAR(r13) -EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD + EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD EXC_VIRT_END(data_access_slb, 0x4380, 0x80) TRAMP_KVM_SKIP(PACA_EXSLB, 0x380) @@ -1415,25 +1415,25 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN - BEGIN_FTR_SECTION - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED - EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_HV, 1 - FTR_SECTION_ELSE - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED - EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_STD, 1 - ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) +BEGIN_FTR_SECTION + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_HV, 1 +FTR_SECTION_ELSE + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_STD, 1 +ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) EXC_REAL_END(hardware_interrupt, 0x500, 0x100) EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN - BEGIN_FTR_SECTION - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED - EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_HV - FTR_SECTION_ELSE - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED - EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_STD - ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) +BEGIN_FTR_SECTION + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_HV +FTR_SECTION_ELSE + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_STD +ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) TRAMP_KVM(PACA_EXGEN, 0x500) @@ -1442,25 +1442,25 @@ EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ) EXC_REAL_BEGIN(alignment, 0x600, 0x100) -SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0 PACA_EXGEN -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 0 + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2_REAL alignment_common, EXC_STD, 1 + EXCEPTION_PROLOG_2_REAL alignment_common, EXC_STD, 1 EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) -SET_SCRATCH0(r13) /* save r13 */ -EXCEPTION_PROLOG_0 PACA_EXGEN -EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 0 + SET_SCRATCH0(r13) /* save r13 */ + EXCEPTION_PROLOG_0 PACA_EXGEN + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 0 mfspr r10,SPRN_DAR mfspr r11,SPRN_DSISR std r10,PACA_EXGEN+EX_DAR(r13) stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2_VIRT alignment_common, EXC_STD + EXCEPTION_PROLOG_2_VIRT alignment_common, EXC_STD EXC_VIRT_END(alignment, 0x4600, 0x100) TRAMP_KVM(PACA_EXGEN, 0x600) -- cgit v1.2.3 From f3c8b6c63e816e4770abf073ee004e63593b5260 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:33 +1000 Subject: powerpc/64s/exception: generate regs clear instructions using .rept No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index fcc88b5446c9..7de4b61bde37 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -2010,12 +2010,11 @@ BEGIN_FTR_SECTION mtmsrd r10 sync -#define FMR2(n) fmr (n), (n) ; fmr n+1, n+1 -#define FMR4(n) FMR2(n) ; FMR2(n+2) -#define FMR8(n) FMR4(n) ; FMR4(n+4) -#define FMR16(n) FMR8(n) ; FMR8(n+8) -#define FMR32(n) FMR16(n) ; FMR16(n+16) - FMR32(0) + .Lreg=0 + .rept 32 + fmr .Lreg,.Lreg + .Lreg=.Lreg+1 + .endr FTR_SECTION_ELSE /* @@ -2027,12 +2026,11 @@ FTR_SECTION_ELSE mtmsrd r10 sync -#define XVCPSGNDP2(n) XVCPSGNDP(n,n,n) ; XVCPSGNDP(n+1,n+1,n+1) -#define XVCPSGNDP4(n) XVCPSGNDP2(n) ; XVCPSGNDP2(n+2) -#define XVCPSGNDP8(n) XVCPSGNDP4(n) ; XVCPSGNDP4(n+4) -#define XVCPSGNDP16(n) XVCPSGNDP8(n) ; XVCPSGNDP8(n+8) -#define XVCPSGNDP32(n) XVCPSGNDP16(n) ; XVCPSGNDP16(n+16) - XVCPSGNDP32(0) + .Lreg=0 + .rept 32 + XVCPSGNDP(.Lreg,.Lreg,.Lreg) + .Lreg=.Lreg+1 + .endr ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_206) @@ -2043,7 +2041,12 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) * To denormalise we need to move a copy of the register to itself. * For POWER8 we need to do that for all 64 VSX registers */ - XVCPSGNDP32(32) + .Lreg=32 + .rept 32 + XVCPSGNDP(.Lreg,.Lreg,.Lreg) + .Lreg=.Lreg+1 + .endr + denorm_done: mfspr r11,SPRN_HSRR0 subi r11,r11,4 -- cgit v1.2.3 From d064151fd33b1b0ebeb732ae978cb3587e934ff9 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:34 +1000 Subject: powerpc/64s/exception: remove pointless EXCEPTION_PROLOG macro indirection No generated code change. Final vmlinux is changed only due to change in bug table line numbers. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 108 +++++++++++++++++------------------ 1 file changed, 51 insertions(+), 57 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 7de4b61bde37..9b60fa040aa6 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -326,40 +326,6 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948) std r0,GPR0(r1); /* save r0 in stackframe */ \ std r10,GPR1(r1); /* save r1 in stackframe */ \ - -/* - * The common exception prolog is used for all except a few exceptions - * such as a segment miss on a kernel address. We have to be prepared - * to take another exception from the point where we first touch the - * kernel stack onwards. - * - * On entry r13 points to the paca, r9-r13 are saved in the paca, - * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and - * SRR1, and relocation is on. - */ -#define EXCEPTION_PROLOG_COMMON(n, area) \ - andi. r10,r12,MSR_PR; /* See if coming from user */ \ - mr r10,r1; /* Save r1 */ \ - subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \ - beq- 1f; \ - ld r1,PACAKSAVE(r13); /* kernel stack to use */ \ -1: cmpdi cr1,r1,-INT_FRAME_SIZE; /* check if r1 is in userspace */ \ - blt+ cr1,3f; /* abort if it is */ \ - li r1,(n); /* will be reloaded later */ \ - sth r1,PACA_TRAP_SAVE(r13); \ - std r3,area+EX_R3(r13); \ - addi r3,r13,area; /* r3 -> where regs are saved*/ \ - RESTORE_CTR(r1, area); \ - b bad_stack; \ -3: EXCEPTION_PROLOG_COMMON_1(); \ - kuap_save_amr_and_lock r9, r10, cr1, cr0; \ - beq 4f; /* if from kernel mode */ \ - ACCOUNT_CPU_USER_ENTRY(r13, r9, r10); \ - SAVE_PPR(area, r9); \ -4: EXCEPTION_PROLOG_COMMON_2(area) \ - EXCEPTION_PROLOG_COMMON_3(n) \ - ACCOUNT_STOLEN_TIME - /* Save original regs values from save area to stack frame. */ #define EXCEPTION_PROLOG_COMMON_2(area) \ ld r9,area+EX_R9(r13); /* move r9, r10 to stackframe */ \ @@ -379,7 +345,7 @@ END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ GET_CTR(r10, area); \ std r10,_CTR(r1); -#define EXCEPTION_PROLOG_COMMON_3(n) \ +#define EXCEPTION_PROLOG_COMMON_3(trap) \ std r2,GPR2(r1); /* save r2 in stackframe */ \ SAVE_4GPRS(3, r1); /* save r3 - r6 in stackframe */ \ SAVE_2GPRS(7, r1); /* save r7, r8 in stackframe */ \ @@ -390,26 +356,45 @@ END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ mfspr r11,SPRN_XER; /* save XER in stackframe */ \ std r10,SOFTE(r1); \ std r11,_XER(r1); \ - li r9,(n)+1; \ + li r9,(trap)+1; \ std r9,_TRAP(r1); /* set trap number */ \ li r10,0; \ ld r11,exception_marker@toc(r2); \ std r10,RESULT(r1); /* clear regs->result */ \ std r11,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ -#define RUNLATCH_ON \ -BEGIN_FTR_SECTION \ - ld r3, PACA_THREAD_INFO(r13); \ - ld r4,TI_LOCAL_FLAGS(r3); \ - andi. r0,r4,_TLF_RUNLATCH; \ - beql ppc64_runlatch_on_trampoline; \ -END_FTR_SECTION_IFSET(CPU_FTR_CTRL) +/* + * On entry r13 points to the paca, r9-r13 are saved in the paca, + * r9 contains the saved CR, r11 and r12 contain the saved SRR0 and + * SRR1, and relocation is on. + */ +#define EXCEPTION_COMMON(area, trap) \ + andi. r10,r12,MSR_PR; /* See if coming from user */ \ + mr r10,r1; /* Save r1 */ \ + subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \ + beq- 1f; \ + ld r1,PACAKSAVE(r13); /* kernel stack to use */ \ +1: cmpdi cr1,r1,-INT_FRAME_SIZE; /* check if r1 is in userspace */ \ + blt+ cr1,3f; /* abort if it is */ \ + li r1,(trap); /* will be reloaded later */ \ + sth r1,PACA_TRAP_SAVE(r13); \ + std r3,area+EX_R3(r13); \ + addi r3,r13,area; /* r3 -> where regs are saved*/ \ + RESTORE_CTR(r1, area); \ + b bad_stack; \ +3: EXCEPTION_PROLOG_COMMON_1(); \ + kuap_save_amr_and_lock r9, r10, cr1, cr0; \ + beq 4f; /* if from kernel mode */ \ + ACCOUNT_CPU_USER_ENTRY(r13, r9, r10); \ + SAVE_PPR(area, r9); \ +4: EXCEPTION_PROLOG_COMMON_2(area); \ + EXCEPTION_PROLOG_COMMON_3(trap); \ + ACCOUNT_STOLEN_TIME -#define EXCEPTION_COMMON(area, trap) \ - EXCEPTION_PROLOG_COMMON(trap, area); \ /* - * Exception where stack is already set in r1, r1 is saved in r10 + * Exception where stack is already set in r1, r1 is saved in r10. + * PPR save and CPU accounting is not done (for some reason). */ #define EXCEPTION_COMMON_STACK(area, trap) \ EXCEPTION_PROLOG_COMMON_1(); \ @@ -417,6 +402,15 @@ END_FTR_SECTION_IFSET(CPU_FTR_CTRL) EXCEPTION_PROLOG_COMMON_2(area); \ EXCEPTION_PROLOG_COMMON_3(trap) + +#define RUNLATCH_ON \ +BEGIN_FTR_SECTION \ + ld r3, PACA_THREAD_INFO(r13); \ + ld r4,TI_LOCAL_FLAGS(r3); \ + andi. r0,r4,_TLF_RUNLATCH; \ + beql ppc64_runlatch_on_trampoline; \ +END_FTR_SECTION_IFSET(CPU_FTR_CTRL) + /* * When the idle code in power4_idle puts the CPU into NAP mode, * it has to do so in a loop, and relies on the external interrupt @@ -1048,7 +1042,7 @@ EXC_COMMON_BEGIN(machine_check_common) std r10,PACA_EXMC+EX_DAR(r13) mfspr r10,SPRN_DSISR stw r10,PACA_EXMC+EX_DSISR(r13) - EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC) + EXCEPTION_COMMON(PACA_EXMC, 0x200) FINISH_NAP RECONCILE_IRQ_STATE(r10, r11) ld r3,PACA_EXMC+EX_DAR(r13) @@ -1299,7 +1293,7 @@ EXC_COMMON_BEGIN(data_access_common) * r9 - r13 are saved in paca->exgen. * EX_DAR and EX_DSISR have saved DAR/DSISR */ - EXCEPTION_PROLOG_COMMON(0x300, PACA_EXGEN) + EXCEPTION_COMMON(PACA_EXGEN, 0x300) RECONCILE_IRQ_STATE(r10, r11) ld r12,_MSR(r1) ld r3,PACA_EXGEN+EX_DAR(r13) @@ -1338,7 +1332,7 @@ EXC_VIRT_END(data_access_slb, 0x4380, 0x80) TRAMP_KVM_SKIP(PACA_EXSLB, 0x380) EXC_COMMON_BEGIN(data_access_slb_common) - EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB) + EXCEPTION_COMMON(PACA_EXSLB, 0x380) ld r4,PACA_EXSLB+EX_DAR(r13) std r4,_DAR(r1) addi r3,r1,STACK_FRAME_OVERHEAD @@ -1368,7 +1362,7 @@ EXC_VIRT(instruction_access, 0x4400, 0x80, 0x400) TRAMP_KVM(PACA_EXGEN, 0x400) EXC_COMMON_BEGIN(instruction_access_common) - EXCEPTION_PROLOG_COMMON(0x400, PACA_EXGEN) + EXCEPTION_COMMON(PACA_EXGEN, 0x400) RECONCILE_IRQ_STATE(r10, r11) ld r12,_MSR(r1) ld r3,_NIP(r1) @@ -1388,7 +1382,7 @@ __EXC_VIRT(instruction_access_slb, 0x4480, 0x80, 0x480, PACA_EXSLB) TRAMP_KVM(PACA_EXSLB, 0x480) EXC_COMMON_BEGIN(instruction_access_slb_common) - EXCEPTION_PROLOG_COMMON(0x480, PACA_EXSLB) + EXCEPTION_COMMON(PACA_EXSLB, 0x480) ld r4,_NIP(r1) addi r3,r1,STACK_FRAME_OVERHEAD BEGIN_MMU_FTR_SECTION @@ -1465,7 +1459,7 @@ EXC_VIRT_END(alignment, 0x4600, 0x100) TRAMP_KVM(PACA_EXGEN, 0x600) EXC_COMMON_BEGIN(alignment_common) - EXCEPTION_PROLOG_COMMON(0x600, PACA_EXGEN) + EXCEPTION_COMMON(PACA_EXGEN, 0x600) ld r3,PACA_EXGEN+EX_DAR(r13) lwz r4,PACA_EXGEN+EX_DSISR(r13) std r3,_DAR(r1) @@ -1503,7 +1497,7 @@ EXC_COMMON_BEGIN(program_check_common) ld r1,PACAEMERGSP(r13) /* Use emergency stack */ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ b 3f /* Jump into the macro !! */ -1: EXCEPTION_PROLOG_COMMON(0x700, PACA_EXGEN) +1: EXCEPTION_COMMON(PACA_EXGEN, 0x700) bl save_nvgprs RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD @@ -1515,7 +1509,7 @@ EXC_REAL(fp_unavailable, 0x800, 0x100) EXC_VIRT(fp_unavailable, 0x4800, 0x100, 0x800) TRAMP_KVM(PACA_EXGEN, 0x800) EXC_COMMON_BEGIN(fp_unavailable_common) - EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN) + EXCEPTION_COMMON(PACA_EXGEN, 0x800) bne 1f /* if from user, just load it up */ bl save_nvgprs RECONCILE_IRQ_STATE(r10, r11) @@ -1728,7 +1722,7 @@ EXC_COMMON_BEGIN(h_data_storage_common) std r10,PACA_EXGEN+EX_DAR(r13) mfspr r10,SPRN_HDSISR stw r10,PACA_EXGEN+EX_DSISR(r13) - EXCEPTION_PROLOG_COMMON(0xe00, PACA_EXGEN) + EXCEPTION_COMMON(PACA_EXGEN, 0xe00) bl save_nvgprs RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD @@ -1860,7 +1854,7 @@ EXC_REAL_OOL(altivec_unavailable, 0xf20, 0x20) EXC_VIRT_OOL(altivec_unavailable, 0x4f20, 0x20, 0xf20) TRAMP_KVM(PACA_EXGEN, 0xf20) EXC_COMMON_BEGIN(altivec_unavailable_common) - EXCEPTION_PROLOG_COMMON(0xf20, PACA_EXGEN) + EXCEPTION_COMMON(PACA_EXGEN, 0xf20) #ifdef CONFIG_ALTIVEC BEGIN_FTR_SECTION beq 1f @@ -1897,7 +1891,7 @@ EXC_REAL_OOL(vsx_unavailable, 0xf40, 0x20) EXC_VIRT_OOL(vsx_unavailable, 0x4f40, 0x20, 0xf40) TRAMP_KVM(PACA_EXGEN, 0xf40) EXC_COMMON_BEGIN(vsx_unavailable_common) - EXCEPTION_PROLOG_COMMON(0xf40, PACA_EXGEN) + EXCEPTION_COMMON(PACA_EXGEN, 0xf40) #ifdef CONFIG_VSX BEGIN_FTR_SECTION beq 1f -- cgit v1.2.3 From 1582009113bc1336d4e54de1d94014ca3569c483 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Jun 2019 23:15:35 +1000 Subject: powerpc/64s/exception: move paca save area offsets into exception-64s.S No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 17 +++-------------- arch/powerpc/kernel/exceptions-64s.S | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 14 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 3585e1d7e898..40d114e69cbe 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -36,22 +36,11 @@ */ #include -/* PACA save area offsets (exgen, exmc, etc) */ -#define EX_R9 0 -#define EX_R10 8 -#define EX_R11 16 -#define EX_R12 24 -#define EX_R13 32 -#define EX_DAR 40 -#define EX_DSISR 48 -#define EX_CCR 52 -#define EX_CFAR 56 -#define EX_PPR 64 +/* PACA save area size in u64 units (exgen, exmc, etc) */ #if defined(CONFIG_RELOCATABLE) -#define EX_CTR 72 -#define EX_SIZE 10 /* size in u64 units */ +#define EX_SIZE 10 #else -#define EX_SIZE 9 /* size in u64 units */ +#define EX_SIZE 9 #endif /* diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 9b60fa040aa6..4dfccc2efc95 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -21,6 +21,28 @@ #include #include +/* PACA save area offsets (exgen, exmc, etc) */ +#define EX_R9 0 +#define EX_R10 8 +#define EX_R11 16 +#define EX_R12 24 +#define EX_R13 32 +#define EX_DAR 40 +#define EX_DSISR 48 +#define EX_CCR 52 +#define EX_CFAR 56 +#define EX_PPR 64 +#if defined(CONFIG_RELOCATABLE) +#define EX_CTR 72 +.if EX_SIZE != 10 + .error "EX_SIZE is wrong" +.endif +#else +.if EX_SIZE != 9 + .error "EX_SIZE is wrong" +.endif +#endif + /* * We're short on space and time in the exception prolog, so we can't * use the normal LOAD_REG_IMMEDIATE macro to load the address of label. -- cgit v1.2.3 From b0b2a93da4c95ac808c4c43f24a1873ae3166a1a Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:20 +1000 Subject: powerpc/64s/exception: clean up system call entry syscall / hcall entry unnecessarily differs between KVM and non-KVM builds. Move the SMT priority instruction to the same location (after INTERRUPT_TO_KERNEL). Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 4dfccc2efc95..564a77039701 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1638,10 +1638,8 @@ EXC_COMMON(trap_0b_common, 0xb00, unknown_exception) std r10,PACA_EXGEN+EX_R10(r13) INTERRUPT_TO_KERNEL KVMTEST EXC_STD 0xc00 /* uses r10, branch to do_kvm_0xc00_system_call */ - HMT_MEDIUM mfctr r9 #else - HMT_MEDIUM mr r9,r13 GET_PACA(r13) INTERRUPT_TO_KERNEL @@ -1653,11 +1651,14 @@ BEGIN_FTR_SECTION beq- 1f END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) #endif - /* We reach here with PACA in r13, r13 in r9, and HMT_MEDIUM. */ - .if ! \virt + /* We reach here with PACA in r13, r13 in r9. */ mfspr r11,SPRN_SRR0 mfspr r12,SPRN_SRR1 + + HMT_MEDIUM + + .if ! \virt __LOAD_HANDLER(r10, system_call_common) mtspr SPRN_SRR0,r10 ld r10,PACAKMSR(r13) @@ -1665,24 +1666,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_REAL_LE) RFI_TO_KERNEL b . /* prevent speculative execution */ .else + li r10,MSR_RI + mtmsrd r10,1 /* Set RI (EE=0) */ #ifdef CONFIG_RELOCATABLE - /* - * We can't branch directly so we do it via the CTR which - * is volatile across system calls. - */ __LOAD_HANDLER(r10, system_call_common) mtctr r10 - mfspr r11,SPRN_SRR0 - mfspr r12,SPRN_SRR1 - li r10,MSR_RI - mtmsrd r10,1 bctr #else - /* We can branch directly */ - mfspr r11,SPRN_SRR0 - mfspr r12,SPRN_SRR1 - li r10,MSR_RI - mtmsrd r10,1 /* Set RI (EE=0) */ b system_call_common #endif .endif -- cgit v1.2.3 From 63d60d0c6946fd6874a099540e0ac9e26661f8dd Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:21 +1000 Subject: powerpc/64s/exception: avoid SPR RAW scoreboard stall in real mode entry Move SPR reads ahead of writes. Real mode entry that is not a KVM guest is rare these days, but bad practice propagates. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 564a77039701..3735b9d1bb65 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -183,19 +183,19 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) .endif .if \hsrr mfspr r11,SPRN_HSRR0 /* save HSRR0 */ + mfspr r12,SPRN_HSRR1 /* and HSRR1 */ + mtspr SPRN_HSRR1,r10 .else mfspr r11,SPRN_SRR0 /* save SRR0 */ + mfspr r12,SPRN_SRR1 /* and SRR1 */ + mtspr SPRN_SRR1,r10 .endif - LOAD_HANDLER(r12, \label\()) + LOAD_HANDLER(r10, \label\()) .if \hsrr - mtspr SPRN_HSRR0,r12 - mfspr r12,SPRN_HSRR1 /* and HSRR1 */ - mtspr SPRN_HSRR1,r10 + mtspr SPRN_HSRR0,r10 HRFI_TO_KERNEL .else - mtspr SPRN_SRR0,r12 - mfspr r12,SPRN_SRR1 /* and SRR1 */ - mtspr SPRN_SRR1,r10 + mtspr SPRN_SRR0,r10 RFI_TO_KERNEL .endif b . /* prevent speculative execution */ -- cgit v1.2.3 From fbc50063a2358f685deaa2f8370a55e3df5ef3ab Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:22 +1000 Subject: powerpc/64s/exception: mtmsrd L=1 cleanup All supported 64s CPUs support mtmsrd L=1 instruction, so a cleanup can be made in sreset and mce handlers. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 3735b9d1bb65..33c253fe8a68 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -882,11 +882,8 @@ EXC_COMMON_BEGIN(system_reset_common) addi r3,r1,STACK_FRAME_OVERHEAD bl system_reset_exception - /* This (and MCE) can be simplified with mtmsrd L=1 */ /* Clear MSR_RI before setting SRR0 and SRR1. */ - li r0,MSR_RI - mfmsr r9 - andc r9,r9,r0 + li r9,0 mtmsrd r9,1 /* @@ -1081,9 +1078,7 @@ EXC_COMMON_BEGIN(machine_check_common) #define MACHINE_CHECK_HANDLER_WINDUP \ /* Clear MSR_RI before setting SRR0 and SRR1. */\ - li r0,MSR_RI; \ - mfmsr r9; /* get MSR value */ \ - andc r9,r9,r0; \ + li r9,0; \ mtmsrd r9,1; /* Clear MSR_RI */ \ /* Move original SRR0 and SRR1 into the respective regs */ \ ld r9,_MSR(r1); \ -- cgit v1.2.3 From 9592b29a9cf14db596c752349e7d5b7a29d3bf90 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:23 +1000 Subject: powerpc/64s/exception: windup use r9 consistently to restore SPRs Trivial code change, r3->r9. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 33c253fe8a68..823e930b6e3e 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -909,8 +909,8 @@ EXC_COMMON_BEGIN(system_reset_common) /* Move original SRR0 and SRR1 into the respective regs */ ld r9,_MSR(r1) mtspr SPRN_SRR1,r9 - ld r3,_NIP(r1) - mtspr SPRN_SRR0,r3 + ld r9,_NIP(r1) + mtspr SPRN_SRR0,r9 ld r9,_CTR(r1) mtctr r9 ld r9,_XER(r1) @@ -1083,8 +1083,8 @@ EXC_COMMON_BEGIN(machine_check_common) /* Move original SRR0 and SRR1 into the respective regs */ \ ld r9,_MSR(r1); \ mtspr SPRN_SRR1,r9; \ - ld r3,_NIP(r1); \ - mtspr SPRN_SRR0,r3; \ + ld r9,_NIP(r1); \ + mtspr SPRN_SRR0,r9; \ ld r9,_CTR(r1); \ mtctr r9; \ ld r9,_XER(r1); \ @@ -1786,8 +1786,8 @@ TRAMP_REAL_BEGIN(hmi_exception_early) /* Move original HSRR0 and HSRR1 into the respective regs */ ld r9,_MSR(r1) mtspr SPRN_HSRR1,r9 - ld r3,_NIP(r1) - mtspr SPRN_HSRR0,r3 + ld r9,_NIP(r1) + mtspr SPRN_HSRR0,r9 ld r9,_CTR(r1) mtctr r9 ld r9,_XER(r1) -- cgit v1.2.3 From ad73d8d4f4a7994b234020fc6484c87672b0c404 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:24 +1000 Subject: powerpc/64s/exception: move machine check windup in_mce handling Move in_mce decrement earlier before registers are restored (but still after RI=0). This helps with later consolidation. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 823e930b6e3e..d8f1d70c675f 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1080,6 +1080,10 @@ EXC_COMMON_BEGIN(machine_check_common) /* Clear MSR_RI before setting SRR0 and SRR1. */\ li r9,0; \ mtmsrd r9,1; /* Clear MSR_RI */ \ + /* Decrement paca->in_mce now RI is clear. */ \ + lhz r12,PACA_IN_MCE(r13); \ + subi r12,r12,1; \ + sth r12,PACA_IN_MCE(r13); \ /* Move original SRR0 and SRR1 into the respective regs */ \ ld r9,_MSR(r1); \ mtspr SPRN_SRR1,r9; \ @@ -1096,10 +1100,6 @@ EXC_COMMON_BEGIN(machine_check_common) REST_GPR(10, r1); \ ld r11,_CCR(r1); \ mtcr r11; \ - /* Decrement paca->in_mce. */ \ - lhz r12,PACA_IN_MCE(r13); \ - subi r12,r12,1; \ - sth r12,PACA_IN_MCE(r13); \ REST_GPR(11, r1); \ REST_2GPRS(12, r1); \ /* restore original r1. */ \ -- cgit v1.2.3 From 67d4160a61a0dd63eef10392e4154433a0a64ac3 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:25 +1000 Subject: powerpc/64s/exception: simplify hmi windup code Duplicate the hmi windup code for both cases, rather than to put a special case branch in the middle of it. Remove unused label. This helps with later code consolidation. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index d8f1d70c675f..cbed1e42d85c 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1781,6 +1781,7 @@ TRAMP_REAL_BEGIN(hmi_exception_early) addi r3,r1,STACK_FRAME_OVERHEAD BRANCH_LINK_TO_FAR(DOTSYM(hmi_exception_realmode)) /* Function call ABI */ cmpdi cr0,r3,0 + bne 1f /* Windup the stack. */ /* Move original HSRR0 and HSRR1 into the respective regs */ @@ -1799,13 +1800,28 @@ TRAMP_REAL_BEGIN(hmi_exception_early) REST_GPR(10, r1) ld r11,_CCR(r1) REST_2GPRS(12, r1) - bne 1f mtcr r11 REST_GPR(11, r1) ld r1,GPR1(r1) HRFI_TO_USER_OR_KERNEL -1: mtcr r11 +1: + ld r9,_MSR(r1) + mtspr SPRN_HSRR1,r9 + ld r9,_NIP(r1) + mtspr SPRN_HSRR0,r9 + ld r9,_CTR(r1) + mtctr r9 + ld r9,_XER(r1) + mtxer r9 + ld r9,_LINK(r1) + mtlr r9 + REST_GPR(0, r1) + REST_8GPRS(2, r1) + REST_GPR(10, r1) + ld r11,_CCR(r1) + REST_2GPRS(12, r1) + mtcr r11 REST_GPR(11, r1) ld r1,GPR1(r1) @@ -1813,8 +1829,6 @@ TRAMP_REAL_BEGIN(hmi_exception_early) * Go to virtual mode and pull the HMI event information from * firmware. */ - .globl hmi_exception_after_realmode -hmi_exception_after_realmode: SET_SCRATCH0(r13) EXCEPTION_PROLOG_0 PACA_EXGEN b tramp_real_hmi_exception -- cgit v1.2.3 From b113c083414ed91c116483ebe74fb20c8b1ddcd3 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:26 +1000 Subject: powerpc/64s/exception: shuffle windup code around Restore all SPRs and CR up-front, these are longer latency instructions. Move register restore around to maximise pairs of adjacent loads (e.g., restore r0 next to r1). Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 40 +++++++++++++++--------------------- 1 file changed, 16 insertions(+), 24 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index cbed1e42d85c..b5060824303b 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -917,13 +917,11 @@ EXC_COMMON_BEGIN(system_reset_common) mtxer r9 ld r9,_LINK(r1) mtlr r9 - REST_GPR(0, r1) + ld r9,_CCR(r1) + mtcr r9 REST_8GPRS(2, r1) - REST_GPR(10, r1) - ld r11,_CCR(r1) - mtcr r11 - REST_GPR(11, r1) - REST_2GPRS(12, r1) + REST_4GPRS(10, r1) + REST_GPR(0, r1) /* restore original r1. */ ld r1,GPR1(r1) RFI_TO_USER_OR_KERNEL @@ -1095,13 +1093,11 @@ EXC_COMMON_BEGIN(machine_check_common) mtxer r9; \ ld r9,_LINK(r1); \ mtlr r9; \ - REST_GPR(0, r1); \ + ld r9,_CCR(r1); \ + mtcr r9; \ REST_8GPRS(2, r1); \ - REST_GPR(10, r1); \ - ld r11,_CCR(r1); \ - mtcr r11; \ - REST_GPR(11, r1); \ - REST_2GPRS(12, r1); \ + REST_4GPRS(10, r1); \ + REST_GPR(0, r1); \ /* restore original r1. */ \ ld r1,GPR1(r1) @@ -1795,13 +1791,11 @@ TRAMP_REAL_BEGIN(hmi_exception_early) mtxer r9 ld r9,_LINK(r1) mtlr r9 - REST_GPR(0, r1) + ld r9,_CCR(r1) + mtcr r9 REST_8GPRS(2, r1) - REST_GPR(10, r1) - ld r11,_CCR(r1) - REST_2GPRS(12, r1) - mtcr r11 - REST_GPR(11, r1) + REST_4GPRS(10, r1) + REST_GPR(0, r1) ld r1,GPR1(r1) HRFI_TO_USER_OR_KERNEL @@ -1816,13 +1810,11 @@ TRAMP_REAL_BEGIN(hmi_exception_early) mtxer r9 ld r9,_LINK(r1) mtlr r9 - REST_GPR(0, r1) + ld r9,_CCR(r1) + mtcr r9 REST_8GPRS(2, r1) - REST_GPR(10, r1) - ld r11,_CCR(r1) - REST_2GPRS(12, r1) - mtcr r11 - REST_GPR(11, r1) + REST_4GPRS(10, r1) + REST_GPR(0, r1) ld r1,GPR1(r1) /* -- cgit v1.2.3 From 391e941b891543f5d79c6be2f8f678cc15f7da5a Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:27 +1000 Subject: powerpc/64s/exception: use common macro for windup No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 112 +++++++++++------------------------ 1 file changed, 36 insertions(+), 76 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index b5060824303b..20e758ee704b 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -424,6 +424,38 @@ END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ EXCEPTION_PROLOG_COMMON_2(area); \ EXCEPTION_PROLOG_COMMON_3(trap) +/* + * Restore all registers including H/SRR0/1 saved in a stack frame of a + * standard exception. + */ +.macro EXCEPTION_RESTORE_REGS hsrr + /* Move original SRR0 and SRR1 into the respective regs */ + ld r9,_MSR(r1) + .if \hsrr + mtspr SPRN_HSRR1,r9 + .else + mtspr SPRN_SRR1,r9 + .endif + ld r9,_NIP(r1) + .if \hsrr + mtspr SPRN_HSRR0,r9 + .else + mtspr SPRN_SRR0,r9 + .endif + ld r9,_CTR(r1) + mtctr r9 + ld r9,_XER(r1) + mtxer r9 + ld r9,_LINK(r1) + mtlr r9 + ld r9,_CCR(r1) + mtcr r9 + REST_8GPRS(2, r1) + REST_4GPRS(10, r1) + REST_GPR(0, r1) + /* restore original r1. */ + ld r1,GPR1(r1) +.endm #define RUNLATCH_ON \ BEGIN_FTR_SECTION \ @@ -901,29 +933,7 @@ EXC_COMMON_BEGIN(system_reset_common) ld r10,SOFTE(r1) stb r10,PACAIRQSOFTMASK(r13) - /* - * Keep below code in synch with MACHINE_CHECK_HANDLER_WINDUP. - * Should share common bits... - */ - - /* Move original SRR0 and SRR1 into the respective regs */ - ld r9,_MSR(r1) - mtspr SPRN_SRR1,r9 - ld r9,_NIP(r1) - mtspr SPRN_SRR0,r9 - ld r9,_CTR(r1) - mtctr r9 - ld r9,_XER(r1) - mtxer r9 - ld r9,_LINK(r1) - mtlr r9 - ld r9,_CCR(r1) - mtcr r9 - REST_8GPRS(2, r1) - REST_4GPRS(10, r1) - REST_GPR(0, r1) - /* restore original r1. */ - ld r1,GPR1(r1) + EXCEPTION_RESTORE_REGS EXC_STD RFI_TO_USER_OR_KERNEL #ifdef CONFIG_PPC_PSERIES @@ -1082,24 +1092,7 @@ EXC_COMMON_BEGIN(machine_check_common) lhz r12,PACA_IN_MCE(r13); \ subi r12,r12,1; \ sth r12,PACA_IN_MCE(r13); \ - /* Move original SRR0 and SRR1 into the respective regs */ \ - ld r9,_MSR(r1); \ - mtspr SPRN_SRR1,r9; \ - ld r9,_NIP(r1); \ - mtspr SPRN_SRR0,r9; \ - ld r9,_CTR(r1); \ - mtctr r9; \ - ld r9,_XER(r1); \ - mtxer r9; \ - ld r9,_LINK(r1); \ - mtlr r9; \ - ld r9,_CCR(r1); \ - mtcr r9; \ - REST_8GPRS(2, r1); \ - REST_4GPRS(10, r1); \ - REST_GPR(0, r1); \ - /* restore original r1. */ \ - ld r1,GPR1(r1) + EXCEPTION_RESTORE_REGS EXC_STD #ifdef CONFIG_PPC_P7_NAP /* @@ -1779,48 +1772,15 @@ TRAMP_REAL_BEGIN(hmi_exception_early) cmpdi cr0,r3,0 bne 1f - /* Windup the stack. */ - /* Move original HSRR0 and HSRR1 into the respective regs */ - ld r9,_MSR(r1) - mtspr SPRN_HSRR1,r9 - ld r9,_NIP(r1) - mtspr SPRN_HSRR0,r9 - ld r9,_CTR(r1) - mtctr r9 - ld r9,_XER(r1) - mtxer r9 - ld r9,_LINK(r1) - mtlr r9 - ld r9,_CCR(r1) - mtcr r9 - REST_8GPRS(2, r1) - REST_4GPRS(10, r1) - REST_GPR(0, r1) - ld r1,GPR1(r1) + EXCEPTION_RESTORE_REGS EXC_HV HRFI_TO_USER_OR_KERNEL 1: - ld r9,_MSR(r1) - mtspr SPRN_HSRR1,r9 - ld r9,_NIP(r1) - mtspr SPRN_HSRR0,r9 - ld r9,_CTR(r1) - mtctr r9 - ld r9,_XER(r1) - mtxer r9 - ld r9,_LINK(r1) - mtlr r9 - ld r9,_CCR(r1) - mtcr r9 - REST_8GPRS(2, r1) - REST_4GPRS(10, r1) - REST_GPR(0, r1) - ld r1,GPR1(r1) - /* * Go to virtual mode and pull the HMI event information from * firmware. */ + EXCEPTION_RESTORE_REGS EXC_HV SET_SCRATCH0(r13) EXCEPTION_PROLOG_0 PACA_EXGEN b tramp_real_hmi_exception -- cgit v1.2.3 From 5312c4941e0d0185bb2c95d666d607292b7e3e57 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:28 +1000 Subject: powerpc/64s/exception: add dar and dsisr options to exception macro Some exception entry requires DAR and/or DSISR to be saved into the paca exception save area. Add options to the standard exception macros for these. Generated code changes slightly due to code structure. - 554: a6 02 72 7d mfdsisr r11 - 558: a8 00 4d f9 std r10,168(r13) - 55c: b0 00 6d 91 stw r11,176(r13) + 554: a8 00 4d f9 std r10,168(r13) + 558: a6 02 52 7d mfdsisr r10 + 55c: b0 00 4d 91 stw r10,176(r13) Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 101 ++++++++++++++++------------------- 1 file changed, 45 insertions(+), 56 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 20e758ee704b..aca162ed12e1 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -136,7 +136,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR) .endm -.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, bitmask +.macro EXCEPTION_PROLOG_1 hsrr, area, kvm, vec, dar, dsisr, bitmask OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR) OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR) INTERRUPT_TO_KERNEL @@ -172,8 +172,22 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) std r11,\area\()+EX_R11(r13) std r12,\area\()+EX_R12(r13) + + /* + * DAR/DSISR, SCRATCH0 must be read before setting MSR[RI], + * because a d-side MCE will clobber those registers so is + * not recoverable if they are live. + */ GET_SCRATCH0(r10) std r10,\area\()+EX_R13(r13) + .if \dar + mfspr r10,SPRN_DAR + std r10,\area\()+EX_DAR(r13) + .endif + .if \dsisr + mfspr r10,SPRN_DSISR + stw r10,\area\()+EX_DSISR(r13) + .endif .endm .macro EXCEPTION_PROLOG_2_REAL label, hsrr, set_ri @@ -535,7 +549,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) EXC_REAL_BEGIN(name, start, size); \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 area ; \ - EXCEPTION_PROLOG_1 EXC_STD, area, 1, start, 0 ; \ + EXCEPTION_PROLOG_1 EXC_STD, area, 1, start, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ EXC_REAL_END(name, start, size) @@ -546,7 +560,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) EXC_VIRT_BEGIN(name, start, size); \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 area ; \ - EXCEPTION_PROLOG_1 EXC_STD, area, 0, realvec, 0; \ + EXCEPTION_PROLOG_1 EXC_STD, area, 0, realvec, 0, 0, 0; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ EXC_VIRT_END(name, start, size) @@ -557,7 +571,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) EXC_REAL_BEGIN(name, start, size); \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, start, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, start, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ EXC_REAL_END(name, start, size) @@ -565,7 +579,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) EXC_VIRT_BEGIN(name, start, size); \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ EXC_VIRT_END(name, start, size) @@ -573,7 +587,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) EXC_REAL_BEGIN(name, start, size); \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN; \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, start, 0 ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, start, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 ; \ EXC_REAL_END(name, start, size) @@ -581,7 +595,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) EXC_VIRT_BEGIN(name, start, size); \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN; \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV ; \ EXC_VIRT_END(name, start, size) @@ -594,7 +608,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_REAL_OOL(name, vec) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0 ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 #define EXC_REAL_OOL(name, start, size) \ @@ -606,7 +620,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, vec, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 #define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask) \ @@ -625,7 +639,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_REAL_OOL_HV(name, vec) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0 ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 #define EXC_REAL_OOL_HV(name, start, size) \ @@ -637,7 +651,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, vec, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 #define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask) \ @@ -653,7 +667,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_VIRT_OOL(name, realvec) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0 ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, vec, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD #define EXC_VIRT_OOL(name, start, size, realvec) \ @@ -665,7 +679,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 #define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask) \ @@ -677,7 +691,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_VIRT_OOL_HV(name, realvec) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0 ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV #define EXC_VIRT_OOL_HV(name, start, size, realvec) \ @@ -689,7 +703,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, bitmask ; \ + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV #define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask) \ @@ -944,7 +958,7 @@ TRAMP_REAL_BEGIN(system_reset_fwnmi) SET_SCRATCH0(r13) /* save r13 */ /* See comment at system_reset exception, don't turn on RI */ EXCEPTION_PROLOG_0 PACA_EXNMI - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0, 0, 0 EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0 #endif /* CONFIG_PPC_PSERIES */ @@ -965,7 +979,7 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_REAL_END(machine_check, 0x200, 0x100) EXC_VIRT_NONE(0x4200, 0x100) TRAMP_REAL_BEGIN(machine_check_common_early) - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 0, 0x200, 0 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 0, 0x200, 0, 0, 0 /* * Register contents: * R13 = PACA @@ -1050,7 +1064,7 @@ BEGIN_FTR_SECTION b machine_check_common_early END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) machine_check_pSeries_0: - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200, 0 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200, 0, 0, 0 /* * MSR_RI is not enabled, because PACA_EXMC is being used, so a * nested machine check corrupts it. machine_check_common enables @@ -1267,26 +1281,13 @@ EXC_REAL_BEGIN(data_access, 0x300, 0x80) EXC_REAL_END(data_access, 0x300, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access) - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 0 - /* - * DAR/DSISR must be read before setting MSR[RI], because - * a d-side MCE will clobber those registers so is not - * recoverable if they are live. - */ - mfspr r10,SPRN_DAR - mfspr r11,SPRN_DSISR - std r10,PACA_EXGEN+EX_DAR(r13) - stw r11,PACA_EXGEN+EX_DSISR(r13) -EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x300, 1, 1, 0 + EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 0 - mfspr r10,SPRN_DAR - mfspr r11,SPRN_DSISR - std r10,PACA_EXGEN+EX_DAR(r13) - stw r11,PACA_EXGEN+EX_DSISR(r13) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 1, 1, 0 EXCEPTION_PROLOG_2_VIRT data_access_common, EXC_STD EXC_VIRT_END(data_access, 0x4300, 0x80) @@ -1321,17 +1322,13 @@ EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) EXC_REAL_END(data_access_slb, 0x380, 0x80) TRAMP_REAL_BEGIN(tramp_real_data_access_slb) - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380, 0 - mfspr r10,SPRN_DAR - std r10,PACA_EXSLB+EX_DAR(r13) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 1, 0x380, 1, 0, 0 EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXSLB - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 0 - mfspr r10,SPRN_DAR - std r10,PACA_EXSLB+EX_DAR(r13) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 1, 0, 0 EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD EXC_VIRT_END(data_access_slb, 0x4380, 0x80) @@ -1416,10 +1413,10 @@ EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN BEGIN_FTR_SECTION - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_HV, 1 FTR_SECTION_ELSE - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED EXCEPTION_PROLOG_2_REAL hardware_interrupt_common, EXC_STD, 1 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) EXC_REAL_END(hardware_interrupt, 0x500, 0x100) @@ -1428,10 +1425,10 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN BEGIN_FTR_SECTION - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_HV FTR_SECTION_ELSE - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, IRQS_DISABLED + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED EXCEPTION_PROLOG_2_VIRT hardware_interrupt_common, EXC_STD ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) @@ -1444,22 +1441,14 @@ EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ) EXC_REAL_BEGIN(alignment, 0x600, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 0 - mfspr r10,SPRN_DAR - mfspr r11,SPRN_DSISR - std r10,PACA_EXGEN+EX_DAR(r13) - stw r11,PACA_EXGEN+EX_DSISR(r13) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 1, 1, 0 EXCEPTION_PROLOG_2_REAL alignment_common, EXC_STD, 1 EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 0 - mfspr r10,SPRN_DAR - mfspr r11,SPRN_DSISR - std r10,PACA_EXGEN+EX_DAR(r13) - stw r11,PACA_EXGEN+EX_DSISR(r13) + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 1, 1, 0 EXCEPTION_PROLOG_2_VIRT alignment_common, EXC_STD EXC_VIRT_END(alignment, 0x4600, 0x100) @@ -1757,7 +1746,7 @@ __TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED) EXC_VIRT_NONE(0x4e60, 0x20) TRAMP_KVM_HV(PACA_EXGEN, 0xe60) TRAMP_REAL_BEGIN(hmi_exception_early) - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0 + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0, 0, 0 mr r10,r1 /* Save r1 */ ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ @@ -1942,7 +1931,7 @@ EXC_VIRT_NONE(0x5400, 0x100) EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) mtspr SPRN_SPRG_HSCRATCH0,r13 EXCEPTION_PROLOG_0 PACA_EXGEN - EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0 + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0, 0, 0 #ifdef CONFIG_PPC_DENORMALISATION mfspr r10,SPRN_HSRR1 -- cgit v1.2.3 From 02a1258154eec78783bd7e90d4ee1361c853db66 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:29 +1000 Subject: powerpc/64s/exception: machine check use standard macros to save dar/dsisr Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index aca162ed12e1..a39d5b47acbc 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1064,7 +1064,7 @@ BEGIN_FTR_SECTION b machine_check_common_early END_FTR_SECTION_IFCLR(CPU_FTR_HVMODE) machine_check_pSeries_0: - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200, 0, 0, 0 + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXMC, 1, 0x200, 1, 1, 0 /* * MSR_RI is not enabled, because PACA_EXMC is being used, so a * nested machine check corrupts it. machine_check_common enables @@ -1079,10 +1079,6 @@ EXC_COMMON_BEGIN(machine_check_common) * Machine check is different because we use a different * save area: PACA_EXMC instead of PACA_EXGEN. */ - mfspr r10,SPRN_DAR - std r10,PACA_EXMC+EX_DAR(r13) - mfspr r10,SPRN_DSISR - stw r10,PACA_EXMC+EX_DSISR(r13) EXCEPTION_COMMON(PACA_EXMC, 0x200) FINISH_NAP RECONCILE_IRQ_STATE(r10, r11) -- cgit v1.2.3 From 904f81f3f36d84a4d26c712a9c7741a397a4f348 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:30 +1000 Subject: powerpc/64s/exception: denorm handler use standard scratch save macro Although the 0x1500 interrupt only applies to bare metal, it is better to just use the standard macro for scratch save. Runtime code path remains unchanged (due to instruction patching). Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index a39d5b47acbc..836cf36df475 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1925,7 +1925,7 @@ EXC_REAL_NONE(0x1400, 0x100) EXC_VIRT_NONE(0x5400, 0x100) EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) - mtspr SPRN_SPRG_HSCRATCH0,r13 + SET_SCRATCH0(r13) EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0, 0, 0 -- cgit v1.2.3 From d7fb34c704719d3c91455297678aa75d9da95817 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:31 +1000 Subject: powerpc/64s/exception: move SET_SCRATCH0 into EXCEPTION_PROLOG_0 No generated code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 836cf36df475..b7c15a2df4a0 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -128,6 +128,7 @@ BEGIN_FTR_SECTION_NESTED(943) \ END_FTR_SECTION_NESTED(ftr,ftr,943) .macro EXCEPTION_PROLOG_0 area + SET_SCRATCH0(r13) /* save r13 */ GET_PACA(r13) std r9,\area\()+EX_R9(r13) /* save r9 */ OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR) @@ -547,7 +548,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __EXC_REAL(name, start, size, area) \ EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 area ; \ EXCEPTION_PROLOG_1 EXC_STD, area, 1, start, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ @@ -558,7 +558,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __EXC_VIRT(name, start, size, realvec, area) \ EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 area ; \ EXCEPTION_PROLOG_1 EXC_STD, area, 0, realvec, 0, 0, 0; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ @@ -569,7 +568,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define EXC_REAL_MASKABLE(name, start, size, bitmask) \ EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN ; \ EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, start, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_STD, 1 ; \ @@ -577,7 +575,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask) \ EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN ; \ EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, realvec, 0, 0, bitmask ; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_STD ; \ @@ -585,7 +582,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define EXC_REAL_HV(name, start, size) \ EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN; \ EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, start, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_REAL name##_common, EXC_HV, 1 ; \ @@ -593,7 +589,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define EXC_VIRT_HV(name, start, size, realvec) \ EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0 PACA_EXGEN; \ EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, realvec, 0, 0, 0 ; \ EXCEPTION_PROLOG_2_VIRT name##_common, EXC_HV ; \ @@ -601,7 +596,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __EXC_REAL_OOL(name, start, size) \ EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); \ EXCEPTION_PROLOG_0 PACA_EXGEN ; \ b tramp_real_##name ; \ EXC_REAL_END(name, start, size) @@ -629,7 +623,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler) \ EXC_REAL_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); \ EXCEPTION_PROLOG_0 PACA_EXGEN ; \ b handler; \ EXC_REAL_END(name, start, size) @@ -660,7 +653,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) #define __EXC_VIRT_OOL(name, start, size) \ EXC_VIRT_BEGIN(name, start, size); \ - SET_SCRATCH0(r13); \ EXCEPTION_PROLOG_0 PACA_EXGEN ; \ b tramp_virt_##name; \ EXC_VIRT_END(name, start, size) @@ -837,7 +829,6 @@ EXC_VIRT_NONE(0x4000, 0x100) EXC_REAL_BEGIN(system_reset, 0x100, 0x100) - SET_SCRATCH0(r13) EXCEPTION_PROLOG_0 PACA_EXNMI /* This is EXCEPTION_PROLOG_1 with the idle feature section added */ @@ -955,7 +946,6 @@ EXC_COMMON_BEGIN(system_reset_common) * Vectors for the FWNMI option. Share common code. */ TRAMP_REAL_BEGIN(system_reset_fwnmi) - SET_SCRATCH0(r13) /* save r13 */ /* See comment at system_reset exception, don't turn on RI */ EXCEPTION_PROLOG_0 PACA_EXNMI EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0, 0, 0 @@ -969,7 +959,6 @@ EXC_REAL_BEGIN(machine_check, 0x200, 0x100) * some code path might still want to branch into the original * vector */ - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXMC BEGIN_FTR_SECTION b machine_check_common_early @@ -1058,7 +1047,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) TRAMP_REAL_BEGIN(machine_check_pSeries) .globl machine_check_fwnmi machine_check_fwnmi: - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXMC BEGIN_FTR_SECTION b machine_check_common_early @@ -1246,7 +1234,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) 9: /* Deliver the machine check to host kernel in V mode. */ MACHINE_CHECK_HANDLER_WINDUP - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXMC b machine_check_pSeries_0 @@ -1271,7 +1258,6 @@ EXC_COMMON_BEGIN(mce_return) b . EXC_REAL_BEGIN(data_access, 0x300, 0x80) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN b tramp_real_data_access EXC_REAL_END(data_access, 0x300, 0x80) @@ -1281,7 +1267,6 @@ TRAMP_REAL_BEGIN(tramp_real_data_access) EXCEPTION_PROLOG_2_REAL data_access_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x300, 1, 1, 0 EXCEPTION_PROLOG_2_VIRT data_access_common, EXC_STD @@ -1312,7 +1297,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXSLB b tramp_real_data_access_slb EXC_REAL_END(data_access_slb, 0x380, 0x80) @@ -1322,7 +1306,6 @@ TRAMP_REAL_BEGIN(tramp_real_data_access_slb) EXCEPTION_PROLOG_2_REAL data_access_slb_common, EXC_STD, 1 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXSLB EXCEPTION_PROLOG_1 EXC_STD, PACA_EXSLB, 0, 0x380, 1, 0, 0 EXCEPTION_PROLOG_2_VIRT data_access_slb_common, EXC_STD @@ -1406,7 +1389,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN BEGIN_FTR_SECTION EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED @@ -1418,7 +1400,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) EXC_REAL_END(hardware_interrupt, 0x500, 0x100) EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN BEGIN_FTR_SECTION EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0x500, 0, 0, IRQS_DISABLED @@ -1435,14 +1416,12 @@ EXC_COMMON_ASYNC(hardware_interrupt_common, 0x500, do_IRQ) EXC_REAL_BEGIN(alignment, 0x600, 0x100) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 1, 0x600, 1, 1, 0 EXCEPTION_PROLOG_2_REAL alignment_common, EXC_STD, 1 EXC_REAL_END(alignment, 0x600, 0x100) EXC_VIRT_BEGIN(alignment, 0x4600, 0x100) - SET_SCRATCH0(r13) /* save r13 */ EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_STD, PACA_EXGEN, 0, 0x600, 1, 1, 0 EXCEPTION_PROLOG_2_VIRT alignment_common, EXC_STD @@ -1766,7 +1745,6 @@ TRAMP_REAL_BEGIN(hmi_exception_early) * firmware. */ EXCEPTION_RESTORE_REGS EXC_HV - SET_SCRATCH0(r13) EXCEPTION_PROLOG_0 PACA_EXGEN b tramp_real_hmi_exception @@ -1925,7 +1903,6 @@ EXC_REAL_NONE(0x1400, 0x100) EXC_VIRT_NONE(0x5400, 0x100) EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) - SET_SCRATCH0(r13) EXCEPTION_PROLOG_0 PACA_EXGEN EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 0, 0x1500, 0, 0, 0 -- cgit v1.2.3 From f30a5e68f026f3214a9392391537adaa79996b24 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 15:33:32 +1000 Subject: powerpc/tm: update comment about interrupt re-entrancy Since the system reset interrupt began to use its own stack, and machine check interrupts have done so for some time, r1 can be changed without clearing MSR[RI], provided no other interrupts (including SLB misses) are taken. MSR[RI] does have to be cleared when using SCRATCH0, however. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/tm.S | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/tm.S b/arch/powerpc/kernel/tm.S index 9fabdce255cd..6ba0fdd1e7f8 100644 --- a/arch/powerpc/kernel/tm.S +++ b/arch/powerpc/kernel/tm.S @@ -148,7 +148,7 @@ _GLOBAL(tm_reclaim) /* Stash the stack pointer away for use after reclaim */ std r1, PACAR1(r13) - /* Clear MSR RI since we are about to change r1, EE is already off. */ + /* Clear MSR RI since we are about to use SCRATCH0, EE is already off */ li r5, 0 mtmsrd r5, 1 @@ -474,7 +474,7 @@ restore_gprs: REST_GPR(7, r7) - /* Clear MSR RI since we are about to change r1. EE is already off */ + /* Clear MSR RI since we are about to use SCRATCH0. EE is already off */ li r5, 0 mtmsrd r5, 1 -- cgit v1.2.3 From 0a882e28468f48ab3d9a36dde0a5723ea29ed1ed Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 16:33:18 +1000 Subject: powerpc/64s/exception: remove bad stack branch The bad stack test in interrupt handlers has a few problems. For performance it is taken in the common case, which is a fetch bubble and a waste of i-cache. For code development and maintainence, it requires yet another stack frame setup routine, and that constrains all exception handlers to follow the same register save pattern which inhibits future optimisation. Remove the test/branch and replace it with a trap. Teach the program check handler to use the emergency stack for this case. This does not result in quite so nice a message, however the SRR0 and SRR1 of the crashed interrupt can be seen in r11 and r12, as is the original r1 (adjusted by INT_FRAME_SIZE). These are the most important parts to debugging the issue. The original r9-12 and cr0 is lost, which is the main downside. kernel BUG at linux/arch/powerpc/kernel/exceptions-64s.S:847! Oops: Exception in kernel mode, sig: 5 [#1] BE SMP NR_CPUS=2048 NUMA PowerNV Modules linked in: CPU: 0 PID: 1 Comm: swapper/0 Not tainted NIP: c000000000009108 LR: c000000000cadbcc CTR: c0000000000090f0 REGS: c0000000fffcbd70 TRAP: 0700 Not tainted MSR: 9000000000021032 CR: 28222448 XER: 20040000 CFAR: c000000000009100 IRQMASK: 0 GPR00: 000000000000003d fffffffffffffd00 c0000000018cfb00 c0000000f02b3166 GPR04: fffffffffffffffd 0000000000000007 fffffffffffffffb 0000000000000030 GPR08: 0000000000000037 0000000028222448 0000000000000000 c000000000ca8de0 GPR12: 9000000002009032 c000000001ae0000 c000000000010a00 0000000000000000 GPR16: 0000000000000000 0000000000000000 0000000000000000 0000000000000000 GPR20: c0000000f00322c0 c000000000f85200 0000000000000004 ffffffffffffffff GPR24: fffffffffffffffe 0000000000000000 0000000000000000 000000000000000a GPR28: 0000000000000000 0000000000000000 c0000000f02b391c c0000000f02b3167 NIP [c000000000009108] decrementer_common+0x18/0x160 LR [c000000000cadbcc] .vsnprintf+0x3ec/0x4f0 Call Trace: Instruction dump: 996d098a 994d098b 38610070 480246ed 48005518 60000000 38200000 718a4000 7c2a0b78 3821fd00 41c20008 e82d0970 <0981fd00> f92101a0 f9610170 f9810178 Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 7 --- arch/powerpc/include/asm/paca.h | 2 + arch/powerpc/kernel/asm-offsets.c | 2 + arch/powerpc/kernel/exceptions-64s.S | 96 ++++++-------------------------- arch/powerpc/xmon/xmon.c | 2 + 5 files changed, 22 insertions(+), 87 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 40d114e69cbe..a77cdb07b152 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -48,13 +48,6 @@ */ #define MAX_MCE_DEPTH 4 -/* - * EX_R3 is only used by the bad_stack handler. bad_stack reloads and - * saves DAR from SPRN_DAR, and EX_DAR is not used. So EX_R3 can overlap - * with EX_DAR. - */ -#define EX_R3 EX_DAR - #ifdef __ASSEMBLY__ #define STF_ENTRY_BARRIER_SLOT \ diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 62f27e0aef7c..a2f713034ed3 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -170,7 +170,9 @@ struct paca_struct { u64 kstack; /* Saved Kernel stack addr */ u64 saved_r1; /* r1 save for RTAS calls or PM or EE=0 */ u64 saved_msr; /* MSR saved here by enter_rtas */ +#ifdef CONFIG_PPC_BOOK3E u16 trap_save; /* Used when bad stack is encountered */ +#endif u8 irq_soft_mask; /* mask for irq soft masking */ u8 irq_happened; /* irq happened while soft-disabled */ u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */ diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 8e02444e9d3d..524a7bba0ee5 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -270,7 +270,9 @@ int main(void) OFFSET(ACCOUNT_STARTTIME_USER, paca_struct, accounting.starttime_user); OFFSET(ACCOUNT_USER_TIME, paca_struct, accounting.utime); OFFSET(ACCOUNT_SYSTEM_TIME, paca_struct, accounting.stime); +#ifdef CONFIG_PPC_BOOK3E OFFSET(PACA_TRAP_SAVE, paca_struct, trap_save); +#endif OFFSET(PACA_SPRG_VDSO, paca_struct, sprg_vdso); #else /* CONFIG_PPC64 */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index b7c15a2df4a0..c73e909470e3 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -411,14 +411,8 @@ END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ subi r1,r1,INT_FRAME_SIZE; /* alloc frame on kernel stack */ \ beq- 1f; \ ld r1,PACAKSAVE(r13); /* kernel stack to use */ \ -1: cmpdi cr1,r1,-INT_FRAME_SIZE; /* check if r1 is in userspace */ \ - blt+ cr1,3f; /* abort if it is */ \ - li r1,(trap); /* will be reloaded later */ \ - sth r1,PACA_TRAP_SAVE(r13); \ - std r3,area+EX_R3(r13); \ - addi r3,r13,area; /* r3 -> where regs are saved*/ \ - RESTORE_CTR(r1, area); \ - b bad_stack; \ +1: tdgei r1,-INT_FRAME_SIZE; /* trap if r1 is in userspace */ \ + EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0; \ 3: EXCEPTION_PROLOG_COMMON_1(); \ kuap_save_amr_and_lock r9, r10, cr1, cr0; \ beq 4f; /* if from kernel mode */ \ @@ -428,7 +422,6 @@ END_FTR_SECTION_NESTED(CPU_FTR_CFAR, CPU_FTR_CFAR, 66); \ EXCEPTION_PROLOG_COMMON_3(trap); \ ACCOUNT_STOLEN_TIME - /* * Exception where stack is already set in r1, r1 is saved in r10. * PPR save and CPU accounting is not done (for some reason). @@ -1453,21 +1446,25 @@ EXC_COMMON_BEGIN(program_check_common) * we switch to the emergency stack if we're taking a TM Bad Thing from * the kernel. */ - li r10,MSR_PR /* Build a mask of MSR_PR .. */ - oris r10,r10,0x200000@h /* .. and SRR1_PROGTM */ - and r10,r10,r12 /* Mask SRR1 with that. */ - srdi r10,r10,8 /* Shift it so we can compare */ - cmpldi r10,(0x200000 >> 8) /* .. with an immediate. */ - bne 1f /* If != go to normal path. */ - - /* SRR1 had PR=0 and SRR1_PROGTM=1, so use the emergency stack */ - andi. r10,r12,MSR_PR; /* Set CR0 correctly for label */ + + andi. r10,r12,MSR_PR + bne 2f /* If userspace, go normal path */ + + andis. r10,r12,(SRR1_PROGTM)@h + bne 1f /* If TM, emergency */ + + cmpdi r1,-INT_FRAME_SIZE /* check if r1 is in userspace */ + blt 2f /* normal path if not */ + + /* Use the emergency stack */ +1: andi. r10,r12,MSR_PR /* Set CR0 correctly for label */ /* 3 in EXCEPTION_PROLOG_COMMON */ mr r10,r1 /* Save r1 */ ld r1,PACAEMERGSP(r13) /* Use emergency stack */ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ b 3f /* Jump into the macro !! */ -1: EXCEPTION_COMMON(PACA_EXGEN, 0x700) +2: + EXCEPTION_COMMON(PACA_EXGEN, 0x700) bl save_nvgprs RECONCILE_IRQ_STATE(r10, r11) addi r3,r1,STACK_FRAME_OVERHEAD @@ -2408,67 +2405,6 @@ handle_dabr_fault: bl bad_page_fault b ret_from_except -/* - * Here we have detected that the kernel stack pointer is bad. - * R9 contains the saved CR, r13 points to the paca, - * r10 contains the (bad) kernel stack pointer, - * r11 and r12 contain the saved SRR0 and SRR1. - * We switch to using an emergency stack, save the registers there, - * and call kernel_bad_stack(), which panics. - */ -bad_stack: - ld r1,PACAEMERGSP(r13) - subi r1,r1,64+INT_FRAME_SIZE - std r9,_CCR(r1) - std r10,GPR1(r1) - std r11,_NIP(r1) - std r12,_MSR(r1) - mfspr r11,SPRN_DAR - mfspr r12,SPRN_DSISR - std r11,_DAR(r1) - std r12,_DSISR(r1) - mflr r10 - mfctr r11 - mfxer r12 - std r10,_LINK(r1) - std r11,_CTR(r1) - std r12,_XER(r1) - SAVE_GPR(0,r1) - SAVE_GPR(2,r1) - ld r10,EX_R3(r3) - std r10,GPR3(r1) - SAVE_GPR(4,r1) - SAVE_4GPRS(5,r1) - ld r9,EX_R9(r3) - ld r10,EX_R10(r3) - SAVE_2GPRS(9,r1) - ld r9,EX_R11(r3) - ld r10,EX_R12(r3) - ld r11,EX_R13(r3) - std r9,GPR11(r1) - std r10,GPR12(r1) - std r11,GPR13(r1) -BEGIN_FTR_SECTION - ld r10,EX_CFAR(r3) - std r10,ORIG_GPR3(r1) -END_FTR_SECTION_IFSET(CPU_FTR_CFAR) - SAVE_8GPRS(14,r1) - SAVE_10GPRS(22,r1) - lhz r12,PACA_TRAP_SAVE(r13) - std r12,_TRAP(r1) - addi r11,r1,INT_FRAME_SIZE - std r11,0(r1) - li r12,0 - std r12,0(r11) - ld r2,PACATOC(r13) - ld r11,exception_marker@toc(r2) - std r12,RESULT(r1) - std r11,STACK_FRAME_OVERHEAD-16(r1) -1: addi r3,r1,STACK_FRAME_OVERHEAD - bl kernel_bad_stack - b 1b -_ASM_NOKPROBE_SYMBOL(bad_stack); - /* * When doorbell is triggered from system reset wakeup, the message is * not cleared, so it would fire again when EE is enabled. diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index f1c4e1601b9d..f879e9fe9733 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -2454,7 +2454,9 @@ static void dump_one_paca(int cpu) DUMP(p, canary, "%#-*lx"); #endif DUMP(p, saved_r1, "%#-*llx"); +#ifdef CONFIG_PPC_BOOK3E DUMP(p, trap_save, "%#-*x"); +#endif DUMP(p, irq_soft_mask, "%#-*x"); DUMP(p, irq_happened, "%#-*x"); #ifdef CONFIG_MMIOWB -- cgit v1.2.3 From 0e10be2bb9cf1f552f5d64bfe786926bba9d11d2 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 16:33:19 +1000 Subject: powerpc/64s/exception: optimise system_reset for idle, clean up non-idle case The idle wake up code in the system reset interrupt is not very optimal. There are two requirements: perform idle wake up quickly; and save everything including CFAR for non-idle interrupts, with no performance requirement. The problem with placing the idle test in the middle of the handler and using the normal handler code to save CFAR, is that it's quite costly (e.g., mfcfar is serialising, speculative workarounds get applied, SRR1 has to be reloaded, etc). It also prevents the standard interrupt handler boilerplate being used. This pain can be avoided by using a dedicated idle interrupt handler at the start of the interrupt handler, which restores all registers back to the way they were in case it was not an idle wake up. CFAR is preserved without saving it before the non-idle case by making that the fall-through, and idle is a taken branch. Performance seems to be in the noise, but possibly around 0.5% faster, the executed instructions certainly look better. The bigger benefit is being able to drop in standard interrupt handlers after the idle code, which helps with subsequent cleanup and consolidation. Signed-off-by: Nicholas Piggin [mpe: Fixup BE by using DOTSYM for idle_return_gpr_loss call] Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 69 ++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 31 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index c73e909470e3..e5660f3e8331 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -256,7 +256,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) * load KBASE for a slight optimisation. */ #define BRANCH_TO_C000(reg, label) \ - __LOAD_HANDLER(reg, label); \ + __LOAD_FAR_HANDLER(reg, label); \ mtctr reg; \ bctr @@ -822,15 +822,6 @@ EXC_VIRT_NONE(0x4000, 0x100) EXC_REAL_BEGIN(system_reset, 0x100, 0x100) - EXCEPTION_PROLOG_0 PACA_EXNMI - - /* This is EXCEPTION_PROLOG_1 with the idle feature section added */ - OPT_SAVE_REG_TO_PACA(PACA_EXNMI+EX_PPR, r9, CPU_FTR_HAS_PPR) - OPT_SAVE_REG_TO_PACA(PACA_EXNMI+EX_CFAR, r10, CPU_FTR_CFAR) - INTERRUPT_TO_KERNEL - SAVE_CTR(r10, PACA_EXNMI) - mfcr r9 - #ifdef CONFIG_PPC_P7_NAP /* * If running native on arch 2.06 or later, check if we are waking up @@ -838,43 +829,59 @@ EXC_REAL_BEGIN(system_reset, 0x100, 0x100) * bits 46:47. A non-0 value indicates that we are coming from a power * saving state. The idle wakeup handler initially runs in real mode, * but we branch to the 0xc000... address so we can turn on relocation - * with mtmsr. + * with mtmsrd later, after SPRs are restored. + * + * Careful to minimise cost for the fast path (idle wakeup) while + * also avoiding clobbering CFAR for the debug path (non-idle). + * + * For the idle wake case volatile registers can be clobbered, which + * is why we use those initially. If it turns out to not be an idle + * wake, carefully put everything back the way it was, so we can use + * common exception macros to handle it. */ BEGIN_FTR_SECTION - mfspr r10,SPRN_SRR1 - rlwinm. r10,r10,47-31,30,31 - beq- 1f - cmpwi cr1,r10,2 + SET_SCRATCH0(r13) + GET_PACA(r13) + std r3,PACA_EXNMI+0*8(r13) + std r4,PACA_EXNMI+1*8(r13) + std r5,PACA_EXNMI+2*8(r13) mfspr r3,SPRN_SRR1 - bltlr cr1 /* no state loss, return to idle caller */ - BRANCH_TO_C000(r10, system_reset_idle_common) -1: + mfocrf r4,0x80 + rlwinm. r5,r3,47-31,30,31 + bne+ system_reset_idle_wake + /* Not powersave wakeup. Restore regs for regular interrupt handler. */ + mtocrf 0x80,r4 + ld r3,PACA_EXNMI+0*8(r13) + ld r4,PACA_EXNMI+1*8(r13) + ld r5,PACA_EXNMI+2*8(r13) + GET_SCRATCH0(r13) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) #endif - KVMTEST EXC_STD 0x100 - std r11,PACA_EXNMI+EX_R11(r13) - std r12,PACA_EXNMI+EX_R12(r13) - GET_SCRATCH0(r10) - std r10,PACA_EXNMI+EX_R13(r13) - + EXCEPTION_PROLOG_0 PACA_EXNMI + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 1, 0x100, 0, 0, 0 EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0 /* * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is * being used, so a nested NMI exception would corrupt it. + * + * In theory, we should not enable relocation here if it was disabled + * in SRR1, because the MMU may not be configured to support it (e.g., + * SLB may have been cleared). In practice, there should only be a few + * small windows where that's the case, and sreset is considered to + * be dangerous anyway. */ - EXC_REAL_END(system_reset, 0x100, 0x100) + EXC_VIRT_NONE(0x4100, 0x100) TRAMP_KVM(PACA_EXNMI, 0x100) #ifdef CONFIG_PPC_P7_NAP -EXC_COMMON_BEGIN(system_reset_idle_common) - /* - * This must be a direct branch (without linker branch stub) because - * we can not use TOC at this point as r2 may not be restored yet. - */ - b idle_return_gpr_loss +TRAMP_REAL_BEGIN(system_reset_idle_wake) + /* We are waking up from idle, so may clobber any volatile register */ + cmpwi cr1,r5,2 + bltlr cr1 /* no state loss, return to idle caller with r3=SRR1 */ + BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) #endif EXC_COMMON_BEGIN(system_reset_common) -- cgit v1.2.3 From acc8da4492ef9b9d60bbdf5f857ead9e84031b5c Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 16:33:20 +1000 Subject: powerpc/64s/exception: sreset move trampoline ahead of common code Follow convention and move tramp ahead of common. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index e5660f3e8331..848d39ccf8c0 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -884,6 +884,18 @@ TRAMP_REAL_BEGIN(system_reset_idle_wake) BRANCH_TO_C000(r12, DOTSYM(idle_return_gpr_loss)) #endif +#ifdef CONFIG_PPC_PSERIES +/* + * Vectors for the FWNMI option. Share common code. + */ +TRAMP_REAL_BEGIN(system_reset_fwnmi) + /* See comment at system_reset exception, don't turn on RI */ + EXCEPTION_PROLOG_0 PACA_EXNMI + EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0, 0, 0 + EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0 + +#endif /* CONFIG_PPC_PSERIES */ + EXC_COMMON_BEGIN(system_reset_common) /* * Increment paca->in_nmi then enable MSR_RI. SLB or MCE will be able @@ -941,18 +953,6 @@ EXC_COMMON_BEGIN(system_reset_common) EXCEPTION_RESTORE_REGS EXC_STD RFI_TO_USER_OR_KERNEL -#ifdef CONFIG_PPC_PSERIES -/* - * Vectors for the FWNMI option. Share common code. - */ -TRAMP_REAL_BEGIN(system_reset_fwnmi) - /* See comment at system_reset exception, don't turn on RI */ - EXCEPTION_PROLOG_0 PACA_EXNMI - EXCEPTION_PROLOG_1 EXC_STD, PACA_EXNMI, 0, 0x100, 0, 0, 0 - EXCEPTION_PROLOG_2_REAL system_reset_common, EXC_STD, 0 - -#endif /* CONFIG_PPC_PSERIES */ - EXC_REAL_BEGIN(machine_check, 0x200, 0x100) /* This is moved out of line as it can be patched by FW, but -- cgit v1.2.3 From f34c9675ca594bea6f813872a470e1b8b077d454 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 16:33:21 +1000 Subject: powerpc/64s/exception: hmi remove special case macro No code change. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 848d39ccf8c0..c6e1d1a88701 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -519,11 +519,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) * * There can be combinations, e.g., EXC_VIRT_OOL_MASKABLE_HV * - * The one unusual case is __EXC_REAL_OOL_HV_DIRECT, which is - * an OOL vector that branches to a specified handler rather than the usual - * trampoline that goes to common. It, and other underscore macros, should - * be used with care. - * * KVM handlers come in the following verieties: * TRAMP_KVM * TRAMP_KVM_SKIP @@ -614,12 +609,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP) __EXC_REAL_OOL_MASKABLE(name, start, size); \ __TRAMP_REAL_OOL_MASKABLE(name, start, bitmask) -#define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler) \ - EXC_REAL_BEGIN(name, start, size); \ - EXCEPTION_PROLOG_0 PACA_EXGEN ; \ - b handler; \ - EXC_REAL_END(name, start, size) - #define __EXC_REAL_OOL_HV(name, start, size) \ __EXC_REAL_OOL(name, start, size) @@ -1720,7 +1709,10 @@ EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt) * first, and then eventaully from there to the trampoline to get into virtual * mode. */ -__EXC_REAL_OOL_HV_DIRECT(hmi_exception, 0xe60, 0x20, hmi_exception_early) +EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) + EXCEPTION_PROLOG_0 PACA_EXGEN + b hmi_exception_early +EXC_REAL_END(hmi_exception, 0xe60, 0x20) __TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED) EXC_VIRT_NONE(0x4e60, 0x20) TRAMP_KVM_HV(PACA_EXGEN, 0xe60) -- cgit v1.2.3 From 293c2e27b999e431ad5f9df3386bea37b18fce53 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 28 Jun 2019 16:33:22 +1000 Subject: powerpc/64s/exception: simplify hmi control flow Branch to the relocated 0xc000 address early (still in real mode), to simplify subsequent branches. Have the virt mode handler avoid just 'windup' and redo the exception from scratch, rather than branching back to the trampoline. Rearrange the stack setup instruction location to match the system reset handler (e.g., right before EXCEPTION_PROLOG_COMMON). Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/exceptions-64s.S | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index c6e1d1a88701..eee5bef736c8 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -260,17 +260,6 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) mtctr reg; \ bctr -#ifdef CONFIG_RELOCATABLE -#define BRANCH_LINK_TO_FAR(label) \ - __LOAD_FAR_HANDLER(r12, label); \ - mtctr r12; \ - bctrl - -#else -#define BRANCH_LINK_TO_FAR(label) \ - bl label -#endif - #ifdef CONFIG_KVM_BOOK3S_64_HANDLER #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* @@ -1713,22 +1702,26 @@ EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20) EXCEPTION_PROLOG_0 PACA_EXGEN b hmi_exception_early EXC_REAL_END(hmi_exception, 0xe60, 0x20) -__TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED) EXC_VIRT_NONE(0x4e60, 0x20) TRAMP_KVM_HV(PACA_EXGEN, 0xe60) TRAMP_REAL_BEGIN(hmi_exception_early) EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0, 0, 0 + mfctr r10 /* save ctr, even for !RELOCATABLE */ + BRANCH_TO_C000(r11, hmi_exception_early_common) + +EXC_COMMON_BEGIN(hmi_exception_early_common) + mtctr r10 /* Restore ctr */ + mfspr r11,SPRN_HSRR0 /* Save HSRR0 */ + mfspr r12,SPRN_HSRR1 /* Save HSRR1 */ mr r10,r1 /* Save r1 */ ld r1,PACAEMERGSP(r13) /* Use emergency stack for realmode */ subi r1,r1,INT_FRAME_SIZE /* alloc stack frame */ - mfspr r11,SPRN_HSRR0 /* Save HSRR0 */ - mfspr r12,SPRN_HSRR1 /* Save HSRR1 */ EXCEPTION_PROLOG_COMMON_1() /* We don't touch AMR here, we never go to virtual mode */ EXCEPTION_PROLOG_COMMON_2(PACA_EXGEN) EXCEPTION_PROLOG_COMMON_3(0xe60) addi r3,r1,STACK_FRAME_OVERHEAD - BRANCH_LINK_TO_FAR(DOTSYM(hmi_exception_realmode)) /* Function call ABI */ + bl hmi_exception_realmode cmpdi cr0,r3,0 bne 1f @@ -1742,7 +1735,8 @@ TRAMP_REAL_BEGIN(hmi_exception_early) */ EXCEPTION_RESTORE_REGS EXC_HV EXCEPTION_PROLOG_0 PACA_EXGEN - b tramp_real_hmi_exception + EXCEPTION_PROLOG_1 EXC_HV, PACA_EXGEN, 1, 0xe60, 0, 0, IRQS_DISABLED + EXCEPTION_PROLOG_2_REAL hmi_exception_common, EXC_HV, 1 EXC_COMMON_BEGIN(hmi_exception_common) EXCEPTION_COMMON(PACA_EXGEN, 0xe60) -- cgit v1.2.3 From fe7946ce0808eb0e43711f5db7d2d1599b362d02 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sun, 23 Jun 2019 20:41:51 +1000 Subject: powerpc/64s: Rename PPC_INVALIDATE_ERAT to PPC_ISA_3_0_INVALIDATE_ERAT This makes it clear to the caller that it can only be used on POWER9 and later CPUs. Signed-off-by: Nicholas Piggin [mpe: Use "ISA_3_0" rather than "ARCH_300"] Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/ppc-opcode.h | 2 +- arch/powerpc/kernel/mce_power.c | 3 +-- arch/powerpc/kvm/book3s_hv_builtin.c | 2 +- arch/powerpc/mm/book3s64/hash_native.c | 2 +- arch/powerpc/mm/book3s64/radix_tlb.c | 8 ++++---- arch/powerpc/platforms/powernv/idle.c | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 23f7ed796f38..14f43b6cf7ec 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -591,7 +591,7 @@ #define PPC_SLBIA(IH) stringify_in_c(.long PPC_INST_SLBIA | \ ((IH & 0x7) << 21)) -#define PPC_INVALIDATE_ERAT PPC_SLBIA(7) +#define PPC_ISA_3_0_INVALIDATE_ERAT PPC_SLBIA(7) #define VCMPEQUD_RC(vrt, vra, vrb) stringify_in_c(.long PPC_INST_VCMPEQUD | \ ___PPC_RT(vrt) | ___PPC_RA(vra) | \ diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c index b5e876efe864..451f3e90e958 100644 --- a/arch/powerpc/kernel/mce_power.c +++ b/arch/powerpc/kernel/mce_power.c @@ -95,8 +95,7 @@ static void flush_erat(void) return; } #endif - /* PPC_INVALIDATE_ERAT can only be used on ISA v3 and newer */ - asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); } #define MCE_FLUSH_SLB 1 diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index a46286f73eec..ca4f006abd77 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -833,7 +833,7 @@ static void flush_guest_tlb(struct kvm *kvm) } } asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); } void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu, diff --git a/arch/powerpc/mm/book3s64/hash_native.c b/arch/powerpc/mm/book3s64/hash_native.c index 47caecdbbbac..0b447d96b911 100644 --- a/arch/powerpc/mm/book3s64/hash_native.c +++ b/arch/powerpc/mm/book3s64/hash_native.c @@ -116,7 +116,7 @@ static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); } void hash__tlbiel_all(unsigned int action) diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index 4d841369399f..7539c714438e 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -87,7 +87,7 @@ void radix__tlbiel_all(unsigned int action) else WARN(1, "%s called on pre-POWER9 CPU\n", __func__); - asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); } static __always_inline void __tlbiel_pid(unsigned long pid, int set, @@ -262,7 +262,7 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) __tlbiel_pid(pid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); } static inline void _tlbie_pid(unsigned long pid, unsigned long ric) @@ -314,7 +314,7 @@ static inline void _tlbiel_lpid(unsigned long lpid, unsigned long ric) __tlbiel_lpid(lpid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT "; isync" : : :"memory"); } static inline void _tlbie_lpid(unsigned long lpid, unsigned long ric) @@ -366,7 +366,7 @@ static inline void _tlbiel_lpid_guest(unsigned long lpid, unsigned long ric) __tlbiel_lpid_guest(lpid, set, RIC_FLUSH_TLB); asm volatile("ptesync": : :"memory"); - asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT : : :"memory"); } diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c index 77f2e0a4ee37..f9ab89510766 100644 --- a/arch/powerpc/platforms/powernv/idle.c +++ b/arch/powerpc/platforms/powernv/idle.c @@ -720,7 +720,7 @@ static unsigned long power9_idle_stop(unsigned long psscr, bool mmu_on) * to reload MMCR0 (see mmcr0 comment above). */ if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) { - asm volatile(PPC_INVALIDATE_ERAT); + asm volatile(PPC_ISA_3_0_INVALIDATE_ERAT); mtspr(SPRN_MMCR0, mmcr0); } -- cgit v1.2.3 From 548c54acba5bd1388d50727a9a126a42d0cd4ad0 Mon Sep 17 00:00:00 2001 From: Mathieu Malaterre Date: Tue, 4 Jun 2019 13:00:36 +1000 Subject: powerpc: silence a -Wcast-function-type warning in dawr_write_file_bool In commit c1fe190c0672 ("powerpc: Add force enable of DAWR on P9 option") the following piece of code was added: smp_call_function((smp_call_func_t)set_dawr, &null_brk, 0); Since GCC 8 this triggers the following warning about incompatible function types: arch/powerpc/kernel/hw_breakpoint.c:408:21: error: cast between incompatible function types from 'int (*)(struct arch_hw_breakpoint *)' to 'void (*)(void *)' [-Werror=cast-function-type] Since the warning is there for a reason, and should not be hidden behind a cast, provide an intermediate callback function to avoid the warning. Fixes: c1fe190c0672 ("powerpc: Add force enable of DAWR on P9 option") Suggested-by: Christoph Hellwig Signed-off-by: Mathieu Malaterre Signed-off-by: Michael Neuling Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/hw_breakpoint.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c index da307dd93ee3..ca3a2358b768 100644 --- a/arch/powerpc/kernel/hw_breakpoint.c +++ b/arch/powerpc/kernel/hw_breakpoint.c @@ -384,6 +384,11 @@ void hw_breakpoint_pmu_read(struct perf_event *bp) bool dawr_force_enable; EXPORT_SYMBOL_GPL(dawr_force_enable); +static void set_dawr_cb(void *info) +{ + set_dawr(info); +} + static ssize_t dawr_write_file_bool(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) @@ -403,7 +408,7 @@ static ssize_t dawr_write_file_bool(struct file *file, /* If we are clearing, make sure all CPUs have the DAWR cleared */ if (!dawr_force_enable) - smp_call_function((smp_call_func_t)set_dawr, &null_brk, 0); + smp_call_function(set_dawr_cb, &null_brk, 0); return rc; } -- cgit v1.2.3 From a278e7ea608bea5fe6df9b6ae91fa134655c5d2c Mon Sep 17 00:00:00 2001 From: Michael Neuling Date: Tue, 4 Jun 2019 13:00:37 +1000 Subject: powerpc: Fix compile issue with force DAWR If you compile with KVM but without CONFIG_HAVE_HW_BREAKPOINT you fail at linking with: arch/powerpc/kvm/book3s_hv_rmhandlers.o:(.text+0x708): undefined reference to `dawr_force_enable' This was caused by commit c1fe190c0672 ("powerpc: Add force enable of DAWR on P9 option"). This moves a bunch of code around to fix this. It moves a lot of the DAWR code in a new file and creates a new CONFIG_PPC_DAWR to enable compiling it. Fixes: c1fe190c0672 ("powerpc: Add force enable of DAWR on P9 option") Signed-off-by: Michael Neuling [mpe: Minor formatting in set_dawr()] Signed-off-by: Michael Ellerman --- arch/powerpc/Kconfig | 4 ++ arch/powerpc/include/asm/hw_breakpoint.h | 21 ++++--- arch/powerpc/kernel/Makefile | 1 + arch/powerpc/kernel/dawr.c | 101 +++++++++++++++++++++++++++++++ arch/powerpc/kernel/hw_breakpoint.c | 61 ------------------- arch/powerpc/kernel/process.c | 28 --------- arch/powerpc/kvm/Kconfig | 1 + 7 files changed, 121 insertions(+), 96 deletions(-) create mode 100644 arch/powerpc/kernel/dawr.c (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 3adff91bc856..876902661c04 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -236,6 +236,7 @@ config PPC select OLD_SIGSUSPEND select PCI_DOMAINS if PCI select PCI_SYSCALL if PCI + select PPC_DAWR if PPC64 select RTC_LIB select SPARSE_IRQ select SYSCTL_EXCEPTION_TRACE @@ -372,6 +373,9 @@ config PPC_ADV_DEBUG_DAC_RANGE depends on PPC_ADV_DEBUG_REGS && 44x default y +config PPC_DAWR + bool + config ZONE_DMA bool default y if PPC_BOOK3E_64 diff --git a/arch/powerpc/include/asm/hw_breakpoint.h b/arch/powerpc/include/asm/hw_breakpoint.h index 0fe8c1e46bbc..41abdae6d079 100644 --- a/arch/powerpc/include/asm/hw_breakpoint.h +++ b/arch/powerpc/include/asm/hw_breakpoint.h @@ -90,18 +90,25 @@ static inline void hw_breakpoint_disable(void) extern void thread_change_pc(struct task_struct *tsk, struct pt_regs *regs); int hw_breakpoint_handler(struct die_args *args); -extern int set_dawr(struct arch_hw_breakpoint *brk); +#else /* CONFIG_HAVE_HW_BREAKPOINT */ +static inline void hw_breakpoint_disable(void) { } +static inline void thread_change_pc(struct task_struct *tsk, + struct pt_regs *regs) { } + +#endif /* CONFIG_HAVE_HW_BREAKPOINT */ + + +#ifdef CONFIG_PPC_DAWR extern bool dawr_force_enable; static inline bool dawr_enabled(void) { return dawr_force_enable; } - -#else /* CONFIG_HAVE_HW_BREAKPOINT */ -static inline void hw_breakpoint_disable(void) { } -static inline void thread_change_pc(struct task_struct *tsk, - struct pt_regs *regs) { } +int set_dawr(struct arch_hw_breakpoint *brk); +#else static inline bool dawr_enabled(void) { return false; } -#endif /* CONFIG_HAVE_HW_BREAKPOINT */ +static inline int set_dawr(struct arch_hw_breakpoint *brk) { return -1; } +#endif + #endif /* __KERNEL__ */ #endif /* _PPC_BOOK3S_64_HW_BREAKPOINT_H */ diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 0ea6c4aa3a20..56dfa7a2a6f2 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -56,6 +56,7 @@ obj-$(CONFIG_PPC64) += setup_64.o sys_ppc32.o \ obj-$(CONFIG_VDSO32) += vdso32/ obj-$(CONFIG_PPC_WATCHDOG) += watchdog.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o +obj-$(CONFIG_PPC_DAWR) += dawr.o obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_ppc970.o cpu_setup_pa6t.o obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_power.o obj-$(CONFIG_PPC_BOOK3S_64) += mce.o mce_power.o diff --git a/arch/powerpc/kernel/dawr.c b/arch/powerpc/kernel/dawr.c new file mode 100644 index 000000000000..5f66b95b6858 --- /dev/null +++ b/arch/powerpc/kernel/dawr.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * DAWR infrastructure + * + * Copyright 2019, Michael Neuling, IBM Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include + +bool dawr_force_enable; +EXPORT_SYMBOL_GPL(dawr_force_enable); + +int set_dawr(struct arch_hw_breakpoint *brk) +{ + unsigned long dawr, dawrx, mrd; + + dawr = brk->address; + + dawrx = (brk->type & (HW_BRK_TYPE_READ | HW_BRK_TYPE_WRITE)) + << (63 - 58); + dawrx |= ((brk->type & (HW_BRK_TYPE_TRANSLATE)) >> 2) << (63 - 59); + dawrx |= (brk->type & (HW_BRK_TYPE_PRIV_ALL)) >> 3; + /* + * DAWR length is stored in field MDR bits 48:53. Matches range in + * doublewords (64 bits) baised by -1 eg. 0b000000=1DW and + * 0b111111=64DW. + * brk->len is in bytes. + * This aligns up to double word size, shifts and does the bias. + */ + mrd = ((brk->len + 7) >> 3) - 1; + dawrx |= (mrd & 0x3f) << (63 - 53); + + if (ppc_md.set_dawr) + return ppc_md.set_dawr(dawr, dawrx); + + mtspr(SPRN_DAWR, dawr); + mtspr(SPRN_DAWRX, dawrx); + + return 0; +} + +static void set_dawr_cb(void *info) +{ + set_dawr(info); +} + +static ssize_t dawr_write_file_bool(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct arch_hw_breakpoint null_brk = {0, 0, 0}; + size_t rc; + + /* Send error to user if they hypervisor won't allow us to write DAWR */ + if (!dawr_force_enable && + firmware_has_feature(FW_FEATURE_LPAR) && + set_dawr(&null_brk) != H_SUCCESS) + return -ENODEV; + + rc = debugfs_write_file_bool(file, user_buf, count, ppos); + if (rc) + return rc; + + /* If we are clearing, make sure all CPUs have the DAWR cleared */ + if (!dawr_force_enable) + smp_call_function(set_dawr_cb, &null_brk, 0); + + return rc; +} + +static const struct file_operations dawr_enable_fops = { + .read = debugfs_read_file_bool, + .write = dawr_write_file_bool, + .open = simple_open, + .llseek = default_llseek, +}; + +static int __init dawr_force_setup(void) +{ + if (cpu_has_feature(CPU_FTR_DAWR)) { + /* Don't setup sysfs file for user control on P8 */ + dawr_force_enable = true; + return 0; + } + + if (PVR_VER(mfspr(SPRN_PVR)) == PVR_POWER9) { + /* Turn DAWR off by default, but allow admin to turn it on */ + debugfs_create_file_unsafe("dawr_enable_dangerous", 0600, + powerpc_debugfs_root, + &dawr_force_enable, + &dawr_enable_fops); + } + return 0; +} +arch_initcall(dawr_force_setup); diff --git a/arch/powerpc/kernel/hw_breakpoint.c b/arch/powerpc/kernel/hw_breakpoint.c index ca3a2358b768..95605a9c9a1e 100644 --- a/arch/powerpc/kernel/hw_breakpoint.c +++ b/arch/powerpc/kernel/hw_breakpoint.c @@ -380,64 +380,3 @@ void hw_breakpoint_pmu_read(struct perf_event *bp) { /* TODO */ } - -bool dawr_force_enable; -EXPORT_SYMBOL_GPL(dawr_force_enable); - -static void set_dawr_cb(void *info) -{ - set_dawr(info); -} - -static ssize_t dawr_write_file_bool(struct file *file, - const char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct arch_hw_breakpoint null_brk = {0, 0, 0}; - size_t rc; - - /* Send error to user if they hypervisor won't allow us to write DAWR */ - if ((!dawr_force_enable) && - (firmware_has_feature(FW_FEATURE_LPAR)) && - (set_dawr(&null_brk) != H_SUCCESS)) - return -1; - - rc = debugfs_write_file_bool(file, user_buf, count, ppos); - if (rc) - return rc; - - /* If we are clearing, make sure all CPUs have the DAWR cleared */ - if (!dawr_force_enable) - smp_call_function(set_dawr_cb, &null_brk, 0); - - return rc; -} - -static const struct file_operations dawr_enable_fops = { - .read = debugfs_read_file_bool, - .write = dawr_write_file_bool, - .open = simple_open, - .llseek = default_llseek, -}; - -static int __init dawr_force_setup(void) -{ - dawr_force_enable = false; - - if (cpu_has_feature(CPU_FTR_DAWR)) { - /* Don't setup sysfs file for user control on P8 */ - dawr_force_enable = true; - return 0; - } - - if (PVR_VER(mfspr(SPRN_PVR)) == PVR_POWER9) { - /* Turn DAWR off by default, but allow admin to turn it on */ - dawr_force_enable = false; - debugfs_create_file_unsafe("dawr_enable_dangerous", 0600, - powerpc_debugfs_root, - &dawr_force_enable, - &dawr_enable_fops); - } - return 0; -} -arch_initcall(dawr_force_setup); diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 87da40129927..03a2da35ce61 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -797,34 +797,6 @@ static inline int set_dabr(struct arch_hw_breakpoint *brk) return __set_dabr(dabr, dabrx); } -int set_dawr(struct arch_hw_breakpoint *brk) -{ - unsigned long dawr, dawrx, mrd; - - dawr = brk->address; - - dawrx = (brk->type & (HW_BRK_TYPE_READ | HW_BRK_TYPE_WRITE)) \ - << (63 - 58); //* read/write bits */ - dawrx |= ((brk->type & (HW_BRK_TYPE_TRANSLATE)) >> 2) \ - << (63 - 59); //* translate */ - dawrx |= (brk->type & (HW_BRK_TYPE_PRIV_ALL)) \ - >> 3; //* PRIM bits */ - /* dawr length is stored in field MDR bits 48:53. Matches range in - doublewords (64 bits) baised by -1 eg. 0b000000=1DW and - 0b111111=64DW. - brk->len is in bytes. - This aligns up to double word size, shifts and does the bias. - */ - mrd = ((brk->len + 7) >> 3) - 1; - dawrx |= (mrd & 0x3f) << (63 - 53); - - if (ppc_md.set_dawr) - return ppc_md.set_dawr(dawr, dawrx); - mtspr(SPRN_DAWR, dawr); - mtspr(SPRN_DAWRX, dawrx); - return 0; -} - void __set_breakpoint(struct arch_hw_breakpoint *brk) { memcpy(this_cpu_ptr(¤t_brk), brk, sizeof(*brk)); diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index f53997a8ca62..b8e13d5a4a31 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -38,6 +38,7 @@ config KVM_BOOK3S_32_HANDLER config KVM_BOOK3S_64_HANDLER bool select KVM_BOOK3S_HANDLER + select PPC_DAWR_FORCE_ENABLE config KVM_BOOK3S_PR_POSSIBLE bool -- cgit v1.2.3 From 24911acd64cee411c9e626d3d0ca0733805b009b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 29 Jun 2019 10:03:59 +0200 Subject: powerpc: remove device_to_mask() Use the dma_get_mask() helper from dma-mapping.h instead, as they are functionally identical. Signed-off-by: Christoph Hellwig Reviewed-by: Alexey Kardashevskiy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/iommu.h | 8 -------- arch/powerpc/kernel/dma-iommu.c | 4 ++-- arch/powerpc/platforms/pseries/vio.c | 4 ++-- 3 files changed, 4 insertions(+), 12 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index 0ac52392ed99..f98f2864b66a 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -327,13 +327,5 @@ extern bool iommu_fixed_is_weak; extern const struct dma_map_ops dma_iommu_ops; -static inline unsigned long device_to_mask(struct device *dev) -{ - if (dev->dma_mask && *dev->dma_mask) - return *dev->dma_mask; - /* Assume devices without mask can take 32 bit addresses */ - return 0xfffffffful; -} - #endif /* __KERNEL__ */ #endif /* _ASM_IOMMU_H */ diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 09231ef06d01..168af3a5b4b1 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -71,7 +71,7 @@ static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page, return dma_direct_map_page(dev, page, offset, size, direction, attrs); return iommu_map_page(dev, get_iommu_table_base(dev), page, offset, - size, device_to_mask(dev), direction, attrs); + size, dma_get_mask(dev), direction, attrs); } @@ -92,7 +92,7 @@ static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, if (dma_iommu_map_bypass(dev, attrs)) return dma_direct_map_sg(dev, sglist, nelems, direction, attrs); return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems, - device_to_mask(dev), direction, attrs); + dma_get_mask(dev), direction, attrs); } static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, diff --git a/arch/powerpc/platforms/pseries/vio.c b/arch/powerpc/platforms/pseries/vio.c index 141795275ccb..97c0e3e5eae5 100644 --- a/arch/powerpc/platforms/pseries/vio.c +++ b/arch/powerpc/platforms/pseries/vio.c @@ -524,7 +524,7 @@ static dma_addr_t vio_dma_iommu_map_page(struct device *dev, struct page *page, if (vio_cmo_alloc(viodev, roundup(size, IOMMU_PAGE_SIZE(tbl)))) goto out_fail; - ret = iommu_map_page(dev, tbl, page, offset, size, device_to_mask(dev), + ret = iommu_map_page(dev, tbl, page, offset, size, dma_get_mask(dev), direction, attrs); if (unlikely(ret == DMA_MAPPING_ERROR)) goto out_deallocate; @@ -564,7 +564,7 @@ static int vio_dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, if (vio_cmo_alloc(viodev, alloc_size)) goto out_fail; - ret = ppc_iommu_map_sg(dev, tbl, sglist, nelems, device_to_mask(dev), + ret = ppc_iommu_map_sg(dev, tbl, sglist, nelems, dma_get_mask(dev), direction, attrs); if (unlikely(!ret)) goto out_deallocate; -- cgit v1.2.3 From efd176a04bef41aab5b3087e977fea2b69915174 Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Tue, 7 May 2019 16:25:58 +1000 Subject: powerpc/pseries/dma: Allow SWIOTLB The commit 8617a5c5bc00 ("powerpc/dma: handle iommu bypass in dma_iommu_ops") merged direct DMA ops into the IOMMU DMA ops allowing SWIOTLB as well but only for mapping; the unmapping and bouncing parts were left unmodified. This adds missing direct unmapping calls to .unmap_page() and .unmap_sg(). This adds missing sync callbacks and directs them to the direct DMA hooks. Fixes: 8617a5c5bc00 ("powerpc/dma: handle iommu bypass in dma_iommu_ops") Signed-off-by: Alexey Kardashevskiy Signed-off-by: Thiago Jung Bauermann Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/dma-iommu.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 168af3a5b4b1..a0879674a9c8 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -82,6 +82,8 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, if (!dma_iommu_map_bypass(dev, attrs)) iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction, attrs); + else + dma_direct_unmap_page(dev, dma_handle, size, direction, attrs); } @@ -102,6 +104,8 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, if (!dma_iommu_map_bypass(dev, attrs)) ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems, direction, attrs); + else + dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs); } static bool dma_iommu_bypass_supported(struct device *dev, u64 mask) @@ -163,6 +167,34 @@ u64 dma_iommu_get_required_mask(struct device *dev) return mask; } +static void dma_iommu_sync_for_cpu(struct device *dev, dma_addr_t addr, + size_t size, enum dma_data_direction dir) +{ + if (dma_iommu_alloc_bypass(dev)) + dma_direct_sync_single_for_cpu(dev, addr, size, dir); +} + +static void dma_iommu_sync_for_device(struct device *dev, dma_addr_t addr, + size_t sz, enum dma_data_direction dir) +{ + if (dma_iommu_alloc_bypass(dev)) + dma_direct_sync_single_for_device(dev, addr, sz, dir); +} + +extern void dma_iommu_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sgl, int nents, enum dma_data_direction dir) +{ + if (dma_iommu_alloc_bypass(dev)) + dma_direct_sync_sg_for_cpu(dev, sgl, nents, dir); +} + +extern void dma_iommu_sync_sg_for_device(struct device *dev, + struct scatterlist *sgl, int nents, enum dma_data_direction dir) +{ + if (dma_iommu_alloc_bypass(dev)) + dma_direct_sync_sg_for_device(dev, sgl, nents, dir); +} + const struct dma_map_ops dma_iommu_ops = { .alloc = dma_iommu_alloc_coherent, .free = dma_iommu_free_coherent, @@ -172,4 +204,8 @@ const struct dma_map_ops dma_iommu_ops = { .map_page = dma_iommu_map_page, .unmap_page = dma_iommu_unmap_page, .get_required_mask = dma_iommu_get_required_mask, + .sync_single_for_cpu = dma_iommu_sync_for_cpu, + .sync_single_for_device = dma_iommu_sync_for_device, + .sync_sg_for_cpu = dma_iommu_sync_sg_for_cpu, + .sync_sg_for_device = dma_iommu_sync_sg_for_device, }; -- cgit v1.2.3 From dead1c845dbe97e0061dae2017eaf3bd8f8f06ee Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Wed, 26 Jun 2019 12:37:46 +1000 Subject: powerpc/pci/of: Parse unassigned resources The pseries platform uses the PCI_PROBE_DEVTREE method of PCI probing which reads "assigned-addresses" of every PCI device and initializes the device resources. However if the property is missing or zero sized, then there is no fallback of any kind and the PCI resources remain undiscovered, i.e. pdev->resource[] array remains empty. This adds a fallback which parses the "reg" property in pretty much same way except it marks resources as "unset" which later make Linux assign those resources proper addresses. This has an effect when: 1. a hypervisor failed to assign any resource for a device; 2. /chosen/linux,pci-probe-only=0 is in the DT so the system may try assigning a resource. Neither is likely to happen under PowerVM. Signed-off-by: Alexey Kardashevskiy Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/pci_of_scan.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/pci_of_scan.c b/arch/powerpc/kernel/pci_of_scan.c index 64ad92016b63..bd78f325a636 100644 --- a/arch/powerpc/kernel/pci_of_scan.c +++ b/arch/powerpc/kernel/pci_of_scan.c @@ -82,10 +82,16 @@ static void of_pci_parse_addrs(struct device_node *node, struct pci_dev *dev) const __be32 *addrs; u32 i; int proplen; + bool mark_unset = false; addrs = of_get_property(node, "assigned-addresses", &proplen); - if (!addrs) - return; + if (!addrs || !proplen) { + addrs = of_get_property(node, "reg", &proplen); + if (!addrs || !proplen) + return; + mark_unset = true; + } + pr_debug(" parse addresses (%d bytes) @ %p\n", proplen, addrs); for (; proplen >= 20; proplen -= 20, addrs += 5) { flags = pci_parse_of_flags(of_read_number(addrs, 1), 0); @@ -110,6 +116,8 @@ static void of_pci_parse_addrs(struct device_node *node, struct pci_dev *dev) continue; } res->flags = flags; + if (mark_unset) + res->flags |= IORESOURCE_UNSET; res->name = pci_name(dev); region.start = base; region.end = base + size - 1; -- cgit v1.2.3 From 1cfb725fb1899dc6fdc88f8b5354a65e8ad260c6 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 14 May 2019 09:05:13 +0000 Subject: powerpc/64: flush_inval_dcache_range() becomes flush_dcache_range() On most arches having function flush_dcache_range(), including PPC32, this function does a writeback and invalidation of the cache bloc. On PPC64, flush_dcache_range() only does a writeback while flush_inval_dcache_range() does the invalidation in addition. In addition it looks like within arch/powerpc/, there are no PPC64 platforms using flush_dcache_range() This patch drops the existing 64 bits version of flush_dcache_range() and renames flush_inval_dcache_range() into flush_dcache_range(). Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/cacheflush.h | 1 - arch/powerpc/kernel/misc_64.S | 27 ++------------------------- arch/powerpc/lib/pmem.c | 8 ++++---- arch/powerpc/mm/mem.c | 4 ++-- arch/powerpc/sysdev/dart_iommu.c | 2 +- drivers/macintosh/smu.c | 4 ++-- 6 files changed, 11 insertions(+), 35 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h index b189f7aee222..69b6c3d78d6c 100644 --- a/arch/powerpc/include/asm/cacheflush.h +++ b/arch/powerpc/include/asm/cacheflush.h @@ -112,7 +112,6 @@ static inline void invalidate_dcache_range(unsigned long start, #endif /* CONFIG_PPC32 */ #ifdef CONFIG_PPC64 extern void flush_dcache_range(unsigned long start, unsigned long stop); -extern void flush_inval_dcache_range(unsigned long start, unsigned long stop); #endif #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S index 262ba9481781..a4fd536efb44 100644 --- a/arch/powerpc/kernel/misc_64.S +++ b/arch/powerpc/kernel/misc_64.S @@ -121,31 +121,8 @@ EXPORT_SYMBOL(flush_icache_range) * * flush all bytes from start to stop-1 inclusive */ -_GLOBAL_TOC(flush_dcache_range) -/* - * Flush the data cache to memory - * - * Different systems have different cache line sizes - */ - ld r10,PPC64_CACHES@toc(r2) - lwz r7,DCACHEL1BLOCKSIZE(r10) /* Get dcache block size */ - addi r5,r7,-1 - andc r6,r3,r5 /* round low to line bdy */ - subf r8,r6,r4 /* compute length */ - add r8,r8,r5 /* ensure we get enough */ - lwz r9,DCACHEL1LOGBLOCKSIZE(r10) /* Get log-2 of dcache block size */ - srw. r8,r8,r9 /* compute line count */ - beqlr /* nothing to do? */ - mtctr r8 -0: dcbst 0,r6 - add r6,r6,r7 - bdnz 0b - sync - blr -EXPORT_SYMBOL(flush_dcache_range) - -_GLOBAL(flush_inval_dcache_range) +_GLOBAL_TOC(flush_dcache_range) ld r10,PPC64_CACHES@toc(r2) lwz r7,DCACHEL1BLOCKSIZE(r10) /* Get dcache block size */ addi r5,r7,-1 @@ -164,7 +141,7 @@ _GLOBAL(flush_inval_dcache_range) sync isync blr - +EXPORT_SYMBOL(flush_dcache_range) /* * Flush a particular page from the data cache to RAM. diff --git a/arch/powerpc/lib/pmem.c b/arch/powerpc/lib/pmem.c index 53c018762e1c..36e08bf850e0 100644 --- a/arch/powerpc/lib/pmem.c +++ b/arch/powerpc/lib/pmem.c @@ -23,14 +23,14 @@ void arch_wb_cache_pmem(void *addr, size_t size) { unsigned long start = (unsigned long) addr; - flush_inval_dcache_range(start, start + size); + flush_dcache_range(start, start + size); } EXPORT_SYMBOL(arch_wb_cache_pmem); void arch_invalidate_pmem(void *addr, size_t size) { unsigned long start = (unsigned long) addr; - flush_inval_dcache_range(start, start + size); + flush_dcache_range(start, start + size); } EXPORT_SYMBOL(arch_invalidate_pmem); @@ -43,7 +43,7 @@ long __copy_from_user_flushcache(void *dest, const void __user *src, unsigned long copied, start = (unsigned long) dest; copied = __copy_from_user(dest, src, size); - flush_inval_dcache_range(start, start + size); + flush_dcache_range(start, start + size); return copied; } @@ -53,7 +53,7 @@ void *memcpy_flushcache(void *dest, const void *src, size_t size) unsigned long start = (unsigned long) dest; memcpy(dest, src, size); - flush_inval_dcache_range(start, start + size); + flush_dcache_range(start, start + size); return dest; } diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 40bd4153ab09..096c87836e29 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -125,7 +125,7 @@ int __ref arch_add_memory(int nid, u64 start, u64 size, start, start + size, rc); return -EFAULT; } - flush_inval_dcache_range(start, start + size); + flush_dcache_range(start, start + size); return __add_pages(nid, start_pfn, nr_pages, restrictions); } @@ -151,7 +151,7 @@ void __ref arch_remove_memory(int nid, u64 start, u64 size, /* Remove htab bolted mappings for this section of memory */ start = (unsigned long)__va(start); - flush_inval_dcache_range(start, start + size); + flush_dcache_range(start, start + size); ret = remove_section_mapping(start, start + size); WARN_ON_ONCE(ret); diff --git a/arch/powerpc/sysdev/dart_iommu.c b/arch/powerpc/sysdev/dart_iommu.c index 2a751795ec1e..bc259a8d3f2d 100644 --- a/arch/powerpc/sysdev/dart_iommu.c +++ b/arch/powerpc/sysdev/dart_iommu.c @@ -158,7 +158,7 @@ static void dart_cache_sync(unsigned int *base, unsigned int count) unsigned int tmp; /* Perform a standard cache flush */ - flush_inval_dcache_range(start, end); + flush_dcache_range(start, end); /* * Perform the sequence described in the CPC925 manual to diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c index 6a844125cf2d..97758eed03f2 100644 --- a/drivers/macintosh/smu.c +++ b/drivers/macintosh/smu.c @@ -133,7 +133,7 @@ static void smu_start_cmd(void) /* Flush command and data to RAM */ faddr = (unsigned long)smu->cmd_buf; fend = faddr + smu->cmd_buf->length + 2; - flush_inval_dcache_range(faddr, fend); + flush_dcache_range(faddr, fend); /* We also disable NAP mode for the duration of the command @@ -195,7 +195,7 @@ static irqreturn_t smu_db_intr(int irq, void *arg) * reply length (it's only 2 cache lines anyway) */ faddr = (unsigned long)smu->cmd_buf; - flush_inval_dcache_range(faddr, faddr + 256); + flush_dcache_range(faddr, faddr + 256); /* Now check ack */ ack = (~cmd->cmd) & 0xff; -- cgit v1.2.3 From 22e9c88d486a0536d337d6e0973968be0a4cd4b2 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Tue, 14 May 2019 09:05:16 +0000 Subject: powerpc/64: reuse PPC32 static inline flush_dcache_range() This patch drops the assembly PPC64 version of flush_dcache_range() and re-uses the PPC32 static inline version. With GCC 8.1, the following code is generated: void flush_test(unsigned long start, unsigned long stop) { flush_dcache_range(start, stop); } 0000000000000130 <.flush_test>: 130: 3d 22 00 00 addis r9,r2,0 132: R_PPC64_TOC16_HA .data+0x8 134: 81 09 00 00 lwz r8,0(r9) 136: R_PPC64_TOC16_LO .data+0x8 138: 3d 22 00 00 addis r9,r2,0 13a: R_PPC64_TOC16_HA .data+0xc 13c: 80 e9 00 00 lwz r7,0(r9) 13e: R_PPC64_TOC16_LO .data+0xc 140: 7d 48 00 d0 neg r10,r8 144: 7d 43 18 38 and r3,r10,r3 148: 7c 00 04 ac hwsync 14c: 4c 00 01 2c isync 150: 39 28 ff ff addi r9,r8,-1 154: 7c 89 22 14 add r4,r9,r4 158: 7c 83 20 50 subf r4,r3,r4 15c: 7c 89 3c 37 srd. r9,r4,r7 160: 41 82 00 1c beq 17c <.flush_test+0x4c> 164: 7d 29 03 a6 mtctr r9 168: 60 00 00 00 nop 16c: 60 00 00 00 nop 170: 7c 00 18 ac dcbf 0,r3 174: 7c 63 42 14 add r3,r3,r8 178: 42 00 ff f8 bdnz 170 <.flush_test+0x40> 17c: 7c 00 04 ac hwsync 180: 4c 00 01 2c isync 184: 4e 80 00 20 blr 188: 60 00 00 00 nop 18c: 60 00 00 00 nop Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/cache.h | 10 ++++++++++ arch/powerpc/include/asm/cacheflush.h | 14 ++++++++------ arch/powerpc/kernel/misc_64.S | 29 ----------------------------- 3 files changed, 18 insertions(+), 35 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/cache.h b/arch/powerpc/include/asm/cache.h index e84d1622eeb6..b3388d95f451 100644 --- a/arch/powerpc/include/asm/cache.h +++ b/arch/powerpc/include/asm/cache.h @@ -54,6 +54,16 @@ struct ppc64_caches { }; extern struct ppc64_caches ppc64_caches; + +static inline u32 l1_cache_shift(void) +{ + return ppc64_caches.l1d.log_block_size; +} + +static inline u32 l1_cache_bytes(void) +{ + return ppc64_caches.l1d.block_size; +} #else static inline u32 l1_cache_shift(void) { diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h index 217f183aa8c4..5c5a4538fc47 100644 --- a/arch/powerpc/include/asm/cacheflush.h +++ b/arch/powerpc/include/asm/cacheflush.h @@ -60,7 +60,6 @@ static inline void __flush_dcache_icache_phys(unsigned long physaddr) } #endif -#ifdef CONFIG_PPC32 /* * Write any modified data cache blocks out to memory and invalidate them. * Does not invalidate the corresponding instruction cache blocks. @@ -73,9 +72,17 @@ static inline void flush_dcache_range(unsigned long start, unsigned long stop) unsigned long size = stop - (unsigned long)addr + (bytes - 1); unsigned long i; + if (IS_ENABLED(CONFIG_PPC64)) { + mb(); /* sync */ + isync(); + } + for (i = 0; i < size >> shift; i++, addr += bytes) dcbf(addr); mb(); /* sync */ + + if (IS_ENABLED(CONFIG_PPC64)) + isync(); } /* @@ -115,11 +122,6 @@ static inline void invalidate_dcache_range(unsigned long start, mb(); /* sync */ } -#endif /* CONFIG_PPC32 */ -#ifdef CONFIG_PPC64 -extern void flush_dcache_range(unsigned long start, unsigned long stop); -#endif - #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ memcpy(dst, src, len); \ diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S index a4fd536efb44..1b0a42c50ef1 100644 --- a/arch/powerpc/kernel/misc_64.S +++ b/arch/powerpc/kernel/misc_64.S @@ -114,35 +114,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE) _ASM_NOKPROBE_SYMBOL(flush_icache_range) EXPORT_SYMBOL(flush_icache_range) -/* - * Like above, but only do the D-cache. - * - * flush_dcache_range(unsigned long start, unsigned long stop) - * - * flush all bytes from start to stop-1 inclusive - */ - -_GLOBAL_TOC(flush_dcache_range) - ld r10,PPC64_CACHES@toc(r2) - lwz r7,DCACHEL1BLOCKSIZE(r10) /* Get dcache block size */ - addi r5,r7,-1 - andc r6,r3,r5 /* round low to line bdy */ - subf r8,r6,r4 /* compute length */ - add r8,r8,r5 /* ensure we get enough */ - lwz r9,DCACHEL1LOGBLOCKSIZE(r10)/* Get log-2 of dcache block size */ - srw. r8,r8,r9 /* compute line count */ - beqlr /* nothing to do? */ - sync - isync - mtctr r8 -0: dcbf 0,r6 - add r6,r6,r7 - bdnz 0b - sync - isync - blr -EXPORT_SYMBOL(flush_dcache_range) - /* * Flush a particular page from the data cache to RAM. * Note: this is necessary because the instruction cache does *not* -- cgit v1.2.3 From 2fb0a2c989837c976b68233496bbaefb47cd3d6f Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Sat, 6 Jul 2019 00:18:53 +1000 Subject: powerpc/module64: Fix comment in R_PPC64_ENTRY handling The comment here is wrong, the addi reads from r2 not r12. The code is correct, 0x38420000 = addi r2,r2,0. Fixes: a61674bdfc7c ("powerpc/module: Handle R_PPC64_ENTRY relocations") Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/module_64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c index 8661eea78503..4bf81a111179 100644 --- a/arch/powerpc/kernel/module_64.c +++ b/arch/powerpc/kernel/module_64.c @@ -719,7 +719,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, /* * If found, replace it with: * addis r2, r12, (.TOC.-func)@ha - * addi r2, r12, (.TOC.-func)@l + * addi r2, r2, (.TOC.-func)@l */ ((uint32_t *)location)[0] = 0x3c4c0000 + PPC_HA(value); ((uint32_t *)location)[1] = 0x38420000 + PPC_LO(value); -- cgit v1.2.3 From 7f9c929a7ff203eae60b4225bb6824c3eb31796c Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 3 May 2019 06:40:15 +0000 Subject: powerpc: Move PPC_HA() PPC_HI() and PPC_LO() to ppc-opcode.h PPC_HA() PPC_HI() and PPC_LO() macros are nice macros. Move them from module64.c to ppc-opcode.h in order to use them in other places. Signed-off-by: Christophe Leroy [mpe: Clean up formatting in new code, drop duplicates in ftrace.c] Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/ppc-opcode.h | 9 +++++++++ arch/powerpc/kernel/module_64.c | 7 ------- arch/powerpc/kernel/trace/ftrace.c | 4 ---- 3 files changed, 9 insertions(+), 11 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 971bdf84f6fc..f544432aef82 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -412,6 +412,15 @@ #define __PPC_SPR(r) ((((r) & 0x1f) << 16) | ((((r) >> 5) & 0x1f) << 11)) #define __PPC_RC21 (0x1 << 10) +/* + * Both low and high 16 bits are added as SIGNED additions, so if low 16 bits + * has high bit set, high 16 bits must be adjusted. These macros do that (stolen + * from binutils). + */ +#define PPC_LO(v) ((v) & 0xffff) +#define PPC_HI(v) (((v) >> 16) & 0xffff) +#define PPC_HA(v) PPC_HI((v) + 0x8000) + /* * Only use the larx hint bit on 64bit CPUs. e500v1/v2 based CPUs will treat a * larx with EH set as an illegal instruction. diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c index 4bf81a111179..80ceb458e79c 100644 --- a/arch/powerpc/kernel/module_64.c +++ b/arch/powerpc/kernel/module_64.c @@ -400,13 +400,6 @@ static inline unsigned long my_r2(const Elf64_Shdr *sechdrs, struct module *me) return (sechdrs[me->arch.toc_section].sh_addr & ~0xfful) + 0x8000; } -/* Both low and high 16 bits are added as SIGNED additions, so if low - 16 bits has high bit set, high 16 bits must be adjusted. These - macros do that (stolen from binutils). */ -#define PPC_LO(v) ((v) & 0xffff) -#define PPC_HI(v) (((v) >> 16) & 0xffff) -#define PPC_HA(v) PPC_HI ((v) + 0x8000) - /* Patch stub to reference function and correct r2 value. */ static inline int create_stub(const Elf64_Shdr *sechdrs, struct ppc64_stub_entry *entry, diff --git a/arch/powerpc/kernel/trace/ftrace.c b/arch/powerpc/kernel/trace/ftrace.c index 517662a56bdc..be1ca98fce5c 100644 --- a/arch/powerpc/kernel/trace/ftrace.c +++ b/arch/powerpc/kernel/trace/ftrace.c @@ -866,10 +866,6 @@ void arch_ftrace_update_code(int command) #ifdef CONFIG_PPC64 #define PACATOC offsetof(struct paca_struct, kernel_toc) -#define PPC_LO(v) ((v) & 0xffff) -#define PPC_HI(v) (((v) >> 16) & 0xffff) -#define PPC_HA(v) PPC_HI ((v) + 0x8000) - extern unsigned int ftrace_tramp_text[], ftrace_tramp_init[]; int __init ftrace_dyn_arch_init(void) -- cgit v1.2.3 From 4eb4516eada3ae469cd9d97c6dbe730ad3cdf2b6 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 3 May 2019 06:40:16 +0000 Subject: powerpc/module32: Use symbolic instructions names. To increase readability/maintainability, replace hard coded instructions values by symbolic names. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/module_32.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/module_32.c b/arch/powerpc/kernel/module_32.c index 88d83771f462..9cf201111d6c 100644 --- a/arch/powerpc/kernel/module_32.c +++ b/arch/powerpc/kernel/module_32.c @@ -172,10 +172,12 @@ int module_frob_arch_sections(Elf32_Ehdr *hdr, static inline int entry_matches(struct ppc_plt_entry *entry, Elf32_Addr val) { - if (entry->jump[0] == 0x3d800000 + ((val + 0x8000) >> 16) - && entry->jump[1] == 0x398c0000 + (val & 0xffff)) - return 1; - return 0; + if (entry->jump[0] != (PPC_INST_ADDIS | __PPC_RT(R12) | PPC_HA(val))) + return 0; + if (entry->jump[1] != (PPC_INST_ADDI | __PPC_RT(R12) | __PPC_RA(R12) | + PPC_LO(val))) + return 0; + return 1; } /* Set up a trampoline in the PLT to bounce us to the distant function */ @@ -200,10 +202,16 @@ static uint32_t do_plt_call(void *location, entry++; } - entry->jump[0] = 0x3d800000+((val+0x8000)>>16); /* lis r12,sym@ha */ - entry->jump[1] = 0x398c0000 + (val&0xffff); /* addi r12,r12,sym@l*/ - entry->jump[2] = 0x7d8903a6; /* mtctr r12 */ - entry->jump[3] = 0x4e800420; /* bctr */ + /* + * lis r12, sym@ha + * addi r12, r12, sym@l + * mtctr r12 + * bctr + */ + entry->jump[0] = PPC_INST_ADDIS | __PPC_RT(R12) | PPC_HA(val); + entry->jump[1] = PPC_INST_ADDI | __PPC_RT(R12) | __PPC_RA(R12) | PPC_LO(val); + entry->jump[2] = PPC_INST_MTCTR | __PPC_RS(R12); + entry->jump[3] = PPC_INST_BCTR; pr_debug("Initialized plt for 0x%x at %p\n", val, entry); return (uint32_t)entry; -- cgit v1.2.3 From a2b6f26c264e510f1d76ce347e34a21fbda9d8b2 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 3 May 2019 06:40:17 +0000 Subject: powerpc/module64: Use symbolic instructions names. To increase readability/maintainability, replace hard coded instructions values by symbolic names. Signed-off-by: Christophe Leroy [mpe: Fix R_PPC64_ENTRY case, the addi reads from r2 not r12] Signed-off-by: Michael Ellerman --- arch/powerpc/kernel/module_64.c | 53 +++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 18 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/module_64.c b/arch/powerpc/kernel/module_64.c index 80ceb458e79c..ef75f590ad3e 100644 --- a/arch/powerpc/kernel/module_64.c +++ b/arch/powerpc/kernel/module_64.c @@ -133,20 +133,27 @@ struct ppc64_stub_entry * the stub, but it's significantly shorter to put these values at the * end of the stub code, and patch the stub address (32-bits relative * to the TOC ptr, r2) into the stub. + * + * addis r11,r2, + * addi r11,r11, + * std r2,R2_STACK_OFFSET(r1) + * ld r12,32(r11) + * ld r2,40(r11) + * mtctr r12 + * bctr */ - static u32 ppc64_stub_insns[] = { - 0x3d620000, /* addis r11,r2, */ - 0x396b0000, /* addi r11,r11, */ + PPC_INST_ADDIS | __PPC_RT(R11) | __PPC_RA(R2), + PPC_INST_ADDI | __PPC_RT(R11) | __PPC_RA(R11), /* Save current r2 value in magic place on the stack. */ - 0xf8410000|R2_STACK_OFFSET, /* std r2,R2_STACK_OFFSET(r1) */ - 0xe98b0020, /* ld r12,32(r11) */ + PPC_INST_STD | __PPC_RS(R2) | __PPC_RA(R1) | R2_STACK_OFFSET, + PPC_INST_LD | __PPC_RT(R12) | __PPC_RA(R11) | 32, #ifdef PPC64_ELF_ABI_v1 /* Set up new r2 from function descriptor */ - 0xe84b0028, /* ld r2,40(r11) */ + PPC_INST_LD | __PPC_RT(R2) | __PPC_RA(R11) | 40, #endif - 0x7d8903a6, /* mtctr r12 */ - 0x4e800420 /* bctr */ + PPC_INST_MTCTR | __PPC_RS(R12), + PPC_INST_BCTR, }; #ifdef CONFIG_DYNAMIC_FTRACE @@ -704,18 +711,21 @@ int apply_relocate_add(Elf64_Shdr *sechdrs, * ld r2, ...(r12) * add r2, r2, r12 */ - if ((((uint32_t *)location)[0] & ~0xfffc) - != 0xe84c0000) + if ((((uint32_t *)location)[0] & ~0xfffc) != + (PPC_INST_LD | __PPC_RT(R2) | __PPC_RA(R12))) break; - if (((uint32_t *)location)[1] != 0x7c426214) + if (((uint32_t *)location)[1] != + (PPC_INST_ADD | __PPC_RT(R2) | __PPC_RA(R2) | __PPC_RB(R12))) break; /* * If found, replace it with: * addis r2, r12, (.TOC.-func)@ha * addi r2, r2, (.TOC.-func)@l */ - ((uint32_t *)location)[0] = 0x3c4c0000 + PPC_HA(value); - ((uint32_t *)location)[1] = 0x38420000 + PPC_LO(value); + ((uint32_t *)location)[0] = PPC_INST_ADDIS | __PPC_RT(R2) | + __PPC_RA(R12) | PPC_HA(value); + ((uint32_t *)location)[1] = PPC_INST_ADDI | __PPC_RT(R2) | + __PPC_RA(R2) | PPC_LO(value); break; case R_PPC64_REL16_HA: @@ -769,12 +779,19 @@ static unsigned long create_ftrace_stub(const Elf64_Shdr *sechdrs, { struct ppc64_stub_entry *entry; unsigned int i, num_stubs; + /* + * ld r12,PACATOC(r13) + * addis r12,r12, + * addi r12,r12, + * mtctr r12 + * bctr + */ static u32 stub_insns[] = { - 0xe98d0000 | PACATOC, /* ld r12,PACATOC(r13) */ - 0x3d8c0000, /* addis r12,r12, */ - 0x398c0000, /* addi r12,r12, */ - 0x7d8903a6, /* mtctr r12 */ - 0x4e800420, /* bctr */ + PPC_INST_LD | __PPC_RT(R12) | __PPC_RA(R13) | PACATOC, + PPC_INST_ADDIS | __PPC_RT(R12) | __PPC_RA(R12), + PPC_INST_ADDI | __PPC_RT(R12) | __PPC_RA(R12), + PPC_INST_MTCTR | __PPC_RS(R12), + PPC_INST_BCTR, }; long reladdr; -- cgit v1.2.3 From 0fc12c022ad25532b66bf6f6c818ee1c1d63e702 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Mon, 8 Jul 2019 16:02:19 +1000 Subject: powerpc/irq: Don't WARN continuously in arch_local_irq_restore() When CONFIG_PPC_IRQ_SOFT_MASK_DEBUG is enabled (uncommon), we have a series of WARN_ON's in arch_local_irq_restore(). These are "should never happen" conditions, but if they do happen they can flood the console and render the system unusable. So switch them to WARN_ON_ONCE(). Fixes: e2b36d591720 ("powerpc/64: Don't trace code that runs with the soft irq mask unreconciled") Fixes: 9b81c0211c24 ("powerpc/64s: make PACA_IRQ_HARD_DIS track MSR[EE] closely") Fixes: 7c0482e3d055 ("powerpc/irq: Fix another case of lazy IRQ state getting out of sync") Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20190708061046.7075-1-mpe@ellerman.id.au --- arch/powerpc/kernel/irq.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index ada901af4950..c9a6eac3075c 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -259,7 +259,7 @@ notrace void arch_local_irq_restore(unsigned long mask) irq_happened = get_irq_happened(); if (!irq_happened) { #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG - WARN_ON(!(mfmsr() & MSR_EE)); + WARN_ON_ONCE(!(mfmsr() & MSR_EE)); #endif return; } @@ -272,7 +272,7 @@ notrace void arch_local_irq_restore(unsigned long mask) */ if (!(irq_happened & PACA_IRQ_HARD_DIS)) { #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG - WARN_ON(!(mfmsr() & MSR_EE)); + WARN_ON_ONCE(!(mfmsr() & MSR_EE)); #endif __hard_irq_disable(); #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG @@ -283,7 +283,7 @@ notrace void arch_local_irq_restore(unsigned long mask) * warn if we are wrong. Only do that when IRQ tracing * is enabled as mfmsr() can be costly. */ - if (WARN_ON(mfmsr() & MSR_EE)) + if (WARN_ON_ONCE(mfmsr() & MSR_EE)) __hard_irq_disable(); #endif } -- cgit v1.2.3 From 33439620680be5225c1b8806579a291e0d761ca0 Mon Sep 17 00:00:00 2001 From: Oliver O'Halloran Date: Thu, 11 Jul 2019 01:05:17 +1000 Subject: powerpc/eeh: Handle hugepages in ioremap space In commit 4a7b06c157a2 ("powerpc/eeh: Handle hugepages in ioremap space") support for using hugepages in the vmalloc and ioremap areas was enabled for radix. Unfortunately this broke EEH MMIO error checking. Detection works by inserting a hook which checks the results of the ioreadXX() set of functions. When a read returns a 0xFFs response we need to check for an error which we do by mapping the (virtual) MMIO address back to a physical address, then mapping physical address to a PCI device via an interval tree. When translating virt -> phys we currently assume the ioremap space is only populated by PAGE_SIZE mappings. If a hugepage mapping is found we emit a WARN_ON(), but otherwise handles the check as though a normal page was found. In pathalogical cases such as copying a buffer containing a lot of 0xFFs from BAR memory this can result in the system not booting because it's too busy printing WARN_ON()s. There's no real reason to assume huge pages can't be present and we're prefectly capable of handling them, so do that. Fixes: 4a7b06c157a2 ("powerpc/eeh: Handle hugepages in ioremap space") Reported-by: Sachin Sant Signed-off-by: Oliver O'Halloran Tested-by: Sachin Sant Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20190710150517.27114-1-oohall@gmail.com --- arch/powerpc/kernel/eeh.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/kernel') diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 289c0b37d845..0dc1865c84ce 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -367,10 +367,19 @@ static inline unsigned long eeh_token_to_phys(unsigned long token) ptep = find_init_mm_pte(token, &hugepage_shift); if (!ptep) return token; - WARN_ON(hugepage_shift); - pa = pte_pfn(*ptep) << PAGE_SHIFT; - return pa | (token & (PAGE_SIZE-1)); + pa = pte_pfn(*ptep); + + /* On radix we can do hugepage mappings for io, so handle that */ + if (hugepage_shift) { + pa <<= hugepage_shift; + pa |= token & ((1ul << hugepage_shift) - 1); + } else { + pa <<= PAGE_SHIFT; + pa |= token & (PAGE_SIZE - 1); + } + + return pa; } /* -- cgit v1.2.3