From b9eab08d012fa093947b230f9a87257c27fb829b Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Thu, 16 Nov 2017 11:45:37 -0600 Subject: powerpc/modules: Don't try to restore r2 after a sibling call When attempting to load a livepatch module, I got the following error: module_64: patch_module: Expect noop after relocate, got 3c820000 The error was triggered by the following code in unregister_netdevice_queue(): 14c: 00 00 00 48 b 14c 14c: R_PPC64_REL24 net_set_todo 150: 00 00 82 3c addis r4,r2,0 GCC didn't insert a nop after the branch to net_set_todo() because it's a sibling call, so it never returns. The nop isn't needed after the branch in that case. Signed-off-by: Josh Poimboeuf Acked-by: Naveen N. Rao Reviewed-and-tested-by: Kamalesh Babulal Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/code-patching.h | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/code-patching.h b/arch/powerpc/include/asm/code-patching.h index abef812de7f8..2c895e8d07f7 100644 --- a/arch/powerpc/include/asm/code-patching.h +++ b/arch/powerpc/include/asm/code-patching.h @@ -33,6 +33,7 @@ int patch_branch(unsigned int *addr, unsigned long target, int flags); int patch_instruction(unsigned int *addr, unsigned int instr); int instr_is_relative_branch(unsigned int instr); +int instr_is_relative_link_branch(unsigned int instr); int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr); unsigned long branch_target(const unsigned int *instr); unsigned int translate_branch(const unsigned int *dest, -- cgit v1.2.3 From 988fc3ba5653278a8c14d6ccf687371775930d2b Mon Sep 17 00:00:00 2001 From: "Bryant G. Ly" Date: Thu, 9 Nov 2017 08:00:33 -0600 Subject: powerpc/pci: Separate SR-IOV Calls SR-IOV can now be enabled for the powernv platform and pseries platform. Therefore move the appropriate calls to machine dependent code instead of relying on definition at compile time. Signed-off-by: Bryant G. Ly Signed-off-by: Juan J. Alvarez Acked-by: Russell Currey Reviewed-by: Alexey Kardashevskiy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/machdep.h | 8 +++++++ arch/powerpc/include/asm/pci-bridge.h | 4 +--- arch/powerpc/kernel/eeh_driver.c | 4 ++-- arch/powerpc/kernel/pci-common.c | 23 +++++++++++++++++++ arch/powerpc/kernel/pci_dn.c | 6 ----- arch/powerpc/platforms/powernv/eeh-powernv.c | 33 ++++++++++++++-------------- arch/powerpc/platforms/powernv/pci-ioda.c | 6 +++-- 7 files changed, 55 insertions(+), 29 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h index 73b92017b6d7..4298ceae6db5 100644 --- a/arch/powerpc/include/asm/machdep.h +++ b/arch/powerpc/include/asm/machdep.h @@ -172,11 +172,19 @@ struct machdep_calls { /* Called after scan and before resource survey */ void (*pcibios_fixup_phb)(struct pci_controller *hose); + /* + * Called after device has been added to bus and + * before sysfs has been created. + */ + void (*pcibios_bus_add_device)(struct pci_dev *pdev); + resource_size_t (*pcibios_default_alignment)(void); #ifdef CONFIG_PCI_IOV void (*pcibios_fixup_sriov)(struct pci_dev *pdev); resource_size_t (*pcibios_iov_resource_alignment)(struct pci_dev *, int resno); + int (*pcibios_sriov_enable)(struct pci_dev *pdev, u16 num_vfs); + int (*pcibios_sriov_disable)(struct pci_dev *pdev); #endif /* CONFIG_PCI_IOV */ /* Called to shutdown machine specific hardware not already controlled diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h index 62ed83db04ae..9f66ddebb799 100644 --- a/arch/powerpc/include/asm/pci-bridge.h +++ b/arch/powerpc/include/asm/pci-bridge.h @@ -203,10 +203,9 @@ struct pci_dn { struct eeh_dev *edev; /* eeh device */ #endif #define IODA_INVALID_PE 0xFFFFFFFF -#ifdef CONFIG_PPC_POWERNV unsigned int pe_number; - int vf_index; /* VF index in the PF */ #ifdef CONFIG_PCI_IOV + int vf_index; /* VF index in the PF */ u16 vfs_expanded; /* number of VFs IOV BAR expanded */ u16 num_vfs; /* number of VFs enabled*/ unsigned int *pe_num_map; /* PE# for the first VF PE or array */ @@ -215,7 +214,6 @@ struct pci_dn { int (*m64_map)[PCI_SRIOV_NUM_BARS]; #endif /* CONFIG_PCI_IOV */ int mps; /* Maximum Payload Size */ -#endif struct list_head child_list; struct list_head list; struct resource holes[PCI_SRIOV_NUM_BARS]; diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 4f71e4c9beb7..3c0fa99c5533 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -440,7 +440,7 @@ static void *eeh_add_virt_device(void *data, void *userdata) return NULL; } -#ifdef CONFIG_PPC_POWERNV +#ifdef CONFIG_PCI_IOV pci_iov_add_virtfn(edev->physfn, pdn->vf_index); #endif return NULL; @@ -496,7 +496,7 @@ static void *eeh_rmv_device(void *data, void *userdata) (*removed)++; if (edev->physfn) { -#ifdef CONFIG_PPC_POWERNV +#ifdef CONFIG_PCI_IOV struct pci_dn *pdn = eeh_dev_to_pdn(edev); pci_iov_remove_virtfn(edev->physfn, pdn->vf_index); diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 7cd2803e2cc3..68d18ff38808 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -249,8 +249,31 @@ resource_size_t pcibios_iov_resource_alignment(struct pci_dev *pdev, int resno) return pci_iov_resource_size(pdev, resno); } + +int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +{ + if (ppc_md.pcibios_sriov_enable) + return ppc_md.pcibios_sriov_enable(pdev, num_vfs); + + return 0; +} + +int pcibios_sriov_disable(struct pci_dev *pdev) +{ + if (ppc_md.pcibios_sriov_disable) + return ppc_md.pcibios_sriov_disable(pdev); + + return 0; +} + #endif /* CONFIG_PCI_IOV */ +void pcibios_bus_add_device(struct pci_dev *pdev) +{ + if (ppc_md.pcibios_bus_add_device) + ppc_md.pcibios_bus_add_device(pdev); +} + static resource_size_t pcibios_io_size(const struct pci_controller *hose) { #ifdef CONFIG_PPC64 diff --git a/arch/powerpc/kernel/pci_dn.c b/arch/powerpc/kernel/pci_dn.c index 0e395afbf0f4..ab147a1909c8 100644 --- a/arch/powerpc/kernel/pci_dn.c +++ b/arch/powerpc/kernel/pci_dn.c @@ -156,10 +156,8 @@ static struct pci_dn *add_one_dev_pci_data(struct pci_dn *parent, pdn->parent = parent; pdn->busno = busno; pdn->devfn = devfn; -#ifdef CONFIG_PPC_POWERNV pdn->vf_index = vf_index; pdn->pe_number = IODA_INVALID_PE; -#endif INIT_LIST_HEAD(&pdn->child_list); INIT_LIST_HEAD(&pdn->list); list_add_tail(&pdn->list, &parent->child_list); @@ -226,9 +224,7 @@ void remove_dev_pci_data(struct pci_dev *pdev) */ if (pdev->is_virtfn) { pdn = pci_get_pdn(pdev); -#ifdef CONFIG_PPC_POWERNV pdn->pe_number = IODA_INVALID_PE; -#endif return; } @@ -294,9 +290,7 @@ struct pci_dn *pci_add_device_node_info(struct pci_controller *hose, return NULL; dn->data = pdn; pdn->phb = hose; -#ifdef CONFIG_PPC_POWERNV pdn->pe_number = IODA_INVALID_PE; -#endif regs = of_get_property(dn, "reg", NULL); if (regs) { u32 addr = of_read_number(regs, 1); diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c index 4650fb294e7a..961e64115d92 100644 --- a/arch/powerpc/platforms/powernv/eeh-powernv.c +++ b/arch/powerpc/platforms/powernv/eeh-powernv.c @@ -43,6 +43,22 @@ static int eeh_event_irq = -EINVAL; +void pnv_pcibios_bus_add_device(struct pci_dev *pdev) +{ + struct pci_dn *pdn = pci_get_pdn(pdev); + + if (!pdev->is_virtfn) + return; + + /* + * The following operations will fail if VF's sysfs files + * aren't created or its resources aren't finalized. + */ + eeh_add_device_early(pdn); + eeh_add_device_late(pdev); + eeh_sysfs_add_device(pdev); +} + static int pnv_eeh_init(void) { struct pci_controller *hose; @@ -86,6 +102,7 @@ static int pnv_eeh_init(void) } eeh_set_pe_aux_size(max_diag_size); + ppc_md.pcibios_bus_add_device = pnv_pcibios_bus_add_device; return 0; } @@ -1749,22 +1766,6 @@ static struct eeh_ops pnv_eeh_ops = { .restore_config = pnv_eeh_restore_config }; -void pcibios_bus_add_device(struct pci_dev *pdev) -{ - struct pci_dn *pdn = pci_get_pdn(pdev); - - if (!pdev->is_virtfn) - return; - - /* - * The following operations will fail if VF's sysfs files - * aren't created or its resources aren't finalized. - */ - eeh_add_device_early(pdn); - eeh_add_device_late(pdev); - eeh_sysfs_add_device(pdev); -} - #ifdef CONFIG_PCI_IOV static void pnv_pci_fixup_vf_mps(struct pci_dev *pdev) { diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 080ee202a173..b9146d5aef81 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1692,7 +1692,7 @@ m64_failed: return ret; } -int pcibios_sriov_disable(struct pci_dev *pdev) +int pnv_pcibios_sriov_disable(struct pci_dev *pdev) { pnv_pci_sriov_disable(pdev); @@ -1701,7 +1701,7 @@ int pcibios_sriov_disable(struct pci_dev *pdev) return 0; } -int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) +int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) { /* Allocate PCI data */ add_dev_pci_data(pdev); @@ -4019,6 +4019,8 @@ static void __init pnv_pci_init_ioda_phb(struct device_node *np, #ifdef CONFIG_PCI_IOV ppc_md.pcibios_fixup_sriov = pnv_pci_ioda_fixup_iov_resources; ppc_md.pcibios_iov_resource_alignment = pnv_pci_iov_resource_alignment; + ppc_md.pcibios_sriov_enable = pnv_pcibios_sriov_enable; + ppc_md.pcibios_sriov_disable = pnv_pcibios_sriov_disable; #endif pci_add_flags(PCI_REASSIGN_ALL_RSRC); -- cgit v1.2.3 From 59aa31fd6f964ac63ac9cb90c8468f60e9bb0756 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Mon, 6 Nov 2017 00:50:45 -0800 Subject: powerpc: introduce pte_set_hidx() helper Introduce pte_set_hidx().It sets the (H_PAGE_F_SECOND|H_PAGE_F_GIX) bits at the appropriate location in the PTE of 4K PTE. For 64K PTE, it sets the bits in the second part of the PTE. Though the implementation for the former just needs the slot parameter, it does take some additional parameters to keep the prototype consistent. This function will be handy as we work towards re-arranging the bits in the subsequent patches. Acked-by: Balbir Singh Reviewed-by: Aneesh Kumar K.V Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 14 ++++++++++++++ arch/powerpc/include/asm/book3s/64/hash-64k.h | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index 197ced1eaaa0..2975fc172d1a 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -49,6 +49,20 @@ static inline int hash__hugepd_ok(hugepd_t hpd) } #endif +/* + * 4K PTE format is different from 64K PTE format. Saving the hash_slot is just + * a matter of returning the PTE bits that need to be modified. On 64K PTE, + * things are a little more involved and hence needs many more parameters to + * accomplish the same. However we want to abstract this out from the caller by + * keeping the prototype consistent across the two formats. + */ +static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte, + unsigned int subpg_index, unsigned long hidx) +{ + return (hidx << H_PAGE_F_GIX_SHIFT) & + (H_PAGE_F_SECOND | H_PAGE_F_GIX); +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline char *get_hpte_slot_array(pmd_t *pmdp) diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index 8d40cf03cb67..c1bd258b7303 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -68,6 +68,8 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep) return rpte; } +#define HIDX_BITS(x, index) (x << (index << 2)) + static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) { if ((pte_val(rpte.pte) & H_PAGE_COMBO)) @@ -75,6 +77,29 @@ static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf; } +/* + * Commit the hidx and return PTE bits that needs to be modified. The caller is + * expected to modify the PTE bits accordingly and commit the PTE to memory. + */ +static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte, + unsigned int subpg_index, unsigned long hidx) +{ + unsigned long *hidxp = (unsigned long *)(ptep + PTRS_PER_PTE); + + rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index); + *hidxp = rpte.hidx | HIDX_BITS(hidx, subpg_index); + + /* + * Anyone reading PTE must ensure hidx bits are read after reading the + * PTE by using the read-side barrier smp_rmb(). __real_pte() can be + * used for that. + */ + smp_wmb(); + + /* No PTE bits to be modified, return 0x0UL */ + return 0x0UL; +} + #define __rpte_to_pte(r) ((r).pte) extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); /* -- cgit v1.2.3 From 318995b4f5fa814b9f9aa434ca845b9a2cb0ae02 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Mon, 6 Nov 2017 00:50:46 -0800 Subject: powerpc: introduce pte_get_hash_gslot() helper Introduce pte_get_hash_gslot()() which returns the global slot number of the HPTE in the global hash table. This function will come in handy as we work towards re-arranging the PTE bits in the later patches. Reviewed-by: Aneesh Kumar K.V Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash.h | 3 +++ arch/powerpc/mm/hash_utils_64.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index ecb1239d74f4..9099c1f3a54e 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -167,6 +167,9 @@ static inline int hash__pte_none(pte_t pte) return (pte_val(pte) & ~H_PTE_NONE_MASK) == 0; } +unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift, + int ssize, real_pte_t rpte, unsigned int subpg_index); + /* This low level function performs the actual PTE insertion * Setting the PTE depends on the MMU type and other factors. It's * an horrible mess that I'm not going to try to clean up now but diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 655a5a9a183d..b35bd0202934 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -1592,6 +1592,24 @@ static inline void tm_flush_hash_page(int local) } #endif +/* + * Return the global hash slot, corresponding to the given PTE, which contains + * the HPTE. + */ +unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift, + int ssize, real_pte_t rpte, unsigned int subpg_index) +{ + unsigned long hash, gslot, hidx; + + hash = hpt_hash(vpn, shift, ssize); + hidx = __rpte_to_hidx(rpte, subpg_index); + if (hidx & _PTEIDX_SECONDARY) + hash = ~hash; + gslot = (hash & htab_hash_mask) * HPTES_PER_GROUP; + gslot += hidx & _PTEIDX_GROUP_IX; + return gslot; +} + /* WARNING: This is called from hash_low_64.S, if you change this prototype, * do not forget to update the assembly call site ! */ -- cgit v1.2.3 From 9d2edb18486febea930ea028dd128544cc28f3fe Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Mon, 6 Nov 2017 00:50:47 -0800 Subject: powerpc: Free up four 64K PTE bits in 4K backed HPTE pages Rearrange 64K PTE bits to free up bits 3, 4, 5 and 6, in the 4K backed HPTE pages.These bits continue to be used for 64K backed HPTE pages in this patch, but will be freed up in the next patch. The bit numbers are big-endian as defined in the ISA3.0 The patch does the following change to the 4k HTPE backed 64K PTE's format. H_PAGE_BUSY moves from bit 3 to bit 9 (B bit in the figure below) V0 which occupied bit 4 is not used anymore. V1 which occupied bit 5 is not used anymore. V2 which occupied bit 6 is not used anymore. V3 which occupied bit 7 is not used anymore. Before the patch, the 4k backed 64k PTE format was as follows 0 1 2 3 4 5 6 7 8 9 10...........................63 : : : : : : : : : : : : v v v v v v v v v v v v ,-,-,-,-,--,--,--,--,-,-,-,-,-,------------------,-,-,-, |x|x|x|B|V0|V1|V2|V3|x| | |x|x|................|x|x|x|x| <- primary pte '_'_'_'_'__'__'__'__'_'_'_'_'_'________________'_'_'_'_' |S|G|I|X|S |G |I |X |S|G|I|X|..................|S|G|I|X| <- secondary pte '_'_'_'_'__'__'__'__'_'_'_'_'__________________'_'_'_'_' After the patch, the 4k backed 64k PTE format is as follows 0 1 2 3 4 5 6 7 8 9 10...........................63 : : : : : : : : : : : : v v v v v v v v v v v v ,-,-,-,-,--,--,--,--,-,-,-,-,-,------------------,-,-,-, |x|x|x| | | | | |x|B| |x|x|................|.|.|.|.| <- primary pte '_'_'_'_'__'__'__'__'_'_'_'_'_'________________'_'_'_'_' |S|G|I|X|S |G |I |X |S|G|I|X|..................|S|G|I|X| <- secondary pte '_'_'_'_'__'__'__'__'_'_'_'_'__________________'_'_'_'_' the four bits S,G,I,X (one quadruplet per 4k HPTE) that cache the hash-bucket slot value, is initialized to 1,1,1,1 indicating -- an invalid slot. If a HPTE gets cached in a 1111 slot(i.e 7th slot of secondary hash bucket), it is released immediately. In other words, even though 1111 is a valid slot value in the hash bucket, we consider it invalid and release the slot and the HPTE. This gives us the opportunity to determine the validity of S,G,I,X bits based on its contents and not on any of the bits V0,V1,V2 or V3 in the primary PTE When we release a HPTE cached in the 1111 slot we also release a legitimate slot in the primary hash bucket and unmap its corresponding HPTE. This is to ensure that we do get a HPTE cached in a slot of the primary hash bucket, the next time we retry. Though treating 1111 slot as invalid, reduces the number of available slots in the hash bucket and may have an effect on the performance, the probabilty of hitting a 1111 slot is extermely low. Compared to the current scheme, the above scheme reduces the number of false hash table updates significantly and has the added advantage of releasing four valuable PTE bits for other purpose. NOTE:even though bits 3, 4, 5, 6, 7 are not used when the 64K PTE is backed by 4k HPTE, they continue to be used if the PTE gets backed by 64k HPTE. The next patch will decouple that aswell, and truely release the bits. This idea was jointly developed by Paul Mackerras, Aneesh, Michael Ellermen and myself. 4K PTE format remains unchanged currently. The patch does the following code changes a) PTE flags are split between 64k and 4k header files. b) __hash_page_4K() is reimplemented to reflect the above logic. Acked-by: Balbir Singh Reviewed-by: Aneesh Kumar K.V Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 2 + arch/powerpc/include/asm/book3s/64/hash-64k.h | 9 +-- arch/powerpc/include/asm/book3s/64/hash.h | 1 - arch/powerpc/mm/hash64_64k.c | 102 +++++++++++++------------- arch/powerpc/mm/hash_utils_64.c | 4 +- 5 files changed, 61 insertions(+), 57 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index 2975fc172d1a..e0130c34766e 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -17,6 +17,8 @@ #define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE) #define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE) +#define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */ + /* PTE flags to conserve for HPTE identification */ #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \ H_PAGE_F_SECOND | H_PAGE_F_GIX) diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index c1bd258b7303..c929e6437a60 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -13,18 +13,14 @@ */ #define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */ #define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */ +#define H_PAGE_BUSY _RPAGE_RPN42 /* software: PTE & hash are busy */ + /* * We need to differentiate between explicit huge page and THP huge * page, since THP huge page also need to track real subpage details */ #define H_PAGE_THP_HUGE H_PAGE_4K_PFN -/* - * Used to track subpage group valid if H_PAGE_COMBO is set - * This overloads H_PAGE_F_GIX and H_PAGE_F_SECOND - */ -#define H_PAGE_COMBO_VALID (H_PAGE_F_GIX | H_PAGE_F_SECOND) - /* PTE flags to conserve for HPTE identification */ #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \ H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO) @@ -69,6 +65,7 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep) } #define HIDX_BITS(x, index) (x << (index << 2)) +#define INVALID_RPTE_HIDX ~(0x0UL) static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) { diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 9099c1f3a54e..4e19a612a963 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -10,7 +10,6 @@ */ #define H_PTE_NONE_MASK _PAGE_HPTEFLAGS #define H_PAGE_F_GIX_SHIFT 56 -#define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */ #define H_PAGE_F_SECOND _RPAGE_RSV2 /* HPTE is in 2ndary HPTEG */ #define H_PAGE_F_GIX (_RPAGE_RSV3 | _RPAGE_RSV4 | _RPAGE_RPN44) #define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */ diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c index 1a68cb19b0e3..b79f50c693cf 100644 --- a/arch/powerpc/mm/hash64_64k.c +++ b/arch/powerpc/mm/hash64_64k.c @@ -15,34 +15,22 @@ #include #include #include + /* - * index from 0 - 15 + * Return true, if the entry has a slot value which + * the software considers as invalid. */ -bool __rpte_sub_valid(real_pte_t rpte, unsigned long index) +static inline bool hpte_soft_invalid(unsigned long hidx) { - unsigned long g_idx; - unsigned long ptev = pte_val(rpte.pte); - - g_idx = (ptev & H_PAGE_COMBO_VALID) >> H_PAGE_F_GIX_SHIFT; - index = index >> 2; - if (g_idx & (0x1 << index)) - return true; - else - return false; + return ((hidx & 0xfUL) == 0xfUL); } + /* * index from 0 - 15 */ -static unsigned long mark_subptegroup_valid(unsigned long ptev, unsigned long index) +bool __rpte_sub_valid(real_pte_t rpte, unsigned long index) { - unsigned long g_idx; - - if (!(ptev & H_PAGE_COMBO)) - return ptev; - index = index >> 2; - g_idx = 0x1 << index; - - return ptev | (g_idx << H_PAGE_F_GIX_SHIFT); + return !(hpte_soft_invalid(__rpte_to_hidx(rpte, index))); } int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, @@ -50,12 +38,11 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, int ssize, int subpg_prot) { real_pte_t rpte; - unsigned long *hidxp; unsigned long hpte_group; unsigned int subpg_index; - unsigned long rflags, pa, hidx; + unsigned long rflags, pa; unsigned long old_pte, new_pte, subpg_pte; - unsigned long vpn, hash, slot; + unsigned long vpn, hash, slot, gslot; unsigned long shift = mmu_psize_defs[MMU_PAGE_4K].shift; /* @@ -126,18 +113,14 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, if (__rpte_sub_valid(rpte, subpg_index)) { int ret; - hash = hpt_hash(vpn, shift, ssize); - hidx = __rpte_to_hidx(rpte, subpg_index); - if (hidx & _PTEIDX_SECONDARY) - hash = ~hash; - slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; - slot += hidx & _PTEIDX_GROUP_IX; - - ret = mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, + gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, + subpg_index); + ret = mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, MMU_PAGE_4K, MMU_PAGE_4K, ssize, flags); + /* - *if we failed because typically the HPTE wasn't really here + * If we failed because typically the HPTE wasn't really here * we try an insertion. */ if (ret == -1) @@ -148,6 +131,14 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, } htab_insert_hpte: + + /* + * Initialize all hidx entries to invalid value, the first time + * the PTE is about to allocate a 4K HPTE. + */ + if (!(old_pte & H_PAGE_COMBO)) + rpte.hidx = INVALID_RPTE_HIDX; + /* * handle H_PAGE_4K_PFN case */ @@ -172,15 +163,39 @@ repeat: * Primary is full, try the secondary */ if (unlikely(slot == -1)) { + bool soft_invalid; + hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, HPTE_V_SECONDARY, MMU_PAGE_4K, MMU_PAGE_4K, ssize); - if (slot == -1) { - if (mftb() & 0x1) + + soft_invalid = hpte_soft_invalid(slot); + if (unlikely(soft_invalid)) { + /* + * We got a valid slot from a hardware point of view. + * but we cannot use it, because we use this special + * value; as defined by hpte_soft_invalid(), to track + * invalid slots. We cannot use it. So invalidate it. + */ + gslot = slot & _PTEIDX_GROUP_IX; + mmu_hash_ops.hpte_invalidate(hpte_group + gslot, vpn, + MMU_PAGE_4K, MMU_PAGE_4K, + ssize, 0); + } + + if (unlikely(slot == -1 || soft_invalid)) { + /* + * For soft invalid slot, let's ensure that we release a + * slot from the primary, with the hope that we will + * acquire that slot next time we try. This will ensure + * that we do not get the same soft-invalid slot. + */ + if (soft_invalid || (mftb() & 0x1)) hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + mmu_hash_ops.hpte_remove(hpte_group); /* * FIXME!! Should be try the group from which we removed ? @@ -198,21 +213,10 @@ repeat: MMU_PAGE_4K, MMU_PAGE_4K, old_pte); return -1; } - /* - * Insert slot number & secondary bit in PTE second half, - * clear H_PAGE_BUSY and set appropriate HPTE slot bit - * Since we have H_PAGE_BUSY set on ptep, we can be sure - * nobody is undating hidx. - */ - hidxp = (unsigned long *)(ptep + PTRS_PER_PTE); - rpte.hidx &= ~(0xfUL << (subpg_index << 2)); - *hidxp = rpte.hidx | (slot << (subpg_index << 2)); - new_pte = mark_subptegroup_valid(new_pte, subpg_index); - new_pte |= H_PAGE_HASHPTE; - /* - * check __real_pte for details on matching smp_rmb() - */ - smp_wmb(); + + new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot); + new_pte |= H_PAGE_HASHPTE; + *ptep = __pte(new_pte & ~H_PAGE_BUSY); return 0; } diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index b35bd0202934..88102b6c664c 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -979,8 +979,9 @@ void __init hash__early_init_devtree(void) void __init hash__early_init_mmu(void) { +#ifndef CONFIG_PPC_64K_PAGES /* - * We have code in __hash_page_64K() and elsewhere, which assumes it can + * We have code in __hash_page_4K() and elsewhere, which assumes it can * do the following: * new_pte |= (slot << H_PAGE_F_GIX_SHIFT) & (H_PAGE_F_SECOND | H_PAGE_F_GIX); * @@ -991,6 +992,7 @@ void __init hash__early_init_mmu(void) * with a BUILD_BUG_ON(). */ BUILD_BUG_ON(H_PAGE_F_SECOND != (1ul << (H_PAGE_F_GIX_SHIFT + 3))); +#endif /* CONFIG_PPC_64K_PAGES */ htab_init_page_sizes(); -- cgit v1.2.3 From bf9a95f9a6481bc6ec98ef7b328c14177eeb3492 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Mon, 6 Nov 2017 00:50:48 -0800 Subject: powerpc: Free up four 64K PTE bits in 64K backed HPTE pages Rearrange 64K PTE bits to free up bits 3, 4, 5 and 6 in the 64K backed HPTE pages. This along with the earlier patch will entirely free up the four bits from 64K PTE. The bit numbers are big-endian as defined in the ISA3.0 This patch does the following change to 64K PTE backed by 64K HPTE. H_PAGE_F_SECOND (S) which occupied bit 4 moves to the second part of the pte to bit 60. H_PAGE_F_GIX (G,I,X) which occupied bit 5, 6 and 7 also moves to the second part of the pte to bit 61, 62, 63, 64 respectively since bit 7 is now freed up, we move H_PAGE_BUSY (B) from bit 9 to bit 7. The second part of the PTE will hold (H_PAGE_F_SECOND|H_PAGE_F_GIX) at bit 60,61,62,63. NOTE: None of the bits in the secondary PTE were not used by 64k-HPTE backed PTE. Before the patch, the 64K HPTE backed 64k PTE format was as follows 0 1 2 3 4 5 6 7 8 9 10...........................63 : : : : : : : : : : : : v v v v v v v v v v v v ,-,-,-,-,--,--,--,--,-,-,-,-,-,------------------,-,-,-, |x|x|x| |S |G |I |X |x|B| |x|x|................|x|x|x|x| <- primary pte '_'_'_'_'__'__'__'__'_'_'_'_'_'________________'_'_'_'_' | | | | | | | | | | | | |..................| | | | | <- secondary pte '_'_'_'_'__'__'__'__'_'_'_'_'__________________'_'_'_'_' After the patch, the 64k HPTE backed 64k PTE format is as follows 0 1 2 3 4 5 6 7 8 9 10...........................63 : : : : : : : : : : : : v v v v v v v v v v v v ,-,-,-,-,--,--,--,--,-,-,-,-,-,------------------,-,-,-, |x|x|x| | | | |B |x| | |x|x|................|.|.|.|.| <- primary pte '_'_'_'_'__'__'__'__'_'_'_'_'_'________________'_'_'_'_' | | | | | | | | | | | | |..................|S|G|I|X| <- secondary pte '_'_'_'_'__'__'__'__'_'_'_'_'__________________'_'_'_'_' The above PTE changes is applicable to hugetlbpages aswell. The patch does the following code changes: a) moves the H_PAGE_F_SECOND and H_PAGE_F_GIX to 4k PTE header since it is no more needed b the 64k PTEs. b) abstracts out __real_pte() and __rpte_to_hidx() so the caller need not know the bit location of the slot. c) moves the slot bits to the secondary pte. Reviewed-by: Aneesh Kumar K.V Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 3 +++ arch/powerpc/include/asm/book3s/64/hash-64k.h | 30 +++++++++++++-------------- arch/powerpc/include/asm/book3s/64/hash.h | 3 --- arch/powerpc/mm/hash64_64k.c | 21 +++++++++---------- arch/powerpc/mm/hugetlbpage-hash64.c | 16 ++++++-------- 5 files changed, 33 insertions(+), 40 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index e0130c34766e..ed0d1cd98907 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -17,6 +17,9 @@ #define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE) #define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE) +#define H_PAGE_F_GIX_SHIFT 56 +#define H_PAGE_F_SECOND _RPAGE_RSV2 /* HPTE is in 2ndary HPTEG */ +#define H_PAGE_F_GIX (_RPAGE_RSV3 | _RPAGE_RSV4 | _RPAGE_RPN44) #define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */ /* PTE flags to conserve for HPTE identification */ diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index c929e6437a60..73ed988e6da5 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -13,7 +13,7 @@ */ #define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */ #define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */ -#define H_PAGE_BUSY _RPAGE_RPN42 /* software: PTE & hash are busy */ +#define H_PAGE_BUSY _RPAGE_RPN44 /* software: PTE & hash are busy */ /* * We need to differentiate between explicit huge page and THP huge @@ -22,8 +22,7 @@ #define H_PAGE_THP_HUGE H_PAGE_4K_PFN /* PTE flags to conserve for HPTE identification */ -#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \ - H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO) +#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | H_PAGE_COMBO) /* * we support 16 fragments per PTE page of 64K size. */ @@ -51,27 +50,26 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep) unsigned long *hidxp; rpte.pte = pte; - rpte.hidx = 0; - if (pte_val(pte) & H_PAGE_COMBO) { - /* - * Make sure we order the hidx load against the H_PAGE_COMBO - * check. The store side ordering is done in __hash_page_4K - */ - smp_rmb(); - hidxp = (unsigned long *)(ptep + PTRS_PER_PTE); - rpte.hidx = *hidxp; - } + + /* + * Ensure that we do not read the hidx before we read the PTE. Because + * the writer side is expected to finish writing the hidx first followed + * by the PTE, by using smp_wmb(). pte_set_hash_slot() ensures that. + */ + smp_rmb(); + + hidxp = (unsigned long *)(ptep + PTRS_PER_PTE); + rpte.hidx = *hidxp; return rpte; } #define HIDX_BITS(x, index) (x << (index << 2)) +#define BITS_TO_HIDX(x, index) ((x >> (index << 2)) & 0xfUL) #define INVALID_RPTE_HIDX ~(0x0UL) static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) { - if ((pte_val(rpte.pte) & H_PAGE_COMBO)) - return (rpte.hidx >> (index<<2)) & 0xf; - return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf; + return BITS_TO_HIDX(rpte.hidx, index); } /* diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 4e19a612a963..e72474f13936 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -9,9 +9,6 @@ * */ #define H_PTE_NONE_MASK _PAGE_HPTEFLAGS -#define H_PAGE_F_GIX_SHIFT 56 -#define H_PAGE_F_SECOND _RPAGE_RSV2 /* HPTE is in 2ndary HPTEG */ -#define H_PAGE_F_GIX (_RPAGE_RSV3 | _RPAGE_RSV4 | _RPAGE_RPN44) #define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */ #ifdef CONFIG_PPC_64K_PAGES diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c index b79f50c693cf..2253bbc6a599 100644 --- a/arch/powerpc/mm/hash64_64k.c +++ b/arch/powerpc/mm/hash64_64k.c @@ -103,8 +103,8 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, * On hash insert failure we use old pte value and we don't * want slot information there if we have a insert failure. */ - old_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND); - new_pte &= ~(H_PAGE_HASHPTE | H_PAGE_F_GIX | H_PAGE_F_SECOND); + old_pte &= ~H_PAGE_HASHPTE; + new_pte &= ~H_PAGE_HASHPTE; goto htab_insert_hpte; } /* @@ -225,6 +225,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize) { + real_pte_t rpte; unsigned long hpte_group; unsigned long rflags, pa; unsigned long old_pte, new_pte; @@ -261,6 +262,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access, } while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); rflags = htab_convert_pte_flags(new_pte); + rpte = __real_pte(__pte(old_pte), ptep); if (cpu_has_feature(CPU_FTR_NOEXECUTE) && !cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) @@ -268,16 +270,13 @@ int __hash_page_64K(unsigned long ea, unsigned long access, vpn = hpt_vpn(ea, vsid, ssize); if (unlikely(old_pte & H_PAGE_HASHPTE)) { + unsigned long gslot; + /* * There MIGHT be an HPTE for this pte */ - hash = hpt_hash(vpn, shift, ssize); - if (old_pte & H_PAGE_F_SECOND) - hash = ~hash; - slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; - slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; - - if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, MMU_PAGE_64K, + gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0); + if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, MMU_PAGE_64K, MMU_PAGE_64K, ssize, flags) == -1) old_pte &= ~_PAGE_HPTEFLAGS; @@ -326,9 +325,9 @@ repeat: MMU_PAGE_64K, MMU_PAGE_64K, old_pte); return -1; } + new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE; - new_pte |= (slot << H_PAGE_F_GIX_SHIFT) & - (H_PAGE_F_SECOND | H_PAGE_F_GIX); + new_pte |= pte_set_hidx(ptep, rpte, 0, slot); } *ptep = __pte(new_pte & ~H_PAGE_BUSY); return 0; diff --git a/arch/powerpc/mm/hugetlbpage-hash64.c b/arch/powerpc/mm/hugetlbpage-hash64.c index 0c2a91df3210..12511f5a015f 100644 --- a/arch/powerpc/mm/hugetlbpage-hash64.c +++ b/arch/powerpc/mm/hugetlbpage-hash64.c @@ -23,6 +23,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, unsigned long flags, int ssize, unsigned int shift, unsigned int mmu_psize) { + real_pte_t rpte; unsigned long vpn; unsigned long old_pte, new_pte; unsigned long rflags, pa, sz; @@ -62,6 +63,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, } while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte))); rflags = htab_convert_pte_flags(new_pte); + rpte = __real_pte(__pte(old_pte), ptep); sz = ((1UL) << shift); if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) @@ -72,15 +74,10 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, /* Check if pte already has an hpte (case 2) */ if (unlikely(old_pte & H_PAGE_HASHPTE)) { /* There MIGHT be an HPTE for this pte */ - unsigned long hash, slot; + unsigned long gslot; - hash = hpt_hash(vpn, shift, ssize); - if (old_pte & H_PAGE_F_SECOND) - hash = ~hash; - slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; - slot += (old_pte & H_PAGE_F_GIX) >> H_PAGE_F_GIX_SHIFT; - - if (mmu_hash_ops.hpte_updatepp(slot, rflags, vpn, mmu_psize, + gslot = pte_get_hash_gslot(vpn, shift, ssize, rpte, 0); + if (mmu_hash_ops.hpte_updatepp(gslot, rflags, vpn, mmu_psize, mmu_psize, ssize, flags) == -1) old_pte &= ~_PAGE_HPTEFLAGS; } @@ -107,8 +104,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, return -1; } - new_pte |= (slot << H_PAGE_F_GIX_SHIFT) & - (H_PAGE_F_SECOND | H_PAGE_F_GIX); + new_pte |= pte_set_hidx(ptep, rpte, 0, slot); } /* -- cgit v1.2.3 From 7b84947cadf18f9a7763f281509db1f24073b4af Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Mon, 6 Nov 2017 00:50:49 -0800 Subject: powerpc: shifted-by-one hidx value 0xf is considered invalid hidx value. It indicates absence of a backing HPTE. A PTE is initialized to 0xf either a) when it is new it is newly allocated to hold 4k-backing-HPTE or b) Any time it gets demoted to a 4k-backing-HPTE This patch shifts the representation by one-modulo-0xf; i.e hidx 0 is represented as 1, 1 as 2,... , and 0xf as 0. This convention lets us initialize the secondary-part of the PTE to all zeroes. PTEs are anyway zero'd when allocated. We do not have to zero them again; thus saving on the initialization. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-64k.h | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index 73ed988e6da5..b78d94e5249b 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -63,13 +63,21 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep) return rpte; } +/* + * shift the hidx representation by one-modulo-0xf; i.e hidx 0 is respresented + * as 1, 1 as 2,... , and 0xf as 0. This convention lets us represent a + * invalid hidx 0xf with a 0x0 bit value. PTEs are anyway zero'd when + * allocated. We dont have to zero them gain; thus save on the initialization. + */ +#define HIDX_UNSHIFT_BY_ONE(x) ((x + 0xfUL) & 0xfUL) /* shift backward by one */ +#define HIDX_SHIFT_BY_ONE(x) ((x + 0x1UL) & 0xfUL) /* shift forward by one */ #define HIDX_BITS(x, index) (x << (index << 2)) #define BITS_TO_HIDX(x, index) ((x >> (index << 2)) & 0xfUL) -#define INVALID_RPTE_HIDX ~(0x0UL) +#define INVALID_RPTE_HIDX 0x0UL static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index) { - return BITS_TO_HIDX(rpte.hidx, index); + return HIDX_UNSHIFT_BY_ONE(BITS_TO_HIDX(rpte.hidx, index)); } /* @@ -82,7 +90,7 @@ static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte, unsigned long *hidxp = (unsigned long *)(ptep + PTRS_PER_PTE); rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index); - *hidxp = rpte.hidx | HIDX_BITS(hidx, subpg_index); + *hidxp = rpte.hidx | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index); /* * Anyone reading PTE must ensure hidx bits are read after reading the -- cgit v1.2.3 From 273b49368966559dd155323c64ec3421e8f4049d Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Mon, 6 Nov 2017 00:50:50 -0800 Subject: powerpc: Swizzle around 4K PTE bits to free up bit 5 and bit 6 We need PTE bits 3 ,4, 5, 6 and 57 to support protection-keys, because these are the bits we want to consolidate on across all configuration to support protection keys. Bit 3,4,5 and 6 are currently used on 4K-pte kernels. But bit 9 and 10 are available. Hence we use the two available bits and free up bit 5 and 6. We will still not be able to free up bit 3 and 4. In the absence of any other free bits, we will have to stay satisfied with what we have :-(. This means we will not be able to support 32 protection keys, but only 8. The bit numbers are big-endian as defined in the ISA3.0 This patch does the following change to 4K PTE. H_PAGE_F_SECOND (S) which occupied bit 4 moves to bit 7. H_PAGE_F_GIX (G,I,X) which occupied bit 5, 6 and 7 also moves to bit 8,9, 10 respectively. H_PAGE_HASHPTE (H) which occupied bit 8 moves to bit 4. Before the patch, the 4k PTE format was as follows 0 1 2 3 4 5 6 7 8 9 10....................57.....63 : : : : : : : : : : : : : v v v v v v v v v v v v v ,-,-,-,-,--,--,--,--,-,-,-,-,-,------------------,-,-,-, |x|x|x|B|S |G |I |X |H| | |x|x|................| |x|x|x| '_'_'_'_'__'__'__'__'_'_'_'_'_'________________'_'_'_'_' After the patch, the 4k PTE format is as follows 0 1 2 3 4 5 6 7 8 9 10....................57.....63 : : : : : : : : : : : : : v v v v v v v v v v v v v ,-,-,-,-,--,--,--,--,-,-,-,-,-,------------------,-,-,-, |x|x|x|B|H | | |S |G|I|X|x|x|................| |.|.|.| '_'_'_'_'__'__'__'__'_'_'_'_'_'________________'_'_'_'_' The patch has no code changes; just swizzles around bits. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 7 ++++--- arch/powerpc/include/asm/book3s/64/hash-64k.h | 1 + arch/powerpc/include/asm/book3s/64/hash.h | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index ed0d1cd98907..8a91f2475ea3 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -17,10 +17,11 @@ #define H_PUD_TABLE_SIZE (sizeof(pud_t) << H_PUD_INDEX_SIZE) #define H_PGD_TABLE_SIZE (sizeof(pgd_t) << H_PGD_INDEX_SIZE) -#define H_PAGE_F_GIX_SHIFT 56 -#define H_PAGE_F_SECOND _RPAGE_RSV2 /* HPTE is in 2ndary HPTEG */ -#define H_PAGE_F_GIX (_RPAGE_RSV3 | _RPAGE_RSV4 | _RPAGE_RPN44) +#define H_PAGE_F_GIX_SHIFT 53 +#define H_PAGE_F_SECOND _RPAGE_RPN44 /* HPTE is in 2ndary HPTEG */ +#define H_PAGE_F_GIX (_RPAGE_RPN43 | _RPAGE_RPN42 | _RPAGE_RPN41) #define H_PAGE_BUSY _RPAGE_RSV1 /* software: PTE & hash are busy */ +#define H_PAGE_HASHPTE _RPAGE_RSV2 /* software: PTE & hash are busy */ /* PTE flags to conserve for HPTE identification */ #define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | \ diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index b78d94e5249b..62e101b43cf6 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -14,6 +14,7 @@ #define H_PAGE_COMBO _RPAGE_RPN0 /* this is a combo 4k page */ #define H_PAGE_4K_PFN _RPAGE_RPN1 /* PFN is for a single 4k page */ #define H_PAGE_BUSY _RPAGE_RPN44 /* software: PTE & hash are busy */ +#define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */ /* * We need to differentiate between explicit huge page and THP huge diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index e72474f13936..0920eff731b3 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -9,7 +9,6 @@ * */ #define H_PTE_NONE_MASK _PAGE_HPTEFLAGS -#define H_PAGE_HASHPTE _RPAGE_RPN43 /* PTE has associated HPTE */ #ifdef CONFIG_PPC_64K_PAGES #include -- cgit v1.2.3 From f72a85e347810c76eb246e68751233aad3b84ac8 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Mon, 4 Dec 2017 07:49:11 +0530 Subject: powerpc/mm/book3s/64: Add proper pte access check helper pte_access_premitted get called in get_user_pages_fast path. If we have marked the pte PROT_NONE, we should not allow a read access on the address. With the current implementation we are not checking the READ and only check for WRITE. This is needed on archs like ppc64 that implement PROT_NONE using RWX access instead of _PAGE_PRESENT. Also add pte_user check just to make sure we are not accessing kernel mapping. Signed-off-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/pgtable.h | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 44697817ccc6..23b03449e2f1 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -546,6 +546,30 @@ static inline int pte_present(pte_t pte) { return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT)); } + +#define pte_access_permitted pte_access_permitted +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + unsigned long pteval = pte_val(pte); + /* Also check for pte_user */ + unsigned long clear_pte_bits = _PAGE_PRIVILEGED; + /* + * _PAGE_READ is needed for any access and will be + * cleared for PROT_NONE + */ + unsigned long need_pte_bits = _PAGE_PRESENT | _PAGE_READ; + + if (write) + need_pte_bits |= _PAGE_WRITE; + + if ((pteval & need_pte_bits) != need_pte_bits) + return false; + + if ((pteval & clear_pte_bits) == clear_pte_bits) + return false; + return true; +} + /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. @@ -850,6 +874,11 @@ static inline int pud_bad(pud_t pud) return hash__pud_bad(pud); } +#define pud_access_permitted pud_access_permitted +static inline bool pud_access_permitted(pud_t pud, bool write) +{ + return pte_access_permitted(pud_pte(pud), write); +} #define pgd_write(pgd) pte_write(pgd_pte(pgd)) static inline void pgd_set(pgd_t *pgdp, unsigned long val) @@ -889,6 +918,12 @@ static inline int pgd_bad(pgd_t pgd) return hash__pgd_bad(pgd); } +#define pgd_access_permitted pgd_access_permitted +static inline bool pgd_access_permitted(pgd_t pgd, bool write) +{ + return pte_access_permitted(pgd_pte(pgd), write); +} + extern struct page *pgd_page(pgd_t pgd); /* Pointers in the page table tree are physical addresses */ @@ -1009,6 +1044,12 @@ static inline int pmd_protnone(pmd_t pmd) #define __pmd_write(pmd) __pte_write(pmd_pte(pmd)) #define pmd_savedwrite(pmd) pte_savedwrite(pmd_pte(pmd)) +#define pmd_access_permitted pmd_access_permitted +static inline bool pmd_access_permitted(pmd_t pmd, bool write) +{ + return pte_access_permitted(pmd_pte(pmd), write); +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot); extern pmd_t mk_pmd(struct page *page, pgprot_t pgprot); -- cgit v1.2.3 From 5769beaf180a892c3fea92937954727fb912bded Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Mon, 4 Dec 2017 07:49:12 +0530 Subject: powerpc/mm: Add proper pte access check helper for other platforms pte_access_premitted get called in get_user_pages_fast path. If we have marked the pte PROT_NONE, we should not allow a read access on the address. With the current implementation we are not checking the READ and only check for WRITE. This is needed on archs like ppc64 that implement PROT_NONE using _PAGE_USER access instead of _PAGE_PRESENT. Also add pte_user check just to make sure we are not accessing kernel mapping. Even though there is code duplication, keeping the low level pte accessors different for different platforms helps in code readability. Signed-off-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/32/pgtable.h | 23 +++++++++++++++++++++++ arch/powerpc/include/asm/nohash/pgtable.h | 23 +++++++++++++++++++++++ 2 files changed, 46 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 016579ef16d3..30a155c0a6b0 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -311,6 +311,29 @@ static inline int pte_present(pte_t pte) return pte_val(pte) & _PAGE_PRESENT; } +/* + * We only find page table entry in the last level + * Hence no need for other accessors + */ +#define pte_access_permitted pte_access_permitted +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + unsigned long pteval = pte_val(pte); + /* + * A read-only access is controlled by _PAGE_USER bit. + * We have _PAGE_READ set for WRITE and EXECUTE + */ + unsigned long need_pte_bits = _PAGE_PRESENT | _PAGE_USER; + + if (write) + need_pte_bits |= _PAGE_WRITE; + + if ((pteval & need_pte_bits) != need_pte_bits) + return false; + + return true; +} + /* Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. * diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index 5c68f4a59f75..fc4376c8d444 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -45,6 +45,29 @@ static inline int pte_present(pte_t pte) return pte_val(pte) & _PAGE_PRESENT; } +/* + * We only find page table entry in the last level + * Hence no need for other accessors + */ +#define pte_access_permitted pte_access_permitted +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + unsigned long pteval = pte_val(pte); + /* + * A read-only access is controlled by _PAGE_USER bit. + * We have _PAGE_READ set for WRITE and EXECUTE + */ + unsigned long need_pte_bits = _PAGE_PRESENT | _PAGE_USER; + + if (write) + need_pte_bits |= _PAGE_WRITE; + + if ((pteval & need_pte_bits) != need_pte_bits) + return false; + + return true; +} + /* Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. * -- cgit v1.2.3 From 12c1f339cd49119e39063ae67f02d936f988c079 Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Fri, 12 Jan 2018 13:39:27 +1100 Subject: powerpc/xive: Move definition of ESB bits From xive.h to xive-regs.h since it's a HW register definition and it can be used from assembly Signed-off-by: Benjamin Herrenschmidt Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/xive-regs.h | 35 +++++++++++++++++++++++++++++++++++ arch/powerpc/include/asm/xive.h | 35 ----------------------------------- 2 files changed, 35 insertions(+), 35 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/xive-regs.h b/arch/powerpc/include/asm/xive-regs.h index 1d3f2be5ae39..fa4288822b68 100644 --- a/arch/powerpc/include/asm/xive-regs.h +++ b/arch/powerpc/include/asm/xive-regs.h @@ -9,6 +9,41 @@ #ifndef _ASM_POWERPC_XIVE_REGS_H #define _ASM_POWERPC_XIVE_REGS_H +/* + * "magic" Event State Buffer (ESB) MMIO offsets. + * + * Each interrupt source has a 2-bit state machine called ESB + * which can be controlled by MMIO. It's made of 2 bits, P and + * Q. P indicates that an interrupt is pending (has been sent + * to a queue and is waiting for an EOI). Q indicates that the + * interrupt has been triggered while pending. + * + * This acts as a coalescing mechanism in order to guarantee + * that a given interrupt only occurs at most once in a queue. + * + * When doing an EOI, the Q bit will indicate if the interrupt + * needs to be re-triggered. + * + * The following offsets into the ESB MMIO allow to read or + * manipulate the PQ bits. They must be used with an 8-bytes + * load instruction. They all return the previous state of the + * interrupt (atomically). + * + * Additionally, some ESB pages support doing an EOI via a + * store at 0 and some ESBs support doing a trigger via a + * separate trigger page. + */ +#define XIVE_ESB_STORE_EOI 0x400 /* Store */ +#define XIVE_ESB_LOAD_EOI 0x000 /* Load */ +#define XIVE_ESB_GET 0x800 /* Load */ +#define XIVE_ESB_SET_PQ_00 0xc00 /* Load */ +#define XIVE_ESB_SET_PQ_01 0xd00 /* Load */ +#define XIVE_ESB_SET_PQ_10 0xe00 /* Load */ +#define XIVE_ESB_SET_PQ_11 0xf00 /* Load */ + +#define XIVE_ESB_VAL_P 0x2 +#define XIVE_ESB_VAL_Q 0x1 + /* * Thread Management (aka "TM") registers */ diff --git a/arch/powerpc/include/asm/xive.h b/arch/powerpc/include/asm/xive.h index 371fbebf1ec9..0e77005cf021 100644 --- a/arch/powerpc/include/asm/xive.h +++ b/arch/powerpc/include/asm/xive.h @@ -72,41 +72,6 @@ struct xive_q { atomic_t pending_count; }; -/* - * "magic" Event State Buffer (ESB) MMIO offsets. - * - * Each interrupt source has a 2-bit state machine called ESB - * which can be controlled by MMIO. It's made of 2 bits, P and - * Q. P indicates that an interrupt is pending (has been sent - * to a queue and is waiting for an EOI). Q indicates that the - * interrupt has been triggered while pending. - * - * This acts as a coalescing mechanism in order to guarantee - * that a given interrupt only occurs at most once in a queue. - * - * When doing an EOI, the Q bit will indicate if the interrupt - * needs to be re-triggered. - * - * The following offsets into the ESB MMIO allow to read or - * manipulate the PQ bits. They must be used with an 8-bytes - * load instruction. They all return the previous state of the - * interrupt (atomically). - * - * Additionally, some ESB pages support doing an EOI via a - * store at 0 and some ESBs support doing a trigger via a - * separate trigger page. - */ -#define XIVE_ESB_STORE_EOI 0x400 /* Store */ -#define XIVE_ESB_LOAD_EOI 0x000 /* Load */ -#define XIVE_ESB_GET 0x800 /* Load */ -#define XIVE_ESB_SET_PQ_00 0xc00 /* Load */ -#define XIVE_ESB_SET_PQ_01 0xd00 /* Load */ -#define XIVE_ESB_SET_PQ_10 0xe00 /* Load */ -#define XIVE_ESB_SET_PQ_11 0xf00 /* Load */ - -#define XIVE_ESB_VAL_P 0x2 -#define XIVE_ESB_VAL_Q 0x1 - /* Global enable flags for the XIVE support */ extern bool __xive_enabled; -- cgit v1.2.3 From 7f1c410da59090f9bb2300efebbc3b717594d64c Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Fri, 12 Jan 2018 13:39:28 +1100 Subject: powerpc/xive: Add interrupt flag to disable automatic EOI This will be used by KVM in order to keep escalation interrupts in the non-EOI (masked) state after they fire. They will be re-enabled directly in HW by KVM when needed. Signed-off-by: Benjamin Herrenschmidt Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/xive.h | 3 +++ arch/powerpc/sysdev/xive/common.c | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/xive.h b/arch/powerpc/include/asm/xive.h index 0e77005cf021..b619a5585cd6 100644 --- a/arch/powerpc/include/asm/xive.h +++ b/arch/powerpc/include/asm/xive.h @@ -58,6 +58,9 @@ struct xive_irq_data { #define XIVE_IRQ_FLAG_EOI_FW 0x10 #define XIVE_IRQ_FLAG_H_INT_ESB 0x20 +/* Special flag set by KVM for excalation interrupts */ +#define XIVE_IRQ_NO_EOI 0x80 + #define XIVE_INVALID_CHIP_ID -1 /* A queue tracking structure in a CPU */ diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index a3b8d7d1316e..2547b6021e6a 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -367,7 +367,8 @@ static void xive_irq_eoi(struct irq_data *d) * EOI the source if it hasn't been disabled and hasn't * been passed-through to a KVM guest */ - if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d)) + if (!irqd_irq_disabled(d) && !irqd_is_forwarded_to_vcpu(d) && + !(xd->flags & XIVE_IRQ_NO_EOI)) xive_do_source_eoi(irqd_to_hwirq(d), xd); /* -- cgit v1.2.3 From 6c6ea53725b357fa3deac96d8d2d4ee785b67c6b Mon Sep 17 00:00:00 2001 From: Nathan Fontenot Date: Fri, 1 Dec 2017 10:47:08 -0600 Subject: powerpc/mm: Separate ibm, dynamic-memory data from DT format We currently have code to parse the dynamic reconfiguration LMB information from the ibm,dynamic-meory device tree property in multiple locations; numa.c, prom.c, and pseries/hotplug-memory.c. In anticipation of adding support for a version 2 of the ibm,dynamic-memory property this patch aims to separate the device tree information from the device tree format. Doing this requires a two step process to avoid a possibly very large bootmem allocation early in boot. During initial boot, new routines are provided to walk the device tree property and make a call-back for each LMB. The second step (introduced in later patches) will allocate an array of LMB information that can be used directly without needing to know the DT format. This approach provides the benefit of consolidating the device tree property parsing to a single location and (eventually) providing a common data structure for retrieving LMB information. This patch introduces a routine to walk the ibm,dynamic-memory property in the flattened device tree and updates the prom.c code to use this to initialize memory. Signed-off-by: Nathan Fontenot Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/drmem.h | 48 ++++++++++++++++ arch/powerpc/kernel/prom.c | 115 +++++++++++++++++---------------------- arch/powerpc/mm/Makefile | 2 +- arch/powerpc/mm/drmem.c | 76 ++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 67 deletions(-) create mode 100644 arch/powerpc/include/asm/drmem.h create mode 100644 arch/powerpc/mm/drmem.c (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h new file mode 100644 index 000000000000..8b1fe465ec9d --- /dev/null +++ b/arch/powerpc/include/asm/drmem.h @@ -0,0 +1,48 @@ +/* + * drmem.h: Power specific logical memory block representation + * + * Copyright 2017 IBM Corporation + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#ifndef _ASM_POWERPC_LMB_H +#define _ASM_POWERPC_LMB_H + +struct drmem_lmb { + u64 base_addr; + u32 drc_index; + u32 aa_index; + u32 flags; +}; + +struct drmem_lmb_info { + struct drmem_lmb *lmbs; + int n_lmbs; + u32 lmb_size; +}; + +extern struct drmem_lmb_info *drmem_info; + +#define for_each_drmem_lmb_in_range(lmb, start, end) \ + for ((lmb) = (start); (lmb) <= (end); (lmb)++) + +#define for_each_drmem_lmb(lmb) \ + for_each_drmem_lmb_in_range((lmb), \ + &drmem_info->lmbs[0], \ + &drmem_info->lmbs[drmem_info->n_lmbs - 1]) + +static inline u32 drmem_lmb_size(void) +{ + return drmem_info->lmb_size; +} + +#ifdef CONFIG_PPC_PSERIES +void __init walk_drmem_lmbs_early(unsigned long node, + void (*func)(struct drmem_lmb *, const __be32 **)); +#endif + +#endif /* _ASM_POWERPC_LMB_H */ diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index b15bae265c90..4dffef947b8a 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -59,6 +59,7 @@ #include #include #include +#include #include @@ -455,92 +456,74 @@ static int __init early_init_dt_scan_chosen_ppc(unsigned long node, #ifdef CONFIG_PPC_PSERIES /* - * Interpret the ibm,dynamic-memory property in the - * /ibm,dynamic-reconfiguration-memory node. + * Interpret the ibm dynamic reconfiguration memory LMBs. * This contains a list of memory blocks along with NUMA affinity * information. */ -static int __init early_init_dt_scan_drconf_memory(unsigned long node) +static void __init early_init_drmem_lmb(struct drmem_lmb *lmb, + const __be32 **usm) { - const __be32 *dm, *ls, *usm; - int l; - unsigned long n, flags; - u64 base, size, memblock_size; - unsigned int is_kexec_kdump = 0, rngs; - - ls = of_get_flat_dt_prop(node, "ibm,lmb-size", &l); - if (ls == NULL || l < dt_root_size_cells * sizeof(__be32)) - return 0; - memblock_size = dt_mem_next_cell(dt_root_size_cells, &ls); + u64 base, size; + int is_kexec_kdump = 0, rngs; - dm = of_get_flat_dt_prop(node, "ibm,dynamic-memory", &l); - if (dm == NULL || l < sizeof(__be32)) - return 0; + base = lmb->base_addr; + size = drmem_lmb_size(); + rngs = 1; - n = of_read_number(dm++, 1); /* number of entries */ - if (l < (n * (dt_root_addr_cells + 4) + 1) * sizeof(__be32)) - return 0; + /* + * Skip this block if the reserved bit is set in flags + * or if the block is not assigned to this partition. + */ + if ((lmb->flags & DRCONF_MEM_RESERVED) || + !(lmb->flags & DRCONF_MEM_ASSIGNED)) + return; - /* check if this is a kexec/kdump kernel. */ - usm = of_get_flat_dt_prop(node, "linux,drconf-usable-memory", - &l); - if (usm != NULL) + if (*usm) is_kexec_kdump = 1; - for (; n != 0; --n) { - base = dt_mem_next_cell(dt_root_addr_cells, &dm); - flags = of_read_number(&dm[3], 1); - /* skip DRC index, pad, assoc. list index, flags */ - dm += 4; - /* skip this block if the reserved bit is set in flags - or if the block is not assigned to this partition */ - if ((flags & DRCONF_MEM_RESERVED) || - !(flags & DRCONF_MEM_ASSIGNED)) - continue; - size = memblock_size; - rngs = 1; + if (is_kexec_kdump) { + /* + * For each memblock in ibm,dynamic-memory, a + * corresponding entry in linux,drconf-usable-memory + * property contains a counter 'p' followed by 'p' + * (base, size) duple. Now read the counter from + * linux,drconf-usable-memory property + */ + rngs = dt_mem_next_cell(dt_root_size_cells, usm); + if (!rngs) /* there are no (base, size) duple */ + return; + } + + do { if (is_kexec_kdump) { - /* - * For each memblock in ibm,dynamic-memory, a corresponding - * entry in linux,drconf-usable-memory property contains - * a counter 'p' followed by 'p' (base, size) duple. - * Now read the counter from - * linux,drconf-usable-memory property - */ - rngs = dt_mem_next_cell(dt_root_size_cells, &usm); - if (!rngs) /* there are no (base, size) duple */ + base = dt_mem_next_cell(dt_root_addr_cells, usm); + size = dt_mem_next_cell(dt_root_size_cells, usm); + } + + if (iommu_is_off) { + if (base >= 0x80000000ul) continue; + if ((base + size) > 0x80000000ul) + size = 0x80000000ul - base; } - do { - if (is_kexec_kdump) { - base = dt_mem_next_cell(dt_root_addr_cells, - &usm); - size = dt_mem_next_cell(dt_root_size_cells, - &usm); - } - if (iommu_is_off) { - if (base >= 0x80000000ul) - continue; - if ((base + size) > 0x80000000ul) - size = 0x80000000ul - base; - } - memblock_add(base, size); - } while (--rngs); - } - memblock_dump_all(); - return 0; + + DBG("Adding: %llx -> %llx\n", base, size); + memblock_add(base, size); + } while (--rngs); } -#else -#define early_init_dt_scan_drconf_memory(node) 0 #endif /* CONFIG_PPC_PSERIES */ static int __init early_init_dt_scan_memory_ppc(unsigned long node, const char *uname, int depth, void *data) { +#ifdef CONFIG_PPC_PSERIES if (depth == 1 && - strcmp(uname, "ibm,dynamic-reconfiguration-memory") == 0) - return early_init_dt_scan_drconf_memory(node); + strcmp(uname, "ibm,dynamic-reconfiguration-memory") == 0) { + walk_drmem_lmbs_early(node, early_init_drmem_lmb); + return 0; + } +#endif return early_init_dt_scan_memory(node, uname, depth, data); } diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index 76a6b057d454..8d271bfe2d94 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile @@ -9,7 +9,7 @@ ccflags-$(CONFIG_PPC64) := $(NO_MINIMAL_TOC) obj-y := fault.o mem.o pgtable.o mmap.o \ init_$(BITS).o pgtable_$(BITS).o \ - init-common.o mmu_context.o + init-common.o mmu_context.o drmem.o obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \ tlb_nohash_low.o obj-$(CONFIG_PPC_BOOK3E) += tlb_low_$(BITS)e.o diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c new file mode 100644 index 000000000000..f8ee0f355405 --- /dev/null +++ b/arch/powerpc/mm/drmem.c @@ -0,0 +1,76 @@ +/* + * Dynamic reconfiguration memory support + * + * Copyright 2017 IBM Corporation + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#define pr_fmt(fmt) "drmem: " fmt + +#include +#include +#include +#include +#include +#include + +static struct drmem_lmb_info __drmem_info; +struct drmem_lmb_info *drmem_info = &__drmem_info; + +#ifdef CONFIG_PPC_PSERIES +static void __init read_drconf_v1_cell(struct drmem_lmb *lmb, + const __be32 **prop) +{ + const __be32 *p = *prop; + + lmb->base_addr = dt_mem_next_cell(dt_root_addr_cells, &p); + lmb->drc_index = of_read_number(p++, 1); + + p++; /* skip reserved field */ + + lmb->aa_index = of_read_number(p++, 1); + lmb->flags = of_read_number(p++, 1); + + *prop = p; +} + +static void __init __walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm, + void (*func)(struct drmem_lmb *, const __be32 **)) +{ + struct drmem_lmb lmb; + u32 i, n_lmbs; + + n_lmbs = of_read_number(prop++, 1); + + for (i = 0; i < n_lmbs; i++) { + read_drconf_v1_cell(&lmb, &prop); + func(&lmb, &usm); + } +} + +void __init walk_drmem_lmbs_early(unsigned long node, + void (*func)(struct drmem_lmb *, const __be32 **)) +{ + const __be32 *prop, *usm; + int len; + + prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len); + if (!prop || len < dt_root_size_cells * sizeof(__be32)) + return; + + drmem_info->lmb_size = dt_mem_next_cell(dt_root_size_cells, &prop); + + usm = of_get_flat_dt_prop(node, "linux,drconf-usable-memory", &len); + + prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory", &len); + if (prop) + __walk_drmem_v1_lmbs(prop, usm, func); + + memblock_dump_all(); +} + +#endif -- cgit v1.2.3 From 514a9cb3316a08d63063a40a70f11b4318d3c06c Mon Sep 17 00:00:00 2001 From: Nathan Fontenot Date: Fri, 1 Dec 2017 10:47:21 -0600 Subject: powerpc/numa: Update numa code use walk_drmem_lmbs Update code in powerpc/numa.c to use the walk_drmem_lmbs() routine instead of parsing the device tree directly. This is in anticipation of introducing a new ibm,dynamic-memory-v2 property with a different format. This will allow the numa code to use a single initialization routine per-LMB irregardless of the device tree format. Additionally, to support additional routines in numa.c that need to look up LMB information, an late_init routine is added to drmem.c to allocate the array of LMB information. This LMB array will provide per-LMB information to separate the LMB data from the device tree format. Signed-off-by: Nathan Fontenot Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/drmem.h | 4 + arch/powerpc/mm/drmem.c | 100 +++++++++++++++++- arch/powerpc/mm/numa.c | 223 +++++++++------------------------------ 3 files changed, 155 insertions(+), 172 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h index 8b1fe465ec9d..9dbfd38fa385 100644 --- a/arch/powerpc/include/asm/drmem.h +++ b/arch/powerpc/include/asm/drmem.h @@ -40,6 +40,10 @@ static inline u32 drmem_lmb_size(void) return drmem_info->lmb_size; } +u64 drmem_lmb_memory_max(void); +void __init walk_drmem_lmbs(struct device_node *dn, + void (*func)(struct drmem_lmb *, const __be32 **)); + #ifdef CONFIG_PPC_PSERIES void __init walk_drmem_lmbs_early(unsigned long node, void (*func)(struct drmem_lmb *, const __be32 **)); diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c index f8ee0f355405..5888ac3ca8a9 100644 --- a/arch/powerpc/mm/drmem.c +++ b/arch/powerpc/mm/drmem.c @@ -21,7 +21,14 @@ static struct drmem_lmb_info __drmem_info; struct drmem_lmb_info *drmem_info = &__drmem_info; -#ifdef CONFIG_PPC_PSERIES +u64 drmem_lmb_memory_max(void) +{ + struct drmem_lmb *last_lmb; + + last_lmb = &drmem_info->lmbs[drmem_info->n_lmbs - 1]; + return last_lmb->base_addr + drmem_lmb_size(); +} + static void __init read_drconf_v1_cell(struct drmem_lmb *lmb, const __be32 **prop) { @@ -52,6 +59,7 @@ static void __init __walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm, } } +#ifdef CONFIG_PPC_PSERIES void __init walk_drmem_lmbs_early(unsigned long node, void (*func)(struct drmem_lmb *, const __be32 **)) { @@ -74,3 +82,93 @@ void __init walk_drmem_lmbs_early(unsigned long node, } #endif + +static int __init init_drmem_lmb_size(struct device_node *dn) +{ + const __be32 *prop; + int len; + + if (drmem_info->lmb_size) + return 0; + + prop = of_get_property(dn, "ibm,lmb-size", &len); + if (!prop || len < dt_root_size_cells * sizeof(__be32)) { + pr_info("Could not determine LMB size\n"); + return -1; + } + + drmem_info->lmb_size = dt_mem_next_cell(dt_root_size_cells, &prop); + return 0; +} + +/* + * Returns the property linux,drconf-usable-memory if + * it exists (the property exists only in kexec/kdump kernels, + * added by kexec-tools) + */ +static const __be32 *of_get_usable_memory(struct device_node *dn) +{ + const __be32 *prop; + u32 len; + + prop = of_get_property(dn, "linux,drconf-usable-memory", &len); + if (!prop || len < sizeof(unsigned int)) + return NULL; + + return prop; +} + +void __init walk_drmem_lmbs(struct device_node *dn, + void (*func)(struct drmem_lmb *, const __be32 **)) +{ + const __be32 *prop, *usm; + + if (init_drmem_lmb_size(dn)) + return; + + usm = of_get_usable_memory(dn); + + prop = of_get_property(dn, "ibm,dynamic-memory", NULL); + if (prop) + __walk_drmem_v1_lmbs(prop, usm, func); +} + +static void __init init_drmem_v1_lmbs(const __be32 *prop) +{ + struct drmem_lmb *lmb; + + drmem_info->n_lmbs = of_read_number(prop++, 1); + + drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb), + GFP_KERNEL); + if (!drmem_info->lmbs) + return; + + for_each_drmem_lmb(lmb) + read_drconf_v1_cell(lmb, &prop); +} + +static int __init drmem_init(void) +{ + struct device_node *dn; + const __be32 *prop; + + dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); + if (!dn) { + pr_info("No dynamic reconfiguration memory found\n"); + return 0; + } + + if (init_drmem_lmb_size(dn)) { + of_node_put(dn); + return 0; + } + + prop = of_get_property(dn, "ibm,dynamic-memory", NULL); + if (prop) + init_drmem_v1_lmbs(prop); + + of_node_put(dn); + return 0; +} +late_initcall(drmem_init); diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index d25278adaead..268c7a2d9a5b 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -40,6 +40,7 @@ #include #include #include +#include static int numa_enabled = 1; @@ -179,29 +180,6 @@ static const __be32 *of_get_associativity(struct device_node *dev) return of_get_property(dev, "ibm,associativity", NULL); } -/* - * Returns the property linux,drconf-usable-memory if - * it exists (the property exists only in kexec/kdump kernels, - * added by kexec-tools) - */ -static const __be32 *of_get_usable_memory(void) -{ - struct device_node *memory; - const __be32 *prop; - u32 len; - - memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); - if (!memory) - return NULL; - - prop = of_get_property(memory, "linux,drconf-usable-memory", &len); - of_node_put(memory); - - if (!prop || len < sizeof(unsigned int)) - return NULL; - return prop; -} - int __node_distance(int a, int b) { int i; @@ -395,69 +373,6 @@ static unsigned long read_n_cells(int n, const __be32 **buf) return result; } -/* - * Read the next memblock list entry from the ibm,dynamic-memory property - * and return the information in the provided of_drconf_cell structure. - */ -static void read_drconf_cell(struct of_drconf_cell *drmem, const __be32 **cellp) -{ - const __be32 *cp; - - drmem->base_addr = read_n_cells(n_mem_addr_cells, cellp); - - cp = *cellp; - drmem->drc_index = of_read_number(cp, 1); - drmem->reserved = of_read_number(&cp[1], 1); - drmem->aa_index = of_read_number(&cp[2], 1); - drmem->flags = of_read_number(&cp[3], 1); - - *cellp = cp + 4; -} - -/* - * Retrieve and validate the ibm,dynamic-memory property of the device tree. - * - * The layout of the ibm,dynamic-memory property is a number N of memblock - * list entries followed by N memblock list entries. Each memblock list entry - * contains information as laid out in the of_drconf_cell struct above. - */ -static int of_get_drconf_memory(struct device_node *memory, const __be32 **dm) -{ - const __be32 *prop; - u32 len, entries; - - prop = of_get_property(memory, "ibm,dynamic-memory", &len); - if (!prop || len < sizeof(unsigned int)) - return 0; - - entries = of_read_number(prop++, 1); - - /* Now that we know the number of entries, revalidate the size - * of the property read in to ensure we have everything - */ - if (len < (entries * (n_mem_addr_cells + 4) + 1) * sizeof(unsigned int)) - return 0; - - *dm = prop; - return entries; -} - -/* - * Retrieve and validate the ibm,lmb-size property for drconf memory - * from the device tree. - */ -static u64 of_get_lmb_size(struct device_node *memory) -{ - const __be32 *prop; - u32 len; - - prop = of_get_property(memory, "ibm,lmb-size", &len); - if (!prop || len < sizeof(unsigned int)) - return 0; - - return read_n_cells(n_mem_size_cells, &prop); -} - struct assoc_arrays { u32 n_arrays; u32 array_sz; @@ -509,7 +424,7 @@ static int of_get_assoc_arrays(struct assoc_arrays *aa) * This is like of_node_to_nid_single() for memory represented in the * ibm,dynamic-reconfiguration-memory node. */ -static int of_drconf_to_nid_single(struct of_drconf_cell *drmem) +static int of_drconf_to_nid_single(struct drmem_lmb *lmb) { struct assoc_arrays aa = { .arrays = NULL }; int default_nid = 0; @@ -521,16 +436,16 @@ static int of_drconf_to_nid_single(struct of_drconf_cell *drmem) return default_nid; if (min_common_depth > 0 && min_common_depth <= aa.array_sz && - !(drmem->flags & DRCONF_MEM_AI_INVALID) && - drmem->aa_index < aa.n_arrays) { - index = drmem->aa_index * aa.array_sz + min_common_depth - 1; + !(lmb->flags & DRCONF_MEM_AI_INVALID) && + lmb->aa_index < aa.n_arrays) { + index = lmb->aa_index * aa.array_sz + min_common_depth - 1; nid = of_read_number(&aa.arrays[index], 1); if (nid == 0xffff || nid >= MAX_NUMNODES) nid = default_nid; if (nid > 0) { - index = drmem->aa_index * aa.array_sz; + index = lmb->aa_index * aa.array_sz; initialize_distance_lookup_table(nid, &aa.arrays[index]); } @@ -665,62 +580,48 @@ static inline int __init read_usm_ranges(const __be32 **usm) * Extract NUMA information from the ibm,dynamic-reconfiguration-memory * node. This assumes n_mem_{addr,size}_cells have been set. */ -static void __init parse_drconf_memory(struct device_node *memory) +static void __init numa_setup_drmem_lmb(struct drmem_lmb *lmb, + const __be32 **usm) { - const __be32 *uninitialized_var(dm), *usm; - unsigned int n, ranges, is_kexec_kdump = 0; - unsigned long lmb_size, base, size, sz; + unsigned int ranges, is_kexec_kdump = 0; + unsigned long base, size, sz; int nid; - n = of_get_drconf_memory(memory, &dm); - if (!n) - return; - - lmb_size = of_get_lmb_size(memory); - if (!lmb_size) + /* + * Skip this block if the reserved bit is set in flags (0x80) + * or if the block is not assigned to this partition (0x8) + */ + if ((lmb->flags & DRCONF_MEM_RESERVED) + || !(lmb->flags & DRCONF_MEM_ASSIGNED)) return; - /* check if this is a kexec/kdump kernel */ - usm = of_get_usable_memory(); - if (usm != NULL) + if (*usm) is_kexec_kdump = 1; - for (; n != 0; --n) { - struct of_drconf_cell drmem; - - read_drconf_cell(&drmem, &dm); + base = lmb->base_addr; + size = drmem_lmb_size(); + ranges = 1; - /* skip this block if the reserved bit is set in flags (0x80) - or if the block is not assigned to this partition (0x8) */ - if ((drmem.flags & DRCONF_MEM_RESERVED) - || !(drmem.flags & DRCONF_MEM_ASSIGNED)) - continue; - - base = drmem.base_addr; - size = lmb_size; - ranges = 1; + if (is_kexec_kdump) { + ranges = read_usm_ranges(usm); + if (!ranges) /* there are no (base, size) duple */ + return; + } + do { if (is_kexec_kdump) { - ranges = read_usm_ranges(&usm); - if (!ranges) /* there are no (base, size) duple */ - continue; + base = read_n_cells(n_mem_addr_cells, usm); + size = read_n_cells(n_mem_size_cells, usm); } - do { - if (is_kexec_kdump) { - base = read_n_cells(n_mem_addr_cells, &usm); - size = read_n_cells(n_mem_size_cells, &usm); - } - nid = of_drconf_to_nid_single(&drmem); - fake_numa_create_new_node( - ((base + size) >> PAGE_SHIFT), - &nid); - node_set_online(nid); - sz = numa_enforce_memory_limit(base, size); - if (sz) - memblock_set_node(base, sz, - &memblock.memory, nid); - } while (--ranges); - } + + nid = of_drconf_to_nid_single(lmb); + fake_numa_create_new_node(((base + size) >> PAGE_SHIFT), + &nid); + node_set_online(nid); + sz = numa_enforce_memory_limit(base, size); + if (sz) + memblock_set_node(base, sz, &memblock.memory, nid); + } while (--ranges); } static int __init parse_numa_properties(void) @@ -815,8 +716,10 @@ new_range: * ibm,dynamic-reconfiguration-memory node. */ memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); - if (memory) - parse_drconf_memory(memory); + if (memory) { + walk_drmem_lmbs(memory, numa_setup_drmem_lmb); + of_node_put(memory); + } return 0; } @@ -994,38 +897,26 @@ early_param("topology_updates", early_topology_updates); * memory represented in the device tree by the property * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory. */ -static int hot_add_drconf_scn_to_nid(struct device_node *memory, - unsigned long scn_addr) +static int hot_add_drconf_scn_to_nid(unsigned long scn_addr) { - const __be32 *dm; - unsigned int drconf_cell_cnt; + struct drmem_lmb *lmb; unsigned long lmb_size; int nid = -1; - drconf_cell_cnt = of_get_drconf_memory(memory, &dm); - if (!drconf_cell_cnt) - return -1; - - lmb_size = of_get_lmb_size(memory); - if (!lmb_size) - return -1; - - for (; drconf_cell_cnt != 0; --drconf_cell_cnt) { - struct of_drconf_cell drmem; - - read_drconf_cell(&drmem, &dm); + lmb_size = drmem_lmb_size(); + for_each_drmem_lmb(lmb) { /* skip this block if it is reserved or not assigned to * this partition */ - if ((drmem.flags & DRCONF_MEM_RESERVED) - || !(drmem.flags & DRCONF_MEM_ASSIGNED)) + if ((lmb->flags & DRCONF_MEM_RESERVED) + || !(lmb->flags & DRCONF_MEM_ASSIGNED)) continue; - if ((scn_addr < drmem.base_addr) - || (scn_addr >= (drmem.base_addr + lmb_size))) + if ((scn_addr < lmb->base_addr) + || (scn_addr >= (lmb->base_addr + lmb_size))) continue; - nid = of_drconf_to_nid_single(&drmem); + nid = of_drconf_to_nid_single(lmb); break; } @@ -1090,7 +981,7 @@ int hot_add_scn_to_nid(unsigned long scn_addr) memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); if (memory) { - nid = hot_add_drconf_scn_to_nid(memory, scn_addr); + nid = hot_add_drconf_scn_to_nid(scn_addr); of_node_put(memory); } else { nid = hot_add_node_scn_to_nid(scn_addr); @@ -1106,11 +997,7 @@ static u64 hot_add_drconf_memory_max(void) { struct device_node *memory = NULL; struct device_node *dn = NULL; - unsigned int drconf_cell_cnt = 0; - u64 lmb_size = 0; - const __be32 *dm = NULL; const __be64 *lrdr = NULL; - struct of_drconf_cell drmem; dn = of_find_node_by_path("/rtas"); if (dn) { @@ -1122,14 +1009,8 @@ static u64 hot_add_drconf_memory_max(void) memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); if (memory) { - drconf_cell_cnt = of_get_drconf_memory(memory, &dm); - lmb_size = of_get_lmb_size(memory); - - /* Advance to the last cell, each cell has 6 32 bit integers */ - dm += (drconf_cell_cnt - 1) * 6; - read_drconf_cell(&drmem, &dm); of_node_put(memory); - return drmem.base_addr + lmb_size; + return drmem_lmb_memory_max(); } return 0; } -- cgit v1.2.3 From 6195a5001f1d11e1ff6a7e47a865f4b42c1bb28c Mon Sep 17 00:00:00 2001 From: Nathan Fontenot Date: Fri, 1 Dec 2017 10:47:31 -0600 Subject: powerpc/pseries: Update memory hotplug code to use drmem LMB array Update the pseries memory hotplug code to use the newly added dynamic reconfiguration LMB array. Doing this is required for the upcoming support of version 2 of the dynamic reconfiguration device tree property. In addition, making this change cleans up the code that parses the LMB information as we no longer need to worry about device tree format. This allows us to discard one of the first steps on memory hotplug where we make a working copy of the device tree property and convert the entire property to cpu format. Instead we just use the LMB array directly while holding the memory hotplug lock. This patch also moves the updating of the device tree property to powerpc/mm/drmem.c. This allows to the hotplug code to work without needing to know the device tree format and provides a single routine for updating the device tree property. This new routine will handle determination of the proper device tree format and generate a properly formatted device tree property. Signed-off-by: Nathan Fontenot Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/drmem.h | 18 + arch/powerpc/mm/drmem.c | 81 ++++ arch/powerpc/platforms/pseries/hotplug-memory.c | 516 +++++++++--------------- 3 files changed, 297 insertions(+), 318 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h index 9dbfd38fa385..464e061a6616 100644 --- a/arch/powerpc/include/asm/drmem.h +++ b/arch/powerpc/include/asm/drmem.h @@ -40,9 +40,27 @@ static inline u32 drmem_lmb_size(void) return drmem_info->lmb_size; } +#define DRMEM_LMB_RESERVED 0x80000000 + +static inline void drmem_mark_lmb_reserved(struct drmem_lmb *lmb) +{ + lmb->flags |= DRMEM_LMB_RESERVED; +} + +static inline void drmem_remove_lmb_reservation(struct drmem_lmb *lmb) +{ + lmb->flags &= ~DRMEM_LMB_RESERVED; +} + +static inline bool drmem_lmb_reserved(struct drmem_lmb *lmb) +{ + return lmb->flags & DRMEM_LMB_RESERVED; +} + u64 drmem_lmb_memory_max(void); void __init walk_drmem_lmbs(struct device_node *dn, void (*func)(struct drmem_lmb *, const __be32 **)); +int drmem_update_dt(void); #ifdef CONFIG_PPC_PSERIES void __init walk_drmem_lmbs_early(unsigned long node, diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c index 5888ac3ca8a9..05ba0c7dcbed 100644 --- a/arch/powerpc/mm/drmem.c +++ b/arch/powerpc/mm/drmem.c @@ -29,6 +29,87 @@ u64 drmem_lmb_memory_max(void) return last_lmb->base_addr + drmem_lmb_size(); } +static u32 drmem_lmb_flags(struct drmem_lmb *lmb) +{ + /* + * Return the value of the lmb flags field minus the reserved + * bit used internally for hotplug processing. + */ + return lmb->flags & ~DRMEM_LMB_RESERVED; +} + +static struct property *clone_property(struct property *prop, u32 prop_sz) +{ + struct property *new_prop; + + new_prop = kzalloc(sizeof(*new_prop), GFP_KERNEL); + if (!new_prop) + return NULL; + + new_prop->name = kstrdup(prop->name, GFP_KERNEL); + new_prop->value = kzalloc(prop_sz, GFP_KERNEL); + if (!new_prop->name || !new_prop->value) { + kfree(new_prop->name); + kfree(new_prop->value); + kfree(new_prop); + return NULL; + } + + new_prop->length = prop_sz; +#if defined(CONFIG_OF_DYNAMIC) + of_property_set_flag(new_prop, OF_DYNAMIC); +#endif + return new_prop; +} + +static int drmem_update_dt_v1(struct device_node *memory, + struct property *prop) +{ + struct property *new_prop; + struct of_drconf_cell *dr_cell; + struct drmem_lmb *lmb; + u32 *p; + + new_prop = clone_property(prop, prop->length); + if (!new_prop) + return -1; + + p = new_prop->value; + *p++ = cpu_to_be32(drmem_info->n_lmbs); + + dr_cell = (struct of_drconf_cell *)p; + + for_each_drmem_lmb(lmb) { + dr_cell->base_addr = cpu_to_be64(lmb->base_addr); + dr_cell->drc_index = cpu_to_be32(lmb->drc_index); + dr_cell->aa_index = cpu_to_be32(lmb->aa_index); + dr_cell->flags = cpu_to_be32(drmem_lmb_flags(lmb)); + + dr_cell++; + } + + of_update_property(memory, new_prop); + return 0; +} + +int drmem_update_dt(void) +{ + struct device_node *memory; + struct property *prop; + int rc = -1; + + memory = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); + if (!memory) + return -1; + + prop = of_find_property(memory, "ibm,dynamic-memory", NULL); + if (prop) + rc = drmem_update_dt_v1(memory, prop); + + of_node_put(memory); + return rc; +} + static void __init read_drconf_v1_cell(struct drmem_lmb *lmb, const __be32 **prop) { diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c index 1d48ab424bd9..2043bc2b77b3 100644 --- a/arch/powerpc/platforms/pseries/hotplug-memory.c +++ b/arch/powerpc/platforms/pseries/hotplug-memory.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "pseries.h" static bool rtas_hp_event; @@ -100,100 +101,6 @@ static struct property *dlpar_clone_property(struct property *prop, return new_prop; } -static struct property *dlpar_clone_drconf_property(struct device_node *dn) -{ - struct property *prop, *new_prop; - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; - int i; - - prop = of_find_property(dn, "ibm,dynamic-memory", NULL); - if (!prop) - return NULL; - - new_prop = dlpar_clone_property(prop, prop->length); - if (!new_prop) - return NULL; - - /* Convert the property to cpu endian-ness */ - p = new_prop->value; - *p = be32_to_cpu(*p); - - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - - for (i = 0; i < num_lmbs; i++) { - lmbs[i].base_addr = be64_to_cpu(lmbs[i].base_addr); - lmbs[i].drc_index = be32_to_cpu(lmbs[i].drc_index); - lmbs[i].aa_index = be32_to_cpu(lmbs[i].aa_index); - lmbs[i].flags = be32_to_cpu(lmbs[i].flags); - } - - return new_prop; -} - -static void dlpar_update_drconf_property(struct device_node *dn, - struct property *prop) -{ - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; - int i; - - /* Convert the property back to BE */ - p = prop->value; - num_lmbs = *p; - *p = cpu_to_be32(*p); - p++; - - lmbs = (struct of_drconf_cell *)p; - for (i = 0; i < num_lmbs; i++) { - lmbs[i].base_addr = cpu_to_be64(lmbs[i].base_addr); - lmbs[i].drc_index = cpu_to_be32(lmbs[i].drc_index); - lmbs[i].aa_index = cpu_to_be32(lmbs[i].aa_index); - lmbs[i].flags = cpu_to_be32(lmbs[i].flags); - } - - rtas_hp_event = true; - of_update_property(dn, prop); - rtas_hp_event = false; -} - -static int dlpar_update_device_tree_lmb(struct of_drconf_cell *lmb) -{ - struct device_node *dn; - struct property *prop; - struct of_drconf_cell *lmbs; - u32 *p, num_lmbs; - int i; - - dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); - if (!dn) - return -ENODEV; - - prop = dlpar_clone_drconf_property(dn); - if (!prop) { - of_node_put(dn); - return -ENODEV; - } - - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - - for (i = 0; i < num_lmbs; i++) { - if (lmbs[i].drc_index == lmb->drc_index) { - lmbs[i].flags = lmb->flags; - lmbs[i].aa_index = lmb->aa_index; - - dlpar_update_drconf_property(dn, prop); - break; - } - } - - of_node_put(dn); - return 0; -} - static u32 find_aa_index(struct device_node *dr_node, struct property *ala_prop, const u32 *lmb_assoc) { @@ -256,7 +163,7 @@ static u32 find_aa_index(struct device_node *dr_node, return aa_index; } -static u32 lookup_lmb_associativity_index(struct of_drconf_cell *lmb) +static u32 lookup_lmb_associativity_index(struct drmem_lmb *lmb) { struct device_node *parent, *lmb_node, *dr_node; struct property *ala_prop; @@ -299,9 +206,9 @@ static u32 lookup_lmb_associativity_index(struct of_drconf_cell *lmb) return aa_index; } -static int dlpar_add_device_tree_lmb(struct of_drconf_cell *lmb) +static int dlpar_add_device_tree_lmb(struct drmem_lmb *lmb) { - int aa_index; + int rc, aa_index; lmb->flags |= DRCONF_MEM_ASSIGNED; @@ -313,17 +220,29 @@ static int dlpar_add_device_tree_lmb(struct of_drconf_cell *lmb) } lmb->aa_index = aa_index; - return dlpar_update_device_tree_lmb(lmb); + + rtas_hp_event = true; + rc = drmem_update_dt(); + rtas_hp_event = false; + + return rc; } -static int dlpar_remove_device_tree_lmb(struct of_drconf_cell *lmb) +static int dlpar_remove_device_tree_lmb(struct drmem_lmb *lmb) { + int rc; + lmb->flags &= ~DRCONF_MEM_ASSIGNED; lmb->aa_index = 0xffffffff; - return dlpar_update_device_tree_lmb(lmb); + + rtas_hp_event = true; + rc = drmem_update_dt(); + rtas_hp_event = false; + + return rc; } -static struct memory_block *lmb_to_memblock(struct of_drconf_cell *lmb) +static struct memory_block *lmb_to_memblock(struct drmem_lmb *lmb) { unsigned long section_nr; struct mem_section *mem_sect; @@ -336,7 +255,36 @@ static struct memory_block *lmb_to_memblock(struct of_drconf_cell *lmb) return mem_block; } -static int dlpar_change_lmb_state(struct of_drconf_cell *lmb, bool online) +static int get_lmb_range(u32 drc_index, int n_lmbs, + struct drmem_lmb **start_lmb, + struct drmem_lmb **end_lmb) +{ + struct drmem_lmb *lmb, *start, *end; + struct drmem_lmb *last_lmb; + + start = NULL; + for_each_drmem_lmb(lmb) { + if (lmb->drc_index == drc_index) { + start = lmb; + break; + } + } + + if (!start) + return -EINVAL; + + end = &start[n_lmbs - 1]; + + last_lmb = &drmem_info->lmbs[drmem_info->n_lmbs - 1]; + if (end > last_lmb) + return -EINVAL; + + *start_lmb = start; + *end_lmb = end; + return 0; +} + +static int dlpar_change_lmb_state(struct drmem_lmb *lmb, bool online) { struct memory_block *mem_block; int rc; @@ -357,13 +305,13 @@ static int dlpar_change_lmb_state(struct of_drconf_cell *lmb, bool online) return rc; } -static int dlpar_online_lmb(struct of_drconf_cell *lmb) +static int dlpar_online_lmb(struct drmem_lmb *lmb) { return dlpar_change_lmb_state(lmb, true); } #ifdef CONFIG_MEMORY_HOTREMOVE -static int dlpar_offline_lmb(struct of_drconf_cell *lmb) +static int dlpar_offline_lmb(struct drmem_lmb *lmb) { return dlpar_change_lmb_state(lmb, false); } @@ -426,7 +374,7 @@ static int pseries_remove_mem_node(struct device_node *np) return 0; } -static bool lmb_is_removable(struct of_drconf_cell *lmb) +static bool lmb_is_removable(struct drmem_lmb *lmb) { int i, scns_per_block; int rc = 1; @@ -458,9 +406,9 @@ static bool lmb_is_removable(struct of_drconf_cell *lmb) return rc ? true : false; } -static int dlpar_add_lmb(struct of_drconf_cell *); +static int dlpar_add_lmb(struct drmem_lmb *); -static int dlpar_remove_lmb(struct of_drconf_cell *lmb) +static int dlpar_remove_lmb(struct drmem_lmb *lmb) { unsigned long block_sz; int nid, rc; @@ -484,28 +432,25 @@ static int dlpar_remove_lmb(struct of_drconf_cell *lmb) return 0; } -static int dlpar_memory_remove_by_count(u32 lmbs_to_remove, - struct property *prop) +static int dlpar_memory_remove_by_count(u32 lmbs_to_remove) { - struct of_drconf_cell *lmbs; + struct drmem_lmb *lmb; int lmbs_removed = 0; int lmbs_available = 0; - u32 num_lmbs, *p; - int i, rc; + int rc; pr_info("Attempting to hot-remove %d LMB(s)\n", lmbs_to_remove); if (lmbs_to_remove == 0) return -EINVAL; - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - /* Validate that there are enough LMBs to satisfy the request */ - for (i = 0; i < num_lmbs; i++) { - if (lmb_is_removable(&lmbs[i])) + for_each_drmem_lmb(lmb) { + if (lmb_is_removable(lmb)) lmbs_available++; + + if (lmbs_available == lmbs_to_remove) + break; } if (lmbs_available < lmbs_to_remove) { @@ -514,45 +459,47 @@ static int dlpar_memory_remove_by_count(u32 lmbs_to_remove, return -EINVAL; } - for (i = 0; i < num_lmbs && lmbs_removed < lmbs_to_remove; i++) { - rc = dlpar_remove_lmb(&lmbs[i]); + for_each_drmem_lmb(lmb) { + rc = dlpar_remove_lmb(lmb); if (rc) continue; - lmbs_removed++; - /* Mark this lmb so we can add it later if all of the * requested LMBs cannot be removed. */ - lmbs[i].reserved = 1; + drmem_mark_lmb_reserved(lmb); + + lmbs_removed++; + if (lmbs_removed == lmbs_to_remove) + break; } if (lmbs_removed != lmbs_to_remove) { pr_err("Memory hot-remove failed, adding LMB's back\n"); - for (i = 0; i < num_lmbs; i++) { - if (!lmbs[i].reserved) + for_each_drmem_lmb(lmb) { + if (!drmem_lmb_reserved(lmb)) continue; - rc = dlpar_add_lmb(&lmbs[i]); + rc = dlpar_add_lmb(lmb); if (rc) pr_err("Failed to add LMB back, drc index %x\n", - lmbs[i].drc_index); + lmb->drc_index); - lmbs[i].reserved = 0; + drmem_remove_lmb_reservation(lmb); } rc = -EINVAL; } else { - for (i = 0; i < num_lmbs; i++) { - if (!lmbs[i].reserved) + for_each_drmem_lmb(lmb) { + if (!drmem_lmb_reserved(lmb)) continue; - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); pr_info("Memory at %llx was hot-removed\n", - lmbs[i].base_addr); + lmb->base_addr); - lmbs[i].reserved = 0; + drmem_remove_lmb_reservation(lmb); } rc = 0; } @@ -560,26 +507,21 @@ static int dlpar_memory_remove_by_count(u32 lmbs_to_remove, return rc; } -static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop) +static int dlpar_memory_remove_by_index(u32 drc_index) { - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; + struct drmem_lmb *lmb; int lmb_found; - int i, rc; + int rc; pr_info("Attempting to hot-remove LMB, drc index %x\n", drc_index); - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - lmb_found = 0; - for (i = 0; i < num_lmbs; i++) { - if (lmbs[i].drc_index == drc_index) { + for_each_drmem_lmb(lmb) { + if (lmb->drc_index == drc_index) { lmb_found = 1; - rc = dlpar_remove_lmb(&lmbs[i]); + rc = dlpar_remove_lmb(lmb); if (!rc) - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); break; } @@ -590,35 +532,30 @@ static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop) if (rc) pr_info("Failed to hot-remove memory at %llx\n", - lmbs[i].base_addr); + lmb->base_addr); else - pr_info("Memory at %llx was hot-removed\n", lmbs[i].base_addr); + pr_info("Memory at %llx was hot-removed\n", lmb->base_addr); return rc; } -static int dlpar_memory_readd_by_index(u32 drc_index, struct property *prop) +static int dlpar_memory_readd_by_index(u32 drc_index) { - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; + struct drmem_lmb *lmb; int lmb_found; - int i, rc; + int rc; pr_info("Attempting to update LMB, drc index %x\n", drc_index); - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - lmb_found = 0; - for (i = 0; i < num_lmbs; i++) { - if (lmbs[i].drc_index == drc_index) { + for_each_drmem_lmb(lmb) { + if (lmb->drc_index == drc_index) { lmb_found = 1; - rc = dlpar_remove_lmb(&lmbs[i]); + rc = dlpar_remove_lmb(lmb); if (!rc) { - rc = dlpar_add_lmb(&lmbs[i]); + rc = dlpar_add_lmb(lmb); if (rc) - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); } break; } @@ -629,20 +566,18 @@ static int dlpar_memory_readd_by_index(u32 drc_index, struct property *prop) if (rc) pr_info("Failed to update memory at %llx\n", - lmbs[i].base_addr); + lmb->base_addr); else - pr_info("Memory at %llx was updated\n", lmbs[i].base_addr); + pr_info("Memory at %llx was updated\n", lmb->base_addr); return rc; } -static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index, - struct property *prop) +static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index) { - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; - int i, rc, start_lmb_found; - int lmbs_available = 0, start_index = 0, end_index; + struct drmem_lmb *lmb, *start_lmb, *end_lmb; + int lmbs_available = 0; + int rc; pr_info("Attempting to hot-remove %u LMB(s) at %x\n", lmbs_to_remove, drc_index); @@ -650,29 +585,13 @@ static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index, if (lmbs_to_remove == 0) return -EINVAL; - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - start_lmb_found = 0; - - /* Navigate to drc_index */ - while (start_index < num_lmbs) { - if (lmbs[start_index].drc_index == drc_index) { - start_lmb_found = 1; - break; - } - - start_index++; - } - - if (!start_lmb_found) + rc = get_lmb_range(drc_index, lmbs_to_remove, &start_lmb, &end_lmb); + if (rc) return -EINVAL; - end_index = start_index + lmbs_to_remove; - /* Validate that there are enough LMBs to satisfy the request */ - for (i = start_index; i < end_index; i++) { - if (lmbs[i].flags & DRCONF_MEM_RESERVED) + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (lmb->flags & DRCONF_MEM_RESERVED) break; lmbs_available++; @@ -681,42 +600,43 @@ static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index, if (lmbs_available < lmbs_to_remove) return -EINVAL; - for (i = start_index; i < end_index; i++) { - if (!(lmbs[i].flags & DRCONF_MEM_ASSIGNED)) + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (!(lmb->flags & DRCONF_MEM_ASSIGNED)) continue; - rc = dlpar_remove_lmb(&lmbs[i]); + rc = dlpar_remove_lmb(lmb); if (rc) break; - lmbs[i].reserved = 1; + drmem_mark_lmb_reserved(lmb); } if (rc) { pr_err("Memory indexed-count-remove failed, adding any removed LMBs\n"); - for (i = start_index; i < end_index; i++) { - if (!lmbs[i].reserved) + + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (!drmem_lmb_reserved(lmb)) continue; - rc = dlpar_add_lmb(&lmbs[i]); + rc = dlpar_add_lmb(lmb); if (rc) pr_err("Failed to add LMB, drc index %x\n", - be32_to_cpu(lmbs[i].drc_index)); + lmb->drc_index); - lmbs[i].reserved = 0; + drmem_remove_lmb_reservation(lmb); } rc = -EINVAL; } else { - for (i = start_index; i < end_index; i++) { - if (!lmbs[i].reserved) + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (!drmem_lmb_reserved(lmb)) continue; - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); pr_info("Memory at %llx (drc index %x) was hot-removed\n", - lmbs[i].base_addr, lmbs[i].drc_index); + lmb->base_addr, lmb->drc_index); - lmbs[i].reserved = 0; + drmem_remove_lmb_reservation(lmb); } } @@ -737,32 +657,30 @@ static inline int dlpar_memory_remove(struct pseries_hp_errorlog *hp_elog) { return -EOPNOTSUPP; } -static int dlpar_remove_lmb(struct of_drconf_cell *lmb) +static int dlpar_remove_lmb(struct drmem_lmb *lmb) { return -EOPNOTSUPP; } -static int dlpar_memory_remove_by_count(u32 lmbs_to_remove, - struct property *prop) +static int dlpar_memory_remove_by_count(u32 lmbs_to_remove) { return -EOPNOTSUPP; } -static int dlpar_memory_remove_by_index(u32 drc_index, struct property *prop) +static int dlpar_memory_remove_by_index(u32 drc_index) { return -EOPNOTSUPP; } -static int dlpar_memory_readd_by_index(u32 drc_index, struct property *prop) +static int dlpar_memory_readd_by_index(u32 drc_index) { return -EOPNOTSUPP; } -static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index, - struct property *prop) +static int dlpar_memory_remove_by_ic(u32 lmbs_to_remove, u32 drc_index) { return -EOPNOTSUPP; } #endif /* CONFIG_MEMORY_HOTREMOVE */ -static int dlpar_add_lmb(struct of_drconf_cell *lmb) +static int dlpar_add_lmb(struct drmem_lmb *lmb) { unsigned long block_sz; int nid, rc; @@ -801,77 +719,79 @@ static int dlpar_add_lmb(struct of_drconf_cell *lmb) return rc; } -static int dlpar_memory_add_by_count(u32 lmbs_to_add, struct property *prop) +static int dlpar_memory_add_by_count(u32 lmbs_to_add) { - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; + struct drmem_lmb *lmb; int lmbs_available = 0; int lmbs_added = 0; - int i, rc; + int rc; pr_info("Attempting to hot-add %d LMB(s)\n", lmbs_to_add); if (lmbs_to_add == 0) return -EINVAL; - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - /* Validate that there are enough LMBs to satisfy the request */ - for (i = 0; i < num_lmbs; i++) { - if (!(lmbs[i].flags & DRCONF_MEM_ASSIGNED)) + for_each_drmem_lmb(lmb) { + if (!(lmb->flags & DRCONF_MEM_ASSIGNED)) lmbs_available++; + + if (lmbs_available == lmbs_to_add) + break; } if (lmbs_available < lmbs_to_add) return -EINVAL; - for (i = 0; i < num_lmbs && lmbs_to_add != lmbs_added; i++) { - if (lmbs[i].flags & DRCONF_MEM_ASSIGNED) + for_each_drmem_lmb(lmb) { + if (lmb->flags & DRCONF_MEM_ASSIGNED) continue; - rc = dlpar_acquire_drc(lmbs[i].drc_index); + rc = dlpar_acquire_drc(lmb->drc_index); if (rc) continue; - rc = dlpar_add_lmb(&lmbs[i]); + rc = dlpar_add_lmb(lmb); if (rc) { - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); continue; } - lmbs_added++; - /* Mark this lmb so we can remove it later if all of the * requested LMBs cannot be added. */ - lmbs[i].reserved = 1; + drmem_mark_lmb_reserved(lmb); + + lmbs_added++; + if (lmbs_added == lmbs_to_add) + break; } if (lmbs_added != lmbs_to_add) { pr_err("Memory hot-add failed, removing any added LMBs\n"); - for (i = 0; i < num_lmbs; i++) { - if (!lmbs[i].reserved) + for_each_drmem_lmb(lmb) { + if (!drmem_lmb_reserved(lmb)) continue; - rc = dlpar_remove_lmb(&lmbs[i]); + rc = dlpar_remove_lmb(lmb); if (rc) pr_err("Failed to remove LMB, drc index %x\n", - be32_to_cpu(lmbs[i].drc_index)); + lmb->drc_index); else - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); + + drmem_remove_lmb_reservation(lmb); } rc = -EINVAL; } else { - for (i = 0; i < num_lmbs; i++) { - if (!lmbs[i].reserved) + for_each_drmem_lmb(lmb) { + if (!drmem_lmb_reserved(lmb)) continue; pr_info("Memory at %llx (drc index %x) was hot-added\n", - lmbs[i].base_addr, lmbs[i].drc_index); - lmbs[i].reserved = 0; + lmb->base_addr, lmb->drc_index); + drmem_remove_lmb_reservation(lmb); } rc = 0; } @@ -879,28 +799,22 @@ static int dlpar_memory_add_by_count(u32 lmbs_to_add, struct property *prop) return rc; } -static int dlpar_memory_add_by_index(u32 drc_index, struct property *prop) +static int dlpar_memory_add_by_index(u32 drc_index) { - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; - int i, lmb_found; - int rc; + struct drmem_lmb *lmb; + int rc, lmb_found; pr_info("Attempting to hot-add LMB, drc index %x\n", drc_index); - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - lmb_found = 0; - for (i = 0; i < num_lmbs; i++) { - if (lmbs[i].drc_index == drc_index) { + for_each_drmem_lmb(lmb) { + if (lmb->drc_index == drc_index) { lmb_found = 1; - rc = dlpar_acquire_drc(lmbs[i].drc_index); + rc = dlpar_acquire_drc(lmb->drc_index); if (!rc) { - rc = dlpar_add_lmb(&lmbs[i]); + rc = dlpar_add_lmb(lmb); if (rc) - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); } break; @@ -914,18 +828,16 @@ static int dlpar_memory_add_by_index(u32 drc_index, struct property *prop) pr_info("Failed to hot-add memory, drc index %x\n", drc_index); else pr_info("Memory at %llx (drc index %x) was hot-added\n", - lmbs[i].base_addr, drc_index); + lmb->base_addr, drc_index); return rc; } -static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index, - struct property *prop) +static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index) { - struct of_drconf_cell *lmbs; - u32 num_lmbs, *p; - int i, rc, start_lmb_found; - int lmbs_available = 0, start_index = 0, end_index; + struct drmem_lmb *lmb, *start_lmb, *end_lmb; + int lmbs_available = 0; + int rc; pr_info("Attempting to hot-add %u LMB(s) at index %x\n", lmbs_to_add, drc_index); @@ -933,29 +845,13 @@ static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index, if (lmbs_to_add == 0) return -EINVAL; - p = prop->value; - num_lmbs = *p++; - lmbs = (struct of_drconf_cell *)p; - start_lmb_found = 0; - - /* Navigate to drc_index */ - while (start_index < num_lmbs) { - if (lmbs[start_index].drc_index == drc_index) { - start_lmb_found = 1; - break; - } - - start_index++; - } - - if (!start_lmb_found) + rc = get_lmb_range(drc_index, lmbs_to_add, &start_lmb, &end_lmb); + if (rc) return -EINVAL; - end_index = start_index + lmbs_to_add; - /* Validate that the LMBs in this range are not reserved */ - for (i = start_index; i < end_index; i++) { - if (lmbs[i].flags & DRCONF_MEM_RESERVED) + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (lmb->flags & DRCONF_MEM_RESERVED) break; lmbs_available++; @@ -964,46 +860,48 @@ static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index, if (lmbs_available < lmbs_to_add) return -EINVAL; - for (i = start_index; i < end_index; i++) { - if (lmbs[i].flags & DRCONF_MEM_ASSIGNED) + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (lmb->flags & DRCONF_MEM_ASSIGNED) continue; - rc = dlpar_acquire_drc(lmbs[i].drc_index); + rc = dlpar_acquire_drc(lmb->drc_index); if (rc) break; - rc = dlpar_add_lmb(&lmbs[i]); + rc = dlpar_add_lmb(lmb); if (rc) { - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); break; } - lmbs[i].reserved = 1; + drmem_mark_lmb_reserved(lmb); } if (rc) { pr_err("Memory indexed-count-add failed, removing any added LMBs\n"); - for (i = start_index; i < end_index; i++) { - if (!lmbs[i].reserved) + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (!drmem_lmb_reserved(lmb)) continue; - rc = dlpar_remove_lmb(&lmbs[i]); + rc = dlpar_remove_lmb(lmb); if (rc) pr_err("Failed to remove LMB, drc index %x\n", - be32_to_cpu(lmbs[i].drc_index)); + lmb->drc_index); else - dlpar_release_drc(lmbs[i].drc_index); + dlpar_release_drc(lmb->drc_index); + + drmem_remove_lmb_reservation(lmb); } rc = -EINVAL; } else { - for (i = start_index; i < end_index; i++) { - if (!lmbs[i].reserved) + for_each_drmem_lmb_in_range(lmb, start_lmb, end_lmb) { + if (!drmem_lmb_reserved(lmb)) continue; pr_info("Memory at %llx (drc index %x) was hot-added\n", - lmbs[i].base_addr, lmbs[i].drc_index); - lmbs[i].reserved = 0; + lmb->base_addr, lmb->drc_index); + drmem_remove_lmb_reservation(lmb); } } @@ -1012,37 +910,23 @@ static int dlpar_memory_add_by_ic(u32 lmbs_to_add, u32 drc_index, int dlpar_memory(struct pseries_hp_errorlog *hp_elog) { - struct device_node *dn; - struct property *prop; u32 count, drc_index; int rc; lock_device_hotplug(); - dn = of_find_node_by_path("/ibm,dynamic-reconfiguration-memory"); - if (!dn) { - rc = -EINVAL; - goto dlpar_memory_out; - } - - prop = dlpar_clone_drconf_property(dn); - if (!prop) { - rc = -EINVAL; - goto dlpar_memory_out; - } - switch (hp_elog->action) { case PSERIES_HP_ELOG_ACTION_ADD: if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT) { count = hp_elog->_drc_u.drc_count; - rc = dlpar_memory_add_by_count(count, prop); + rc = dlpar_memory_add_by_count(count); } else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) { drc_index = hp_elog->_drc_u.drc_index; - rc = dlpar_memory_add_by_index(drc_index, prop); + rc = dlpar_memory_add_by_index(drc_index); } else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_IC) { count = hp_elog->_drc_u.ic.count; drc_index = hp_elog->_drc_u.ic.index; - rc = dlpar_memory_add_by_ic(count, drc_index, prop); + rc = dlpar_memory_add_by_ic(count, drc_index); } else { rc = -EINVAL; } @@ -1051,14 +935,14 @@ int dlpar_memory(struct pseries_hp_errorlog *hp_elog) case PSERIES_HP_ELOG_ACTION_REMOVE: if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_COUNT) { count = hp_elog->_drc_u.drc_count; - rc = dlpar_memory_remove_by_count(count, prop); + rc = dlpar_memory_remove_by_count(count); } else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_INDEX) { drc_index = hp_elog->_drc_u.drc_index; - rc = dlpar_memory_remove_by_index(drc_index, prop); + rc = dlpar_memory_remove_by_index(drc_index); } else if (hp_elog->id_type == PSERIES_HP_ELOG_ID_DRC_IC) { count = hp_elog->_drc_u.ic.count; drc_index = hp_elog->_drc_u.ic.index; - rc = dlpar_memory_remove_by_ic(count, drc_index, prop); + rc = dlpar_memory_remove_by_ic(count, drc_index); } else { rc = -EINVAL; } @@ -1066,7 +950,7 @@ int dlpar_memory(struct pseries_hp_errorlog *hp_elog) break; case PSERIES_HP_ELOG_ACTION_READD: drc_index = hp_elog->_drc_u.drc_index; - rc = dlpar_memory_readd_by_index(drc_index, prop); + rc = dlpar_memory_readd_by_index(drc_index); break; default: pr_err("Invalid action (%d) specified\n", hp_elog->action); @@ -1074,10 +958,6 @@ int dlpar_memory(struct pseries_hp_errorlog *hp_elog) break; } - dlpar_free_property(prop); - -dlpar_memory_out: - of_node_put(dn); unlock_device_hotplug(); return rc; } -- cgit v1.2.3 From 2c77721552e565e900705f4499067f8d37be3976 Mon Sep 17 00:00:00 2001 From: Nathan Fontenot Date: Fri, 1 Dec 2017 10:47:42 -0600 Subject: powerpc: Move of_drconf_cell struct to asm/drmem.h Now that the powerpc code parses dynamic reconfiguration memory LMB information from the LMB array and not the device tree directly we can move the of_drconf_cell struct to drmem.h where it fits better. In addition, the struct is renamed to of_drconf_cell_v1 in anticipation of upcoming support for version 2 of the dynamic reconfiguration property and the members are typed as __be* values to reflect how they exist in the device tree. Signed-off-by: Nathan Fontenot Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/drmem.h | 19 +++++++++++++++++++ arch/powerpc/include/asm/prom.h | 16 ---------------- arch/powerpc/mm/drmem.c | 4 ++-- arch/powerpc/platforms/pseries/hotplug-memory.c | 6 +++--- 4 files changed, 24 insertions(+), 21 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h index 464e061a6616..54241456420e 100644 --- a/arch/powerpc/include/asm/drmem.h +++ b/arch/powerpc/include/asm/drmem.h @@ -35,6 +35,25 @@ extern struct drmem_lmb_info *drmem_info; &drmem_info->lmbs[0], \ &drmem_info->lmbs[drmem_info->n_lmbs - 1]) +/* + * The of_drconf_cell_v1 struct defines the layout of the LMB data + * specified in the ibm,dynamic-memory device tree property. + * The property itself is a 32-bit value specifying the number of + * LMBs followed by an array of of_drconf_cell_v1 entries, one + * per LMB. + */ +struct of_drconf_cell_v1 { + __be64 base_addr; + __be32 drc_index; + __be32 reserved; + __be32 aa_index; + __be32 flags; +}; + +#define DRCONF_MEM_ASSIGNED 0x00000008 +#define DRCONF_MEM_AI_INVALID 0x00000040 +#define DRCONF_MEM_RESERVED 0x00000080 + static inline u32 drmem_lmb_size(void) { return drmem_info->lmb_size; diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index 825bd5998701..f0a30a003bd8 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -80,22 +80,6 @@ extern void of_instantiate_rtc(void); extern int of_get_ibm_chip_id(struct device_node *np); -/* The of_drconf_cell struct defines the layout of the LMB array - * specified in the device tree property - * ibm,dynamic-reconfiguration-memory/ibm,dynamic-memory - */ -struct of_drconf_cell { - u64 base_addr; - u32 drc_index; - u32 reserved; - u32 aa_index; - u32 flags; -}; - -#define DRCONF_MEM_ASSIGNED 0x00000008 -#define DRCONF_MEM_AI_INVALID 0x00000040 -#define DRCONF_MEM_RESERVED 0x00000080 - /* * There are two methods for telling firmware what our capabilities are. * Newer machines have an "ibm,client-architecture-support" method on the diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c index 05ba0c7dcbed..61bc183013ae 100644 --- a/arch/powerpc/mm/drmem.c +++ b/arch/powerpc/mm/drmem.c @@ -66,7 +66,7 @@ static int drmem_update_dt_v1(struct device_node *memory, struct property *prop) { struct property *new_prop; - struct of_drconf_cell *dr_cell; + struct of_drconf_cell_v1 *dr_cell; struct drmem_lmb *lmb; u32 *p; @@ -77,7 +77,7 @@ static int drmem_update_dt_v1(struct device_node *memory, p = new_prop->value; *p++ = cpu_to_be32(drmem_info->n_lmbs); - dr_cell = (struct of_drconf_cell *)p; + dr_cell = (struct of_drconf_cell_v1 *)p; for_each_drmem_lmb(lmb) { dr_cell->base_addr = cpu_to_be64(lmb->base_addr); diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c index 2043bc2b77b3..c1578f54c626 100644 --- a/arch/powerpc/platforms/pseries/hotplug-memory.c +++ b/arch/powerpc/platforms/pseries/hotplug-memory.c @@ -996,7 +996,7 @@ static int pseries_add_mem_node(struct device_node *np) static int pseries_update_drconf_memory(struct of_reconfig_data *pr) { - struct of_drconf_cell *new_drmem, *old_drmem; + struct of_drconf_cell_v1 *new_drmem, *old_drmem; unsigned long memblock_size; u32 entries; __be32 *p; @@ -1019,11 +1019,11 @@ static int pseries_update_drconf_memory(struct of_reconfig_data *pr) * of_drconf_cell's. */ entries = be32_to_cpu(*p++); - old_drmem = (struct of_drconf_cell *)p; + old_drmem = (struct of_drconf_cell_v1 *)p; p = (__be32 *)pr->prop->value; p++; - new_drmem = (struct of_drconf_cell *)p; + new_drmem = (struct of_drconf_cell_v1 *)p; for (i = 0; i < entries; i++) { if ((be32_to_cpu(old_drmem[i].flags) & DRCONF_MEM_ASSIGNED) && -- cgit v1.2.3 From 2b31e3aec1dbaae5f0a6f0e485fcb1ff1aceb6cf Mon Sep 17 00:00:00 2001 From: Nathan Fontenot Date: Fri, 1 Dec 2017 10:47:53 -0600 Subject: powerpc/drmem: Add support for ibm, dynamic-memory-v2 property The Power Hypervisor has introduced a new device tree format for the property describing the dynamic reconfiguration LMBs for a system, ibm,dynamic-memory-v2. This new format condenses the size of the property, especially on large memory systems, by reporting sets of LMBs that have the same properties (flags and associativity array index). This patch updates the powerpc/mm/drmem.c code to provide routines that can parse the new device tree format during the walk_drmem_lmb* routines used during boot, the creation of the LMB array, and updating the device tree to create a new property in the proper format for ibm,dynamic-memory-v2. Signed-off-by: Nathan Fontenot Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/drmem.h | 13 +++ arch/powerpc/mm/drmem.c | 192 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 201 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/drmem.h b/arch/powerpc/include/asm/drmem.h index 54241456420e..ce242b9ea8c6 100644 --- a/arch/powerpc/include/asm/drmem.h +++ b/arch/powerpc/include/asm/drmem.h @@ -50,6 +50,19 @@ struct of_drconf_cell_v1 { __be32 flags; }; +/* + * Version 2 of the ibm,dynamic-memory property is defined as a + * 32-bit value specifying the number of LMB sets followed by an + * array of of_drconf_cell_v2 entries, one per LMB set. + */ +struct of_drconf_cell_v2 { + u32 seq_lmbs; + u64 base_addr; + u32 drc_index; + u32 aa_index; + u32 flags; +} __packed; + #define DRCONF_MEM_ASSIGNED 0x00000008 #define DRCONF_MEM_AI_INVALID 0x00000040 #define DRCONF_MEM_RESERVED 0x00000080 diff --git a/arch/powerpc/mm/drmem.c b/arch/powerpc/mm/drmem.c index 61bc183013ae..1604110c4238 100644 --- a/arch/powerpc/mm/drmem.c +++ b/arch/powerpc/mm/drmem.c @@ -92,6 +92,84 @@ static int drmem_update_dt_v1(struct device_node *memory, return 0; } +static void init_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell, + struct drmem_lmb *lmb) +{ + dr_cell->base_addr = cpu_to_be64(lmb->base_addr); + dr_cell->drc_index = cpu_to_be32(lmb->drc_index); + dr_cell->aa_index = cpu_to_be32(lmb->aa_index); + dr_cell->flags = cpu_to_be32(lmb->flags); +} + +static int drmem_update_dt_v2(struct device_node *memory, + struct property *prop) +{ + struct property *new_prop; + struct of_drconf_cell_v2 *dr_cell; + struct drmem_lmb *lmb, *prev_lmb; + u32 lmb_sets, prop_sz, seq_lmbs; + u32 *p; + + /* First pass, determine how many LMB sets are needed. */ + lmb_sets = 0; + prev_lmb = NULL; + for_each_drmem_lmb(lmb) { + if (!prev_lmb) { + prev_lmb = lmb; + lmb_sets++; + continue; + } + + if (prev_lmb->aa_index != lmb->aa_index || + prev_lmb->flags != lmb->flags) + lmb_sets++; + + prev_lmb = lmb; + } + + prop_sz = lmb_sets * sizeof(*dr_cell) + sizeof(__be32); + new_prop = clone_property(prop, prop_sz); + if (!new_prop) + return -1; + + p = new_prop->value; + *p++ = cpu_to_be32(lmb_sets); + + dr_cell = (struct of_drconf_cell_v2 *)p; + + /* Second pass, populate the LMB set data */ + prev_lmb = NULL; + seq_lmbs = 0; + for_each_drmem_lmb(lmb) { + if (prev_lmb == NULL) { + /* Start of first LMB set */ + prev_lmb = lmb; + init_drconf_v2_cell(dr_cell, lmb); + seq_lmbs++; + continue; + } + + if (prev_lmb->aa_index != lmb->aa_index || + prev_lmb->flags != lmb->flags) { + /* end of one set, start of another */ + dr_cell->seq_lmbs = cpu_to_be32(seq_lmbs); + dr_cell++; + + init_drconf_v2_cell(dr_cell, lmb); + seq_lmbs = 1; + } else { + seq_lmbs++; + } + + prev_lmb = lmb; + } + + /* close out last LMB set */ + dr_cell->seq_lmbs = cpu_to_be32(seq_lmbs); + of_update_property(memory, new_prop); + return 0; +} + int drmem_update_dt(void) { struct device_node *memory; @@ -103,8 +181,13 @@ int drmem_update_dt(void) return -1; prop = of_find_property(memory, "ibm,dynamic-memory", NULL); - if (prop) + if (prop) { rc = drmem_update_dt_v1(memory, prop); + } else { + prop = of_find_property(memory, "ibm,dynamic-memory-v2", NULL); + if (prop) + rc = drmem_update_dt_v2(memory, prop); + } of_node_put(memory); return rc; @@ -140,6 +223,47 @@ static void __init __walk_drmem_v1_lmbs(const __be32 *prop, const __be32 *usm, } } +static void __init read_drconf_v2_cell(struct of_drconf_cell_v2 *dr_cell, + const __be32 **prop) +{ + const __be32 *p = *prop; + + dr_cell->seq_lmbs = of_read_number(p++, 1); + dr_cell->base_addr = dt_mem_next_cell(dt_root_addr_cells, &p); + dr_cell->drc_index = of_read_number(p++, 1); + dr_cell->aa_index = of_read_number(p++, 1); + dr_cell->flags = of_read_number(p++, 1); + + *prop = p; +} + +static void __init __walk_drmem_v2_lmbs(const __be32 *prop, const __be32 *usm, + void (*func)(struct drmem_lmb *, const __be32 **)) +{ + struct of_drconf_cell_v2 dr_cell; + struct drmem_lmb lmb; + u32 i, j, lmb_sets; + + lmb_sets = of_read_number(prop++, 1); + + for (i = 0; i < lmb_sets; i++) { + read_drconf_v2_cell(&dr_cell, &prop); + + for (j = 0; j < dr_cell.seq_lmbs; j++) { + lmb.base_addr = dr_cell.base_addr; + dr_cell.base_addr += drmem_lmb_size(); + + lmb.drc_index = dr_cell.drc_index; + dr_cell.drc_index++; + + lmb.aa_index = dr_cell.aa_index; + lmb.flags = dr_cell.flags; + + func(&lmb, &usm); + } + } +} + #ifdef CONFIG_PPC_PSERIES void __init walk_drmem_lmbs_early(unsigned long node, void (*func)(struct drmem_lmb *, const __be32 **)) @@ -156,8 +280,14 @@ void __init walk_drmem_lmbs_early(unsigned long node, usm = of_get_flat_dt_prop(node, "linux,drconf-usable-memory", &len); prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory", &len); - if (prop) + if (prop) { __walk_drmem_v1_lmbs(prop, usm, func); + } else { + prop = of_get_flat_dt_prop(node, "ibm,dynamic-memory-v2", + &len); + if (prop) + __walk_drmem_v2_lmbs(prop, usm, func); + } memblock_dump_all(); } @@ -210,8 +340,13 @@ void __init walk_drmem_lmbs(struct device_node *dn, usm = of_get_usable_memory(dn); prop = of_get_property(dn, "ibm,dynamic-memory", NULL); - if (prop) + if (prop) { __walk_drmem_v1_lmbs(prop, usm, func); + } else { + prop = of_get_property(dn, "ibm,dynamic-memory-v2", NULL); + if (prop) + __walk_drmem_v2_lmbs(prop, usm, func); + } } static void __init init_drmem_v1_lmbs(const __be32 *prop) @@ -229,6 +364,50 @@ static void __init init_drmem_v1_lmbs(const __be32 *prop) read_drconf_v1_cell(lmb, &prop); } +static void __init init_drmem_v2_lmbs(const __be32 *prop) +{ + struct drmem_lmb *lmb; + struct of_drconf_cell_v2 dr_cell; + const __be32 *p; + u32 i, j, lmb_sets; + int lmb_index; + + lmb_sets = of_read_number(prop++, 1); + + /* first pass, calculate the number of LMBs */ + p = prop; + for (i = 0; i < lmb_sets; i++) { + read_drconf_v2_cell(&dr_cell, &p); + drmem_info->n_lmbs += dr_cell.seq_lmbs; + } + + drmem_info->lmbs = kcalloc(drmem_info->n_lmbs, sizeof(*lmb), + GFP_KERNEL); + if (!drmem_info->lmbs) + return; + + /* second pass, read in the LMB information */ + lmb_index = 0; + p = prop; + + for (i = 0; i < lmb_sets; i++) { + read_drconf_v2_cell(&dr_cell, &p); + + for (j = 0; j < dr_cell.seq_lmbs; j++) { + lmb = &drmem_info->lmbs[lmb_index++]; + + lmb->base_addr = dr_cell.base_addr; + dr_cell.base_addr += drmem_info->lmb_size; + + lmb->drc_index = dr_cell.drc_index; + dr_cell.drc_index++; + + lmb->aa_index = dr_cell.aa_index; + lmb->flags = dr_cell.flags; + } + } +} + static int __init drmem_init(void) { struct device_node *dn; @@ -246,8 +425,13 @@ static int __init drmem_init(void) } prop = of_get_property(dn, "ibm,dynamic-memory", NULL); - if (prop) + if (prop) { init_drmem_v1_lmbs(prop); + } else { + prop = of_get_property(dn, "ibm,dynamic-memory-v2", NULL); + if (prop) + init_drmem_v2_lmbs(prop); + } of_node_put(dn); return 0; -- cgit v1.2.3 From 0c38ed6f6f0b78a404fe46767d21504b37af8705 Mon Sep 17 00:00:00 2001 From: Nathan Fontenot Date: Fri, 1 Dec 2017 10:48:03 -0600 Subject: powerpc/pseries: Enable support of ibm,dynamic-memory-v2 Add required bits to the architecture vector to enable support of the ibm,dynamic-memory-v2 device tree property. Signed-off-by: Nathan Fontenot Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/firmware.h | 3 ++- arch/powerpc/include/asm/prom.h | 1 + arch/powerpc/kernel/prom_init.c | 1 + arch/powerpc/platforms/pseries/firmware.c | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index 8645897472b1..832df61f30ef 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -51,6 +51,7 @@ #define FW_FEATURE_BEST_ENERGY ASM_CONST(0x0000000080000000) #define FW_FEATURE_TYPE1_AFFINITY ASM_CONST(0x0000000100000000) #define FW_FEATURE_PRRN ASM_CONST(0x0000000200000000) +#define FW_FEATURE_DRMEM_V2 ASM_CONST(0x0000000400000000) #ifndef __ASSEMBLY__ @@ -67,7 +68,7 @@ enum { FW_FEATURE_CMO | FW_FEATURE_VPHN | FW_FEATURE_XCMO | FW_FEATURE_SET_MODE | FW_FEATURE_BEST_ENERGY | FW_FEATURE_TYPE1_AFFINITY | FW_FEATURE_PRRN | - FW_FEATURE_HPT_RESIZE, + FW_FEATURE_HPT_RESIZE | FW_FEATURE_DRMEM_V2, FW_FEATURE_PSERIES_ALWAYS = 0, FW_FEATURE_POWERNV_POSSIBLE = FW_FEATURE_OPAL, FW_FEATURE_POWERNV_ALWAYS = 0, diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index f0a30a003bd8..9f27866e3126 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -143,6 +143,7 @@ extern int of_get_ibm_chip_id(struct device_node *np); #define OV5_PFO_HW_842 0x1140 /* PFO Compression Accelerator */ #define OV5_PFO_HW_ENCR 0x1120 /* PFO Encryption Accelerator */ #define OV5_SUB_PROCESSORS 0x1501 /* 1,2,or 4 Sub-Processors supported */ +#define OV5_DRMEM_V2 0x1680 /* ibm,dynamic-reconfiguration-v2 */ #define OV5_XIVE_SUPPORT 0x17C0 /* XIVE Exploitation Support Mask */ #define OV5_XIVE_LEGACY 0x1700 /* XIVE legacy mode Only */ #define OV5_XIVE_EXPLOIT 0x1740 /* XIVE exploitation mode Only */ diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 02190e90c7ae..acf4b2e0530c 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -869,6 +869,7 @@ struct ibm_arch_vec __cacheline_aligned ibm_architecture_vec = { .reserved2 = 0, .reserved3 = 0, .subprocessors = 1, + .byte22 = OV5_FEAT(OV5_DRMEM_V2), .intarch = 0, .mmu = 0, .hash_ext = 0, diff --git a/arch/powerpc/platforms/pseries/firmware.c b/arch/powerpc/platforms/pseries/firmware.c index 63cc82ad58ac..aac3ea2911b2 100644 --- a/arch/powerpc/platforms/pseries/firmware.c +++ b/arch/powerpc/platforms/pseries/firmware.c @@ -114,6 +114,7 @@ static __initdata struct vec5_fw_feature vec5_fw_features_table[] = { {FW_FEATURE_TYPE1_AFFINITY, OV5_TYPE1_AFFINITY}, {FW_FEATURE_PRRN, OV5_PRRN}, + {FW_FEATURE_DRMEM_V2, OV5_DRMEM_V2}, }; static void __init fw_vec5_feature_init(const char *vec5, unsigned long len) -- cgit v1.2.3 From 4145f358644b970fcff293c09fdcc7939e8527d2 Mon Sep 17 00:00:00 2001 From: Balbir Singh Date: Fri, 15 Dec 2017 19:14:55 +1100 Subject: powernv/kdump: Fix cases where the kdump kernel can get HMI's Certain HMI's such as malfunction error propagate through all threads/core on the system. If a thread was offline prior to us crashing the system and jumping to the kdump kernel, bad things happen when it wakes up due to an HMI in the kdump kernel. There are several possible ways to solve this problem 1. Put the offline cores in a state such that they are not woken up for machine check and HMI errors. This does not work, since we might need to wake up offline threads to handle TB errors 2. Ignore HMI errors, setup HMEER to mask HMI errors, but this still leads the window open for any MCEs and masking them for the duration of the dump might be a concern 3. Wake up offline CPUs, as in send them to crash_ipi_callback (not wake them up as in mark them online as seen by the hotplug). kexec does a wake_online_cpus() call, this patch does something similar, but instead sends an IPI and forces them to crash_ipi_callback() This patch takes approach #3. Care is taken to enable this only for powenv platforms via crash_wake_offline (a global value set at setup time). The crash code sends out IPI's to all CPU's which then move to crash_ipi_callback and kexec_smp_wait(). Signed-off-by: Balbir Singh Reviewed-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/kexec.h | 2 ++ arch/powerpc/kernel/crash.c | 13 ++++++++++++- arch/powerpc/kernel/smp.c | 18 ++++++++++++++++++ arch/powerpc/platforms/powernv/smp.c | 28 ++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/kexec.h b/arch/powerpc/include/asm/kexec.h index 4419d435639a..9dcbfa6bbb91 100644 --- a/arch/powerpc/include/asm/kexec.h +++ b/arch/powerpc/include/asm/kexec.h @@ -73,6 +73,8 @@ extern void kexec_smp_wait(void); /* get and clear naca physid, wait for master to copy new code to 0 */ extern int crashing_cpu; extern void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)); +extern void crash_ipi_callback(struct pt_regs *); +extern int crash_wake_offline; struct kimage; struct pt_regs; diff --git a/arch/powerpc/kernel/crash.c b/arch/powerpc/kernel/crash.c index 29c56ca2ddfd..00b215125d3e 100644 --- a/arch/powerpc/kernel/crash.c +++ b/arch/powerpc/kernel/crash.c @@ -44,6 +44,14 @@ #define REAL_MODE_TIMEOUT 10000 static int time_to_dump; +/* + * crash_wake_offline should be set to 1 by platforms that intend to wake + * up offline cpus prior to jumping to a kdump kernel. Currently powernv + * sets it to 1, since we want to avoid things from happening when an + * offline CPU wakes up due to something like an HMI (malfunction error), + * which propagates to all threads. + */ +int crash_wake_offline; #define CRASH_HANDLER_MAX 3 /* List of shutdown handles */ @@ -63,7 +71,7 @@ static int handle_fault(struct pt_regs *regs) #ifdef CONFIG_SMP static atomic_t cpus_in_crash; -static void crash_ipi_callback(struct pt_regs *regs) +void crash_ipi_callback(struct pt_regs *regs) { static cpumask_t cpus_state_saved = CPU_MASK_NONE; @@ -106,6 +114,9 @@ static void crash_kexec_prepare_cpus(int cpu) printk(KERN_EMERG "Sending IPI to other CPUs\n"); + if (crash_wake_offline) + ncpus = num_present_cpus() - 1; + crash_send_ipi(crash_ipi_callback); smp_wmb(); diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index e0a4c1f82e25..bbe7634b3a43 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -543,7 +543,25 @@ void smp_send_debugger_break(void) #ifdef CONFIG_KEXEC_CORE void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) { + int cpu; + smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000); + if (kdump_in_progress() && crash_wake_offline) { + for_each_present_cpu(cpu) { + if (cpu_online(cpu)) + continue; + /* + * crash_ipi_callback will wait for + * all cpus, including offline CPUs. + * We don't care about nmi_ipi_function. + * Offline cpus will jump straight into + * crash_ipi_callback, we can skip the + * entire NMI dance and waiting for + * cpus to clear pending mask, etc. + */ + do_smp_send_nmi_ipi(cpu); + } + } } #endif diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index ba030669eca1..9664c8461f03 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -37,6 +37,8 @@ #include #include #include +#include +#include #include "powernv.h" @@ -209,9 +211,32 @@ static void pnv_smp_cpu_kill_self(void) } else if ((srr1 & wmask) == SRR1_WAKEHDBELL) { unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); asm volatile(PPC_MSGCLR(%0) : : "r" (msg)); + } else if ((srr1 & wmask) == SRR1_WAKERESET) { + irq_set_pending_from_srr1(srr1); + /* Does not return */ } + smp_mb(); + /* + * For kdump kernels, we process the ipi and jump to + * crash_ipi_callback + */ + if (kdump_in_progress()) { + /* + * If we got to this point, we've not used + * NMI's, otherwise we would have gone + * via the SRR1_WAKERESET path. We are + * using regular IPI's for waking up offline + * threads. + */ + struct pt_regs regs; + + ppc_save_regs(®s); + crash_ipi_callback(®s); + /* Does not return */ + } + if (cpu_core_split_required()) continue; @@ -371,5 +396,8 @@ void __init pnv_smp_init(void) #ifdef CONFIG_HOTPLUG_CPU ppc_md.cpu_die = pnv_smp_cpu_kill_self; +#ifdef CONFIG_KEXEC_CORE + crash_wake_offline = 1; +#endif #endif } -- cgit v1.2.3 From 2a45addd21de25f41c8f21a6f08f0980e583d596 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Jan 2018 13:45:19 +0100 Subject: powerpc/8xx: Remove CPU6 ERRATA Workaround CPU6 ERRATA affects only MPC860 revisions prior to C.0. Manufacturing of those revisiosn was stopped in 1999-2000. Therefore, it has been almost 20 years since this ERRATA has been fixed in the silicon. This patch removes the workaround for that ERRATA. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/configs/mpc866_ads_defconfig | 1 - arch/powerpc/include/asm/reg_8xx.h | 82 ------------------------------- arch/powerpc/kernel/head_8xx.S | 54 +++++--------------- arch/powerpc/platforms/8xx/Kconfig | 12 ----- 4 files changed, 12 insertions(+), 137 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/configs/mpc866_ads_defconfig b/arch/powerpc/configs/mpc866_ads_defconfig index f1f176c29fa3..5320735395e7 100644 --- a/arch/powerpc/configs/mpc866_ads_defconfig +++ b/arch/powerpc/configs/mpc866_ads_defconfig @@ -13,7 +13,6 @@ CONFIG_EXPERT=y CONFIG_PARTITION_ADVANCED=y CONFIG_MPC86XADS=y CONFIG_8xx_COPYBACK=y -CONFIG_8xx_CPU6=y CONFIG_GEN_RTC=y CONFIG_HZ_1000=y CONFIG_MATH_EMULATION=y diff --git a/arch/powerpc/include/asm/reg_8xx.h b/arch/powerpc/include/asm/reg_8xx.h index 53a7e2955d3e..7192eece6c3e 100644 --- a/arch/powerpc/include/asm/reg_8xx.h +++ b/arch/powerpc/include/asm/reg_8xx.h @@ -66,86 +66,4 @@ #define DC_DFWT 0x40000000 /* Data cache is forced write through */ #define DC_LES 0x20000000 /* Caches are little endian mode */ -#ifdef CONFIG_8xx_CPU6 -#define do_mtspr_cpu6(rn, rn_addr, v) \ - do { \ - int _reg_cpu6 = rn_addr, _tmp_cpu6; \ - asm volatile("stw %0, %1;" \ - "lwz %0, %1;" \ - "mtspr " __stringify(rn) ",%2" : \ - : "r" (_reg_cpu6), "m"(_tmp_cpu6), \ - "r" ((unsigned long)(v)) \ - : "memory"); \ - } while (0) - -#define do_mtspr(rn, v) asm volatile("mtspr " __stringify(rn) ",%0" : \ - : "r" ((unsigned long)(v)) \ - : "memory") -#define mtspr(rn, v) \ - do { \ - if (rn == SPRN_IMMR) \ - do_mtspr_cpu6(rn, 0x3d30, v); \ - else if (rn == SPRN_IC_CST) \ - do_mtspr_cpu6(rn, 0x2110, v); \ - else if (rn == SPRN_IC_ADR) \ - do_mtspr_cpu6(rn, 0x2310, v); \ - else if (rn == SPRN_IC_DAT) \ - do_mtspr_cpu6(rn, 0x2510, v); \ - else if (rn == SPRN_DC_CST) \ - do_mtspr_cpu6(rn, 0x3110, v); \ - else if (rn == SPRN_DC_ADR) \ - do_mtspr_cpu6(rn, 0x3310, v); \ - else if (rn == SPRN_DC_DAT) \ - do_mtspr_cpu6(rn, 0x3510, v); \ - else if (rn == SPRN_MI_CTR) \ - do_mtspr_cpu6(rn, 0x2180, v); \ - else if (rn == SPRN_MI_AP) \ - do_mtspr_cpu6(rn, 0x2580, v); \ - else if (rn == SPRN_MI_EPN) \ - do_mtspr_cpu6(rn, 0x2780, v); \ - else if (rn == SPRN_MI_TWC) \ - do_mtspr_cpu6(rn, 0x2b80, v); \ - else if (rn == SPRN_MI_RPN) \ - do_mtspr_cpu6(rn, 0x2d80, v); \ - else if (rn == SPRN_MI_CAM) \ - do_mtspr_cpu6(rn, 0x2190, v); \ - else if (rn == SPRN_MI_RAM0) \ - do_mtspr_cpu6(rn, 0x2390, v); \ - else if (rn == SPRN_MI_RAM1) \ - do_mtspr_cpu6(rn, 0x2590, v); \ - else if (rn == SPRN_MD_CTR) \ - do_mtspr_cpu6(rn, 0x3180, v); \ - else if (rn == SPRN_M_CASID) \ - do_mtspr_cpu6(rn, 0x3380, v); \ - else if (rn == SPRN_MD_AP) \ - do_mtspr_cpu6(rn, 0x3580, v); \ - else if (rn == SPRN_MD_EPN) \ - do_mtspr_cpu6(rn, 0x3780, v); \ - else if (rn == SPRN_M_TWB) \ - do_mtspr_cpu6(rn, 0x3980, v); \ - else if (rn == SPRN_MD_TWC) \ - do_mtspr_cpu6(rn, 0x3b80, v); \ - else if (rn == SPRN_MD_RPN) \ - do_mtspr_cpu6(rn, 0x3d80, v); \ - else if (rn == SPRN_M_TW) \ - do_mtspr_cpu6(rn, 0x3f80, v); \ - else if (rn == SPRN_MD_CAM) \ - do_mtspr_cpu6(rn, 0x3190, v); \ - else if (rn == SPRN_MD_RAM0) \ - do_mtspr_cpu6(rn, 0x3390, v); \ - else if (rn == SPRN_MD_RAM1) \ - do_mtspr_cpu6(rn, 0x3590, v); \ - else if (rn == SPRN_DEC) \ - do_mtspr_cpu6(rn, 0x2c00, v); \ - else if (rn == SPRN_TBWL) \ - do_mtspr_cpu6(rn, 0x3880, v); \ - else if (rn == SPRN_TBWU) \ - do_mtspr_cpu6(rn, 0x3a80, v); \ - else if (rn == SPRN_DPDR) \ - do_mtspr_cpu6(rn, 0x2d30, v); \ - else \ - do_mtspr(rn, v); \ - } while (0) -#endif - #endif /* _ASM_POWERPC_REG_8xx_H */ diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 4fee00d414e8..728b513c07b8 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -33,23 +33,6 @@ #include #include -/* Macro to make the code more readable. */ -#ifdef CONFIG_8xx_CPU6 -#define SPRN_MI_TWC_ADDR 0x2b80 -#define SPRN_MI_RPN_ADDR 0x2d80 -#define SPRN_MD_TWC_ADDR 0x3b80 -#define SPRN_MD_RPN_ADDR 0x3d80 - -#define MTSPR_CPU6(spr, reg, treg) \ - li treg, spr##_ADDR; \ - stw treg, 12(r0); \ - lwz treg, 12(r0); \ - mtspr spr, reg -#else -#define MTSPR_CPU6(spr, reg, treg) \ - mtspr spr, reg -#endif - #if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000 /* By simply checking Address >= 0x80000000, we know if its a kernel address */ #define SIMPLE_KERNEL_ADDRESS 1 @@ -326,7 +309,7 @@ SystemCall: #endif InstructionTLBMiss: -#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) +#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) mtspr SPRN_SPRG_SCRATCH2, r3 #endif EXCEPTION_PROLOG_0 @@ -393,7 +376,7 @@ _ENTRY(ITLBMiss_cmp) /* Insert the APG into the TWC from the Linux PTE. */ rlwimi r11, r10, 0, 25, 26 /* Load the MI_TWC with the attributes for this "segment." */ - MTSPR_CPU6(SPRN_MI_TWC, r11, r3) /* Set segment attributes */ + mtspr SPRN_MI_TWC, r11 /* Set segment attributes */ #if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES) rlwimi r10, r11, 1, MI_SPS16K @@ -415,10 +398,10 @@ _ENTRY(ITLBMiss_cmp) #else rlwimi r10, r11, 0, 0x0ff8 /* Set 24-27, clear 20-23,28 */ #endif - MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */ + mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ /* Restore registers */ -#if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) +#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) mfspr r3, SPRN_SPRG_SCRATCH2 #endif EXCEPTION_EPILOG_0 @@ -512,7 +495,7 @@ _ENTRY(DTLBMiss_jmp) * It is bit 25 in the Linux PTE and bit 30 in the TWC */ rlwimi r11, r10, 32-5, 30, 30 - MTSPR_CPU6(SPRN_MD_TWC, r11, r3) + mtspr SPRN_MD_TWC, r11 /* In 4k pages mode, SPS (bit 28) in RPN must match PS[1] (bit 29) * In 16k pages mode, SPS is always 1 */ @@ -546,7 +529,7 @@ _ENTRY(DTLBMiss_jmp) rlwimi r10, r11, 0, 24, 28 /* Set 24-27, clear 28 */ #endif rlwimi r10, r11, 0, 20, 20 /* clear 20 */ - MTSPR_CPU6(SPRN_MD_RPN, r10, r3) /* Update TLB entry */ + mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ /* Restore registers */ mfspr r3, SPRN_SPRG_SCRATCH2 @@ -684,12 +667,12 @@ DTLBMissIMMR: mtcr r3 /* Set 512k byte guarded page and mark it valid */ li r10, MD_PS512K | MD_GUARDED | MD_SVALID - MTSPR_CPU6(SPRN_MD_TWC, r10, r11) + mtspr SPRN_MD_TWC, r10 mfspr r10, SPRN_IMMR /* Get current IMMR */ rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */ ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \ _PAGE_PRESENT | _PAGE_NO_CACHE - MTSPR_CPU6(SPRN_MD_RPN, r10, r11) /* Update TLB entry */ + mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ li r11, RPN_PATTERN mtspr SPRN_DAR, r11 /* Tag DAR */ @@ -701,11 +684,11 @@ DTLBMissLinear: mtcr r3 /* Set 8M byte page and mark it valid */ li r11, MD_PS8MEG | MD_SVALID - MTSPR_CPU6(SPRN_MD_TWC, r11, r3) + mtspr SPRN_MD_TWC, r11 rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \ _PAGE_PRESENT - MTSPR_CPU6(SPRN_MD_RPN, r10, r11) /* Update TLB entry */ + mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ li r11, RPN_PATTERN mtspr SPRN_DAR, r11 /* Tag DAR */ @@ -718,11 +701,11 @@ ITLBMissLinear: mtcr r3 /* Set 8M byte page and mark it valid */ li r11, MI_PS8MEG | MI_SVALID | _PAGE_EXEC - MTSPR_CPU6(SPRN_MI_TWC, r11, r3) + mtspr SPRN_MI_TWC, r11 rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \ _PAGE_PRESENT - MTSPR_CPU6(SPRN_MI_RPN, r10, r11) /* Update TLB entry */ + mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ mfspr r3, SPRN_SPRG_SCRATCH2 EXCEPTION_EPILOG_0 @@ -933,13 +916,6 @@ start_here: */ lis r6, swapper_pg_dir@ha tophys(r6,r6) -#ifdef CONFIG_8xx_CPU6 - lis r4, cpu6_errata_word@h - ori r4, r4, cpu6_errata_word@l - li r3, 0x3f80 - stw r3, 12(r4) - lwz r3, 12(r4) -#endif mtspr SPRN_M_TW, r6 lis r4,2f@h ori r4,r4,2f@l @@ -1094,12 +1070,6 @@ swapper_pg_dir: abatron_pteptrs: .space 8 -#ifdef CONFIG_8xx_CPU6 - .globl cpu6_errata_word -cpu6_errata_word: - .space 16 -#endif - #ifdef CONFIG_PPC_8xx_PERF_EVENT .globl itlb_miss_counter itlb_miss_counter: diff --git a/arch/powerpc/platforms/8xx/Kconfig b/arch/powerpc/platforms/8xx/Kconfig index e2089d3de00c..d408162d5af4 100644 --- a/arch/powerpc/platforms/8xx/Kconfig +++ b/arch/powerpc/platforms/8xx/Kconfig @@ -116,18 +116,6 @@ config 8xx_GPIO If in doubt, say Y here. -config 8xx_CPU6 - bool "CPU6 Silicon Errata (860 Pre Rev. C)" - help - MPC860 CPUs, prior to Rev C have some bugs in the silicon, which - require workarounds for Linux (and most other OSes to work). If you - get a BUG() very early in boot, this might fix the problem. For - more details read the document entitled "MPC860 Family Device Errata - Reference" on Freescale's website. This option also incurs a - performance hit. - - If in doubt, say N here. - config 8xx_CPU15 bool "CPU15 Silicon Errata" depends on !HUGETLB_PAGE -- cgit v1.2.3 From cd99ddbea250ee79027df6c469f51ad9e5452738 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Jan 2018 13:45:23 +0100 Subject: powerpc/8xx: Only perform perf counting when perf is in use. In TLB miss handlers, updating the perf counter is only useful when performing a perf analysis. As it has a noticeable overhead, let's only do it when needed. In order to do so, the exit of the miss handlers will be patched when starting/stopping 'perf': the first register restore instruction of each exit point will be replaced by a jump to the counting code. Once this is done, CONFIG_PPC_8xx_PERF_EVENT becomes useless as this feature doesn't add any overhead. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/ppc-opcode.h | 2 ++ arch/powerpc/kernel/entry_32.S | 10 +++---- arch/powerpc/kernel/head_8xx.S | 47 ++++++++++++++++++++---------- arch/powerpc/perf/8xx-pmu.c | 52 +++++++++++++++++++++++++++++++--- arch/powerpc/perf/Makefile | 2 +- arch/powerpc/platforms/Kconfig.cputype | 7 ----- 6 files changed, 88 insertions(+), 32 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index ce0930d68857..ab5c1588b487 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -236,6 +236,7 @@ #define PPC_INST_RFCI 0x4c000066 #define PPC_INST_RFDI 0x4c00004e #define PPC_INST_RFMCI 0x4c00004c +#define PPC_INST_MFSPR 0x7c0002a6 #define PPC_INST_MFSPR_DSCR 0x7c1102a6 #define PPC_INST_MFSPR_DSCR_MASK 0xfc1ffffe #define PPC_INST_MTSPR_DSCR 0x7c1103a6 @@ -383,6 +384,7 @@ #define __PPC_ME64(s) __PPC_MB64(s) #define __PPC_BI(s) (((s) & 0x1f) << 16) #define __PPC_CT(t) (((t) & 0x0f) << 21) +#define __PPC_SPR(r) ((((r) & 0x1f) << 16) | ((((r) >> 5) & 0x1f) << 11)) /* * Only use the larx hint bit on 64bit CPUs. e500v1/v2 based CPUs will treat a diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index e780e1fbf6c2..eb8d01bae8c6 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -211,7 +211,7 @@ transfer_to_handler_cont: mflr r9 lwz r11,0(r9) /* virtual address of handler */ lwz r9,4(r9) /* where to go when done */ -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) mtspr SPRN_NRI, r0 #endif #ifdef CONFIG_TRACE_IRQFLAGS @@ -301,7 +301,7 @@ stack_ovf: lis r9,StackOverflow@ha addi r9,r9,StackOverflow@l LOAD_MSR_KERNEL(r10,MSR_KERNEL) -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) mtspr SPRN_NRI, r0 #endif mtspr SPRN_SRR0,r9 @@ -430,7 +430,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) lwz r7,_NIP(r1) lwz r2,GPR2(r1) lwz r1,GPR1(r1) -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) mtspr SPRN_NRI, r0 #endif mtspr SPRN_SRR0,r7 @@ -727,7 +727,7 @@ fast_exception_return: lwz r10,_LINK(r11) mtlr r10 REST_GPR(10, r11) -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) mtspr SPRN_NRI, r0 #endif mtspr SPRN_SRR1,r9 @@ -978,7 +978,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX) .globl exc_exit_restart exc_exit_restart: lwz r12,_NIP(r1) -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PERF_EVENTS) mtspr SPRN_NRI, r0 #endif mtspr SPRN_SRR0,r12 diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index eda582b96dbf..641c9a9d4db2 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -304,12 +304,6 @@ InstructionTLBMiss: #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) mtspr SPRN_SPRG_SCRATCH2, r12 #endif -#ifdef CONFIG_PPC_8xx_PERF_EVENT - lis r10, (itlb_miss_counter - PAGE_OFFSET)@ha - lwz r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10) - addi r11, r11, 1 - stw r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10) -#endif /* If we are faulting a kernel address, we have to use the * kernel page tables. @@ -392,6 +386,20 @@ _ENTRY(ITLBMiss_cmp) mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ /* Restore registers */ +_ENTRY(itlb_miss_exit_1) + mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH1 +#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) + mfspr r12, SPRN_SPRG_SCRATCH2 +#endif + rfi +#ifdef CONFIG_PERF_EVENTS +_ENTRY(itlb_miss_perf) + lis r10, (itlb_miss_counter - PAGE_OFFSET)@ha + lwz r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10) + addi r11, r11, 1 + stw r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10) +#endif mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1 #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) @@ -429,12 +437,6 @@ DataStoreTLBMiss: mtspr SPRN_SPRG_SCRATCH0, r10 mtspr SPRN_SPRG_SCRATCH1, r11 mtspr SPRN_SPRG_SCRATCH2, r12 -#ifdef CONFIG_PPC_8xx_PERF_EVENT - lis r10, (dtlb_miss_counter - PAGE_OFFSET)@ha - lwz r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10) - addi r11, r11, 1 - stw r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10) -#endif mfcr r12 /* If we are faulting a kernel address, we have to use the @@ -526,6 +528,18 @@ _ENTRY(DTLBMiss_jmp) /* Restore registers */ mtspr SPRN_DAR, r11 /* Tag DAR */ +_ENTRY(dtlb_miss_exit_1) + mfspr r10, SPRN_SPRG_SCRATCH0 + mfspr r11, SPRN_SPRG_SCRATCH1 + mfspr r12, SPRN_SPRG_SCRATCH2 + rfi +#ifdef CONFIG_PERF_EVENTS +_ENTRY(dtlb_miss_perf) + lis r10, (dtlb_miss_counter - PAGE_OFFSET)@ha + lwz r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10) + addi r11, r11, 1 + stw r11, (dtlb_miss_counter - PAGE_OFFSET)@l(r10) +#endif mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1 mfspr r12, SPRN_SPRG_SCRATCH2 @@ -635,7 +649,7 @@ DataBreakpoint: mfspr r11, SPRN_SPRG_SCRATCH1 rfi -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#ifdef CONFIG_PERF_EVENTS . = 0x1d00 InstructionBreakpoint: mtspr SPRN_SPRG_SCRATCH0, r10 @@ -675,6 +689,7 @@ DTLBMissIMMR: li r11, RPN_PATTERN mtspr SPRN_DAR, r11 /* Tag DAR */ +_ENTRY(dtlb_miss_exit_2) mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1 mfspr r12, SPRN_SPRG_SCRATCH2 @@ -692,6 +707,7 @@ DTLBMissLinear: li r11, RPN_PATTERN mtspr SPRN_DAR, r11 /* Tag DAR */ +_ENTRY(dtlb_miss_exit_3) mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1 mfspr r12, SPRN_SPRG_SCRATCH2 @@ -708,6 +724,7 @@ ITLBMissLinear: _PAGE_PRESENT mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ +_ENTRY(itlb_miss_exit_2) mfspr r10, SPRN_SPRG_SCRATCH0 mfspr r11, SPRN_SPRG_SCRATCH1 mfspr r12, SPRN_SPRG_SCRATCH2 @@ -1039,7 +1056,7 @@ initial_mmu: #endif /* Disable debug mode entry on breakpoints */ mfspr r8, SPRN_DER -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#ifdef CONFIG_PERF_EVENTS rlwinm r8, r8, 0, ~0xc #else rlwinm r8, r8, 0, ~0x8 @@ -1072,7 +1089,7 @@ swapper_pg_dir: abatron_pteptrs: .space 8 -#ifdef CONFIG_PPC_8xx_PERF_EVENT +#ifdef CONFIG_PERF_EVENTS .globl itlb_miss_counter itlb_miss_counter: .space 4 diff --git a/arch/powerpc/perf/8xx-pmu.c b/arch/powerpc/perf/8xx-pmu.c index 3c39f05f0af3..6c0020d1c561 100644 --- a/arch/powerpc/perf/8xx-pmu.c +++ b/arch/powerpc/perf/8xx-pmu.c @@ -18,6 +18,7 @@ #include #include #include +#include #define PERF_8xx_ID_CPU_CYCLES 1 #define PERF_8xx_ID_HW_INSTRUCTIONS 2 @@ -30,8 +31,13 @@ extern unsigned long itlb_miss_counter, dtlb_miss_counter; extern atomic_t instruction_counter; +extern unsigned int itlb_miss_perf, dtlb_miss_perf; +extern unsigned int itlb_miss_exit_1, itlb_miss_exit_2; +extern unsigned int dtlb_miss_exit_1, dtlb_miss_exit_2, dtlb_miss_exit_3; static atomic_t insn_ctr_ref; +static atomic_t itlb_miss_ref; +static atomic_t dtlb_miss_ref; static s64 get_insn_ctr(void) { @@ -96,9 +102,24 @@ static int mpc8xx_pmu_add(struct perf_event *event, int flags) val = get_insn_ctr(); break; case PERF_8xx_ID_ITLB_LOAD_MISS: + if (atomic_inc_return(&itlb_miss_ref) == 1) { + unsigned long target = (unsigned long)&itlb_miss_perf; + + patch_branch(&itlb_miss_exit_1, target, 0); +#ifndef CONFIG_PIN_TLB_TEXT + patch_branch(&itlb_miss_exit_2, target, 0); +#endif + } val = itlb_miss_counter; break; case PERF_8xx_ID_DTLB_LOAD_MISS: + if (atomic_inc_return(&dtlb_miss_ref) == 1) { + unsigned long target = (unsigned long)&dtlb_miss_perf; + + patch_branch(&dtlb_miss_exit_1, target, 0); + patch_branch(&dtlb_miss_exit_2, target, 0); + patch_branch(&dtlb_miss_exit_3, target, 0); + } val = dtlb_miss_counter; break; } @@ -143,13 +164,36 @@ static void mpc8xx_pmu_read(struct perf_event *event) static void mpc8xx_pmu_del(struct perf_event *event, int flags) { + /* mfspr r10, SPRN_SPRG_SCRATCH0 */ + unsigned int insn = PPC_INST_MFSPR | __PPC_RS(R10) | + __PPC_SPR(SPRN_SPRG_SCRATCH0); + mpc8xx_pmu_read(event); - if (event_type(event) != PERF_8xx_ID_HW_INSTRUCTIONS) - return; /* If it was the last user, stop counting to avoid useles overhead */ - if (atomic_dec_return(&insn_ctr_ref) == 0) - mtspr(SPRN_ICTRL, 7); + switch (event_type(event)) { + case PERF_8xx_ID_CPU_CYCLES: + break; + case PERF_8xx_ID_HW_INSTRUCTIONS: + if (atomic_dec_return(&insn_ctr_ref) == 0) + mtspr(SPRN_ICTRL, 7); + break; + case PERF_8xx_ID_ITLB_LOAD_MISS: + if (atomic_dec_return(&itlb_miss_ref) == 0) { + patch_instruction(&itlb_miss_exit_1, insn); +#ifndef CONFIG_PIN_TLB_TEXT + patch_instruction(&itlb_miss_exit_2, insn); +#endif + } + break; + case PERF_8xx_ID_DTLB_LOAD_MISS: + if (atomic_dec_return(&dtlb_miss_ref) == 0) { + patch_instruction(&dtlb_miss_exit_1, insn); + patch_instruction(&dtlb_miss_exit_2, insn); + patch_instruction(&dtlb_miss_exit_3, insn); + } + break; + } } static struct pmu mpc8xx_pmu = { diff --git a/arch/powerpc/perf/Makefile b/arch/powerpc/perf/Makefile index 225c9c86d7c0..57ebc655d2ac 100644 --- a/arch/powerpc/perf/Makefile +++ b/arch/powerpc/perf/Makefile @@ -15,7 +15,7 @@ obj-$(CONFIG_FSL_EMB_PERF_EVENT_E500) += e500-pmu.o e6500-pmu.o obj-$(CONFIG_HV_PERF_CTRS) += hv-24x7.o hv-gpci.o hv-common.o -obj-$(CONFIG_PPC_8xx_PERF_EVENT) += 8xx-pmu.o +obj-$(CONFIG_PPC_8xx) += 8xx-pmu.o obj-$(CONFIG_PPC64) += $(obj64-y) obj-$(CONFIG_PPC32) += $(obj32-y) diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index c838581f81f1..a429d859f15d 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -167,13 +167,6 @@ config PPC_FPU bool default y if PPC64 -config PPC_8xx_PERF_EVENT - bool "PPC 8xx perf events" - depends on PPC_8xx && PERF_EVENTS - help - This is Performance Events support for PPC 8xx. The 8xx doesn't - have a PMU but some events are emulated using 8xx features. - config FSL_EMB_PERFMON bool "Freescale Embedded Perfmon" depends on E500 || PPC_83xx -- cgit v1.2.3 From 5f356497c38448874f2d491cd214e9c283f5592d Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Jan 2018 13:45:25 +0100 Subject: powerpc/8xx: remove unused _PAGE_WRITETHRU _PAGE_WRITETHRU is only used in: * AMIGA_Z2RAM block driver which is never activated on powerPC * Video/FB driver which is for PPC_PMAC Therefore, no need to spend time in 8xx TLB miss handlers for handling it. And by removing it, we free up bit 20 which then avoids having to clear it on each TLB miss. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 3 +-- arch/powerpc/include/asm/nohash/pgtable.h | 2 ++ arch/powerpc/kernel/head_8xx.S | 5 ----- 3 files changed, 3 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index 6dc0180fd5c7..19a5ecaef265 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -41,8 +41,7 @@ #define _PAGE_GUARDED 0x0010 /* Copied to L1 G entry in DTLB */ #define _PAGE_USER 0x0020 /* Copied to L1 APG lsb */ #define _PAGE_EXEC 0x0040 /* Copied to L1 APG */ -#define _PAGE_WRITETHRU 0x0080 /* software: caching is write through */ -#define _PAGE_ACCESSED 0x0800 /* software: page referenced */ +#define _PAGE_ACCESSED 0x0080 /* software: page referenced */ #define _PAGE_RO 0x0600 /* Supervisor RO, User no access */ diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index fc4376c8d444..e0864e683cf9 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -235,8 +235,10 @@ extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addre #define pgprot_cached(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ _PAGE_COHERENT)) +#if _PAGE_WRITETHRU != 0 #define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \ _PAGE_COHERENT | _PAGE_WRITETHRU)) +#endif #define pgprot_cached_noncoherent(prot) \ (__pgprot(pgprot_val(prot) & ~_PAGE_CACHE_CTL)) diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 641c9a9d4db2..6399dcadf51d 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -486,10 +486,6 @@ _ENTRY(DTLBMiss_jmp) * above. */ rlwimi r11, r10, 0, 26, 27 - /* Insert the WriteThru flag into the TWC from the Linux PTE. - * It is bit 25 in the Linux PTE and bit 30 in the TWC - */ - rlwimi r11, r10, 32-5, 30, 30 mtspr SPRN_MD_TWC, r11 /* In 4k pages mode, SPS (bit 28) in RPN must match PS[1] (bit 29) @@ -523,7 +519,6 @@ _ENTRY(DTLBMiss_jmp) #else rlwimi r10, r11, 0, 24, 28 /* Set 24-27, clear 28 */ #endif - rlwimi r10, r11, 0, 20, 20 /* clear 20 */ mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ /* Restore registers */ -- cgit v1.2.3 From 812fadcb941a81d1f3948b10a95a4dce663da3e4 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Jan 2018 13:45:27 +0100 Subject: powerpc/mm: extend _PAGE_PRIVILEGED to all CPUs commit ac29c64089b74 ("powerpc/mm: Replace _PAGE_USER with _PAGE_PRIVILEGED") introduced _PAGE_PRIVILEGED for BOOK3S/64 This patch generalises _PAGE_PRIVILEGED for all CPUs, allowing to have either _PAGE_PRIVILEGED or _PAGE_USER or both. PPC_8xx has a _PAGE_SHARED flag which is set for and only for all non user pages. Lets rename it _PAGE_PRIVILEGED to remove confusion as it has nothing to do with Linux shared pages. On BookE, there's a _PAGE_BAP_SR which has to be set for kernel pages: defining _PAGE_PRIVILEGED as _PAGE_BAP_SR will make this generic Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 +- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 10 +--------- arch/powerpc/include/asm/nohash/pte-book3e.h | 1 + arch/powerpc/include/asm/pte-common.h | 24 ++++++++++++++++-------- arch/powerpc/kernel/head_8xx.S | 6 +++--- arch/powerpc/mm/8xx_mmu.c | 2 +- arch/powerpc/mm/dump_linuxpagetables.c | 11 +---------- arch/powerpc/mm/pgtable.c | 3 ++- arch/powerpc/mm/pgtable_32.c | 9 +-------- arch/powerpc/mm/pgtable_64.c | 14 +------------- 10 files changed, 28 insertions(+), 54 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 23b03449e2f1..8e28febedac3 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -15,7 +15,7 @@ #define _PAGE_BIT_SWAP_TYPE 0 #define _PAGE_RO 0 -#define _PAGE_SHARED 0 +#define _PAGE_USER 0 #define _PAGE_EXEC 0x00001 /* execute permission */ #define _PAGE_WRITE 0x00002 /* write access allowed */ diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index 19a5ecaef265..7c7040f015e2 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -31,7 +31,7 @@ /* Definitions for 8xx embedded chips. */ #define _PAGE_PRESENT 0x0001 /* Page is valid */ #define _PAGE_NO_CACHE 0x0002 /* I: cache inhibit */ -#define _PAGE_SHARED 0x0004 /* No ASID (context) compare */ +#define _PAGE_PRIVILEGED 0x0004 /* No ASID (context) compare */ #define _PAGE_SPECIAL 0x0008 /* SW entry, forced to 0 by the TLB miss */ #define _PAGE_DIRTY 0x0100 /* C: page changed */ @@ -54,13 +54,5 @@ /* Until my rework is finished, 8xx still needs atomic PTE updates */ #define PTE_ATOMIC_UPDATES 1 -/* We need to add _PAGE_SHARED to kernel pages */ -#define _PAGE_KERNEL_RO (_PAGE_SHARED | _PAGE_RO) -#define _PAGE_KERNEL_ROX (_PAGE_SHARED | _PAGE_RO | _PAGE_EXEC) -#define _PAGE_KERNEL_RW (_PAGE_SHARED | _PAGE_DIRTY | _PAGE_RW | \ - _PAGE_HWWRITE) -#define _PAGE_KERNEL_RWX (_PAGE_SHARED | _PAGE_DIRTY | _PAGE_RW | \ - _PAGE_HWWRITE | _PAGE_EXEC) - #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_NOHASH_32_PTE_8xx_H */ diff --git a/arch/powerpc/include/asm/nohash/pte-book3e.h b/arch/powerpc/include/asm/nohash/pte-book3e.h index 2da4532ca377..ccee8eb509bb 100644 --- a/arch/powerpc/include/asm/nohash/pte-book3e.h +++ b/arch/powerpc/include/asm/nohash/pte-book3e.h @@ -55,6 +55,7 @@ #define _PAGE_KERNEL_RWX (_PAGE_BAP_SW | _PAGE_BAP_SR | _PAGE_DIRTY | _PAGE_BAP_SX) #define _PAGE_KERNEL_ROX (_PAGE_BAP_SR | _PAGE_BAP_SX) #define _PAGE_USER (_PAGE_BAP_UR | _PAGE_BAP_SR) /* Can be read */ +#define _PAGE_PRIVILEGED (_PAGE_BAP_SR) #define _PAGE_HASHPTE 0 #define _PAGE_BUSY 0 diff --git a/arch/powerpc/include/asm/pte-common.h b/arch/powerpc/include/asm/pte-common.h index ce142ef99ba7..0e6595a1b9d8 100644 --- a/arch/powerpc/include/asm/pte-common.h +++ b/arch/powerpc/include/asm/pte-common.h @@ -8,9 +8,6 @@ #ifndef _PAGE_HASHPTE #define _PAGE_HASHPTE 0 #endif -#ifndef _PAGE_SHARED -#define _PAGE_SHARED 0 -#endif #ifndef _PAGE_HWWRITE #define _PAGE_HWWRITE 0 #endif @@ -45,6 +42,14 @@ #ifndef _PAGE_PTE #define _PAGE_PTE 0 #endif +/* At least one of _PAGE_PRIVILEGED or _PAGE_USER must be defined */ +#ifndef _PAGE_PRIVILEGED +#define _PAGE_PRIVILEGED 0 +#else +#ifndef _PAGE_USER +#define _PAGE_USER 0 +#endif +#endif #ifndef _PMD_PRESENT_MASK #define _PMD_PRESENT_MASK _PMD_PRESENT @@ -54,16 +59,18 @@ #define PMD_PAGE_SIZE(pmd) bad_call_to_PMD_PAGE_SIZE() #endif #ifndef _PAGE_KERNEL_RO -#define _PAGE_KERNEL_RO (_PAGE_RO) +#define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_RO) #endif #ifndef _PAGE_KERNEL_ROX -#define _PAGE_KERNEL_ROX (_PAGE_EXEC | _PAGE_RO) +#define _PAGE_KERNEL_ROX (_PAGE_PRIVILEGED | _PAGE_RO | _PAGE_EXEC) #endif #ifndef _PAGE_KERNEL_RW -#define _PAGE_KERNEL_RW (_PAGE_DIRTY | _PAGE_RW | _PAGE_HWWRITE) +#define _PAGE_KERNEL_RW (_PAGE_PRIVILEGED | _PAGE_DIRTY | _PAGE_RW | \ + _PAGE_HWWRITE) #endif #ifndef _PAGE_KERNEL_RWX -#define _PAGE_KERNEL_RWX (_PAGE_DIRTY | _PAGE_RW | _PAGE_HWWRITE | _PAGE_EXEC) +#define _PAGE_KERNEL_RWX (_PAGE_PRIVILEGED | _PAGE_DIRTY | _PAGE_RW | \ + _PAGE_HWWRITE | _PAGE_EXEC) #endif #ifndef _PAGE_HPTEFLAGS #define _PAGE_HPTEFLAGS _PAGE_HASHPTE @@ -85,7 +92,7 @@ extern unsigned long bad_call_to_PMD_PAGE_SIZE(void); */ static inline bool pte_user(pte_t pte) { - return (pte_val(pte) & _PAGE_USER) == _PAGE_USER; + return (pte_val(pte) & (_PAGE_USER | _PAGE_PRIVILEGED)) == _PAGE_USER; } #endif /* __ASSEMBLY__ */ @@ -116,6 +123,7 @@ static inline bool pte_user(pte_t pte) #define PAGE_PROT_BITS (_PAGE_GUARDED | _PAGE_COHERENT | _PAGE_NO_CACHE | \ _PAGE_WRITETHRU | _PAGE_ENDIAN | _PAGE_4K_PFN | \ _PAGE_USER | _PAGE_ACCESSED | _PAGE_RO | \ + _PAGE_PRIVILEGED | \ _PAGE_RW | _PAGE_HWWRITE | _PAGE_DIRTY | _PAGE_EXEC) /* diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 6399dcadf51d..642680389b7e 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -678,7 +678,7 @@ DTLBMissIMMR: mtspr SPRN_MD_TWC, r10 mfspr r10, SPRN_IMMR /* Get current IMMR */ rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */ - ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \ + ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \ _PAGE_PRESENT | _PAGE_NO_CACHE mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ @@ -696,7 +696,7 @@ DTLBMissLinear: li r11, MD_PS8MEG | MD_SVALID mtspr SPRN_MD_TWC, r11 rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ - ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \ + ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \ _PAGE_PRESENT mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ @@ -715,7 +715,7 @@ ITLBMissLinear: li r11, MI_PS8MEG | MI_SVALID | _PAGE_EXEC mtspr SPRN_MI_TWC, r11 rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ - ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \ + ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \ _PAGE_PRESENT mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c index f29212e40f40..c2d9a6d709c3 100644 --- a/arch/powerpc/mm/8xx_mmu.c +++ b/arch/powerpc/mm/8xx_mmu.c @@ -67,7 +67,7 @@ void __init MMU_init_hw(void) /* PIN up to the 3 first 8Mb after IMMR in DTLB table */ #ifdef CONFIG_PIN_TLB_DATA unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000; - unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY; + unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY; #ifdef CONFIG_PIN_TLB_IMMR int i = 29; #else diff --git a/arch/powerpc/mm/dump_linuxpagetables.c b/arch/powerpc/mm/dump_linuxpagetables.c index 139dd1993eeb..d87ab2cabaa6 100644 --- a/arch/powerpc/mm/dump_linuxpagetables.c +++ b/arch/powerpc/mm/dump_linuxpagetables.c @@ -112,13 +112,8 @@ struct flag_info { static const struct flag_info flag_array[] = { { -#ifdef CONFIG_PPC_BOOK3S_64 - .mask = _PAGE_PRIVILEGED, - .val = 0, -#else - .mask = _PAGE_USER, + .mask = _PAGE_USER | _PAGE_PRIVILEGED, .val = _PAGE_USER, -#endif .set = "user", .clear = " ", }, { @@ -229,10 +224,6 @@ static const struct flag_info flag_array[] = { .mask = _PAGE_SPECIAL, .val = _PAGE_SPECIAL, .set = "special", - }, { - .mask = _PAGE_SHARED, - .val = _PAGE_SHARED, - .set = "shared", } }; diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index a03ff3d99e0c..9f361ae571e9 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -54,7 +54,8 @@ static inline int pte_looks_normal(pte_t pte) return 0; #else return (pte_val(pte) & - (_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE | _PAGE_USER)) == + (_PAGE_PRESENT | _PAGE_SPECIAL | _PAGE_NO_CACHE | _PAGE_USER | + _PAGE_PRIVILEGED)) == (_PAGE_PRESENT | _PAGE_USER); #endif } diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index f6c7f54c0515..d35d9ad3c1cd 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -98,14 +98,7 @@ ioremap_prot(phys_addr_t addr, unsigned long size, unsigned long flags) /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */ flags &= ~(_PAGE_USER | _PAGE_EXEC); - -#ifdef _PAGE_BAP_SR - /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format - * which means that we just cleared supervisor access... oops ;-) This - * restores it - */ - flags |= _PAGE_BAP_SR; -#endif + flags |= _PAGE_PRIVILEGED; return __ioremap_caller(addr, size, flags, __builtin_return_address(0)); } diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 813ea22c3e00..c9a623c2d8a2 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -244,20 +244,8 @@ void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size, /* * Force kernel mapping. */ -#if defined(CONFIG_PPC_BOOK3S_64) - flags |= _PAGE_PRIVILEGED; -#else flags &= ~_PAGE_USER; -#endif - - -#ifdef _PAGE_BAP_SR - /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format - * which means that we just cleared supervisor access... oops ;-) This - * restores it - */ - flags |= _PAGE_BAP_SR; -#endif + flags |= _PAGE_PRIVILEGED; if (ppc_md.ioremap) return ppc_md.ioremap(addr, size, flags, caller); -- cgit v1.2.3 From 351750331fc1580cdb60d2efef04238f5faa89fe Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Jan 2018 13:45:29 +0100 Subject: powerpc/mm: Introduce _PAGE_NA Today, PAGE_NONE is defined as a page not having _PAGE_USER. In some circunstances, when the CPU supports it, it might be better to be able to flag a page with NO ACCESS. In a following patch, the 8xx will switch user access being flagged in the PMD, therefore it will not be possible anymore to use _PAGE_USER as a way to flag a page with no access. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/pgtable.h | 1 + arch/powerpc/include/asm/nohash/32/pgtable.h | 2 +- arch/powerpc/include/asm/pte-common.h | 7 +++++-- arch/powerpc/mm/dump_linuxpagetables.c | 18 +++++++++++------- 4 files changed, 18 insertions(+), 10 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 8e28febedac3..60a0d7fd82bc 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -14,6 +14,7 @@ */ #define _PAGE_BIT_SWAP_TYPE 0 +#define _PAGE_NA 0 #define _PAGE_RO 0 #define _PAGE_USER 0 diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index cc2bfec3aa3b..504a3c36ce5c 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -282,7 +282,7 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm, { unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_RW | _PAGE_EXEC); - unsigned long clr = ~pte_val(entry) & _PAGE_RO; + unsigned long clr = ~pte_val(entry) & (_PAGE_RO | _PAGE_NA); pte_update(ptep, clr, set); } diff --git a/arch/powerpc/include/asm/pte-common.h b/arch/powerpc/include/asm/pte-common.h index 0e6595a1b9d8..426a902816c5 100644 --- a/arch/powerpc/include/asm/pte-common.h +++ b/arch/powerpc/include/asm/pte-common.h @@ -50,6 +50,9 @@ #define _PAGE_USER 0 #endif #endif +#ifndef _PAGE_NA +#define _PAGE_NA 0 +#endif #ifndef _PMD_PRESENT_MASK #define _PMD_PRESENT_MASK _PMD_PRESENT @@ -122,7 +125,7 @@ static inline bool pte_user(pte_t pte) /* Mask of bits returned by pte_pgprot() */ #define PAGE_PROT_BITS (_PAGE_GUARDED | _PAGE_COHERENT | _PAGE_NO_CACHE | \ _PAGE_WRITETHRU | _PAGE_ENDIAN | _PAGE_4K_PFN | \ - _PAGE_USER | _PAGE_ACCESSED | _PAGE_RO | \ + _PAGE_USER | _PAGE_ACCESSED | _PAGE_RO | _PAGE_NA | \ _PAGE_PRIVILEGED | \ _PAGE_RW | _PAGE_HWWRITE | _PAGE_DIRTY | _PAGE_EXEC) @@ -150,7 +153,7 @@ static inline bool pte_user(pte_t pte) * * Note due to the way vm flags are laid out, the bits are XWR */ -#define PAGE_NONE __pgprot(_PAGE_BASE) +#define PAGE_NONE __pgprot(_PAGE_BASE | _PAGE_NA) #define PAGE_SHARED __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW) #define PAGE_SHARED_X __pgprot(_PAGE_BASE | _PAGE_USER | _PAGE_RW | \ _PAGE_EXEC) diff --git a/arch/powerpc/mm/dump_linuxpagetables.c b/arch/powerpc/mm/dump_linuxpagetables.c index d87ab2cabaa6..876e2a3c79f2 100644 --- a/arch/powerpc/mm/dump_linuxpagetables.c +++ b/arch/powerpc/mm/dump_linuxpagetables.c @@ -117,16 +117,20 @@ static const struct flag_info flag_array[] = { .set = "user", .clear = " ", }, { -#if _PAGE_RO == 0 - .mask = _PAGE_RW, + .mask = _PAGE_RW | _PAGE_RO | _PAGE_NA, .val = _PAGE_RW, -#else - .mask = _PAGE_RO, - .val = 0, -#endif .set = "rw", - .clear = "ro", }, { + .mask = _PAGE_RW | _PAGE_RO | _PAGE_NA, + .val = _PAGE_RO, + .set = "ro", + }, { +#if _PAGE_NA != 0 + .mask = _PAGE_RW | _PAGE_RO | _PAGE_NA, + .val = _PAGE_RO, + .set = "na", + }, { +#endif .mask = _PAGE_EXEC, .val = _PAGE_EXEC, .set = " X ", -- cgit v1.2.3 From de0f93873937e999fadaba011d368bc042af37b2 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Jan 2018 13:45:31 +0100 Subject: powerpc/8xx: Remove _PAGE_USER and handle user access at PMD level As Linux kernel separates KERNEL and USER address spaces, there is therefore no need to flag USER access at page level. Today, the 8xx TLB handlers already handle user access in the L1 entry through Access Protection Groups, it is then natural to move the user access handling at PMD level once _PAGE_NA allows to handle PAGE_NONE protection without _PAGE_USER In the mean time, as we free up one bit in the PTE, we can use it to include SPS (page size flag) in the PTE and avoid handling it at every TLB miss hence removing special handling based on compiled page size. For _PAGE_EXEC, we rework it to use PP PTE bits, avoiding the copy of _PAGE_EXEC bit into the L1 entry. Unfortunatly we are not able to put it at the correct location as it conflicts with NA/RO/RW bits for data entries. Upper bits of APG in L1 entry overlap with PMD base address. In order to avoid having to filter that out, we set up all groups so that upper bits can have any value. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hugetlb.h | 3 +- arch/powerpc/include/asm/mmu-8xx.h | 34 ++++++++++---------- arch/powerpc/include/asm/nohash/32/pgalloc.h | 3 +- arch/powerpc/include/asm/nohash/32/pte-8xx.h | 14 ++++++--- arch/powerpc/include/asm/nohash/pgtable.h | 2 +- arch/powerpc/include/asm/pte-common.h | 6 ++++ arch/powerpc/kernel/head_8xx.S | 46 ++++++---------------------- arch/powerpc/mm/hugetlbpage.c | 2 +- 8 files changed, 48 insertions(+), 62 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index 14c9d44f355b..1a4847f67ea8 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -47,8 +47,7 @@ static inline pte_t *hugepd_page(hugepd_t hpd) { BUG_ON(!hugepd_ok(hpd)); #ifdef CONFIG_PPC_8xx - return (pte_t *)__va(hpd_val(hpd) & - ~(_PMD_PAGE_MASK | _PMD_PRESENT_MASK)); + return (pte_t *)__va(hpd_val(hpd) & ~HUGEPD_SHIFT_MASK); #else return (pte_t *)((hpd_val(hpd) & ~HUGEPD_SHIFT_MASK) | PD_HUGE); diff --git a/arch/powerpc/include/asm/mmu-8xx.h b/arch/powerpc/include/asm/mmu-8xx.h index 5bb3dbede41a..a568ff5c5fb8 100644 --- a/arch/powerpc/include/asm/mmu-8xx.h +++ b/arch/powerpc/include/asm/mmu-8xx.h @@ -29,17 +29,17 @@ #define MI_Kp 0x40000000 /* Should always be set */ /* - * All pages' PP exec bits are set to 000, which means Execute for Supervisor - * and no Execute for User. - * Then we use the APG to say whether accesses are according to Page rules, - * "all Supervisor" rules (Exec for all) and "all User" rules (Exec for noone) - * Therefore, we define 4 APG groups. msb is _PAGE_EXEC, lsb is _PAGE_USER - * 0 (00) => Not User, no exec => 11 (all accesses performed as user) - * 1 (01) => User but no exec => 11 (all accesses performed as user) - * 2 (10) => Not User, exec => 01 (rights according to page definition) - * 3 (11) => User, exec => 00 (all accesses performed as supervisor) - */ -#define MI_APG_INIT 0xf4ffffff + * All pages' PP data bits are set to either 001 or 011 by copying _PAGE_EXEC + * into bit 21 in the ITLBmiss handler (bit 21 is the middle bit), which means + * respectively NA for All or X for Supervisor and no access for User. + * Then we use the APG to say whether accesses are according to Page rules or + * "all Supervisor" rules (Access to all) + * Therefore, we define 2 APG groups. lsb is _PMD_USER + * 0 => No user => 01 (all accesses performed according to page definition) + * 1 => User => 00 (all accesses performed as supervisor iaw page definition) + * We define all 16 groups so that all other bits of APG can take any value + */ +#define MI_APG_INIT 0x44444444 /* The effective page number register. When read, contains the information * about the last instruction TLB miss. When MI_RPN is written, bits in @@ -102,17 +102,17 @@ #define MD_Kp 0x40000000 /* Should always be set */ /* - * All pages' PP data bits are set to either 000 or 011, which means + * All pages' PP data bits are set to either 000 or 011 or 001, which means * respectively RW for Supervisor and no access for User, or RO for - * Supervisor and no access for user. + * Supervisor and no access for user and NA for ALL. * Then we use the APG to say whether accesses are according to Page rules or * "all Supervisor" rules (Access to all) - * Therefore, we define 2 APG groups. lsb is _PAGE_USER + * Therefore, we define 2 APG groups. lsb is _PMD_USER * 0 => No user => 01 (all accesses performed according to page definition) - * 1 => User => 00 (all accesses performed as supervisor - * according to page definition) + * 1 => User => 00 (all accesses performed as supervisor iaw page definition) + * We define all 16 groups so that all other bits of APG can take any value */ -#define MD_APG_INIT 0x4fffffff +#define MD_APG_INIT 0x44444444 /* The effective page number register. When read, contains the information * about the last instruction TLB miss. When MD_RPN is written, bits in diff --git a/arch/powerpc/include/asm/nohash/32/pgalloc.h b/arch/powerpc/include/asm/nohash/32/pgalloc.h index d072139ff2e5..29d37bd1f3b3 100644 --- a/arch/powerpc/include/asm/nohash/32/pgalloc.h +++ b/arch/powerpc/include/asm/nohash/32/pgalloc.h @@ -61,7 +61,8 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pte_page) { - *pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_PRESENT); + *pmdp = __pmd((page_to_pfn(pte_page) << PAGE_SHIFT) | _PMD_USER | + _PMD_PRESENT); } #define pmd_pgtable(pmd) pmd_page(pmd) diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h index 7c7040f015e2..f04cb46ae8a1 100644 --- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h @@ -32,27 +32,33 @@ #define _PAGE_PRESENT 0x0001 /* Page is valid */ #define _PAGE_NO_CACHE 0x0002 /* I: cache inhibit */ #define _PAGE_PRIVILEGED 0x0004 /* No ASID (context) compare */ -#define _PAGE_SPECIAL 0x0008 /* SW entry, forced to 0 by the TLB miss */ +#define _PAGE_HUGE 0x0008 /* SPS: Small Page Size (1 if 16k, 512k or 8M)*/ #define _PAGE_DIRTY 0x0100 /* C: page changed */ /* These 4 software bits must be masked out when the L2 entry is loaded * into the TLB. */ #define _PAGE_GUARDED 0x0010 /* Copied to L1 G entry in DTLB */ -#define _PAGE_USER 0x0020 /* Copied to L1 APG lsb */ -#define _PAGE_EXEC 0x0040 /* Copied to L1 APG */ +#define _PAGE_SPECIAL 0x0020 /* SW entry */ +#define _PAGE_EXEC 0x0040 /* Copied to PP (bit 21) in ITLB */ #define _PAGE_ACCESSED 0x0080 /* software: page referenced */ +#define _PAGE_NA 0x0200 /* Supervisor NA, User no access */ #define _PAGE_RO 0x0600 /* Supervisor RO, User no access */ #define _PMD_PRESENT 0x0001 -#define _PMD_BAD 0x0ff0 +#define _PMD_BAD 0x0fd0 #define _PMD_PAGE_MASK 0x000c #define _PMD_PAGE_8M 0x000c #define _PMD_PAGE_512K 0x0004 +#define _PMD_USER 0x0020 /* APG 1 */ /* Until my rework is finished, 8xx still needs atomic PTE updates */ #define PTE_ATOMIC_UPDATES 1 +#ifdef CONFIG_PPC_16K_PAGES +#define _PAGE_PSIZE _PAGE_HUGE +#endif + #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_NOHASH_32_PTE_8xx_H */ diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h index e0864e683cf9..c56de1e8026f 100644 --- a/arch/powerpc/include/asm/nohash/pgtable.h +++ b/arch/powerpc/include/asm/nohash/pgtable.h @@ -126,7 +126,7 @@ static inline pte_t pte_mkspecial(pte_t pte) static inline pte_t pte_mkhuge(pte_t pte) { - return pte; + return __pte(pte_val(pte) | _PAGE_HUGE); } static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) diff --git a/arch/powerpc/include/asm/pte-common.h b/arch/powerpc/include/asm/pte-common.h index 426a902816c5..c4a72c7a8c83 100644 --- a/arch/powerpc/include/asm/pte-common.h +++ b/arch/powerpc/include/asm/pte-common.h @@ -53,6 +53,9 @@ #ifndef _PAGE_NA #define _PAGE_NA 0 #endif +#ifndef _PAGE_HUGE +#define _PAGE_HUGE 0 +#endif #ifndef _PMD_PRESENT_MASK #define _PMD_PRESENT_MASK _PMD_PRESENT @@ -61,6 +64,9 @@ #define _PMD_SIZE 0 #define PMD_PAGE_SIZE(pmd) bad_call_to_PMD_PAGE_SIZE() #endif +#ifndef _PMD_USER +#define _PMD_USER 0 +#endif #ifndef _PAGE_KERNEL_RO #define _PAGE_KERNEL_RO (_PAGE_PRIVILEGED | _PAGE_RO) #endif diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 642680389b7e..c3b831bb8bad 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -52,11 +52,7 @@ * Value for the bits that have fixed value in RPN entries. * Also used for tagging DAR for DTLBerror. */ -#ifdef CONFIG_PPC_16K_PAGES -#define RPN_PATTERN (0x00f0 | MD_SPS16K) -#else #define RPN_PATTERN 0x00f0 -#endif #define PAGE_SHIFT_512K 19 #define PAGE_SHIFT_8M 23 @@ -358,31 +354,23 @@ _ENTRY(ITLBMiss_cmp) #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) mtcr r12 #endif - /* Insert the APG into the TWC from the Linux PTE. */ - rlwimi r11, r10, 0, 25, 26 /* Load the MI_TWC with the attributes for this "segment." */ mtspr SPRN_MI_TWC, r11 /* Set segment attributes */ -#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES) - rlwimi r10, r11, 1, MI_SPS16K -#endif #ifdef CONFIG_SWAP rlwinm r11, r10, 32-5, _PAGE_PRESENT and r11, r11, r10 rlwimi r10, r11, 0, _PAGE_PRESENT #endif - li r11, RPN_PATTERN + li r11, RPN_PATTERN | 0x200 /* The Linux PTE won't go exactly into the MMU TLB. - * Software indicator bits 20-23 and 28 must be clear. - * Software indicator bits 24, 25, 26, and 27 must be + * Software indicator bits 20 and 23 must be clear. + * Software indicator bits 22, 24, 25, 26, and 27 must be * set. All other Linux PTE bits control the behavior * of the MMU. */ -#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES) - rlwimi r10, r11, 0, 0x0ff0 /* Set 24-27, clear 20-23 */ -#else - rlwimi r10, r11, 0, 0x0ff8 /* Set 24-27, clear 20-23,28 */ -#endif + rlwimi r11, r10, 4, 0x0400 /* Copy _PAGE_EXEC into bit 21 */ + rlwimi r10, r11, 0, 0x0ff0 /* Set 22, 24-27, clear 20,23 */ mtspr SPRN_MI_RPN, r10 /* Update TLB entry */ /* Restore registers */ @@ -419,7 +407,6 @@ _ENTRY(itlb_miss_perf) rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK #endif lwz r10, 0(r10) /* Get the pte */ - rlwinm r11, r11, 0, 0xf b 4b 20: /* 512k pages */ @@ -428,7 +415,6 @@ _ENTRY(itlb_miss_perf) /* Add level 2 base */ rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1 lwz r10, 0(r10) /* Get the pte */ - rlwinm r11, r11, 0, 0xf b 4b #endif @@ -479,20 +465,15 @@ _ENTRY(DTLBMiss_jmp) 4: mtcr r12 - /* Insert the Guarded flag and APG into the TWC from the Linux PTE. - * It is bit 26-27 of both the Linux PTE and the TWC (at least + /* Insert the Guarded flag into the TWC from the Linux PTE. + * It is bit 27 of both the Linux PTE and the TWC (at least * I got that right :-). It will be better when we can put * this into the Linux pgd/pmd and load it in the operation * above. */ - rlwimi r11, r10, 0, 26, 27 + rlwimi r11, r10, 0, _PAGE_GUARDED mtspr SPRN_MD_TWC, r11 - /* In 4k pages mode, SPS (bit 28) in RPN must match PS[1] (bit 29) - * In 16k pages mode, SPS is always 1 */ -#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES) - rlwimi r10, r11, 1, MD_SPS16K -#endif /* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set. * We also need to know if the insn is a load/store, so: * Clear _PAGE_PRESENT and load that which will @@ -508,17 +489,12 @@ _ENTRY(DTLBMiss_jmp) rlwimi r10, r11, 0, _PAGE_PRESENT #endif /* The Linux PTE won't go exactly into the MMU TLB. - * Software indicator bits 22 and 28 must be clear. * Software indicator bits 24, 25, 26, and 27 must be * set. All other Linux PTE bits control the behavior * of the MMU. */ li r11, RPN_PATTERN -#if defined (CONFIG_HUGETLB_PAGE) && defined (CONFIG_PPC_4K_PAGES) rlwimi r10, r11, 0, 24, 27 /* Set 24-27 */ -#else - rlwimi r10, r11, 0, 24, 28 /* Set 24-27, clear 28 */ -#endif mtspr SPRN_MD_RPN, r10 /* Update TLB entry */ /* Restore registers */ @@ -552,7 +528,6 @@ _ENTRY(dtlb_miss_perf) rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK #endif lwz r10, 0(r10) /* Get the pte */ - rlwinm r11, r11, 0, 0xf b 4b 20: /* 512k pages */ @@ -561,7 +536,6 @@ _ENTRY(dtlb_miss_perf) /* Add level 2 base */ rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1 lwz r10, 0(r10) /* Get the pte */ - rlwinm r11, r11, 0, 0xf b 4b #endif @@ -712,7 +686,7 @@ _ENTRY(dtlb_miss_exit_3) ITLBMissLinear: mtcr r12 /* Set 8M byte page and mark it valid */ - li r11, MI_PS8MEG | MI_SVALID | _PAGE_EXEC + li r11, MI_PS8MEG | MI_SVALID mtspr SPRN_MI_TWC, r11 rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \ @@ -994,7 +968,7 @@ initial_mmu: lis r8, KERNELBASE@h /* Create vaddr for TLB */ ori r8, r8, MI_EVALID /* Mark it valid */ mtspr SPRN_MI_EPN, r8 - li r8, MI_PS8MEG | (2 << 5) /* Set 8M byte page, APG 2 */ + li r8, MI_PS8MEG /* Set 8M byte page */ ori r8, r8, MI_SVALID /* Make it valid */ mtspr SPRN_MI_TWC, r8 li r8, MI_BOOTINIT /* Create RPN for address 0 */ diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index c7e5afe5e118..f5d1008f8574 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -96,7 +96,7 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, *hpdp = __hugepd(__pa(new) | (shift_to_mmu_psize(pshift) << 2)); #elif defined(CONFIG_PPC_8xx) - *hpdp = __hugepd(__pa(new) | + *hpdp = __hugepd(__pa(new) | _PMD_USER | (pshift == PAGE_SHIFT_8M ? _PMD_PAGE_8M : _PMD_PAGE_512K) | _PMD_PRESENT); #else -- cgit v1.2.3 From 4f94b2c7462d9720b2afa7e8e8d4c19446bb31ce Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 12 Jan 2018 13:45:33 +0100 Subject: powerpc/8xx: Use L1 entry APG to handle _PAGE_ACCESSED for CONFIG_SWAP When CONFIG_SWAP is set, the TLB miss handlers have to also take into account _PAGE_ACCESSED flag. At the moment it is done by anding _PAGE_ACCESSED into _PAGE_PRESENT using 3 instructions. This patch uses APG for handling _PAGE_ACCESSED, allowing to just copy _PAGE_ACCESSED bit into APG field, hence reducing the action to a single instruction. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/mmu-8xx.h | 34 +++++++++++++++++++++++----- arch/powerpc/kernel/head_8xx.S | 45 +++++++++++++++----------------------- arch/powerpc/mm/8xx_mmu.c | 2 +- 3 files changed, 47 insertions(+), 34 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/mmu-8xx.h b/arch/powerpc/include/asm/mmu-8xx.h index a568ff5c5fb8..2f806e329648 100644 --- a/arch/powerpc/include/asm/mmu-8xx.h +++ b/arch/powerpc/include/asm/mmu-8xx.h @@ -34,12 +34,20 @@ * respectively NA for All or X for Supervisor and no access for User. * Then we use the APG to say whether accesses are according to Page rules or * "all Supervisor" rules (Access to all) - * Therefore, we define 2 APG groups. lsb is _PMD_USER - * 0 => No user => 01 (all accesses performed according to page definition) - * 1 => User => 00 (all accesses performed as supervisor iaw page definition) + * We also use the 2nd APG bit for _PAGE_ACCESSED when having SWAP: + * When that bit is not set access is done iaw "all user" + * which means no access iaw page rules. + * Therefore, we define 4 APG groups. lsb is _PMD_USER, 2nd is _PAGE_ACCESSED + * 0x => No access => 11 (all accesses performed as user iaw page definition) + * 10 => No user => 01 (all accesses performed according to page definition) + * 11 => User => 00 (all accesses performed as supervisor iaw page definition) * We define all 16 groups so that all other bits of APG can take any value */ +#ifdef CONFIG_SWAP +#define MI_APG_INIT 0xf4f4f4f4 +#else #define MI_APG_INIT 0x44444444 +#endif /* The effective page number register. When read, contains the information * about the last instruction TLB miss. When MI_RPN is written, bits in @@ -107,12 +115,20 @@ * Supervisor and no access for user and NA for ALL. * Then we use the APG to say whether accesses are according to Page rules or * "all Supervisor" rules (Access to all) - * Therefore, we define 2 APG groups. lsb is _PMD_USER - * 0 => No user => 01 (all accesses performed according to page definition) - * 1 => User => 00 (all accesses performed as supervisor iaw page definition) + * We also use the 2nd APG bit for _PAGE_ACCESSED when having SWAP: + * When that bit is not set access is done iaw "all user" + * which means no access iaw page rules. + * Therefore, we define 4 APG groups. lsb is _PMD_USER, 2nd is _PAGE_ACCESSED + * 0x => No access => 11 (all accesses performed as user iaw page definition) + * 10 => No user => 01 (all accesses performed according to page definition) + * 11 => User => 00 (all accesses performed as supervisor iaw page definition) * We define all 16 groups so that all other bits of APG can take any value */ +#ifdef CONFIG_SWAP +#define MD_APG_INIT 0xf4f4f4f4 +#else #define MD_APG_INIT 0x44444444 +#endif /* The effective page number register. When read, contains the information * about the last instruction TLB miss. When MD_RPN is written, bits in @@ -164,6 +180,12 @@ */ #define SPRN_M_TW 799 +/* APGs */ +#define M_APG0 0x00000000 +#define M_APG1 0x00000020 +#define M_APG2 0x00000040 +#define M_APG3 0x00000060 + #ifndef __ASSEMBLY__ typedef struct { unsigned int id; diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index c3b831bb8bad..d8670a37d70c 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -354,14 +354,13 @@ _ENTRY(ITLBMiss_cmp) #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE) mtcr r12 #endif - /* Load the MI_TWC with the attributes for this "segment." */ - mtspr SPRN_MI_TWC, r11 /* Set segment attributes */ #ifdef CONFIG_SWAP - rlwinm r11, r10, 32-5, _PAGE_PRESENT - and r11, r11, r10 - rlwimi r10, r11, 0, _PAGE_PRESENT + rlwinm r11, r10, 31, _PAGE_ACCESSED >> 1 #endif + /* Load the MI_TWC with the attributes for this "segment." */ + mtspr SPRN_MI_TWC, r11 /* Set segment attributes */ + li r11, RPN_PATTERN | 0x200 /* The Linux PTE won't go exactly into the MMU TLB. * Software indicator bits 20 and 23 must be clear. @@ -472,22 +471,14 @@ _ENTRY(DTLBMiss_jmp) * above. */ rlwimi r11, r10, 0, _PAGE_GUARDED - mtspr SPRN_MD_TWC, r11 - - /* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set. - * We also need to know if the insn is a load/store, so: - * Clear _PAGE_PRESENT and load that which will - * trap into DTLB Error with store bit set accordinly. - */ - /* PRESENT=0x1, ACCESSED=0x20 - * r11 = ((r10 & PRESENT) & ((r10 & ACCESSED) >> 5)); - * r10 = (r10 & ~PRESENT) | r11; - */ #ifdef CONFIG_SWAP - rlwinm r11, r10, 32-5, _PAGE_PRESENT - and r11, r11, r10 - rlwimi r10, r11, 0, _PAGE_PRESENT + /* _PAGE_ACCESSED has to be set. We use second APG bit for that, 0 + * on that bit will represent a Non Access group + */ + rlwinm r11, r10, 31, _PAGE_ACCESSED >> 1 #endif + mtspr SPRN_MD_TWC, r11 + /* The Linux PTE won't go exactly into the MMU TLB. * Software indicator bits 24, 25, 26, and 27 must be * set. All other Linux PTE bits control the behavior @@ -647,8 +638,8 @@ InstructionBreakpoint: */ DTLBMissIMMR: mtcr r12 - /* Set 512k byte guarded page and mark it valid */ - li r10, MD_PS512K | MD_GUARDED | MD_SVALID + /* Set 512k byte guarded page and mark it valid and accessed */ + li r10, MD_PS512K | MD_GUARDED | MD_SVALID | M_APG2 mtspr SPRN_MD_TWC, r10 mfspr r10, SPRN_IMMR /* Get current IMMR */ rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */ @@ -666,8 +657,8 @@ _ENTRY(dtlb_miss_exit_2) DTLBMissLinear: mtcr r12 - /* Set 8M byte page and mark it valid */ - li r11, MD_PS8MEG | MD_SVALID + /* Set 8M byte page and mark it valid and accessed */ + li r11, MD_PS8MEG | MD_SVALID | M_APG2 mtspr SPRN_MD_TWC, r11 rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \ @@ -685,8 +676,8 @@ _ENTRY(dtlb_miss_exit_3) #ifndef CONFIG_PIN_TLB_TEXT ITLBMissLinear: mtcr r12 - /* Set 8M byte page and mark it valid */ - li r11, MI_PS8MEG | MI_SVALID + /* Set 8M byte page and mark it valid,accessed */ + li r11, MI_PS8MEG | MI_SVALID | M_APG2 mtspr SPRN_MI_TWC, r11 rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */ ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \ @@ -969,7 +960,7 @@ initial_mmu: ori r8, r8, MI_EVALID /* Mark it valid */ mtspr SPRN_MI_EPN, r8 li r8, MI_PS8MEG /* Set 8M byte page */ - ori r8, r8, MI_SVALID /* Make it valid */ + ori r8, r8, MI_SVALID | M_APG2 /* Make it valid, APG 2 */ mtspr SPRN_MI_TWC, r8 li r8, MI_BOOTINIT /* Create RPN for address 0 */ mtspr SPRN_MI_RPN, r8 /* Store TLB entry */ @@ -996,7 +987,7 @@ initial_mmu: ori r8, r8, MD_EVALID /* Mark it valid */ mtspr SPRN_MD_EPN, r8 li r8, MD_PS512K | MD_GUARDED /* Set 512k byte page */ - ori r8, r8, MD_SVALID /* Make it valid */ + ori r8, r8, MD_SVALID | M_APG2 /* Make it valid and accessed */ mtspr SPRN_MD_TWC, r8 mr r8, r9 /* Create paddr for TLB */ ori r8, r8, MI_BOOTINIT|0x2 /* Inhibit cache -- Cort */ diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c index c2d9a6d709c3..849f50cd62f2 100644 --- a/arch/powerpc/mm/8xx_mmu.c +++ b/arch/powerpc/mm/8xx_mmu.c @@ -79,7 +79,7 @@ void __init MMU_init_hw(void) for (; i < 32 && mem >= LARGE_PAGE_SIZE_8M; i++) { mtspr(SPRN_MD_CTR, ctr | (i << 8)); mtspr(SPRN_MD_EPN, (unsigned long)__va(addr) | MD_EVALID); - mtspr(SPRN_MD_TWC, MD_PS8MEG | MD_SVALID); + mtspr(SPRN_MD_TWC, MD_PS8MEG | MD_SVALID | M_APG2); mtspr(SPRN_MD_RPN, addr | flags | _PAGE_PRESENT); addr += LARGE_PAGE_SIZE_8M; mem -= LARGE_PAGE_SIZE_8M; -- cgit v1.2.3 From 872e2ae4bdd4c244f7617ffc08c39462192460ee Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Fri, 12 Jan 2018 13:28:48 +1100 Subject: powerpc: Remove useless EXC_COMMON_HV The only difference between EXC_COMMON_HV and EXC_COMMON is that the former adds "2" to the trap number which is supposed to represent the fact that this is an "HV" interrupt which uses HSRR0/1. However KVM is the only one who cares and it has its own separate macros. In fact, we only have one user of EXC_COMMON_HV and it's for an unknown interrupt case. All the other ones already using EXC_COMMON. Signed-off-by: Benjamin Herrenschmidt Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/head-64.h | 7 +------ arch/powerpc/kernel/exceptions-64s.S | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index fdcff76e9a25..0a663dfc28b5 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -178,7 +178,7 @@ name: * TRAMP_REAL_* - real, unrelocated helpers (virt can call these) * TRAMP_VIRT_* - virt, unreloc helpers (in practice, real can use) * TRAMP_KVM - KVM handlers that get put into real, unrelocated - * EXC_COMMON_* - virt, relocated common handlers + * EXC_COMMON - virt, relocated common handlers * * The EXC handlers are given a name, and branch to name_common, or the * appropriate KVM or masking function. Vector handler verieties are as @@ -211,7 +211,6 @@ name: * EXC_COMMON_BEGIN/END - used to open-code the handler * EXC_COMMON * EXC_COMMON_ASYNC - * EXC_COMMON_HV * * TRAMP_REAL and TRAMP_VIRT can be used with BEGIN/END. KVM * and OOL handlers are implemented as types of TRAMP and TRAMP_VIRT handlers. @@ -413,10 +412,6 @@ name: EXC_COMMON_BEGIN(name); \ STD_EXCEPTION_COMMON_ASYNC(realvec, name, hdlr); \ -#define EXC_COMMON_HV(name, realvec, hdlr) \ - EXC_COMMON_BEGIN(name); \ - STD_EXCEPTION_COMMON(realvec + 0x2, name, hdlr); \ - #endif /* __ASSEMBLY__ */ #endif /* _ASM_POWERPC_HEAD_64_H */ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index e441b469dc8f..175891c6909c 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1318,7 +1318,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) b . #endif -EXC_COMMON_HV(denorm_common, 0x1500, unknown_exception) +EXC_COMMON(denorm_common, 0x1500, unknown_exception) #ifdef CONFIG_CBE_RAS -- cgit v1.2.3 From d4748276ae14ce951a3254852dddc3675797c277 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sun, 24 Dec 2017 01:15:50 +1000 Subject: powerpc/64s: Improve local TLB flush for boot and MCE on POWER9 There are several cases outside the normal address space management where a CPU's entire local TLB is to be flushed: 1. Booting the kernel, in case something has left stale entries in the TLB (e.g., kexec). 2. Machine check, to clean corrupted TLB entries. One other place where the TLB is flushed, is waking from deep idle states. The flush is a side-effect of calling ->cpu_restore with the intention of re-setting various SPRs. The flush itself is unnecessary because in the first case, the TLB should not acquire new corrupted TLB entries as part of sleep/wake (though they may be lost). This type of TLB flush is coded inflexibly, several times for each CPU type, and they have a number of problems with ISA v3.0B: - The current radix mode of the MMU is not taken into account, it is always done as a hash flushn For IS=2 (LPID-matching flush from host) and IS=3 with HV=0 (guest kernel flush), tlbie(l) is undefined if the R field does not match the current radix mode. - ISA v3.0B hash must flush the partition and process table caches as well. - ISA v3.0B radix must flush partition and process scoped translations, partition and process table caches, and also the page walk cache. So consolidate the flushing code and implement it in C and inline asm under the mm/ directory with the rest of the flush code. Add ISA v3.0B cases for radix and hash, and use the radix flush in radix environment. Provide a way for IS=2 (LPID flush) to specify the radix mode of the partition. Have KVM pass in the radix mode of the guest. Take out the flushes from early cputable/dt_cpu_ftrs detection hooks, and move it later in the boot process after, the MMU registers are set up and before relocation is first turned on. The TLB flush is no longer called when restoring from deep idle states. This was not be done as a separate step because booting secondaries uses the same cpu_restore as idle restore, which needs the TLB flush. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 1 + .../powerpc/include/asm/book3s/64/tlbflush-radix.h | 3 + arch/powerpc/include/asm/book3s/64/tlbflush.h | 34 ++++++ arch/powerpc/include/asm/cputable.h | 12 --- arch/powerpc/kernel/cpu_setup_power.S | 50 --------- arch/powerpc/kernel/cputable.c | 15 --- arch/powerpc/kernel/dt_cpu_ftrs.c | 30 ------ arch/powerpc/kernel/mce_power.c | 115 +-------------------- arch/powerpc/kvm/book3s_hv_ras.c | 6 +- arch/powerpc/mm/hash_native_64.c | 97 +++++++++++++++++ arch/powerpc/mm/hash_utils_64.c | 8 ++ arch/powerpc/mm/pgtable-radix.c | 6 ++ arch/powerpc/mm/tlb-radix.c | 66 ++++++++++++ 13 files changed, 219 insertions(+), 224 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h index 849ecaae9e79..64d02a704bcb 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -51,6 +51,7 @@ static inline void arch_leave_lazy_mmu_mode(void) #define arch_flush_lazy_mmu_mode() do {} while (0) +extern void hash__tlbiel_all(unsigned int action); extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize, unsigned long flags); diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index 6a9e68003387..b8f9ad587087 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -11,6 +11,8 @@ static inline int mmu_get_ap(int psize) return mmu_psize_defs[psize].ap; } +extern void radix__tlbiel_all(unsigned int action); + extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start, @@ -47,4 +49,5 @@ extern void radix__flush_tlb_lpid(unsigned long lpid); extern void radix__flush_tlb_all(void); extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm, unsigned long address); + #endif diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index 58b576f654b3..9befb4df235c 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -8,6 +8,40 @@ #include #include +/* TLB flush actions. Used as argument to tlbiel_all() */ +enum { + TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ + TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */ +}; + +static inline void tlbiel_all(void) +{ + /* + * This is used for host machine check and bootup. + * + * This uses early_radix_enabled and implementations use + * early_cpu_has_feature etc because that works early in boot + * and this is the machine check path which is not performance + * critical. + */ + if (early_radix_enabled()) + radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); + else + hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); +} + +static inline void tlbiel_all_lpid(bool radix) +{ + /* + * This is used for guest machine check. + */ + if (radix) + radix__tlbiel_all(TLB_INVAL_SCOPE_LPID); + else + hash__tlbiel_all(TLB_INVAL_SCOPE_LPID); +} + + #define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE static inline void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index 0546663a98db..78ca2a721d04 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -107,12 +107,6 @@ struct cpu_spec { * called in real mode to handle SLB and TLB errors. */ long (*machine_check_early)(struct pt_regs *regs); - - /* - * Processor specific routine to flush tlbs. - */ - void (*flush_tlb)(unsigned int action); - }; extern struct cpu_spec *cur_cpu_spec; @@ -133,12 +127,6 @@ extern void cpu_feature_keys_init(void); static inline void cpu_feature_keys_init(void) { } #endif -/* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */ -enum { - TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */ - TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */ -}; - #endif /* __ASSEMBLY__ */ /* CPU kernel features */ diff --git a/arch/powerpc/kernel/cpu_setup_power.S b/arch/powerpc/kernel/cpu_setup_power.S index 610955fe8b81..730ade48329b 100644 --- a/arch/powerpc/kernel/cpu_setup_power.S +++ b/arch/powerpc/kernel/cpu_setup_power.S @@ -31,7 +31,6 @@ _GLOBAL(__setup_cpu_power7) mfspr r3,SPRN_LPCR li r4,(LPCR_LPES1 >> LPCR_LPES_SH) bl __init_LPCR_ISA206 - bl __init_tlb_power7 mtlr r11 blr @@ -45,7 +44,6 @@ _GLOBAL(__restore_cpu_power7) mfspr r3,SPRN_LPCR li r4,(LPCR_LPES1 >> LPCR_LPES_SH) bl __init_LPCR_ISA206 - bl __init_tlb_power7 mtlr r11 blr @@ -64,7 +62,6 @@ _GLOBAL(__setup_cpu_power8) li r4,0 /* LPES = 0 */ bl __init_LPCR_ISA206 bl __init_HFSCR - bl __init_tlb_power8 bl __init_PMU_HV bl __init_PMU_HV_ISA207 mtlr r11 @@ -86,7 +83,6 @@ _GLOBAL(__restore_cpu_power8) li r4,0 /* LPES = 0 */ bl __init_LPCR_ISA206 bl __init_HFSCR - bl __init_tlb_power8 bl __init_PMU_HV bl __init_PMU_HV_ISA207 mtlr r11 @@ -110,7 +106,6 @@ _GLOBAL(__setup_cpu_power9) li r4,0 /* LPES = 0 */ bl __init_LPCR_ISA300 bl __init_HFSCR - bl __init_tlb_power9 bl __init_PMU_HV mtlr r11 blr @@ -134,7 +129,6 @@ _GLOBAL(__restore_cpu_power9) li r4,0 /* LPES = 0 */ bl __init_LPCR_ISA300 bl __init_HFSCR - bl __init_tlb_power9 bl __init_PMU_HV mtlr r11 blr @@ -192,50 +186,6 @@ __init_HFSCR: mtspr SPRN_HFSCR,r3 blr -/* - * Clear the TLB using the specified IS form of tlbiel instruction - * (invalidate by congruence class). P7 has 128 CCs., P8 has 512. - */ -__init_tlb_power7: - li r6,POWER7_TLB_SETS - mtctr r6 - li r7,0xc00 /* IS field = 0b11 */ - ptesync -2: tlbiel r7 - addi r7,r7,0x1000 - bdnz 2b - ptesync -1: blr - -__init_tlb_power8: - li r6,POWER8_TLB_SETS - mtctr r6 - li r7,0xc00 /* IS field = 0b11 */ - ptesync -2: tlbiel r7 - addi r7,r7,0x1000 - bdnz 2b - ptesync -1: blr - -/* - * Flush the TLB in hash mode. Hash must flush with RIC=2 once for process - * and one for partition scope to clear process and partition table entries. - */ -__init_tlb_power9: - li r6,POWER9_TLB_SETS_HASH - 1 - mtctr r6 - li r7,0xc00 /* IS field = 0b11 */ - li r8,0 - ptesync - PPC_TLBIEL(7, 8, 2, 1, 0) - PPC_TLBIEL(7, 8, 2, 0, 0) -2: addi r7,r7,0x1000 - PPC_TLBIEL(7, 8, 0, 0, 0) - bdnz 2b - ptesync -1: blr - __init_PMU_HV: li r5,0 mtspr SPRN_MMCRC,r5 diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index 1350f49d81a8..c40a9fc1e5d1 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -74,9 +74,6 @@ extern void __setup_cpu_power8(unsigned long offset, struct cpu_spec* spec); extern void __restore_cpu_power8(void); extern void __setup_cpu_power9(unsigned long offset, struct cpu_spec* spec); extern void __restore_cpu_power9(void); -extern void __flush_tlb_power7(unsigned int action); -extern void __flush_tlb_power8(unsigned int action); -extern void __flush_tlb_power9(unsigned int action); extern long __machine_check_early_realmode_p7(struct pt_regs *regs); extern long __machine_check_early_realmode_p8(struct pt_regs *regs); extern long __machine_check_early_realmode_p9(struct pt_regs *regs); @@ -368,7 +365,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_cpu_type = "ppc64/ibm-compat-v1", .cpu_setup = __setup_cpu_power7, .cpu_restore = __restore_cpu_power7, - .flush_tlb = __flush_tlb_power7, .machine_check_early = __machine_check_early_realmode_p7, .platform = "power7", }, @@ -386,7 +382,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_cpu_type = "ppc64/ibm-compat-v1", .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", }, @@ -404,7 +399,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_cpu_type = "ppc64/ibm-compat-v1", .cpu_setup = __setup_cpu_power9, .cpu_restore = __restore_cpu_power9, - .flush_tlb = __flush_tlb_power9, .platform = "power9", }, { /* Power7 */ @@ -423,7 +417,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_POWER4, .cpu_setup = __setup_cpu_power7, .cpu_restore = __restore_cpu_power7, - .flush_tlb = __flush_tlb_power7, .machine_check_early = __machine_check_early_realmode_p7, .platform = "power7", }, @@ -443,7 +436,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_POWER4, .cpu_setup = __setup_cpu_power7, .cpu_restore = __restore_cpu_power7, - .flush_tlb = __flush_tlb_power7, .machine_check_early = __machine_check_early_realmode_p7, .platform = "power7+", }, @@ -463,7 +455,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", }, @@ -483,7 +474,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", }, @@ -503,7 +493,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", }, @@ -523,7 +512,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power8, .cpu_restore = __restore_cpu_power8, - .flush_tlb = __flush_tlb_power8, .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", }, @@ -543,7 +531,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power9, .cpu_restore = __restore_cpu_power9, - .flush_tlb = __flush_tlb_power9, .machine_check_early = __machine_check_early_realmode_p9, .platform = "power9", }, @@ -563,7 +550,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power9, .cpu_restore = __restore_cpu_power9, - .flush_tlb = __flush_tlb_power9, .machine_check_early = __machine_check_early_realmode_p9, .platform = "power9", }, @@ -583,7 +569,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = __setup_cpu_power9, .cpu_restore = __restore_cpu_power9, - .flush_tlb = __flush_tlb_power9, .machine_check_early = __machine_check_early_realmode_p9, .platform = "power9", }, diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 8bdc2f96c5d6..945e2c29ad2d 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -77,8 +77,6 @@ struct dt_cpu_feature { * Set up the base CPU */ -extern void __flush_tlb_power8(unsigned int action); -extern void __flush_tlb_power9(unsigned int action); extern long __machine_check_early_realmode_p8(struct pt_regs *regs); extern long __machine_check_early_realmode_p9(struct pt_regs *regs); @@ -92,27 +90,6 @@ static struct { static void (*init_pmu_registers)(void); -static void cpufeatures_flush_tlb(void) -{ - /* - * This is a temporary measure to keep equivalent TLB flush as the - * cputable based setup code. - */ - switch (PVR_VER(mfspr(SPRN_PVR))) { - case PVR_POWER8: - case PVR_POWER8E: - case PVR_POWER8NVL: - __flush_tlb_power8(TLB_INVAL_SCOPE_GLOBAL); - break; - case PVR_POWER9: - __flush_tlb_power9(TLB_INVAL_SCOPE_GLOBAL); - break; - default: - pr_err("unknown CPU version for boot TLB flush\n"); - break; - } -} - static void __restore_cpu_cpufeatures(void) { /* @@ -137,8 +114,6 @@ static void __restore_cpu_cpufeatures(void) if (init_pmu_registers) init_pmu_registers(); - - cpufeatures_flush_tlb(); } static char dt_cpu_name[64]; @@ -157,7 +132,6 @@ static struct cpu_spec __initdata base_cpu_spec = { .oprofile_type = PPC_OPROFILE_INVALID, .cpu_setup = NULL, .cpu_restore = __restore_cpu_cpufeatures, - .flush_tlb = NULL, .machine_check_early = NULL, .platform = NULL, }; @@ -412,7 +386,6 @@ static void init_pmu_power8(void) static int __init feat_enable_mce_power8(struct dt_cpu_feature *f) { cur_cpu_spec->platform = "power8"; - cur_cpu_spec->flush_tlb = __flush_tlb_power8; cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8; return 1; @@ -451,7 +424,6 @@ static void init_pmu_power9(void) static int __init feat_enable_mce_power9(struct dt_cpu_feature *f) { cur_cpu_spec->platform = "power9"; - cur_cpu_spec->flush_tlb = __flush_tlb_power9; cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9; return 1; @@ -752,8 +724,6 @@ static void __init cpufeatures_setup_finished(void) system_registers.hfscr = mfspr(SPRN_HFSCR); system_registers.fscr = mfspr(SPRN_FSCR); - cpufeatures_flush_tlb(); - pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n", cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features); } diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c index 644f7040b91c..fe6fc63251fe 100644 --- a/arch/powerpc/kernel/mce_power.c +++ b/arch/powerpc/kernel/mce_power.c @@ -58,115 +58,6 @@ static unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr) return pte_pfn(*ptep); } -static void flush_tlb_206(unsigned int num_sets, unsigned int action) -{ - unsigned long rb; - unsigned int i; - - switch (action) { - case TLB_INVAL_SCOPE_GLOBAL: - rb = TLBIEL_INVAL_SET; - break; - case TLB_INVAL_SCOPE_LPID: - rb = TLBIEL_INVAL_SET_LPID; - break; - default: - BUG(); - break; - } - - asm volatile("ptesync" : : : "memory"); - for (i = 0; i < num_sets; i++) { - asm volatile("tlbiel %0" : : "r" (rb)); - rb += 1 << TLBIEL_INVAL_SET_SHIFT; - } - asm volatile("ptesync" : : : "memory"); -} - -static void flush_tlb_300(unsigned int num_sets, unsigned int action) -{ - unsigned long rb; - unsigned int i; - unsigned int r; - - switch (action) { - case TLB_INVAL_SCOPE_GLOBAL: - rb = TLBIEL_INVAL_SET; - break; - case TLB_INVAL_SCOPE_LPID: - rb = TLBIEL_INVAL_SET_LPID; - break; - default: - BUG(); - break; - } - - asm volatile("ptesync" : : : "memory"); - - if (early_radix_enabled()) - r = 1; - else - r = 0; - - /* - * First flush table/PWC caches with set 0, then flush the - * rest of the sets, partition scope. Radix must then do it - * all again with process scope. Hash just has to flush - * process table. - */ - asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) : : - "r"(rb), "r"(0), "i"(2), "i"(0), "r"(r)); - for (i = 1; i < num_sets; i++) { - unsigned long set = i * (1< TLB_INVAL_SCOPE_GLOBAL: Invalidate all TLBs. - * TLB_INVAL_SCOPE_LPID: Invalidate TLB for current LPID. - */ -void __flush_tlb_power7(unsigned int action) -{ - flush_tlb_206(POWER7_TLB_SETS, action); -} - -void __flush_tlb_power8(unsigned int action) -{ - flush_tlb_206(POWER8_TLB_SETS, action); -} - -void __flush_tlb_power9(unsigned int action) -{ - unsigned int num_sets; - - if (early_radix_enabled()) - num_sets = POWER9_TLB_SETS_RADIX; - else - num_sets = POWER9_TLB_SETS_HASH; - - flush_tlb_300(num_sets, action); -} - - /* flush SLBs and reload */ #ifdef CONFIG_PPC_BOOK3S_64 static void flush_and_reload_slb(void) @@ -226,10 +117,8 @@ static int mce_flush(int what) return 1; } if (what == MCE_FLUSH_TLB) { - if (cur_cpu_spec && cur_cpu_spec->flush_tlb) { - cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_GLOBAL); - return 1; - } + tlbiel_all(); + return 1; } return 0; diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c index c356f9a40b24..e61066bb6725 100644 --- a/arch/powerpc/kvm/book3s_hv_ras.c +++ b/arch/powerpc/kvm/book3s_hv_ras.c @@ -87,8 +87,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) DSISR_MC_SLB_PARITY | DSISR_MC_DERAT_MULTI); } if (dsisr & DSISR_MC_TLB_MULTI) { - if (cur_cpu_spec && cur_cpu_spec->flush_tlb) - cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); + tlbiel_all_lpid(vcpu->kvm->arch.radix); dsisr &= ~DSISR_MC_TLB_MULTI; } /* Any other errors we don't understand? */ @@ -105,8 +104,7 @@ static long kvmppc_realmode_mc_power7(struct kvm_vcpu *vcpu) reload_slb(vcpu); break; case SRR1_MC_IFETCH_TLBMULTI: - if (cur_cpu_spec && cur_cpu_spec->flush_tlb) - cur_cpu_spec->flush_tlb(TLB_INVAL_SCOPE_LPID); + tlbiel_all_lpid(vcpu->kvm->arch.radix); break; default: handled = 0; diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c index 640cf566e986..a0675e91ad7d 100644 --- a/arch/powerpc/mm/hash_native_64.c +++ b/arch/powerpc/mm/hash_native_64.c @@ -47,6 +47,103 @@ DEFINE_RAW_SPINLOCK(native_tlbie_lock); +static inline void tlbiel_hash_set_isa206(unsigned int set, unsigned int is) +{ + unsigned long rb; + + rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); + + asm volatile("tlbiel %0" : : "r" (rb)); +} + +/* + * tlbiel instruction for hash, set invalidation + * i.e., r=1 and is=01 or is=10 or is=11 + */ +static inline void tlbiel_hash_set_isa300(unsigned int set, unsigned int is, + unsigned int pid, + unsigned int ric, unsigned int prs) +{ + unsigned long rb; + unsigned long rs; + unsigned int r = 0; /* hash format */ + + rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); + rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); + + asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) + : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r) + : "memory"); +} + + +static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is) +{ + unsigned int set; + + asm volatile("ptesync": : :"memory"); + + for (set = 0; set < num_sets; set++) + tlbiel_hash_set_isa206(set, is); + + asm volatile("ptesync": : :"memory"); +} + +static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) +{ + unsigned int set; + + asm volatile("ptesync": : :"memory"); + + /* + * Flush the first set of the TLB, and any caching of partition table + * entries. Then flush the remaining sets of the TLB. Hash mode uses + * partition scoped TLB translations. + */ + tlbiel_hash_set_isa300(0, is, 0, 2, 0); + for (set = 1; set < num_sets; set++) + tlbiel_hash_set_isa300(set, is, 0, 0, 0); + + /* + * Now invalidate the process table cache. + * + * From ISA v3.0B p. 1078: + * The following forms are invalid. + * * PRS=1, R=0, and RIC!=2 (The only process-scoped + * HPT caching is of the Process Table.) + */ + tlbiel_hash_set_isa300(0, is, 0, 2, 1); + + asm volatile("ptesync": : :"memory"); +} + +void hash__tlbiel_all(unsigned int action) +{ + unsigned int is; + + switch (action) { + case TLB_INVAL_SCOPE_GLOBAL: + is = 3; + break; + case TLB_INVAL_SCOPE_LPID: + is = 2; + break; + default: + BUG(); + } + + if (early_cpu_has_feature(CPU_FTR_ARCH_300)) + tlbiel_all_isa300(POWER9_TLB_SETS_HASH, is); + else if (early_cpu_has_feature(CPU_FTR_ARCH_207S)) + tlbiel_all_isa206(POWER8_TLB_SETS, is); + else if (early_cpu_has_feature(CPU_FTR_ARCH_206)) + tlbiel_all_isa206(POWER7_TLB_SETS, is); + else + WARN(1, "%s called on pre-POWER7 CPU\n", __func__); + + asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); +} + static inline unsigned long ___tlbie(unsigned long vpn, int psize, int apsize, int ssize) { diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 0c802ded6f21..69fa01e02b63 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -1051,6 +1051,10 @@ void __init hash__early_init_mmu(void) pr_info("Initializing hash mmu with SLB\n"); /* Initialize SLB management */ slb_initialize(); + + if (cpu_has_feature(CPU_FTR_ARCH_206) + && cpu_has_feature(CPU_FTR_HVMODE)) + tlbiel_all(); } #ifdef CONFIG_SMP @@ -1070,6 +1074,10 @@ void hash__early_init_mmu_secondary(void) } /* Initialize SLB */ slb_initialize(); + + if (cpu_has_feature(CPU_FTR_ARCH_206) + && cpu_has_feature(CPU_FTR_HVMODE)) + tlbiel_all(); } #endif /* CONFIG_SMP */ diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c index cfbbee941a76..27ccab28665b 100644 --- a/arch/powerpc/mm/pgtable-radix.c +++ b/arch/powerpc/mm/pgtable-radix.c @@ -579,6 +579,9 @@ void __init radix__early_init_mmu(void) radix_init_iamr(); radix_init_pgtable(); + + if (cpu_has_feature(CPU_FTR_HVMODE)) + tlbiel_all(); } void radix__early_init_mmu_secondary(void) @@ -600,6 +603,9 @@ void radix__early_init_mmu_secondary(void) radix_init_amor(); } radix_init_iamr(); + + if (cpu_has_feature(CPU_FTR_HVMODE)) + tlbiel_all(); } void radix__mmu_cleanup_all(void) diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c index 884f4b705b57..fbaa429471a1 100644 --- a/arch/powerpc/mm/tlb-radix.c +++ b/arch/powerpc/mm/tlb-radix.c @@ -23,6 +23,72 @@ #define RIC_FLUSH_PWC 1 #define RIC_FLUSH_ALL 2 +/* + * tlbiel instruction for radix, set invalidation + * i.e., r=1 and is=01 or is=10 or is=11 + */ +static inline void tlbiel_radix_set_isa300(unsigned int set, unsigned int is, + unsigned int pid, + unsigned int ric, unsigned int prs) +{ + unsigned long rb; + unsigned long rs; + unsigned int r = 1; /* radix format */ + + rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53)); + rs = ((unsigned long)pid << PPC_BITLSHIFT(31)); + + asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4) + : : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r) + : "memory"); +} + +static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is) +{ + unsigned int set; + + asm volatile("ptesync": : :"memory"); + + /* + * Flush the first set of the TLB, and the entire Page Walk Cache + * and partition table entries. Then flush the remaining sets of the + * TLB. + */ + tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 0); + for (set = 1; set < num_sets; set++) + tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 0); + + /* Do the same for process scoped entries. */ + tlbiel_radix_set_isa300(0, is, 0, RIC_FLUSH_ALL, 1); + for (set = 1; set < num_sets; set++) + tlbiel_radix_set_isa300(set, is, 0, RIC_FLUSH_TLB, 1); + + asm volatile("ptesync": : :"memory"); +} + +void radix__tlbiel_all(unsigned int action) +{ + unsigned int is; + + switch (action) { + case TLB_INVAL_SCOPE_GLOBAL: + is = 3; + break; + case TLB_INVAL_SCOPE_LPID: + is = 2; + break; + default: + BUG(); + } + + if (early_cpu_has_feature(CPU_FTR_ARCH_300)) + tlbiel_all_isa300(POWER9_TLB_SETS_RADIX, is); + else + WARN(1, "%s called on pre-POWER9 CPU\n", __func__); + + asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory"); +} + static inline void __tlbiel_pid(unsigned long pid, int set, unsigned long ric) { -- cgit v1.2.3 From d075745d893c78730e4a3b7a60fca23c2f764081 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Wed, 17 Jan 2018 20:51:13 +1100 Subject: KVM: PPC: Book3S HV: Improve handling of debug-trigger HMIs on POWER9 Hypervisor maintenance interrupts (HMIs) are generated by various causes, signalled by bits in the hypervisor maintenance exception register (HMER). In most cases calling OPAL to handle the interrupt is the correct thing to do, but the "debug trigger" HMIs signalled by PPC bit 17 (bit 46) of HMER are used to invoke software workarounds for hardware bugs, and OPAL does not have any code to handle this cause. The debug trigger HMI is used in POWER9 DD2.0 and DD2.1 chips to work around a hardware bug in executing vector load instructions to cache inhibited memory. In POWER9 DD2.2 chips, it is generated when conditions are detected relating to threads being in TM (transactional memory) suspended mode when the core SMT configuration needs to be reconfigured. The kernel currently has code to detect the vector CI load condition, but only when the HMI occurs in the host, not when it occurs in a guest. If a HMI occurs in the guest, it is always passed to OPAL, and then we always re-sync the timebase, because the HMI cause might have been a timebase error, for which OPAL would re-sync the timebase, thus removing the timebase offset which KVM applied for the guest. Since we don't know what OPAL did, we don't know whether to subtract the timebase offset from the timebase, so instead we re-sync the timebase. This adds code to determine explicitly what the cause of a debug trigger HMI will be. This is based on a new device-tree property under the CPU nodes called ibm,hmi-special-triggers, if it is present, or otherwise based on the PVR (processor version register). The handling of debug trigger HMIs is pulled out into a separate function which can be called from the KVM guest exit code. If this function handles and clears the HMI, and no other HMI causes remain, then we skip calling OPAL and we proceed to subtract the guest timebase offset from the timebase. The overall handling for HMIs that occur in the host (i.e. not in a KVM guest) is largely unchanged, except that we now don't set the flag for the vector CI load workaround on DD2.2 processors. This also removes a BUG_ON in the KVM code. BUG_ON is generally not useful in KVM guest entry/exit code since it is difficult to handle the resulting trap gracefully. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hmi.h | 4 + arch/powerpc/include/asm/reg.h | 5 +- arch/powerpc/kernel/mce.c | 142 +++++++++++++++++++++++++------- arch/powerpc/kvm/book3s_hv_ras.c | 8 +- arch/powerpc/kvm/book3s_hv_rmhandlers.S | 9 +- 5 files changed, 131 insertions(+), 37 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hmi.h b/arch/powerpc/include/asm/hmi.h index 85b7a1a21e22..9c14f7b5c46c 100644 --- a/arch/powerpc/include/asm/hmi.h +++ b/arch/powerpc/include/asm/hmi.h @@ -42,4 +42,8 @@ extern void wait_for_tb_resync(void); static inline void wait_for_subcore_guest_exit(void) { } static inline void wait_for_tb_resync(void) { } #endif + +struct pt_regs; +extern long hmi_handle_debugtrig(struct pt_regs *regs); + #endif /* __ASM_PPC64_HMI_H__ */ diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index b779f3ccd412..14e41b843952 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -432,8 +432,9 @@ #define SPRN_LPID 0x13F /* Logical Partition Identifier */ #endif #define LPID_RSVD 0x3ff /* Reserved LPID for partn switching */ -#define SPRN_HMER 0x150 /* Hardware m? error recovery */ -#define SPRN_HMEER 0x151 /* Hardware m? enable error recovery */ +#define SPRN_HMER 0x150 /* Hypervisor maintenance exception reg */ +#define HMER_DEBUG_TRIG (1ul << (63 - 17)) /* Debug trigger */ +#define SPRN_HMEER 0x151 /* Hyp maintenance exception enable reg */ #define SPRN_PCR 0x152 /* Processor compatibility register */ #define PCR_VEC_DIS (1ul << (63-0)) /* Vec. disable (bit NA since POWER8) */ #define PCR_VSX_DIS (1ul << (63-1)) /* VSX disable (bit NA since POWER8) */ diff --git a/arch/powerpc/kernel/mce.c b/arch/powerpc/kernel/mce.c index 742e4658c5dc..d2fecaec4fec 100644 --- a/arch/powerpc/kernel/mce.c +++ b/arch/powerpc/kernel/mce.c @@ -495,37 +495,123 @@ long machine_check_early(struct pt_regs *regs) return handled; } -long hmi_exception_realmode(struct pt_regs *regs) +/* Possible meanings for HMER_DEBUG_TRIG bit being set on POWER9 */ +static enum { + DTRIG_UNKNOWN, + DTRIG_VECTOR_CI, /* need to emulate vector CI load instr */ + DTRIG_SUSPEND_ESCAPE, /* need to escape from TM suspend mode */ +} hmer_debug_trig_function; + +static int init_debug_trig_function(void) { - __this_cpu_inc(irq_stat.hmi_exceptions); - -#ifdef CONFIG_PPC_BOOK3S_64 - /* Workaround for P9 vector CI loads (see p9_hmi_special_emu) */ - if (pvr_version_is(PVR_POWER9)) { - unsigned long hmer = mfspr(SPRN_HMER); - - /* Do we have the debug bit set */ - if (hmer & PPC_BIT(17)) { - hmer &= ~PPC_BIT(17); - mtspr(SPRN_HMER, hmer); - - /* - * Now to avoid problems with soft-disable we - * only do the emulation if we are coming from - * user space - */ - if (user_mode(regs)) - local_paca->hmi_p9_special_emu = 1; - - /* - * Don't bother going to OPAL if that's the - * only relevant bit. - */ - if (!(hmer & mfspr(SPRN_HMEER))) - return local_paca->hmi_p9_special_emu; + int pvr; + struct device_node *cpun; + struct property *prop = NULL; + const char *str; + + /* First look in the device tree */ + preempt_disable(); + cpun = of_get_cpu_node(smp_processor_id(), NULL); + if (cpun) { + of_property_for_each_string(cpun, "ibm,hmi-special-triggers", + prop, str) { + if (strcmp(str, "bit17-vector-ci-load") == 0) + hmer_debug_trig_function = DTRIG_VECTOR_CI; + else if (strcmp(str, "bit17-tm-suspend-escape") == 0) + hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE; } + of_node_put(cpun); + } + preempt_enable(); + + /* If we found the property, don't look at PVR */ + if (prop) + goto out; + + pvr = mfspr(SPRN_PVR); + /* Check for POWER9 Nimbus (scale-out) */ + if ((PVR_VER(pvr) == PVR_POWER9) && (pvr & 0xe000) == 0) { + /* DD2.2 and later */ + if ((pvr & 0xfff) >= 0x202) + hmer_debug_trig_function = DTRIG_SUSPEND_ESCAPE; + /* DD2.0 and DD2.1 - used for vector CI load emulation */ + else if ((pvr & 0xfff) >= 0x200) + hmer_debug_trig_function = DTRIG_VECTOR_CI; + } + + out: + switch (hmer_debug_trig_function) { + case DTRIG_VECTOR_CI: + pr_debug("HMI debug trigger used for vector CI load\n"); + break; + case DTRIG_SUSPEND_ESCAPE: + pr_debug("HMI debug trigger used for TM suspend escape\n"); + break; + default: + break; } -#endif /* CONFIG_PPC_BOOK3S_64 */ + return 0; +} +__initcall(init_debug_trig_function); + +/* + * Handle HMIs that occur as a result of a debug trigger. + * Return values: + * -1 means this is not a HMI cause that we know about + * 0 means no further handling is required + * 1 means further handling is required + */ +long hmi_handle_debugtrig(struct pt_regs *regs) +{ + unsigned long hmer = mfspr(SPRN_HMER); + long ret = 0; + + /* HMER_DEBUG_TRIG bit is used for various workarounds on P9 */ + if (!((hmer & HMER_DEBUG_TRIG) + && hmer_debug_trig_function != DTRIG_UNKNOWN)) + return -1; + + hmer &= ~HMER_DEBUG_TRIG; + /* HMER is a write-AND register */ + mtspr(SPRN_HMER, ~HMER_DEBUG_TRIG); + + switch (hmer_debug_trig_function) { + case DTRIG_VECTOR_CI: + /* + * Now to avoid problems with soft-disable we + * only do the emulation if we are coming from + * host user space + */ + if (regs && user_mode(regs)) + ret = local_paca->hmi_p9_special_emu = 1; + + break; + + default: + break; + } + + /* + * See if any other HMI causes remain to be handled + */ + if (hmer & mfspr(SPRN_HMEER)) + return -1; + + return ret; +} + +/* + * Return values: + */ +long hmi_exception_realmode(struct pt_regs *regs) +{ + int ret; + + __this_cpu_inc(irq_stat.hmi_exceptions); + + ret = hmi_handle_debugtrig(regs); + if (ret >= 0) + return ret; wait_for_subcore_guest_exit(); diff --git a/arch/powerpc/kvm/book3s_hv_ras.c b/arch/powerpc/kvm/book3s_hv_ras.c index c356f9a40b24..c296343d0dcc 100644 --- a/arch/powerpc/kvm/book3s_hv_ras.c +++ b/arch/powerpc/kvm/book3s_hv_ras.c @@ -268,17 +268,19 @@ static void kvmppc_tb_resync_done(void) * secondary threads to proceed. * - All secondary threads will eventually call opal hmi handler on * their exit path. + * + * Returns 1 if the timebase offset should be applied, 0 if not. */ long kvmppc_realmode_hmi_handler(void) { - int ptid = local_paca->kvm_hstate.ptid; bool resync_req; - /* This is only called on primary thread. */ - BUG_ON(ptid != 0); __this_cpu_inc(irq_stat.hmi_exceptions); + if (hmi_handle_debugtrig(NULL) >= 0) + return 1; + /* * By now primary thread has already completed guest->host * partition switch but haven't signaled secondaries yet. diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 2659844784b8..bd0b623335af 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -1909,16 +1909,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) bne 27f bl kvmppc_realmode_hmi_handler nop + cmpdi r3, 0 li r12, BOOK3S_INTERRUPT_HMI /* - * At this point kvmppc_realmode_hmi_handler would have resync-ed - * the TB. Hence it is not required to subtract guest timebase - * offset from timebase. So, skip it. + * At this point kvmppc_realmode_hmi_handler may have resync-ed + * the TB, and if it has, we must not subtract the guest timebase + * offset from the timebase. So, skip it. * * Also, do not call kvmppc_subcore_exit_guest() because it has * been invoked as part of kvmppc_realmode_hmi_handler(). */ - b 30f + beq 30f 27: /* Subtract timebase offset from timebase */ -- cgit v1.2.3 From 47712a921bb781caf69fca9eae43be19968816cb Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 17 Jan 2018 22:47:22 +1000 Subject: powerpc/watchdog: remove arch_trigger_cpumask_backtrace The powerpc NMI IPIs may not be recoverable if they are taken in some sections of code, and also there have been and still are issues with taking NMIs (in KVM guest code, in firmware, etc) which makes them a bit dangerous to use. Generic code like softlockup detector and rcu stall detectors really hammer on trigger_*_backtrace, which has lead to further problems because we've implemented it with the NMI. So stop providing NMI backtraces for now. Importantly, the powerpc code uses NMI IPIs in crash/debug, and the SMP hardlockup watchdog. So if the softlockup and rcu hang detection traces are not being printed because the CPU is stuck with interrupts off, then the hard lockup watchdog should get it with the NMI IPI. Fixes: 2104180a5369 ("powerpc/64s: implement arch-specific hardlockup watchdog") Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/nmi.h | 4 ---- arch/powerpc/kernel/watchdog.c | 22 ---------------------- 2 files changed, 26 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/nmi.h b/arch/powerpc/include/asm/nmi.h index e97f58689ca7..9c80939b4d14 100644 --- a/arch/powerpc/include/asm/nmi.h +++ b/arch/powerpc/include/asm/nmi.h @@ -4,10 +4,6 @@ #ifdef CONFIG_PPC_WATCHDOG extern void arch_touch_nmi_watchdog(void); -extern void arch_trigger_cpumask_backtrace(const cpumask_t *mask, - bool exclude_self); -#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace - #else static inline void arch_touch_nmi_watchdog(void) {} #endif diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c index 87da80ccced1..3963baaba92d 100644 --- a/arch/powerpc/kernel/watchdog.c +++ b/arch/powerpc/kernel/watchdog.c @@ -393,25 +393,3 @@ int __init watchdog_nmi_probe(void) } return 0; } - -static void handle_backtrace_ipi(struct pt_regs *regs) -{ - nmi_cpu_backtrace(regs); -} - -static void raise_backtrace_ipi(cpumask_t *mask) -{ - unsigned int cpu; - - for_each_cpu(cpu, mask) { - if (cpu == smp_processor_id()) - handle_backtrace_ipi(NULL); - else - smp_send_nmi_ipi(cpu, handle_backtrace_ipi, 1000000); - } -} - -void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) -{ - nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_backtrace_ipi); -} -- cgit v1.2.3 From c16bee4bded5449ec3b3ec73579ba29881f2e978 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 17 Nov 2017 02:00:49 +1000 Subject: powerpc: define __ARCH_IRQ_EXIT_IRQS_DISABLED powerpc calls irq_exit() with local irqs disabled, therefore it can define __ARCH_IRQ_EXIT_IRQS_DISABLED. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hardirq.h | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hardirq.h b/arch/powerpc/include/asm/hardirq.h index 456f9e7b8d83..5986d473722b 100644 --- a/arch/powerpc/include/asm/hardirq.h +++ b/arch/powerpc/include/asm/hardirq.h @@ -29,6 +29,7 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); #define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending) #define __ARCH_SET_SOFTIRQ_PENDING +#define __ARCH_IRQ_EXIT_IRQS_DISABLED #define set_softirq_pending(x) __this_cpu_write(irq_stat.__softirq_pending, (x)) #define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x)) -- cgit v1.2.3 From 7a074fc08389f43b448ebef74bf56ba0f6f087d9 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Fri, 19 Jan 2018 15:20:12 +1100 Subject: powerpc/64s: Fix ps3 build error due to tlbiel_all() The recent changes to TLB handling broke the PS3 build: arch/powerpc/include/asm/book3s/64/tlbflush.h:30: undefined reference to `.hash__tlbiel_all' Fix it by adding an fallback version of tlbiel_all() for non-native builds. It should never be called, due to checks in callers so it calls BUG(). We should probably clean it up further but this will suffice for now. Fixes: d4748276ae14 ("powerpc/64s: Improve local TLB flush for boot and MCE on POWER9") Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/tlbflush.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h index 9befb4df235c..0cac17253513 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h @@ -14,6 +14,7 @@ enum { TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */ }; +#ifdef CONFIG_PPC_NATIVE static inline void tlbiel_all(void) { /* @@ -29,6 +30,9 @@ static inline void tlbiel_all(void) else hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL); } +#else +static inline void tlbiel_all(void) { BUG(); }; +#endif static inline void tlbiel_all_lpid(bool radix) { -- cgit v1.2.3 From c2e480ba822718190e58849b79a76db13c3dac18 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:42 +0530 Subject: powerpc/64: Add #defines for paca->soft_enabled flags Two #defines IRQS_ENABLED and IRQS_DISABLED are added to be used when updating paca->soft_enabled. Replace the hardcoded values used when updating paca->soft_enabled with IRQ_(EN|DIS)ABLED #define. No logic change. Reviewed-by: Nicholas Piggin Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 2 +- arch/powerpc/include/asm/hw_irq.h | 21 ++++++++++++++------- arch/powerpc/include/asm/irqflags.h | 6 +++--- arch/powerpc/include/asm/kvm_ppc.h | 2 +- arch/powerpc/kernel/entry_64.S | 16 ++++++++-------- arch/powerpc/kernel/exceptions-64e.S | 6 +++--- arch/powerpc/kernel/head_64.S | 5 +++-- arch/powerpc/kernel/idle_book3e.S | 3 ++- arch/powerpc/kernel/idle_power4.S | 3 ++- arch/powerpc/kernel/irq.c | 9 +++++---- arch/powerpc/kernel/process.c | 3 ++- arch/powerpc/kernel/setup_64.c | 3 +++ arch/powerpc/kernel/time.c | 2 +- arch/powerpc/mm/hugetlbpage.c | 2 +- arch/powerpc/perf/core-book3s.c | 2 +- 15 files changed, 50 insertions(+), 35 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index b27205297e1d..69c4e3d35e02 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -499,7 +499,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define __SOFTEN_TEST(h, vec) \ lbz r10,PACASOFTIRQEN(r13); \ - cmpwi r10,0; \ + cmpwi r10,IRQS_DISABLED; \ li r10,SOFTEN_VALUE_##vec; \ beq masked_##h##interrupt diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 3818fa0164f0..7c2717dfd89a 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -28,6 +28,12 @@ #define PACA_IRQ_EE_EDGE 0x10 /* BookE only */ #define PACA_IRQ_HMI 0x20 +/* + * flags for paca->soft_enabled + */ +#define IRQS_ENABLED 1 +#define IRQS_DISABLED 0 + #endif /* CONFIG_PPC64 */ #ifndef __ASSEMBLY__ @@ -60,9 +66,10 @@ static inline unsigned long arch_local_irq_disable(void) unsigned long flags, zero; asm volatile( - "li %1,0; lbz %0,%2(13); stb %1,%2(13)" + "li %1,%3; lbz %0,%2(13); stb %1,%2(13)" : "=r" (flags), "=&r" (zero) - : "i" (offsetof(struct paca_struct, soft_enabled)) + : "i" (offsetof(struct paca_struct, soft_enabled)),\ + "i" (IRQS_DISABLED) : "memory"); return flags; @@ -72,7 +79,7 @@ extern void arch_local_irq_restore(unsigned long); static inline void arch_local_irq_enable(void) { - arch_local_irq_restore(1); + arch_local_irq_restore(IRQS_ENABLED); } static inline unsigned long arch_local_irq_save(void) @@ -82,7 +89,7 @@ static inline unsigned long arch_local_irq_save(void) static inline bool arch_irqs_disabled_flags(unsigned long flags) { - return flags == 0; + return flags == IRQS_DISABLED; } static inline bool arch_irqs_disabled(void) @@ -102,9 +109,9 @@ static inline bool arch_irqs_disabled(void) u8 _was_enabled; \ __hard_irq_disable(); \ _was_enabled = local_paca->soft_enabled; \ - local_paca->soft_enabled = 0; \ + local_paca->soft_enabled = IRQS_DISABLED;\ local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ - if (_was_enabled) \ + if (_was_enabled == IRQS_ENABLED) \ trace_hardirqs_off(); \ } while(0) @@ -127,7 +134,7 @@ static inline void may_hard_irq_enable(void) static inline bool arch_irq_disabled_regs(struct pt_regs *regs) { - return !regs->softe; + return (regs->softe == IRQS_DISABLED); } extern bool prep_irq_for_idle(void); diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h index 1aeb5f13b8c4..8d9fdc84828d 100644 --- a/arch/powerpc/include/asm/irqflags.h +++ b/arch/powerpc/include/asm/irqflags.h @@ -49,8 +49,8 @@ #define RECONCILE_IRQ_STATE(__rA, __rB) \ lbz __rA,PACASOFTIRQEN(r13); \ lbz __rB,PACAIRQHAPPENED(r13); \ - cmpwi cr0,__rA,0; \ - li __rA,0; \ + cmpwi cr0,__rA,IRQS_DISABLED;\ + li __rA,IRQS_DISABLED; \ ori __rB,__rB,PACA_IRQ_HARD_DIS; \ stb __rB,PACAIRQHAPPENED(r13); \ beq 44f; \ @@ -64,7 +64,7 @@ #define RECONCILE_IRQ_STATE(__rA, __rB) \ lbz __rA,PACAIRQHAPPENED(r13); \ - li __rB,0; \ + li __rB,IRQS_DISABLED; \ ori __rA,__rA,PACA_IRQ_HARD_DIS; \ stb __rB,PACASOFTIRQEN(r13); \ stb __rA,PACAIRQHAPPENED(r13) diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 941c2a3f231b..68484d77a3cb 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -873,7 +873,7 @@ static inline void kvmppc_fix_ee_before_entry(void) /* Only need to enable IRQs by hard enabling them after this */ local_paca->irq_happened = 0; - local_paca->soft_enabled = 1; + local_paca->soft_enabled = IRQS_ENABLED; #endif } diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 371c05fe250a..f8b32224dc11 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -130,7 +130,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) */ #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG) lbz r10,PACASOFTIRQEN(r13) - xori r10,r10,1 + xori r10,r10,IRQS_ENABLED 1: tdnei r10,0 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif @@ -147,7 +147,7 @@ system_call: /* label this so stack traces look sane */ /* We do need to set SOFTE in the stack frame or the return * from interrupt will be painful */ - li r10,1 + li r10,IRQS_ENABLED std r10,SOFTE(r1) CURRENT_THREAD_INFO(r11, r1) @@ -743,7 +743,7 @@ resume_kernel: lwz r8,TI_PREEMPT(r9) cmpwi cr1,r8,0 ld r0,SOFTE(r1) - cmpdi r0,0 + cmpdi r0,IRQS_DISABLED crandc eq,cr1*4+eq,eq bne restore @@ -783,11 +783,11 @@ restore: */ ld r5,SOFTE(r1) lbz r6,PACASOFTIRQEN(r13) - cmpwi cr0,r5,0 + cmpwi cr0,r5,IRQS_DISABLED beq .Lrestore_irq_off /* We are enabling, were we already enabled ? Yes, just return */ - cmpwi cr0,r6,1 + cmpwi cr0,r6,IRQS_ENABLED beq cr0,.Ldo_restore /* @@ -806,7 +806,7 @@ restore: */ .Lrestore_no_replay: TRACE_ENABLE_INTS - li r0,1 + li r0,IRQS_ENABLED stb r0,PACASOFTIRQEN(r13); /* @@ -915,7 +915,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG) /* The interrupt should not have soft enabled. */ lbz r7,PACASOFTIRQEN(r13) -1: tdnei r7,0 +1: tdnei r7,IRQS_DISABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif b .Ldo_restore @@ -1036,7 +1036,7 @@ _GLOBAL(enter_rtas) * check it with the asm equivalent of WARN_ON */ lbz r0,PACASOFTIRQEN(r13) -1: tdnei r0,0 +1: tdnei r0,IRQS_DISABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index acd8ca76233e..80cc91bbcf1a 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -210,9 +210,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) ld r5,SOFTE(r1) /* Interrupts had better not already be enabled... */ - twnei r6,0 + twnei r6,IRQS_DISABLED - cmpwi cr0,r5,0 + cmpwi cr0,r5,IRQS_DISABLED beq 1f TRACE_ENABLE_INTS @@ -352,7 +352,7 @@ ret_from_mc_except: #define PROLOG_ADDITION_MASKABLE_GEN(n) \ lbz r10,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \ - cmpwi cr0,r10,0; /* yes -> go out of line */ \ + cmpwi cr0,r10,IRQS_DISABLED; /* yes -> go out of line */ \ beq masked_interrupt_book3e_##n #define PROLOG_ADDITION_2REGS_GEN(n) \ diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index aa71a90f5222..0deef350004f 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -765,7 +765,7 @@ _GLOBAL(pmac_secondary_start) /* Mark interrupts soft and hard disabled (they might be enabled * in the PACA when doing hotplug) */ - li r0,0 + li r0,IRQS_DISABLED stb r0,PACASOFTIRQEN(r13) li r0,PACA_IRQ_HARD_DIS stb r0,PACAIRQHAPPENED(r13) @@ -822,6 +822,7 @@ __secondary_start: /* Mark interrupts soft and hard disabled (they might be enabled * in the PACA when doing hotplug) */ + li r7,IRQS_DISABLED stb r7,PACASOFTIRQEN(r13) li r0,PACA_IRQ_HARD_DIS stb r0,PACAIRQHAPPENED(r13) @@ -988,7 +989,7 @@ start_here_common: /* Mark interrupts soft and hard disabled (they might be enabled * in the PACA when doing hotplug) */ - li r0,0 + li r0,IRQS_DISABLED stb r0,PACASOFTIRQEN(r13) li r0,PACA_IRQ_HARD_DIS stb r0,PACAIRQHAPPENED(r13) diff --git a/arch/powerpc/kernel/idle_book3e.S b/arch/powerpc/kernel/idle_book3e.S index 48c21acef915..1bb1d152aa46 100644 --- a/arch/powerpc/kernel/idle_book3e.S +++ b/arch/powerpc/kernel/idle_book3e.S @@ -17,6 +17,7 @@ #include #include #include +#include /* 64-bit version only for now */ #ifdef CONFIG_PPC64 @@ -46,7 +47,7 @@ _GLOBAL(\name) bl trace_hardirqs_on addi r1,r1,128 #endif - li r0,1 + li r0,IRQS_ENABLED stb r0,PACASOFTIRQEN(r13) /* Interrupts will make use return to LR, so get something we want diff --git a/arch/powerpc/kernel/idle_power4.S b/arch/powerpc/kernel/idle_power4.S index f57a19348bdd..8b4702d93701 100644 --- a/arch/powerpc/kernel/idle_power4.S +++ b/arch/powerpc/kernel/idle_power4.S @@ -15,6 +15,7 @@ #include #include #include +#include #undef DEBUG @@ -53,7 +54,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP) mfmsr r7 #endif /* CONFIG_TRACE_IRQFLAGS */ - li r0,1 + li r0,IRQS_ENABLED stb r0,PACASOFTIRQEN(r13) /* we'll hard-enable shortly */ BEGIN_FTR_SECTION DSSALL diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index b7a84522e652..483a9206554f 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -67,6 +67,7 @@ #include #include #include +#include #ifdef CONFIG_PPC64 #include @@ -231,7 +232,7 @@ notrace void arch_local_irq_restore(unsigned long en) /* Write the new soft-enabled value */ set_soft_enabled(en); - if (!en) + if (en == IRQS_DISABLED) return; /* * From this point onward, we can take interrupts, preempt, @@ -276,7 +277,7 @@ notrace void arch_local_irq_restore(unsigned long en) } #endif /* CONFIG_TRACE_IRQFLAGS */ - set_soft_enabled(0); + set_soft_enabled(IRQS_DISABLED); trace_hardirqs_off(); /* @@ -288,7 +289,7 @@ notrace void arch_local_irq_restore(unsigned long en) /* We can soft-enable now */ trace_hardirqs_on(); - set_soft_enabled(1); + set_soft_enabled(IRQS_ENABLED); /* * And replay if we have to. This will return with interrupts @@ -363,7 +364,7 @@ bool prep_irq_for_idle(void) * of entering the low power state. */ local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; - local_paca->soft_enabled = 1; + local_paca->soft_enabled = IRQS_ENABLED; /* Tell the caller to enter the low power state */ return true; diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index bcd4441304a5..1563d1190da3 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -57,6 +57,7 @@ #include #ifdef CONFIG_PPC64 #include +#include #endif #include #include @@ -1674,7 +1675,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childregs->gpr[14] = ppc_function_entry((void *)usp); #ifdef CONFIG_PPC64 clear_tsk_thread_flag(p, TIF_32BIT); - childregs->softe = 1; + childregs->softe = IRQS_ENABLED; #endif childregs->gpr[15] = kthread_arg; p->thread.regs = NULL; /* no user register state */ diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 8f285d6a3db1..2fd4e167ef09 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -66,6 +66,7 @@ #include #include #include +#include #include "setup.h" @@ -187,6 +188,8 @@ static void __init fixup_boot_paca(void) get_paca()->cpu_start = 1; /* Allow percpu accesses to work until we setup percpu data */ get_paca()->data_offset = 0; + /* Mark interrupts disabled in PACA */ + get_paca()->soft_enabled = IRQS_DISABLED; } static void __init configure_exceptions(void) diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index fe6f3a285455..070092b1ba8a 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -253,7 +253,7 @@ void accumulate_stolen_time(void) * needs to reflect that so various debug stuff doesn't * complain */ - local_paca->soft_enabled = 0; + local_paca->soft_enabled = IRQS_DISABLED; sst = scan_dispatch_log(acct->starttime_user); ust = scan_dispatch_log(acct->starttime); diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index f5d1008f8574..760bb5a2b8bd 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -752,7 +752,7 @@ void flush_dcache_icache_hugepage(struct page *page) * So long as we atomically load page table pointers we are safe against teardown, * we can follow the address down to the the page and take a ref on it. * This function need to be called with interrupts disabled. We use this variant - * when we have MSR[EE] = 0 but the paca->soft_enabled = 1 + * when we have MSR[EE] = 0 but the paca->soft_enabled = IRQS_ENABLED */ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, bool *is_thp, unsigned *hpage_shift) diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 9e3da168d54c..594261e308b3 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -322,7 +322,7 @@ static inline void perf_read_regs(struct pt_regs *regs) */ static inline int perf_intr_is_nmi(struct pt_regs *regs) { - return !regs->softe; + return (regs->softe == IRQS_DISABLED); } /* -- cgit v1.2.3 From 9f83e00f4cc1c560b6847acecba2b3746c8d8c06 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 20 Dec 2017 09:25:43 +0530 Subject: powerpc/64: Improve inline asm in arch_local_irq_disable arch_local_irq_disable is implemented strangely, with a temporary output register being set to the desired soft_enabled value via an immediate input, which is then used to store to memory. This is not required, the immediate can be specified directly as a register input. For simple cases at least, assembly is unchanged except register mapping. Reviewed-by: Madhavan Srinivasan Signed-off-by: Nicholas Piggin Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hw_irq.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 7c2717dfd89a..4210ddbf38b0 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -63,13 +63,13 @@ static inline unsigned long arch_local_save_flags(void) static inline unsigned long arch_local_irq_disable(void) { - unsigned long flags, zero; + unsigned long flags; asm volatile( - "li %1,%3; lbz %0,%2(13); stb %1,%2(13)" - : "=r" (flags), "=&r" (zero) - : "i" (offsetof(struct paca_struct, soft_enabled)),\ - "i" (IRQS_DISABLED) + "lbz %0,%1(13); stb %2,%1(13)" + : "=&r" (flags) + : "i" (offsetof(struct paca_struct, soft_enabled)), + "r" (IRQS_DISABLED) : "memory"); return flags; -- cgit v1.2.3 From b5c1bd62c054f3cff1a672f9bf1dddefafadffec Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:44 +0530 Subject: powerpc/64: Fix arch_local_irq_disable() prototype In powerpc/64, the arch_local_irq_disable() function returns unsigned long, which is not consistent with other architectures. Move that set-return asm implementation into arch_local_irq_save(), and make arch_local_irq_disable() return void, simplifying the assembly. Suggested-by: Nicholas Piggin Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hw_irq.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 4210ddbf38b0..4c54db29104f 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -61,18 +61,14 @@ static inline unsigned long arch_local_save_flags(void) return flags; } -static inline unsigned long arch_local_irq_disable(void) +static inline void arch_local_irq_disable(void) { - unsigned long flags; - asm volatile( - "lbz %0,%1(13); stb %2,%1(13)" - : "=&r" (flags) - : "i" (offsetof(struct paca_struct, soft_enabled)), - "r" (IRQS_DISABLED) + "stb %0,%1(13)" + : + : "r" (IRQS_DISABLED), + "i" (offsetof(struct paca_struct, soft_enabled)) : "memory"); - - return flags; } extern void arch_local_irq_restore(unsigned long); @@ -84,7 +80,16 @@ static inline void arch_local_irq_enable(void) static inline unsigned long arch_local_irq_save(void) { - return arch_local_irq_disable(); + unsigned long flags; + + asm volatile( + "lbz %0,%1(13); stb %2,%1(13)" + : "=&r" (flags) + : "i" (offsetof(struct paca_struct, soft_enabled)), + "r" (IRQS_DISABLED) + : "memory"); + + return flags; } static inline bool arch_irqs_disabled_flags(unsigned long flags) -- cgit v1.2.3 From 0b63acf4a0eb8843f83954ea1bd29ccdfcbaa778 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:45 +0530 Subject: powerpc/64: Move set_soft_enabled() and rename Move set_soft_enabled() from powerpc/kernel/irq.c to asm/hw_irq.c, to encourage updates to paca->soft_enabled done via these access function. Add "memory" clobber to hint compiler since paca->soft_enabled memory is the target here. Renaming it as soft_enabled_set() will make namespaces works better as prefix than a postfix when new soft_enabled manipulation functions are introduced. Reviewed-by: Nicholas Piggin Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hw_irq.h | 22 ++++++++++++++++------ arch/powerpc/include/asm/kvm_ppc.h | 2 +- arch/powerpc/kernel/irq.c | 14 ++++---------- arch/powerpc/kernel/setup_64.c | 4 ++-- arch/powerpc/kernel/time.c | 4 ++-- 5 files changed, 25 insertions(+), 21 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 4c54db29104f..d046d9f3b777 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -49,6 +49,21 @@ extern void unknown_exception(struct pt_regs *regs); #ifdef CONFIG_PPC64 #include +/* + * The "memory" clobber acts as both a compiler barrier + * for the critical section and as a clobber because + * we changed paca->soft_enabled + */ +static inline notrace void soft_enabled_set(unsigned long enable) +{ + asm volatile( + "stb %0,%1(13)" + : + : "r" (enable), + "i" (offsetof(struct paca_struct, soft_enabled)) + : "memory"); +} + static inline unsigned long arch_local_save_flags(void) { unsigned long flags; @@ -63,12 +78,7 @@ static inline unsigned long arch_local_save_flags(void) static inline void arch_local_irq_disable(void) { - asm volatile( - "stb %0,%1(13)" - : - : "r" (IRQS_DISABLED), - "i" (offsetof(struct paca_struct, soft_enabled)) - : "memory"); + soft_enabled_set(IRQS_DISABLED); } extern void arch_local_irq_restore(unsigned long); diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 68484d77a3cb..369f0640826c 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -873,7 +873,7 @@ static inline void kvmppc_fix_ee_before_entry(void) /* Only need to enable IRQs by hard enabling them after this */ local_paca->irq_happened = 0; - local_paca->soft_enabled = IRQS_ENABLED; + soft_enabled_set(IRQS_ENABLED); #endif } diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 483a9206554f..6c04e465caf5 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -107,12 +107,6 @@ static inline notrace unsigned long get_irq_happened(void) return happened; } -static inline notrace void set_soft_enabled(unsigned long enable) -{ - __asm__ __volatile__("stb %0,%1(13)" - : : "r" (enable), "i" (offsetof(struct paca_struct, soft_enabled))); -} - static inline notrace int decrementer_check_overflow(void) { u64 now = get_tb_or_rtc(); @@ -231,7 +225,7 @@ notrace void arch_local_irq_restore(unsigned long en) unsigned int replay; /* Write the new soft-enabled value */ - set_soft_enabled(en); + soft_enabled_set(en); if (en == IRQS_DISABLED) return; /* @@ -277,7 +271,7 @@ notrace void arch_local_irq_restore(unsigned long en) } #endif /* CONFIG_TRACE_IRQFLAGS */ - set_soft_enabled(IRQS_DISABLED); + soft_enabled_set(IRQS_DISABLED); trace_hardirqs_off(); /* @@ -289,7 +283,7 @@ notrace void arch_local_irq_restore(unsigned long en) /* We can soft-enable now */ trace_hardirqs_on(); - set_soft_enabled(IRQS_ENABLED); + soft_enabled_set(IRQS_ENABLED); /* * And replay if we have to. This will return with interrupts @@ -364,7 +358,7 @@ bool prep_irq_for_idle(void) * of entering the low power state. */ local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; - local_paca->soft_enabled = IRQS_ENABLED; + soft_enabled_set(IRQS_ENABLED); /* Tell the caller to enter the low power state */ return true; diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 2fd4e167ef09..f6bedebda90a 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -189,7 +189,7 @@ static void __init fixup_boot_paca(void) /* Allow percpu accesses to work until we setup percpu data */ get_paca()->data_offset = 0; /* Mark interrupts disabled in PACA */ - get_paca()->soft_enabled = IRQS_DISABLED; + soft_enabled_set(IRQS_DISABLED); } static void __init configure_exceptions(void) @@ -352,7 +352,7 @@ void __init early_setup(unsigned long dt_ptr) void early_setup_secondary(void) { /* Mark interrupts disabled in PACA */ - get_paca()->soft_enabled = 0; + soft_enabled_set(IRQS_DISABLED); /* Initialize the hash table or TLB handling */ early_init_mmu_secondary(); diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 070092b1ba8a..320b8459c74e 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -253,7 +253,7 @@ void accumulate_stolen_time(void) * needs to reflect that so various debug stuff doesn't * complain */ - local_paca->soft_enabled = IRQS_DISABLED; + soft_enabled_set(IRQS_DISABLED); sst = scan_dispatch_log(acct->starttime_user); ust = scan_dispatch_log(acct->starttime); @@ -261,7 +261,7 @@ void accumulate_stolen_time(void) acct->utime -= ust; acct->steal_time += ust + sst; - local_paca->soft_enabled = save_soft_enabled; + soft_enabled_set(save_soft_enabled); } static inline u64 calculate_stolen_time(u64 stop_tb) -- cgit v1.2.3 From e0b5687bed13ddbf99985d77910b9adfd429bf12 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:46 +0530 Subject: powerpc/64: Implement and use soft_enabled_return API Add a new wrapper function, soft_enabled_return(), added to return paca->soft_enabled value. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hw_irq.h | 21 +++++++++++++-------- arch/powerpc/kernel/time.c | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index d046d9f3b777..f9791884af08 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -49,6 +49,18 @@ extern void unknown_exception(struct pt_regs *regs); #ifdef CONFIG_PPC64 #include +static inline notrace unsigned long soft_enabled_return(void) +{ + unsigned long flags; + + asm volatile( + "lbz %0,%1(13)" + : "=r" (flags) + : "i" (offsetof(struct paca_struct, soft_enabled))); + + return flags; +} + /* * The "memory" clobber acts as both a compiler barrier * for the critical section and as a clobber because @@ -66,14 +78,7 @@ static inline notrace void soft_enabled_set(unsigned long enable) static inline unsigned long arch_local_save_flags(void) { - unsigned long flags; - - asm volatile( - "lbz %0,%1(13)" - : "=r" (flags) - : "i" (offsetof(struct paca_struct, soft_enabled))); - - return flags; + return soft_enabled_return(); } static inline void arch_local_irq_disable(void) diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index 320b8459c74e..daa6e9e35ab9 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -244,7 +244,7 @@ static u64 scan_dispatch_log(u64 stop_tb) void accumulate_stolen_time(void) { u64 sst, ust; - u8 save_soft_enabled = local_paca->soft_enabled; + unsigned long save_soft_enabled = soft_enabled_return(); struct cpu_accounting_data *acct = &local_paca->accounting; /* We are called early in the exception entry, before -- cgit v1.2.3 From a67c543aacb247a74e9d48fd9fde6d8369cea10b Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:47 +0530 Subject: powerpc/64: Implement and use soft_enabled_set_return API Add a new wrapper function, soft_enabled_set_return(), added to do the paca->soft_enabled updates requiring a set-return. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hw_irq.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index f9791884af08..c1764fabf181 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -76,6 +76,20 @@ static inline notrace void soft_enabled_set(unsigned long enable) : "memory"); } +static inline notrace unsigned long soft_enabled_set_return(unsigned long enable) +{ + unsigned long flags; + + asm volatile( + "lbz %0,%1(13); stb %2,%1(13)" + : "=&r" (flags) + : "i" (offsetof(struct paca_struct, soft_enabled)), + "r" (enable) + : "memory"); + + return flags; +} + static inline unsigned long arch_local_save_flags(void) { return soft_enabled_return(); @@ -95,16 +109,7 @@ static inline void arch_local_irq_enable(void) static inline unsigned long arch_local_irq_save(void) { - unsigned long flags; - - asm volatile( - "lbz %0,%1(13); stb %2,%1(13)" - : "=&r" (flags) - : "i" (offsetof(struct paca_struct, soft_enabled)), - "r" (IRQS_DISABLED) - : "memory"); - - return flags; + return soft_enabled_set_return(IRQS_DISABLED); } static inline bool arch_irqs_disabled_flags(unsigned long flags) -- cgit v1.2.3 From acb396d7c2b8b5f0f567c980185bbddd10534b56 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:48 +0530 Subject: powerpc/64: Cleanup hard_irq_disable() macro Minor cleanup to use helper function for manipulating paca->soft_enabled variable. Suggested-by: Nicholas Piggin Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hw_irq.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index c1764fabf181..52afb1595cb0 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -131,12 +131,11 @@ static inline bool arch_irqs_disabled(void) #endif #define hard_irq_disable() do { \ - u8 _was_enabled; \ + unsigned long flags; \ __hard_irq_disable(); \ - _was_enabled = local_paca->soft_enabled; \ - local_paca->soft_enabled = IRQS_DISABLED;\ + flags = soft_enabled_set_return(IRQS_DISABLED); \ local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ - if (_was_enabled == IRQS_ENABLED) \ + if (!arch_irqs_disabled_flags(flags)) \ trace_hardirqs_off(); \ } while(0) -- cgit v1.2.3 From 01417c6cc7dc9195f721f7f9e9ea066090ccc99d Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:49 +0530 Subject: powerpc/64: Change soft_enabled from flag to bitmask "paca->soft_enabled" is used as a flag to mask some of interrupts. Currently supported flags values and their details: soft_enabled MSR[EE] 0 0 Disabled (PMI and HMI not masked) 1 1 Enabled "paca->soft_enabled" is initialized to 1 to make the interripts as enabled. arch_local_irq_disable() will toggle the value when interrupts needs to disbled. At this point, the interrupts are not actually disabled, instead, interrupt vector has code to check for the flag and mask it when it occurs. By "mask it", it update interrupt paca->irq_happened and return. arch_local_irq_restore() is called to re-enable interrupts, which checks and replays interrupts if any occured. Now, as mentioned, current logic doesnot mask "performance monitoring interrupts" and PMIs are implemented as NMI. But this patchset depends on local_irq_* for a successful local_* update. Meaning, mask all possible interrupts during local_* update and replay them after the update. So the idea here is to reserve the "paca->soft_enabled" logic. New values and details: soft_enabled MSR[EE] 1 0 Disabled (PMI and HMI not masked) 0 1 Enabled Reason for the this change is to create foundation for a third mask value "0x2" for "soft_enabled" to add support to mask PMIs. When ->soft_enabled is set to a value "3", PMI interrupts are mask and when set to a value of "1", PMI are not mask. With this patch also extends soft_enabled as interrupt disable mask. Current flags are renamed from IRQ_[EN?DIS}ABLED to IRQS_ENABLED and IRQS_DISABLED. Patch also fixes the ptrace call to force the user to see the softe value to be alway 1. Reason being, even though userspace has no business knowing about softe, it is part of pt_regs. Like-wise in signal context. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 4 ++-- arch/powerpc/include/asm/hw_irq.h | 30 +++++++++++++++++++++++------- arch/powerpc/include/asm/irqflags.h | 4 ++-- arch/powerpc/kernel/entry_64.S | 21 ++++++++++----------- arch/powerpc/kernel/exceptions-64e.S | 10 +++++----- arch/powerpc/kernel/irq.c | 20 +++++++++++++++++--- arch/powerpc/perf/core-book3s.c | 2 +- 7 files changed, 60 insertions(+), 31 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 69c4e3d35e02..a21adcb75fc6 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -499,9 +499,9 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define __SOFTEN_TEST(h, vec) \ lbz r10,PACASOFTIRQEN(r13); \ - cmpwi r10,IRQS_DISABLED; \ + andi. r10,r10,IRQS_DISABLED; \ li r10,SOFTEN_VALUE_##vec; \ - beq masked_##h##interrupt + bne masked_##h##interrupt #define _SOFTEN_TEST(h, vec) __SOFTEN_TEST(h, vec) diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 52afb1595cb0..f96280ee9540 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -31,8 +31,8 @@ /* * flags for paca->soft_enabled */ -#define IRQS_ENABLED 1 -#define IRQS_DISABLED 0 +#define IRQS_ENABLED 0 +#define IRQS_DISABLED 1 #endif /* CONFIG_PPC64 */ @@ -68,6 +68,18 @@ static inline notrace unsigned long soft_enabled_return(void) */ static inline notrace void soft_enabled_set(unsigned long enable) { +#ifdef CONFIG_TRACE_IRQFLAGS + /* + * mask must always include LINUX bit if any are set, and + * interrupts don't get replayed until the Linux interrupt is + * unmasked. This could be changed to replay partial unmasks + * in future, which would allow Linux masks to nest inside + * other masks, among other things. For now, be very dumb and + * simple. + */ + WARN_ON(mask && !(mask & IRQS_DISABLED)); +#endif + asm volatile( "stb %0,%1(13)" : @@ -76,15 +88,19 @@ static inline notrace void soft_enabled_set(unsigned long enable) : "memory"); } -static inline notrace unsigned long soft_enabled_set_return(unsigned long enable) +static inline notrace unsigned long soft_enabled_set_return(unsigned long mask) { unsigned long flags; +#ifdef CONFIG_TRACE_IRQFLAGS + WARN_ON(mask && !(mask & IRQS_DISABLED)); +#endif + asm volatile( "lbz %0,%1(13); stb %2,%1(13)" : "=&r" (flags) : "i" (offsetof(struct paca_struct, soft_enabled)), - "r" (enable) + "r" (mask) : "memory"); return flags; @@ -114,7 +130,7 @@ static inline unsigned long arch_local_irq_save(void) static inline bool arch_irqs_disabled_flags(unsigned long flags) { - return flags == IRQS_DISABLED; + return flags & IRQS_DISABLED; } static inline bool arch_irqs_disabled(void) @@ -133,7 +149,7 @@ static inline bool arch_irqs_disabled(void) #define hard_irq_disable() do { \ unsigned long flags; \ __hard_irq_disable(); \ - flags = soft_enabled_set_return(IRQS_DISABLED); \ + flags = soft_enabled_set_return(IRQS_DISABLED);\ local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ if (!arch_irqs_disabled_flags(flags)) \ trace_hardirqs_off(); \ @@ -158,7 +174,7 @@ static inline void may_hard_irq_enable(void) static inline bool arch_irq_disabled_regs(struct pt_regs *regs) { - return (regs->softe == IRQS_DISABLED); + return (regs->softe & IRQS_DISABLED); } extern bool prep_irq_for_idle(void); diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h index 8d9fdc84828d..f1ec012232d9 100644 --- a/arch/powerpc/include/asm/irqflags.h +++ b/arch/powerpc/include/asm/irqflags.h @@ -49,11 +49,11 @@ #define RECONCILE_IRQ_STATE(__rA, __rB) \ lbz __rA,PACASOFTIRQEN(r13); \ lbz __rB,PACAIRQHAPPENED(r13); \ - cmpwi cr0,__rA,IRQS_DISABLED;\ + andi. __rA,__rA,IRQS_DISABLED;\ li __rA,IRQS_DISABLED; \ ori __rB,__rB,PACA_IRQ_HARD_DIS; \ stb __rB,PACAIRQHAPPENED(r13); \ - beq 44f; \ + bne 44f; \ stb __rA,PACASOFTIRQEN(r13); \ TRACE_DISABLE_INTS; \ 44: diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index f8b32224dc11..51e4cc4dba4a 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -130,8 +130,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) */ #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG) lbz r10,PACASOFTIRQEN(r13) - xori r10,r10,IRQS_ENABLED -1: tdnei r10,0 +1: tdnei r10,IRQS_ENABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif @@ -741,10 +740,10 @@ resume_kernel: beq+ restore /* Check that preempt_count() == 0 and interrupts are enabled */ lwz r8,TI_PREEMPT(r9) - cmpwi cr1,r8,0 + cmpwi cr0,r8,0 + bne restore ld r0,SOFTE(r1) - cmpdi r0,IRQS_DISABLED - crandc eq,cr1*4+eq,eq + andi. r0,r0,IRQS_DISABLED bne restore /* @@ -783,11 +782,11 @@ restore: */ ld r5,SOFTE(r1) lbz r6,PACASOFTIRQEN(r13) - cmpwi cr0,r5,IRQS_DISABLED - beq .Lrestore_irq_off + andi. r5,r5,IRQS_DISABLED + bne .Lrestore_irq_off /* We are enabling, were we already enabled ? Yes, just return */ - cmpwi cr0,r6,IRQS_ENABLED + andi. r6,r6,IRQS_DISABLED beq cr0,.Ldo_restore /* @@ -1031,15 +1030,15 @@ _GLOBAL(enter_rtas) li r0,0 mtcr r0 -#ifdef CONFIG_BUG +#ifdef CONFIG_BUG /* There is no way it is acceptable to get here with interrupts enabled, * check it with the asm equivalent of WARN_ON */ lbz r0,PACASOFTIRQEN(r13) -1: tdnei r0,IRQS_DISABLED +1: tdeqi r0,IRQS_ENABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif - + /* Hard-disable interrupts */ mfmsr r6 rldicl r7,r6,48,1 diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 80cc91bbcf1a..9216ece7672c 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -210,10 +210,10 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) ld r5,SOFTE(r1) /* Interrupts had better not already be enabled... */ - twnei r6,IRQS_DISABLED + tweqi r6,IRQS_ENABLED - cmpwi cr0,r5,IRQS_DISABLED - beq 1f + andi. r6,r5,IRQS_DISABLED + bne 1f TRACE_ENABLE_INTS stb r5,PACASOFTIRQEN(r13) @@ -352,8 +352,8 @@ ret_from_mc_except: #define PROLOG_ADDITION_MASKABLE_GEN(n) \ lbz r10,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \ - cmpwi cr0,r10,IRQS_DISABLED; /* yes -> go out of line */ \ - beq masked_interrupt_book3e_##n + andi. r10,r10,IRQS_DISABLED; /* yes -> go out of line */ \ + bne masked_interrupt_book3e_##n #define PROLOG_ADDITION_2REGS_GEN(n) \ std r14,PACA_EXGEN+EX_R14(r13); \ diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 6c04e465caf5..406c4329b535 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -219,15 +219,29 @@ notrace unsigned int __check_irq_replay(void) return 0; } -notrace void arch_local_irq_restore(unsigned long en) +notrace void arch_local_irq_restore(unsigned long mask) { unsigned char irq_happened; unsigned int replay; /* Write the new soft-enabled value */ - soft_enabled_set(en); - if (en == IRQS_DISABLED) + soft_enabled_set(mask); + if (mask) { +#ifdef CONFIG_TRACE_IRQFLAGS + /* + * mask must always include LINUX bit if any + * are set, and interrupts don't get replayed until + * the Linux interrupt is unmasked. This could be + * changed to replay partial unmasks in future, + * which would allow Linux masks to nest inside + * other masks, among other things. For now, be very + * dumb and simple. + */ + WARN_ON(!(mask & IRQS_DISABLED)); +#endif return; + } + /* * From this point onward, we can take interrupts, preempt, * etc... unless we got hard-disabled. We check if an event diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 594261e308b3..ea4c709f481b 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -322,7 +322,7 @@ static inline void perf_read_regs(struct pt_regs *regs) */ static inline int perf_intr_is_nmi(struct pt_regs *regs) { - return (regs->softe == IRQS_DISABLED); + return (regs->softe & IRQS_DISABLED); } /* -- cgit v1.2.3 From 4e26bc4a4ed683c42ba45f09050575a671c6f1f4 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:50 +0530 Subject: powerpc/64: Rename soft_enabled to irq_soft_mask Rename the paca->soft_enabled to paca->irq_soft_mask as it is no longer used as a flag for interrupt state, but a mask. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 4 +-- arch/powerpc/include/asm/hw_irq.h | 56 ++++++++++++++++++-------------- arch/powerpc/include/asm/irqflags.h | 12 +++---- arch/powerpc/include/asm/kvm_ppc.h | 2 +- arch/powerpc/include/asm/paca.h | 2 +- arch/powerpc/kernel/asm-offsets.c | 2 +- arch/powerpc/kernel/entry_64.S | 12 +++---- arch/powerpc/kernel/exceptions-64e.S | 10 +++--- arch/powerpc/kernel/head_64.S | 6 ++-- arch/powerpc/kernel/idle_book3e.S | 2 +- arch/powerpc/kernel/idle_power4.S | 2 +- arch/powerpc/kernel/irq.c | 23 +++---------- arch/powerpc/kernel/optprobes_head.S | 2 +- arch/powerpc/kernel/ptrace.c | 2 +- arch/powerpc/kernel/setup_64.c | 4 +-- arch/powerpc/kernel/time.c | 6 ++-- arch/powerpc/kvm/book3s_hv_rmhandlers.S | 2 +- arch/powerpc/mm/hugetlbpage.c | 2 +- arch/powerpc/xmon/xmon.c | 4 +-- 19 files changed, 74 insertions(+), 81 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index a21adcb75fc6..b090db4ca37e 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -432,7 +432,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) mflr r9; /* Get LR, later save to stack */ \ ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \ std r9,_LINK(r1); \ - lbz r10,PACASOFTIRQEN(r13); \ + lbz r10,PACAIRQSOFTMASK(r13); \ mfspr r11,SPRN_XER; /* save XER in stackframe */ \ std r10,SOFTE(r1); \ std r11,_XER(r1); \ @@ -498,7 +498,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define SOFTEN_VALUE_0xea0 PACA_IRQ_EE #define __SOFTEN_TEST(h, vec) \ - lbz r10,PACASOFTIRQEN(r13); \ + lbz r10,PACAIRQSOFTMASK(r13); \ andi. r10,r10,IRQS_DISABLED; \ li r10,SOFTEN_VALUE_##vec; \ bne masked_##h##interrupt diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index f96280ee9540..5c82bb116832 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -29,7 +29,7 @@ #define PACA_IRQ_HMI 0x20 /* - * flags for paca->soft_enabled + * flags for paca->irq_soft_mask */ #define IRQS_ENABLED 0 #define IRQS_DISABLED 1 @@ -49,14 +49,14 @@ extern void unknown_exception(struct pt_regs *regs); #ifdef CONFIG_PPC64 #include -static inline notrace unsigned long soft_enabled_return(void) +static inline notrace unsigned long irq_soft_mask_return(void) { unsigned long flags; asm volatile( "lbz %0,%1(13)" : "=r" (flags) - : "i" (offsetof(struct paca_struct, soft_enabled))); + : "i" (offsetof(struct paca_struct, irq_soft_mask))); return flags; } @@ -64,18 +64,24 @@ static inline notrace unsigned long soft_enabled_return(void) /* * The "memory" clobber acts as both a compiler barrier * for the critical section and as a clobber because - * we changed paca->soft_enabled + * we changed paca->irq_soft_mask */ -static inline notrace void soft_enabled_set(unsigned long enable) +static inline notrace void irq_soft_mask_set(unsigned long mask) { #ifdef CONFIG_TRACE_IRQFLAGS /* - * mask must always include LINUX bit if any are set, and - * interrupts don't get replayed until the Linux interrupt is - * unmasked. This could be changed to replay partial unmasks - * in future, which would allow Linux masks to nest inside - * other masks, among other things. For now, be very dumb and - * simple. + * The irq mask must always include the STD bit if any are set. + * + * and interrupts don't get replayed until the standard + * interrupt (local_irq_disable()) is unmasked. + * + * Other masks must only provide additional masking beyond + * the standard, and they are also not replayed until the + * standard interrupt becomes unmasked. + * + * This could be changed, but it will require partial + * unmasks to be replayed, among other things. For now, take + * the simple approach. */ WARN_ON(mask && !(mask & IRQS_DISABLED)); #endif @@ -83,12 +89,12 @@ static inline notrace void soft_enabled_set(unsigned long enable) asm volatile( "stb %0,%1(13)" : - : "r" (enable), - "i" (offsetof(struct paca_struct, soft_enabled)) + : "r" (mask), + "i" (offsetof(struct paca_struct, irq_soft_mask)) : "memory"); } -static inline notrace unsigned long soft_enabled_set_return(unsigned long mask) +static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask) { unsigned long flags; @@ -99,7 +105,7 @@ static inline notrace unsigned long soft_enabled_set_return(unsigned long mask) asm volatile( "lbz %0,%1(13); stb %2,%1(13)" : "=&r" (flags) - : "i" (offsetof(struct paca_struct, soft_enabled)), + : "i" (offsetof(struct paca_struct, irq_soft_mask)), "r" (mask) : "memory"); @@ -108,12 +114,12 @@ static inline notrace unsigned long soft_enabled_set_return(unsigned long mask) static inline unsigned long arch_local_save_flags(void) { - return soft_enabled_return(); + return irq_soft_mask_return(); } static inline void arch_local_irq_disable(void) { - soft_enabled_set(IRQS_DISABLED); + irq_soft_mask_set(IRQS_DISABLED); } extern void arch_local_irq_restore(unsigned long); @@ -125,7 +131,7 @@ static inline void arch_local_irq_enable(void) static inline unsigned long arch_local_irq_save(void) { - return soft_enabled_set_return(IRQS_DISABLED); + return irq_soft_mask_set_return(IRQS_DISABLED); } static inline bool arch_irqs_disabled_flags(unsigned long flags) @@ -146,13 +152,13 @@ static inline bool arch_irqs_disabled(void) #define __hard_irq_disable() __mtmsrd(local_paca->kernel_msr, 1) #endif -#define hard_irq_disable() do { \ - unsigned long flags; \ - __hard_irq_disable(); \ - flags = soft_enabled_set_return(IRQS_DISABLED);\ - local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ - if (!arch_irqs_disabled_flags(flags)) \ - trace_hardirqs_off(); \ +#define hard_irq_disable() do { \ + unsigned long flags; \ + __hard_irq_disable(); \ + flags = irq_soft_mask_set_return(IRQS_DISABLED); \ + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ + if (!arch_irqs_disabled_flags(flags)) \ + trace_hardirqs_off(); \ } while(0) static inline bool lazy_irq_pending(void) diff --git a/arch/powerpc/include/asm/irqflags.h b/arch/powerpc/include/asm/irqflags.h index f1ec012232d9..1a6c1ce17735 100644 --- a/arch/powerpc/include/asm/irqflags.h +++ b/arch/powerpc/include/asm/irqflags.h @@ -47,14 +47,14 @@ * be clobbered. */ #define RECONCILE_IRQ_STATE(__rA, __rB) \ - lbz __rA,PACASOFTIRQEN(r13); \ + lbz __rA,PACAIRQSOFTMASK(r13); \ lbz __rB,PACAIRQHAPPENED(r13); \ - andi. __rA,__rA,IRQS_DISABLED;\ - li __rA,IRQS_DISABLED; \ + andi. __rA,__rA,IRQS_DISABLED; \ + li __rA,IRQS_DISABLED; \ ori __rB,__rB,PACA_IRQ_HARD_DIS; \ stb __rB,PACAIRQHAPPENED(r13); \ bne 44f; \ - stb __rA,PACASOFTIRQEN(r13); \ + stb __rA,PACAIRQSOFTMASK(r13); \ TRACE_DISABLE_INTS; \ 44: @@ -64,9 +64,9 @@ #define RECONCILE_IRQ_STATE(__rA, __rB) \ lbz __rA,PACAIRQHAPPENED(r13); \ - li __rB,IRQS_DISABLED; \ + li __rB,IRQS_DISABLED; \ ori __rA,__rA,PACA_IRQ_HARD_DIS; \ - stb __rB,PACASOFTIRQEN(r13); \ + stb __rB,PACAIRQSOFTMASK(r13); \ stb __rA,PACAIRQHAPPENED(r13) #endif #endif diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 369f0640826c..9db18287b5f4 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -873,7 +873,7 @@ static inline void kvmppc_fix_ee_before_entry(void) /* Only need to enable IRQs by hard enabling them after this */ local_paca->irq_happened = 0; - soft_enabled_set(IRQS_ENABLED); + irq_soft_mask_set(IRQS_ENABLED); #endif } diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 3892db93b837..e2ee193eb24d 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -159,7 +159,7 @@ struct paca_struct { u64 saved_r1; /* r1 save for RTAS calls or PM */ u64 saved_msr; /* MSR saved here by enter_rtas */ u16 trap_save; /* Used when bad stack is encountered */ - u8 soft_enabled; /* irq soft-enable flag */ + u8 irq_soft_mask; /* mask for irq soft masking */ u8 irq_happened; /* irq happened while soft-disabled */ u8 io_sync; /* writel() needs spin_unlock sync */ u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */ diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 3f6316bcde4e..43a6967d48ad 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -178,7 +178,7 @@ int main(void) OFFSET(PACATOC, paca_struct, kernel_toc); OFFSET(PACAKBASE, paca_struct, kernelbase); OFFSET(PACAKMSR, paca_struct, kernel_msr); - OFFSET(PACASOFTIRQEN, paca_struct, soft_enabled); + OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask); OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened); #ifdef CONFIG_PPC_BOOK3S OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id); diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 51e4cc4dba4a..7c51f022d0de 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -129,7 +129,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) * is correct */ #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG) - lbz r10,PACASOFTIRQEN(r13) + lbz r10,PACAIRQSOFTMASK(r13) 1: tdnei r10,IRQS_ENABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif @@ -781,7 +781,7 @@ restore: * are about to re-enable interrupts */ ld r5,SOFTE(r1) - lbz r6,PACASOFTIRQEN(r13) + lbz r6,PACAIRQSOFTMASK(r13) andi. r5,r5,IRQS_DISABLED bne .Lrestore_irq_off @@ -806,7 +806,7 @@ restore: .Lrestore_no_replay: TRACE_ENABLE_INTS li r0,IRQS_ENABLED - stb r0,PACASOFTIRQEN(r13); + stb r0,PACAIRQSOFTMASK(r13); /* * Final return path. BookE is handled in a different file @@ -913,8 +913,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) 1: #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG) /* The interrupt should not have soft enabled. */ - lbz r7,PACASOFTIRQEN(r13) -1: tdnei r7,IRQS_DISABLED + lbz r7,PACAIRQSOFTMASK(r13) +1: tdeqi r7,IRQS_ENABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif b .Ldo_restore @@ -1034,7 +1034,7 @@ _GLOBAL(enter_rtas) /* There is no way it is acceptable to get here with interrupts enabled, * check it with the asm equivalent of WARN_ON */ - lbz r0,PACASOFTIRQEN(r13) + lbz r0,PACAIRQSOFTMASK(r13) 1: tdeqi r0,IRQS_ENABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING #endif diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 9216ece7672c..ee832d344a5a 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -139,7 +139,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) mfspr r10,SPRN_ESR SPECIAL_EXC_STORE(r10,ESR) - lbz r10,PACASOFTIRQEN(r13) + lbz r10,PACAIRQSOFTMASK(r13) SPECIAL_EXC_STORE(r10,SOFTE) ld r10,_NIP(r1) SPECIAL_EXC_STORE(r10,CSRR0) @@ -206,7 +206,7 @@ BEGIN_FTR_SECTION mtspr SPRN_MAS8,r10 END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) - lbz r6,PACASOFTIRQEN(r13) + lbz r6,PACAIRQSOFTMASK(r13) ld r5,SOFTE(r1) /* Interrupts had better not already be enabled... */ @@ -216,7 +216,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV) bne 1f TRACE_ENABLE_INTS - stb r5,PACASOFTIRQEN(r13) + stb r5,PACAIRQSOFTMASK(r13) 1: /* * Restore PACAIRQHAPPENED rather than setting it based on @@ -351,7 +351,7 @@ ret_from_mc_except: #define PROLOG_ADDITION_NONE_MC(n) #define PROLOG_ADDITION_MASKABLE_GEN(n) \ - lbz r10,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \ + lbz r10,PACAIRQSOFTMASK(r13); /* are irqs soft-masked? */ \ andi. r10,r10,IRQS_DISABLED; /* yes -> go out of line */ \ bne masked_interrupt_book3e_##n @@ -397,7 +397,7 @@ exc_##n##_common: \ mfspr r8,SPRN_XER; /* save XER in stackframe */ \ ld r9,excf+EX_R1(r13); /* load orig r1 back from PACA */ \ lwz r10,excf+EX_CR(r13); /* load orig CR back from PACA */ \ - lbz r11,PACASOFTIRQEN(r13); /* get current IRQ softe */ \ + lbz r11,PACAIRQSOFTMASK(r13); /* get current IRQ softe */ \ ld r12,exception_marker@toc(r2); \ li r0,0; \ std r3,GPR10(r1); /* save r10 to stackframe */ \ diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index 0deef350004f..a61151a6ea5e 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -766,7 +766,7 @@ _GLOBAL(pmac_secondary_start) * in the PACA when doing hotplug) */ li r0,IRQS_DISABLED - stb r0,PACASOFTIRQEN(r13) + stb r0,PACAIRQSOFTMASK(r13) li r0,PACA_IRQ_HARD_DIS stb r0,PACAIRQHAPPENED(r13) @@ -823,7 +823,7 @@ __secondary_start: * in the PACA when doing hotplug) */ li r7,IRQS_DISABLED - stb r7,PACASOFTIRQEN(r13) + stb r7,PACAIRQSOFTMASK(r13) li r0,PACA_IRQ_HARD_DIS stb r0,PACAIRQHAPPENED(r13) @@ -990,7 +990,7 @@ start_here_common: * in the PACA when doing hotplug) */ li r0,IRQS_DISABLED - stb r0,PACASOFTIRQEN(r13) + stb r0,PACAIRQSOFTMASK(r13) li r0,PACA_IRQ_HARD_DIS stb r0,PACAIRQHAPPENED(r13) diff --git a/arch/powerpc/kernel/idle_book3e.S b/arch/powerpc/kernel/idle_book3e.S index 1bb1d152aa46..2b269315d377 100644 --- a/arch/powerpc/kernel/idle_book3e.S +++ b/arch/powerpc/kernel/idle_book3e.S @@ -48,7 +48,7 @@ _GLOBAL(\name) addi r1,r1,128 #endif li r0,IRQS_ENABLED - stb r0,PACASOFTIRQEN(r13) + stb r0,PACAIRQSOFTMASK(r13) /* Interrupts will make use return to LR, so get something we want * in there diff --git a/arch/powerpc/kernel/idle_power4.S b/arch/powerpc/kernel/idle_power4.S index 8b4702d93701..08faa93755f9 100644 --- a/arch/powerpc/kernel/idle_power4.S +++ b/arch/powerpc/kernel/idle_power4.S @@ -55,7 +55,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP) #endif /* CONFIG_TRACE_IRQFLAGS */ li r0,IRQS_ENABLED - stb r0,PACASOFTIRQEN(r13) /* we'll hard-enable shortly */ + stb r0,PACAIRQSOFTMASK(r13) /* we'll hard-enable shortly */ BEGIN_FTR_SECTION DSSALL sync diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 406c4329b535..e137b98dc5f8 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -225,22 +225,9 @@ notrace void arch_local_irq_restore(unsigned long mask) unsigned int replay; /* Write the new soft-enabled value */ - soft_enabled_set(mask); - if (mask) { -#ifdef CONFIG_TRACE_IRQFLAGS - /* - * mask must always include LINUX bit if any - * are set, and interrupts don't get replayed until - * the Linux interrupt is unmasked. This could be - * changed to replay partial unmasks in future, - * which would allow Linux masks to nest inside - * other masks, among other things. For now, be very - * dumb and simple. - */ - WARN_ON(!(mask & IRQS_DISABLED)); -#endif + irq_soft_mask_set(mask); + if (mask) return; - } /* * From this point onward, we can take interrupts, preempt, @@ -285,7 +272,7 @@ notrace void arch_local_irq_restore(unsigned long mask) } #endif /* CONFIG_TRACE_IRQFLAGS */ - soft_enabled_set(IRQS_DISABLED); + irq_soft_mask_set(IRQS_DISABLED); trace_hardirqs_off(); /* @@ -297,7 +284,7 @@ notrace void arch_local_irq_restore(unsigned long mask) /* We can soft-enable now */ trace_hardirqs_on(); - soft_enabled_set(IRQS_ENABLED); + irq_soft_mask_set(IRQS_ENABLED); /* * And replay if we have to. This will return with interrupts @@ -372,7 +359,7 @@ bool prep_irq_for_idle(void) * of entering the low power state. */ local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; - soft_enabled_set(IRQS_ENABLED); + irq_soft_mask_set(IRQS_ENABLED); /* Tell the caller to enter the low power state */ return true; diff --git a/arch/powerpc/kernel/optprobes_head.S b/arch/powerpc/kernel/optprobes_head.S index 52fc864cdec4..98a3aeeb3c8c 100644 --- a/arch/powerpc/kernel/optprobes_head.S +++ b/arch/powerpc/kernel/optprobes_head.S @@ -58,7 +58,7 @@ optprobe_template_entry: std r5,_XER(r1) mfcr r5 std r5,_CCR(r1) - lbz r5,PACASOFTIRQEN(r13) + lbz r5,PACAIRQSOFTMASK(r13) std r5,SOFTE(r1) /* diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index bd2c49475473..aef08e579946 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -285,7 +285,7 @@ int ptrace_get_reg(struct task_struct *task, int regno, unsigned long *data) #ifdef CONFIG_PPC64 /* - * softe copies paca->soft_enabled variable state. Since soft_enabled is + * softe copies paca->irq_soft_mask variable state. Since irq_soft_mask is * no more used as a flag, lets force usr to alway see the softe value as 1 * which means interrupts are not soft disabled. */ diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index f6bedebda90a..896dacef2f2d 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -189,7 +189,7 @@ static void __init fixup_boot_paca(void) /* Allow percpu accesses to work until we setup percpu data */ get_paca()->data_offset = 0; /* Mark interrupts disabled in PACA */ - soft_enabled_set(IRQS_DISABLED); + irq_soft_mask_set(IRQS_DISABLED); } static void __init configure_exceptions(void) @@ -352,7 +352,7 @@ void __init early_setup(unsigned long dt_ptr) void early_setup_secondary(void) { /* Mark interrupts disabled in PACA */ - soft_enabled_set(IRQS_DISABLED); + irq_soft_mask_set(IRQS_DISABLED); /* Initialize the hash table or TLB handling */ early_init_mmu_secondary(); diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index daa6e9e35ab9..a32823dcd9a4 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -244,7 +244,7 @@ static u64 scan_dispatch_log(u64 stop_tb) void accumulate_stolen_time(void) { u64 sst, ust; - unsigned long save_soft_enabled = soft_enabled_return(); + unsigned long save_irq_soft_mask = irq_soft_mask_return(); struct cpu_accounting_data *acct = &local_paca->accounting; /* We are called early in the exception entry, before @@ -253,7 +253,7 @@ void accumulate_stolen_time(void) * needs to reflect that so various debug stuff doesn't * complain */ - soft_enabled_set(IRQS_DISABLED); + irq_soft_mask_set(IRQS_DISABLED); sst = scan_dispatch_log(acct->starttime_user); ust = scan_dispatch_log(acct->starttime); @@ -261,7 +261,7 @@ void accumulate_stolen_time(void) acct->utime -= ust; acct->steal_time += ust + sst; - soft_enabled_set(save_soft_enabled); + irq_soft_mask_set(save_irq_soft_mask); } static inline u64 calculate_stolen_time(u64 stop_tb) diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 2659844784b8..a92ad8500917 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -3249,7 +3249,7 @@ kvmppc_bad_host_intr: mfctr r4 #endif mfxer r5 - lbz r6, PACASOFTIRQEN(r13) + lbz r6, PACAIRQSOFTMASK(r13) std r3, _LINK(r1) std r4, _CTR(r1) std r5, _XER(r1) diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 760bb5a2b8bd..876da2bc1796 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -752,7 +752,7 @@ void flush_dcache_icache_hugepage(struct page *page) * So long as we atomically load page table pointers we are safe against teardown, * we can follow the address down to the the page and take a ref on it. * This function need to be called with interrupts disabled. We use this variant - * when we have MSR[EE] = 0 but the paca->soft_enabled = IRQS_ENABLED + * when we have MSR[EE] = 0 but the paca->irq_soft_mask = IRQS_ENABLED */ pte_t *__find_linux_pte(pgd_t *pgdir, unsigned long ea, bool *is_thp, unsigned *hpage_shift) diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index 1b2d8cb49abb..2695f8dd2359 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -1623,7 +1623,7 @@ static void excprint(struct pt_regs *fp) printf(" current = 0x%lx\n", current); #ifdef CONFIG_PPC64 printf(" paca = 0x%lx\t softe: %d\t irq_happened: 0x%02x\n", - local_paca, local_paca->soft_enabled, local_paca->irq_happened); + local_paca, local_paca->irq_soft_mask, local_paca->irq_happened); #endif if (current) { printf(" pid = %ld, comm = %s\n", @@ -2391,7 +2391,7 @@ static void dump_one_paca(int cpu) DUMP(p, stab_rr, "lx"); DUMP(p, saved_r1, "lx"); DUMP(p, trap_save, "x"); - DUMP(p, soft_enabled, "x"); + DUMP(p, irq_soft_mask, "x"); DUMP(p, irq_happened, "x"); DUMP(p, io_sync, "x"); DUMP(p, irq_work_pending, "x"); -- cgit v1.2.3 From d32eb1b550d4ecb791c6311fcafa08ce91b6c06b Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:51 +0530 Subject: powerpc/64s: Avoid using EXCEPTION_PROLOG_1 macro in MASKABLE_* Currently we use both EXCEPTION_PROLOG_1 and __EXCEPTION_PROLOG_1 in the MASKABLE_* macros. As a cleanup, this patch makes MASKABLE_* to use only __EXCEPTION_PROLOG_1. There is not logic change. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index b090db4ca37e..593d7f9da5da 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -536,7 +536,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) EXC_STD, SOFTEN_TEST_PR) #define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label) \ - EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec); \ + __EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec); \ EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD) #define MASKABLE_EXCEPTION_HV(loc, vec, label) \ @@ -544,7 +544,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) EXC_HV, SOFTEN_TEST_HV) #define MASKABLE_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \ + __EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \ EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV) #define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \ @@ -565,7 +565,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) EXC_HV, SOFTEN_TEST_HV) #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label) \ - EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \ + __EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \ EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV) /* -- cgit v1.2.3 From f14e953b191fcb0da82ef066597df731b683a957 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:52 +0530 Subject: powerpc/64s: Add support to take additional parameter in MASKABLE_* macro To support addition of "bitmask" to MASKABLE_* macros, factor out the EXCPETION_PROLOG_1 macro. Make it explicit the interrupt masking supported by a gievn interrupt handler. Patch correspondingly extends the MASKABLE_* macros with an addition's parameter. "bitmask" parameter is passed to SOFTEN_TEST macro to decide on masking the interrupt. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 92 ++++++++++++++++++++------------ arch/powerpc/include/asm/head-64.h | 40 +++++++------- arch/powerpc/kernel/exceptions-64s.S | 32 ++++++----- 3 files changed, 96 insertions(+), 68 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index 593d7f9da5da..d005a1c19b68 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -198,18 +198,40 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) std r10,area+EX_R10(r13); /* save r10 - r12 */ \ OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR) -#define __EXCEPTION_PROLOG_1(area, extra, vec) \ +#define __EXCEPTION_PROLOG_1_PRE(area) \ OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR); \ OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR); \ SAVE_CTR(r10, area); \ - mfcr r9; \ - extra(vec); \ + mfcr r9; + +#define __EXCEPTION_PROLOG_1_POST(area) \ std r11,area+EX_R11(r13); \ std r12,area+EX_R12(r13); \ GET_SCRATCH0(r10); \ std r10,area+EX_R13(r13) + +/* + * This version of the EXCEPTION_PROLOG_1 will carry + * addition parameter called "bitmask" to support + * checking of the interrupt maskable level in the SOFTEN_TEST. + * Intended to be used in MASKABLE_EXCPETION_* macros. + */ +#define MASKABLE_EXCEPTION_PROLOG_1(area, extra, vec, bitmask) \ + __EXCEPTION_PROLOG_1_PRE(area); \ + extra(vec, bitmask); \ + __EXCEPTION_PROLOG_1_POST(area); + +/* + * This version of the EXCEPTION_PROLOG_1 is intended + * to be used in STD_EXCEPTION* macros + */ +#define _EXCEPTION_PROLOG_1(area, extra, vec) \ + __EXCEPTION_PROLOG_1_PRE(area); \ + extra(vec); \ + __EXCEPTION_PROLOG_1_POST(area); + #define EXCEPTION_PROLOG_1(area, extra, vec) \ - __EXCEPTION_PROLOG_1(area, extra, vec) + _EXCEPTION_PROLOG_1(area, extra, vec) #define __EXCEPTION_PROLOG_PSERIES_1(label, h) \ ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ @@ -497,21 +519,21 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define SOFTEN_VALUE_0xe60 PACA_IRQ_HMI #define SOFTEN_VALUE_0xea0 PACA_IRQ_EE -#define __SOFTEN_TEST(h, vec) \ +#define __SOFTEN_TEST(h, vec, bitmask) \ lbz r10,PACAIRQSOFTMASK(r13); \ - andi. r10,r10,IRQS_DISABLED; \ + andi. r10,r10,bitmask; \ li r10,SOFTEN_VALUE_##vec; \ bne masked_##h##interrupt -#define _SOFTEN_TEST(h, vec) __SOFTEN_TEST(h, vec) +#define _SOFTEN_TEST(h, vec, bitmask) __SOFTEN_TEST(h, vec, bitmask) -#define SOFTEN_TEST_PR(vec) \ +#define SOFTEN_TEST_PR(vec, bitmask) \ KVMTEST(EXC_STD, vec); \ - _SOFTEN_TEST(EXC_STD, vec) + _SOFTEN_TEST(EXC_STD, vec, bitmask) -#define SOFTEN_TEST_HV(vec) \ +#define SOFTEN_TEST_HV(vec, bitmask) \ KVMTEST(EXC_HV, vec); \ - _SOFTEN_TEST(EXC_HV, vec) + _SOFTEN_TEST(EXC_HV, vec, bitmask) #define KVMTEST_PR(vec) \ KVMTEST(EXC_STD, vec) @@ -519,53 +541,53 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define KVMTEST_HV(vec) \ KVMTEST(EXC_HV, vec) -#define SOFTEN_NOTEST_PR(vec) _SOFTEN_TEST(EXC_STD, vec) -#define SOFTEN_NOTEST_HV(vec) _SOFTEN_TEST(EXC_HV, vec) +#define SOFTEN_NOTEST_PR(vec, bitmask) _SOFTEN_TEST(EXC_STD, vec, bitmask) +#define SOFTEN_NOTEST_HV(vec, bitmask) _SOFTEN_TEST(EXC_HV, vec, bitmask) -#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \ +#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ - __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \ + MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ EXCEPTION_PROLOG_PSERIES_1(label, h); -#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) \ - __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra) +#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \ + __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) -#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label) \ +#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label, bitmask) \ _MASKABLE_EXCEPTION_PSERIES(vec, label, \ - EXC_STD, SOFTEN_TEST_PR) + EXC_STD, SOFTEN_TEST_PR, bitmask) -#define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label) \ - __EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec); \ +#define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label, bitmask) \ + MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\ EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD) -#define MASKABLE_EXCEPTION_HV(loc, vec, label) \ +#define MASKABLE_EXCEPTION_HV(loc, vec, label, bitmask) \ _MASKABLE_EXCEPTION_PSERIES(vec, label, \ - EXC_HV, SOFTEN_TEST_HV) + EXC_HV, SOFTEN_TEST_HV, bitmask) -#define MASKABLE_EXCEPTION_HV_OOL(vec, label) \ - __EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \ +#define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ + MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV) -#define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \ +#define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ - __EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec); \ + MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) -#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) \ - __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra) +#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)\ + __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) -#define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label) \ +#define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label, bitmask) \ _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ - EXC_STD, SOFTEN_NOTEST_PR) + EXC_STD, SOFTEN_NOTEST_PR, bitmask) -#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label) \ +#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label, bitmask) \ _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ - EXC_HV, SOFTEN_TEST_HV) + EXC_HV, SOFTEN_TEST_HV, bitmask) -#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label) \ - __EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec); \ +#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \ + MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_HV, vec, bitmask);\ EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV) /* diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index 0a663dfc28b5..7e0e93f24cb7 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -268,14 +268,14 @@ name: STD_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \ EXC_VIRT_END(name, start, size); -#define EXC_REAL_MASKABLE(name, start, size) \ +#define EXC_REAL_MASKABLE(name, start, size, bitmask) \ EXC_REAL_BEGIN(name, start, size); \ - MASKABLE_EXCEPTION_PSERIES(start, start, name##_common); \ + MASKABLE_EXCEPTION_PSERIES(start, start, name##_common, bitmask);\ EXC_REAL_END(name, start, size); -#define EXC_VIRT_MASKABLE(name, start, size, realvec) \ +#define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask) \ EXC_VIRT_BEGIN(name, start, size); \ - MASKABLE_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \ + MASKABLE_RELON_EXCEPTION_PSERIES(start, realvec, name##_common, bitmask);\ EXC_VIRT_END(name, start, size); #define EXC_REAL_HV(name, start, size) \ @@ -304,13 +304,13 @@ name: #define __EXC_REAL_OOL_MASKABLE(name, start, size) \ __EXC_REAL_OOL(name, start, size); -#define __TRAMP_REAL_OOL_MASKABLE(name, vec) \ +#define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - MASKABLE_EXCEPTION_PSERIES_OOL(vec, name##_common); \ + MASKABLE_EXCEPTION_PSERIES_OOL(vec, name##_common, bitmask); \ -#define EXC_REAL_OOL_MASKABLE(name, start, size) \ +#define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask) \ __EXC_REAL_OOL_MASKABLE(name, start, size); \ - __TRAMP_REAL_OOL_MASKABLE(name, start); + __TRAMP_REAL_OOL_MASKABLE(name, start, bitmask); #define __EXC_REAL_OOL_HV_DIRECT(name, start, size, handler) \ EXC_REAL_BEGIN(name, start, size); \ @@ -331,13 +331,13 @@ name: #define __EXC_REAL_OOL_MASKABLE_HV(name, start, size) \ __EXC_REAL_OOL(name, start, size); -#define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec) \ +#define __TRAMP_REAL_OOL_MASKABLE_HV(name, vec, bitmask) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - MASKABLE_EXCEPTION_HV_OOL(vec, name##_common); \ + MASKABLE_EXCEPTION_HV_OOL(vec, name##_common, bitmask); \ -#define EXC_REAL_OOL_MASKABLE_HV(name, start, size) \ +#define EXC_REAL_OOL_MASKABLE_HV(name, start, size, bitmask) \ __EXC_REAL_OOL_MASKABLE_HV(name, start, size); \ - __TRAMP_REAL_OOL_MASKABLE_HV(name, start); + __TRAMP_REAL_OOL_MASKABLE_HV(name, start, bitmask); #define __EXC_VIRT_OOL(name, start, size) \ EXC_VIRT_BEGIN(name, start, size); \ @@ -355,13 +355,13 @@ name: #define __EXC_VIRT_OOL_MASKABLE(name, start, size) \ __EXC_VIRT_OOL(name, start, size); -#define __TRAMP_VIRT_OOL_MASKABLE(name, realvec) \ +#define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - MASKABLE_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common); \ + MASKABLE_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common, bitmask);\ -#define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec) \ +#define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask) \ __EXC_VIRT_OOL_MASKABLE(name, start, size); \ - __TRAMP_VIRT_OOL_MASKABLE(name, realvec); + __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask); #define __EXC_VIRT_OOL_HV(name, start, size) \ __EXC_VIRT_OOL(name, start, size); @@ -377,13 +377,13 @@ name: #define __EXC_VIRT_OOL_MASKABLE_HV(name, start, size) \ __EXC_VIRT_OOL(name, start, size); -#define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec) \ +#define __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common); \ + MASKABLE_RELON_EXCEPTION_HV_OOL(realvec, name##_common, bitmask);\ -#define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec) \ +#define EXC_VIRT_OOL_MASKABLE_HV(name, start, size, realvec, bitmask) \ __EXC_VIRT_OOL_MASKABLE_HV(name, start, size); \ - __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec); + __TRAMP_VIRT_OOL_MASKABLE_HV(name, realvec, bitmask); #define TRAMP_KVM(area, n) \ TRAMP_KVM_BEGIN(do_kvm_##n); \ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 175891c6909c..f99021f3b47b 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -691,10 +691,12 @@ EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) hardware_interrupt_hv: BEGIN_FTR_SECTION _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, - EXC_HV, SOFTEN_TEST_HV) + EXC_HV, SOFTEN_TEST_HV, + IRQS_DISABLED) FTR_SECTION_ELSE _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, - EXC_STD, SOFTEN_TEST_PR) + EXC_STD, SOFTEN_TEST_PR, + IRQS_DISABLED) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) EXC_REAL_END(hardware_interrupt, 0x500, 0x100) @@ -702,9 +704,13 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) .globl hardware_interrupt_relon_hv; hardware_interrupt_relon_hv: BEGIN_FTR_SECTION - _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_HV, SOFTEN_TEST_HV) + _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, + EXC_HV, SOFTEN_TEST_HV, + IRQS_DISABLED) FTR_SECTION_ELSE - _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, EXC_STD, SOFTEN_TEST_PR) + _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, + EXC_STD, SOFTEN_TEST_PR, + IRQS_DISABLED) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) @@ -800,8 +806,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) #endif -EXC_REAL_MASKABLE(decrementer, 0x900, 0x80) -EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900) +EXC_REAL_MASKABLE(decrementer, 0x900, 0x80, IRQS_DISABLED) +EXC_VIRT_MASKABLE(decrementer, 0x4900, 0x80, 0x900, IRQS_DISABLED) TRAMP_KVM(PACA_EXGEN, 0x900) EXC_COMMON_ASYNC(decrementer_common, 0x900, timer_interrupt) @@ -812,8 +818,8 @@ TRAMP_KVM_HV(PACA_EXGEN, 0x980) EXC_COMMON(hdecrementer_common, 0x980, hdec_interrupt) -EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0x100) -EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x100, 0xa00) +EXC_REAL_MASKABLE(doorbell_super, 0xa00, 0x100, IRQS_DISABLED) +EXC_VIRT_MASKABLE(doorbell_super, 0x4a00, 0x100, 0xa00, IRQS_DISABLED) TRAMP_KVM(PACA_EXGEN, 0xa00) #ifdef CONFIG_PPC_DOORBELL EXC_COMMON_ASYNC(doorbell_super_common, 0xa00, doorbell_exception) @@ -1025,7 +1031,7 @@ EXC_COMMON(emulation_assist_common, 0xe40, emulation_assist_interrupt) * mode. */ __EXC_REAL_OOL_HV_DIRECT(hmi_exception, 0xe60, 0x20, hmi_exception_early) -__TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60) +__TRAMP_REAL_OOL_MASKABLE_HV(hmi_exception, 0xe60, IRQS_DISABLED) EXC_VIRT_NONE(0x4e60, 0x20) TRAMP_KVM_HV(PACA_EXGEN, 0xe60) TRAMP_REAL_BEGIN(hmi_exception_early) @@ -1083,8 +1089,8 @@ EXC_COMMON_BEGIN(hmi_exception_common) EXCEPTION_COMMON(PACA_EXGEN, 0xe60, hmi_exception_common, handle_hmi_exception, ret_from_except, FINISH_NAP;ADD_NVGPRS;ADD_RECONCILE;RUNLATCH_ON) -EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20) -EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80) +EXC_REAL_OOL_MASKABLE_HV(h_doorbell, 0xe80, 0x20, IRQS_DISABLED) +EXC_VIRT_OOL_MASKABLE_HV(h_doorbell, 0x4e80, 0x20, 0xe80, IRQS_DISABLED) TRAMP_KVM_HV(PACA_EXGEN, 0xe80) #ifdef CONFIG_PPC_DOORBELL EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, doorbell_exception) @@ -1093,8 +1099,8 @@ EXC_COMMON_ASYNC(h_doorbell_common, 0xe80, unknown_exception) #endif -EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20) -EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0) +EXC_REAL_OOL_MASKABLE_HV(h_virt_irq, 0xea0, 0x20, IRQS_DISABLED) +EXC_VIRT_OOL_MASKABLE_HV(h_virt_irq, 0x4ea0, 0x20, 0xea0, IRQS_DISABLED) TRAMP_KVM_HV(PACA_EXGEN, 0xea0) EXC_COMMON_ASYNC(h_virt_irq_common, 0xea0, do_IRQ) -- cgit v1.2.3 From f442d004806e31fe5aab614ec48e53f7b38f7c2d Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:53 +0530 Subject: powerpc/64s: Add support to mask perf interrupts and replay them Two new bit mask field "IRQ_DISABLE_MASK_PMU" is introduced to support the masking of PMI and "IRQ_DISABLE_MASK_ALL" to aid interrupt masking checking. Couple of new irq #defs "PACA_IRQ_PMI" and "SOFTEN_VALUE_0xf0*" added to use in the exception code to check for PMI interrupts. In the masked_interrupt handler, for PMIs we reset the MSR[EE] and return. In the __check_irq_replay(), replay the PMI interrupt by calling performance_monitor_common handler. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/exception-64s.h | 5 +++++ arch/powerpc/include/asm/hw_irq.h | 19 +++++++++++-------- arch/powerpc/kernel/entry_64.S | 5 +++++ arch/powerpc/kernel/exceptions-64s.S | 6 ++++-- arch/powerpc/kernel/irq.c | 7 ++++++- 5 files changed, 31 insertions(+), 11 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index d005a1c19b68..54afd1f140a4 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -518,6 +518,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define SOFTEN_VALUE_0xe80 PACA_IRQ_DBELL #define SOFTEN_VALUE_0xe60 PACA_IRQ_HMI #define SOFTEN_VALUE_0xea0 PACA_IRQ_EE +#define SOFTEN_VALUE_0xf00 PACA_IRQ_PMI #define __SOFTEN_TEST(h, vec, bitmask) \ lbz r10,PACAIRQSOFTMASK(r13); \ @@ -582,6 +583,10 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ EXC_STD, SOFTEN_NOTEST_PR, bitmask) +#define MASKABLE_RELON_EXCEPTION_PSERIES_OOL(vec, label, bitmask) \ + MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\ + EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD); + #define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label, bitmask) \ _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ EXC_HV, SOFTEN_TEST_HV, bitmask) diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 5c82bb116832..552a2c6e428f 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -27,12 +27,15 @@ #define PACA_IRQ_DEC 0x08 /* Or FIT */ #define PACA_IRQ_EE_EDGE 0x10 /* BookE only */ #define PACA_IRQ_HMI 0x20 +#define PACA_IRQ_PMI 0x40 /* * flags for paca->irq_soft_mask */ #define IRQS_ENABLED 0 -#define IRQS_DISABLED 1 +#define IRQS_DISABLED 1 /* local_irq_disable() interrupts */ +#define IRQS_PMI_DISABLED 2 +#define IRQS_ALL_DISABLED (IRQS_DISABLED | IRQS_PMI_DISABLED) #endif /* CONFIG_PPC64 */ @@ -152,13 +155,13 @@ static inline bool arch_irqs_disabled(void) #define __hard_irq_disable() __mtmsrd(local_paca->kernel_msr, 1) #endif -#define hard_irq_disable() do { \ - unsigned long flags; \ - __hard_irq_disable(); \ - flags = irq_soft_mask_set_return(IRQS_DISABLED); \ - local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ - if (!arch_irqs_disabled_flags(flags)) \ - trace_hardirqs_off(); \ +#define hard_irq_disable() do { \ + unsigned long flags; \ + __hard_irq_disable(); \ + flags = irq_soft_mask_set_return(IRQS_ALL_DISABLED); \ + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; \ + if (!arch_irqs_disabled_flags(flags)) \ + trace_hardirqs_off(); \ } while(0) static inline bool lazy_irq_pending(void) diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 7c51f022d0de..3b90a2f1018e 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -954,6 +954,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) addi r3,r1,STACK_FRAME_OVERHEAD; bl do_IRQ b ret_from_except +1: cmpwi cr0,r3,0xf00 + bne 1f + addi r3,r1,STACK_FRAME_OVERHEAD; + bl performance_monitor_exception + b ret_from_except 1: cmpwi cr0,r3,0xe60 bne 1f addi r3,r1,STACK_FRAME_OVERHEAD; diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index f99021f3b47b..1a4f51f35870 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1111,8 +1111,8 @@ EXC_REAL_NONE(0xee0, 0x20) EXC_VIRT_NONE(0x4ee0, 0x20) -EXC_REAL_OOL(performance_monitor, 0xf00, 0x20) -EXC_VIRT_OOL(performance_monitor, 0x4f00, 0x20, 0xf00) +EXC_REAL_OOL_MASKABLE(performance_monitor, 0xf00, 0x20, IRQS_PMI_DISABLED) +EXC_VIRT_OOL_MASKABLE(performance_monitor, 0x4f00, 0x20, 0xf00, IRQS_PMI_DISABLED) TRAMP_KVM(PACA_EXGEN, 0xf00) EXC_COMMON_ASYNC(performance_monitor_common, 0xf00, performance_monitor_exception) @@ -1723,6 +1723,8 @@ BEGIN_FTR_SECTION FTR_SECTION_ELSE beq hardware_interrupt_common ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_300) + cmpwi r3,0xf00 + beq performance_monitor_common BEGIN_FTR_SECTION cmpwi r3,0xa00 beq h_doorbell_common_msgclr diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index e137b98dc5f8..1a41cceddfe2 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -186,6 +186,11 @@ notrace unsigned int __check_irq_replay(void) return 0x900; } + if (happened & PACA_IRQ_PMI) { + local_paca->irq_happened &= ~PACA_IRQ_PMI; + return 0xf00; + } + if (happened & PACA_IRQ_EE) { local_paca->irq_happened &= ~PACA_IRQ_EE; return 0x500; @@ -272,7 +277,7 @@ notrace void arch_local_irq_restore(unsigned long mask) } #endif /* CONFIG_TRACE_IRQFLAGS */ - irq_soft_mask_set(IRQS_DISABLED); + irq_soft_mask_set(IRQS_ALL_DISABLED); trace_hardirqs_off(); /* -- cgit v1.2.3 From 9aa88188ee87f4cc5c4e1bce15754bfcde8e900f Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:54 +0530 Subject: powerpc: Add new kconfig CONFIG_PPC_IRQ_SOFT_MASK_DEBUG New Kconfig is added "CONFIG_PPC_IRQ_SOFT_MASK_DEBUG" to add WARN_ON to alert the invalid transitions. Also moved the code under the CONFIG_TRACE_IRQFLAGS in arch_local_irq_restore() to new Kconfig. Reviewed-by: Nicholas Piggin Signed-off-by: Madhavan Srinivasan [mpe: Fix name of CONFIG option in change log] Signed-off-by: Michael Ellerman --- arch/powerpc/Kconfig.debug | 4 ++++ arch/powerpc/include/asm/hw_irq.h | 4 ++-- arch/powerpc/kernel/entry_64.S | 4 ++-- arch/powerpc/kernel/irq.c | 4 ++-- 4 files changed, 10 insertions(+), 6 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug index 5d18169fff58..c45424c64e19 100644 --- a/arch/powerpc/Kconfig.debug +++ b/arch/powerpc/Kconfig.debug @@ -90,6 +90,10 @@ config MSI_BITMAP_SELFTEST depends on DEBUG_KERNEL default n +config PPC_IRQ_SOFT_MASK_DEBUG + bool "Include extra checks for powerpc irq soft masking" + default n + config XMON bool "Include xmon kernel debugger" depends on DEBUG_KERNEL diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index 552a2c6e428f..f40da3bd079b 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -71,7 +71,7 @@ static inline notrace unsigned long irq_soft_mask_return(void) */ static inline notrace void irq_soft_mask_set(unsigned long mask) { -#ifdef CONFIG_TRACE_IRQFLAGS +#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG /* * The irq mask must always include the STD bit if any are set. * @@ -101,7 +101,7 @@ static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask) { unsigned long flags; -#ifdef CONFIG_TRACE_IRQFLAGS +#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG WARN_ON(mask && !(mask & IRQS_DISABLED)); #endif diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 3b90a2f1018e..b2995509b01d 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -128,7 +128,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR) * of irq tracing is used, we additionally check that condition * is correct */ -#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG) +#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG) lbz r10,PACAIRQSOFTMASK(r13) 1: tdnei r10,IRQS_ENABLED EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING @@ -911,7 +911,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS stb r7,PACAIRQHAPPENED(r13) 1: -#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG) +#if defined(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG) && defined(CONFIG_BUG) /* The interrupt should not have soft enabled. */ lbz r7,PACAIRQSOFTMASK(r13) 1: tdeqi r7,IRQS_ENABLED diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 1a41cceddfe2..f88038847790 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -264,7 +264,7 @@ notrace void arch_local_irq_restore(unsigned long mask) */ if (unlikely(irq_happened != PACA_IRQ_HARD_DIS)) __hard_irq_disable(); -#ifdef CONFIG_TRACE_IRQFLAGS +#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG else { /* * We should already be hard disabled here. We had bugs @@ -275,7 +275,7 @@ notrace void arch_local_irq_restore(unsigned long mask) if (WARN_ON(mfmsr() & MSR_EE)) __hard_irq_disable(); } -#endif /* CONFIG_TRACE_IRQFLAGS */ +#endif irq_soft_mask_set(IRQS_ALL_DISABLED); trace_hardirqs_off(); -- cgit v1.2.3 From c6424382de996c216e81cfccf5cf2371157df651 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:55 +0530 Subject: powerpc/64s: Add new set of irq_soft_mask_ functions for PMI masking To support soft-masking of the performance monitor interrupt, a set of new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore() functions are added. And powerpc_local_irq_save() implemented, by adding a new irq_soft_mask manipulation function irq_soft_mask_or_return(). Local_irq_pmu_* macros are provided to access these powerpc_local_irq_pmu* functions which includes trace_hardirqs_on|off() to match what we have in include/linux/irqflags.h. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/hw_irq.h | 67 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index f40da3bd079b..88e5e8f17e98 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -115,6 +115,24 @@ static inline notrace unsigned long irq_soft_mask_set_return(unsigned long mask) return flags; } +static inline notrace unsigned long irq_soft_mask_or_return(unsigned long mask) +{ + unsigned long flags, tmp; + + asm volatile( + "lbz %0,%2(13); or %1,%0,%3; stb %1,%2(13)" + : "=&r" (flags), "=r" (tmp) + : "i" (offsetof(struct paca_struct, irq_soft_mask)), + "r" (mask) + : "memory"); + +#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG + WARN_ON((mask | flags) && !((mask | flags) & IRQS_DISABLED)); +#endif + + return flags; +} + static inline unsigned long arch_local_save_flags(void) { return irq_soft_mask_return(); @@ -147,6 +165,55 @@ static inline bool arch_irqs_disabled(void) return arch_irqs_disabled_flags(arch_local_save_flags()); } +#ifdef CONFIG_PPC_BOOK3S +/* + * To support disabling and enabling of irq with PMI, set of + * new powerpc_local_irq_pmu_save() and powerpc_local_irq_restore() + * functions are added. These macros are implemented using generic + * linux local_irq_* code from include/linux/irqflags.h. + */ +#define raw_local_irq_pmu_save(flags) \ + do { \ + typecheck(unsigned long, flags); \ + flags = irq_soft_mask_or_return(IRQS_DISABLED | \ + IRQS_PMI_DISABLED); \ + } while(0) + +#define raw_local_irq_pmu_restore(flags) \ + do { \ + typecheck(unsigned long, flags); \ + arch_local_irq_restore(flags); \ + } while(0) + +#ifdef CONFIG_TRACE_IRQFLAGS +#define powerpc_local_irq_pmu_save(flags) \ + do { \ + raw_local_irq_pmu_save(flags); \ + trace_hardirqs_off(); \ + } while(0) +#define powerpc_local_irq_pmu_restore(flags) \ + do { \ + if (raw_irqs_disabled_flags(flags)) { \ + raw_local_irq_pmu_restore(flags); \ + trace_hardirqs_off(); \ + } else { \ + trace_hardirqs_on(); \ + raw_local_irq_pmu_restore(flags); \ + } \ + } while(0) +#else +#define powerpc_local_irq_pmu_save(flags) \ + do { \ + raw_local_irq_pmu_save(flags); \ + } while(0) +#define powerpc_local_irq_pmu_restore(flags) \ + do { \ + raw_local_irq_pmu_restore(flags); \ + } while (0) +#endif /* CONFIG_TRACE_IRQFLAGS */ + +#endif /* CONFIG_PPC_BOOK3S */ + #ifdef CONFIG_PPC_BOOK3E #define __hard_irq_enable() asm volatile("wrteei 1" : : : "memory") #define __hard_irq_disable() asm volatile("wrteei 0" : : : "memory") -- cgit v1.2.3 From 3b7e30208699e23a70565601cac54b1f0741b6d7 Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:56 +0530 Subject: powerpc: use generic atomic implementation for local_t powerpc implements local_t with atomic operations. There is already an asm-generic implementation which does this using atomic_t. Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/local.h | 171 +-------------------------------------- 1 file changed, 1 insertion(+), 170 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h index 600a68bd77f5..cfbcc31d43ad 100644 --- a/arch/powerpc/include/asm/local.h +++ b/arch/powerpc/include/asm/local.h @@ -2,175 +2,6 @@ #ifndef _ARCH_POWERPC_LOCAL_H #define _ARCH_POWERPC_LOCAL_H -#include -#include - -typedef struct -{ - atomic_long_t a; -} local_t; - -#define LOCAL_INIT(i) { ATOMIC_LONG_INIT(i) } - -#define local_read(l) atomic_long_read(&(l)->a) -#define local_set(l,i) atomic_long_set(&(l)->a, (i)) - -#define local_add(i,l) atomic_long_add((i),(&(l)->a)) -#define local_sub(i,l) atomic_long_sub((i),(&(l)->a)) -#define local_inc(l) atomic_long_inc(&(l)->a) -#define local_dec(l) atomic_long_dec(&(l)->a) - -static __inline__ long local_add_return(long a, local_t *l) -{ - long t; - - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%2,0) " # local_add_return\n\ - add %0,%1,%0\n" - PPC405_ERR77(0,%2) - PPC_STLCX "%0,0,%2 \n\ - bne- 1b" - : "=&r" (t) - : "r" (a), "r" (&(l->a.counter)) - : "cc", "memory"); - - return t; -} - -#define local_add_negative(a, l) (local_add_return((a), (l)) < 0) - -static __inline__ long local_sub_return(long a, local_t *l) -{ - long t; - - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%2,0) " # local_sub_return\n\ - subf %0,%1,%0\n" - PPC405_ERR77(0,%2) - PPC_STLCX "%0,0,%2 \n\ - bne- 1b" - : "=&r" (t) - : "r" (a), "r" (&(l->a.counter)) - : "cc", "memory"); - - return t; -} - -static __inline__ long local_inc_return(local_t *l) -{ - long t; - - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%1,0) " # local_inc_return\n\ - addic %0,%0,1\n" - PPC405_ERR77(0,%1) - PPC_STLCX "%0,0,%1 \n\ - bne- 1b" - : "=&r" (t) - : "r" (&(l->a.counter)) - : "cc", "xer", "memory"); - - return t; -} - -/* - * local_inc_and_test - increment and test - * @l: pointer of type local_t - * - * Atomically increments @l by 1 - * and returns true if the result is zero, or false for all - * other cases. - */ -#define local_inc_and_test(l) (local_inc_return(l) == 0) - -static __inline__ long local_dec_return(local_t *l) -{ - long t; - - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%1,0) " # local_dec_return\n\ - addic %0,%0,-1\n" - PPC405_ERR77(0,%1) - PPC_STLCX "%0,0,%1\n\ - bne- 1b" - : "=&r" (t) - : "r" (&(l->a.counter)) - : "cc", "xer", "memory"); - - return t; -} - -#define local_cmpxchg(l, o, n) \ - (cmpxchg_local(&((l)->a.counter), (o), (n))) -#define local_xchg(l, n) (xchg_local(&((l)->a.counter), (n))) - -/** - * local_add_unless - add unless the number is a given value - * @l: pointer of type local_t - * @a: the amount to add to v... - * @u: ...unless v is equal to u. - * - * Atomically adds @a to @l, so long as it was not @u. - * Returns non-zero if @l was not @u, and zero otherwise. - */ -static __inline__ int local_add_unless(local_t *l, long a, long u) -{ - long t; - - __asm__ __volatile__ ( -"1:" PPC_LLARX(%0,0,%1,0) " # local_add_unless\n\ - cmpw 0,%0,%3 \n\ - beq- 2f \n\ - add %0,%2,%0 \n" - PPC405_ERR77(0,%2) - PPC_STLCX "%0,0,%1 \n\ - bne- 1b \n" -" subf %0,%2,%0 \n\ -2:" - : "=&r" (t) - : "r" (&(l->a.counter)), "r" (a), "r" (u) - : "cc", "memory"); - - return t != u; -} - -#define local_inc_not_zero(l) local_add_unless((l), 1, 0) - -#define local_sub_and_test(a, l) (local_sub_return((a), (l)) == 0) -#define local_dec_and_test(l) (local_dec_return((l)) == 0) - -/* - * Atomically test *l and decrement if it is greater than 0. - * The function returns the old value of *l minus 1. - */ -static __inline__ long local_dec_if_positive(local_t *l) -{ - long t; - - __asm__ __volatile__( -"1:" PPC_LLARX(%0,0,%1,0) " # local_dec_if_positive\n\ - cmpwi %0,1\n\ - addi %0,%0,-1\n\ - blt- 2f\n" - PPC405_ERR77(0,%1) - PPC_STLCX "%0,0,%1\n\ - bne- 1b" - "\n\ -2:" : "=&b" (t) - : "r" (&(l->a.counter)) - : "cc", "memory"); - - return t; -} - -/* Use these for per-cpu local_t variables: on some archs they are - * much more efficient than these naive implementations. Note they take - * a variable, not an address. - */ - -#define __local_inc(l) ((l)->a.counter++) -#define __local_dec(l) ((l)->a.counter++) -#define __local_add(i,l) ((l)->a.counter+=(i)) -#define __local_sub(i,l) ((l)->a.counter-=(i)) +#include #endif /* _ARCH_POWERPC_LOCAL_H */ -- cgit v1.2.3 From 6cd74d2b6174957103c8f6f2cfcefe086a51e6eb Mon Sep 17 00:00:00 2001 From: Madhavan Srinivasan Date: Wed, 20 Dec 2017 09:25:57 +0530 Subject: powerpc/64s: Implement local_t using irq soft masking local_t is used for atomic modifications for per-CPU data, versus re-entrant modifications via interrupts. local_t read-modify-write atomic operations are currently implemented with hardware atomics (larx/stcx), which are quite slow. This patch implements them by masking all types of interrupts that may do local_t operations ("standard" and perf interrupts). Rusty's benchmark (https://lkml.org/lkml/2008/12/16/450) gives the following timings for the local_t test, in nanoseconds per iteration: larx/stcx irq+pmu disable _inc 38 10 _add 38 10 _read 4 4 _add_return 38 10 There are still some interrupt types (system reset, machine check, and watchdog), which can not safely use local_t operations, because they are not masked. An alternative approach was proposed, using a CR bit to mark a critical section, which is tested in the interrupt return path, and would then branch to a fixup handler (similar to exception fixups), which re-starts the operation. The problem with this was the complexity of the fixup handler and the latency of the slow path. https://lists.ozlabs.org/pipermail/linuxppc-dev/2014-November/123024.html Signed-off-by: Madhavan Srinivasan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/local.h | 141 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/local.h b/arch/powerpc/include/asm/local.h index cfbcc31d43ad..fdd00939270b 100644 --- a/arch/powerpc/include/asm/local.h +++ b/arch/powerpc/include/asm/local.h @@ -2,6 +2,147 @@ #ifndef _ARCH_POWERPC_LOCAL_H #define _ARCH_POWERPC_LOCAL_H +#ifdef CONFIG_PPC_BOOK3S_64 + +#include +#include +#include + +#include + +typedef struct +{ + long v; +} local_t; + +#define LOCAL_INIT(i) { (i) } + +static __inline__ long local_read(local_t *l) +{ + return READ_ONCE(l->v); +} + +static __inline__ void local_set(local_t *l, long i) +{ + WRITE_ONCE(l->v, i); +} + +#define LOCAL_OP(op, c_op) \ +static __inline__ void local_##op(long i, local_t *l) \ +{ \ + unsigned long flags; \ + \ + powerpc_local_irq_pmu_save(flags); \ + l->v c_op i; \ + powerpc_local_irq_pmu_restore(flags); \ +} + +#define LOCAL_OP_RETURN(op, c_op) \ +static __inline__ long local_##op##_return(long a, local_t *l) \ +{ \ + long t; \ + unsigned long flags; \ + \ + powerpc_local_irq_pmu_save(flags); \ + t = (l->v c_op a); \ + powerpc_local_irq_pmu_restore(flags); \ + \ + return t; \ +} + +#define LOCAL_OPS(op, c_op) \ + LOCAL_OP(op, c_op) \ + LOCAL_OP_RETURN(op, c_op) + +LOCAL_OPS(add, +=) +LOCAL_OPS(sub, -=) + +#define local_add_negative(a, l) (local_add_return((a), (l)) < 0) +#define local_inc_return(l) local_add_return(1LL, l) +#define local_inc(l) local_inc_return(l) + +/* + * local_inc_and_test - increment and test + * @l: pointer of type local_t + * + * Atomically increments @l by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +#define local_inc_and_test(l) (local_inc_return(l) == 0) + +#define local_dec_return(l) local_sub_return(1LL, l) +#define local_dec(l) local_dec_return(l) +#define local_sub_and_test(a, l) (local_sub_return((a), (l)) == 0) +#define local_dec_and_test(l) (local_dec_return((l)) == 0) + +static __inline__ long local_cmpxchg(local_t *l, long o, long n) +{ + long t; + unsigned long flags; + + powerpc_local_irq_pmu_save(flags); + t = l->v; + if (t == o) + l->v = n; + powerpc_local_irq_pmu_restore(flags); + + return t; +} + +static __inline__ long local_xchg(local_t *l, long n) +{ + long t; + unsigned long flags; + + powerpc_local_irq_pmu_save(flags); + t = l->v; + l->v = n; + powerpc_local_irq_pmu_restore(flags); + + return t; +} + +/** + * local_add_unless - add unless the number is a given value + * @l: pointer of type local_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @l, so long as it was not @u. + * Returns non-zero if @l was not @u, and zero otherwise. + */ +static __inline__ int local_add_unless(local_t *l, long a, long u) +{ + unsigned long flags; + int ret = 0; + + powerpc_local_irq_pmu_save(flags); + if (l->v != u) { + l->v += a; + ret = 1; + } + powerpc_local_irq_pmu_restore(flags); + + return ret; +} + +#define local_inc_not_zero(l) local_add_unless((l), 1, 0) + +/* Use these for per-cpu local_t variables: on some archs they are + * much more efficient than these naive implementations. Note they take + * a variable, not an address. + */ + +#define __local_inc(l) ((l)->v++) +#define __local_dec(l) ((l)->v++) +#define __local_add(i,l) ((l)->v+=(i)) +#define __local_sub(i,l) ((l)->v-=(i)) + +#else /* CONFIG_PPC64 */ + #include +#endif /* CONFIG_PPC64 */ + #endif /* _ARCH_POWERPC_LOCAL_H */ -- cgit v1.2.3 From 8b4e6deaff7822c76c94336c40f8d1f244c6f6ed Mon Sep 17 00:00:00 2001 From: Anju T Sudhakar Date: Mon, 11 Dec 2017 11:28:37 +0530 Subject: powerpc/perf: Pass struct imc_events as a parameter to imc_parse_event() Remove the allocation of struct imc_events from imc_parse_event(). Instead pass imc_events as a parameter to imc_parse_event(), which is a pointer to a slot in the array allocated in update_events_in_group(). Reported-by: Dan Carpenter ("powerpc/perf: Fix a sizeof() typo so we allocate less memory") Suggested-by: Michael Ellerman Signed-off-by: Anju T Sudhakar Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/imc-pmu.h | 2 +- arch/powerpc/perf/imc-pmu.c | 66 +++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 27 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/imc-pmu.h b/arch/powerpc/include/asm/imc-pmu.h index fad0e6ff460f..080731dc883a 100644 --- a/arch/powerpc/include/asm/imc-pmu.h +++ b/arch/powerpc/include/asm/imc-pmu.h @@ -71,7 +71,7 @@ struct imc_events { struct imc_pmu { struct pmu pmu; struct imc_mem_info *mem_info; - struct imc_events **events; + struct imc_events *events; /* * Attribute groups for the PMU. Slot 0 used for * format attribute, slot 1 used for cpusmask attribute, diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index 9db2529df3ed..9ec7e87056e5 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -116,17 +116,13 @@ static struct attribute *device_str_attr_create(const char *name, const char *st return &attr->attr.attr; } -struct imc_events *imc_parse_event(struct device_node *np, const char *scale, - const char *unit, const char *prefix, u32 base) +static int imc_parse_event(struct device_node *np, const char *scale, + const char *unit, const char *prefix, + u32 base, struct imc_events *event) { - struct imc_events *event; const char *s; u32 reg; - event = kzalloc(sizeof(struct imc_events), GFP_KERNEL); - if (!event) - return NULL; - if (of_property_read_u32(np, "reg", ®)) goto error; /* Add the base_reg value to the "reg" */ @@ -157,14 +153,32 @@ struct imc_events *imc_parse_event(struct device_node *np, const char *scale, goto error; } - return event; + return 0; error: kfree(event->unit); kfree(event->scale); kfree(event->name); - kfree(event); + return -EINVAL; +} + +/* + * imc_free_events: Function to cleanup the events list, having + * "nr_entries". + */ +static void imc_free_events(struct imc_events *events, int nr_entries) +{ + int i; + + /* Nothing to clean, return */ + if (!events) + return; + for (i = 0; i < nr_entries; i++) { + kfree(events[i].unit); + kfree(events[i].scale); + kfree(events[i].name); + } - return NULL; + kfree(events); } /* @@ -176,9 +190,8 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu) struct attribute_group *attr_group; struct attribute **attrs, *dev_str; struct device_node *np, *pmu_events; - struct imc_events *ev; u32 handle, base_reg; - int i=0, j=0, ct; + int i = 0, j = 0, ct, ret; const char *prefix, *g_scale, *g_unit; const char *ev_val_str, *ev_scale_str, *ev_unit_str; @@ -216,15 +229,17 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu) ct = 0; /* Parse the events and update the struct */ for_each_child_of_node(pmu_events, np) { - ev = imc_parse_event(np, g_scale, g_unit, prefix, base_reg); - if (ev) - pmu->events[ct++] = ev; + ret = imc_parse_event(np, g_scale, g_unit, prefix, base_reg, &pmu->events[ct]); + if (!ret) + ct++; } /* Allocate memory for attribute group */ attr_group = kzalloc(sizeof(*attr_group), GFP_KERNEL); - if (!attr_group) + if (!attr_group) { + imc_free_events(pmu->events, ct); return -ENOMEM; + } /* * Allocate memory for attributes. @@ -237,31 +252,31 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu) attrs = kcalloc(((ct * 3) + 1), sizeof(struct attribute *), GFP_KERNEL); if (!attrs) { kfree(attr_group); - kfree(pmu->events); + imc_free_events(pmu->events, ct); return -ENOMEM; } attr_group->name = "events"; attr_group->attrs = attrs; do { - ev_val_str = kasprintf(GFP_KERNEL, "event=0x%x", pmu->events[i]->value); - dev_str = device_str_attr_create(pmu->events[i]->name, ev_val_str); + ev_val_str = kasprintf(GFP_KERNEL, "event=0x%x", pmu->events[i].value); + dev_str = device_str_attr_create(pmu->events[i].name, ev_val_str); if (!dev_str) continue; attrs[j++] = dev_str; - if (pmu->events[i]->scale) { - ev_scale_str = kasprintf(GFP_KERNEL, "%s.scale",pmu->events[i]->name); - dev_str = device_str_attr_create(ev_scale_str, pmu->events[i]->scale); + if (pmu->events[i].scale) { + ev_scale_str = kasprintf(GFP_KERNEL, "%s.scale", pmu->events[i].name); + dev_str = device_str_attr_create(ev_scale_str, pmu->events[i].scale); if (!dev_str) continue; attrs[j++] = dev_str; } - if (pmu->events[i]->unit) { - ev_unit_str = kasprintf(GFP_KERNEL, "%s.unit",pmu->events[i]->name); - dev_str = device_str_attr_create(ev_unit_str, pmu->events[i]->unit); + if (pmu->events[i].unit) { + ev_unit_str = kasprintf(GFP_KERNEL, "%s.unit", pmu->events[i].name); + dev_str = device_str_attr_create(ev_unit_str, pmu->events[i].unit); if (!dev_str) continue; @@ -272,7 +287,6 @@ static int update_events_in_group(struct device_node *node, struct imc_pmu *pmu) /* Save the event attribute */ pmu->attr_groups[IMC_EVENT_ATTR] = attr_group; - kfree(pmu->events); return 0; } -- cgit v1.2.3 From 684d984038aa2b8ec3ab4b9d314257879a3fbfbd Mon Sep 17 00:00:00 2001 From: Anju T Sudhakar Date: Wed, 13 Dec 2017 11:39:54 +0530 Subject: powerpc/powernv: Add debugfs interface for imc-mode and imc-command In memory Collection (IMC) counter pmu driver controls the ucode's execution state. At the system boot, IMC perf driver pause the ucode. Ucode state is changed to "running" only when any of the nest units are monitored or profiled using perf tool. Nest units support only limited set of hardware counters and ucode is always programmed in the "production mode" ("accumulation") mode. This mode is configured to provide key performance metric data for most of the nest units. But ucode also supports other modes which would be used for "debug" to drill down specific nest units. That is, ucode when switched to "powerbus" debug mode (for example), will dynamically reconfigure the nest counters to target only "powerbus" related events in the hardware counters. This allows the IMC nest unit to focus on powerbus related transactions in the system in more detail. At this point, production mode events may or may not be counted. IMC nest counters has both in-band (ucode access) and out of band access to it. Since not all nest counter configurations are supported by ucode, out of band tools are used to characterize other nest counter configurations. Patch provides an interface via "debugfs" to enable the switching of ucode modes in the system. To switch ucode mode, one has to first pause the microcode (imc_cmd), and then write the target mode value to the "imc_mode" file. Proposed Approach: In the proposed approach, the function (export_imc_mode_and_cmd) which creates the debugfs interface for imc mode and command is implemented in opal-imc.c. Thus we can use imc_get_mem_addr() to get the homer base address for each chip. The interface to expose imc mode and command is required only if we have nest pmu units registered. Employing the existing data structures to track whether we have any nest units registered will require to extend data from perf side to opal-imc.c. Instead an integer is introduced to hold that information by counting successful nest unit registration. Debugfs interface is removed based on the integer count. Example for the interface: $ ls /sys/kernel/debug/imc imc_cmd_0 imc_cmd_8 imc_mode_0 imc_mode_8 Signed-off-by: Anju T Sudhakar Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/imc-pmu.h | 7 +++ arch/powerpc/platforms/powernv/opal-imc.c | 77 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/imc-pmu.h b/arch/powerpc/include/asm/imc-pmu.h index 080731dc883a..d76cb11be3e3 100644 --- a/arch/powerpc/include/asm/imc-pmu.h +++ b/arch/powerpc/include/asm/imc-pmu.h @@ -34,6 +34,13 @@ #define THREAD_IMC_LDBAR_MASK 0x0003ffffffffe000ULL #define THREAD_IMC_ENABLE 0x8000000000000000ULL +/* + * For debugfs interface for imc-mode and imc-command + */ +#define IMC_CNTL_BLK_OFFSET 0x3FC00 +#define IMC_CNTL_BLK_CMD_OFFSET 8 +#define IMC_CNTL_BLK_MODE_OFFSET 32 + /* * Structure to hold memory address information for imc units. */ diff --git a/arch/powerpc/platforms/powernv/opal-imc.c b/arch/powerpc/platforms/powernv/opal-imc.c index 465ea105b771..dd4c9b8b8a81 100644 --- a/arch/powerpc/platforms/powernv/opal-imc.c +++ b/arch/powerpc/platforms/powernv/opal-imc.c @@ -21,6 +21,78 @@ #include #include #include +#include + +static struct dentry *imc_debugfs_parent; + +/* Helpers to export imc command and mode via debugfs */ +static int imc_mem_get(void *data, u64 *val) +{ + *val = cpu_to_be64(*(u64 *)data); + return 0; +} + +static int imc_mem_set(void *data, u64 val) +{ + *(u64 *)data = cpu_to_be64(val); + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_imc_x64, imc_mem_get, imc_mem_set, "0x%016llx\n"); + +static struct dentry *imc_debugfs_create_x64(const char *name, umode_t mode, + struct dentry *parent, u64 *value) +{ + return debugfs_create_file_unsafe(name, mode, parent, + value, &fops_imc_x64); +} + +/* + * export_imc_mode_and_cmd: Create a debugfs interface + * for imc_cmd and imc_mode + * for each node in the system. + * imc_mode and imc_cmd can be changed by echo into + * this interface. + */ +static void export_imc_mode_and_cmd(struct device_node *node, + struct imc_pmu *pmu_ptr) +{ + static u64 loc, *imc_mode_addr, *imc_cmd_addr; + int chip = 0, nid; + char mode[16], cmd[16]; + u32 cb_offset; + + imc_debugfs_parent = debugfs_create_dir("imc", powerpc_debugfs_root); + + /* + * Return here, either because 'imc' directory already exists, + * Or failed to create a new one. + */ + if (!imc_debugfs_parent) + return; + + if (of_property_read_u32(node, "cb_offset", &cb_offset)) + cb_offset = IMC_CNTL_BLK_OFFSET; + + for_each_node(nid) { + loc = (u64)(pmu_ptr->mem_info[chip].vbase) + cb_offset; + imc_mode_addr = (u64 *)(loc + IMC_CNTL_BLK_MODE_OFFSET); + sprintf(mode, "imc_mode_%d", nid); + if (!imc_debugfs_create_x64(mode, 0600, imc_debugfs_parent, + imc_mode_addr)) + goto err; + + imc_cmd_addr = (u64 *)(loc + IMC_CNTL_BLK_CMD_OFFSET); + sprintf(cmd, "imc_cmd_%d", nid); + if (!imc_debugfs_create_x64(cmd, 0600, imc_debugfs_parent, + imc_cmd_addr)) + goto err; + chip++; + } + return; + +err: + debugfs_remove_recursive(imc_debugfs_parent); +} /* * imc_get_mem_addr_nest: Function to get nest counter memory region @@ -65,6 +137,7 @@ static int imc_get_mem_addr_nest(struct device_node *node, } pmu_ptr->imc_counter_mmaped = true; + export_imc_mode_and_cmd(node, pmu_ptr); kfree(base_addr_arr); kfree(chipid_arr); return 0; @@ -213,6 +286,10 @@ static int opal_imc_counters_probe(struct platform_device *pdev) } } + /* If none of the nest units are registered, remove debugfs interface */ + if (pmu_count == 0) + debugfs_remove_recursive(imc_debugfs_parent); + return 0; } -- cgit v1.2.3 From 92e3da3cf193fd27996909956c12a23c0333da44 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:24 -0800 Subject: powerpc: initial pkey plumbing Basic plumbing to initialize the pkey system. Nothing is enabled yet. A later patch will enable it once all the infrastructure is in place. Signed-off-by: Ram Pai [mpe: Rework copyrights to use SPDX tags] Signed-off-by: Michael Ellerman --- arch/powerpc/Kconfig | 15 ++++++++++ arch/powerpc/include/asm/mmu_context.h | 1 + arch/powerpc/include/asm/pkeys.h | 51 ++++++++++++++++++++++++++++++++++ arch/powerpc/mm/Makefile | 1 + arch/powerpc/mm/hash_utils_64.c | 1 + arch/powerpc/mm/pkeys.c | 29 +++++++++++++++++++ 6 files changed, 98 insertions(+) create mode 100644 arch/powerpc/include/asm/pkeys.h create mode 100644 arch/powerpc/mm/pkeys.c (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index c51e6ce42e7a..c9660a18c093 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -867,6 +867,21 @@ config SECCOMP If unsure, say Y. Only embedded should say N here. +config PPC_MEM_KEYS + prompt "PowerPC Memory Protection Keys" + def_bool y + depends on PPC_BOOK3S_64 + select ARCH_USES_HIGH_VMA_FLAGS + select ARCH_HAS_PKEYS + help + Memory Protection Keys provides a mechanism for enforcing + page-based protections, but without requiring modification of the + page tables when an application changes protection domains. + + For details, see Documentation/vm/protection-keys.txt + + If unsure, say y. + endmenu config ISA_DMA_API diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 6177d43f0ce8..fb5e6a3ce127 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -192,5 +192,6 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, /* by default, allow everything */ return true; } + #endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_MMU_CONTEXT_H */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h new file mode 100644 index 000000000000..dadea7d4324d --- /dev/null +++ b/arch/powerpc/include/asm/pkeys.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * PowerPC Memory Protection Keys management + * + * Copyright 2017, Ram Pai, IBM Corporation. + */ + +#ifndef _ASM_POWERPC_KEYS_H +#define _ASM_POWERPC_KEYS_H + +#include + +DECLARE_STATIC_KEY_TRUE(pkey_disabled); +#define ARCH_VM_PKEY_FLAGS 0 + +static inline bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey) +{ + return false; +} + +static inline int mm_pkey_alloc(struct mm_struct *mm) +{ + return -1; +} + +static inline int mm_pkey_free(struct mm_struct *mm, int pkey) +{ + return -EINVAL; +} + +/* + * Try to dedicate one of the protection keys to be used as an + * execute-only protection key. + */ +static inline int execute_only_pkey(struct mm_struct *mm) +{ + return 0; +} + +static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma, + int prot, int pkey) +{ + return 0; +} + +static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, + unsigned long init_val) +{ + return 0; +} +#endif /*_ASM_POWERPC_KEYS_H */ diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index 8d271bfe2d94..f06f3577d8d1 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile @@ -44,3 +44,4 @@ obj-$(CONFIG_PPC_COPRO_BASE) += copro_fault.o obj-$(CONFIG_SPAPR_TCE_IOMMU) += mmu_context_iommu.o obj-$(CONFIG_PPC_PTDUMP) += dump_linuxpagetables.o obj-$(CONFIG_PPC_HTDUMP) += dump_hashpagetable.o +obj-$(CONFIG_PPC_MEM_KEYS) += pkeys.o diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 64e704eefad1..8cd31ea9b243 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c new file mode 100644 index 000000000000..a11afba584b9 --- /dev/null +++ b/arch/powerpc/mm/pkeys.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PowerPC Memory Protection Keys management + * + * Copyright 2017, Ram Pai, IBM Corporation. + */ + +#include + +DEFINE_STATIC_KEY_TRUE(pkey_disabled); +bool pkey_execute_disable_supported; + +int pkey_initialize(void) +{ + /* + * Disable the pkey system till everything is in place. A subsequent + * patch will enable it. + */ + static_branch_enable(&pkey_disabled); + + /* + * Disable execute_disable support for now. A subsequent patch will + * enable it. + */ + pkey_execute_disable_supported = false; + return 0; +} + +arch_initcall(pkey_initialize); -- cgit v1.2.3 From 4fb158f65ac5556b9b4a6f63f38272853ed99b22 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:25 -0800 Subject: powerpc: track allocation status of all pkeys Total 32 keys are available on power7 and above. However pkey 0,1 are reserved. So effectively we have 30 pkeys. On 4K kernels, we do not have 5 bits in the PTE to represent all the keys; we only have 3bits. Two of those keys are reserved; pkey 0 and pkey 1. So effectively we have 6 pkeys. This patch keeps track of reserved keys, allocated keys and keys that are currently free. Also it adds skeletal functions and macros, that the architecture-independent code expects to be available. Reviewed-by: Thiago Jung Bauermann Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/mmu.h | 9 ++++ arch/powerpc/include/asm/mmu_context.h | 4 ++ arch/powerpc/include/asm/pkeys.h | 83 ++++++++++++++++++++++++++++++-- arch/powerpc/mm/mmu_context_book3s64.c | 2 + arch/powerpc/mm/pkeys.c | 40 +++++++++++++++ 5 files changed, 134 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index c9448e19847a..37ef23cc1136 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -108,6 +108,15 @@ typedef struct { #ifdef CONFIG_SPAPR_TCE_IOMMU struct list_head iommu_group_mem_list; #endif + +#ifdef CONFIG_PPC_MEM_KEYS + /* + * Each bit represents one protection key. + * bit set -> key allocated + * bit unset -> key available for allocation + */ + u32 pkey_allocation_map; +#endif } mm_context_t; /* diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index fb5e6a3ce127..7d0f2d05189b 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -193,5 +193,9 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, return true; } +#ifndef CONFIG_PPC_MEM_KEYS +#define pkey_mm_init(mm) +#endif /* CONFIG_PPC_MEM_KEYS */ + #endif /* __KERNEL__ */ #endif /* __ASM_POWERPC_MMU_CONTEXT_H */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index dadea7d4324d..79b105d1ff73 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -11,21 +11,94 @@ #include DECLARE_STATIC_KEY_TRUE(pkey_disabled); -#define ARCH_VM_PKEY_FLAGS 0 +extern int pkeys_total; /* total pkeys as per device tree */ +extern u32 initial_allocation_mask; /* bits set for reserved keys */ + +/* + * Define these here temporarily so we're not dependent on patching linux/mm.h. + * Once it's updated we can drop these. + */ +#ifndef VM_PKEY_BIT0 +# define VM_PKEY_SHIFT VM_HIGH_ARCH_BIT_0 +# define VM_PKEY_BIT0 VM_HIGH_ARCH_0 +# define VM_PKEY_BIT1 VM_HIGH_ARCH_1 +# define VM_PKEY_BIT2 VM_HIGH_ARCH_2 +# define VM_PKEY_BIT3 VM_HIGH_ARCH_3 +# define VM_PKEY_BIT4 VM_HIGH_ARCH_4 +#endif + +#define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | \ + VM_PKEY_BIT3 | VM_PKEY_BIT4) + +#define arch_max_pkey() pkeys_total + +#define pkey_alloc_mask(pkey) (0x1 << pkey) + +#define mm_pkey_allocation_map(mm) (mm->context.pkey_allocation_map) + +#define __mm_pkey_allocated(mm, pkey) { \ + mm_pkey_allocation_map(mm) |= pkey_alloc_mask(pkey); \ +} + +#define __mm_pkey_free(mm, pkey) { \ + mm_pkey_allocation_map(mm) &= ~pkey_alloc_mask(pkey); \ +} + +#define __mm_pkey_is_allocated(mm, pkey) \ + (mm_pkey_allocation_map(mm) & pkey_alloc_mask(pkey)) + +#define __mm_pkey_is_reserved(pkey) (initial_allocation_mask & \ + pkey_alloc_mask(pkey)) static inline bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey) { - return false; + /* A reserved key is never considered as 'explicitly allocated' */ + return ((pkey < arch_max_pkey()) && + !__mm_pkey_is_reserved(pkey) && + __mm_pkey_is_allocated(mm, pkey)); } +/* + * Returns a positive, 5-bit key on success, or -1 on failure. + * Relies on the mmap_sem to protect against concurrency in mm_pkey_alloc() and + * mm_pkey_free(). + */ static inline int mm_pkey_alloc(struct mm_struct *mm) { - return -1; + /* + * Note: this is the one and only place we make sure that the pkey is + * valid as far as the hardware is concerned. The rest of the kernel + * trusts that only good, valid pkeys come out of here. + */ + u32 all_pkeys_mask = (u32)(~(0x0)); + int ret; + + if (static_branch_likely(&pkey_disabled)) + return -1; + + /* + * Are we out of pkeys? We must handle this specially because ffz() + * behavior is undefined if there are no zeros. + */ + if (mm_pkey_allocation_map(mm) == all_pkeys_mask) + return -1; + + ret = ffz((u32)mm_pkey_allocation_map(mm)); + __mm_pkey_allocated(mm, ret); + return ret; } static inline int mm_pkey_free(struct mm_struct *mm, int pkey) { - return -EINVAL; + if (static_branch_likely(&pkey_disabled)) + return -1; + + if (!mm_pkey_is_allocated(mm, pkey)) + return -EINVAL; + + __mm_pkey_free(mm, pkey); + + return 0; } /* @@ -48,4 +121,6 @@ static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, { return 0; } + +extern void pkey_mm_init(struct mm_struct *mm); #endif /*_ASM_POWERPC_KEYS_H */ diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c index 59c0766ae4e0..929d9ef7083f 100644 --- a/arch/powerpc/mm/mmu_context_book3s64.c +++ b/arch/powerpc/mm/mmu_context_book3s64.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -118,6 +119,7 @@ static int hash__init_new_context(struct mm_struct *mm) subpage_prot_init_new_context(mm); + pkey_mm_init(mm); return index; } diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index a11afba584b9..e68e367dcdd7 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -9,21 +9,61 @@ DEFINE_STATIC_KEY_TRUE(pkey_disabled); bool pkey_execute_disable_supported; +int pkeys_total; /* Total pkeys as per device tree */ +u32 initial_allocation_mask; /* Bits set for reserved keys */ int pkey_initialize(void) { + int os_reserved, i; + /* * Disable the pkey system till everything is in place. A subsequent * patch will enable it. */ static_branch_enable(&pkey_disabled); + /* Lets assume 32 keys */ + pkeys_total = 32; + + /* + * Adjust the upper limit, based on the number of bits supported by + * arch-neutral code. + */ + pkeys_total = min_t(int, pkeys_total, + (ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)); + /* * Disable execute_disable support for now. A subsequent patch will * enable it. */ pkey_execute_disable_supported = false; + +#ifdef CONFIG_PPC_4K_PAGES + /* + * The OS can manage only 8 pkeys due to its inability to represent them + * in the Linux 4K PTE. + */ + os_reserved = pkeys_total - 8; +#else + os_reserved = 0; +#endif + /* + * Bits are in LE format. NOTE: 1, 0 are reserved. + * key 0 is the default key, which allows read/write/execute. + * key 1 is recommended not to be used. PowerISA(3.0) page 1015, + * programming note. + */ + initial_allocation_mask = ~0x0; + for (i = 2; i < (pkeys_total - os_reserved); i++) + initial_allocation_mask &= ~(0x1 << i); return 0; } arch_initcall(pkey_initialize); + +void pkey_mm_init(struct mm_struct *mm) +{ + if (static_branch_likely(&pkey_disabled)) + return; + mm_pkey_allocation_map(mm) = initial_allocation_mask; +} -- cgit v1.2.3 From 0685f217afe8279a754490954b8dcc02cb763a5d Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:28 -0800 Subject: powerpc: cleanup AMR, IAMR when a key is allocated or freed Cleanup the bits corresponding to a key in the AMR, and IAMR register, when the key is newly allocated/activated or is freed. We dont want some residual bits cause the hardware enforce unintended behavior when the key is activated or freed. Reviewed-by: Thiago Jung Bauermann Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pkeys.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 79b105d1ff73..5ed52ac39bab 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -58,6 +58,8 @@ static inline bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey) __mm_pkey_is_allocated(mm, pkey)); } +extern void __arch_activate_pkey(int pkey); +extern void __arch_deactivate_pkey(int pkey); /* * Returns a positive, 5-bit key on success, or -1 on failure. * Relies on the mmap_sem to protect against concurrency in mm_pkey_alloc() and @@ -85,6 +87,12 @@ static inline int mm_pkey_alloc(struct mm_struct *mm) ret = ffz((u32)mm_pkey_allocation_map(mm)); __mm_pkey_allocated(mm, ret); + + /* + * Enable the key in the hardware + */ + if (ret > 0) + __arch_activate_pkey(ret); return ret; } @@ -96,6 +104,10 @@ static inline int mm_pkey_free(struct mm_struct *mm, int pkey) if (!mm_pkey_is_allocated(mm, pkey)) return -EINVAL; + /* + * Disable the key in the hardware + */ + __arch_deactivate_pkey(pkey); __mm_pkey_free(mm, pkey); return 0; -- cgit v1.2.3 From 2ddc53f3a7516b3e2c4ab9c36854a227641a4fde Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:29 -0800 Subject: powerpc: implementation for arch_set_user_pkey_access() This patch provides the detailed implementation for a user to allocate a key and enable it in the hardware. It provides the plumbing, but it cannot be used till the system call is implemented. The next patch will do so. Reviewed-by: Thiago Jung Bauermann Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pkeys.h | 6 +++++- arch/powerpc/mm/pkeys.c | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 5ed52ac39bab..bf8e706dcb0f 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -128,10 +128,14 @@ static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma, return 0; } +extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, + unsigned long init_val); static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, unsigned long init_val) { - return 0; + if (static_branch_likely(&pkey_disabled)) + return -EINVAL; + return __arch_set_user_pkey_access(tsk, pkey, init_val); } extern void pkey_mm_init(struct mm_struct *mm); diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index 0cfe901594c7..b01238bd88c1 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -5,6 +5,7 @@ * Copyright 2017, Ram Pai, IBM Corporation. */ +#include #include DEFINE_STATIC_KEY_TRUE(pkey_disabled); @@ -13,6 +14,9 @@ int pkeys_total; /* Total pkeys as per device tree */ u32 initial_allocation_mask; /* Bits set for reserved keys */ #define AMR_BITS_PER_PKEY 2 +#define AMR_RD_BIT 0x1UL +#define AMR_WR_BIT 0x2UL +#define IAMR_EX_BIT 0x1UL #define PKEY_REG_BITS (sizeof(u64)*8) #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY)) @@ -108,6 +112,20 @@ static inline void write_uamor(u64 value) mtspr(SPRN_UAMOR, value); } +static bool is_pkey_enabled(int pkey) +{ + u64 uamor = read_uamor(); + u64 pkey_bits = 0x3ul << pkeyshift(pkey); + u64 uamor_pkey_bits = (uamor & pkey_bits); + + /* + * Both the bits in UAMOR corresponding to the key should be set or + * reset. + */ + WARN_ON(uamor_pkey_bits && (uamor_pkey_bits != pkey_bits)); + return !!(uamor_pkey_bits); +} + static inline void init_amr(int pkey, u8 init_bits) { u64 new_amr_bits = (((u64)init_bits & 0x3UL) << pkeyshift(pkey)); @@ -150,3 +168,25 @@ void __arch_deactivate_pkey(int pkey) { pkey_status_change(pkey, false); } + +/* + * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that + * specified in @init_val. + */ +int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, + unsigned long init_val) +{ + u64 new_amr_bits = 0x0ul; + + if (!is_pkey_enabled(pkey)) + return -EINVAL; + + /* Set the bits we need in AMR: */ + if (init_val & PKEY_DISABLE_ACCESS) + new_amr_bits |= AMR_RD_BIT | AMR_WR_BIT; + else if (init_val & PKEY_DISABLE_WRITE) + new_amr_bits |= AMR_WR_BIT; + + init_amr(pkey, new_amr_bits); + return 0; +} -- cgit v1.2.3 From 06bb53b33804613627c7ca1eda246459a7be2803 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:31 -0800 Subject: powerpc: store and restore the pkey state across context switches Store and restore the AMR, IAMR and UAMOR register state of the task before scheduling out and after scheduling in, respectively. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/mmu_context.h | 3 ++ arch/powerpc/include/asm/pkeys.h | 4 +++ arch/powerpc/include/asm/processor.h | 5 ++++ arch/powerpc/kernel/process.c | 7 +++++ arch/powerpc/mm/pkeys.c | 52 +++++++++++++++++++++++++++++++++- 5 files changed, 70 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 7d0f2d05189b..4d69223d217a 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -195,6 +195,9 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, #ifndef CONFIG_PPC_MEM_KEYS #define pkey_mm_init(mm) +#define thread_pkey_regs_save(thread) +#define thread_pkey_regs_restore(new_thread, old_thread) +#define thread_pkey_regs_init(thread) #endif /* CONFIG_PPC_MEM_KEYS */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index bf8e706dcb0f..41c303a01e1a 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -139,4 +139,8 @@ static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, } extern void pkey_mm_init(struct mm_struct *mm); +extern void thread_pkey_regs_save(struct thread_struct *thread); +extern void thread_pkey_regs_restore(struct thread_struct *new_thread, + struct thread_struct *old_thread); +extern void thread_pkey_regs_init(struct thread_struct *thread); #endif /*_ASM_POWERPC_KEYS_H */ diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h index bdab3b74eb98..01299cdc9806 100644 --- a/arch/powerpc/include/asm/processor.h +++ b/arch/powerpc/include/asm/processor.h @@ -309,6 +309,11 @@ struct thread_struct { struct thread_vr_state ckvr_state; /* Checkpointed VR state */ unsigned long ckvrsave; /* Checkpointed VRSAVE */ #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ +#ifdef CONFIG_PPC_MEM_KEYS + unsigned long amr; + unsigned long iamr; + unsigned long uamor; +#endif #ifdef CONFIG_KVM_BOOK3S_32_HANDLER void* kvm_shadow_vcpu; /* KVM internal data */ #endif /* CONFIG_KVM_BOOK3S_32_HANDLER */ diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 9eb78ec0ce5b..755fd7e23ed4 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -1103,6 +1104,8 @@ static inline void save_sprs(struct thread_struct *t) t->tar = mfspr(SPRN_TAR); } #endif + + thread_pkey_regs_save(t); } static inline void restore_sprs(struct thread_struct *old_thread, @@ -1142,6 +1145,8 @@ static inline void restore_sprs(struct thread_struct *old_thread, old_thread->tidr != new_thread->tidr) mtspr(SPRN_TIDR, new_thread->tidr); #endif + + thread_pkey_regs_restore(new_thread, old_thread); } #ifdef CONFIG_PPC_BOOK3S_64 @@ -1867,6 +1872,8 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp) current->thread.tm_tfiar = 0; current->thread.load_tm = 0; #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ + + thread_pkey_regs_init(¤t->thread); } EXPORT_SYMBOL(start_thread); diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index e61bea4a3606..91ec6c467074 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -12,6 +12,8 @@ DEFINE_STATIC_KEY_TRUE(pkey_disabled); bool pkey_execute_disable_supported; int pkeys_total; /* Total pkeys as per device tree */ u32 initial_allocation_mask; /* Bits set for reserved keys */ +u64 pkey_amr_uamor_mask; /* Bits in AMR/UMOR not to be touched */ +u64 pkey_iamr_mask; /* Bits in AMR not to be touched */ #define AMR_BITS_PER_PKEY 2 #define AMR_RD_BIT 0x1UL @@ -70,8 +72,16 @@ int pkey_initialize(void) * programming note. */ initial_allocation_mask = ~0x0; - for (i = 2; i < (pkeys_total - os_reserved); i++) + + /* register mask is in BE format */ + pkey_amr_uamor_mask = ~0x0ul; + pkey_iamr_mask = ~0x0ul; + + for (i = 2; i < (pkeys_total - os_reserved); i++) { initial_allocation_mask &= ~(0x1 << i); + pkey_amr_uamor_mask &= ~(0x3ul << pkeyshift(i)); + pkey_iamr_mask &= ~(0x1ul << pkeyshift(i)); + } return 0; } @@ -206,3 +216,43 @@ int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, init_amr(pkey, new_amr_bits); return 0; } + +void thread_pkey_regs_save(struct thread_struct *thread) +{ + if (static_branch_likely(&pkey_disabled)) + return; + + /* + * TODO: Skip saving registers if @thread hasn't used any keys yet. + */ + thread->amr = read_amr(); + thread->iamr = read_iamr(); + thread->uamor = read_uamor(); +} + +void thread_pkey_regs_restore(struct thread_struct *new_thread, + struct thread_struct *old_thread) +{ + if (static_branch_likely(&pkey_disabled)) + return; + + /* + * TODO: Just set UAMOR to zero if @new_thread hasn't used any keys yet. + */ + if (old_thread->amr != new_thread->amr) + write_amr(new_thread->amr); + if (old_thread->iamr != new_thread->iamr) + write_iamr(new_thread->iamr); + if (old_thread->uamor != new_thread->uamor) + write_uamor(new_thread->uamor); +} + +void thread_pkey_regs_init(struct thread_struct *thread) +{ + if (static_branch_likely(&pkey_disabled)) + return; + + write_amr(read_amr() & pkey_amr_uamor_mask); + write_iamr(read_iamr() & pkey_iamr_mask); + write_uamor(read_uamor() & pkey_amr_uamor_mask); +} -- cgit v1.2.3 From 5586cf61e108019565bb936daeb296e53df1c1d6 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:32 -0800 Subject: powerpc: introduce execute-only pkey This patch provides the implementation of execute-only pkey. The architecture-independent layer expects the arch-dependent layer, to support the ability to create and enable a special key which has execute-only permission. Acked-by: Balbir Singh Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/mmu.h | 1 + arch/powerpc/include/asm/pkeys.h | 6 +++- arch/powerpc/mm/pkeys.c | 58 ++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index 37ef23cc1136..0abeb0e2d616 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -116,6 +116,7 @@ typedef struct { * bit unset -> key available for allocation */ u32 pkey_allocation_map; + s16 execute_only_pkey; /* key holding execute-only protection */ #endif } mm_context_t; diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 41c303a01e1a..b8d2a8e3f2e1 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -117,9 +117,13 @@ static inline int mm_pkey_free(struct mm_struct *mm, int pkey) * Try to dedicate one of the protection keys to be used as an * execute-only protection key. */ +extern int __execute_only_pkey(struct mm_struct *mm); static inline int execute_only_pkey(struct mm_struct *mm) { - return 0; + if (static_branch_likely(&pkey_disabled)) + return -1; + + return __execute_only_pkey(mm); } static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma, diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index 91ec6c467074..6fece6e6fc97 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -92,6 +92,8 @@ void pkey_mm_init(struct mm_struct *mm) if (static_branch_likely(&pkey_disabled)) return; mm_pkey_allocation_map(mm) = initial_allocation_mask; + /* -1 means unallocated or invalid */ + mm->context.execute_only_pkey = -1; } static inline u64 read_amr(void) @@ -256,3 +258,59 @@ void thread_pkey_regs_init(struct thread_struct *thread) write_iamr(read_iamr() & pkey_iamr_mask); write_uamor(read_uamor() & pkey_amr_uamor_mask); } + +static inline bool pkey_allows_readwrite(int pkey) +{ + int pkey_shift = pkeyshift(pkey); + + if (!is_pkey_enabled(pkey)) + return true; + + return !(read_amr() & ((AMR_RD_BIT|AMR_WR_BIT) << pkey_shift)); +} + +int __execute_only_pkey(struct mm_struct *mm) +{ + bool need_to_set_mm_pkey = false; + int execute_only_pkey = mm->context.execute_only_pkey; + int ret; + + /* Do we need to assign a pkey for mm's execute-only maps? */ + if (execute_only_pkey == -1) { + /* Go allocate one to use, which might fail */ + execute_only_pkey = mm_pkey_alloc(mm); + if (execute_only_pkey < 0) + return -1; + need_to_set_mm_pkey = true; + } + + /* + * We do not want to go through the relatively costly dance to set AMR + * if we do not need to. Check it first and assume that if the + * execute-only pkey is readwrite-disabled than we do not have to set it + * ourselves. + */ + if (!need_to_set_mm_pkey && !pkey_allows_readwrite(execute_only_pkey)) + return execute_only_pkey; + + /* + * Set up AMR so that it denies access for everything other than + * execution. + */ + ret = __arch_set_user_pkey_access(current, execute_only_pkey, + PKEY_DISABLE_ACCESS | + PKEY_DISABLE_WRITE); + /* + * If the AMR-set operation failed somehow, just return 0 and + * effectively disable execute-only support. + */ + if (ret) { + mm_pkey_free(mm, execute_only_pkey); + return -1; + } + + /* We got one, store it and use it from here on out */ + if (need_to_set_mm_pkey) + mm->context.execute_only_pkey = execute_only_pkey; + return execute_only_pkey; +} -- cgit v1.2.3 From 013a91b39c2d5158cdc5529803f245bbb0526c7c Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:33 -0800 Subject: powerpc: ability to associate pkey to a vma arch-independent code expects the arch to map a pkey into the vma's protection bit setting. The patch provides that ability. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/mman.h | 7 ++++++- arch/powerpc/include/asm/pkeys.h | 11 +++++++++++ arch/powerpc/mm/pkeys.c | 8 ++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/mman.h b/arch/powerpc/include/asm/mman.h index 30922f699341..29994788b9f3 100644 --- a/arch/powerpc/include/asm/mman.h +++ b/arch/powerpc/include/asm/mman.h @@ -13,6 +13,7 @@ #include #include +#include #include /* @@ -22,7 +23,11 @@ static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, unsigned long pkey) { - return (prot & PROT_SAO) ? VM_SAO : 0; +#ifdef CONFIG_PPC_MEM_KEYS + return (((prot & PROT_SAO) ? VM_SAO : 0) | pkey_to_vmflag_bits(pkey)); +#else + return ((prot & PROT_SAO) ? VM_SAO : 0); +#endif } #define arch_calc_vm_prot_bits(prot, pkey) arch_calc_vm_prot_bits(prot, pkey) diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index b8d2a8e3f2e1..df372557ec33 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -30,6 +30,17 @@ extern u32 initial_allocation_mask; /* bits set for reserved keys */ #define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | \ VM_PKEY_BIT3 | VM_PKEY_BIT4) +/* Override any generic PKEY permission defines */ +#define PKEY_DISABLE_EXECUTE 0x4 +#define PKEY_ACCESS_MASK (PKEY_DISABLE_ACCESS | \ + PKEY_DISABLE_WRITE | \ + PKEY_DISABLE_EXECUTE) + +static inline u64 pkey_to_vmflag_bits(u16 pkey) +{ + return (((u64)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS); +} + #define arch_max_pkey() pkeys_total #define pkey_alloc_mask(pkey) (0x1 << pkey) diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index 6fece6e6fc97..a4aac75f1a93 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -34,6 +34,14 @@ int pkey_initialize(void) BUILD_BUG_ON(PKEY_DISABLE_EXECUTE & (PKEY_DISABLE_ACCESS | PKEY_DISABLE_WRITE)); + /* + * pkey_to_vmflag_bits() assumes that the pkey bits are contiguous + * in the vmaflag. Make sure that is really the case. + */ + BUILD_BUG_ON(__builtin_clzl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + + __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) + != (sizeof(u64) * BITS_PER_BYTE)); + /* * Disable the pkey system till everything is in place. A subsequent * patch will enable it. -- cgit v1.2.3 From 87bbabbed8a77092135f6442b8d5619906a81255 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:34 -0800 Subject: powerpc: implementation for arch_override_mprotect_pkey() arch independent code calls arch_override_mprotect_pkey() to return a pkey that best matches the requested protection. This patch provides the implementation. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/mmu_context.h | 5 +++++ arch/powerpc/include/asm/pkeys.h | 21 +++++++++++++++++++- arch/powerpc/mm/pkeys.c | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 4d69223d217a..3ba571dfdfd9 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -198,6 +198,11 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, #define thread_pkey_regs_save(thread) #define thread_pkey_regs_restore(new_thread, old_thread) #define thread_pkey_regs_init(thread) + +static inline int vma_pkey(struct vm_area_struct *vma) +{ + return 0; +} #endif /* CONFIG_PPC_MEM_KEYS */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index df372557ec33..24e82260052f 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -41,6 +41,13 @@ static inline u64 pkey_to_vmflag_bits(u16 pkey) return (((u64)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS); } +static inline int vma_pkey(struct vm_area_struct *vma) +{ + if (static_branch_likely(&pkey_disabled)) + return 0; + return (vma->vm_flags & ARCH_VM_PKEY_FLAGS) >> VM_PKEY_SHIFT; +} + #define arch_max_pkey() pkeys_total #define pkey_alloc_mask(pkey) (0x1 << pkey) @@ -137,10 +144,22 @@ static inline int execute_only_pkey(struct mm_struct *mm) return __execute_only_pkey(mm); } +extern int __arch_override_mprotect_pkey(struct vm_area_struct *vma, + int prot, int pkey); static inline int arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, int pkey) { - return 0; + if (static_branch_likely(&pkey_disabled)) + return 0; + + /* + * Is this an mprotect_pkey() call? If so, never override the value that + * came from the user. + */ + if (pkey != -1) + return pkey; + + return __arch_override_mprotect_pkey(vma, prot, pkey); } extern int __arch_set_user_pkey_access(struct task_struct *tsk, int pkey, diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index a4aac75f1a93..010c90c1f5da 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -322,3 +322,39 @@ int __execute_only_pkey(struct mm_struct *mm) mm->context.execute_only_pkey = execute_only_pkey; return execute_only_pkey; } + +static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma) +{ + /* Do this check first since the vm_flags should be hot */ + if ((vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC)) != VM_EXEC) + return false; + + return (vma_pkey(vma) == vma->vm_mm->context.execute_only_pkey); +} + +/* + * This should only be called for *plain* mprotect calls. + */ +int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, + int pkey) +{ + /* + * If the currently associated pkey is execute-only, but the requested + * protection requires read or write, move it back to the default pkey. + */ + if (vma_is_pkey_exec_only(vma) && (prot & (PROT_READ | PROT_WRITE))) + return 0; + + /* + * The requested protection is execute-only. Hence let's use an + * execute-only pkey. + */ + if (prot == PROT_EXEC) { + pkey = execute_only_pkey(vma->vm_mm); + if (pkey > 0) + return pkey; + } + + /* Nothing to override. */ + return vma_pkey(vma); +} -- cgit v1.2.3 From eb95d016ce3d3404e061c7c85d4362094dc34b3f Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:35 -0800 Subject: powerpc: map vma key-protection bits to pte key bits. Map the key protection bits of the vma to the pkey bits in the PTE. The PTE bits used for pkey are 3,4,5,6 and 57. The first four bits are the same four bits that were freed up initially in this patch series. remember? :-) Without those four bits this patch wouldn't be possible. BUT, on 4k kernel, bit 3, and 4 could not be freed up. remember? Hence we have to be satisfied with 5, 6 and 7. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/pgtable.h | 25 ++++++++++++++++++++++++- arch/powerpc/include/asm/mman.h | 6 ++++++ arch/powerpc/include/asm/pkeys.h | 12 ++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 60a0d7fd82bc..26285fc17baf 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -40,6 +40,7 @@ #define _RPAGE_RSV2 0x0800000000000000UL #define _RPAGE_RSV3 0x0400000000000000UL #define _RPAGE_RSV4 0x0200000000000000UL +#define _RPAGE_RSV5 0x00040UL #define _PAGE_PTE 0x4000000000000000UL /* distinguishes PTEs from pointers */ #define _PAGE_PRESENT 0x8000000000000000UL /* pte contains a translation */ @@ -59,6 +60,25 @@ /* Max physical address bit as per radix table */ #define _RPAGE_PA_MAX 57 +#ifdef CONFIG_PPC_MEM_KEYS +#ifdef CONFIG_PPC_64K_PAGES +#define H_PTE_PKEY_BIT0 _RPAGE_RSV1 +#define H_PTE_PKEY_BIT1 _RPAGE_RSV2 +#else /* CONFIG_PPC_64K_PAGES */ +#define H_PTE_PKEY_BIT0 0 /* _RPAGE_RSV1 is not available */ +#define H_PTE_PKEY_BIT1 0 /* _RPAGE_RSV2 is not available */ +#endif /* CONFIG_PPC_64K_PAGES */ +#define H_PTE_PKEY_BIT2 _RPAGE_RSV3 +#define H_PTE_PKEY_BIT3 _RPAGE_RSV4 +#define H_PTE_PKEY_BIT4 _RPAGE_RSV5 +#else /* CONFIG_PPC_MEM_KEYS */ +#define H_PTE_PKEY_BIT0 0 +#define H_PTE_PKEY_BIT1 0 +#define H_PTE_PKEY_BIT2 0 +#define H_PTE_PKEY_BIT3 0 +#define H_PTE_PKEY_BIT4 0 +#endif /* CONFIG_PPC_MEM_KEYS */ + /* * Max physical address bit we will use for now. * @@ -122,13 +142,16 @@ #define _PAGE_CHG_MASK (PTE_RPN_MASK | _PAGE_HPTEFLAGS | _PAGE_DIRTY | \ _PAGE_ACCESSED | _PAGE_SPECIAL | _PAGE_PTE | \ _PAGE_SOFT_DIRTY) + +#define H_PTE_PKEY (H_PTE_PKEY_BIT0 | H_PTE_PKEY_BIT1 | H_PTE_PKEY_BIT2 | \ + H_PTE_PKEY_BIT3 | H_PTE_PKEY_BIT4) /* * Mask of bits returned by pte_pgprot() */ #define PAGE_PROT_BITS (_PAGE_SAO | _PAGE_NON_IDEMPOTENT | _PAGE_TOLERANT | \ H_PAGE_4K_PFN | _PAGE_PRIVILEGED | _PAGE_ACCESSED | \ _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY | _PAGE_EXEC | \ - _PAGE_SOFT_DIRTY) + _PAGE_SOFT_DIRTY | H_PTE_PKEY) /* * We define 2 sets of base prot bits, one for basic pages (ie, * cacheable kernel and user pages) and one for non cacheable diff --git a/arch/powerpc/include/asm/mman.h b/arch/powerpc/include/asm/mman.h index 29994788b9f3..07e3f54de9e3 100644 --- a/arch/powerpc/include/asm/mman.h +++ b/arch/powerpc/include/asm/mman.h @@ -33,7 +33,13 @@ static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot, static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags) { +#ifdef CONFIG_PPC_MEM_KEYS + return (vm_flags & VM_SAO) ? + __pgprot(_PAGE_SAO | vmflag_to_pte_pkey_bits(vm_flags)) : + __pgprot(0 | vmflag_to_pte_pkey_bits(vm_flags)); +#else return (vm_flags & VM_SAO) ? __pgprot(_PAGE_SAO) : __pgprot(0); +#endif } #define arch_vm_get_page_prot(vm_flags) arch_vm_get_page_prot(vm_flags) diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 24e82260052f..03d8d5053126 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -41,6 +41,18 @@ static inline u64 pkey_to_vmflag_bits(u16 pkey) return (((u64)pkey << VM_PKEY_SHIFT) & ARCH_VM_PKEY_FLAGS); } +static inline u64 vmflag_to_pte_pkey_bits(u64 vm_flags) +{ + if (static_branch_likely(&pkey_disabled)) + return 0x0UL; + + return (((vm_flags & VM_PKEY_BIT0) ? H_PTE_PKEY_BIT4 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT1) ? H_PTE_PKEY_BIT3 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT2) ? H_PTE_PKEY_BIT2 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT3) ? H_PTE_PKEY_BIT1 : 0x0UL) | + ((vm_flags & VM_PKEY_BIT4) ? H_PTE_PKEY_BIT0 : 0x0UL)); +} + static inline int vma_pkey(struct vm_area_struct *vma) { if (static_branch_likely(&pkey_disabled)) -- cgit v1.2.3 From a6590ca55f1f49912e17e4811ccae9a51bda8859 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:36 -0800 Subject: powerpc: Program HPTE key protection bits Map the PTE protection key bits to the HPTE key protection bits, while creating HPTE entries. Acked-by: Balbir Singh Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/mmu-hash.h | 5 +++++ arch/powerpc/include/asm/mmu_context.h | 6 ++++++ arch/powerpc/include/asm/pkeys.h | 9 +++++++++ arch/powerpc/mm/hash_utils_64.c | 1 + 4 files changed, 21 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index e91e115a816f..50ed64fba4ae 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -90,6 +90,8 @@ #define HPTE_R_PP0 ASM_CONST(0x8000000000000000) #define HPTE_R_TS ASM_CONST(0x4000000000000000) #define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) +#define HPTE_R_KEY_BIT0 ASM_CONST(0x2000000000000000) +#define HPTE_R_KEY_BIT1 ASM_CONST(0x1000000000000000) #define HPTE_R_RPN_SHIFT 12 #define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) #define HPTE_R_RPN_3_0 ASM_CONST(0x01fffffffffff000) @@ -104,6 +106,9 @@ #define HPTE_R_C ASM_CONST(0x0000000000000080) #define HPTE_R_R ASM_CONST(0x0000000000000100) #define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00) +#define HPTE_R_KEY_BIT2 ASM_CONST(0x0000000000000800) +#define HPTE_R_KEY_BIT3 ASM_CONST(0x0000000000000400) +#define HPTE_R_KEY_BIT4 ASM_CONST(0x0000000000000200) #define HPTE_R_KEY (HPTE_R_KEY_LO | HPTE_R_KEY_HI) #define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000) diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 3ba571dfdfd9..209f12705fbb 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -203,6 +203,12 @@ static inline int vma_pkey(struct vm_area_struct *vma) { return 0; } + +static inline u64 pte_to_hpte_pkey_bits(u64 pteflags) +{ + return 0x0UL; +} + #endif /* CONFIG_PPC_MEM_KEYS */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 03d8d5053126..a8ef7fa5360e 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -62,6 +62,15 @@ static inline int vma_pkey(struct vm_area_struct *vma) #define arch_max_pkey() pkeys_total +static inline u64 pte_to_hpte_pkey_bits(u64 pteflags) +{ + return (((pteflags & H_PTE_PKEY_BIT0) ? HPTE_R_KEY_BIT0 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT1) ? HPTE_R_KEY_BIT1 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT2) ? HPTE_R_KEY_BIT2 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT3) ? HPTE_R_KEY_BIT3 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL)); +} + #define pkey_alloc_mask(pkey) (0x1 << pkey) #define mm_pkey_allocation_map(mm) (mm->context.pkey_allocation_map) diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 8cd31ea9b243..a78e24cf93ff 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -233,6 +233,7 @@ unsigned long htab_convert_pte_flags(unsigned long pteflags) */ rflags |= HPTE_R_M; + rflags |= pte_to_hpte_pkey_bits(pteflags); return rflags; } -- cgit v1.2.3 From f2407ef3ba225665ee24965f69bc84435fb590cf Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:37 -0800 Subject: powerpc: helper to validate key-access permissions of a pte helper function that checks if the read/write/execute is allowed on the pte. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/pgtable.h | 2 ++ arch/powerpc/include/asm/pkeys.h | 9 +++++++++ arch/powerpc/mm/pkeys.c | 28 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 26285fc17baf..449b797f8b7b 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -571,6 +571,8 @@ static inline int pte_present(pte_t pte) return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT)); } +extern bool arch_pte_access_permitted(u64 pte, bool write, bool execute); + #define pte_access_permitted pte_access_permitted static inline bool pte_access_permitted(pte_t pte, bool write) { diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index a8ef7fa5360e..2298771b066b 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -71,6 +71,15 @@ static inline u64 pte_to_hpte_pkey_bits(u64 pteflags) ((pteflags & H_PTE_PKEY_BIT4) ? HPTE_R_KEY_BIT4 : 0x0UL)); } +static inline u16 pte_to_pkey_bits(u64 pteflags) +{ + return (((pteflags & H_PTE_PKEY_BIT0) ? 0x10 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT1) ? 0x8 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT2) ? 0x4 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT3) ? 0x2 : 0x0UL) | + ((pteflags & H_PTE_PKEY_BIT4) ? 0x1 : 0x0UL)); +} + #define pkey_alloc_mask(pkey) (0x1 << pkey) #define mm_pkey_allocation_map(mm) (mm->context.pkey_allocation_map) diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index 010c90c1f5da..5071778fdd20 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -358,3 +358,31 @@ int __arch_override_mprotect_pkey(struct vm_area_struct *vma, int prot, /* Nothing to override. */ return vma_pkey(vma); } + +static bool pkey_access_permitted(int pkey, bool write, bool execute) +{ + int pkey_shift; + u64 amr; + + if (!pkey) + return true; + + if (!is_pkey_enabled(pkey)) + return true; + + pkey_shift = pkeyshift(pkey); + if (execute && !(read_iamr() & (IAMR_EX_BIT << pkey_shift))) + return true; + + amr = read_amr(); /* Delay reading amr until absolutely needed */ + return ((!write && !(amr & (AMR_RD_BIT << pkey_shift))) || + (write && !(amr & (AMR_WR_BIT << pkey_shift)))); +} + +bool arch_pte_access_permitted(u64 pte, bool write, bool execute) +{ + if (static_branch_likely(&pkey_disabled)) + return true; + + return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute); +} -- cgit v1.2.3 From bca7aacfe8be121ecefb3be609420220b0820c2c Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:38 -0800 Subject: powerpc: check key protection for user page access Make sure that the kernel does not access user pages without checking their key-protection. Signed-off-by: Ram Pai [mpe: Integrate with upstream version of pte_access_permitted()] Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/pgtable.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 449b797f8b7b..1c495068bcfa 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -571,7 +571,14 @@ static inline int pte_present(pte_t pte) return !!(pte_raw(pte) & cpu_to_be64(_PAGE_PRESENT)); } +#ifdef CONFIG_PPC_MEM_KEYS extern bool arch_pte_access_permitted(u64 pte, bool write, bool execute); +#else +static inline bool arch_pte_access_permitted(u64 pte, bool write, bool execute) +{ + return true; +} +#endif /* CONFIG_PPC_MEM_KEYS */ #define pte_access_permitted pte_access_permitted static inline bool pte_access_permitted(pte_t pte, bool write) @@ -593,7 +600,8 @@ static inline bool pte_access_permitted(pte_t pte, bool write) if ((pteval & clear_pte_bits) == clear_pte_bits) return false; - return true; + + return arch_pte_access_permitted(pte_val(pte), write, 0); } /* -- cgit v1.2.3 From 1137573acfe4c67cdd265bbfbd2d66ebe87d6325 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:39 -0800 Subject: powerpc: implementation for arch_vma_access_permitted() This patch provides the implementation for arch_vma_access_permitted(). Returns true if the requested access is allowed by pkey associated with the vma. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/mmu_context.h | 5 ++++- arch/powerpc/mm/pkeys.c | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 209f12705fbb..cd2bd739c0df 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -186,6 +186,10 @@ static inline void arch_bprm_mm_init(struct mm_struct *mm, { } +#ifdef CONFIG_PPC_MEM_KEYS +bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, + bool execute, bool foreign); +#else /* CONFIG_PPC_MEM_KEYS */ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, bool execute, bool foreign) { @@ -193,7 +197,6 @@ static inline bool arch_vma_access_permitted(struct vm_area_struct *vma, return true; } -#ifndef CONFIG_PPC_MEM_KEYS #define pkey_mm_init(mm) #define thread_pkey_regs_save(thread) #define thread_pkey_regs_restore(new_thread, old_thread) diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index 5071778fdd20..4a885cc165e5 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -386,3 +386,37 @@ bool arch_pte_access_permitted(u64 pte, bool write, bool execute) return pkey_access_permitted(pte_to_pkey_bits(pte), write, execute); } + +/* + * We only want to enforce protection keys on the current thread because we + * effectively have no access to AMR/IAMR for other threads or any way to tell + * which AMR/IAMR in a threaded process we could use. + * + * So do not enforce things if the VMA is not from the current mm, or if we are + * in a kernel thread. + */ +static inline bool vma_is_foreign(struct vm_area_struct *vma) +{ + if (!current->mm) + return true; + + /* if it is not our ->mm, it has to be foreign */ + if (current->mm != vma->vm_mm) + return true; + + return false; +} + +bool arch_vma_access_permitted(struct vm_area_struct *vma, bool write, + bool execute, bool foreign) +{ + if (static_branch_likely(&pkey_disabled)) + return true; + /* + * Do not enforce our key-permissions on a foreign vma. + */ + if (foreign || vma_is_foreign(vma)) + return true; + + return pkey_access_permitted(vma_pkey(vma), write, execute); +} -- cgit v1.2.3 From e6c2a4797e101a25eced94aa9e1fc42c30247aec Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:40 -0800 Subject: powerpc: Handle exceptions caused by pkey violation Handle Data and Instruction exceptions caused by memory protection-key. The CPU will detect the key fault if the HPTE is already programmed with the key. However if the HPTE is not hashed, a key fault will not be detected by the hardware. The software will detect pkey violation in such a case. Signed-off-by: Ram Pai Signed-off-by: Thiago Jung Bauermann Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/reg.h | 1 - arch/powerpc/kernel/exceptions-64s.S | 2 +- arch/powerpc/mm/fault.c | 22 ++++++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index b779f3ccd412..ffc9990e8040 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -312,7 +312,6 @@ DSISR_BAD_EXT_CTRL) #define DSISR_BAD_FAULT_64S (DSISR_BAD_FAULT_32S | \ DSISR_ATTR_CONFLICT | \ - DSISR_KEYFAULT | \ DSISR_UNSUPP_MMU | \ DSISR_PRTABLE_FAULT | \ DSISR_ICSWX_NO_CT | \ diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 1a4f51f35870..42a47d83282c 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1527,7 +1527,7 @@ USE_TEXT_SECTION() .balign IFETCH_ALIGN_BYTES do_hash_page: #ifdef CONFIG_PPC_BOOK3S_64 - lis r0,(DSISR_BAD_FAULT_64S|DSISR_DABRMATCH)@h + lis r0,(DSISR_BAD_FAULT_64S | DSISR_DABRMATCH | DSISR_KEYFAULT)@h ori r0,r0,DSISR_BAD_FAULT_64S@l and. r0,r4,r0 /* weird error? */ bne- handle_page_fault /* if not, try to insert a HPTE */ diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index c07896c19517..6417a659b02d 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -427,6 +427,11 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); + if (error_code & DSISR_KEYFAULT) { + _exception(SIGSEGV, regs, SEGV_PKUERR, address); + return 0; + } + /* * We want to do this outside mmap_sem, because reading code around nip * can result in fault, which will cause a deadlock when called with @@ -498,6 +503,23 @@ good_area: * the fault. */ fault = handle_mm_fault(vma, address, flags); + +#ifdef CONFIG_PPC_MEM_KEYS + /* + * if the HPTE is not hashed, hardware will not detect + * a key fault. Lets check if we failed because of a + * software detected key fault. + */ + if (unlikely(fault & VM_FAULT_SIGSEGV) && + !arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE, + is_exec, 0)) { + int pkey = vma_pkey(vma); + + if (likely(pkey)) + return __bad_area(regs, address, SEGV_PKUERR); + } +#endif /* CONFIG_PPC_MEM_KEYS */ + major |= fault & VM_FAULT_MAJOR; /* -- cgit v1.2.3 From 087003e9ef7c1c5bec932387e47511429eff2a54 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:41 -0800 Subject: powerpc: introduce get_mm_addr_key() helper get_mm_addr_key() helper returns the pkey associated with an address corresponding to a given mm_struct. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/mmu.h | 9 +++++++++ arch/powerpc/mm/hash_utils_64.c | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 6364f5c2cc3e..bb38312cff28 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -260,6 +260,15 @@ static inline bool early_radix_enabled(void) } #endif +#ifdef CONFIG_PPC_MEM_KEYS +extern u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address); +#else +static inline u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address) +{ + return 0; +} +#endif /* CONFIG_PPC_MEM_KEYS */ + #endif /* !__ASSEMBLY__ */ /* The kernel use the constants below to index in the page sizes array. diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index a78e24cf93ff..462c34e7b01d 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -1581,6 +1581,30 @@ out_exit: local_irq_restore(flags); } +#ifdef CONFIG_PPC_MEM_KEYS +/* + * Return the protection key associated with the given address and the + * mm_struct. + */ +u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address) +{ + pte_t *ptep; + u16 pkey = 0; + unsigned long flags; + + if (!mm || !mm->pgd) + return 0; + + local_irq_save(flags); + ptep = find_linux_pte(mm->pgd, address, NULL, NULL); + if (ptep) + pkey = pte_to_pkey_bits(pte_val(READ_ONCE(*ptep))); + local_irq_restore(flags); + + return pkey; +} +#endif /* CONFIG_PPC_MEM_KEYS */ + #ifdef CONFIG_PPC_TRANSACTIONAL_MEM static inline void tm_flush_hash_page(int local) { -- cgit v1.2.3 From 99cd1302327a2ccaedf905e9f6a8d8fd234bd485 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:42 -0800 Subject: powerpc: Deliver SEGV signal on pkey violation The value of the pkey, whose protection got violated, is made available in si_pkey field of the siginfo structure. Signed-off-by: Ram Pai Signed-off-by: Thiago Jung Bauermann Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/bug.h | 1 + arch/powerpc/kernel/traps.c | 12 +++++++++++- arch/powerpc/mm/fault.c | 39 +++++++++++++++++++++++++++------------ 3 files changed, 39 insertions(+), 13 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h index 3c04249bcf39..97c38472b924 100644 --- a/arch/powerpc/include/asm/bug.h +++ b/arch/powerpc/include/asm/bug.h @@ -133,6 +133,7 @@ struct pt_regs; extern int do_page_fault(struct pt_regs *, unsigned long, unsigned long); extern void bad_page_fault(struct pt_regs *, unsigned long, int); extern void _exception(int, struct pt_regs *, int, unsigned long); +extern void _exception_pkey(int, struct pt_regs *, int, unsigned long, int); extern void die(const char *, struct pt_regs *, long); extern bool die_will_crash(void); diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index d139117d3339..4b1a8e2ec023 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -266,7 +267,9 @@ void user_single_step_siginfo(struct task_struct *tsk, info->si_addr = (void __user *)regs->nip; } -void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr) + +void _exception_pkey(int signr, struct pt_regs *regs, int code, + unsigned long addr, int key) { siginfo_t info; const char fmt32[] = KERN_INFO "%s[%d]: unhandled signal %d " \ @@ -293,9 +296,16 @@ void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr) info.si_signo = signr; info.si_code = code; info.si_addr = (void __user *) addr; + info.si_pkey = key; + force_sig_info(signr, &info, current); } +void _exception(int signr, struct pt_regs *regs, int code, unsigned long addr) +{ + _exception_pkey(signr, regs, code, addr, 0); +} + void system_reset_exception(struct pt_regs *regs) { /* diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 6417a659b02d..d3a6e0395eaa 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -107,7 +107,8 @@ static bool store_updates_sp(struct pt_regs *regs) */ static int -__bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code) +__bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code, + int pkey) { /* * If we are in kernel mode, bail out with a SEGV, this will @@ -117,17 +118,18 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long address, int si_code) if (!user_mode(regs)) return SIGSEGV; - _exception(SIGSEGV, regs, si_code, address); + _exception_pkey(SIGSEGV, regs, si_code, address, pkey); return 0; } static noinline int bad_area_nosemaphore(struct pt_regs *regs, unsigned long address) { - return __bad_area_nosemaphore(regs, address, SEGV_MAPERR); + return __bad_area_nosemaphore(regs, address, SEGV_MAPERR, 0); } -static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code) +static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code, + int pkey) { struct mm_struct *mm = current->mm; @@ -137,12 +139,18 @@ static int __bad_area(struct pt_regs *regs, unsigned long address, int si_code) */ up_read(&mm->mmap_sem); - return __bad_area_nosemaphore(regs, address, si_code); + return __bad_area_nosemaphore(regs, address, si_code, pkey); } static noinline int bad_area(struct pt_regs *regs, unsigned long address) { - return __bad_area(regs, address, SEGV_MAPERR); + return __bad_area(regs, address, SEGV_MAPERR, 0); +} + +static int bad_key_fault_exception(struct pt_regs *regs, unsigned long address, + int pkey) +{ + return __bad_area_nosemaphore(regs, address, SEGV_PKUERR, pkey); } static int do_sigbus(struct pt_regs *regs, unsigned long address, @@ -427,10 +435,9 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); - if (error_code & DSISR_KEYFAULT) { - _exception(SIGSEGV, regs, SEGV_PKUERR, address); - return 0; - } + if (error_code & DSISR_KEYFAULT) + return bad_key_fault_exception(regs, address, + get_mm_addr_key(mm, address)); /* * We want to do this outside mmap_sem, because reading code around nip @@ -513,10 +520,18 @@ good_area: if (unlikely(fault & VM_FAULT_SIGSEGV) && !arch_vma_access_permitted(vma, flags & FAULT_FLAG_WRITE, is_exec, 0)) { + /* + * The PGD-PDT...PMD-PTE tree may not have been fully setup. + * Hence we cannot walk the tree to locate the PTE, to locate + * the key. Hence let's use vma_pkey() to get the key; instead + * of get_mm_addr_key(). + */ int pkey = vma_pkey(vma); - if (likely(pkey)) - return __bad_area(regs, address, SEGV_PKUERR); + if (likely(pkey)) { + up_read(&mm->mmap_sem); + return bad_key_fault_exception(regs, address, pkey); + } } #endif /* CONFIG_PPC_MEM_KEYS */ -- cgit v1.2.3 From c5cc1f4df6b16646f8fae7aab523c1820bf916e8 Mon Sep 17 00:00:00 2001 From: Thiago Jung Bauermann Date: Thu, 18 Jan 2018 17:50:43 -0800 Subject: powerpc/ptrace: Add memory protection key regset The AMR/IAMR/UAMOR are part of the program context. Allow it to be accessed via ptrace and through core files. Signed-off-by: Ram Pai Signed-off-by: Thiago Jung Bauermann Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pkeys.h | 5 +++ arch/powerpc/include/uapi/asm/elf.h | 1 + arch/powerpc/kernel/ptrace.c | 66 +++++++++++++++++++++++++++++++++++++ arch/powerpc/kernel/traps.c | 7 ++++ include/uapi/linux/elf.h | 1 + 5 files changed, 80 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 2298771b066b..c3cbad824e5a 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -202,6 +202,11 @@ static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, return __arch_set_user_pkey_access(tsk, pkey, init_val); } +static inline bool arch_pkeys_enabled(void) +{ + return !static_branch_likely(&pkey_disabled); +} + extern void pkey_mm_init(struct mm_struct *mm); extern void thread_pkey_regs_save(struct thread_struct *thread); extern void thread_pkey_regs_restore(struct thread_struct *new_thread, diff --git a/arch/powerpc/include/uapi/asm/elf.h b/arch/powerpc/include/uapi/asm/elf.h index 5f201d40bcca..860c59291bfc 100644 --- a/arch/powerpc/include/uapi/asm/elf.h +++ b/arch/powerpc/include/uapi/asm/elf.h @@ -97,6 +97,7 @@ #define ELF_NTMSPRREG 3 /* include tfhar, tfiar, texasr */ #define ELF_NEBB 3 /* includes ebbrr, ebbhr, bescr */ #define ELF_NPMU 5 /* includes siar, sdar, sier, mmcr2, mmcr0 */ +#define ELF_NPKEY 3 /* includes amr, iamr, uamor */ typedef unsigned long elf_greg_t64; typedef elf_greg_t64 elf_gregset_t64[ELF_NGREG]; diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c index aef08e579946..ca72d7391d40 100644 --- a/arch/powerpc/kernel/ptrace.c +++ b/arch/powerpc/kernel/ptrace.c @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -1787,6 +1788,61 @@ static int pmu_set(struct task_struct *target, return ret; } #endif + +#ifdef CONFIG_PPC_MEM_KEYS +static int pkey_active(struct task_struct *target, + const struct user_regset *regset) +{ + if (!arch_pkeys_enabled()) + return -ENODEV; + + return regset->n; +} + +static int pkey_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + BUILD_BUG_ON(TSO(amr) + sizeof(unsigned long) != TSO(iamr)); + BUILD_BUG_ON(TSO(iamr) + sizeof(unsigned long) != TSO(uamor)); + + if (!arch_pkeys_enabled()) + return -ENODEV; + + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, + &target->thread.amr, 0, + ELF_NPKEY * sizeof(unsigned long)); +} + +static int pkey_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + u64 new_amr; + int ret; + + if (!arch_pkeys_enabled()) + return -ENODEV; + + /* Only the AMR can be set from userspace */ + if (pos != 0 || count != sizeof(new_amr)) + return -EINVAL; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &new_amr, 0, sizeof(new_amr)); + if (ret) + return ret; + + /* UAMOR determines which bits of the AMR can be set from userspace. */ + target->thread.amr = (new_amr & target->thread.uamor) | + (target->thread.amr & ~target->thread.uamor); + + return 0; +} +#endif /* CONFIG_PPC_MEM_KEYS */ + /* * These are our native regset flavors. */ @@ -1821,6 +1877,9 @@ enum powerpc_regset { REGSET_EBB, /* EBB registers */ REGSET_PMR, /* Performance Monitor Registers */ #endif +#ifdef CONFIG_PPC_MEM_KEYS + REGSET_PKEY, /* AMR register */ +#endif }; static const struct user_regset native_regsets[] = { @@ -1926,6 +1985,13 @@ static const struct user_regset native_regsets[] = { .active = pmu_active, .get = pmu_get, .set = pmu_set }, #endif +#ifdef CONFIG_PPC_MEM_KEYS + [REGSET_PKEY] = { + .core_note_type = NT_PPC_PKEY, .n = ELF_NPKEY, + .size = sizeof(u64), .align = sizeof(u64), + .active = pkey_active, .get = pkey_get, .set = pkey_set + }, +#endif }; static const struct user_regset_view user_ppc_native_view = { diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 4b1a8e2ec023..122a3c883f4e 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -292,6 +292,13 @@ void _exception_pkey(int signr, struct pt_regs *regs, int code, local_irq_enable(); current->thread.trap_nr = code; + + /* + * Save all the pkey registers AMR/IAMR/UAMOR. Eg: Core dumps need + * to capture the content, if the task gets killed. + */ + thread_pkey_regs_save(¤t->thread); + memset(&info, 0, sizeof(info)); info.si_signo = signr; info.si_code = code; diff --git a/include/uapi/linux/elf.h b/include/uapi/linux/elf.h index bb6836986200..3bf73fb58045 100644 --- a/include/uapi/linux/elf.h +++ b/include/uapi/linux/elf.h @@ -396,6 +396,7 @@ typedef struct elf64_shdr { #define NT_PPC_TM_CTAR 0x10d /* TM checkpointed Target Address Register */ #define NT_PPC_TM_CPPR 0x10e /* TM checkpointed Program Priority Register */ #define NT_PPC_TM_CDSCR 0x10f /* TM checkpointed Data Stream Control Register */ +#define NT_PPC_PKEY 0x110 /* Memory Protection Keys registers */ #define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */ #define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */ #define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */ -- cgit v1.2.3 From cf43d3b26452a332d7e1d64a00079c607613e944 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:44 -0800 Subject: powerpc: Enable pkey subsystem PAPR defines 'ibm,processor-storage-keys' property. It exports two values. The first value holds the number of data-access keys and the second holds the number of instruction-access keys. Due to a bug in the firmware, instruction-access keys is always reported as zero. However any key can be configured to disable data-access and/or disable execution-access. The inavailablity of the second value is not a big handicap, though it could have been used to determine if the platform supported disable-execution-access. Non-PAPR platforms do not define this property in the device tree yet. Fortunately power8 is the only released Non-PAPR platform that is supported. Here, we hardcode the number of supported pkey to 32, by consulting the PowerISA3.0 This patch calculates the number of keys supported by the platform. Also it determines the platform support for read/write/execution access support for pkeys. Signed-off-by: Ram Pai [mpe: Use a PVR check instead of CPU_FTR for execute. Restrict to Power7/8/9 for now until older CPUs are tested.] Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/cputable.h | 9 +++--- arch/powerpc/include/asm/pkeys.h | 3 ++ arch/powerpc/mm/pkeys.c | 64 +++++++++++++++++++++++++++++++------ 3 files changed, 63 insertions(+), 13 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index 78ca2a721d04..a2c5c95882cf 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -195,7 +195,7 @@ static inline void cpu_feature_keys_init(void) { } #define CPU_FTR_STCX_CHECKS_ADDRESS LONG_ASM_CONST(0x0004000000000000) #define CPU_FTR_POPCNTB LONG_ASM_CONST(0x0008000000000000) #define CPU_FTR_POPCNTD LONG_ASM_CONST(0x0010000000000000) -/* Free LONG_ASM_CONST(0x0020000000000000) */ +#define CPU_FTR_PKEY LONG_ASM_CONST(0x0020000000000000) #define CPU_FTR_VMX_COPY LONG_ASM_CONST(0x0040000000000000) #define CPU_FTR_TM LONG_ASM_CONST(0x0080000000000000) #define CPU_FTR_CFAR LONG_ASM_CONST(0x0100000000000000) @@ -442,7 +442,7 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_DSCR | CPU_FTR_SAO | CPU_FTR_ASYM_SMT | \ CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ CPU_FTR_CFAR | CPU_FTR_HVMODE | \ - CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX) + CPU_FTR_VMX_COPY | CPU_FTR_HAS_PPR | CPU_FTR_DABRX | CPU_FTR_PKEY) #define CPU_FTRS_POWER8 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\ CPU_FTR_MMCRA | CPU_FTR_SMT | \ @@ -452,7 +452,7 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \ - CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP) + CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_PKEY) #define CPU_FTRS_POWER8E (CPU_FTRS_POWER8 | CPU_FTR_PMAO_BUG) #define CPU_FTRS_POWER8_DD1 (CPU_FTRS_POWER8 & ~CPU_FTR_DBELL) #define CPU_FTRS_POWER9 (CPU_FTR_USE_TB | CPU_FTR_LWSYNC | \ @@ -464,7 +464,8 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \ - CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300) + CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \ + CPU_FTR_PKEY) #define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \ (~CPU_FTR_SAO)) #define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9 diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index c3cbad824e5a..0409c80c32c0 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -9,6 +9,7 @@ #define _ASM_POWERPC_KEYS_H #include +#include DECLARE_STATIC_KEY_TRUE(pkey_disabled); extern int pkeys_total; /* total pkeys as per device tree */ @@ -208,6 +209,8 @@ static inline bool arch_pkeys_enabled(void) } extern void pkey_mm_init(struct mm_struct *mm); +extern bool arch_supports_pkeys(int cap); +extern unsigned int arch_usable_pkeys(void); extern void thread_pkey_regs_save(struct thread_struct *thread); extern void thread_pkey_regs_restore(struct thread_struct *new_thread, struct thread_struct *old_thread); diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index 4a885cc165e5..ba71c5481f42 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -6,11 +6,14 @@ */ #include +#include #include +#include DEFINE_STATIC_KEY_TRUE(pkey_disabled); bool pkey_execute_disable_supported; int pkeys_total; /* Total pkeys as per device tree */ +bool pkeys_devtree_defined; /* pkey property exported by device tree */ u32 initial_allocation_mask; /* Bits set for reserved keys */ u64 pkey_amr_uamor_mask; /* Bits in AMR/UMOR not to be touched */ u64 pkey_iamr_mask; /* Bits in AMR not to be touched */ @@ -22,6 +25,35 @@ u64 pkey_iamr_mask; /* Bits in AMR not to be touched */ #define PKEY_REG_BITS (sizeof(u64)*8) #define pkeyshift(pkey) (PKEY_REG_BITS - ((pkey+1) * AMR_BITS_PER_PKEY)) +static void scan_pkey_feature(void) +{ + u32 vals[2]; + struct device_node *cpu; + + cpu = of_find_node_by_type(NULL, "cpu"); + if (!cpu) + return; + + if (of_property_read_u32_array(cpu, + "ibm,processor-storage-keys", vals, 2)) + return; + + /* + * Since any pkey can be used for data or execute, we will just treat + * all keys as equal and track them as one entity. + */ + pkeys_total = be32_to_cpu(vals[0]); + pkeys_devtree_defined = true; +} + +static inline bool pkey_mmu_enabled(void) +{ + if (firmware_has_feature(FW_FEATURE_LPAR)) + return pkeys_total; + else + return cpu_has_feature(CPU_FTR_PKEY); +} + int pkey_initialize(void) { int os_reserved, i; @@ -42,14 +74,17 @@ int pkey_initialize(void) __builtin_popcountl(ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT) != (sizeof(u64) * BITS_PER_BYTE)); + /* scan the device tree for pkey feature */ + scan_pkey_feature(); + /* - * Disable the pkey system till everything is in place. A subsequent - * patch will enable it. + * Let's assume 32 pkeys on P8 bare metal, if its not defined by device + * tree. We make this exception since skiboot forgot to expose this + * property on power8. */ - static_branch_enable(&pkey_disabled); - - /* Lets assume 32 keys */ - pkeys_total = 32; + if (!pkeys_devtree_defined && !firmware_has_feature(FW_FEATURE_LPAR) && + cpu_has_feature(CPU_FTRS_POWER8)) + pkeys_total = 32; /* * Adjust the upper limit, based on the number of bits supported by @@ -58,11 +93,22 @@ int pkey_initialize(void) pkeys_total = min_t(int, pkeys_total, (ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)); + if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total) + static_branch_enable(&pkey_disabled); + else + static_branch_disable(&pkey_disabled); + + if (static_branch_likely(&pkey_disabled)) + return 0; + /* - * Disable execute_disable support for now. A subsequent patch will - * enable it. + * The device tree cannot be relied to indicate support for + * execute_disable support. Instead we use a PVR check. */ - pkey_execute_disable_supported = false; + if (pvr_version_is(PVR_POWER7) || pvr_version_is(PVR_POWER7p)) + pkey_execute_disable_supported = false; + else + pkey_execute_disable_supported = true; #ifdef CONFIG_PPC_4K_PAGES /* -- cgit v1.2.3 From 9499ec1b5e82321829e1c1510bcc37edc20b6f38 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:45 -0800 Subject: powerpc: sys_pkey_alloc() and sys_pkey_free() system calls Finally this patch provides the ability for a process to allocate and free a protection key. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/systbl.h | 2 ++ arch/powerpc/include/asm/unistd.h | 4 +--- arch/powerpc/include/uapi/asm/unistd.h | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h index 449912f057f6..dea4a952ec53 100644 --- a/arch/powerpc/include/asm/systbl.h +++ b/arch/powerpc/include/asm/systbl.h @@ -389,3 +389,5 @@ COMPAT_SYS_SPU(preadv2) COMPAT_SYS_SPU(pwritev2) SYSCALL(kexec_file_load) SYSCALL(statx) +SYSCALL(pkey_alloc) +SYSCALL(pkey_free) diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h index 9ba11dbcaca9..e0273bc5b095 100644 --- a/arch/powerpc/include/asm/unistd.h +++ b/arch/powerpc/include/asm/unistd.h @@ -12,13 +12,11 @@ #include -#define NR_syscalls 384 +#define NR_syscalls 386 #define __NR__exit __NR_exit #define __IGNORE_pkey_mprotect -#define __IGNORE_pkey_alloc -#define __IGNORE_pkey_free #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/uapi/asm/unistd.h b/arch/powerpc/include/uapi/asm/unistd.h index df8684f31919..5db438516e61 100644 --- a/arch/powerpc/include/uapi/asm/unistd.h +++ b/arch/powerpc/include/uapi/asm/unistd.h @@ -395,5 +395,7 @@ #define __NR_pwritev2 381 #define __NR_kexec_file_load 382 #define __NR_statx 383 +#define __NR_pkey_alloc 384 +#define __NR_pkey_free 385 #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */ -- cgit v1.2.3 From 3350eb2ea127978319ced883523d828046af4045 Mon Sep 17 00:00:00 2001 From: Ram Pai Date: Thu, 18 Jan 2018 17:50:46 -0800 Subject: powerpc: sys_pkey_mprotect() system call Patch provides the ability for a process to associate a pkey with a address range. Signed-off-by: Ram Pai Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/systbl.h | 1 + arch/powerpc/include/asm/unistd.h | 4 +--- arch/powerpc/include/uapi/asm/unistd.h | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/systbl.h b/arch/powerpc/include/asm/systbl.h index dea4a952ec53..d61f9c96d916 100644 --- a/arch/powerpc/include/asm/systbl.h +++ b/arch/powerpc/include/asm/systbl.h @@ -391,3 +391,4 @@ SYSCALL(kexec_file_load) SYSCALL(statx) SYSCALL(pkey_alloc) SYSCALL(pkey_free) +SYSCALL(pkey_mprotect) diff --git a/arch/powerpc/include/asm/unistd.h b/arch/powerpc/include/asm/unistd.h index e0273bc5b095..daf1ba97a00c 100644 --- a/arch/powerpc/include/asm/unistd.h +++ b/arch/powerpc/include/asm/unistd.h @@ -12,12 +12,10 @@ #include -#define NR_syscalls 386 +#define NR_syscalls 387 #define __NR__exit __NR_exit -#define __IGNORE_pkey_mprotect - #ifndef __ASSEMBLY__ #include diff --git a/arch/powerpc/include/uapi/asm/unistd.h b/arch/powerpc/include/uapi/asm/unistd.h index 5db438516e61..389c36fd8299 100644 --- a/arch/powerpc/include/uapi/asm/unistd.h +++ b/arch/powerpc/include/uapi/asm/unistd.h @@ -397,5 +397,6 @@ #define __NR_statx 383 #define __NR_pkey_alloc 384 #define __NR_pkey_free 385 +#define __NR_pkey_mprotect 386 #endif /* _UAPI_ASM_POWERPC_UNISTD_H_ */ -- cgit v1.2.3 From 11ed8c5569b149a065184dc8ce22414aac2f20e9 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Tue, 16 Jan 2018 18:01:50 +0100 Subject: powerpc/mpic_timer: avoid struct timeval In an effort to remove all instances of 'struct timeval' from the kernel, I'm changing the powerpc mpic_timer interface to use plain seconds instead. There is only one user of this interface, and that doesn't use the microseconds portion, so the code gets noticeably simpler in the process. Signed-off-by: Arnd Bergmann Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/mpic_timer.h | 8 ++--- arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c | 16 ++++----- arch/powerpc/sysdev/mpic_timer.c | 55 ++++++----------------------- 3 files changed, 21 insertions(+), 58 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/mpic_timer.h b/arch/powerpc/include/asm/mpic_timer.h index 0e23cd4ac8aa..13e6702ec458 100644 --- a/arch/powerpc/include/asm/mpic_timer.h +++ b/arch/powerpc/include/asm/mpic_timer.h @@ -29,17 +29,17 @@ struct mpic_timer { #ifdef CONFIG_MPIC_TIMER struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev, - const struct timeval *time); + time64_t time); void mpic_start_timer(struct mpic_timer *handle); void mpic_stop_timer(struct mpic_timer *handle); -void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time); +void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time); void mpic_free_timer(struct mpic_timer *handle); #else struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev, - const struct timeval *time) { return NULL; } + time64_t time) { return NULL; } void mpic_start_timer(struct mpic_timer *handle) { } void mpic_stop_timer(struct mpic_timer *handle) { } -void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time) { } +void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time) { } void mpic_free_timer(struct mpic_timer *handle) { } #endif diff --git a/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c b/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c index 1707bf04dec6..94278e8af192 100644 --- a/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c +++ b/arch/powerpc/sysdev/fsl_mpic_timer_wakeup.c @@ -56,17 +56,16 @@ static ssize_t fsl_timer_wakeup_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct timeval interval; - int val = 0; + time64_t interval = 0; mutex_lock(&sysfs_lock); if (fsl_wakeup->timer) { mpic_get_remain_time(fsl_wakeup->timer, &interval); - val = interval.tv_sec + 1; + interval++; } mutex_unlock(&sysfs_lock); - return sprintf(buf, "%d\n", val); + return sprintf(buf, "%lld\n", interval); } static ssize_t fsl_timer_wakeup_store(struct device *dev, @@ -74,11 +73,10 @@ static ssize_t fsl_timer_wakeup_store(struct device *dev, const char *buf, size_t count) { - struct timeval interval; + time64_t interval; int ret; - interval.tv_usec = 0; - if (kstrtol(buf, 0, &interval.tv_sec)) + if (kstrtoll(buf, 0, &interval)) return -EINVAL; mutex_lock(&sysfs_lock); @@ -89,13 +87,13 @@ static ssize_t fsl_timer_wakeup_store(struct device *dev, fsl_wakeup->timer = NULL; } - if (!interval.tv_sec) { + if (!interval) { mutex_unlock(&sysfs_lock); return count; } fsl_wakeup->timer = mpic_request_timer(fsl_mpic_timer_irq, - fsl_wakeup, &interval); + fsl_wakeup, interval); if (!fsl_wakeup->timer) { mutex_unlock(&sysfs_lock); return -EINVAL; diff --git a/arch/powerpc/sysdev/mpic_timer.c b/arch/powerpc/sysdev/mpic_timer.c index a418579591be..87e7c42777a8 100644 --- a/arch/powerpc/sysdev/mpic_timer.c +++ b/arch/powerpc/sysdev/mpic_timer.c @@ -47,9 +47,6 @@ #define MAX_TICKS_CASCADE (~0U) #define TIMER_OFFSET(num) (1 << (TIMERS_PER_GROUP - 1 - num)) -/* tv_usec should be less than ONE_SECOND, otherwise use tv_sec */ -#define ONE_SECOND 1000000 - struct timer_regs { u32 gtccr; u32 res0[3]; @@ -90,51 +87,23 @@ static struct cascade_priv cascade_timer[] = { static LIST_HEAD(timer_group_list); static void convert_ticks_to_time(struct timer_group_priv *priv, - const u64 ticks, struct timeval *time) + const u64 ticks, time64_t *time) { - u64 tmp_sec; - - time->tv_sec = (__kernel_time_t)div_u64(ticks, priv->timerfreq); - tmp_sec = (u64)time->tv_sec * (u64)priv->timerfreq; - - time->tv_usec = 0; - - if (tmp_sec <= ticks) - time->tv_usec = (__kernel_suseconds_t) - div_u64((ticks - tmp_sec) * 1000000, priv->timerfreq); - - return; + *time = (u64)div_u64(ticks, priv->timerfreq); } /* the time set by the user is converted to "ticks" */ static int convert_time_to_ticks(struct timer_group_priv *priv, - const struct timeval *time, u64 *ticks) + time64_t time, u64 *ticks) { u64 max_value; /* prevent u64 overflow */ - u64 tmp = 0; - - u64 tmp_sec; - u64 tmp_ms; - u64 tmp_us; max_value = div_u64(ULLONG_MAX, priv->timerfreq); - if (time->tv_sec > max_value || - (time->tv_sec == max_value && time->tv_usec > 0)) + if (time > max_value) return -EINVAL; - tmp_sec = (u64)time->tv_sec * (u64)priv->timerfreq; - tmp += tmp_sec; - - tmp_ms = time->tv_usec / 1000; - tmp_ms = div_u64((u64)tmp_ms * (u64)priv->timerfreq, 1000); - tmp += tmp_ms; - - tmp_us = time->tv_usec % 1000; - tmp_us = div_u64((u64)tmp_us * (u64)priv->timerfreq, 1000000); - tmp += tmp_us; - - *ticks = tmp; + *ticks = (u64)time * (u64)priv->timerfreq; return 0; } @@ -223,7 +192,7 @@ static struct mpic_timer *get_cascade_timer(struct timer_group_priv *priv, return allocated_timer; } -static struct mpic_timer *get_timer(const struct timeval *time) +static struct mpic_timer *get_timer(time64_t time) { struct timer_group_priv *priv; struct mpic_timer *timer; @@ -277,7 +246,7 @@ static struct mpic_timer *get_timer(const struct timeval *time) * @handle: the timer to be started. * * It will do ->fn(->dev) callback from the hardware interrupt at - * the ->timeval point in the future. + * the 'time64_t' point in the future. */ void mpic_start_timer(struct mpic_timer *handle) { @@ -319,7 +288,7 @@ EXPORT_SYMBOL(mpic_stop_timer); * * Query timer remaining time. */ -void mpic_get_remain_time(struct mpic_timer *handle, struct timeval *time) +void mpic_get_remain_time(struct mpic_timer *handle, time64_t *time) { struct timer_group_priv *priv = container_of(handle, struct timer_group_priv, timer[handle->num]); @@ -391,7 +360,7 @@ EXPORT_SYMBOL(mpic_free_timer); * else "handle" on success. */ struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev, - const struct timeval *time) + time64_t time) { struct mpic_timer *allocated_timer; int ret; @@ -399,11 +368,7 @@ struct mpic_timer *mpic_request_timer(irq_handler_t fn, void *dev, if (list_empty(&timer_group_list)) return NULL; - if (!(time->tv_sec + time->tv_usec) || - time->tv_sec < 0 || time->tv_usec < 0) - return NULL; - - if (time->tv_usec > ONE_SECOND) + if (time < 0) return NULL; allocated_timer = get_timer(time); -- cgit v1.2.3 From 8183d99f4a22c2abbc543847a588df3666ef0c0c Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Fri, 24 Nov 2017 08:31:09 +0100 Subject: powerpc/lib/feature-fixups: use raw_patch_instruction() feature fixups need to use patch_instruction() early in the boot, even before the code is relocated to its final address, requiring patch_instruction() to use PTRRELOC() in order to address data. But feature fixups applies on code before it is set to read only, even for modules. Therefore, feature fixups can use raw_patch_instruction() instead. Signed-off-by: Christophe Leroy Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/code-patching.h | 1 + arch/powerpc/lib/code-patching.c | 4 ++-- arch/powerpc/lib/feature-fixups.c | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/code-patching.h b/arch/powerpc/include/asm/code-patching.h index 2c895e8d07f7..812535f40124 100644 --- a/arch/powerpc/include/asm/code-patching.h +++ b/arch/powerpc/include/asm/code-patching.h @@ -31,6 +31,7 @@ unsigned int create_cond_branch(const unsigned int *addr, unsigned long target, int flags); int patch_branch(unsigned int *addr, unsigned long target, int flags); int patch_instruction(unsigned int *addr, unsigned int instr); +int raw_patch_instruction(unsigned int *addr, unsigned int instr); int instr_is_relative_branch(unsigned int instr); int instr_is_relative_link_branch(unsigned int instr); diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index e1c58937281f..e0d881ab304e 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -38,7 +38,7 @@ static int __patch_instruction(unsigned int *exec_addr, unsigned int instr, return 0; } -static int raw_patch_instruction(unsigned int *addr, unsigned int instr) +int raw_patch_instruction(unsigned int *addr, unsigned int instr) { return __patch_instruction(addr, instr, addr); } @@ -155,7 +155,7 @@ int patch_instruction(unsigned int *addr, unsigned int instr) * when text_poke_area is not ready, but we still need * to allow patching. We just do the plain old patching */ - if (!this_cpu_read(*PTRRELOC(&text_poke_area))) + if (!this_cpu_read(text_poke_area)) return raw_patch_instruction(addr, instr); local_irq_save(flags); diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c index 41cf5ae273cf..0872d60ede10 100644 --- a/arch/powerpc/lib/feature-fixups.c +++ b/arch/powerpc/lib/feature-fixups.c @@ -62,7 +62,7 @@ static int patch_alt_instruction(unsigned int *src, unsigned int *dest, } } - patch_instruction(dest, instr); + raw_patch_instruction(dest, instr); return 0; } @@ -91,7 +91,7 @@ static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) } for (; dest < end; dest++) - patch_instruction(dest, PPC_INST_NOP); + raw_patch_instruction(dest, PPC_INST_NOP); return 0; } @@ -129,7 +129,7 @@ void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) for (; start < end; start++) { dest = (void *)start + *start; - patch_instruction(dest, PPC_INST_LWSYNC); + raw_patch_instruction(dest, PPC_INST_LWSYNC); } } @@ -147,7 +147,7 @@ static void do_final_fixups(void) length = (__end_interrupts - _stext) / sizeof(int); while (length--) { - patch_instruction(dest, *src); + raw_patch_instruction(dest, *src); src++; dest++; } -- cgit v1.2.3 From 3f38000eda485f466153d37c4f9201fb50cb03a3 Mon Sep 17 00:00:00 2001 From: Michael Bringmann Date: Fri, 1 Dec 2017 17:19:40 -0600 Subject: powerpc/firmware: Add definitions for new drc-info firmware feature Firmware Features: Define new bit flag representing the presence of new device tree property "ibm,drc-info". The flag is used to tell the front end processor whether the Linux kernel supports the new property, and by the front end processor to tell the Linux kernel that the new property is present in the device tree. Signed-off-by: Michael Bringmann Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/firmware.h | 4 +++- arch/powerpc/include/asm/prom.h | 1 + arch/powerpc/platforms/pseries/firmware.c | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index 832df61f30ef..511acfd7ab0d 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -52,6 +52,7 @@ #define FW_FEATURE_TYPE1_AFFINITY ASM_CONST(0x0000000100000000) #define FW_FEATURE_PRRN ASM_CONST(0x0000000200000000) #define FW_FEATURE_DRMEM_V2 ASM_CONST(0x0000000400000000) +#define FW_FEATURE_DRC_INFO ASM_CONST(0x0000000400000000) #ifndef __ASSEMBLY__ @@ -68,7 +69,8 @@ enum { FW_FEATURE_CMO | FW_FEATURE_VPHN | FW_FEATURE_XCMO | FW_FEATURE_SET_MODE | FW_FEATURE_BEST_ENERGY | FW_FEATURE_TYPE1_AFFINITY | FW_FEATURE_PRRN | - FW_FEATURE_HPT_RESIZE | FW_FEATURE_DRMEM_V2, + FW_FEATURE_HPT_RESIZE | FW_FEATURE_DRMEM_V2 | + FW_FEATURE_DRC_INFO, FW_FEATURE_PSERIES_ALWAYS = 0, FW_FEATURE_POWERNV_POSSIBLE = FW_FEATURE_OPAL, FW_FEATURE_POWERNV_ALWAYS = 0, diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index 9f27866e3126..2ab5d732fefa 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -160,6 +160,7 @@ extern int of_get_ibm_chip_id(struct device_node *np); #define OV5_HASH_GTSE 0x1940 /* Guest Translation Shoot Down Avail */ /* Radix Table Extensions */ #define OV5_RADIX_GTSE 0x1A40 /* Guest Translation Shoot Down Avail */ +#define OV5_DRC_INFO 0x1640 /* Redef Prop Structures: drc-info */ /* Option Vector 6: IBM PAPR hints */ #define OV6_LINUX 0x02 /* Linux is our OS */ diff --git a/arch/powerpc/platforms/pseries/firmware.c b/arch/powerpc/platforms/pseries/firmware.c index aac3ea2911b2..a3bbeb43689e 100644 --- a/arch/powerpc/platforms/pseries/firmware.c +++ b/arch/powerpc/platforms/pseries/firmware.c @@ -115,6 +115,7 @@ vec5_fw_features_table[] = { {FW_FEATURE_TYPE1_AFFINITY, OV5_TYPE1_AFFINITY}, {FW_FEATURE_PRRN, OV5_PRRN}, {FW_FEATURE_DRMEM_V2, OV5_DRMEM_V2}, + {FW_FEATURE_DRC_INFO, OV5_DRC_INFO}, }; static void __init fw_vec5_feature_init(const char *vec5, unsigned long len) -- cgit v1.2.3 From e83636ac333441a17436a1fcd196308f59cd0b51 Mon Sep 17 00:00:00 2001 From: Michael Bringmann Date: Fri, 1 Dec 2017 17:19:43 -0600 Subject: pseries/drc-info: Search DRC properties for CPU indexes pseries/drc-info: Provide parallel routines to convert between drc_index and CPU numbers at runtime, using the older device-tree properties ("ibm,drc-indexes", "ibm,drc-names", "ibm,drc-types" and "ibm,drc-power-domains"), or the new property "ibm,drc-info". Signed-off-by: Michael Bringmann Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/prom.h | 15 +++ arch/powerpc/platforms/pseries/of_helpers.c | 60 +++++++++++ arch/powerpc/platforms/pseries/pseries_energy.c | 126 ++++++++++++++++++------ 3 files changed, 173 insertions(+), 28 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/prom.h b/arch/powerpc/include/asm/prom.h index 2ab5d732fefa..b04c5ce8191b 100644 --- a/arch/powerpc/include/asm/prom.h +++ b/arch/powerpc/include/asm/prom.h @@ -80,6 +80,21 @@ extern void of_instantiate_rtc(void); extern int of_get_ibm_chip_id(struct device_node *np); +struct of_drc_info { + char *drc_type; + char *drc_name_prefix; + u32 drc_index_start; + u32 drc_name_suffix_start; + u32 num_sequential_elems; + u32 sequential_inc; + u32 drc_power_domain; + u32 last_drc_index; +}; + +extern int of_read_drc_info_cell(struct property **prop, + const __be32 **curval, struct of_drc_info *data); + + /* * There are two methods for telling firmware what our capabilities are. * Newer machines have an "ibm,client-architecture-support" method on the diff --git a/arch/powerpc/platforms/pseries/of_helpers.c b/arch/powerpc/platforms/pseries/of_helpers.c index 7e75101fa522..6df192f38f80 100644 --- a/arch/powerpc/platforms/pseries/of_helpers.c +++ b/arch/powerpc/platforms/pseries/of_helpers.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "of_helpers.h" @@ -37,3 +38,62 @@ struct device_node *pseries_of_derive_parent(const char *path) kfree(parent_path); return parent ? parent : ERR_PTR(-EINVAL); } + + +/* Helper Routines to convert between drc_index to cpu numbers */ + +int of_read_drc_info_cell(struct property **prop, const __be32 **curval, + struct of_drc_info *data) +{ + const char *p; + const __be32 *p2; + + if (!data) + return -EINVAL; + + /* Get drc-type:encode-string */ + p = data->drc_type = (char*) (*curval); + p = of_prop_next_string(*prop, p); + if (!p) + return -EINVAL; + + /* Get drc-name-prefix:encode-string */ + data->drc_name_prefix = (char *)p; + p = of_prop_next_string(*prop, p); + if (!p) + return -EINVAL; + + /* Get drc-index-start:encode-int */ + p2 = (const __be32 *)p; + p2 = of_prop_next_u32(*prop, p2, &data->drc_index_start); + if (!p2) + return -EINVAL; + + /* Get drc-name-suffix-start:encode-int */ + p2 = of_prop_next_u32(*prop, p2, &data->drc_name_suffix_start); + if (!p2) + return -EINVAL; + + /* Get number-sequential-elements:encode-int */ + p2 = of_prop_next_u32(*prop, p2, &data->num_sequential_elems); + if (!p2) + return -EINVAL; + + /* Get sequential-increment:encode-int */ + p2 = of_prop_next_u32(*prop, p2, &data->sequential_inc); + if (!p2) + return -EINVAL; + + /* Get drc-power-domain:encode-int */ + p2 = of_prop_next_u32(*prop, p2, &data->drc_power_domain); + if (!p2) + return -EINVAL; + + /* Should now know end of current entry */ + (*curval) = (void *)p2; + data->last_drc_index = data->drc_index_start + + ((data->num_sequential_elems - 1) * data->sequential_inc); + + return 0; +} +EXPORT_SYMBOL(of_read_drc_info_cell); diff --git a/arch/powerpc/platforms/pseries/pseries_energy.c b/arch/powerpc/platforms/pseries/pseries_energy.c index 35c891aabef0..6ed22127391b 100644 --- a/arch/powerpc/platforms/pseries/pseries_energy.c +++ b/arch/powerpc/platforms/pseries/pseries_energy.c @@ -22,6 +22,7 @@ #include #include #include +#include #define MODULE_VERS "1.0" @@ -38,26 +39,58 @@ static int sysfs_entries; static u32 cpu_to_drc_index(int cpu) { struct device_node *dn = NULL; - const int *indexes; - int i; + int thread_index; int rc = 1; u32 ret = 0; dn = of_find_node_by_path("/cpus"); if (dn == NULL) goto err; - indexes = of_get_property(dn, "ibm,drc-indexes", NULL); - if (indexes == NULL) - goto err_of_node_put; + /* Convert logical cpu number to core number */ - i = cpu_core_index_of_thread(cpu); - /* - * The first element indexes[0] is the number of drc_indexes - * returned in the list. Hence i+1 will get the drc_index - * corresponding to core number i. - */ - WARN_ON(i > indexes[0]); - ret = indexes[i + 1]; + thread_index = cpu_core_index_of_thread(cpu); + + if (firmware_has_feature(FW_FEATURE_DRC_INFO)) { + struct property *info = NULL; + struct of_drc_info drc; + int j; + u32 num_set_entries; + const __be32 *value; + + info = of_find_property(dn, "ibm,drc-info", NULL); + if (info == NULL) + goto err_of_node_put; + + value = of_prop_next_u32(info, NULL, &num_set_entries); + if (!value) + goto err_of_node_put; + + for (j = 0; j < num_set_entries; j++) { + + of_read_drc_info_cell(&info, &value, &drc); + if (strncmp(drc.drc_type, "CPU", 3)) + goto err; + + if (thread_index < drc.last_drc_index) + break; + } + + ret = drc.drc_index_start + (thread_index * drc.sequential_inc); + } else { + const __be32 *indexes; + + indexes = of_get_property(dn, "ibm,drc-indexes", NULL); + if (indexes == NULL) + goto err_of_node_put; + + /* + * The first element indexes[0] is the number of drc_indexes + * returned in the list. Hence thread_index+1 will get the + * drc_index corresponding to core number thread_index. + */ + ret = indexes[thread_index + 1]; + } + rc = 0; err_of_node_put: @@ -72,34 +105,71 @@ static int drc_index_to_cpu(u32 drc_index) { struct device_node *dn = NULL; const int *indexes; - int i, cpu = 0; + int thread_index = 0, cpu = 0; int rc = 1; dn = of_find_node_by_path("/cpus"); if (dn == NULL) goto err; - indexes = of_get_property(dn, "ibm,drc-indexes", NULL); - if (indexes == NULL) - goto err_of_node_put; - /* - * First element in the array is the number of drc_indexes - * returned. Search through the list to find the matching - * drc_index and get the core number - */ - for (i = 0; i < indexes[0]; i++) { - if (indexes[i + 1] == drc_index) + + if (firmware_has_feature(FW_FEATURE_DRC_INFO)) { + struct property *info = NULL; + struct of_drc_info drc; + int j; + u32 num_set_entries; + const __be32 *value; + + info = of_find_property(dn, "ibm,drc-info", NULL); + if (info == NULL) + goto err_of_node_put; + + value = of_prop_next_u32(info, NULL, &num_set_entries); + if (!value) + goto err_of_node_put; + + for (j = 0; j < num_set_entries; j++) { + + of_read_drc_info_cell(&info, &value, &drc); + if (strncmp(drc.drc_type, "CPU", 3)) + goto err; + + if (drc_index > drc.last_drc_index) { + cpu += drc.num_sequential_elems; + continue; + } + cpu += ((drc_index - drc.drc_index_start) / + drc.sequential_inc); + + thread_index = cpu_first_thread_of_core(cpu); + rc = 0; break; + } + } else { + unsigned long int i; + + indexes = of_get_property(dn, "ibm,drc-indexes", NULL); + if (indexes == NULL) + goto err_of_node_put; + /* + * First element in the array is the number of drc_indexes + * returned. Search through the list to find the matching + * drc_index and get the core number + */ + for (i = 0; i < indexes[0]; i++) { + if (indexes[i + 1] == drc_index) + break; + } + /* Convert core number to logical cpu number */ + thread_index = cpu_first_thread_of_core(i); + rc = 0; } - /* Convert core number to logical cpu number */ - cpu = cpu_first_thread_of_core(i); - rc = 0; err_of_node_put: of_node_put(dn); err: if (rc) printk(KERN_WARNING "drc_index_to_cpu(%d) failed", drc_index); - return cpu; + return thread_index; } /* -- cgit v1.2.3 From c095ff93f901c1620b28dce4d813dd548bc5236b Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 13 Dec 2017 12:26:23 +0100 Subject: powerpc/sysdev: change CPM GPIO to platform_device Since commit 9427ecbed46cc ("gpio: Rework of_gpiochip_set_names() to use device property accessors"), gpio chips have to have a parent, otherwise devprop_gpiochip_set_names() prematurely exists with message "GPIO chip parent is NULL" and doesn't proceed 'gpio-line-names' DT property. This patch wraps the CPM GPIO into a platform driver to allow assignment of the parent device. Signed-off-by: Christophe Leroy Signed-off-by: Scott Wood --- arch/powerpc/include/asm/cpm.h | 2 +- arch/powerpc/include/asm/cpm1.h | 2 + arch/powerpc/sysdev/Makefile | 3 +- arch/powerpc/sysdev/cpm1.c | 33 ++++------------- arch/powerpc/sysdev/cpm2.c | 11 ------ arch/powerpc/sysdev/cpm_common.c | 5 ++- arch/powerpc/sysdev/cpm_gpio.c | 80 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 97 insertions(+), 39 deletions(-) create mode 100644 arch/powerpc/sysdev/cpm_gpio.c (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/cpm.h b/arch/powerpc/include/asm/cpm.h index b925df1b87d0..4c24ea8209bb 100644 --- a/arch/powerpc/include/asm/cpm.h +++ b/arch/powerpc/include/asm/cpm.h @@ -166,6 +166,6 @@ static inline int cpm_command(u32 command, u8 opcode) } #endif /* CONFIG_CPM */ -int cpm2_gpiochip_add32(struct device_node *np); +int cpm2_gpiochip_add32(struct device *dev); #endif diff --git a/arch/powerpc/include/asm/cpm1.h b/arch/powerpc/include/asm/cpm1.h index 3db821876d48..a116fe931789 100644 --- a/arch/powerpc/include/asm/cpm1.h +++ b/arch/powerpc/include/asm/cpm1.h @@ -605,5 +605,7 @@ enum cpm_clk { }; int cpm1_clk_setup(enum cpm_clk_target target, int clock, int mode); +int cpm1_gpiochip_add16(struct device *dev); +int cpm1_gpiochip_add32(struct device *dev); #endif /* __CPM1__ */ diff --git a/arch/powerpc/sysdev/Makefile b/arch/powerpc/sysdev/Makefile index 0baba21404dc..fc31a271830b 100644 --- a/arch/powerpc/sysdev/Makefile +++ b/arch/powerpc/sysdev/Makefile @@ -43,7 +43,8 @@ obj-$(CONFIG_OF_RTC) += of_rtc.o obj-$(CONFIG_CPM) += cpm_common.o obj-$(CONFIG_CPM1) += cpm1.o -obj-$(CONFIG_CPM2) += cpm2.o cpm2_pic.o +obj-$(CONFIG_CPM2) += cpm2.o cpm2_pic.o cpm_gpio.o +obj-$(CONFIG_8xx_GPIO) += cpm_gpio.o obj-$(CONFIG_QUICC_ENGINE) += cpm_common.o obj-$(CONFIG_PPC_DCR) += dcr.o obj-$(CONFIG_UCODE_PATCH) += micropatch.o diff --git a/arch/powerpc/sysdev/cpm1.c b/arch/powerpc/sysdev/cpm1.c index c6f154b602fb..5240d3a74a10 100644 --- a/arch/powerpc/sysdev/cpm1.c +++ b/arch/powerpc/sysdev/cpm1.c @@ -629,8 +629,9 @@ static int cpm1_gpio16_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } -int cpm1_gpiochip_add16(struct device_node *np) +int cpm1_gpiochip_add16(struct device *dev) { + struct device_node *np = dev->of_node; struct cpm1_gpio16_chip *cpm1_gc; struct of_mm_gpio_chip *mm_gc; struct gpio_chip *gc; @@ -660,6 +661,8 @@ int cpm1_gpiochip_add16(struct device_node *np) gc->get = cpm1_gpio16_get; gc->set = cpm1_gpio16_set; gc->to_irq = cpm1_gpio16_to_irq; + gc->parent = dev; + gc->owner = THIS_MODULE; return of_mm_gpiochip_add_data(np, mm_gc, cpm1_gc); } @@ -755,8 +758,9 @@ static int cpm1_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } -int cpm1_gpiochip_add32(struct device_node *np) +int cpm1_gpiochip_add32(struct device *dev) { + struct device_node *np = dev->of_node; struct cpm1_gpio32_chip *cpm1_gc; struct of_mm_gpio_chip *mm_gc; struct gpio_chip *gc; @@ -776,31 +780,10 @@ int cpm1_gpiochip_add32(struct device_node *np) gc->direction_output = cpm1_gpio32_dir_out; gc->get = cpm1_gpio32_get; gc->set = cpm1_gpio32_set; + gc->parent = dev; + gc->owner = THIS_MODULE; return of_mm_gpiochip_add_data(np, mm_gc, cpm1_gc); } -static int cpm_init_par_io(void) -{ - struct device_node *np; - - for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-a") - cpm1_gpiochip_add16(np); - - for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-b") - cpm1_gpiochip_add32(np); - - for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-c") - cpm1_gpiochip_add16(np); - - for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-d") - cpm1_gpiochip_add16(np); - - /* Port E uses CPM2 layout */ - for_each_compatible_node(np, NULL, "fsl,cpm1-pario-bank-e") - cpm2_gpiochip_add32(np); - return 0; -} -arch_initcall(cpm_init_par_io); - #endif /* CONFIG_8xx_GPIO */ diff --git a/arch/powerpc/sysdev/cpm2.c b/arch/powerpc/sysdev/cpm2.c index f78ff841652c..07718b9a2c99 100644 --- a/arch/powerpc/sysdev/cpm2.c +++ b/arch/powerpc/sysdev/cpm2.c @@ -354,14 +354,3 @@ void cpm2_set_pin(int port, int pin, int flags) else clrbits32(&iop[port].odr, pin); } - -static int cpm_init_par_io(void) -{ - struct device_node *np; - - for_each_compatible_node(np, NULL, "fsl,cpm2-pario-bank") - cpm2_gpiochip_add32(np); - return 0; -} -arch_initcall(cpm_init_par_io); - diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c index 51bf749a4f3a..b74508175b67 100644 --- a/arch/powerpc/sysdev/cpm_common.c +++ b/arch/powerpc/sysdev/cpm_common.c @@ -190,8 +190,9 @@ static int cpm2_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } -int cpm2_gpiochip_add32(struct device_node *np) +int cpm2_gpiochip_add32(struct device *dev) { + struct device_node *np = dev->of_node; struct cpm2_gpio32_chip *cpm2_gc; struct of_mm_gpio_chip *mm_gc; struct gpio_chip *gc; @@ -211,6 +212,8 @@ int cpm2_gpiochip_add32(struct device_node *np) gc->direction_output = cpm2_gpio32_dir_out; gc->get = cpm2_gpio32_get; gc->set = cpm2_gpio32_set; + gc->parent = dev; + gc->owner = THIS_MODULE; return of_mm_gpiochip_add_data(np, mm_gc, cpm2_gc); } diff --git a/arch/powerpc/sysdev/cpm_gpio.c b/arch/powerpc/sysdev/cpm_gpio.c new file mode 100644 index 000000000000..0badc90be666 --- /dev/null +++ b/arch/powerpc/sysdev/cpm_gpio.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common CPM GPIO wrapper for the CPM GPIO ports + * + * Author: Christophe Leroy + * + * Copyright 2017 CS Systemes d'Information. + * + */ + +#include +#include + +#include +#ifdef CONFIG_8xx_GPIO +#include +#endif + +static int cpm_gpio_probe(struct platform_device *ofdev) +{ + struct device *dev = &ofdev->dev; + int (*gp_add)(struct device *dev) = of_device_get_match_data(dev); + + if (!gp_add) + return -ENODEV; + + return gp_add(dev); +} + +static const struct of_device_id cpm_gpio_match[] = { +#ifdef CONFIG_8xx_GPIO + { + .compatible = "fsl,cpm1-pario-bank-a", + .data = cpm1_gpiochip_add16, + }, + { + .compatible = "fsl,cpm1-pario-bank-b", + .data = cpm1_gpiochip_add32, + }, + { + .compatible = "fsl,cpm1-pario-bank-c", + .data = cpm1_gpiochip_add16, + }, + { + .compatible = "fsl,cpm1-pario-bank-d", + .data = cpm1_gpiochip_add16, + }, + /* Port E uses CPM2 layout */ + { + .compatible = "fsl,cpm1-pario-bank-e", + .data = cpm2_gpiochip_add32, + }, +#endif + { + .compatible = "fsl,cpm2-pario-bank", + .data = cpm2_gpiochip_add32, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, cpm_gpio_match); + +static struct platform_driver cpm_gpio_driver = { + .probe = cpm_gpio_probe, + .driver = { + .name = "cpm-gpio", + .owner = THIS_MODULE, + .of_match_table = cpm_gpio_match, + }, +}; + +static int __init cpm_gpio_init(void) +{ + return platform_driver_register(&cpm_gpio_driver); +} +arch_initcall(cpm_gpio_init); + +MODULE_AUTHOR("Christophe Leroy "); +MODULE_DESCRIPTION("Driver for CPM GPIO"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:cpm-gpio"); -- cgit v1.2.3 From 38833faa112c10dd6e3d888f8b53de2b80cb881a Mon Sep 17 00:00:00 2001 From: Mathieu Malaterre Date: Tue, 26 Dec 2017 14:00:17 +0100 Subject: powerpc/xive: Properly use static keyword for inline function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix fatal warning during compilation: In file included from arch/powerpc/xmon/xmon.c:54:0: ./arch/powerpc/include/asm/xive.h:157:20: error: no previous prototype for ‘xive_smp_prepare_cpu’ [-Werror=missing-prototypes] extern inline int xive_smp_prepare_cpu(unsigned int cpu) { return -EINVAL; } ^ Signed-off-by: Mathieu Malaterre Reviewed-by: Cédric Le Goater Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/xive.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/xive.h b/arch/powerpc/include/asm/xive.h index b619a5585cd6..7624e22f5045 100644 --- a/arch/powerpc/include/asm/xive.h +++ b/arch/powerpc/include/asm/xive.h @@ -122,7 +122,7 @@ static inline bool xive_enabled(void) { return false; } static inline bool xive_spapr_init(void) { return false; } static inline bool xive_native_init(void) { return false; } static inline void xive_smp_probe(void) { } -extern inline int xive_smp_prepare_cpu(unsigned int cpu) { return -EINVAL; } +static inline int xive_smp_prepare_cpu(unsigned int cpu) { return -EINVAL; } static inline void xive_smp_setup_cpu(void) { } static inline void xive_smp_disable_cpu(void) { } static inline void xive_kexec_teardown_cpu(int secondary) { } -- cgit v1.2.3 From 35adacd6fc48d658419522f192a3c8b2785612da Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sun, 24 Dec 2017 02:49:23 +1000 Subject: powerpc/pseries, ps3: panic flush kernel messages before halting system Platforms with a panic handler that halts the system can have problems getting kernel messages out, because the panic notifiers are called before kernel/panic.c does its flushing of printk buffers an console etc. This was attempted to be solved with commit a3b2cb30f252 ("powerpc: Do not call ppc_md.panic in fadump panic notifier"), but that wasn't the right approach and caused other problems, and was reverted by commit ab9dbf771ff9. Instead, the powernv shutdown paths have already had a similar problem, fixed by taking the message flushing sequence from kernel/panic.c. That's a little bit ugly, but while we have the code duplicated, it will work for this case as well. So have ppc panic handlers do the same flushing before they terminate. Without this patch, a qemu pseries_le_defconfig guest stops silently when issued the nmi command when xmon is off and no crash dumpers enabled. Afterwards, an oops is printed by each CPU as expected. Fixes: ab9dbf771ff9 ("Revert "powerpc: Do not call ppc_md.panic in fadump panic notifier"") Signed-off-by: Nicholas Piggin Reviewed-by: David Gibson Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/bug.h | 3 ++- arch/powerpc/kernel/traps.c | 24 ++++++++++++++++++++++++ arch/powerpc/platforms/powernv/opal.c | 18 ++++-------------- arch/powerpc/platforms/ps3/setup.c | 1 + arch/powerpc/platforms/pseries/setup.c | 8 +++++++- 5 files changed, 38 insertions(+), 16 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/bug.h b/arch/powerpc/include/asm/bug.h index 97c38472b924..fd06dbe7d7d3 100644 --- a/arch/powerpc/include/asm/bug.h +++ b/arch/powerpc/include/asm/bug.h @@ -136,7 +136,8 @@ extern void _exception(int, struct pt_regs *, int, unsigned long); extern void _exception_pkey(int, struct pt_regs *, int, unsigned long, int); extern void die(const char *, struct pt_regs *, long); extern bool die_will_crash(void); - +extern void panic_flush_kmsg_start(void); +extern void panic_flush_kmsg_end(void); #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index cb07d6519783..4260c73461a3 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -39,6 +39,8 @@ #include #include #include +#include +#include #include #include @@ -143,6 +145,28 @@ static int die_owner = -1; static unsigned int die_nest_count; static int die_counter; +extern void panic_flush_kmsg_start(void) +{ + /* + * These are mostly taken from kernel/panic.c, but tries to do + * relatively minimal work. Don't use delay functions (TB may + * be broken), don't crash dump (need to set a firmware log), + * don't run notifiers. We do want to get some information to + * Linux console. + */ + console_verbose(); + bust_spinlocks(1); +} + +extern void panic_flush_kmsg_end(void) +{ + printk_safe_flush_on_panic(); + kmsg_dump(KMSG_DUMP_PANIC); + bust_spinlocks(0); + debug_locks_off(); + console_flush_on_panic(); +} + static unsigned long oops_begin(struct pt_regs *regs) { int cpu; diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index 69b5263fc9e3..c15182765ff5 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -461,24 +461,14 @@ static int opal_recover_mce(struct pt_regs *regs, void pnv_platform_error_reboot(struct pt_regs *regs, const char *msg) { - /* - * This is mostly taken from kernel/panic.c, but tries to do - * relatively minimal work. Don't use delay functions (TB may - * be broken), don't crash dump (need to set a firmware log), - * don't run notifiers. We do want to get some information to - * Linux console. - */ - console_verbose(); - bust_spinlocks(1); + panic_flush_kmsg_start(); + pr_emerg("Hardware platform error: %s\n", msg); if (regs) show_regs(regs); smp_send_stop(); - printk_safe_flush_on_panic(); - kmsg_dump(KMSG_DUMP_PANIC); - bust_spinlocks(0); - debug_locks_off(); - console_flush_on_panic(); + + panic_flush_kmsg_end(); /* * Don't bother to shut things down because this will diff --git a/arch/powerpc/platforms/ps3/setup.c b/arch/powerpc/platforms/ps3/setup.c index 6244bc849469..77a37520068d 100644 --- a/arch/powerpc/platforms/ps3/setup.c +++ b/arch/powerpc/platforms/ps3/setup.c @@ -113,6 +113,7 @@ static void ps3_panic(char *str) printk(" System does not reboot automatically.\n"); printk(" Please press POWER button.\n"); printk("\n"); + panic_flush_kmsg_end(); while(1) lv1_pause(1); diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index db76963e693d..42e9cf0b3180 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -533,6 +533,12 @@ static void __init pSeries_setup_arch(void) ppc_md.pcibios_root_bridge_prepare = pseries_root_bridge_prepare; } +static void pseries_panic(char *str) +{ + panic_flush_kmsg_end(); + rtas_os_term(str); +} + static int __init pSeries_init_panel(void) { /* Manually leave the kernel version on the panel. */ @@ -761,7 +767,7 @@ define_machine(pseries) { .pcibios_fixup = pSeries_final_fixup, .restart = rtas_restart, .halt = rtas_halt, - .panic = rtas_os_term, + .panic = pseries_panic, .get_boot_time = rtas_get_boot_time, .get_rtc_time = rtas_get_rtc_time, .set_rtc_time = rtas_set_rtc_time, -- cgit v1.2.3 From bdcb1aefc5b3f7d0f1dc8b02673602bca2ff7a4b Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Wed, 17 Jan 2018 23:58:18 +1000 Subject: powerpc/64s: Improve RFI L1-D cache flush fallback The fallback RFI flush is used when firmware does not provide a way to flush the cache. It's a "displacement flush" that evicts useful data by displacing it with an uninteresting buffer. The flush has to take care to work with implementation specific cache replacment policies, so the recipe has been in flux. The initial slow but conservative approach is to touch all lines of a congruence class, with dependencies between each load. It has since been determined that a linear pattern of loads without dependencies is sufficient, and is significantly faster. Measuring the speed of a null syscall with RFI fallback flush enabled gives the relative improvement: P8 - 1.83x P9 - 1.75x The flush also becomes simpler and more adaptable to different cache geometries. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/paca.h | 3 +- arch/powerpc/kernel/asm-offsets.c | 3 +- arch/powerpc/kernel/exceptions-64s.S | 76 +++++++++++++++++------------------- arch/powerpc/kernel/setup_64.c | 13 +----- arch/powerpc/xmon/xmon.c | 2 - 5 files changed, 39 insertions(+), 58 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 1b7fd535393b..b62c31037cad 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -239,8 +239,7 @@ struct paca_struct { */ u64 exrfi[EX_SIZE] __aligned(0x80); void *rfi_flush_fallback_area; - u64 l1d_flush_congruence; - u64 l1d_flush_sets; + u64 l1d_flush_size; #endif }; diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index fa5c4125f42a..88b84ac76b53 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -239,8 +239,7 @@ int main(void) OFFSET(PACA_IN_NMI, paca_struct, in_nmi); OFFSET(PACA_RFI_FLUSH_FALLBACK_AREA, paca_struct, rfi_flush_fallback_area); OFFSET(PACA_EXRFI, paca_struct, exrfi); - OFFSET(PACA_L1D_FLUSH_CONGRUENCE, paca_struct, l1d_flush_congruence); - OFFSET(PACA_L1D_FLUSH_SETS, paca_struct, l1d_flush_sets); + OFFSET(PACA_L1D_FLUSH_SIZE, paca_struct, l1d_flush_size); #endif OFFSET(PACAHWCPUID, paca_struct, hw_cpu_id); diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 9e6882bdc526..243d072a225a 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -1461,39 +1461,37 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback) std r9,PACA_EXRFI+EX_R9(r13) std r10,PACA_EXRFI+EX_R10(r13) std r11,PACA_EXRFI+EX_R11(r13) - std r12,PACA_EXRFI+EX_R12(r13) - std r8,PACA_EXRFI+EX_R13(r13) mfctr r9 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) - ld r11,PACA_L1D_FLUSH_SETS(r13) - ld r12,PACA_L1D_FLUSH_CONGRUENCE(r13) - /* - * The load adresses are at staggered offsets within cachelines, - * which suits some pipelines better (on others it should not - * hurt). - */ - addi r12,r12,8 + ld r11,PACA_L1D_FLUSH_SIZE(r13) + srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ mtctr r11 DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ /* order ld/st prior to dcbt stop all streams with flushing */ sync -1: li r8,0 - .rept 8 /* 8-way set associative */ - ldx r11,r10,r8 - add r8,r8,r12 - xor r11,r11,r11 // Ensure r11 is 0 even if fallback area is not - add r8,r8,r11 // Add 0, this creates a dependency on the ldx - .endr - addi r10,r10,128 /* 128 byte cache line */ + + /* + * The load adresses are at staggered offsets within cachelines, + * which suits some pipelines better (on others it should not + * hurt). + */ +1: + ld r11,(0x80 + 8)*0(r10) + ld r11,(0x80 + 8)*1(r10) + ld r11,(0x80 + 8)*2(r10) + ld r11,(0x80 + 8)*3(r10) + ld r11,(0x80 + 8)*4(r10) + ld r11,(0x80 + 8)*5(r10) + ld r11,(0x80 + 8)*6(r10) + ld r11,(0x80 + 8)*7(r10) + addi r10,r10,0x80*8 bdnz 1b mtctr r9 ld r9,PACA_EXRFI+EX_R9(r13) ld r10,PACA_EXRFI+EX_R10(r13) ld r11,PACA_EXRFI+EX_R11(r13) - ld r12,PACA_EXRFI+EX_R12(r13) - ld r8,PACA_EXRFI+EX_R13(r13) GET_SCRATCH0(r13); rfid @@ -1503,39 +1501,37 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) std r9,PACA_EXRFI+EX_R9(r13) std r10,PACA_EXRFI+EX_R10(r13) std r11,PACA_EXRFI+EX_R11(r13) - std r12,PACA_EXRFI+EX_R12(r13) - std r8,PACA_EXRFI+EX_R13(r13) mfctr r9 ld r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13) - ld r11,PACA_L1D_FLUSH_SETS(r13) - ld r12,PACA_L1D_FLUSH_CONGRUENCE(r13) - /* - * The load adresses are at staggered offsets within cachelines, - * which suits some pipelines better (on others it should not - * hurt). - */ - addi r12,r12,8 + ld r11,PACA_L1D_FLUSH_SIZE(r13) + srdi r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */ mtctr r11 DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */ /* order ld/st prior to dcbt stop all streams with flushing */ sync -1: li r8,0 - .rept 8 /* 8-way set associative */ - ldx r11,r10,r8 - add r8,r8,r12 - xor r11,r11,r11 // Ensure r11 is 0 even if fallback area is not - add r8,r8,r11 // Add 0, this creates a dependency on the ldx - .endr - addi r10,r10,128 /* 128 byte cache line */ + + /* + * The load adresses are at staggered offsets within cachelines, + * which suits some pipelines better (on others it should not + * hurt). + */ +1: + ld r11,(0x80 + 8)*0(r10) + ld r11,(0x80 + 8)*1(r10) + ld r11,(0x80 + 8)*2(r10) + ld r11,(0x80 + 8)*3(r10) + ld r11,(0x80 + 8)*4(r10) + ld r11,(0x80 + 8)*5(r10) + ld r11,(0x80 + 8)*6(r10) + ld r11,(0x80 + 8)*7(r10) + addi r10,r10,0x80*8 bdnz 1b mtctr r9 ld r9,PACA_EXRFI+EX_R9(r13) ld r10,PACA_EXRFI+EX_R10(r13) ld r11,PACA_EXRFI+EX_R11(r13) - ld r12,PACA_EXRFI+EX_R12(r13) - ld r8,PACA_EXRFI+EX_R13(r13) GET_SCRATCH0(r13); hrfid diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index d1fa0e91f526..c388cc3357fa 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -875,19 +875,8 @@ static void init_fallback_flush(void) memset(l1d_flush_fallback_area, 0, l1d_size * 2); for_each_possible_cpu(cpu) { - /* - * The fallback flush is currently coded for 8-way - * associativity. Different associativity is possible, but it - * will be treated as 8-way and may not evict the lines as - * effectively. - * - * 128 byte lines are mandatory. - */ - u64 c = l1d_size / 8; - paca[cpu].rfi_flush_fallback_area = l1d_flush_fallback_area; - paca[cpu].l1d_flush_congruence = c; - paca[cpu].l1d_flush_sets = c / 128; + paca[cpu].l1d_flush_size = l1d_size; } } diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index 01d9a2dcff20..82e1a3ee6e0f 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -2377,8 +2377,6 @@ static void dump_one_paca(int cpu) printf(" slb_cache[%d]: = 0x%016lx\n", i, p->slb_cache[i]); DUMP(p, rfi_flush_fallback_area, "px"); - DUMP(p, l1d_flush_congruence, "llx"); - DUMP(p, l1d_flush_sets, "llx"); #endif DUMP(p, dscr_default, "llx"); #ifdef CONFIG_PPC_BOOK3E -- cgit v1.2.3 From 74d656d219b98ef3b96f92439337aa6392a7577d Mon Sep 17 00:00:00 2001 From: Frederic Barrat Date: Tue, 23 Jan 2018 12:31:38 +0100 Subject: powerpc/powernv: Add opal calls for opencapi Add opal calls to interact with the NPU: OPAL_NPU_SPA_SETUP: set the Shared Process Area (SPA) The SPA is a table containing one entry (Process Element) per memory context which can be accessed by the opencapi device. OPAL_NPU_SPA_CLEAR_CACHE: clear the context cache The NPU keeps a cache of recently accessed memory contexts. When a Process Element is removed from the SPA, the cache for the link must be cleared. OPAL_NPU_TL_SET: configure the Transaction Layer The Transaction Layer specification defines several templates for messages to be exchanged on the link. During link setup, the host and device must negotiate what templates are supported on both sides and at what rates those messages can be sent. Signed-off-by: Frederic Barrat Acked-by: Andrew Donnellan Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/opal-api.h | 5 ++++- arch/powerpc/include/asm/opal.h | 6 ++++++ arch/powerpc/platforms/powernv/opal-wrappers.S | 3 +++ 3 files changed, 13 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h index 233c7504b1f2..24c73f5575ee 100644 --- a/arch/powerpc/include/asm/opal-api.h +++ b/arch/powerpc/include/asm/opal-api.h @@ -201,7 +201,10 @@ #define OPAL_SET_POWER_SHIFT_RATIO 155 #define OPAL_SENSOR_GROUP_CLEAR 156 #define OPAL_PCI_SET_P2P 157 -#define OPAL_LAST 157 +#define OPAL_NPU_SPA_SETUP 159 +#define OPAL_NPU_SPA_CLEAR_CACHE 160 +#define OPAL_NPU_TL_SET 161 +#define OPAL_LAST 161 /* Device tree flags */ diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index 0c545f7fc77b..12e70fb58700 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -34,6 +34,12 @@ int64_t opal_npu_init_context(uint64_t phb_id, int pasid, uint64_t msr, uint64_t bdf); int64_t opal_npu_map_lpar(uint64_t phb_id, uint64_t bdf, uint64_t lparid, uint64_t lpcr); +int64_t opal_npu_spa_setup(uint64_t phb_id, uint32_t bdfn, + uint64_t addr, uint64_t PE_mask); +int64_t opal_npu_spa_clear_cache(uint64_t phb_id, uint32_t bdfn, + uint64_t PE_handle); +int64_t opal_npu_tl_set(uint64_t phb_id, uint32_t bdfn, long cap, + uint64_t rate_phys, uint32_t size); int64_t opal_console_write(int64_t term_number, __be64 *length, const uint8_t *buffer); int64_t opal_console_read(int64_t term_number, __be64 *length, diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S index 6f4b00a2ac46..1b2936ba6040 100644 --- a/arch/powerpc/platforms/powernv/opal-wrappers.S +++ b/arch/powerpc/platforms/powernv/opal-wrappers.S @@ -320,3 +320,6 @@ OPAL_CALL(opal_set_powercap, OPAL_SET_POWERCAP); OPAL_CALL(opal_get_power_shift_ratio, OPAL_GET_POWER_SHIFT_RATIO); OPAL_CALL(opal_set_power_shift_ratio, OPAL_SET_POWER_SHIFT_RATIO); OPAL_CALL(opal_sensor_group_clear, OPAL_SENSOR_GROUP_CLEAR); +OPAL_CALL(opal_npu_spa_setup, OPAL_NPU_SPA_SETUP); +OPAL_CALL(opal_npu_spa_clear_cache, OPAL_NPU_SPA_CLEAR_CACHE); +OPAL_CALL(opal_npu_tl_set, OPAL_NPU_TL_SET); -- cgit v1.2.3 From 6914c757118e2a60ba826d9959ccf5532779781b Mon Sep 17 00:00:00 2001 From: Frederic Barrat Date: Tue, 23 Jan 2018 12:31:39 +0100 Subject: powerpc/powernv: Add platform-specific services for opencapi Implement a few platform-specific calls which can be used by drivers: - provide the Transaction Layer capabilities of the host, so that the driver can find some common ground and configure the device and host appropriately. - provide the hw interrupt to be used for translation faults raised by the NPU - map/unmap some NPU mmio registers to get the fault context when the NPU raises an address translation fault The rest are wrappers around the previously-introduced opal calls. Signed-off-by: Frederic Barrat Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pnv-ocxl.h | 29 +++++ arch/powerpc/platforms/powernv/Makefile | 1 + arch/powerpc/platforms/powernv/ocxl.c | 180 ++++++++++++++++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 arch/powerpc/include/asm/pnv-ocxl.h create mode 100644 arch/powerpc/platforms/powernv/ocxl.c (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h new file mode 100644 index 000000000000..36868d49aeed --- /dev/null +++ b/arch/powerpc/include/asm/pnv-ocxl.h @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2017 IBM Corp. +#ifndef _ASM_PNV_OCXL_H +#define _ASM_PNV_OCXL_H + +#include + +#define PNV_OCXL_TL_MAX_TEMPLATE 63 +#define PNV_OCXL_TL_BITS_PER_RATE 4 +#define PNV_OCXL_TL_RATE_BUF_SIZE ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8) + +extern int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, + char *rate_buf, int rate_buf_size); +extern int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, + uint64_t rate_buf_phys, int rate_buf_size); + +extern int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq); +extern void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, + void __iomem *tfc, void __iomem *pe_handle); +extern int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, + void __iomem **dar, void __iomem **tfc, + void __iomem **pe_handle); + +extern int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, + void **platform_data); +extern void pnv_ocxl_spa_release(void *platform_data); +extern int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle); + +#endif /* _ASM_PNV_OCXL_H */ diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile index 3732118a0482..6c9d5199a7e2 100644 --- a/arch/powerpc/platforms/powernv/Makefile +++ b/arch/powerpc/platforms/powernv/Makefile @@ -17,3 +17,4 @@ obj-$(CONFIG_PERF_EVENTS) += opal-imc.o obj-$(CONFIG_PPC_MEMTRACE) += memtrace.o obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o obj-$(CONFIG_PPC_FTW) += nx-ftw.o +obj-$(CONFIG_OCXL_BASE) += ocxl.o diff --git a/arch/powerpc/platforms/powernv/ocxl.c b/arch/powerpc/platforms/powernv/ocxl.c new file mode 100644 index 000000000000..d61186805a07 --- /dev/null +++ b/arch/powerpc/platforms/powernv/ocxl.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2017 IBM Corp. +#include +#include +#include "pci.h" + +#define PNV_OCXL_TL_P9_RECV_CAP 0x000000000000000Full +/* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */ +#define PNV_OCXL_PASID_BITS 15 +#define PNV_OCXL_PASID_MAX ((1 << PNV_OCXL_PASID_BITS) - 1) + + +static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf) +{ + int shift, idx; + + WARN_ON(templ > PNV_OCXL_TL_MAX_TEMPLATE); + idx = (PNV_OCXL_TL_MAX_TEMPLATE - templ) / 2; + shift = 4 * (1 - ((PNV_OCXL_TL_MAX_TEMPLATE - templ) % 2)); + buf[idx] |= rate << shift; +} + +int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, + char *rate_buf, int rate_buf_size) +{ + if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) + return -EINVAL; + /* + * The TL capabilities are a characteristic of the NPU, so + * we go with hard-coded values. + * + * The receiving rate of each template is encoded on 4 bits. + * + * On P9: + * - templates 0 -> 3 are supported + * - templates 0, 1 and 3 have a 0 receiving rate + * - template 2 has receiving rate of 1 (extra cycle) + */ + memset(rate_buf, 0, rate_buf_size); + set_templ_rate(2, 1, rate_buf); + *cap = PNV_OCXL_TL_P9_RECV_CAP; + return 0; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_get_tl_cap); + +int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, + uint64_t rate_buf_phys, int rate_buf_size) +{ + struct pci_controller *hose = pci_bus_to_host(dev->bus); + struct pnv_phb *phb = hose->private_data; + int rc; + + if (rate_buf_size != PNV_OCXL_TL_RATE_BUF_SIZE) + return -EINVAL; + + rc = opal_npu_tl_set(phb->opal_id, dev->devfn, cap, + rate_buf_phys, rate_buf_size); + if (rc) { + dev_err(&dev->dev, "Can't configure host TL: %d\n", rc); + return -EINVAL; + } + return 0; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_set_tl_conf); + +int pnv_ocxl_get_xsl_irq(struct pci_dev *dev, int *hwirq) +{ + int rc; + + rc = of_property_read_u32(dev->dev.of_node, "ibm,opal-xsl-irq", hwirq); + if (rc) { + dev_err(&dev->dev, + "Can't get translation interrupt for device\n"); + return rc; + } + return 0; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_get_xsl_irq); + +void pnv_ocxl_unmap_xsl_regs(void __iomem *dsisr, void __iomem *dar, + void __iomem *tfc, void __iomem *pe_handle) +{ + iounmap(dsisr); + iounmap(dar); + iounmap(tfc); + iounmap(pe_handle); +} +EXPORT_SYMBOL_GPL(pnv_ocxl_unmap_xsl_regs); + +int pnv_ocxl_map_xsl_regs(struct pci_dev *dev, void __iomem **dsisr, + void __iomem **dar, void __iomem **tfc, + void __iomem **pe_handle) +{ + u64 reg; + int i, j, rc = 0; + void __iomem *regs[4]; + + /* + * opal stores the mmio addresses of the DSISR, DAR, TFC and + * PE_HANDLE registers in a device tree property, in that + * order + */ + for (i = 0; i < 4; i++) { + rc = of_property_read_u64_index(dev->dev.of_node, + "ibm,opal-xsl-mmio", i, ®); + if (rc) + break; + regs[i] = ioremap(reg, 8); + if (!regs[i]) { + rc = -EINVAL; + break; + } + } + if (rc) { + dev_err(&dev->dev, "Can't map translation mmio registers\n"); + for (j = i - 1; j >= 0; j--) + iounmap(regs[j]); + } else { + *dsisr = regs[0]; + *dar = regs[1]; + *tfc = regs[2]; + *pe_handle = regs[3]; + } + return rc; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_map_xsl_regs); + +struct spa_data { + u64 phb_opal_id; + u32 bdfn; +}; + +int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, + void **platform_data) +{ + struct pci_controller *hose = pci_bus_to_host(dev->bus); + struct pnv_phb *phb = hose->private_data; + struct spa_data *data; + u32 bdfn; + int rc; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + bdfn = (dev->bus->number << 8) | dev->devfn; + rc = opal_npu_spa_setup(phb->opal_id, bdfn, virt_to_phys(spa_mem), + PE_mask); + if (rc) { + dev_err(&dev->dev, "Can't setup Shared Process Area: %d\n", rc); + kfree(data); + return rc; + } + data->phb_opal_id = phb->opal_id; + data->bdfn = bdfn; + *platform_data = (void *) data; + return 0; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_spa_setup); + +void pnv_ocxl_spa_release(void *platform_data) +{ + struct spa_data *data = (struct spa_data *) platform_data; + int rc; + + rc = opal_npu_spa_setup(data->phb_opal_id, data->bdfn, 0, 0); + WARN_ON(rc); + kfree(data); +} +EXPORT_SYMBOL_GPL(pnv_ocxl_spa_release); + +int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle) +{ + struct spa_data *data = (struct spa_data *) platform_data; + int rc; + + rc = opal_npu_spa_clear_cache(data->phb_opal_id, data->bdfn, pe_handle); + return rc; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe); -- cgit v1.2.3 From 2cb3d64b26984703a6bb80e66adcc3727ad37f9f Mon Sep 17 00:00:00 2001 From: Frederic Barrat Date: Tue, 23 Jan 2018 12:31:40 +0100 Subject: powerpc/powernv: Capture actag information for the device In the opencapi protocol, host memory contexts are referenced by a 'actag'. During setup, a driver must tell the device how many actags it can used, and what values are acceptable. On POWER9, the NPU can handle 64 actags per link, so they must be shared between all the PCI functions of the link. To get a global picture of how many actags are used by each AFU of every function, we capture some data at the end of PCI enumeration, so that actags can be shared fairly if needed. This is not powernv specific per say, but rather a consequence of the opencapi configuration specification being quite general. The number of available actags on POWER9 makes it more likely to be hit. This is somewhat mitigated by the fact that existing AFUs are coded by requesting a reasonable count of actags and existing devices carry only one AFU. Signed-off-by: Frederic Barrat Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pnv-ocxl.h | 4 + arch/powerpc/platforms/powernv/ocxl.c | 305 ++++++++++++++++++++++++++++++++++ include/misc/ocxl-config.h | 45 +++++ 3 files changed, 354 insertions(+) create mode 100644 include/misc/ocxl-config.h (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h index 36868d49aeed..398d05b30600 100644 --- a/arch/powerpc/include/asm/pnv-ocxl.h +++ b/arch/powerpc/include/asm/pnv-ocxl.h @@ -9,6 +9,10 @@ #define PNV_OCXL_TL_BITS_PER_RATE 4 #define PNV_OCXL_TL_RATE_BUF_SIZE ((PNV_OCXL_TL_MAX_TEMPLATE+1) * PNV_OCXL_TL_BITS_PER_RATE / 8) +extern int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, + u16 *supported); +extern int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count); + extern int pnv_ocxl_get_tl_cap(struct pci_dev *dev, long *cap, char *rate_buf, int rate_buf_size); extern int pnv_ocxl_set_tl_conf(struct pci_dev *dev, long cap, diff --git a/arch/powerpc/platforms/powernv/ocxl.c b/arch/powerpc/platforms/powernv/ocxl.c index d61186805a07..1faaa4ef6903 100644 --- a/arch/powerpc/platforms/powernv/ocxl.c +++ b/arch/powerpc/platforms/powernv/ocxl.c @@ -2,13 +2,318 @@ // Copyright 2017 IBM Corp. #include #include +#include #include "pci.h" #define PNV_OCXL_TL_P9_RECV_CAP 0x000000000000000Full +#define PNV_OCXL_ACTAG_MAX 64 /* PASIDs are 20-bit, but on P9, NPU can only handle 15 bits */ #define PNV_OCXL_PASID_BITS 15 #define PNV_OCXL_PASID_MAX ((1 << PNV_OCXL_PASID_BITS) - 1) +#define AFU_PRESENT (1 << 31) +#define AFU_INDEX_MASK 0x3F000000 +#define AFU_INDEX_SHIFT 24 +#define ACTAG_MASK 0xFFF + + +struct actag_range { + u16 start; + u16 count; +}; + +struct npu_link { + struct list_head list; + int domain; + int bus; + int dev; + u16 fn_desired_actags[8]; + struct actag_range fn_actags[8]; + bool assignment_done; +}; +static struct list_head links_list = LIST_HEAD_INIT(links_list); +static DEFINE_MUTEX(links_list_lock); + + +/* + * opencapi actags handling: + * + * When sending commands, the opencapi device references the memory + * context it's targeting with an 'actag', which is really an alias + * for a (BDF, pasid) combination. When it receives a command, the NPU + * must do a lookup of the actag to identify the memory context. The + * hardware supports a finite number of actags per link (64 for + * POWER9). + * + * The device can carry multiple functions, and each function can have + * multiple AFUs. Each AFU advertises in its config space the number + * of desired actags. The host must configure in the config space of + * the AFU how many actags the AFU is really allowed to use (which can + * be less than what the AFU desires). + * + * When a PCI function is probed by the driver, it has no visibility + * about the other PCI functions and how many actags they'd like, + * which makes it impossible to distribute actags fairly among AFUs. + * + * Unfortunately, the only way to know how many actags a function + * desires is by looking at the data for each AFU in the config space + * and add them up. Similarly, the only way to know how many actags + * all the functions of the physical device desire is by adding the + * previously computed function counts. Then we can match that against + * what the hardware supports. + * + * To get a comprehensive view, we use a 'pci fixup': at the end of + * PCI enumeration, each function counts how many actags its AFUs + * desire and we save it in a 'npu_link' structure, shared between all + * the PCI functions of a same device. Therefore, when the first + * function is probed by the driver, we can get an idea of the total + * count of desired actags for the device, and assign the actags to + * the AFUs, by pro-rating if needed. + */ + +static int find_dvsec_from_pos(struct pci_dev *dev, int dvsec_id, int pos) +{ + int vsec = pos; + u16 vendor, id; + + while ((vsec = pci_find_next_ext_capability(dev, vsec, + OCXL_EXT_CAP_ID_DVSEC))) { + pci_read_config_word(dev, vsec + OCXL_DVSEC_VENDOR_OFFSET, + &vendor); + pci_read_config_word(dev, vsec + OCXL_DVSEC_ID_OFFSET, &id); + if (vendor == PCI_VENDOR_ID_IBM && id == dvsec_id) + return vsec; + } + return 0; +} + +static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx) +{ + int vsec = 0; + u8 idx; + + while ((vsec = find_dvsec_from_pos(dev, OCXL_DVSEC_AFU_CTRL_ID, + vsec))) { + pci_read_config_byte(dev, vsec + OCXL_DVSEC_AFU_CTRL_AFU_IDX, + &idx); + if (idx == afu_idx) + return vsec; + } + return 0; +} + +static int get_max_afu_index(struct pci_dev *dev, int *afu_idx) +{ + int pos; + u32 val; + + pos = find_dvsec_from_pos(dev, OCXL_DVSEC_FUNC_ID, 0); + if (!pos) + return -ESRCH; + + pci_read_config_dword(dev, pos + OCXL_DVSEC_FUNC_OFF_INDEX, &val); + if (val & AFU_PRESENT) + *afu_idx = (val & AFU_INDEX_MASK) >> AFU_INDEX_SHIFT; + else + *afu_idx = -1; + return 0; +} + +static int get_actag_count(struct pci_dev *dev, int afu_idx, int *actag) +{ + int pos; + u16 actag_sup; + + pos = find_dvsec_afu_ctrl(dev, afu_idx); + if (!pos) + return -ESRCH; + + pci_read_config_word(dev, pos + OCXL_DVSEC_AFU_CTRL_ACTAG_SUP, + &actag_sup); + *actag = actag_sup & ACTAG_MASK; + return 0; +} + +static struct npu_link *find_link(struct pci_dev *dev) +{ + struct npu_link *link; + + list_for_each_entry(link, &links_list, list) { + /* The functions of a device all share the same link */ + if (link->domain == pci_domain_nr(dev->bus) && + link->bus == dev->bus->number && + link->dev == PCI_SLOT(dev->devfn)) { + return link; + } + } + + /* link doesn't exist yet. Allocate one */ + link = kzalloc(sizeof(struct npu_link), GFP_KERNEL); + if (!link) + return NULL; + link->domain = pci_domain_nr(dev->bus); + link->bus = dev->bus->number; + link->dev = PCI_SLOT(dev->devfn); + list_add(&link->list, &links_list); + return link; +} + +static void pnv_ocxl_fixup_actag(struct pci_dev *dev) +{ + struct pci_controller *hose = pci_bus_to_host(dev->bus); + struct pnv_phb *phb = hose->private_data; + struct npu_link *link; + int rc, afu_idx = -1, i, actag; + + if (!machine_is(powernv)) + return; + + if (phb->type != PNV_PHB_NPU_OCAPI) + return; + + mutex_lock(&links_list_lock); + + link = find_link(dev); + if (!link) { + dev_warn(&dev->dev, "couldn't update actag information\n"); + mutex_unlock(&links_list_lock); + return; + } + + /* + * Check how many actags are desired for the AFUs under that + * function and add it to the count for the link + */ + rc = get_max_afu_index(dev, &afu_idx); + if (rc) { + /* Most likely an invalid config space */ + dev_dbg(&dev->dev, "couldn't find AFU information\n"); + afu_idx = -1; + } + + link->fn_desired_actags[PCI_FUNC(dev->devfn)] = 0; + for (i = 0; i <= afu_idx; i++) { + /* + * AFU index 'holes' are allowed. So don't fail if we + * can't read the actag info for an index + */ + rc = get_actag_count(dev, i, &actag); + if (rc) + continue; + link->fn_desired_actags[PCI_FUNC(dev->devfn)] += actag; + } + dev_dbg(&dev->dev, "total actags for function: %d\n", + link->fn_desired_actags[PCI_FUNC(dev->devfn)]); + + mutex_unlock(&links_list_lock); +} +DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pnv_ocxl_fixup_actag); + +static u16 assign_fn_actags(u16 desired, u16 total) +{ + u16 count; + + if (total <= PNV_OCXL_ACTAG_MAX) + count = desired; + else + count = PNV_OCXL_ACTAG_MAX * desired / total; + + return count; +} + +static void assign_actags(struct npu_link *link) +{ + u16 actag_count, range_start = 0, total_desired = 0; + int i; + + for (i = 0; i < 8; i++) + total_desired += link->fn_desired_actags[i]; + + for (i = 0; i < 8; i++) { + if (link->fn_desired_actags[i]) { + actag_count = assign_fn_actags( + link->fn_desired_actags[i], + total_desired); + link->fn_actags[i].start = range_start; + link->fn_actags[i].count = actag_count; + range_start += actag_count; + WARN_ON(range_start >= PNV_OCXL_ACTAG_MAX); + } + pr_debug("link %x:%x:%x fct %d actags: start=%d count=%d (desired=%d)\n", + link->domain, link->bus, link->dev, i, + link->fn_actags[i].start, link->fn_actags[i].count, + link->fn_desired_actags[i]); + } + link->assignment_done = true; +} + +int pnv_ocxl_get_actag(struct pci_dev *dev, u16 *base, u16 *enabled, + u16 *supported) +{ + struct npu_link *link; + + mutex_lock(&links_list_lock); + + link = find_link(dev); + if (!link) { + dev_err(&dev->dev, "actag information not found\n"); + mutex_unlock(&links_list_lock); + return -ENODEV; + } + /* + * On p9, we only have 64 actags per link, so they must be + * shared by all the functions of the same adapter. We counted + * the desired actag counts during PCI enumeration, so that we + * can allocate a pro-rated number of actags to each function. + */ + if (!link->assignment_done) + assign_actags(link); + + *base = link->fn_actags[PCI_FUNC(dev->devfn)].start; + *enabled = link->fn_actags[PCI_FUNC(dev->devfn)].count; + *supported = link->fn_desired_actags[PCI_FUNC(dev->devfn)]; + + mutex_unlock(&links_list_lock); + return 0; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_get_actag); + +int pnv_ocxl_get_pasid_count(struct pci_dev *dev, int *count) +{ + struct npu_link *link; + int i, rc = -EINVAL; + + /* + * The number of PASIDs (process address space ID) which can + * be used by a function depends on how many functions exist + * on the device. The NPU needs to be configured to know how + * many bits are available to PASIDs and how many are to be + * used by the function BDF indentifier. + * + * We only support one AFU-carrying function for now. + */ + mutex_lock(&links_list_lock); + + link = find_link(dev); + if (!link) { + dev_err(&dev->dev, "actag information not found\n"); + mutex_unlock(&links_list_lock); + return -ENODEV; + } + + for (i = 0; i < 8; i++) + if (link->fn_desired_actags[i] && (i == PCI_FUNC(dev->devfn))) { + *count = PNV_OCXL_PASID_MAX; + rc = 0; + break; + } + + mutex_unlock(&links_list_lock); + dev_dbg(&dev->dev, "%d PASIDs available for function\n", + rc ? 0 : *count); + return rc; +} +EXPORT_SYMBOL_GPL(pnv_ocxl_get_pasid_count); static void set_templ_rate(unsigned int templ, unsigned int rate, char *buf) { diff --git a/include/misc/ocxl-config.h b/include/misc/ocxl-config.h new file mode 100644 index 000000000000..3526fa996a22 --- /dev/null +++ b/include/misc/ocxl-config.h @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2017 IBM Corp. +#ifndef _OCXL_CONFIG_H_ +#define _OCXL_CONFIG_H_ + +/* + * This file lists the various constants used to read the + * configuration space of an opencapi adapter. + * + * It follows the specification for opencapi 3.0 + */ + +#define OCXL_EXT_CAP_ID_DVSEC 0x23 + +#define OCXL_DVSEC_VENDOR_OFFSET 0x4 +#define OCXL_DVSEC_ID_OFFSET 0x8 +#define OCXL_DVSEC_TL_ID 0xF000 +#define OCXL_DVSEC_TL_BACKOFF_TIMERS 0x10 +#define OCXL_DVSEC_TL_RECV_CAP 0x18 +#define OCXL_DVSEC_TL_SEND_CAP 0x20 +#define OCXL_DVSEC_TL_RECV_RATE 0x30 +#define OCXL_DVSEC_TL_SEND_RATE 0x50 +#define OCXL_DVSEC_FUNC_ID 0xF001 +#define OCXL_DVSEC_FUNC_OFF_INDEX 0x08 +#define OCXL_DVSEC_FUNC_OFF_ACTAG 0x0C +#define OCXL_DVSEC_AFU_INFO_ID 0xF003 +#define OCXL_DVSEC_AFU_INFO_AFU_IDX 0x0A +#define OCXL_DVSEC_AFU_INFO_OFF 0x0C +#define OCXL_DVSEC_AFU_INFO_DATA 0x10 +#define OCXL_DVSEC_AFU_CTRL_ID 0xF004 +#define OCXL_DVSEC_AFU_CTRL_AFU_IDX 0x0A +#define OCXL_DVSEC_AFU_CTRL_TERM_PASID 0x0C +#define OCXL_DVSEC_AFU_CTRL_ENABLE 0x0F +#define OCXL_DVSEC_AFU_CTRL_PASID_SUP 0x10 +#define OCXL_DVSEC_AFU_CTRL_PASID_EN 0x11 +#define OCXL_DVSEC_AFU_CTRL_PASID_BASE 0x14 +#define OCXL_DVSEC_AFU_CTRL_ACTAG_SUP 0x18 +#define OCXL_DVSEC_AFU_CTRL_ACTAG_EN 0x1A +#define OCXL_DVSEC_AFU_CTRL_ACTAG_BASE 0x1C +#define OCXL_DVSEC_VENDOR_ID 0xF0F0 +#define OCXL_DVSEC_VENDOR_CFG_VERS 0x0C +#define OCXL_DVSEC_VENDOR_TLX_VERS 0x10 +#define OCXL_DVSEC_VENDOR_DLX_VERS 0x20 + +#endif /* _OCXL_CONFIG_H_ */ -- cgit v1.2.3 From aeddad1760aeb206d912b27b230269407efd5b06 Mon Sep 17 00:00:00 2001 From: Frederic Barrat Date: Tue, 23 Jan 2018 12:31:42 +0100 Subject: ocxl: Add AFU interrupt support Add user APIs through ioctl to allocate, free, and be notified of an AFU interrupt. For opencapi, an AFU can trigger an interrupt on the host by sending a specific command targeting a 64-bit object handle. On POWER9, this is implemented by mapping a special page in the address space of a process and a write to that page will trigger an interrupt. Signed-off-by: Frederic Barrat Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pnv-ocxl.h | 3 + arch/powerpc/platforms/powernv/ocxl.c | 30 ++++++ drivers/misc/ocxl/afu_irq.c | 197 ++++++++++++++++++++++++++++++++++ drivers/misc/ocxl/context.c | 51 ++++++++- drivers/misc/ocxl/file.c | 34 ++++++ drivers/misc/ocxl/link.c | 28 +++++ drivers/misc/ocxl/ocxl_internal.h | 7 ++ include/uapi/misc/ocxl.h | 9 ++ 8 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 drivers/misc/ocxl/afu_irq.c (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pnv-ocxl.h b/arch/powerpc/include/asm/pnv-ocxl.h index 398d05b30600..f6945d3bc971 100644 --- a/arch/powerpc/include/asm/pnv-ocxl.h +++ b/arch/powerpc/include/asm/pnv-ocxl.h @@ -30,4 +30,7 @@ extern int pnv_ocxl_spa_setup(struct pci_dev *dev, void *spa_mem, int PE_mask, extern void pnv_ocxl_spa_release(void *platform_data); extern int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle); +extern int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr); +extern void pnv_ocxl_free_xive_irq(u32 irq); + #endif /* _ASM_PNV_OCXL_H */ diff --git a/arch/powerpc/platforms/powernv/ocxl.c b/arch/powerpc/platforms/powernv/ocxl.c index 1faaa4ef6903..fa9b53af3c7b 100644 --- a/arch/powerpc/platforms/powernv/ocxl.c +++ b/arch/powerpc/platforms/powernv/ocxl.c @@ -2,6 +2,7 @@ // Copyright 2017 IBM Corp. #include #include +#include #include #include "pci.h" @@ -483,3 +484,32 @@ int pnv_ocxl_spa_remove_pe(void *platform_data, int pe_handle) return rc; } EXPORT_SYMBOL_GPL(pnv_ocxl_spa_remove_pe); + +int pnv_ocxl_alloc_xive_irq(u32 *irq, u64 *trigger_addr) +{ + __be64 flags, trigger_page; + s64 rc; + u32 hwirq; + + hwirq = xive_native_alloc_irq(); + if (!hwirq) + return -ENOENT; + + rc = opal_xive_get_irq_info(hwirq, &flags, NULL, &trigger_page, NULL, + NULL); + if (rc || !trigger_page) { + xive_native_free_irq(hwirq); + return -ENOENT; + } + *irq = hwirq; + *trigger_addr = be64_to_cpu(trigger_page); + return 0; + +} +EXPORT_SYMBOL_GPL(pnv_ocxl_alloc_xive_irq); + +void pnv_ocxl_free_xive_irq(u32 irq) +{ + xive_native_free_irq(irq); +} +EXPORT_SYMBOL_GPL(pnv_ocxl_free_xive_irq); diff --git a/drivers/misc/ocxl/afu_irq.c b/drivers/misc/ocxl/afu_irq.c new file mode 100644 index 000000000000..f40d853de401 --- /dev/null +++ b/drivers/misc/ocxl/afu_irq.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2017 IBM Corp. +#include +#include +#include +#include "ocxl_internal.h" + +struct afu_irq { + int id; + int hw_irq; + unsigned int virq; + char *name; + u64 trigger_page; + struct eventfd_ctx *ev_ctx; +}; + +static int irq_offset_to_id(struct ocxl_context *ctx, u64 offset) +{ + return (offset - ctx->afu->irq_base_offset) >> PAGE_SHIFT; +} + +static u64 irq_id_to_offset(struct ocxl_context *ctx, int id) +{ + return ctx->afu->irq_base_offset + (id << PAGE_SHIFT); +} + +static irqreturn_t afu_irq_handler(int virq, void *data) +{ + struct afu_irq *irq = (struct afu_irq *) data; + + if (irq->ev_ctx) + eventfd_signal(irq->ev_ctx, 1); + return IRQ_HANDLED; +} + +static int setup_afu_irq(struct ocxl_context *ctx, struct afu_irq *irq) +{ + int rc; + + irq->virq = irq_create_mapping(NULL, irq->hw_irq); + if (!irq->virq) { + pr_err("irq_create_mapping failed\n"); + return -ENOMEM; + } + pr_debug("hw_irq %d mapped to virq %u\n", irq->hw_irq, irq->virq); + + irq->name = kasprintf(GFP_KERNEL, "ocxl-afu-%u", irq->virq); + if (!irq->name) { + irq_dispose_mapping(irq->virq); + return -ENOMEM; + } + + rc = request_irq(irq->virq, afu_irq_handler, 0, irq->name, irq); + if (rc) { + kfree(irq->name); + irq->name = NULL; + irq_dispose_mapping(irq->virq); + pr_err("request_irq failed: %d\n", rc); + return rc; + } + return 0; +} + +static void release_afu_irq(struct afu_irq *irq) +{ + free_irq(irq->virq, irq); + irq_dispose_mapping(irq->virq); + kfree(irq->name); +} + +int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset) +{ + struct afu_irq *irq; + int rc; + + irq = kzalloc(sizeof(struct afu_irq), GFP_KERNEL); + if (!irq) + return -ENOMEM; + + /* + * We limit the number of afu irqs per context and per link to + * avoid a single process or user depleting the pool of IPIs + */ + + mutex_lock(&ctx->irq_lock); + + irq->id = idr_alloc(&ctx->irq_idr, irq, 0, MAX_IRQ_PER_CONTEXT, + GFP_KERNEL); + if (irq->id < 0) { + rc = -ENOSPC; + goto err_unlock; + } + + rc = ocxl_link_irq_alloc(ctx->afu->fn->link, &irq->hw_irq, + &irq->trigger_page); + if (rc) + goto err_idr; + + rc = setup_afu_irq(ctx, irq); + if (rc) + goto err_alloc; + + *irq_offset = irq_id_to_offset(ctx, irq->id); + + mutex_unlock(&ctx->irq_lock); + return 0; + +err_alloc: + ocxl_link_free_irq(ctx->afu->fn->link, irq->hw_irq); +err_idr: + idr_remove(&ctx->irq_idr, irq->id); +err_unlock: + mutex_unlock(&ctx->irq_lock); + kfree(irq); + return rc; +} + +static void afu_irq_free(struct afu_irq *irq, struct ocxl_context *ctx) +{ + if (ctx->mapping) + unmap_mapping_range(ctx->mapping, + irq_id_to_offset(ctx, irq->id), + 1 << PAGE_SHIFT, 1); + release_afu_irq(irq); + if (irq->ev_ctx) + eventfd_ctx_put(irq->ev_ctx); + ocxl_link_free_irq(ctx->afu->fn->link, irq->hw_irq); + kfree(irq); +} + +int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset) +{ + struct afu_irq *irq; + int id = irq_offset_to_id(ctx, irq_offset); + + mutex_lock(&ctx->irq_lock); + + irq = idr_find(&ctx->irq_idr, id); + if (!irq) { + mutex_unlock(&ctx->irq_lock); + return -EINVAL; + } + idr_remove(&ctx->irq_idr, irq->id); + afu_irq_free(irq, ctx); + mutex_unlock(&ctx->irq_lock); + return 0; +} + +void ocxl_afu_irq_free_all(struct ocxl_context *ctx) +{ + struct afu_irq *irq; + int id; + + mutex_lock(&ctx->irq_lock); + idr_for_each_entry(&ctx->irq_idr, irq, id) + afu_irq_free(irq, ctx); + mutex_unlock(&ctx->irq_lock); +} + +int ocxl_afu_irq_set_fd(struct ocxl_context *ctx, u64 irq_offset, int eventfd) +{ + struct afu_irq *irq; + struct eventfd_ctx *ev_ctx; + int rc = 0, id = irq_offset_to_id(ctx, irq_offset); + + mutex_lock(&ctx->irq_lock); + irq = idr_find(&ctx->irq_idr, id); + if (!irq) { + rc = -EINVAL; + goto unlock; + } + + ev_ctx = eventfd_ctx_fdget(eventfd); + if (IS_ERR(ev_ctx)) { + rc = -EINVAL; + goto unlock; + } + + irq->ev_ctx = ev_ctx; +unlock: + mutex_unlock(&ctx->irq_lock); + return rc; +} + +u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, u64 irq_offset) +{ + struct afu_irq *irq; + int id = irq_offset_to_id(ctx, irq_offset); + u64 addr = 0; + + mutex_lock(&ctx->irq_lock); + irq = idr_find(&ctx->irq_idr, id); + if (irq) + addr = irq->trigger_page; + mutex_unlock(&ctx->irq_lock); + return addr; +} diff --git a/drivers/misc/ocxl/context.c b/drivers/misc/ocxl/context.c index b34b836f924c..269149490063 100644 --- a/drivers/misc/ocxl/context.c +++ b/drivers/misc/ocxl/context.c @@ -31,6 +31,8 @@ int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu, mutex_init(&ctx->mapping_lock); init_waitqueue_head(&ctx->events_wq); mutex_init(&ctx->xsl_error_lock); + mutex_init(&ctx->irq_lock); + idr_init(&ctx->irq_idr); /* * Keep a reference on the AFU to make sure it's valid for the * duration of the life of the context @@ -80,6 +82,19 @@ out: return rc; } +static int map_afu_irq(struct vm_area_struct *vma, unsigned long address, + u64 offset, struct ocxl_context *ctx) +{ + u64 trigger_addr; + + trigger_addr = ocxl_afu_irq_get_addr(ctx, offset); + if (!trigger_addr) + return VM_FAULT_SIGBUS; + + vm_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT); + return VM_FAULT_NOPAGE; +} + static int map_pp_mmio(struct vm_area_struct *vma, unsigned long address, u64 offset, struct ocxl_context *ctx) { @@ -118,7 +133,10 @@ static int ocxl_mmap_fault(struct vm_fault *vmf) pr_debug("%s: pasid %d address 0x%lx offset 0x%llx\n", __func__, ctx->pasid, vmf->address, offset); - rc = map_pp_mmio(vma, vmf->address, offset, ctx); + if (offset < ctx->afu->irq_base_offset) + rc = map_pp_mmio(vma, vmf->address, offset, ctx); + else + rc = map_afu_irq(vma, vmf->address, offset, ctx); return rc; } @@ -126,6 +144,30 @@ static const struct vm_operations_struct ocxl_vmops = { .fault = ocxl_mmap_fault, }; +static int check_mmap_afu_irq(struct ocxl_context *ctx, + struct vm_area_struct *vma) +{ + /* only one page */ + if (vma_pages(vma) != 1) + return -EINVAL; + + /* check offset validty */ + if (!ocxl_afu_irq_get_addr(ctx, vma->vm_pgoff << PAGE_SHIFT)) + return -EINVAL; + + /* + * trigger page should only be accessible in write mode. + * + * It's a bit theoretical, as a page mmaped with only + * PROT_WRITE is currently readable, but it doesn't hurt. + */ + if ((vma->vm_flags & VM_READ) || (vma->vm_flags & VM_EXEC) || + !(vma->vm_flags & VM_WRITE)) + return -EINVAL; + vma->vm_flags &= ~(VM_MAYREAD | VM_MAYEXEC); + return 0; +} + static int check_mmap_mmio(struct ocxl_context *ctx, struct vm_area_struct *vma) { @@ -139,7 +181,10 @@ int ocxl_context_mmap(struct ocxl_context *ctx, struct vm_area_struct *vma) { int rc; - rc = check_mmap_mmio(ctx, vma); + if ((vma->vm_pgoff << PAGE_SHIFT) < ctx->afu->irq_base_offset) + rc = check_mmap_mmio(ctx, vma); + else + rc = check_mmap_afu_irq(ctx, vma); if (rc) return rc; @@ -224,6 +269,8 @@ void ocxl_context_free(struct ocxl_context *ctx) idr_remove(&ctx->afu->contexts_idr, ctx->pasid); mutex_unlock(&ctx->afu->contexts_lock); + ocxl_afu_irq_free_all(ctx); + idr_destroy(&ctx->irq_idr); /* reference to the AFU taken in ocxl_context_init */ ocxl_afu_put(ctx->afu); kfree(ctx); diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c index 6f0befda6a8a..c90c1a578d2f 100644 --- a/drivers/misc/ocxl/file.c +++ b/drivers/misc/ocxl/file.c @@ -103,12 +103,17 @@ static long afu_ioctl_attach(struct ocxl_context *ctx, } #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \ + x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \ + x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \ + x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \ "UNKNOWN") static long afu_ioctl(struct file *file, unsigned int cmd, unsigned long args) { struct ocxl_context *ctx = file->private_data; + struct ocxl_ioctl_irq_fd irq_fd; + u64 irq_offset; long rc; pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, @@ -123,6 +128,35 @@ static long afu_ioctl(struct file *file, unsigned int cmd, (struct ocxl_ioctl_attach __user *) args); break; + case OCXL_IOCTL_IRQ_ALLOC: + rc = ocxl_afu_irq_alloc(ctx, &irq_offset); + if (!rc) { + rc = copy_to_user((u64 __user *) args, &irq_offset, + sizeof(irq_offset)); + if (rc) + ocxl_afu_irq_free(ctx, irq_offset); + } + break; + + case OCXL_IOCTL_IRQ_FREE: + rc = copy_from_user(&irq_offset, (u64 __user *) args, + sizeof(irq_offset)); + if (rc) + return -EFAULT; + rc = ocxl_afu_irq_free(ctx, irq_offset); + break; + + case OCXL_IOCTL_IRQ_SET_FD: + rc = copy_from_user(&irq_fd, (u64 __user *) args, + sizeof(irq_fd)); + if (rc) + return -EFAULT; + if (irq_fd.reserved) + return -EINVAL; + rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, + irq_fd.eventfd); + break; + default: rc = -EINVAL; } diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c index 64d7a98c904a..8bdcef9c3cba 100644 --- a/drivers/misc/ocxl/link.c +++ b/drivers/misc/ocxl/link.c @@ -601,3 +601,31 @@ unlock: mutex_unlock(&spa->spa_lock); return rc; } + +int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr) +{ + struct link *link = (struct link *) link_handle; + int rc, irq; + u64 addr; + + if (atomic_dec_if_positive(&link->irq_available) < 0) + return -ENOSPC; + + rc = pnv_ocxl_alloc_xive_irq(&irq, &addr); + if (rc) { + atomic_inc(&link->irq_available); + return rc; + } + + *hw_irq = irq; + *trigger_addr = addr; + return 0; +} + +void ocxl_link_free_irq(void *link_handle, int hw_irq) +{ + struct link *link = (struct link *) link_handle; + + pnv_ocxl_free_xive_irq(hw_irq); + atomic_inc(&link->irq_available); +} diff --git a/drivers/misc/ocxl/ocxl_internal.h b/drivers/misc/ocxl/ocxl_internal.h index 04fc160c7bd5..a89b88ac67eb 100644 --- a/drivers/misc/ocxl/ocxl_internal.h +++ b/drivers/misc/ocxl/ocxl_internal.h @@ -190,4 +190,11 @@ extern void ocxl_context_free(struct ocxl_context *ctx); extern int ocxl_sysfs_add_afu(struct ocxl_afu *afu); extern void ocxl_sysfs_remove_afu(struct ocxl_afu *afu); +extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset); +extern int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset); +extern void ocxl_afu_irq_free_all(struct ocxl_context *ctx); +extern int ocxl_afu_irq_set_fd(struct ocxl_context *ctx, u64 irq_offset, + int eventfd); +extern u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, u64 irq_offset); + #endif /* _OCXL_INTERNAL_H_ */ diff --git a/include/uapi/misc/ocxl.h b/include/uapi/misc/ocxl.h index a37e34edf52f..4b0b0b756f3e 100644 --- a/include/uapi/misc/ocxl.h +++ b/include/uapi/misc/ocxl.h @@ -32,9 +32,18 @@ struct ocxl_ioctl_attach { __u64 reserved3; }; +struct ocxl_ioctl_irq_fd { + __u64 irq_offset; + __s32 eventfd; + __u32 reserved; +}; + /* ioctl numbers */ #define OCXL_MAGIC 0xCA /* AFU devices */ #define OCXL_IOCTL_ATTACH _IOW(OCXL_MAGIC, 0x10, struct ocxl_ioctl_attach) +#define OCXL_IOCTL_IRQ_ALLOC _IOR(OCXL_MAGIC, 0x11, __u64) +#define OCXL_IOCTL_IRQ_FREE _IOW(OCXL_MAGIC, 0x12, __u64) +#define OCXL_IOCTL_IRQ_SET_FD _IOW(OCXL_MAGIC, 0x13, struct ocxl_ioctl_irq_fd) #endif /* _UAPI_MISC_OCXL_H */ -- cgit v1.2.3 From 64ba3dc7bf7cb0cbc89821db54edfe5180726fbf Mon Sep 17 00:00:00 2001 From: "Bryant G. Ly" Date: Fri, 5 Jan 2018 10:45:46 -0600 Subject: powerpc/eeh: Update VF config space after EEH Add EEH platform operations for pseries to update VF config space. With this change after EEH, the VF will have updated config space for pseries platform. Signed-off-by: Bryant G. Ly Signed-off-by: Juan J. Alvarez Acked-by: Russell Currey Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/eeh.h | 1 + arch/powerpc/kernel/eeh.c | 59 +++++++++++++++++++++++++ arch/powerpc/platforms/powernv/eeh-powernv.c | 65 ++-------------------------- arch/powerpc/platforms/pseries/eeh_pseries.c | 26 ++++++++++- 4 files changed, 88 insertions(+), 63 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 5161c37dd039..82829c65f31a 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -297,6 +297,7 @@ int eeh_pe_reset(struct eeh_pe *pe, int option); int eeh_pe_configure(struct eeh_pe *pe); int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, unsigned long addr, unsigned long mask); +int eeh_restore_vf_config(struct pci_dn *pdn); /** * EEH_POSSIBLE_ERROR() -- test for possible MMIO failure. diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index cbca0a667682..cc649809885e 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -740,6 +740,65 @@ static void *eeh_restore_dev_state(void *data, void *userdata) return NULL; } +int eeh_restore_vf_config(struct pci_dn *pdn) +{ + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); + u32 devctl, cmd, cap2, aer_capctl; + int old_mps; + + if (edev->pcie_cap) { + /* Restore MPS */ + old_mps = (ffs(pdn->mps) - 8) << 5; + eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, + 2, &devctl); + devctl &= ~PCI_EXP_DEVCTL_PAYLOAD; + devctl |= old_mps; + eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, + 2, devctl); + + /* Disable Completion Timeout */ + eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP2, + 4, &cap2); + if (cap2 & 0x10) { + eeh_ops->read_config(pdn, + edev->pcie_cap + PCI_EXP_DEVCTL2, + 4, &cap2); + cap2 |= 0x10; + eeh_ops->write_config(pdn, + edev->pcie_cap + PCI_EXP_DEVCTL2, + 4, cap2); + } + } + + /* Enable SERR and parity checking */ + eeh_ops->read_config(pdn, PCI_COMMAND, 2, &cmd); + cmd |= (PCI_COMMAND_PARITY | PCI_COMMAND_SERR); + eeh_ops->write_config(pdn, PCI_COMMAND, 2, cmd); + + /* Enable report various errors */ + if (edev->pcie_cap) { + eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, + 2, &devctl); + devctl &= ~PCI_EXP_DEVCTL_CERE; + devctl |= (PCI_EXP_DEVCTL_NFERE | + PCI_EXP_DEVCTL_FERE | + PCI_EXP_DEVCTL_URRE); + eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, + 2, devctl); + } + + /* Enable ECRC generation and check */ + if (edev->pcie_cap && edev->aer_cap) { + eeh_ops->read_config(pdn, edev->aer_cap + PCI_ERR_CAP, + 4, &aer_capctl); + aer_capctl |= (PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); + eeh_ops->write_config(pdn, edev->aer_cap + PCI_ERR_CAP, + 4, aer_capctl); + } + + return 0; +} + /** * pcibios_set_pcie_reset_state - Set PCI-E reset state * @dev: pci device struct diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c index 961e64115d92..0665b6d03cb3 100644 --- a/arch/powerpc/platforms/powernv/eeh-powernv.c +++ b/arch/powerpc/platforms/powernv/eeh-powernv.c @@ -1655,70 +1655,11 @@ static int pnv_eeh_next_error(struct eeh_pe **pe) return ret; } -static int pnv_eeh_restore_vf_config(struct pci_dn *pdn) -{ - struct eeh_dev *edev = pdn_to_eeh_dev(pdn); - u32 devctl, cmd, cap2, aer_capctl; - int old_mps; - - if (edev->pcie_cap) { - /* Restore MPS */ - old_mps = (ffs(pdn->mps) - 8) << 5; - eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, - 2, &devctl); - devctl &= ~PCI_EXP_DEVCTL_PAYLOAD; - devctl |= old_mps; - eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, - 2, devctl); - - /* Disable Completion Timeout */ - eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP2, - 4, &cap2); - if (cap2 & 0x10) { - eeh_ops->read_config(pdn, - edev->pcie_cap + PCI_EXP_DEVCTL2, - 4, &cap2); - cap2 |= 0x10; - eeh_ops->write_config(pdn, - edev->pcie_cap + PCI_EXP_DEVCTL2, - 4, cap2); - } - } - - /* Enable SERR and parity checking */ - eeh_ops->read_config(pdn, PCI_COMMAND, 2, &cmd); - cmd |= (PCI_COMMAND_PARITY | PCI_COMMAND_SERR); - eeh_ops->write_config(pdn, PCI_COMMAND, 2, cmd); - - /* Enable report various errors */ - if (edev->pcie_cap) { - eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, - 2, &devctl); - devctl &= ~PCI_EXP_DEVCTL_CERE; - devctl |= (PCI_EXP_DEVCTL_NFERE | - PCI_EXP_DEVCTL_FERE | - PCI_EXP_DEVCTL_URRE); - eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL, - 2, devctl); - } - - /* Enable ECRC generation and check */ - if (edev->pcie_cap && edev->aer_cap) { - eeh_ops->read_config(pdn, edev->aer_cap + PCI_ERR_CAP, - 4, &aer_capctl); - aer_capctl |= (PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); - eeh_ops->write_config(pdn, edev->aer_cap + PCI_ERR_CAP, - 4, aer_capctl); - } - - return 0; -} - static int pnv_eeh_restore_config(struct pci_dn *pdn) { struct eeh_dev *edev = pdn_to_eeh_dev(pdn); struct pnv_phb *phb; - s64 ret; + s64 ret = 0; int config_addr = (pdn->busno << 8) | (pdn->devfn); if (!edev) @@ -1732,7 +1673,7 @@ static int pnv_eeh_restore_config(struct pci_dn *pdn) * to be exported by firmware in extendible way. */ if (edev->physfn) { - ret = pnv_eeh_restore_vf_config(pdn); + ret = eeh_restore_vf_config(pdn); } else { phb = pdn->phb->private_data; ret = opal_pci_reinit(phb->opal_id, @@ -1745,7 +1686,7 @@ static int pnv_eeh_restore_config(struct pci_dn *pdn) return -EIO; } - return 0; + return ret; } static struct eeh_ops pnv_eeh_ops = { diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c index 2295f117e2d3..a671ef4f57f5 100644 --- a/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -708,6 +708,30 @@ static int pseries_eeh_write_config(struct pci_dn *pdn, int where, int size, u32 return rtas_write_config(pdn, where, size, val); } +static int pseries_eeh_restore_config(struct pci_dn *pdn) +{ + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); + s64 ret = 0; + + if (!edev) + return -EEXIST; + + /* + * FIXME: The MPS, error routing rules, timeout setting are worthy + * to be exported by firmware in extendible way. + */ + if (edev->physfn) + ret = eeh_restore_vf_config(pdn); + + if (ret) { + pr_warn("%s: Can't reinit PCI dev 0x%x (%lld)\n", + __func__, edev->pe_config_addr, ret); + return -EIO; + } + + return ret; +} + static struct eeh_ops pseries_eeh_ops = { .name = "pseries", .init = pseries_eeh_init, @@ -723,7 +747,7 @@ static struct eeh_ops pseries_eeh_ops = { .read_config = pseries_eeh_read_config, .write_config = pseries_eeh_write_config, .next_error = NULL, - .restore_config = NULL + .restore_config = pseries_eeh_restore_config }; /** -- cgit v1.2.3 From 565a744dd2bedde8a8abe8827748bdceb20444ee Mon Sep 17 00:00:00 2001 From: "Bryant G. Ly" Date: Fri, 5 Jan 2018 10:45:48 -0600 Subject: powerpc/pseries: Set eeh_pe of EEH_PE_VF type To correctly use EEH code one has to make sure that the EEH_PE_VF is set for dynamic created VFs. Therefore this patch allocates an eeh_pe of eeh type EEH_PE_VF and associates PE with parent. Signed-off-by: Bryant G. Ly Signed-off-by: Juan J. Alvarez Acked-by: Russell Currey Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pci-bridge.h | 3 ++- arch/powerpc/platforms/pseries/eeh_pseries.c | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h index 9f66ddebb799..b98a44667333 100644 --- a/arch/powerpc/include/asm/pci-bridge.h +++ b/arch/powerpc/include/asm/pci-bridge.h @@ -211,7 +211,8 @@ struct pci_dn { unsigned int *pe_num_map; /* PE# for the first VF PE or array */ bool m64_single_mode; /* Use M64 BAR in Single Mode */ #define IODA_INVALID_M64 (-1) - int (*m64_map)[PCI_SRIOV_NUM_BARS]; + int (*m64_map)[PCI_SRIOV_NUM_BARS]; /* Only used on powernv */ + int last_allow_rc; /* Only used on pseries */ #endif /* CONFIG_PCI_IOV */ int mps; /* Maximum Payload Size */ struct list_head child_list; diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c index a671ef4f57f5..428bfd10bd5a 100644 --- a/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -55,9 +55,12 @@ static int ibm_get_config_addr_info; static int ibm_get_config_addr_info2; static int ibm_configure_pe; +#ifdef CONFIG_PCI_IOV void pseries_pcibios_bus_add_device(struct pci_dev *pdev) { struct pci_dn *pdn = pci_get_pdn(pdev); + struct pci_dn *physfn_pdn; + struct eeh_dev *edev; if (!pdev->is_virtfn) return; @@ -65,6 +68,15 @@ void pseries_pcibios_bus_add_device(struct pci_dev *pdev) pdn->device_id = pdev->device; pdn->vendor_id = pdev->vendor; pdn->class_code = pdev->class; + /* + * Last allow unfreeze return code used for retrieval + * by user space in eeh-sysfs to show the last command + * completion from platform. + */ + pdn->last_allow_rc = 0; + physfn_pdn = pci_get_pdn(pdev->physfn); + pdn->pe_number = physfn_pdn->pe_num_map[pdn->vf_index]; + edev = pdn_to_eeh_dev(pdn); /* * The following operations will fail if VF's sysfs files @@ -72,9 +84,13 @@ void pseries_pcibios_bus_add_device(struct pci_dev *pdev) */ eeh_add_device_early(pdn); eeh_add_device_late(pdev); + edev->pe_config_addr = (pdn->busno << 16) | (pdn->devfn << 8); + eeh_rmv_from_parent_pe(edev); /* Remove as it is adding to bus pe */ + eeh_add_to_parent_pe(edev); /* Add as VF PE type */ eeh_sysfs_add_device(pdev); } +#endif /* * Buffer for reporting slot-error-detail rtas calls. Its here @@ -141,8 +157,10 @@ static int pseries_eeh_init(void) /* Set EEH probe mode */ eeh_add_flag(EEH_PROBE_MODE_DEVTREE | EEH_ENABLE_IO_FOR_LOG); +#ifdef CONFIG_PCI_IOV /* Set EEH machine dependent code */ ppc_md.pcibios_bus_add_device = pseries_pcibios_bus_add_device; +#endif return 0; } -- cgit v1.2.3 From 67923cfcfa1427351bc5fa9d05f1c2a2e7cb9399 Mon Sep 17 00:00:00 2001 From: "Bryant G. Ly" Date: Fri, 5 Jan 2018 10:45:49 -0600 Subject: powerpc/eeh: Add EEH operations to notify resume When pseries SR-IOV is enabled and after a PF driver has resumed from EEH, platform has to be notified of the event so the child VFs can be allowed to resume their normal recovery path. This patch makes the EEH operation allow unfreeze platform dependent code and adds the call to pseries EEH code. Signed-off-by: Bryant G. Ly Signed-off-by: Juan J. Alvarez Acked-by: Russell Currey Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/eeh.h | 1 + arch/powerpc/platforms/powernv/eeh-powernv.c | 3 +- arch/powerpc/platforms/pseries/eeh_pseries.c | 96 +++++++++++++++++++++++++++- 3 files changed, 98 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 82829c65f31a..fd37cc101f4f 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -214,6 +214,7 @@ struct eeh_ops { int (*write_config)(struct pci_dn *pdn, int where, int size, u32 val); int (*next_error)(struct eeh_pe **pe); int (*restore_config)(struct pci_dn *pdn); + int (*notify_resume)(struct pci_dn *pdn); }; extern int eeh_subsystem_flags; diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c index 0665b6d03cb3..33c86c1a1720 100644 --- a/arch/powerpc/platforms/powernv/eeh-powernv.c +++ b/arch/powerpc/platforms/powernv/eeh-powernv.c @@ -1704,7 +1704,8 @@ static struct eeh_ops pnv_eeh_ops = { .read_config = pnv_eeh_read_config, .write_config = pnv_eeh_write_config, .next_error = pnv_eeh_next_error, - .restore_config = pnv_eeh_restore_config + .restore_config = pnv_eeh_restore_config, + .notify_resume = NULL }; #ifdef CONFIG_PCI_IOV diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c index 428bfd10bd5a..823cb27efa8b 100644 --- a/arch/powerpc/platforms/pseries/eeh_pseries.c +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c @@ -750,6 +750,97 @@ static int pseries_eeh_restore_config(struct pci_dn *pdn) return ret; } +#ifdef CONFIG_PCI_IOV +int pseries_send_allow_unfreeze(struct pci_dn *pdn, + u16 *vf_pe_array, int cur_vfs) +{ + int rc; + int ibm_allow_unfreeze = rtas_token("ibm,open-sriov-allow-unfreeze"); + unsigned long buid, addr; + + addr = rtas_config_addr(pdn->busno, pdn->devfn, 0); + buid = pdn->phb->buid; + spin_lock(&rtas_data_buf_lock); + memcpy(rtas_data_buf, vf_pe_array, RTAS_DATA_BUF_SIZE); + rc = rtas_call(ibm_allow_unfreeze, 5, 1, NULL, + addr, + BUID_HI(buid), + BUID_LO(buid), + rtas_data_buf, cur_vfs * sizeof(u16)); + spin_unlock(&rtas_data_buf_lock); + if (rc) + pr_warn("%s: Failed to allow unfreeze for PHB#%x-PE#%lx, rc=%x\n", + __func__, + pdn->phb->global_number, addr, rc); + return rc; +} + +static int pseries_call_allow_unfreeze(struct eeh_dev *edev) +{ + struct pci_dn *pdn, *tmp, *parent, *physfn_pdn; + int cur_vfs = 0, rc = 0, vf_index, bus, devfn; + u16 *vf_pe_array; + + vf_pe_array = kzalloc(RTAS_DATA_BUF_SIZE, GFP_KERNEL); + if (!vf_pe_array) + return -ENOMEM; + if (pci_num_vf(edev->physfn ? edev->physfn : edev->pdev)) { + if (edev->pdev->is_physfn) { + cur_vfs = pci_num_vf(edev->pdev); + pdn = eeh_dev_to_pdn(edev); + parent = pdn->parent; + for (vf_index = 0; vf_index < cur_vfs; vf_index++) + vf_pe_array[vf_index] = + cpu_to_be16(pdn->pe_num_map[vf_index]); + rc = pseries_send_allow_unfreeze(pdn, vf_pe_array, + cur_vfs); + pdn->last_allow_rc = rc; + for (vf_index = 0; vf_index < cur_vfs; vf_index++) { + list_for_each_entry_safe(pdn, tmp, + &parent->child_list, + list) { + bus = pci_iov_virtfn_bus(edev->pdev, + vf_index); + devfn = pci_iov_virtfn_devfn(edev->pdev, + vf_index); + if (pdn->busno != bus || + pdn->devfn != devfn) + continue; + pdn->last_allow_rc = rc; + } + } + } else { + pdn = pci_get_pdn(edev->pdev); + vf_pe_array[0] = cpu_to_be16(pdn->pe_number); + physfn_pdn = pci_get_pdn(edev->physfn); + rc = pseries_send_allow_unfreeze(physfn_pdn, + vf_pe_array, 1); + pdn->last_allow_rc = rc; + } + } + + kfree(vf_pe_array); + return rc; +} + +static int pseries_notify_resume(struct pci_dn *pdn) +{ + struct eeh_dev *edev = pdn_to_eeh_dev(pdn); + + if (!edev) + return -EEXIST; + + if (rtas_token("ibm,open-sriov-allow-unfreeze") + == RTAS_UNKNOWN_SERVICE) + return -EINVAL; + + if (edev->pdev->is_physfn || edev->pdev->is_virtfn) + return pseries_call_allow_unfreeze(edev); + + return 0; +} +#endif + static struct eeh_ops pseries_eeh_ops = { .name = "pseries", .init = pseries_eeh_init, @@ -765,7 +856,10 @@ static struct eeh_ops pseries_eeh_ops = { .read_config = pseries_eeh_read_config, .write_config = pseries_eeh_write_config, .next_error = NULL, - .restore_config = pseries_eeh_restore_config + .restore_config = pseries_eeh_restore_config, +#ifdef CONFIG_PCI_IOV + .notify_resume = pseries_notify_resume +#endif }; /** -- cgit v1.2.3 From fc5f622163637308a3d520a315527481cff023f5 Mon Sep 17 00:00:00 2001 From: "Bryant G. Ly" Date: Fri, 5 Jan 2018 10:45:52 -0600 Subject: powerpc/pseries: Add Initialization of VF Bars When enabling SR-IOV in pseries platform, the VF bar properties for a PF are reported on the device node in the device tree. This patch adds the IOV Bar resources to Linux structures from the device tree for later use when configuring SR-IOV by PF driver. Signed-off-by: Bryant G. Ly Signed-off-by: Juan J. Alvarez Acked-by: Russell Currey Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pci.h | 2 + arch/powerpc/kernel/pci_of_scan.c | 2 +- arch/powerpc/platforms/pseries/setup.c | 164 +++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pci.h b/arch/powerpc/include/asm/pci.h index 8dc32eacc97c..d82802ff5088 100644 --- a/arch/powerpc/include/asm/pci.h +++ b/arch/powerpc/include/asm/pci.h @@ -121,6 +121,8 @@ extern int remove_phb_dynamic(struct pci_controller *phb); extern struct pci_dev *of_create_pci_dev(struct device_node *node, struct pci_bus *bus, int devfn); +extern unsigned int pci_parse_of_flags(u32 addr0, int bridge); + extern void of_scan_pci_bridge(struct pci_dev *dev); extern void of_scan_bus(struct device_node *node, struct pci_bus *bus); diff --git a/arch/powerpc/kernel/pci_of_scan.c b/arch/powerpc/kernel/pci_of_scan.c index 0d790f8432d2..20ceec4a5f5e 100644 --- a/arch/powerpc/kernel/pci_of_scan.c +++ b/arch/powerpc/kernel/pci_of_scan.c @@ -38,7 +38,7 @@ static u32 get_int_prop(struct device_node *np, const char *name, u32 def) * @addr0: value of 1st cell of a device tree PCI address. * @bridge: Set this flag if the address is from a bridge 'ranges' property */ -static unsigned int pci_parse_of_flags(u32 addr0, int bridge) +unsigned int pci_parse_of_flags(u32 addr0, int bridge) { unsigned int flags = 0; diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 42e9cf0b3180..372d7ada1a0c 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -492,6 +492,162 @@ static void pseries_setup_rfi_flush(void) setup_rfi_flush(types, enable); } +#ifdef CONFIG_PCI_IOV +enum rtas_iov_fw_value_map { + NUM_RES_PROPERTY = 0, /* Number of Resources */ + LOW_INT = 1, /* Lowest 32 bits of Address */ + START_OF_ENTRIES = 2, /* Always start of entry */ + APERTURE_PROPERTY = 2, /* Start of entry+ to Aperture Size */ + WDW_SIZE_PROPERTY = 4, /* Start of entry+ to Window Size */ + NEXT_ENTRY = 7 /* Go to next entry on array */ +}; + +enum get_iov_fw_value_index { + BAR_ADDRS = 1, /* Get Bar Address */ + APERTURE_SIZE = 2, /* Get Aperture Size */ + WDW_SIZE = 3 /* Get Window Size */ +}; + +resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno, + enum get_iov_fw_value_index value) +{ + const int *indexes; + struct device_node *dn = pci_device_to_OF_node(dev); + int i, num_res, ret = 0; + + indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL); + if (!indexes) + return 0; + + /* + * First element in the array is the number of Bars + * returned. Search through the list to find the matching + * bar + */ + num_res = of_read_number(&indexes[NUM_RES_PROPERTY], 1); + if (resno >= num_res) + return 0; /* or an errror */ + + i = START_OF_ENTRIES + NEXT_ENTRY * resno; + switch (value) { + case BAR_ADDRS: + ret = of_read_number(&indexes[i], 2); + break; + case APERTURE_SIZE: + ret = of_read_number(&indexes[i + APERTURE_PROPERTY], 2); + break; + case WDW_SIZE: + ret = of_read_number(&indexes[i + WDW_SIZE_PROPERTY], 2); + break; + } + + return ret; +} + +void of_pci_set_vf_bar_size(struct pci_dev *dev, const int *indexes) +{ + struct resource *res; + resource_size_t base, size; + int i, r, num_res; + + num_res = of_read_number(&indexes[NUM_RES_PROPERTY], 1); + num_res = min_t(int, num_res, PCI_SRIOV_NUM_BARS); + for (i = START_OF_ENTRIES, r = 0; r < num_res && r < PCI_SRIOV_NUM_BARS; + i += NEXT_ENTRY, r++) { + res = &dev->resource[r + PCI_IOV_RESOURCES]; + base = of_read_number(&indexes[i], 2); + size = of_read_number(&indexes[i + APERTURE_PROPERTY], 2); + res->flags = pci_parse_of_flags(of_read_number + (&indexes[i + LOW_INT], 1), 0); + res->flags |= (IORESOURCE_MEM_64 | IORESOURCE_PCI_FIXED); + res->name = pci_name(dev); + res->start = base; + res->end = base + size - 1; + } +} + +void of_pci_parse_iov_addrs(struct pci_dev *dev, const int *indexes) +{ + struct resource *res, *root, *conflict; + resource_size_t base, size; + int i, r, num_res; + + /* + * First element in the array is the number of Bars + * returned. Search through the list to find the matching + * bars assign them from firmware into resources structure. + */ + num_res = of_read_number(&indexes[NUM_RES_PROPERTY], 1); + for (i = START_OF_ENTRIES, r = 0; r < num_res && r < PCI_SRIOV_NUM_BARS; + i += NEXT_ENTRY, r++) { + res = &dev->resource[r + PCI_IOV_RESOURCES]; + base = of_read_number(&indexes[i], 2); + size = of_read_number(&indexes[i + WDW_SIZE_PROPERTY], 2); + res->name = pci_name(dev); + res->start = base; + res->end = base + size - 1; + root = &iomem_resource; + dev_dbg(&dev->dev, + "pSeries IOV BAR %d: trying firmware assignment %pR\n", + r + PCI_IOV_RESOURCES, res); + conflict = request_resource_conflict(root, res); + if (conflict) { + dev_info(&dev->dev, + "BAR %d: %pR conflicts with %s %pR\n", + r + PCI_IOV_RESOURCES, res, + conflict->name, conflict); + res->flags |= IORESOURCE_UNSET; + } + } +} + +static void pseries_pci_fixup_resources(struct pci_dev *pdev) +{ + const int *indexes; + struct device_node *dn = pci_device_to_OF_node(pdev); + + /*Firmware must support open sriov otherwise dont configure*/ + indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL); + if (!indexes) + return; + /* Assign the addresses from device tree*/ + of_pci_set_vf_bar_size(pdev, indexes); +} + +static void pseries_pci_fixup_iov_resources(struct pci_dev *pdev) +{ + const int *indexes; + struct device_node *dn = pci_device_to_OF_node(pdev); + + if (!pdev->is_physfn || pdev->is_added) + return; + /*Firmware must support open sriov otherwise dont configure*/ + indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL); + if (!indexes) + return; + /* Assign the addresses from device tree*/ + of_pci_parse_iov_addrs(pdev, indexes); +} + +static resource_size_t pseries_pci_iov_resource_alignment(struct pci_dev *pdev, + int resno) +{ + const __be32 *reg; + struct device_node *dn = pci_device_to_OF_node(pdev); + + /*Firmware must support open sriov otherwise report regular alignment*/ + reg = of_get_property(dn, "ibm,is-open-sriov-pf", NULL); + if (!reg) + return pci_iov_resource_size(pdev, resno); + + if (!pdev->is_physfn) + return 0; + return pseries_get_iov_fw_value(pdev, + resno - PCI_IOV_RESOURCES, + APERTURE_SIZE); +} +#endif + static void __init pSeries_setup_arch(void) { set_arch_panic_timeout(10, ARCH_PANIC_TIMEOUT); @@ -525,6 +681,14 @@ static void __init pSeries_setup_arch(void) vpa_init(boot_cpuid); ppc_md.power_save = pseries_lpar_idle; ppc_md.enable_pmcs = pseries_lpar_enable_pmcs; +#ifdef CONFIG_PCI_IOV + ppc_md.pcibios_fixup_resources = + pseries_pci_fixup_resources; + ppc_md.pcibios_fixup_sriov = + pseries_pci_fixup_iov_resources; + ppc_md.pcibios_iov_resource_alignment = + pseries_pci_iov_resource_alignment; +#endif } else { /* No special idle routine */ ppc_md.enable_pmcs = power4_enable_pmcs; -- cgit v1.2.3 From 902bdc57451c2c64aa139bbe24067f70a186db0a Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Mon, 23 Oct 2017 19:07:02 +1100 Subject: powerpc/powernv/idoa: Remove unnecessary pcidev from pci_dn The pcidev value stored in pci_dn is only used for NPU/NPU2 initialization. We can easily drop the cached pointer and use an ancient helper - pci_get_domain_bus_and_slot() instead in order to reduce complexity. Signed-off-by: Alexey Kardashevskiy Acked-by: Russell Currey Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/pci-bridge.h | 2 -- arch/powerpc/platforms/powernv/npu-dma.c | 5 ++++- arch/powerpc/platforms/powernv/pci-ioda.c | 3 --- 3 files changed, 4 insertions(+), 6 deletions(-) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/pci-bridge.h b/arch/powerpc/include/asm/pci-bridge.h index b98a44667333..94d449031b18 100644 --- a/arch/powerpc/include/asm/pci-bridge.h +++ b/arch/powerpc/include/asm/pci-bridge.h @@ -197,8 +197,6 @@ struct pci_dn { struct iommu_table_group *table_group; /* for phb's or bridges */ int pci_ext_config_space; /* for pci devices */ - - struct pci_dev *pcidev; /* back-pointer to the pci device */ #ifdef CONFIG_EEH struct eeh_dev *edev; /* eeh device */ #endif diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c index c5899c107d59..0a253b64ac5f 100644 --- a/arch/powerpc/platforms/powernv/npu-dma.c +++ b/arch/powerpc/platforms/powernv/npu-dma.c @@ -39,7 +39,10 @@ */ static struct pci_dev *get_pci_dev(struct device_node *dn) { - return PCI_DN(dn)->pcidev; + struct pci_dn *pdn = PCI_DN(dn); + + return pci_get_domain_bus_and_slot(pci_domain_nr(pdn->phb->bus), + pdn->busno, pdn->devfn); } /* Given a NPU device get the associated PCI device. */ diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 886014ed85fe..398cc5e4b91b 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1082,7 +1082,6 @@ static struct pnv_ioda_pe *pnv_ioda_setup_dev_PE(struct pci_dev *dev) * At some point we want to remove the PDN completely anyways */ pci_dev_get(dev); - pdn->pcidev = dev; pdn->pe_number = pe->pe_number; pe->flags = PNV_IODA_PE_DEV; pe->pdev = dev; @@ -1129,7 +1128,6 @@ static void pnv_ioda_setup_same_PE(struct pci_bus *bus, struct pnv_ioda_pe *pe) continue; pe->device_count++; - pdn->pcidev = dev; pdn->pe_number = pe->pe_number; if ((pe->flags & PNV_IODA_PE_BUS_ALL) && dev->subordinate) pnv_ioda_setup_same_PE(dev->subordinate, pe); @@ -1244,7 +1242,6 @@ static struct pnv_ioda_pe *pnv_ioda_setup_npu_PE(struct pci_dev *npu_pdev) pci_dev_get(npu_pdev); npu_pdn = pci_get_pdn(npu_pdev); rid = npu_pdev->bus->number << 8 | npu_pdn->devfn; - npu_pdn->pcidev = npu_pdev; npu_pdn->pe_number = pe_num; phb->ioda.pe_rmap[rid] = pe->pe_number; -- cgit v1.2.3 From 015eb1b89e959c9349f0a01803fb8ed1ced36f09 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Tue, 30 Jan 2018 20:37:21 +1100 Subject: powerpc/mm/radix: Fix build error when RADIX_MMU=n The recent TLB flush rework broke the build when the Radix MMU is disabled at build time, eg: (.text+0x264): undefined reference to `.radix__tlbiel_all' We could add an empty version, but if we ever called it by accident that would indicate a bad bug, so add a stub that just WARNs if we do. Fixes: d4748276ae14 ("powerpc/64s: Improve local TLB flush for boot and MCE on POWER9") Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/tlbflush-radix.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/powerpc/include/asm') diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index b8f9ad587087..8eea90f80e45 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -11,7 +11,11 @@ static inline int mmu_get_ap(int psize) return mmu_psize_defs[psize].ap; } +#ifdef CONFIG_PPC_RADIX_MMU extern void radix__tlbiel_all(unsigned int action); +#else +static inline void radix__tlbiel_all(unsigned int action) { WARN_ON(1); }; +#endif extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); -- cgit v1.2.3