From b6622a339e8670a1025d4dd84be473c76dabed33 Mon Sep 17 00:00:00 2001 From: Sukadev Bhattiprolu Date: Mon, 28 Aug 2017 23:23:32 -0700 Subject: powerpc/powernv: Move GET_FIELD/SET_FIELD to vas.h Move the GET_FIELD and SET_FIELD macros to vas.h as VAS and other users of VAS, including NX-842 can use those macros. There is a lot of related code between the VAS/NX kernel drivers and skiboot. For consistency, switch the order of parameters in SET_FIELD to match the order in skiboot. Signed-off-by: Sukadev Bhattiprolu Reviewed-by: Dan Streetman Signed-off-by: Michael Ellerman --- drivers/crypto/nx/nx-842-powernv.c | 7 ++++--- drivers/crypto/nx/nx-842.h | 5 ----- 2 files changed, 4 insertions(+), 8 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 1710f80a09ec..3abb045cdba7 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -22,6 +22,7 @@ #include #include +#include MODULE_LICENSE("GPL"); MODULE_AUTHOR("Dan Streetman "); @@ -424,9 +425,9 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, /* set up CCW */ ccw = 0; - ccw = SET_FIELD(ccw, CCW_CT, nx842_ct); - ccw = SET_FIELD(ccw, CCW_CI_842, 0); /* use 0 for hw auto-selection */ - ccw = SET_FIELD(ccw, CCW_FC_842, fc); + ccw = SET_FIELD(CCW_CT, ccw, nx842_ct); + ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */ + ccw = SET_FIELD(CCW_FC_842, ccw, fc); /* set up CRB's CSB addr */ csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS; diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h index a4eee3bba937..30929bd7d1a9 100644 --- a/drivers/crypto/nx/nx-842.h +++ b/drivers/crypto/nx/nx-842.h @@ -100,11 +100,6 @@ static inline unsigned long nx842_get_pa(void *addr) return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr); } -/* Get/Set bit fields */ -#define MASK_LSH(m) (__builtin_ffsl(m) - 1) -#define GET_FIELD(v, m) (((v) & (m)) >> MASK_LSH(m)) -#define SET_FIELD(v, m, val) (((v) & ~(m)) | (((val) << MASK_LSH(m)) & (m))) - /** * This provides the driver's constraints. Different nx842 implementations * may have varying requirements. The constraints are: -- cgit v1.2.3 From c97f8169fb227cae5adeac56cafa980f25978031 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 31 Aug 2017 00:11:29 -0700 Subject: crypto/nx: Rename nx842_powernv_function as icswx function Rename nx842_powernv_function to nx842_powernv_exec. nx842_powernv_exec points to nx842_exec_icswx and will be point to VAS exec function which will be added later for P9 NX support. Signed-off-by: Haren Myneni Signed-off-by: Michael Ellerman --- drivers/crypto/nx/nx-842-powernv.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 3abb045cdba7..161987698bbc 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -54,7 +54,11 @@ struct nx842_coproc { /* no cpu hotplug on powernv, so this list never changes after init */ static LIST_HEAD(nx842_coprocs); -static unsigned int nx842_ct; +static unsigned int nx842_ct; /* used in icswx function */ + +static int (*nx842_powernv_exec)(const unsigned char *in, + unsigned int inlen, unsigned char *out, + unsigned int *outlenp, void *workmem, int fc); /** * setup_indirect_dde - Setup an indirect DDE @@ -355,7 +359,7 @@ static int wait_for_csb(struct nx842_workmem *wmem, } /** - * nx842_powernv_function - compress/decompress data using the 842 algorithm + * nx842_exec_icswx - compress/decompress data using the 842 algorithm * * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems. * This compresses or decompresses the provided input buffer into the provided @@ -385,7 +389,7 @@ static int wait_for_csb(struct nx842_workmem *wmem, * -ETIMEDOUT hardware did not complete operation in reasonable time * -EINTR operation was aborted */ -static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, +static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlenp, void *workmem, int fc) { @@ -489,13 +493,13 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, * @workmem: working memory buffer pointer, size determined by * nx842_powernv_driver.workmem_size * - * Returns: see @nx842_powernv_function() + * Returns: see @nx842_powernv_exec() */ static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlenp, void *wmem) { - return nx842_powernv_function(in, inlen, out, outlenp, + return nx842_powernv_exec(in, inlen, out, outlenp, wmem, CCW_FC_842_COMP_CRC); } @@ -517,13 +521,13 @@ static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen, * @workmem: working memory buffer pointer, size determined by * nx842_powernv_driver.workmem_size * - * Returns: see @nx842_powernv_function() + * Returns: see @nx842_powernv_exec() */ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen, unsigned char *out, unsigned int *outlenp, void *wmem) { - return nx842_powernv_function(in, inlen, out, outlenp, + return nx842_powernv_exec(in, inlen, out, outlenp, wmem, CCW_FC_842_DECOMP_CRC); } @@ -625,6 +629,8 @@ static __init int nx842_powernv_init(void) if (!nx842_ct) return -ENODEV; + nx842_powernv_exec = nx842_exec_icswx; + ret = crypto_register_alg(&nx842_powernv_alg); if (ret) { struct nx842_coproc *coproc, *n; -- cgit v1.2.3 From 56c10d5ea68b9a1425df0dedbe5d2710cc7d8bfa Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 31 Aug 2017 00:12:22 -0700 Subject: crypto/nx: Create nx842_configure_crb function Configure CRB is moved to nx842_configure_crb() so that it can be used for icswx and VAS exec functions. VAS function will be added later with P9 support. Signed-off-by: Haren Myneni Signed-off-by: Michael Ellerman --- drivers/crypto/nx/nx-842-powernv.c | 57 +++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 19 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 161987698bbc..1bd19e03eb7d 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -358,6 +358,40 @@ static int wait_for_csb(struct nx842_workmem *wmem, return 0; } +static int nx842_config_crb(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int outlen, + struct nx842_workmem *wmem) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + u64 csb_addr; + int ret; + + crb = &wmem->crb; + csb = &crb->csb; + + /* Clear any previous values */ + memset(crb, 0, sizeof(*crb)); + + /* set up DDLs */ + ret = setup_ddl(&crb->source, wmem->ddl_in, + (unsigned char *)in, inlen, true); + if (ret) + return ret; + + ret = setup_ddl(&crb->target, wmem->ddl_out, + out, outlen, false); + if (ret) + return ret; + + /* set up CRB's CSB addr */ + csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS; + csb_addr |= CRB_CSB_AT; /* Addrs are phys */ + crb->csb_addr = cpu_to_be64(csb_addr); + + return 0; +} + /** * nx842_exec_icswx - compress/decompress data using the 842 algorithm * @@ -397,7 +431,6 @@ static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen, struct coprocessor_status_block *csb; struct nx842_workmem *wmem; int ret; - u64 csb_addr; u32 ccw; unsigned int outlen = *outlenp; @@ -411,33 +444,19 @@ static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen, return -ENODEV; } - crb = &wmem->crb; - csb = &crb->csb; - - /* Clear any previous values */ - memset(crb, 0, sizeof(*crb)); - - /* set up DDLs */ - ret = setup_ddl(&crb->source, wmem->ddl_in, - (unsigned char *)in, inlen, true); - if (ret) - return ret; - ret = setup_ddl(&crb->target, wmem->ddl_out, - out, outlen, false); + ret = nx842_config_crb(in, inlen, out, outlen, wmem); if (ret) return ret; + crb = &wmem->crb; + csb = &crb->csb; + /* set up CCW */ ccw = 0; ccw = SET_FIELD(CCW_CT, ccw, nx842_ct); ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */ ccw = SET_FIELD(CCW_FC_842, ccw, fc); - /* set up CRB's CSB addr */ - csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS; - csb_addr |= CRB_CSB_AT; /* Addrs are phys */ - crb->csb_addr = cpu_to_be64(csb_addr); - wmem->start = ktime_get(); /* do ICSWX */ -- cgit v1.2.3 From 1ee51b28ee6ad7919da4fbe9672263dd274dcbfe Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 31 Aug 2017 00:13:15 -0700 Subject: crypto/nx: Create nx842_delete_coprocs function Move deleting coprocessors info upon exit or failure to nx842_delete_coprocs(). Signed-off-by: Haren Myneni Signed-off-by: Michael Ellerman --- drivers/crypto/nx/nx-842-powernv.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 1bd19e03eb7d..67dc06f9b557 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -593,6 +593,16 @@ static int __init nx842_powernv_probe(struct device_node *dn) return 0; } +static void nx842_delete_coprocs(void) +{ + struct nx842_coproc *coproc, *n; + + list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + list_del(&coproc->list); + kfree(coproc); + } +} + static struct nx842_constraints nx842_powernv_constraints = { .alignment = DDE_BUFFER_ALIGN, .multiple = DDE_BUFFER_LAST_MULT, @@ -652,13 +662,7 @@ static __init int nx842_powernv_init(void) ret = crypto_register_alg(&nx842_powernv_alg); if (ret) { - struct nx842_coproc *coproc, *n; - - list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { - list_del(&coproc->list); - kfree(coproc); - } - + nx842_delete_coprocs(); return ret; } @@ -668,13 +672,8 @@ module_init(nx842_powernv_init); static void __exit nx842_powernv_exit(void) { - struct nx842_coproc *coproc, *n; - crypto_unregister_alg(&nx842_powernv_alg); - list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { - list_del(&coproc->list); - kfree(coproc); - } + nx842_delete_coprocs(); } module_exit(nx842_powernv_exit); -- cgit v1.2.3 From cd38a8a8a2ab95e43a031410db6a32f2d84e3fc0 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 31 Aug 2017 00:14:06 -0700 Subject: crypto/nx: Add nx842_add_coprocs_list function Updating coprocessor list is moved to nx842_add_coprocs_list(). This function will be used for both icswx and VAS functions. Signed-off-by: Haren Myneni Signed-off-by: Michael Ellerman --- drivers/crypto/nx/nx-842-powernv.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 67dc06f9b557..829b5cad0043 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -550,6 +550,14 @@ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen, wmem, CCW_FC_842_DECOMP_CRC); } +static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc, + int chipid) +{ + coproc->chip_id = chipid; + INIT_LIST_HEAD(&coproc->list); + list_add(&coproc->list, &nx842_coprocs); +} + static int __init nx842_powernv_probe(struct device_node *dn) { struct nx842_coproc *coproc; @@ -576,11 +584,9 @@ static int __init nx842_powernv_probe(struct device_node *dn) if (!coproc) return -ENOMEM; - coproc->chip_id = chip_id; coproc->ct = ct; coproc->ci = ci; - INIT_LIST_HEAD(&coproc->list); - list_add(&coproc->list, &nx842_coprocs); + nx842_add_coprocs_list(coproc, chip_id); pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci); -- cgit v1.2.3 From f05368336b3ae399f66cf511c52c6d69c7bc6b39 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 31 Aug 2017 00:17:25 -0700 Subject: crypto/nx: Use kzalloc for workmem allocation Send window is opened / closed for each crypto session. So initializes txwin in workmem. Signed-off-by: Haren Myneni Signed-off-by: Michael Ellerman --- drivers/crypto/nx/nx-842.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index d94e25df503b..da3cb8c35ec7 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -116,7 +116,7 @@ int nx842_crypto_init(struct crypto_tfm *tfm, struct nx842_driver *driver) spin_lock_init(&ctx->lock); ctx->driver = driver; - ctx->wmem = kmalloc(driver->workmem_size, GFP_KERNEL); + ctx->wmem = kzalloc(driver->workmem_size, GFP_KERNEL); ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER); if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) { -- cgit v1.2.3 From 146e9f1b65478643f2729a97ccb8be60bb4492e5 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 31 Aug 2017 00:18:18 -0700 Subject: crypto/nx: Add P9 NX specific error codes for 842 engine This patch adds changes for checking P9 specific 842 engine error codes. These errros are reported in coprocessor status block (CSB) for failures. Signed-off-by: Haren Myneni Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/icswx.h | 3 +++ drivers/crypto/nx/nx-842-powernv.c | 18 ++++++++++++++++++ drivers/crypto/nx/nx-842.h | 8 ++++++++ 3 files changed, 29 insertions(+) (limited to 'drivers/crypto') diff --git a/arch/powerpc/include/asm/icswx.h b/arch/powerpc/include/asm/icswx.h index 27e588f6c72e..6a2c87577541 100644 --- a/arch/powerpc/include/asm/icswx.h +++ b/arch/powerpc/include/asm/icswx.h @@ -69,7 +69,10 @@ struct coprocessor_completion_block { #define CSB_CC_WR_PROTECTION (16) #define CSB_CC_UNKNOWN_CODE (17) #define CSB_CC_ABORT (18) +#define CSB_CC_EXCEED_BYTE_COUNT (19) /* P9 or later */ #define CSB_CC_TRANSPORT (20) +#define CSB_CC_INVALID_CRB (21) /* P9 or later */ +#define CSB_CC_INVALID_DDE (30) /* P9 or later */ #define CSB_CC_SEGMENTED_DDL (31) #define CSB_CC_PROGRESS_POINT (32) #define CSB_CC_DDE_OVERFLOW (33) diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 829b5cad0043..c0dd4c7e17d3 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -243,6 +243,13 @@ static int wait_for_csb(struct nx842_workmem *wmem, case CSB_CC_TEMPL_OVERFLOW: CSB_ERR(csb, "Compressed data template shows data past end"); return -EINVAL; + case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */ + /* + * DDE byte count exceeds the limit specified in Maximum + * byte count register. + */ + CSB_ERR(csb, "DDE byte count exceeds the limit"); + return -EINVAL; /* these should not happen */ case CSB_CC_INVALID_ALIGN: @@ -284,9 +291,17 @@ static int wait_for_csb(struct nx842_workmem *wmem, CSB_ERR(csb, "Too many DDEs in DDL"); return -EINVAL; case CSB_CC_TRANSPORT: + case CSB_CC_INVALID_CRB: /* P9 or later */ /* shouldn't happen, we setup CRB correctly */ CSB_ERR(csb, "Invalid CRB"); return -EINVAL; + case CSB_CC_INVALID_DDE: /* P9 or later */ + /* + * shouldn't happen, setup_direct/indirect_dde creates + * DDE right + */ + CSB_ERR(csb, "Invalid DDE"); + return -EINVAL; case CSB_CC_SEGMENTED_DDL: /* shouldn't happen, setup_ddl creates DDL right */ CSB_ERR(csb, "Segmented DDL error"); @@ -330,6 +345,9 @@ static int wait_for_csb(struct nx842_workmem *wmem, case CSB_CC_HW: CSB_ERR(csb, "Correctable hardware error"); return -EPROTO; + case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */ + CSB_ERR(csb, "Job did not finish within allowed time"); + return -EPROTO; default: CSB_ERR(csb, "Invalid CC %d", csb->cc); diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h index 30929bd7d1a9..bb2f31792683 100644 --- a/drivers/crypto/nx/nx-842.h +++ b/drivers/crypto/nx/nx-842.h @@ -76,9 +76,17 @@ #define CSB_CC_DECRYPT_OVERFLOW (64) /* asym crypt codes */ #define CSB_CC_MINV_OVERFLOW (128) +/* + * HW error - Job did not finish in the maximum time allowed. + * Job terminated. + */ +#define CSB_CC_HW_EXPIRED_TIMER (224) /* These are reserved for hypervisor use */ #define CSB_CC_HYP_RESERVE_START (240) #define CSB_CC_HYP_RESERVE_END (253) +#define CSB_CC_HYP_RESERVE_P9_END (251) +/* No valid interrupt server (P9 or later). */ +#define CSB_CC_HYP_RESERVE_NO_INTR_SERVER (252) #define CSB_CC_HYP_NO_HW (254) #define CSB_CC_HYP_HANG_ABORTED (255) -- cgit v1.2.3 From b0d6c9bab5e41d07f2bece1ef8c81cd2175b5f88 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 31 Aug 2017 00:19:07 -0700 Subject: crypto/nx: Add P9 NX support for 842 compression engine This patch adds P9 NX support for 842 compression engine. Virtual Accelerator Switchboard (VAS) is used to access 842 engine on P9. For each NX engine per chip, setup receive window using vas_rx_win_open() which configures RxFIFo with FIFO address, lpid, pid and tid values. This unique (lpid, pid, tid) combination will be used to identify the target engine. For crypto open request, open send window on the NX engine for the corresponding chip / cpu where the open request is executed. This send window will be closed upon crypto close request. NX provides high and normal priority FIFOs. For compression / decompression requests, we use only hight priority FIFOs in kernel. Each NX request will be communicated to VAS using copy/paste instructions with vas_copy_crb() / vas_paste_crb() functions. Signed-off-by: Haren Myneni Reviewed-by: Ram Pai Signed-off-by: Michael Ellerman --- drivers/crypto/nx/Kconfig | 1 + drivers/crypto/nx/nx-842-powernv.c | 377 ++++++++++++++++++++++++++++++++++++- 2 files changed, 372 insertions(+), 6 deletions(-) (limited to 'drivers/crypto') diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig index ad7552a6998c..cd5dda9c48f4 100644 --- a/drivers/crypto/nx/Kconfig +++ b/drivers/crypto/nx/Kconfig @@ -38,6 +38,7 @@ config CRYPTO_DEV_NX_COMPRESS_PSERIES config CRYPTO_DEV_NX_COMPRESS_POWERNV tristate "Compression acceleration support on PowerNV platform" depends on PPC_POWERNV + depends on PPC_VAS default y help Support for PowerPC Nest (NX) compression acceleration. This diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index c0dd4c7e17d3..874ddf5e9087 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -23,6 +23,7 @@ #include #include #include +#include MODULE_LICENSE("GPL"); MODULE_AUTHOR("Dan Streetman "); @@ -32,6 +33,9 @@ MODULE_ALIAS_CRYPTO("842-nx"); #define WORKMEM_ALIGN (CRB_ALIGN) #define CSB_WAIT_MAX (5000) /* ms */ +#define VAS_RETRIES (10) +/* # of requests allowed per RxFIFO at a time. 0 for unlimited */ +#define MAX_CREDITS_PER_RXFIFO (1024) struct nx842_workmem { /* Below fields must be properly aligned */ @@ -42,16 +46,27 @@ struct nx842_workmem { ktime_t start; + struct vas_window *txwin; /* Used with VAS function */ char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */ } __packed __aligned(WORKMEM_ALIGN); struct nx842_coproc { unsigned int chip_id; unsigned int ct; - unsigned int ci; + unsigned int ci; /* Coprocessor instance, used with icswx */ + struct { + struct vas_window *rxwin; + int id; + } vas; struct list_head list; }; +/* + * Send the request to NX engine on the chip for the corresponding CPU + * where the process is executing. Use with VAS function. + */ +static DEFINE_PER_CPU(struct nx842_coproc *, coproc_inst); + /* no cpu hotplug on powernv, so this list never changes after init */ static LIST_HEAD(nx842_coprocs); static unsigned int nx842_ct; /* used in icswx function */ @@ -512,6 +527,104 @@ static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen, return ret; } +/** + * nx842_exec_vas - compress/decompress data using the 842 algorithm + * + * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems. + * This compresses or decompresses the provided input buffer into the provided + * output buffer. + * + * Upon return from this function @outlen contains the length of the + * output data. If there is an error then @outlen will be 0 and an + * error will be specified by the return code from this function. + * + * The @workmem buffer should only be used by one function call at a time. + * + * @in: input buffer pointer + * @inlen: input buffer size + * @out: output buffer pointer + * @outlenp: output buffer size pointer + * @workmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size + * @fc: function code, see CCW Function Codes in nx-842.h + * + * Returns: + * 0 Success, output of length @outlenp stored in the buffer + * at @out + * -ENODEV Hardware unavailable + * -ENOSPC Output buffer is to small + * -EMSGSIZE Input buffer too large + * -EINVAL buffer constraints do not fix nx842_constraints + * -EPROTO hardware error during operation + * -ETIMEDOUT hardware did not complete operation in reasonable time + * -EINTR operation was aborted + */ +static int nx842_exec_vas(const unsigned char *in, unsigned int inlen, + unsigned char *out, unsigned int *outlenp, + void *workmem, int fc) +{ + struct coprocessor_request_block *crb; + struct coprocessor_status_block *csb; + struct nx842_workmem *wmem; + struct vas_window *txwin; + int ret, i = 0; + u32 ccw; + unsigned int outlen = *outlenp; + + wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN); + + *outlenp = 0; + + crb = &wmem->crb; + csb = &crb->csb; + + ret = nx842_config_crb(in, inlen, out, outlen, wmem); + if (ret) + return ret; + + ccw = 0; + ccw = SET_FIELD(CCW_FC_842, ccw, fc); + crb->ccw = cpu_to_be32(ccw); + + txwin = wmem->txwin; + /* shoudn't happen, we don't load without a coproc */ + if (!txwin) { + pr_err_ratelimited("NX-842 coprocessor is not available"); + return -ENODEV; + } + + do { + wmem->start = ktime_get(); + preempt_disable(); + /* + * VAS copy CRB into L2 cache. Refer . + * @crb and @offset. + */ + vas_copy_crb(crb, 0); + + /* + * VAS paste previously copied CRB to NX. + * @txwin, @offset and @last (must be true). + */ + ret = vas_paste_crb(txwin, 0, 1); + preempt_enable(); + /* + * Retry copy/paste function for VAS failures. + */ + } while (ret && (i++ < VAS_RETRIES)); + + if (ret) { + pr_err_ratelimited("VAS copy/paste failed\n"); + return ret; + } + + ret = wait_for_csb(wmem, csb); + if (!ret) + *outlenp = be32_to_cpu(csb->count); + + return ret; +} + /** * nx842_powernv_compress - Compress data using the 842 algorithm * @@ -576,6 +689,201 @@ static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc, list_add(&coproc->list, &nx842_coprocs); } +/* + * Identify chip ID for each CPU and save coprocesor adddress for the + * corresponding NX engine in percpu coproc_inst. + * coproc_inst is used in crypto_init to open send window on the NX instance + * for the corresponding CPU / chip where the open request is executed. + */ +static void nx842_set_per_cpu_coproc(struct nx842_coproc *coproc) +{ + unsigned int i, chip_id; + + for_each_possible_cpu(i) { + chip_id = cpu_to_chip_id(i); + + if (coproc->chip_id == chip_id) + per_cpu(coproc_inst, i) = coproc; + } +} + + +static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc) +{ + struct vas_window *txwin = NULL; + struct vas_tx_win_attr txattr; + + /* + * Kernel requests will be high priority. So open send + * windows only for high priority RxFIFO entries. + */ + vas_init_tx_win_attr(&txattr, coproc->ct); + txattr.lpid = 0; /* lpid is 0 for kernel requests */ + txattr.pid = 0; /* pid is 0 for kernel requests */ + + /* + * Open a VAS send window which is used to send request to NX. + */ + txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr); + if (IS_ERR(txwin)) { + pr_err("ibm,nx-842: Can not open TX window: %ld\n", + PTR_ERR(txwin)); + return NULL; + } + + return txwin; +} + +static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, + int vasid) +{ + struct vas_window *rxwin = NULL; + struct vas_rx_win_attr rxattr; + struct nx842_coproc *coproc; + u32 lpid, pid, tid, fifo_size; + u64 rx_fifo; + const char *priority; + int ret; + + ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo); + if (ret) { + pr_err("Missing rx-fifo-address property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size); + if (ret) { + pr_err("Missing rx-fifo-size property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "lpid", &lpid); + if (ret) { + pr_err("Missing lpid property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "pid", &pid); + if (ret) { + pr_err("Missing pid property\n"); + return ret; + } + + ret = of_property_read_u32(dn, "tid", &tid); + if (ret) { + pr_err("Missing tid property\n"); + return ret; + } + + ret = of_property_read_string(dn, "priority", &priority); + if (ret) { + pr_err("Missing priority property\n"); + return ret; + } + + coproc = kzalloc(sizeof(*coproc), GFP_KERNEL); + if (!coproc) + return -ENOMEM; + + if (!strcmp(priority, "High")) + coproc->ct = VAS_COP_TYPE_842_HIPRI; + else if (!strcmp(priority, "Normal")) + coproc->ct = VAS_COP_TYPE_842; + else { + pr_err("Invalid RxFIFO priority value\n"); + ret = -EINVAL; + goto err_out; + } + + vas_init_rx_win_attr(&rxattr, coproc->ct); + rxattr.rx_fifo = (void *)rx_fifo; + rxattr.rx_fifo_size = fifo_size; + rxattr.lnotify_lpid = lpid; + rxattr.lnotify_pid = pid; + rxattr.lnotify_tid = tid; + rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO; + + /* + * Open a VAS receice window which is used to configure RxFIFO + * for NX. + */ + rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr); + if (IS_ERR(rxwin)) { + ret = PTR_ERR(rxwin); + pr_err("setting RxFIFO with VAS failed: %d\n", + ret); + goto err_out; + } + + coproc->vas.rxwin = rxwin; + coproc->vas.id = vasid; + nx842_add_coprocs_list(coproc, chip_id); + + /* + * Kernel requests use only high priority FIFOs. So save coproc + * info in percpu coproc_inst which will be used to open send + * windows for crypto open requests later. + */ + if (coproc->ct == VAS_COP_TYPE_842_HIPRI) + nx842_set_per_cpu_coproc(coproc); + + return 0; + +err_out: + kfree(coproc); + return ret; +} + + +static int __init nx842_powernv_probe_vas(struct device_node *pn) +{ + struct device_node *dn; + int chip_id, vasid, ret = 0; + int nx_fifo_found = 0; + + chip_id = of_get_ibm_chip_id(pn); + if (chip_id < 0) { + pr_err("ibm,chip-id missing\n"); + return -EINVAL; + } + + for_each_compatible_node(dn, NULL, "ibm,power9-vas-x") { + if (of_get_ibm_chip_id(dn) == chip_id) + break; + } + + if (!dn) { + pr_err("Missing VAS device node\n"); + return -EINVAL; + } + + if (of_property_read_u32(dn, "ibm,vas-id", &vasid)) { + pr_err("Missing ibm,vas-id device property\n"); + of_node_put(dn); + return -EINVAL; + } + + of_node_put(dn); + + for_each_child_of_node(pn, dn) { + if (of_device_is_compatible(dn, "ibm,p9-nx-842")) { + ret = vas_cfg_coproc_info(dn, chip_id, vasid); + if (ret) { + of_node_put(dn); + return ret; + } + nx_fifo_found++; + } + } + + if (!nx_fifo_found) { + pr_err("NX842 FIFO nodes are missing\n"); + ret = -EINVAL; + } + + return ret; +} + static int __init nx842_powernv_probe(struct device_node *dn) { struct nx842_coproc *coproc; @@ -622,6 +930,9 @@ static void nx842_delete_coprocs(void) struct nx842_coproc *coproc, *n; list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) { + if (coproc->vas.rxwin) + vas_win_close(coproc->vas.rxwin); + list_del(&coproc->list); kfree(coproc); } @@ -643,6 +954,46 @@ static struct nx842_driver nx842_powernv_driver = { .decompress = nx842_powernv_decompress, }; +static int nx842_powernv_crypto_init_vas(struct crypto_tfm *tfm) +{ + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_workmem *wmem; + struct nx842_coproc *coproc; + int ret; + + ret = nx842_crypto_init(tfm, &nx842_powernv_driver); + + if (ret) + return ret; + + wmem = PTR_ALIGN((struct nx842_workmem *)ctx->wmem, WORKMEM_ALIGN); + coproc = per_cpu(coproc_inst, smp_processor_id()); + + ret = -EINVAL; + if (coproc && coproc->vas.rxwin) { + wmem->txwin = nx842_alloc_txwin(coproc); + if (!IS_ERR(wmem->txwin)) + return 0; + + ret = PTR_ERR(wmem->txwin); + } + + return ret; +} + +void nx842_powernv_crypto_exit_vas(struct crypto_tfm *tfm) +{ + struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm); + struct nx842_workmem *wmem; + + wmem = PTR_ALIGN((struct nx842_workmem *)ctx->wmem, WORKMEM_ALIGN); + + if (wmem && wmem->txwin) + vas_win_close(wmem->txwin); + + nx842_crypto_exit(tfm); +} + static int nx842_powernv_crypto_init(struct crypto_tfm *tfm) { return nx842_crypto_init(tfm, &nx842_powernv_driver); @@ -676,13 +1027,27 @@ static __init int nx842_powernv_init(void) BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT); BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT); - for_each_compatible_node(dn, NULL, "ibm,power-nx") - nx842_powernv_probe(dn); + for_each_compatible_node(dn, NULL, "ibm,power9-nx") { + ret = nx842_powernv_probe_vas(dn); + if (ret) { + nx842_delete_coprocs(); + return ret; + } + } - if (!nx842_ct) - return -ENODEV; + if (list_empty(&nx842_coprocs)) { + for_each_compatible_node(dn, NULL, "ibm,power-nx") + nx842_powernv_probe(dn); + + if (!nx842_ct) + return -ENODEV; - nx842_powernv_exec = nx842_exec_icswx; + nx842_powernv_exec = nx842_exec_icswx; + } else { + nx842_powernv_exec = nx842_exec_vas; + nx842_powernv_alg.cra_init = nx842_powernv_crypto_init_vas; + nx842_powernv_alg.cra_exit = nx842_powernv_crypto_exit_vas; + } ret = crypto_register_alg(&nx842_powernv_alg); if (ret) { -- cgit v1.2.3