diff options
Diffstat (limited to 'drivers/crypto/caam')
| -rw-r--r-- | drivers/crypto/caam/Kconfig | 46 | ||||
| -rw-r--r-- | drivers/crypto/caam/Makefile | 18 | ||||
| -rw-r--r-- | drivers/crypto/caam/caamalg.c | 338 | ||||
| -rw-r--r-- | drivers/crypto/caam/caamalg_desc.c | 147 | ||||
| -rw-r--r-- | drivers/crypto/caam/caamalg_desc.h | 4 | ||||
| -rw-r--r-- | drivers/crypto/caam/caamalg_qi.c | 267 | ||||
| -rw-r--r-- | drivers/crypto/caam/caamalg_qi2.c | 202 | ||||
| -rw-r--r-- | drivers/crypto/caam/caamhash.c | 329 | ||||
| -rw-r--r-- | drivers/crypto/caam/caampkc.c | 177 | ||||
| -rw-r--r-- | drivers/crypto/caam/caampkc.h | 9 | ||||
| -rw-r--r-- | drivers/crypto/caam/caamrng.c | 76 | ||||
| -rw-r--r-- | drivers/crypto/caam/ctrl.c | 56 | ||||
| -rw-r--r-- | drivers/crypto/caam/desc_constr.h | 11 | ||||
| -rw-r--r-- | drivers/crypto/caam/error.c | 8 | ||||
| -rw-r--r-- | drivers/crypto/caam/error.h | 2 | ||||
| -rw-r--r-- | drivers/crypto/caam/intern.h | 102 | ||||
| -rw-r--r-- | drivers/crypto/caam/jr.c | 43 | ||||
| -rw-r--r-- | drivers/crypto/caam/key_gen.c | 28 | ||||
| -rw-r--r-- | drivers/crypto/caam/qi.c | 52 | ||||
| -rw-r--r-- | drivers/crypto/caam/sg_sw_qm.h | 18 | ||||
| -rw-r--r-- | drivers/crypto/caam/sg_sw_qm2.h | 18 | ||||
| -rw-r--r-- | drivers/crypto/caam/sg_sw_sec4.h | 26 |
22 files changed, 999 insertions, 978 deletions
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index 577c9844b322..3720ddabb507 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -2,6 +2,12 @@ config CRYPTO_DEV_FSL_CAAM_COMMON tristate +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + tristate + +config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC + tristate + config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore platform driver backend" depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE @@ -25,7 +31,7 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG Selecting this will enable printing of various debug information in the CAAM driver. -config CRYPTO_DEV_FSL_CAAM_JR +menuconfig CRYPTO_DEV_FSL_CAAM_JR tristate "Freescale CAAM Job Ring driver backend" default y help @@ -86,8 +92,9 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD threshold. Range is 1-65535. config CRYPTO_DEV_FSL_CAAM_CRYPTO_API - tristate "Register algorithm implementations with the Crypto API" + bool "Register algorithm implementations with the Crypto API" default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC select CRYPTO_AEAD select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER @@ -97,13 +104,11 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API scatterlist crypto API (such as the linux native IPSec stack) to the SEC4 via job ring. - To compile this as a module, choose M here: the module - will be called caamalg. - config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI - tristate "Queue Interface as Crypto API backend" + bool "Queue Interface as Crypto API backend" depends on FSL_DPAA && NET default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER help @@ -114,33 +119,26 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI assigned to the kernel should also be more than the number of job rings. - To compile this as a module, choose M here: the module - will be called caamalg_qi. - config CRYPTO_DEV_FSL_CAAM_AHASH_API - tristate "Register hash algorithm implementations with Crypto API" + bool "Register hash algorithm implementations with Crypto API" default y + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC select CRYPTO_HASH help Selecting this will offload ahash for users of the scatterlist crypto API to the SEC4 via job ring. - To compile this as a module, choose M here: the module - will be called caamhash. - config CRYPTO_DEV_FSL_CAAM_PKC_API - tristate "Register public key cryptography implementations with Crypto API" + bool "Register public key cryptography implementations with Crypto API" default y select CRYPTO_RSA help Selecting this will allow SEC Public key support for RSA. Supported cryptographic primitives: encryption, decryption, signature and verification. - To compile this as a module, choose M here: the module - will be called caam_pkc. config CRYPTO_DEV_FSL_CAAM_RNG_API - tristate "Register caam device for hwrng API" + bool "Register caam device for hwrng API" default y select CRYPTO_RNG select HW_RANDOM @@ -148,9 +146,6 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API Selecting this will register the SEC4 hardware rng to the hw_random API for suppying the kernel entropy pool. - To compile this as a module, choose M here: the module - will be called caamrng. - endif # CRYPTO_DEV_FSL_CAAM_JR endif # CRYPTO_DEV_FSL_CAAM @@ -160,6 +155,8 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM depends on FSL_MC_DPIO depends on NETDEVICES select CRYPTO_DEV_FSL_CAAM_COMMON + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC select CRYPTO_BLKCIPHER select CRYPTO_AUTHENC select CRYPTO_AEAD @@ -171,12 +168,3 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM To compile this as a module, choose M here: the module will be called dpaa2_caam. - -config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC - def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \ - CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \ - CRYPTO_DEV_FSL_DPAA2_CAAM) - -config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC - def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \ - CRYPTO_DEV_FSL_DPAA2_CAAM) diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 7bbfd06a11ff..9ab4e81ea21e 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -11,20 +11,20 @@ ccflags-y += -DVERSION=\"\" obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o -caam-objs := ctrl.o -caam_jr-objs := jr.o key_gen.o -caam_pkc-y := caampkc.o pkc_desc.o +caam-y := ctrl.o +caam_jr-y := jr.o key_gen.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o + +caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) ccflags-y += -DCONFIG_CAAM_QI - caam-objs += qi.o endif obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index c0ece44f303b..43f18253e5b6 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -77,13 +77,6 @@ #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) -#ifdef DEBUG -/* for print_hex_dumps with line references */ -#define debug(format, arg...) printk(format, arg) -#else -#define debug(format, arg...) -#endif - struct caam_alg_entry { int class1_alg_type; int class2_alg_type; @@ -583,13 +576,11 @@ static int aead_setkey(struct crypto_aead *aead, if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; -#ifdef DEBUG - printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * If DKP is supported, use it in the shared descriptor to generate @@ -623,11 +614,10 @@ static int aead_setkey(struct crypto_aead *aead, memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->adata.keylen_pad + keys.enckeylen, 1); -#endif + + print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); skip_split_key: ctx->cdata.keylen = keys.enckeylen; @@ -678,10 +668,8 @@ static int gcm_setkey(struct crypto_aead *aead, struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); @@ -699,10 +687,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -725,10 +711,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -757,10 +741,8 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) @@ -916,7 +898,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); if (sec4_sg_bytes) dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, DMA_TO_DEVICE); @@ -949,9 +931,7 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct aead_edesc, hw_desc[0]); @@ -971,9 +951,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct aead_edesc, hw_desc[0]); @@ -1001,33 +979,32 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); int ivsize = crypto_skcipher_ivsize(skcipher); -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); -#endif - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); - skcipher_unmap(jrdev, edesc, req); /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - if (ivsize) - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - - ivsize, ivsize, 0); + if (ivsize) { + memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, + ivsize); + + print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + } + + caam_dump_sg("dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); kfree(edesc); @@ -1039,26 +1016,35 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, { struct skcipher_request *req = context; struct skcipher_edesc *edesc; -#ifdef DEBUG struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); int ivsize = crypto_skcipher_ivsize(skcipher); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); -#endif - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + skcipher_unmap(jrdev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + if (ivsize) { + memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, + ivsize); + + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + ivsize, 1); + } + + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); - skcipher_unmap(jrdev, edesc, req); kfree(edesc); skcipher_request_complete(req, err); @@ -1106,6 +1092,7 @@ static void init_aead_job(struct aead_request *req, if (unlikely(req->src != req->dst)) { if (!edesc->mapped_dst_nents) { dst_dma = 0; + out_options = 0; } else if (edesc->mapped_dst_nents == 1) { dst_dma = sg_dma_address(req->dst); out_options = 0; @@ -1249,6 +1236,7 @@ static void init_skcipher_job(struct skcipher_request *req, { struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct device *jrdev = ctx->jrdev; int ivsize = crypto_skcipher_ivsize(skcipher); u32 *desc = edesc->hw_desc; u32 *sh_desc; @@ -1256,13 +1244,12 @@ static void init_skcipher_job(struct skcipher_request *req, dma_addr_t src_dma, dst_dma, ptr; int len, sec4_sg_index = 0; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); - pr_err("asked=%d, cryptlen%d\n", + print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); + dev_dbg(jrdev, "asked=%d, cryptlen%d\n", (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); -#endif - caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", + + caam_dump_sg("src @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->src, edesc->src_nents > 1 ? 100 : req->cryptlen, 1); @@ -1285,7 +1272,7 @@ static void init_skcipher_job(struct skcipher_request *req, if (likely(req->src == req->dst)) { dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); out_options = in_options; - } else if (edesc->mapped_dst_nents == 1) { + } else if (!ivsize && edesc->mapped_dst_nents == 1) { dst_dma = sg_dma_address(req->dst); } else { dst_dma = edesc->sec4_sg_dma + sec4_sg_index * @@ -1293,7 +1280,7 @@ static void init_skcipher_job(struct skcipher_request *req, out_options = LDST_SGF; } - append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); + append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); } /* @@ -1309,37 +1296,36 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; unsigned int authsize = ctx->authsize; if (unlikely(req->dst != req->src)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); return ERR_PTR(dst_nents); } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); return ERR_PTR(src_nents); } } @@ -1380,8 +1366,16 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } } + /* + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; - sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + if (mapped_dst_nents > 1) + sec4_sg_len += pad_sg_nents(mapped_dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ @@ -1403,12 +1397,12 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sec4_sg_index = 0; if (mapped_src_nents > 1) { - sg_to_sec4_sg_last(req->src, mapped_src_nents, + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + sec4_sg_index, 0); sec4_sg_index += mapped_src_nents; } if (mapped_dst_nents > 1) { - sg_to_sec4_sg_last(req->dst, mapped_dst_nents, + sg_to_sec4_sg_last(req->dst, dst_len, edesc->sec4_sg + sec4_sg_index, 0); } @@ -1446,11 +1440,10 @@ static int gcm_encrypt(struct aead_request *req) /* Create and submit job descriptor */ init_gcm_job(req, edesc, all_contig, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); @@ -1556,11 +1549,10 @@ static int aead_encrypt(struct aead_request *req) /* Create and submit job descriptor */ init_authenc_job(req, edesc, all_contig, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); @@ -1591,11 +1583,10 @@ static int gcm_decrypt(struct aead_request *req) /* Create and submit job descriptor*/ init_gcm_job(req, edesc, all_contig, false); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); @@ -1627,7 +1618,7 @@ static int aead_decrypt(struct aead_request *req) u32 *desc; int ret = 0; - caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ", + caam_dump_sg("dec src@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->src, req->assoclen + req->cryptlen, 1); @@ -1639,11 +1630,10 @@ static int aead_decrypt(struct aead_request *req) /* Create and submit job descriptor*/ init_authenc_job(req, edesc, all_contig, false); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); @@ -1719,7 +1709,29 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, else sec4_sg_ents = mapped_src_nents + !!ivsize; dst_sg_idx = sec4_sg_ents; - sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (output S/G) + * pad output S/G, if needed + * else if (input S/G) ... + * pad input S/G, if needed + */ + if (ivsize || mapped_dst_nents > 1) { + if (req->src == req->dst) + sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); + else + sec4_sg_ents += pad_sg_nents(mapped_dst_nents + + !!ivsize); + } else { + sec4_sg_ents = pad_sg_nents(sec4_sg_ents); + } + sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); /* @@ -1744,10 +1756,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, /* Make sure IV is located in a DMAable area */ if (ivsize) { - iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes; + iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(jrdev, iv_dma)) { dev_err(jrdev, "unable to map IV\n"); caam_unmap(jrdev, req->src, req->dst, src_nents, @@ -1759,13 +1771,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); } if (dst_sg_idx) - sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg + - !!ivsize, 0); + sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + + !!ivsize, 0); - if (mapped_dst_nents > 1) { - sg_to_sec4_sg_last(req->dst, mapped_dst_nents, - edesc->sec4_sg + dst_sg_idx, 0); - } + if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) + sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + + dst_sg_idx, 0); + + if (ivsize) + dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents, iv_dma, ivsize, 0); + + if (ivsize || mapped_dst_nents > 1) + sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents); if (sec4_sg_bytes) { edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, @@ -1782,11 +1801,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, edesc->iv_dma = iv_dma; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, - sec4_sg_bytes, 1); -#endif + print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + sec4_sg_bytes, 1); return edesc; } @@ -1807,11 +1824,11 @@ static int skcipher_encrypt(struct skcipher_request *req) /* Create and submit job descriptor*/ init_skcipher_job(req, edesc, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); + desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req); @@ -1830,7 +1847,6 @@ static int skcipher_decrypt(struct skcipher_request *req) struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); - int ivsize = crypto_skcipher_ivsize(skcipher); struct device *jrdev = ctx->jrdev; u32 *desc; int ret = 0; @@ -1840,22 +1856,13 @@ static int skcipher_decrypt(struct skcipher_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - if (ivsize) - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - - ivsize, ivsize, 0); - /* Create and submit job descriptor*/ init_skcipher_job(req, edesc, false); desc = edesc->hw_desc; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req); if (!ret) { @@ -3444,7 +3451,7 @@ static void caam_aead_exit(struct crypto_aead *tfm) caam_exit_common(crypto_aead_ctx(tfm)); } -static void __exit caam_algapi_exit(void) +void caam_algapi_exit(void) { int i; @@ -3489,43 +3496,15 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg) alg->exit = caam_aead_exit; } -static int __init caam_algapi_init(void) +int caam_algapi_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int i = 0, err = 0; u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; u32 arc4_inst; unsigned int md_limit = SHA512_DIGEST_SIZE; bool registered = false, gcm_support; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - - /* * Register crypto algorithms the device supports. * First, detect presence and attributes of DES, AES, and MD blocks. @@ -3668,14 +3647,5 @@ static int __init caam_algapi_init(void) if (registered) pr_info("caam algorithms registered in /proc/crypto\n"); -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_algapi_init); -module_exit(caam_algapi_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for crypto API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 1e1a376edc2f..72531837571e 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -33,12 +33,11 @@ static inline void append_dec_op1(u32 *desc, u32 type) } jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT); uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); set_jump_tgt_here(desc, jump_cmd); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT | OP_ALG_AAI_DK); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT | + OP_ALG_AAI_DK); set_jump_tgt_here(desc, uncond_jump_cmd); } @@ -115,11 +114,9 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); @@ -204,11 +201,9 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); @@ -358,10 +353,9 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_encap); @@ -475,10 +469,9 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_decap); @@ -613,11 +606,9 @@ copy_iv: append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead givenc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); @@ -742,10 +733,9 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); @@ -838,10 +828,9 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); @@ -933,11 +922,9 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4106 enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); @@ -1030,11 +1017,9 @@ void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4106 dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); @@ -1115,11 +1100,9 @@ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4543 enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); @@ -1205,11 +1188,9 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4543 dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); @@ -1410,17 +1391,21 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, LDST_OFFSET_SHIFT)); /* Load operation */ - append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT); /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "skcipher enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap); @@ -1479,7 +1464,7 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Choose operation */ if (ctx1_iv_off) - append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | OP_ALG_DECRYPT); else append_dec_op1(desc, cdata->algtype); @@ -1487,11 +1472,15 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "skcipher dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap); @@ -1538,11 +1527,13 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts skcipher enc shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); } EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap); @@ -1588,11 +1579,13 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts skcipher dec shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); } EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap); diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index d5ca42ff961a..da4a4ee60c80 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -44,9 +44,9 @@ #define DESC_SKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_SKCIPHER_ENC_LEN (DESC_SKCIPHER_BASE + \ - 20 * CAAM_CMD_SZ) + 21 * CAAM_CMD_SZ) #define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \ - 15 * CAAM_CMD_SZ) + 16 * CAAM_CMD_SZ) void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, unsigned int icvsize, int era); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index d290d6b41825..32f0f8a72067 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -4,7 +4,7 @@ * Based on caamalg.c * * Copyright 2013-2016 Freescale Semiconductor, Inc. - * Copyright 2016-2018 NXP + * Copyright 2016-2019 NXP */ #include "compat.h" @@ -214,13 +214,11 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; -#ifdef DEBUG - dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * If DKP is supported, use it in the shared descriptor to generate @@ -237,7 +235,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, memcpy(ctx->key, keys.authkey, keys.authkeylen); memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); goto skip_split_key; @@ -251,8 +249,9 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, /* postpend encryption key to auth split key */ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + - keys.enckeylen, ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->adata.keylen_pad + keys.enckeylen, + ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, @@ -386,13 +385,12 @@ static int gcm_setkey(struct crypto_aead *aead, struct device *jrdev = ctx->jrdev; int ret; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, + ctx->dir); ctx->cdata.keylen = keylen; ret = gcm_set_sh_desc(aead); @@ -485,10 +483,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); /* @@ -496,8 +492,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, * in the nonce. Update the AES key length. */ ctx->cdata.keylen = keylen - 4; - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); ret = rfc4106_set_sh_desc(aead); if (ret) @@ -589,10 +585,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); /* @@ -600,8 +594,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, * in the nonce. Update the AES key length. */ ctx->cdata.keylen = keylen - 4; - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); ret = rfc4543_set_sh_desc(aead); if (ret) @@ -644,10 +638,9 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, const bool is_rfc3686 = alg->caam.rfc3686; int ret = 0; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) @@ -838,7 +831,8 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, int dst_nents, dma_addr_t iv_dma, int ivsize, - dma_addr_t qm_sg_dma, int qm_sg_bytes) + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) { if (dst != src) { if (src_nents) @@ -850,7 +844,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); if (qm_sg_bytes) dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); } @@ -863,7 +857,8 @@ static void aead_unmap(struct device *dev, int ivsize = crypto_aead_ivsize(aead); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); } @@ -874,7 +869,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, int ivsize = crypto_skcipher_ivsize(skcipher); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); } static void aead_done(struct caam_drv_req *drv_req, u32 status) @@ -924,6 +920,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; dma_addr_t qm_sg_dma, iv_dma = 0; int ivsize = 0; @@ -945,13 +942,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } if (likely(req->src == req->dst)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } @@ -964,23 +961,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, return ERR_PTR(-ENOMEM); } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); qi_cache_free(edesc); return ERR_PTR(dst_nents); } @@ -1019,9 +1014,24 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, /* * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed */ - qm_sg_ents = 1 + !!ivsize + mapped_src_nents + - (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + qm_sg_ents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_ents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_ents = max(pad_sg_nents(qm_sg_ents), + 1 + !!ivsize + pad_sg_nents(mapped_src_nents)); + else + qm_sg_ents = pad_sg_nents(qm_sg_ents); + sg_table = &edesc->sgt[0]; qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > @@ -1029,7 +1039,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1044,7 +1054,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(qidev, iv_dma)) { dev_err(qidev, "unable to map IV\n"); caam_unmap(qidev, req->src, req->dst, src_nents, - dst_nents, 0, 0, 0, 0); + dst_nents, 0, 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1063,7 +1073,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(qidev, edesc->assoclen_dma)) { dev_err(qidev, "unable to map assoclen\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1074,19 +1084,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); qm_sg_index++; } - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); qm_sg_index += mapped_src_nents; if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - qm_sg_index, 0); + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(qidev, qm_sg_dma)) { dev_err(qidev, "unable to map S/G table\n"); dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1109,7 +1118,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + (1 + !!ivsize) * sizeof(*sg_table), out_len, 0); - } else if (mapped_dst_nents == 1) { + } else if (mapped_dst_nents <= 1) { dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 0); } else { @@ -1182,33 +1191,28 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status) struct device *qidev = caam_ctx->qidev; int ivsize = crypto_skcipher_ivsize(skcipher); -#ifdef DEBUG - dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); -#endif + dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); edesc = container_of(drv_req, typeof(*edesc), drv_req); if (status) caam_jr_strstatus(qidev, status); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); -#endif skcipher_unmap(qidev, edesc, req); /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - if (edesc->drv_req.drv_ctx->op_type == ENCRYPT) - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - - ivsize, ivsize, 0); + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); qi_cache_free(edesc); skcipher_request_complete(req, status); @@ -1276,14 +1280,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents = 1 + mapped_src_nents; dst_sg_idx = qm_sg_ents; - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); + else + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); + qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + ivsize > CAAM_QI_MEMCACHE_SIZE)) { dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1292,7 +1308,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, if (unlikely(!edesc)) { dev_err(qidev, "could not allocate extended descriptor\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1301,11 +1317,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, iv = (u8 *)(sg_table + qm_sg_ents); memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(qidev, iv_dma)) { dev_err(qidev, "unable to map IV\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1319,18 +1335,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, edesc->drv_req.drv_ctx = drv_ctx; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); - if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - dst_sg_idx, 0); + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { dev_err(qidev, "unable to map S/G table\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1340,16 +1358,14 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, ivsize + req->cryptlen, 0); - if (req->src == req->dst) { + if (req->src == req->dst) dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + - sizeof(*sg_table), req->cryptlen, 0); - } else if (mapped_dst_nents > 1) { + sizeof(*sg_table), req->cryptlen + ivsize, + 0); + else dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * - sizeof(*sg_table), req->cryptlen, 0); - } else { - dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), - req->cryptlen, 0); - } + sizeof(*sg_table), req->cryptlen + ivsize, + 0); return edesc; } @@ -1359,7 +1375,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); - int ivsize = crypto_skcipher_ivsize(skcipher); int ret; if (unlikely(caam_congested)) @@ -1370,14 +1385,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - if (!encrypt) - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - - ivsize, ivsize, 0); - ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); if (!ret) { ret = -EINPROGRESS; @@ -2382,6 +2389,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, bool uses_dkp) { struct caam_drv_private *priv; + struct device *dev; /* * distribute tfms across job rings to ensure in-order @@ -2393,16 +2401,17 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, return PTR_ERR(ctx->jrdev); } - priv = dev_get_drvdata(ctx->jrdev->parent); + dev = ctx->jrdev->parent; + priv = dev_get_drvdata(dev); if (priv->era >= 6 && uses_dkp) ctx->dir = DMA_BIDIRECTIONAL; else ctx->dir = DMA_TO_DEVICE; - ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), + ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key), ctx->dir); - if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { - dev_err(ctx->jrdev, "unable to map key\n"); + if (dma_mapping_error(dev, ctx->key_dma)) { + dev_err(dev, "unable to map key\n"); caam_jr_free(ctx->jrdev); return -ENOMEM; } @@ -2411,7 +2420,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; - ctx->qidev = priv->qidev; + ctx->qidev = dev; spin_lock_init(&ctx->lock); ctx->drv_ctx[ENCRYPT] = NULL; @@ -2445,7 +2454,8 @@ static void caam_exit_common(struct caam_ctx *ctx) caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); - dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); + dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key), + ctx->dir); caam_jr_free(ctx->jrdev); } @@ -2460,7 +2470,7 @@ static void caam_aead_exit(struct crypto_aead *tfm) caam_exit_common(crypto_aead_ctx(tfm)); } -static void __exit caam_qi_algapi_exit(void) +void caam_qi_algapi_exit(void) { int i; @@ -2505,45 +2515,17 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg) alg->exit = caam_aead_exit; } -static int __init caam_qi_algapi_init(void) +int caam_qi_algapi_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int i = 0, err = 0; u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; unsigned int md_limit = SHA512_DIGEST_SIZE; bool registered = false; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - of_node_put(dev_node); - if (!pdev) - return -ENODEV; - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv || !priv->qi_present) { - err = -ENODEV; - goto out_put_dev; - } - if (caam_dpaa2) { dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); - err = -ENODEV; - goto out_put_dev; + return -ENODEV; } /* @@ -2598,7 +2580,7 @@ static int __init caam_qi_algapi_init(void) err = crypto_register_skcipher(&t_alg->skcipher); if (err) { - dev_warn(priv->qidev, "%s alg registration failed\n", + dev_warn(ctrldev, "%s alg registration failed\n", t_alg->skcipher.base.cra_driver_name); continue; } @@ -2654,16 +2636,7 @@ static int __init caam_qi_algapi_init(void) } if (registered) - dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); + dev_info(ctrldev, "algorithms registered in /proc/crypto\n"); -out_put_dev: - put_device(ctrldev); return err; } - -module_init(caam_qi_algapi_init); -module_exit(caam_qi_algapi_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); -MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 2b2980a8a9b9..06bf32c32cbd 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* * Copyright 2015-2016 Freescale Semiconductor Inc. - * Copyright 2017-2018 NXP + * Copyright 2017-2019 NXP */ #include "compat.h" @@ -140,7 +140,8 @@ static struct caam_request *to_caam_req(struct crypto_async_request *areq) static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, int dst_nents, dma_addr_t iv_dma, int ivsize, - dma_addr_t qm_sg_dma, int qm_sg_bytes) + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) { if (dst != src) { if (src_nents) @@ -152,7 +153,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); if (qm_sg_bytes) dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); @@ -371,6 +372,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; dma_addr_t qm_sg_dma, iv_dma = 0; int ivsize = 0; @@ -387,23 +389,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } if (unlikely(req->dst != req->src)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); qi_cache_free(edesc); return ERR_PTR(dst_nents); } @@ -434,13 +434,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, mapped_dst_nents = 0; } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } @@ -460,9 +460,25 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, /* * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed */ - qm_sg_nents = 1 + !!ivsize + mapped_src_nents + - (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + qm_sg_nents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_nents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_nents = max(pad_sg_nents(qm_sg_nents), + 1 + !!ivsize + + pad_sg_nents(mapped_src_nents)); + else + qm_sg_nents = pad_sg_nents(qm_sg_nents); + sg_table = &edesc->sgt[0]; qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > @@ -470,7 +486,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_nents, ivsize); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -485,7 +501,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(dev, iv_dma)) { dev_err(dev, "unable to map IV\n"); caam_unmap(dev, req->src, req->dst, src_nents, - dst_nents, 0, 0, 0, 0); + dst_nents, 0, 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -509,7 +525,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(dev, edesc->assoclen_dma)) { dev_err(dev, "unable to map assoclen\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -520,19 +536,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); qm_sg_index++; } - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); qm_sg_index += mapped_src_nents; if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - qm_sg_index, 0); + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(dev, qm_sg_dma)) { dev_err(dev, "unable to map S/G table\n"); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -559,6 +574,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dpaa2_fl_set_addr(out_fle, qm_sg_dma + (1 + !!ivsize) * sizeof(*sg_table)); } + } else if (!mapped_dst_nents) { + /* + * crypto engine requires the output entry to be present when + * "frame list" FD is used. + * Since engine does not support FMT=2'b11 (unused entry type), + * leaving out_fle zeroized is the best option. + */ + goto skip_out_fle; } else if (mapped_dst_nents == 1) { dpaa2_fl_set_format(out_fle, dpaa2_fl_single); dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); @@ -570,6 +593,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dpaa2_fl_set_len(out_fle, out_len); +skip_out_fle: return edesc; } @@ -1077,14 +1101,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) qm_sg_ents = 1 + mapped_src_nents; dst_sg_idx = qm_sg_ents; - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); + else + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); + qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + ivsize > CAAM_QI_MEMCACHE_SIZE)) { dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1093,7 +1129,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) if (unlikely(!edesc)) { dev_err(dev, "could not allocate extended descriptor\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1102,11 +1138,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) iv = (u8 *)(sg_table + qm_sg_ents); memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, iv_dma)) { dev_err(dev, "unable to map IV\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1117,18 +1153,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) edesc->qm_sg_bytes = qm_sg_bytes; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); - if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - dst_sg_idx, 0); + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(dev, edesc->qm_sg_dma)) { dev_err(dev, "unable to map S/G table\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1136,23 +1174,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); dpaa2_fl_set_final(in_fle, true); dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); - dpaa2_fl_set_len(out_fle, req->cryptlen); + dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); - if (req->src == req->dst) { - dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + + if (req->src == req->dst) dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + sizeof(*sg_table)); - } else if (mapped_dst_nents > 1) { - dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + else dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * sizeof(*sg_table)); - } else { - dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); - } return edesc; } @@ -1164,7 +1198,8 @@ static void aead_unmap(struct device *dev, struct aead_edesc *edesc, int ivsize = crypto_aead_ivsize(aead); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); } @@ -1175,7 +1210,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, int ivsize = crypto_skcipher_ivsize(skcipher); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); } static void aead_encrypt_done(void *cbk_ctx, u32 status) @@ -1324,7 +1360,7 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status) print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1332,10 +1368,10 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status) /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, - ivsize, 0); + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); qi_cache_free(edesc); skcipher_request_complete(req, ecode); @@ -1362,11 +1398,19 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status) print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); skcipher_unmap(ctx->dev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); + qi_cache_free(edesc); skcipher_request_complete(req, ecode); } @@ -1405,7 +1449,6 @@ static int skcipher_decrypt(struct skcipher_request *req) struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct caam_request *caam_req = skcipher_request_ctx(req); - int ivsize = crypto_skcipher_ivsize(skcipher); int ret; /* allocate extended descriptor */ @@ -1413,13 +1456,6 @@ static int skcipher_decrypt(struct skcipher_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize, - ivsize, 0); - caam_req->flc = &ctx->flc[DECRYPT]; caam_req->flc_dma = ctx->flc_dma[DECRYPT]; caam_req->cbk = skcipher_decrypt_done; @@ -3380,9 +3416,9 @@ static int ahash_update_ctx(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -3409,7 +3445,7 @@ static int ahash_update_ctx(struct ahash_request *req) edesc->src_nents = src_nents; qm_sg_src_index = 1 + (*buflen ? 1 : 0); - qm_sg_bytes = (qm_sg_src_index + mapped_nents) * + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; @@ -3423,7 +3459,7 @@ static int ahash_update_ctx(struct ahash_request *req) goto unmap_ctx; if (mapped_nents) { - sg_to_qm_sg_last(req->src, mapped_nents, + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_src_index, 0); if (*next_buflen) scatterwalk_map_and_copy(next_buf, req->src, @@ -3494,7 +3530,7 @@ static int ahash_final_ctx(struct ahash_request *req) gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); - int qm_sg_bytes, qm_sg_src_index; + int qm_sg_bytes; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; struct dpaa2_sg_entry *sg_table; @@ -3505,8 +3541,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (!edesc) return -ENOMEM; - qm_sg_src_index = 1 + (buflen ? 1 : 0); - qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, @@ -3518,7 +3553,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true); + dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -3599,7 +3634,8 @@ static int ahash_finup_ctx(struct ahash_request *req) edesc->src_nents = src_nents; qm_sg_src_index = 1 + (buflen ? 1 : 0); - qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * + sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, @@ -3611,7 +3647,7 @@ static int ahash_finup_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -3696,8 +3732,8 @@ static int ahash_digest(struct ahash_request *req) int qm_sg_bytes; struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; - qm_sg_bytes = mapped_nents * sizeof(*sg_table); - sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); + qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { @@ -3840,9 +3876,9 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - *next_buflen); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -3868,14 +3904,15 @@ static int ahash_update_no_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * + sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); if (ret) goto unmap_ctx; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); if (*next_buflen) scatterwalk_map_and_copy(next_buf, req->src, @@ -3987,14 +4024,14 @@ static int ahash_finup_no_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); if (ret) goto unmap; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -4064,9 +4101,9 @@ static int ahash_update_first(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -4101,8 +4138,9 @@ static int ahash_update_first(struct ahash_request *req) if (mapped_nents > 1) { int qm_sg_bytes; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); - qm_sg_bytes = mapped_nents * sizeof(*sg_table); + sg_to_qm_sg_last(req->src, src_len, sg_table, 0); + qm_sg_bytes = pad_sg_nents(mapped_nents) * + sizeof(*sg_table); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 7205d9f4029e..e4ac5d591ad6 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -82,14 +82,6 @@ #define HASH_MSG_LEN 8 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) -#ifdef DEBUG -/* for print_hex_dumps with line references */ -#define debug(format, arg...) printk(format, arg) -#else -#define debug(format, arg...) -#endif - - static struct list_head hash_list; /* ahash per-session context */ @@ -243,11 +235,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, true, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); /* ahash_update_first shared descriptor */ desc = ctx->sh_desc_update_first; @@ -255,11 +246,9 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update first shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* ahash_final shared descriptor */ desc = ctx->sh_desc_fin; @@ -267,11 +256,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, true, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* ahash_digest shared descriptor */ desc = ctx->sh_desc_digest; @@ -279,12 +267,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash digest shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); return 0; } @@ -328,9 +314,9 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, ctx->key_dma); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); - print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), - 1); + print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* shared descriptor for ahash_digest */ desc = ctx->sh_desc_digest; @@ -377,8 +363,8 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, 0); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); - print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, + print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); /* shared descriptor for ahash_digest */ @@ -429,12 +415,11 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("key_in@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); result.err = 0; init_completion(&result.completion); @@ -444,11 +429,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, /* in progress */ wait_for_completion(&result.completion); ret = result.err; -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "digested key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1); -#endif + + print_hex_dump_debug("digested key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, + digestsize, 1); } dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL); @@ -463,15 +447,14 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, unsigned int keylen) { struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct device *jrdev = ctx->jrdev; int blocksize = crypto_tfm_alg_blocksize(&ahash->base); int digestsize = crypto_ahash_digestsize(ahash); struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); int ret; u8 *hashed_key = NULL; -#ifdef DEBUG - printk(KERN_ERR "keylen %d\n", keylen); -#endif + dev_dbg(jrdev, "keylen %d\n", keylen); if (keylen > blocksize) { hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); @@ -600,11 +583,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); int digestsize = crypto_ahash_digestsize(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -614,11 +595,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err, memcpy(req->result, state->caam_ctx, digestsize); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); -#endif + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); req->base.complete(&req->base, err); } @@ -631,11 +610,9 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -645,15 +622,13 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, switch_buf(state); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); if (req->result) - print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); -#endif + print_hex_dump_debug("result@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); req->base.complete(&req->base, err); } @@ -666,11 +641,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); int digestsize = crypto_ahash_digestsize(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -680,11 +653,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, memcpy(req->result, state->caam_ctx, digestsize); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); -#endif + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); req->base.complete(&req->base, err); } @@ -697,11 +668,9 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -711,15 +680,13 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, switch_buf(state); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); if (req->result) - print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); -#endif + print_hex_dump_debug("result@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); req->base.complete(&req->base, err); } @@ -759,9 +726,10 @@ static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, if (nents > 1 || first_sg) { struct sec4_sg_entry *sg = edesc->sec4_sg; - unsigned int sgsize = sizeof(*sg) * (first_sg + nents); + unsigned int sgsize = sizeof(*sg) * + pad_sg_nents(first_sg + nents); - sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); + sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0); src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); if (dma_mapping_error(ctx->jrdev, src_dma)) { @@ -819,8 +787,10 @@ static int ahash_update_ctx(struct ahash_request *req) } if (to_hash) { - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + int pad_nents; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(jrdev, "Invalid number of src SG.\n"); return src_nents; @@ -838,15 +808,14 @@ static int ahash_update_ctx(struct ahash_request *req) } sec4_sg_src_index = 1 + (*buflen ? 1 : 0); - sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * - sizeof(struct sec4_sg_entry); + pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); /* * allocate space for base edesc and hw desc commands, * link tables */ - edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, - ctx->sh_desc_update, + edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update, ctx->sh_desc_update_dma, flags); if (!edesc) { dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); @@ -866,7 +835,7 @@ static int ahash_update_ctx(struct ahash_request *req) goto unmap_ctx; if (mapped_nents) - sg_to_sec4_sg_last(req->src, mapped_nents, + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + sec4_sg_src_index, 0); else @@ -893,11 +862,9 @@ static int ahash_update_ctx(struct ahash_request *req) append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); if (ret) @@ -910,13 +877,12 @@ static int ahash_update_ctx(struct ahash_request *req) *buflen = *next_buflen; *next_buflen = last_buflen; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, + *next_buflen, 1); return ret; unmap_ctx: @@ -935,18 +901,17 @@ static int ahash_final_ctx(struct ahash_request *req) GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); u32 *desc; - int sec4_sg_bytes, sec4_sg_src_index; + int sec4_sg_bytes; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; int ret; - sec4_sg_src_index = 1 + (buflen ? 1 : 0); - sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); + sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * + sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index, - ctx->sh_desc_fin, ctx->sh_desc_fin_dma, - flags); + edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin, + ctx->sh_desc_fin_dma, flags); if (!edesc) return -ENOMEM; @@ -963,7 +928,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); + sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0)); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); @@ -977,10 +942,9 @@ static int ahash_final_ctx(struct ahash_request *req) LDST_SGF); append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); if (ret) @@ -1058,10 +1022,9 @@ static int ahash_finup_ctx(struct ahash_request *req) append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); if (ret) @@ -1135,10 +1098,9 @@ static int ahash_digest(struct ahash_request *req) return -ENOMEM; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1190,10 +1152,9 @@ static int ahash_final_no_ctx(struct ahash_request *req) if (ret) goto unmap; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1246,8 +1207,10 @@ static int ahash_update_no_ctx(struct ahash_request *req) } if (to_hash) { - src_nents = sg_nents_for_len(req->src, - req->nbytes - *next_buflen); + int pad_nents; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(jrdev, "Invalid number of src SG.\n"); return src_nents; @@ -1264,14 +1227,14 @@ static int ahash_update_no_ctx(struct ahash_request *req) mapped_nents = 0; } - sec4_sg_bytes = (1 + mapped_nents) * - sizeof(struct sec4_sg_entry); + pad_nents = pad_sg_nents(1 + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); /* * allocate space for base edesc and hw desc commands, * link tables */ - edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents, + edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update_first, ctx->sh_desc_update_first_dma, flags); @@ -1287,8 +1250,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_sec4_sg_last(req->src, mapped_nents, - edesc->sec4_sg + 1, 0); + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0); if (*next_buflen) { scatterwalk_map_and_copy(next_buf, req->src, @@ -1313,11 +1275,9 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); if (ret) @@ -1333,13 +1293,12 @@ static int ahash_update_no_ctx(struct ahash_request *req) *buflen = *next_buflen; *next_buflen = 0; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, + 1); return ret; unmap_ctx: @@ -1414,10 +1373,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req) if (ret) goto unmap; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1517,11 +1475,9 @@ static int ahash_update_first(struct ahash_request *req) if (ret) goto unmap_ctx; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); if (ret) @@ -1539,11 +1495,10 @@ static int ahash_update_first(struct ahash_request *req) req->nbytes, 0); switch_buf(state); } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, + 1); return ret; unmap_ctx: @@ -1930,7 +1885,7 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm) caam_jr_free(ctx->jrdev); } -static void __exit caam_algapi_hash_exit(void) +void caam_algapi_hash_exit(void) { struct caam_hash_alg *t_alg, *n; @@ -1988,40 +1943,13 @@ caam_hash_alloc(struct caam_hash_template *template, return t_alg; } -static int __init caam_algapi_hash_init(void) +int caam_algapi_hash_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; int i = 0, err = 0; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); unsigned int md_limit = SHA512_DIGEST_SIZE; u32 md_inst, md_vid; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* * Register crypto algorithms the device supports. First, identify * presence and attributes of MD block. @@ -2042,10 +1970,8 @@ static int __init caam_algapi_hash_init(void) * Skip registration of any hashing algorithms if MD block * is not present. */ - if (!md_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!md_inst) + return -ENODEV; /* Limit digest size based on LP256 */ if (md_vid == CHA_VER_VID_MD_LP256) @@ -2102,14 +2028,5 @@ static int __init caam_algapi_hash_init(void) list_add_tail(&t_alg->entry, &hash_list); } -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_algapi_hash_init); -module_exit(caam_algapi_hash_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG"); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index fe24485274e1..80574106af29 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for Public Key Cryptography * * Copyright 2016 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP * * There is no Shared Descriptor for PKC so that the Job Descriptor must carry * all the desired key parameters, input and output pointers. @@ -24,12 +24,18 @@ sizeof(struct rsa_priv_f2_pdb)) #define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \ sizeof(struct rsa_priv_f3_pdb)) +#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */ + +/* buffer filled with zeros, used for padding */ +static u8 *zero_buffer; static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, struct akcipher_request *req) { + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE); - dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE); if (edesc->sec4_sg_bytes) dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes, @@ -168,6 +174,13 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err, akcipher_request_complete(req, err); } +/** + * Count leading zeros, need it to strip, from a given scatterlist + * + * @sgl : scatterlist to count zeros from + * @nbytes: number of zeros, in bytes, to strip + * @flags : operation flags + */ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl, unsigned int nbytes, unsigned int flags) @@ -187,7 +200,8 @@ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl, lzeros = 0; len = 0; while (nbytes > 0) { - while (len && !*buff) { + /* do not strip more than given bytes */ + while (len && !*buff && lzeros < nbytes) { lzeros++; len--; buff++; @@ -218,6 +232,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct device *dev = ctx->dev; struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct caam_rsa_key *key = &ctx->key; struct rsa_edesc *edesc; gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; @@ -225,22 +240,45 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, int sgc; int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; int src_nents, dst_nents; + unsigned int diff_size = 0; int lzeros; - lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags); - if (lzeros < 0) - return ERR_PTR(lzeros); - - req->src_len -= lzeros; - req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros); + if (req->src_len > key->n_sz) { + /* + * strip leading zeros and + * return the number of zeros to skip + */ + lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len - + key->n_sz, sg_flags); + if (lzeros < 0) + return ERR_PTR(lzeros); + + req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src, + lzeros); + req_ctx->fixup_src_len = req->src_len - lzeros; + } else { + /* + * input src is less then n key modulus, + * so there will be zero padding + */ + diff_size = key->n_sz - req->src_len; + req_ctx->fixup_src = req->src; + req_ctx->fixup_src_len = req->src_len; + } - src_nents = sg_nents_for_len(req->src, req->src_len); + src_nents = sg_nents_for_len(req_ctx->fixup_src, + req_ctx->fixup_src_len); dst_nents = sg_nents_for_len(req->dst, req->dst_len); - if (src_nents > 1) - sec4_sg_len = src_nents; + if (!diff_size && src_nents == 1) + sec4_sg_len = 0; /* no need for an input hw s/g table */ + else + sec4_sg_len = src_nents + !!diff_size; + sec4_sg_index = sec4_sg_len; if (dst_nents > 1) - sec4_sg_len += dst_nents; + sec4_sg_len += pad_sg_nents(dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); @@ -250,7 +288,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, if (!edesc) return ERR_PTR(-ENOMEM); - sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + sgc = dma_map_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE); if (unlikely(!sgc)) { dev_err(dev, "unable to map source\n"); goto src_fail; @@ -263,14 +301,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, } edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen; + if (diff_size) + dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size, + 0); + + if (sec4_sg_index) + sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len, + edesc->sec4_sg + !!diff_size, 0); - sec4_sg_index = 0; - if (src_nents > 1) { - sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); - sec4_sg_index += src_nents; - } if (dst_nents > 1) - sg_to_sec4_sg_last(req->dst, dst_nents, + sg_to_sec4_sg_last(req->dst, req->dst_len, edesc->sec4_sg + sec4_sg_index, 0); /* Save nents for later use in Job Descriptor */ @@ -289,12 +329,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, edesc->sec4_sg_bytes = sec4_sg_bytes; + print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + edesc->sec4_sg_bytes, 1); + return edesc; sec4_sg_fail: dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); dst_fail: - dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE); src_fail: kfree(edesc); return ERR_PTR(-ENOMEM); @@ -304,6 +348,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, struct rsa_edesc *edesc) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct caam_rsa_key *key = &ctx->key; struct device *dev = ctx->dev; @@ -328,7 +373,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, pdb->f_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->f_dma = sg_dma_address(req->src); + pdb->f_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -340,7 +385,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, } pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz; - pdb->f_len = req->src_len; + pdb->f_len = req_ctx->fixup_src_len; return 0; } @@ -373,7 +418,9 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -436,7 +483,9 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -523,7 +572,9 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -978,6 +1029,15 @@ static int caam_rsa_init_tfm(struct crypto_akcipher *tfm) return PTR_ERR(ctx->dev); } + ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer, + CAAM_RSA_MAX_INPUT_SIZE - 1, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, ctx->padding_dma)) { + dev_err(ctx->dev, "unable to map padding\n"); + caam_jr_free(ctx->dev); + return -ENOMEM; + } + return 0; } @@ -987,6 +1047,8 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm) struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct caam_rsa_key *key = &ctx->key; + dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE - + 1, DMA_TO_DEVICE); caam_rsa_free_key(key); caam_jr_free(ctx->dev); } @@ -1010,41 +1072,12 @@ static struct akcipher_alg caam_rsa = { }; /* Public Key Cryptography module initialization handler */ -static int __init caam_pkc_init(void) +int caam_pkc_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); u32 pk_inst; int err; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* Determine public key hardware accelerator presence. */ if (priv->era < 10) pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & @@ -1053,31 +1086,29 @@ static int __init caam_pkc_init(void) pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK; /* Do not register algorithms if PKHA is not present. */ - if (!pk_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!pk_inst) + return 0; + + /* allocate zero buffer, used for padding input */ + zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA | + GFP_KERNEL); + if (!zero_buffer) + return -ENOMEM; err = crypto_register_akcipher(&caam_rsa); - if (err) + if (err) { + kfree(zero_buffer); dev_warn(ctrldev, "%s alg registration failed\n", caam_rsa.base.cra_driver_name); - else + } else { dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n"); + } -out_put_dev: - put_device(ctrldev); return err; } -static void __exit caam_pkc_exit(void) +void caam_pkc_exit(void) { + kfree(zero_buffer); crypto_unregister_akcipher(&caam_rsa); } - -module_init(caam_pkc_init); -module_exit(caam_pkc_exit); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API"); -MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h index 82645bcf8b27..2c488c9a3812 100644 --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -89,18 +89,25 @@ struct caam_rsa_key { * caam_rsa_ctx - per session context. * @key : RSA key in DMA zone * @dev : device structure + * @padding_dma : dma address of padding, for adding it to the input */ struct caam_rsa_ctx { struct caam_rsa_key key; struct device *dev; + dma_addr_t padding_dma; + }; /** * caam_rsa_req_ctx - per request context. - * @src: input scatterlist (stripped of leading zeros) + * @src : input scatterlist (stripped of leading zeros) + * @fixup_src : input scatterlist (that might be stripped of leading zeros) + * @fixup_src_len : length of the fixup_src input scatterlist */ struct caam_rsa_req_ctx { struct scatterlist src[2]; + struct scatterlist *fixup_src; + unsigned int fixup_src_len; }; /** diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 95eb5402c59f..561bcb535184 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for hw_random * * Copyright 2011 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP * * Based on caamalg.c crypto API driver. * @@ -113,10 +113,8 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context) /* Buffer refilled, invalidate cache */ dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng refreshed buf@: ", - DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1); -#endif + print_hex_dump_debug("rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4, + bd->buf, RN_BUF_SIZE, 1); } static inline int submit_job(struct caam_rng_ctx *ctx, int to_current) @@ -209,10 +207,10 @@ static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx) dev_err(jrdev, "unable to map shared descriptor\n"); return -ENOMEM; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); + return 0; } @@ -233,10 +231,10 @@ static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) } append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); + return 0; } @@ -296,47 +294,20 @@ static struct hwrng caam_rng = { .read = caam_read, }; -static void __exit caam_rng_exit(void) +void caam_rng_exit(void) { caam_jr_free(rng_ctx->jrdev); hwrng_unregister(&caam_rng); kfree(rng_ctx); } -static int __init caam_rng_init(void) +int caam_rng_init(struct device *ctrldev) { struct device *dev; - struct device_node *dev_node; - struct platform_device *pdev; - struct caam_drv_private *priv; u32 rng_inst; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int err; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* Check for an instantiated RNG before registration */ if (priv->era < 10) rng_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & @@ -344,16 +315,13 @@ static int __init caam_rng_init(void) else rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK; - if (!rng_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!rng_inst) + return 0; dev = caam_jr_alloc(); if (IS_ERR(dev)) { pr_err("Job Ring Device allocation for transform failed\n"); - err = PTR_ERR(dev); - goto out_put_dev; + return PTR_ERR(dev); } rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL); if (!rng_ctx) { @@ -364,7 +332,6 @@ static int __init caam_rng_init(void) if (err) goto free_rng_ctx; - put_device(&pdev->dev); dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); @@ -372,14 +339,5 @@ free_rng_ctx: kfree(rng_ctx); free_caam_alloc: caam_jr_free(dev); -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_rng_init); -module_exit(caam_rng_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for hw_random API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG"); diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index fec39c35c877..4e43ca4d3656 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -3,7 +3,7 @@ * Controller-level driver, kernel property detection, initialization * * Copyright 2008-2012 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP */ #include <linux/device.h> @@ -323,8 +323,8 @@ static int caam_remove(struct platform_device *pdev) of_platform_depopulate(ctrldev); #ifdef CONFIG_CAAM_QI - if (ctrlpriv->qidev) - caam_qi_shutdown(ctrlpriv->qidev); + if (ctrlpriv->qi_init) + caam_qi_shutdown(ctrldev); #endif /* @@ -540,7 +540,8 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->caam_ipg = clk; if (!of_machine_is_compatible("fsl,imx7d") && - !of_machine_is_compatible("fsl,imx7s")) { + !of_machine_is_compatible("fsl,imx7s") && + !of_machine_is_compatible("fsl,imx7ulp")) { clk = caam_drv_identify_clk(&pdev->dev, "mem"); if (IS_ERR(clk)) { ret = PTR_ERR(clk); @@ -562,7 +563,8 @@ static int caam_probe(struct platform_device *pdev) if (!of_machine_is_compatible("fsl,imx6ul") && !of_machine_is_compatible("fsl,imx7d") && - !of_machine_is_compatible("fsl,imx7s")) { + !of_machine_is_compatible("fsl,imx7s") && + !of_machine_is_compatible("fsl,imx7ulp")) { clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); if (IS_ERR(clk)) { ret = PTR_ERR(clk); @@ -702,12 +704,7 @@ static int caam_probe(struct platform_device *pdev) } ctrlpriv->era = caam_get_era(ctrl); - - ret = of_platform_populate(nprop, caam_match, NULL, dev); - if (ret) { - dev_err(dev, "JR platform devices creation error\n"); - goto iounmap_ctrl; - } + ctrlpriv->domain = iommu_get_domain_for_dev(dev); #ifdef CONFIG_DEBUG_FS /* @@ -721,19 +718,6 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); #endif - ring = 0; - for_each_available_child_of_node(nprop, np) - if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || - of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { - ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) - ((__force uint8_t *)ctrl + - (ring + JR_BLOCK_NUMBER) * - BLOCK_OFFSET - ); - ctrlpriv->total_jobrs++; - ring++; - } - /* Check to see if (DPAA 1.x) QI present. If so, enable */ ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); if (ctrlpriv->qi_present && !caam_dpaa2) { @@ -752,6 +736,25 @@ static int caam_probe(struct platform_device *pdev) #endif } + ret = of_platform_populate(nprop, caam_match, NULL, dev); + if (ret) { + dev_err(dev, "JR platform devices creation error\n"); + goto shutdown_qi; + } + + ring = 0; + for_each_available_child_of_node(nprop, np) + if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || + of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { + ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) + ((__force uint8_t *)ctrl + + (ring + JR_BLOCK_NUMBER) * + BLOCK_OFFSET + ); + ctrlpriv->total_jobrs++; + ring++; + } + /* If no QI and no rings specified, quit and go home */ if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) { dev_err(dev, "no queues configured, terminating\n"); @@ -898,6 +901,11 @@ caam_remove: caam_remove(pdev); return ret; +shutdown_qi: +#ifdef CONFIG_CAAM_QI + if (ctrlpriv->qi_init) + caam_qi_shutdown(dev); +#endif iounmap_ctrl: iounmap(ctrl); disable_caam_emi_slow: diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 2980b8ef1fb1..5988a26a2441 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -3,6 +3,7 @@ * caam descriptor construction helper functions * * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019 NXP */ #ifndef DESC_CONSTR_H @@ -37,6 +38,16 @@ extern bool caam_little_end; +/* + * HW fetches 4 S/G table entries at a time, irrespective of how many entries + * are in the table. It's SW's responsibility to make sure these accesses + * do not have side effects. + */ +static inline int pad_sg_nents(int sg_nents) +{ + return ALIGN(sg_nents, 4); +} + static inline int desc_len(u32 * const desc) { return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 4da844e4b61d..4f0d45865aa2 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -13,7 +13,7 @@ #ifdef DEBUG #include <linux/highmem.h> -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii) { @@ -35,15 +35,15 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, buf = it_page + it->offset; len = min_t(size_t, tlen, it->length); - print_hex_dump(level, prefix_str, prefix_type, rowsize, - groupsize, buf, len, ascii); + print_hex_dump_debug(prefix_str, prefix_type, rowsize, + groupsize, buf, len, ascii); tlen -= len; kunmap_atomic(it_page); } } #else -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii) {} diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h index 8c6b83e02a70..d9726e66edbf 100644 --- a/drivers/crypto/caam/error.h +++ b/drivers/crypto/caam/error.h @@ -17,7 +17,7 @@ void caam_strstatus(struct device *dev, u32 status, bool qi_v2); #define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false) #define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true) -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii); diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 3392615dc91b..6af84bbc612c 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -4,7 +4,7 @@ * Private/internal definitions between modules * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * + * Copyright 2019 NXP */ #ifndef INTERN_H @@ -63,10 +63,6 @@ struct caam_drv_private_jr { * Driver-private storage for a single CAAM block instance */ struct caam_drv_private { -#ifdef CONFIG_CAAM_QI - struct device *qidev; -#endif - /* Physical-presence section */ struct caam_ctrl __iomem *ctrl; /* controller region */ struct caam_deco __iomem *deco; /* DECO/CCB views */ @@ -74,12 +70,17 @@ struct caam_drv_private { struct caam_queue_if __iomem *qi; /* QI control region */ struct caam_job_ring __iomem *jr[4]; /* JobR's register space */ + struct iommu_domain *domain; + /* * Detected geometry block. Filled in from device tree if powerpc, * or from register-based version detection code */ u8 total_jobrs; /* Total Job Rings in device */ u8 qi_present; /* Nonzero if QI present in device */ +#ifdef CONFIG_CAAM_QI + u8 qi_init; /* Nonzero if QI has been initialized */ +#endif u8 mc_en; /* Nonzero if MC f/w is active */ int secvio_irq; /* Security violation interrupt number */ int virt_en; /* Virtualization enabled in CAAM */ @@ -107,8 +108,95 @@ struct caam_drv_private { #endif }; -void caam_jr_algapi_init(struct device *dev); -void caam_jr_algapi_remove(struct device *dev); +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API + +int caam_algapi_init(struct device *dev); +void caam_algapi_exit(void); + +#else + +static inline int caam_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API + +int caam_algapi_hash_init(struct device *dev); +void caam_algapi_hash_exit(void); + +#else + +static inline int caam_algapi_hash_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_hash_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API + +int caam_pkc_init(struct device *dev); +void caam_pkc_exit(void); + +#else + +static inline int caam_pkc_init(struct device *dev) +{ + return 0; +} + +static inline void caam_pkc_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API + +int caam_rng_init(struct device *dev); +void caam_rng_exit(void); + +#else + +static inline int caam_rng_init(struct device *dev) +{ + return 0; +} + +static inline void caam_rng_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */ + +#ifdef CONFIG_CAAM_QI + +int caam_qi_algapi_init(struct device *dev); +void caam_qi_algapi_exit(void); + +#else + +static inline int caam_qi_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_qi_algapi_exit(void) +{ +} + +#endif /* CONFIG_CAAM_QI */ #ifdef CONFIG_DEBUG_FS static int caam_debugfs_u64_get(void *data, u64 *val) diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 1de2562d0982..cea811fed320 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -4,6 +4,7 @@ * JobR backend functionality * * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019 NXP */ #include <linux/of_irq.h> @@ -23,6 +24,43 @@ struct jr_driver_data { } ____cacheline_aligned; static struct jr_driver_data driver_data; +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +static void register_algs(struct device *dev) +{ + mutex_lock(&algs_lock); + + if (++active_devs != 1) + goto algs_unlock; + + caam_algapi_init(dev); + caam_algapi_hash_init(dev); + caam_pkc_init(dev); + caam_rng_init(dev); + caam_qi_algapi_init(dev); + +algs_unlock: + mutex_unlock(&algs_lock); +} + +static void unregister_algs(void) +{ + mutex_lock(&algs_lock); + + if (--active_devs != 0) + goto algs_unlock; + + caam_qi_algapi_exit(); + + caam_rng_exit(); + caam_pkc_exit(); + caam_algapi_hash_exit(); + caam_algapi_exit(); + +algs_unlock: + mutex_unlock(&algs_lock); +} static int caam_reset_hw_jr(struct device *dev) { @@ -109,6 +147,9 @@ static int caam_jr_remove(struct platform_device *pdev) return -EBUSY; } + /* Unregister JR-based RNG & crypto algorithms */ + unregister_algs(); + /* Remove the node from Physical JobR list maintained by driver */ spin_lock(&driver_data.jr_alloc_lock); list_del(&jrpriv->list_node); @@ -541,6 +582,8 @@ static int caam_jr_probe(struct platform_device *pdev) atomic_set(&jrpriv->tfm_count, 0); + register_algs(jrdev->parent); + return 0; } diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index 8d0713fae6ac..48dd3536060d 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -16,9 +16,7 @@ void split_key_done(struct device *dev, u32 *desc, u32 err, { struct split_key_result *res = context; -#ifdef DEBUG - dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); if (err) caam_jr_strstatus(dev, err); @@ -55,12 +53,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out, adata->keylen_pad = split_key_pad_len(adata->algtype & OP_ALG_ALGSEL_MASK); -#ifdef DEBUG - dev_err(jrdev, "split keylen %d split keylen padded %d\n", + dev_dbg(jrdev, "split keylen %d split keylen padded %d\n", adata->keylen, adata->keylen_pad); - print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); -#endif + print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); if (adata->keylen_pad > max_keylen) return -EINVAL; @@ -102,10 +98,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out, append_fifo_store(desc, dma_addr, adata->keylen, LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); result.err = 0; init_completion(&result.completion); @@ -115,11 +110,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out, /* in progress */ wait_for_completion(&result.completion); ret = result.err; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_out, - adata->keylen_pad, 1); -#endif + + print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_out, + adata->keylen_pad, 1); } dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL); diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 9f08f84cca59..0fe618e3804a 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -4,7 +4,7 @@ * Queue Interface backend functionality * * Copyright 2013-2016 Freescale Semiconductor, Inc. - * Copyright 2016-2017 NXP + * Copyright 2016-2017, 2019 NXP */ #include <linux/cpumask.h> @@ -18,6 +18,7 @@ #include "desc_constr.h" #define PREHDR_RSLS_SHIFT 31 +#define PREHDR_ABS BIT(25) /* * Use a reasonable backlog of frames (per CPU) as congestion threshold, @@ -58,11 +59,9 @@ static DEFINE_PER_CPU(int, last_cpu); /* * caam_qi_priv - CAAM QI backend private params * @cgr: QMan congestion group - * @qi_pdev: platform device for QI backend */ struct caam_qi_priv { struct qman_cgr cgr; - struct platform_device *qi_pdev; }; static struct caam_qi_priv qipriv ____cacheline_aligned; @@ -95,6 +94,16 @@ static u64 times_congested; */ static struct kmem_cache *qi_cache; +static void *caam_iova_to_virt(struct iommu_domain *domain, + dma_addr_t iova_addr) +{ + phys_addr_t phys_addr; + + phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; + + return phys_to_virt(phys_addr); +} + int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) { struct qm_fd fd; @@ -135,6 +144,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, const struct qm_fd *fd; struct caam_drv_req *drv_req; struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); fd = &msg->ern.fd; @@ -143,7 +153,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, return; } - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); if (!drv_req) { dev_err(qidev, "Can't find original request for CAAM response\n"); @@ -346,6 +356,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); dma_sync_single_for_device(qidev, drv_ctx->context_a, sizeof(drv_ctx->sh_desc) + @@ -401,6 +412,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, @@ -488,7 +500,7 @@ EXPORT_SYMBOL(caam_drv_ctx_rel); void caam_qi_shutdown(struct device *qidev) { int i; - struct caam_qi_priv *priv = dev_get_drvdata(qidev); + struct caam_qi_priv *priv = &qipriv; const cpumask_t *cpus = qman_affine_cpus(); for_each_cpu(i, cpus) { @@ -506,8 +518,6 @@ void caam_qi_shutdown(struct device *qidev) qman_release_cgrid(priv->cgr.cgrid); kmem_cache_destroy(qi_cache); - - platform_device_unregister(priv->qi_pdev); } static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) @@ -550,6 +560,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, struct caam_drv_req *drv_req; const struct qm_fd *fd; struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); u32 status; if (caam_qi_napi_schedule(p, caam_napi)) @@ -572,7 +583,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, return qman_cb_dqrr_consume; } - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); if (unlikely(!drv_req)) { dev_err(qidev, "Can't find original request for caam response\n"); @@ -692,33 +703,17 @@ static void free_rsp_fqs(void) int caam_qi_init(struct platform_device *caam_pdev) { int err, i; - struct platform_device *qi_pdev; struct device *ctrldev = &caam_pdev->dev, *qidev; struct caam_drv_private *ctrlpriv; const cpumask_t *cpus = qman_affine_cpus(); - static struct platform_device_info qi_pdev_info = { - .name = "caam_qi", - .id = PLATFORM_DEVID_NONE - }; - - qi_pdev_info.parent = ctrldev; - qi_pdev_info.dma_mask = dma_get_mask(ctrldev); - qi_pdev = platform_device_register_full(&qi_pdev_info); - if (IS_ERR(qi_pdev)) - return PTR_ERR(qi_pdev); - set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev)); ctrlpriv = dev_get_drvdata(ctrldev); - qidev = &qi_pdev->dev; - - qipriv.qi_pdev = qi_pdev; - dev_set_drvdata(qidev, &qipriv); + qidev = ctrldev; /* Initialize the congestion detection */ err = init_cgr(qidev); if (err) { dev_err(qidev, "CGR initialization failed: %d\n", err); - platform_device_unregister(qi_pdev); return err; } @@ -727,7 +722,6 @@ int caam_qi_init(struct platform_device *caam_pdev) if (err) { dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err); free_rsp_fqs(); - platform_device_unregister(qi_pdev); return err; } @@ -750,15 +744,11 @@ int caam_qi_init(struct platform_device *caam_pdev) napi_enable(irqtask); } - /* Hook up QI device to parent controlling caam device */ - ctrlpriv->qidev = qidev; - qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0, SLAB_CACHE_DMA, NULL); if (!qi_cache) { dev_err(qidev, "Can't allocate CAAM cache\n"); free_rsp_fqs(); - platform_device_unregister(qi_pdev); return -ENOMEM; } @@ -766,6 +756,8 @@ int caam_qi_init(struct platform_device *caam_pdev) debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, ×_congested, &caam_fops_u64_ro); #endif + + ctrlpriv->qi_init = 1; dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); return 0; } diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h index b3e1aaaeffea..d56cc7efbc13 100644 --- a/drivers/crypto/caam/sg_sw_qm.h +++ b/drivers/crypto/caam/sg_sw_qm.h @@ -54,15 +54,19 @@ static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct qm_sg_entry * -sg_to_qm_sg(struct scatterlist *sg, int sg_count, +sg_to_qm_sg(struct scatterlist *sg, int len, struct qm_sg_entry *qm_sg_ptr, u16 offset) { - while (sg_count && sg) { - dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); qm_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return qm_sg_ptr - 1; } @@ -71,10 +75,10 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count, * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, struct qm_sg_entry *qm_sg_ptr, u16 offset) { - qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr)); } diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h index c9378402a5f8..b8b737d2b0ea 100644 --- a/drivers/crypto/caam/sg_sw_qm2.h +++ b/drivers/crypto/caam/sg_sw_qm2.h @@ -25,15 +25,19 @@ static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct dpaa2_sg_entry * -sg_to_qm_sg(struct scatterlist *sg, int sg_count, +sg_to_qm_sg(struct scatterlist *sg, int len, struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) { - while (sg_count && sg) { - dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); qm_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return qm_sg_ptr - 1; } @@ -42,11 +46,11 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count, * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) { - qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); dpaa2_sg_set_final(qm_sg_ptr, true); } diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index dbfa9fce33e0..07e1ee99273b 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -35,11 +35,9 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", - DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, - sizeof(struct sec4_sg_entry), 1); -#endif + + print_hex_dump_debug("sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, + sec4_sg_ptr, sizeof(struct sec4_sg_entry), 1); } /* @@ -47,15 +45,19 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct sec4_sg_entry * -sg_to_sec4_sg(struct scatterlist *sg, int sg_count, +sg_to_sec4_sg(struct scatterlist *sg, int len, struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { - while (sg_count) { - dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), ent_len, + offset); sec4_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return sec4_sg_ptr - 1; } @@ -72,11 +74,11 @@ static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr) * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int len, struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { - sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); + sec4_sg_ptr = sg_to_sec4_sg(sg, len, sec4_sg_ptr, offset); sg_to_sec4_set_last(sec4_sg_ptr); } |