From a3310d64d7cb1ba0f9279e77d21f13a75fa66ab5 Mon Sep 17 00:00:00 2001 From: Yangbo Lu Date: Wed, 17 Jan 2018 15:29:23 +0800 Subject: [PATCH 16/30] crypto: support layerscape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is an integrated patch for layerscape sec support. Signed-off-by: Radu Alexe Signed-off-by: Fabio Estevam Signed-off-by: Arnd Bergmann Signed-off-by: Radu Alexe Signed-off-by: Tudor Ambarus Signed-off-by: Eric Biggers Signed-off-by: Giovanni Cabiddu Signed-off-by: Xulin Sun Signed-off-by: Ard Biesheuvel Signed-off-by: Marcus Folkesson Signed-off-by: Tudor Ambarus Signed-off-by: Andrew Lutomirski Signed-off-by: Wei Yongjun Signed-off-by: Masahiro Yamada Signed-off-by: Marcelo Cerri Signed-off-by: Arvind Yadav Signed-off-by: Herbert Xu Signed-off-by: Laura Abbott Signed-off-by: Horia Geantă Signed-off-by: Yangbo Lu --- crypto/Kconfig | 30 + crypto/Makefile | 4 + crypto/acompress.c | 169 + crypto/algboss.c | 12 +- crypto/crypto_user.c | 19 + crypto/scompress.c | 356 ++ crypto/tcrypt.c | 17 +- crypto/testmgr.c | 1708 +++--- crypto/testmgr.h | 1125 ++-- crypto/tls.c | 607 +++ drivers/crypto/caam/Kconfig | 77 +- drivers/crypto/caam/Makefile | 16 +- drivers/crypto/caam/caamalg.c | 2171 ++------ drivers/crypto/caam/caamalg_desc.c | 1961 +++++++ drivers/crypto/caam/caamalg_desc.h | 127 + drivers/crypto/caam/caamalg_qi.c | 2929 ++++++++++ drivers/crypto/caam/caamalg_qi2.c | 5920 +++++++++++++++++++++ drivers/crypto/caam/caamalg_qi2.h | 281 + drivers/crypto/caam/caamhash.c | 550 +- drivers/crypto/caam/caamhash_desc.c | 108 + drivers/crypto/caam/caamhash_desc.h | 49 + drivers/crypto/caam/caampkc.c | 471 +- drivers/crypto/caam/caampkc.h | 58 + drivers/crypto/caam/caamrng.c | 16 +- drivers/crypto/caam/compat.h | 1 + drivers/crypto/caam/ctrl.c | 358 +- drivers/crypto/caam/ctrl.h | 2 + drivers/crypto/caam/desc.h | 84 +- drivers/crypto/caam/desc_constr.h | 180 +- drivers/crypto/caam/dpseci.c | 859 +++ drivers/crypto/caam/dpseci.h | 395 ++ drivers/crypto/caam/dpseci_cmd.h | 261 + drivers/crypto/caam/error.c | 127 +- drivers/crypto/caam/error.h | 10 +- drivers/crypto/caam/intern.h | 31 +- drivers/crypto/caam/jr.c | 72 +- drivers/crypto/caam/jr.h | 2 + drivers/crypto/caam/key_gen.c | 32 +- drivers/crypto/caam/key_gen.h | 36 +- drivers/crypto/caam/pdb.h | 62 + drivers/crypto/caam/pkc_desc.c | 36 + drivers/crypto/caam/qi.c | 797 +++ drivers/crypto/caam/qi.h | 204 + drivers/crypto/caam/regs.h | 63 +- drivers/crypto/caam/sg_sw_qm.h | 126 + drivers/crypto/caam/sg_sw_qm2.h | 81 + drivers/crypto/caam/sg_sw_sec4.h | 60 +- drivers/net/wireless/rsi/rsi_91x_usb.c | 2 +- drivers/staging/wilc1000/linux_wlan.c | 2 +- drivers/staging/wilc1000/wilc_wfi_cfgoperations.c | 2 +- include/crypto/acompress.h | 269 + include/crypto/internal/acompress.h | 81 + include/crypto/internal/scompress.h | 136 + include/linux/crypto.h | 3 + include/uapi/linux/cryptouser.h | 5 + scripts/spelling.txt | 3 + sound/soc/amd/acp-pcm-dma.c | 2 +- 57 files changed, 19177 insertions(+), 3988 deletions(-) create mode 100644 crypto/acompress.c create mode 100644 crypto/scompress.c create mode 100644 crypto/tls.c create mode 100644 drivers/crypto/caam/caamalg_desc.c create mode 100644 drivers/crypto/caam/caamalg_desc.h create mode 100644 drivers/crypto/caam/caamalg_qi.c create mode 100644 drivers/crypto/caam/caamalg_qi2.c create mode 100644 drivers/crypto/caam/caamalg_qi2.h create mode 100644 drivers/crypto/caam/caamhash_desc.c create mode 100644 drivers/crypto/caam/caamhash_desc.h create mode 100644 drivers/crypto/caam/dpseci.c create mode 100644 drivers/crypto/caam/dpseci.h create mode 100644 drivers/crypto/caam/dpseci_cmd.h create mode 100644 drivers/crypto/caam/qi.c create mode 100644 drivers/crypto/caam/qi.h create mode 100644 drivers/crypto/caam/sg_sw_qm.h create mode 100644 drivers/crypto/caam/sg_sw_qm2.h create mode 100644 include/crypto/acompress.h create mode 100644 include/crypto/internal/acompress.h create mode 100644 include/crypto/internal/scompress.h --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -102,6 +102,15 @@ config CRYPTO_KPP select CRYPTO_ALGAPI select CRYPTO_KPP2 +config CRYPTO_ACOMP2 + tristate + select CRYPTO_ALGAPI2 + +config CRYPTO_ACOMP + tristate + select CRYPTO_ALGAPI + select CRYPTO_ACOMP2 + config CRYPTO_RSA tristate "RSA algorithm" select CRYPTO_AKCIPHER @@ -138,6 +147,7 @@ config CRYPTO_MANAGER2 select CRYPTO_BLKCIPHER2 if !CRYPTO_MANAGER_DISABLE_TESTS select CRYPTO_AKCIPHER2 if !CRYPTO_MANAGER_DISABLE_TESTS select CRYPTO_KPP2 if !CRYPTO_MANAGER_DISABLE_TESTS + select CRYPTO_ACOMP2 if !CRYPTO_MANAGER_DISABLE_TESTS config CRYPTO_USER tristate "Userspace cryptographic algorithm configuration" @@ -295,6 +305,26 @@ config CRYPTO_ECHAINIV a sequence number xored with a salt. This is the default algorithm for CBC. +config CRYPTO_TLS + tristate "TLS support" + select CRYPTO_AEAD + select CRYPTO_BLKCIPHER + select CRYPTO_MANAGER + select CRYPTO_HASH + select CRYPTO_NULL + select CRYPTO_AUTHENC + help + Support for TLS 1.0 record encryption and decryption + + This module adds support for encryption/decryption of TLS 1.0 frames + using blockcipher algorithms. The name of the resulting algorithm is + "tls10(hmac(),cbc())". By default, the generic base + algorithms are used (e.g. aes-generic, sha1-generic), but hardware + accelerated versions will be used automatically if available. + + User-space applications (OpenSSL, GnuTLS) can offload TLS 1.0 + operations through AF_ALG or cryptodev interfaces + comment "Block modes" config CRYPTO_CBC --- a/crypto/Makefile +++ b/crypto/Makefile @@ -51,6 +51,9 @@ rsa_generic-y += rsa_helper.o rsa_generic-y += rsa-pkcs1pad.o obj-$(CONFIG_CRYPTO_RSA) += rsa_generic.o +obj-$(CONFIG_CRYPTO_ACOMP2) += acompress.o +obj-$(CONFIG_CRYPTO_ACOMP2) += scompress.o + cryptomgr-y := algboss.o testmgr.o obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o @@ -115,6 +118,7 @@ obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_ge obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_common.o crct10dif_generic.o obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o +obj-$(CONFIG_CRYPTO_TLS) += tls.o obj-$(CONFIG_CRYPTO_LZO) += lzo.o obj-$(CONFIG_CRYPTO_LZ4) += lz4.o obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o --- /dev/null +++ b/crypto/acompress.c @@ -0,0 +1,169 @@ +/* + * Asynchronous Compression operations + * + * Copyright (c) 2016, Intel Corporation + * Authors: Weigang Li + * Giovanni Cabiddu + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static const struct crypto_type crypto_acomp_type; + +#ifdef CONFIG_NET +static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + struct crypto_report_acomp racomp; + + strncpy(racomp.type, "acomp", sizeof(racomp.type)); + + if (nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, + sizeof(struct crypto_report_acomp), &racomp)) + goto nla_put_failure; + return 0; + +nla_put_failure: + return -EMSGSIZE; +} +#else +static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + return -ENOSYS; +} +#endif + +static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) + __attribute__ ((unused)); + +static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) +{ + seq_puts(m, "type : acomp\n"); +} + +static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm) +{ + struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); + struct acomp_alg *alg = crypto_acomp_alg(acomp); + + alg->exit(acomp); +} + +static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); + struct acomp_alg *alg = crypto_acomp_alg(acomp); + + if (tfm->__crt_alg->cra_type != &crypto_acomp_type) + return crypto_init_scomp_ops_async(tfm); + + acomp->compress = alg->compress; + acomp->decompress = alg->decompress; + acomp->dst_free = alg->dst_free; + acomp->reqsize = alg->reqsize; + + if (alg->exit) + acomp->base.exit = crypto_acomp_exit_tfm; + + if (alg->init) + return alg->init(acomp); + + return 0; +} + +static unsigned int crypto_acomp_extsize(struct crypto_alg *alg) +{ + int extsize = crypto_alg_extsize(alg); + + if (alg->cra_type != &crypto_acomp_type) + extsize += sizeof(struct crypto_scomp *); + + return extsize; +} + +static const struct crypto_type crypto_acomp_type = { + .extsize = crypto_acomp_extsize, + .init_tfm = crypto_acomp_init_tfm, +#ifdef CONFIG_PROC_FS + .show = crypto_acomp_show, +#endif + .report = crypto_acomp_report, + .maskclear = ~CRYPTO_ALG_TYPE_MASK, + .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK, + .type = CRYPTO_ALG_TYPE_ACOMPRESS, + .tfmsize = offsetof(struct crypto_acomp, base), +}; + +struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type, + u32 mask) +{ + return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_alloc_acomp); + +struct acomp_req *acomp_request_alloc(struct crypto_acomp *acomp) +{ + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); + struct acomp_req *req; + + req = __acomp_request_alloc(acomp); + if (req && (tfm->__crt_alg->cra_type != &crypto_acomp_type)) + return crypto_acomp_scomp_alloc_ctx(req); + + return req; +} +EXPORT_SYMBOL_GPL(acomp_request_alloc); + +void acomp_request_free(struct acomp_req *req) +{ + struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); + + if (tfm->__crt_alg->cra_type != &crypto_acomp_type) + crypto_acomp_scomp_free_ctx(req); + + if (req->flags & CRYPTO_ACOMP_ALLOC_OUTPUT) { + acomp->dst_free(req->dst); + req->dst = NULL; + } + + __acomp_request_free(req); +} +EXPORT_SYMBOL_GPL(acomp_request_free); + +int crypto_register_acomp(struct acomp_alg *alg) +{ + struct crypto_alg *base = &alg->base; + + base->cra_type = &crypto_acomp_type; + base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; + base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS; + + return crypto_register_alg(base); +} +EXPORT_SYMBOL_GPL(crypto_register_acomp); + +int crypto_unregister_acomp(struct acomp_alg *alg) +{ + return crypto_unregister_alg(&alg->base); +} +EXPORT_SYMBOL_GPL(crypto_unregister_acomp); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Asynchronous compression type"); --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -247,17 +247,9 @@ static int cryptomgr_schedule_test(struc memcpy(param->alg, alg->cra_name, sizeof(param->alg)); type = alg->cra_flags; - /* This piece of crap needs to disappear into per-type test hooks. */ -#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS - type |= CRYPTO_ALG_TESTED; -#else - if (!((type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & - CRYPTO_ALG_TYPE_BLKCIPHER_MASK) && !(type & CRYPTO_ALG_GENIV) && - ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : - alg->cra_ablkcipher.ivsize)) + /* Do not test internal algorithms. */ + if (type & CRYPTO_ALG_INTERNAL) type |= CRYPTO_ALG_TESTED; -#endif param->type = type; --- a/crypto/crypto_user.c +++ b/crypto/crypto_user.c @@ -115,6 +115,21 @@ nla_put_failure: return -EMSGSIZE; } +static int crypto_report_acomp(struct sk_buff *skb, struct crypto_alg *alg) +{ + struct crypto_report_acomp racomp; + + strncpy(racomp.type, "acomp", sizeof(racomp.type)); + + if (nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, + sizeof(struct crypto_report_acomp), &racomp)) + goto nla_put_failure; + return 0; + +nla_put_failure: + return -EMSGSIZE; +} + static int crypto_report_akcipher(struct sk_buff *skb, struct crypto_alg *alg) { struct crypto_report_akcipher rakcipher; @@ -189,7 +204,11 @@ static int crypto_report_one(struct cryp goto nla_put_failure; break; + case CRYPTO_ALG_TYPE_ACOMPRESS: + if (crypto_report_acomp(skb, alg)) + goto nla_put_failure; + break; case CRYPTO_ALG_TYPE_AKCIPHER: if (crypto_report_akcipher(skb, alg)) goto nla_put_failure; --- /dev/null +++ b/crypto/scompress.c @@ -0,0 +1,356 @@ +/* + * Synchronous Compression operations + * + * Copyright 2015 LG Electronics Inc. + * Copyright (c) 2016, Intel Corporation + * Author: Giovanni Cabiddu + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static const struct crypto_type crypto_scomp_type; +static void * __percpu *scomp_src_scratches; +static void * __percpu *scomp_dst_scratches; +static int scomp_scratch_users; +static DEFINE_MUTEX(scomp_lock); + +#ifdef CONFIG_NET +static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + struct crypto_report_comp rscomp; + + strncpy(rscomp.type, "scomp", sizeof(rscomp.type)); + + if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, + sizeof(struct crypto_report_comp), &rscomp)) + goto nla_put_failure; + return 0; + +nla_put_failure: + return -EMSGSIZE; +} +#else +static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg) +{ + return -ENOSYS; +} +#endif + +static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) + __attribute__ ((unused)); + +static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) +{ + seq_puts(m, "type : scomp\n"); +} + +static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) +{ + return 0; +} + +static void crypto_scomp_free_scratches(void * __percpu *scratches) +{ + int i; + + if (!scratches) + return; + + for_each_possible_cpu(i) + vfree(*per_cpu_ptr(scratches, i)); + + free_percpu(scratches); +} + +static void * __percpu *crypto_scomp_alloc_scratches(void) +{ + void * __percpu *scratches; + int i; + + scratches = alloc_percpu(void *); + if (!scratches) + return NULL; + + for_each_possible_cpu(i) { + void *scratch; + + scratch = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); + if (!scratch) + goto error; + *per_cpu_ptr(scratches, i) = scratch; + } + + return scratches; + +error: + crypto_scomp_free_scratches(scratches); + return NULL; +} + +static void crypto_scomp_free_all_scratches(void) +{ + if (!--scomp_scratch_users) { + crypto_scomp_free_scratches(scomp_src_scratches); + crypto_scomp_free_scratches(scomp_dst_scratches); + scomp_src_scratches = NULL; + scomp_dst_scratches = NULL; + } +} + +static int crypto_scomp_alloc_all_scratches(void) +{ + if (!scomp_scratch_users++) { + scomp_src_scratches = crypto_scomp_alloc_scratches(); + if (!scomp_src_scratches) + return -ENOMEM; + scomp_dst_scratches = crypto_scomp_alloc_scratches(); + if (!scomp_dst_scratches) + return -ENOMEM; + } + return 0; +} + +static void crypto_scomp_sg_free(struct scatterlist *sgl) +{ + int i, n; + struct page *page; + + if (!sgl) + return; + + n = sg_nents(sgl); + for_each_sg(sgl, sgl, n, i) { + page = sg_page(sgl); + if (page) + __free_page(page); + } + + kfree(sgl); +} + +static struct scatterlist *crypto_scomp_sg_alloc(size_t size, gfp_t gfp) +{ + struct scatterlist *sgl; + struct page *page; + int i, n; + + n = ((size - 1) >> PAGE_SHIFT) + 1; + + sgl = kmalloc_array(n, sizeof(struct scatterlist), gfp); + if (!sgl) + return NULL; + + sg_init_table(sgl, n); + + for (i = 0; i < n; i++) { + page = alloc_page(gfp); + if (!page) + goto err; + sg_set_page(sgl + i, page, PAGE_SIZE, 0); + } + + return sgl; + +err: + sg_mark_end(sgl + i); + crypto_scomp_sg_free(sgl); + return NULL; +} + +static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + void **tfm_ctx = acomp_tfm_ctx(tfm); + struct crypto_scomp *scomp = *tfm_ctx; + void **ctx = acomp_request_ctx(req); + const int cpu = get_cpu(); + u8 *scratch_src = *per_cpu_ptr(scomp_src_scratches, cpu); + u8 *scratch_dst = *per_cpu_ptr(scomp_dst_scratches, cpu); + int ret; + + if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) { + ret = -EINVAL; + goto out; + } + + if (req->dst && !req->dlen) { + ret = -EINVAL; + goto out; + } + + if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE) + req->dlen = SCOMP_SCRATCH_SIZE; + + scatterwalk_map_and_copy(scratch_src, req->src, 0, req->slen, 0); + if (dir) + ret = crypto_scomp_compress(scomp, scratch_src, req->slen, + scratch_dst, &req->dlen, *ctx); + else + ret = crypto_scomp_decompress(scomp, scratch_src, req->slen, + scratch_dst, &req->dlen, *ctx); + if (!ret) { + if (!req->dst) { + req->dst = crypto_scomp_sg_alloc(req->dlen, + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC); + if (!req->dst) + goto out; + } + scatterwalk_map_and_copy(scratch_dst, req->dst, 0, req->dlen, + 1); + } +out: + put_cpu(); + return ret; +} + +static int scomp_acomp_compress(struct acomp_req *req) +{ + return scomp_acomp_comp_decomp(req, 1); +} + +static int scomp_acomp_decompress(struct acomp_req *req) +{ + return scomp_acomp_comp_decomp(req, 0); +} + +static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) +{ + struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); + + crypto_free_scomp(*ctx); +} + +int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) +{ + struct crypto_alg *calg = tfm->__crt_alg; + struct crypto_acomp *crt = __crypto_acomp_tfm(tfm); + struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); + struct crypto_scomp *scomp; + + if (!crypto_mod_get(calg)) + return -EAGAIN; + + scomp = crypto_create_tfm(calg, &crypto_scomp_type); + if (IS_ERR(scomp)) { + crypto_mod_put(calg); + return PTR_ERR(scomp); + } + + *ctx = scomp; + tfm->exit = crypto_exit_scomp_ops_async; + + crt->compress = scomp_acomp_compress; + crt->decompress = scomp_acomp_decompress; + crt->dst_free = crypto_scomp_sg_free; + crt->reqsize = sizeof(void *); + + return 0; +} + +struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req) +{ + struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); + struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); + struct crypto_scomp *scomp = *tfm_ctx; + void *ctx; + + ctx = crypto_scomp_alloc_ctx(scomp); + if (IS_ERR(ctx)) { + kfree(req); + return NULL; + } + + *req->__ctx = ctx; + + return req; +} + +void crypto_acomp_scomp_free_ctx(struct acomp_req *req) +{ + struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); + struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); + struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); + struct crypto_scomp *scomp = *tfm_ctx; + void *ctx = *req->__ctx; + + if (ctx) + crypto_scomp_free_ctx(scomp, ctx); +} + +static const struct crypto_type crypto_scomp_type = { + .extsize = crypto_alg_extsize, + .init_tfm = crypto_scomp_init_tfm, +#ifdef CONFIG_PROC_FS + .show = crypto_scomp_show, +#endif + .report = crypto_scomp_report, + .maskclear = ~CRYPTO_ALG_TYPE_MASK, + .maskset = CRYPTO_ALG_TYPE_MASK, + .type = CRYPTO_ALG_TYPE_SCOMPRESS, + .tfmsize = offsetof(struct crypto_scomp, base), +}; + +int crypto_register_scomp(struct scomp_alg *alg) +{ + struct crypto_alg *base = &alg->base; + int ret = -ENOMEM; + + mutex_lock(&scomp_lock); + if (crypto_scomp_alloc_all_scratches()) + goto error; + + base->cra_type = &crypto_scomp_type; + base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; + base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS; + + ret = crypto_register_alg(base); + if (ret) + goto error; + + mutex_unlock(&scomp_lock); + return ret; + +error: + crypto_scomp_free_all_scratches(); + mutex_unlock(&scomp_lock); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_register_scomp); + +int crypto_unregister_scomp(struct scomp_alg *alg) +{ + int ret; + + mutex_lock(&scomp_lock); + ret = crypto_unregister_alg(&alg->base); + crypto_scomp_free_all_scratches(); + mutex_unlock(&scomp_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(crypto_unregister_scomp); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Synchronous compression type"); --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -74,7 +74,7 @@ static char *check[] = { "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt", "camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320", "lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512", - NULL + "rsa", NULL }; struct tcrypt_result { @@ -1336,6 +1336,10 @@ static int do_test(const char *alg, u32 ret += tcrypt_test("hmac(sha3-512)"); break; + case 115: + ret += tcrypt_test("rsa"); + break; + case 150: ret += tcrypt_test("ansi_cprng"); break; @@ -1397,6 +1401,9 @@ static int do_test(const char *alg, u32 case 190: ret += tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))"); break; + case 191: + ret += tcrypt_test("tls10(hmac(sha1),cbc(aes))"); + break; case 200: test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); @@ -1411,9 +1418,9 @@ static int do_test(const char *alg, u32 test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, speed_template_32_40_48); test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, @@ -1844,9 +1851,9 @@ static int do_test(const char *alg, u32 test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, speed_template_32_40_48); test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, - speed_template_32_48_64); + speed_template_32_64); test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, speed_template_16_24_32); test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "internal.h" @@ -62,7 +63,7 @@ int alg_test(const char *driver, const c */ #define IDX1 32 #define IDX2 32400 -#define IDX3 1 +#define IDX3 1511 #define IDX4 8193 #define IDX5 22222 #define IDX6 17101 @@ -82,47 +83,54 @@ struct tcrypt_result { struct aead_test_suite { struct { - struct aead_testvec *vecs; + const struct aead_testvec *vecs; unsigned int count; } enc, dec; }; struct cipher_test_suite { struct { - struct cipher_testvec *vecs; + const struct cipher_testvec *vecs; unsigned int count; } enc, dec; }; struct comp_test_suite { struct { - struct comp_testvec *vecs; + const struct comp_testvec *vecs; unsigned int count; } comp, decomp; }; struct hash_test_suite { - struct hash_testvec *vecs; + const struct hash_testvec *vecs; unsigned int count; }; struct cprng_test_suite { - struct cprng_testvec *vecs; + const struct cprng_testvec *vecs; unsigned int count; }; struct drbg_test_suite { - struct drbg_testvec *vecs; + const struct drbg_testvec *vecs; unsigned int count; }; +struct tls_test_suite { + struct { + struct tls_testvec *vecs; + unsigned int count; + } enc, dec; +}; + struct akcipher_test_suite { - struct akcipher_testvec *vecs; + const struct akcipher_testvec *vecs; unsigned int count; }; struct kpp_test_suite { - struct kpp_testvec *vecs; + const struct kpp_testvec *vecs; unsigned int count; }; @@ -139,12 +147,14 @@ struct alg_test_desc { struct hash_test_suite hash; struct cprng_test_suite cprng; struct drbg_test_suite drbg; + struct tls_test_suite tls; struct akcipher_test_suite akcipher; struct kpp_test_suite kpp; } suite; }; -static unsigned int IDX[8] = { IDX1, IDX2, IDX3, IDX4, IDX5, IDX6, IDX7, IDX8 }; +static const unsigned int IDX[8] = { + IDX1, IDX2, IDX3, IDX4, IDX5, IDX6, IDX7, IDX8 }; static void hexdump(unsigned char *buf, unsigned int len) { @@ -202,7 +212,7 @@ static int wait_async_op(struct tcrypt_r } static int ahash_partial_update(struct ahash_request **preq, - struct crypto_ahash *tfm, struct hash_testvec *template, + struct crypto_ahash *tfm, const struct hash_testvec *template, void *hash_buff, int k, int temp, struct scatterlist *sg, const char *algo, char *result, struct tcrypt_result *tresult) { @@ -259,11 +269,12 @@ out_nostate: return ret; } -static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, - unsigned int tcount, bool use_digest, - const int align_offset) +static int __test_hash(struct crypto_ahash *tfm, + const struct hash_testvec *template, unsigned int tcount, + bool use_digest, const int align_offset) { const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); + size_t digest_size = crypto_ahash_digestsize(tfm); unsigned int i, j, k, temp; struct scatterlist sg[8]; char *result; @@ -274,7 +285,7 @@ static int __test_hash(struct crypto_aha char *xbuf[XBUFSIZE]; int ret = -ENOMEM; - result = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL); + result = kmalloc(digest_size, GFP_KERNEL); if (!result) return ret; key = kmalloc(MAX_KEYLEN, GFP_KERNEL); @@ -304,7 +315,7 @@ static int __test_hash(struct crypto_aha goto out; j++; - memset(result, 0, MAX_DIGEST_SIZE); + memset(result, 0, digest_size); hash_buff = xbuf[0]; hash_buff += align_offset; @@ -379,7 +390,7 @@ static int __test_hash(struct crypto_aha continue; j++; - memset(result, 0, MAX_DIGEST_SIZE); + memset(result, 0, digest_size); temp = 0; sg_init_table(sg, template[i].np); @@ -457,7 +468,7 @@ static int __test_hash(struct crypto_aha continue; j++; - memset(result, 0, MAX_DIGEST_SIZE); + memset(result, 0, digest_size); ret = -EINVAL; hash_buff = xbuf[0]; @@ -536,7 +547,8 @@ out_nobuf: return ret; } -static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, +static int test_hash(struct crypto_ahash *tfm, + const struct hash_testvec *template, unsigned int tcount, bool use_digest) { unsigned int alignmask; @@ -564,7 +576,7 @@ static int test_hash(struct crypto_ahash } static int __test_aead(struct crypto_aead *tfm, int enc, - struct aead_testvec *template, unsigned int tcount, + const struct aead_testvec *template, unsigned int tcount, const bool diff_dst, const int align_offset) { const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); @@ -955,7 +967,7 @@ out_noxbuf: } static int test_aead(struct crypto_aead *tfm, int enc, - struct aead_testvec *template, unsigned int tcount) + const struct aead_testvec *template, unsigned int tcount) { unsigned int alignmask; int ret; @@ -987,8 +999,236 @@ static int test_aead(struct crypto_aead return 0; } +static int __test_tls(struct crypto_aead *tfm, int enc, + struct tls_testvec *template, unsigned int tcount, + const bool diff_dst) +{ + const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); + unsigned int i, k, authsize; + char *q; + struct aead_request *req; + struct scatterlist *sg; + struct scatterlist *sgout; + const char *e, *d; + struct tcrypt_result result; + void *input; + void *output; + void *assoc; + char *iv; + char *key; + char *xbuf[XBUFSIZE]; + char *xoutbuf[XBUFSIZE]; + char *axbuf[XBUFSIZE]; + int ret = -ENOMEM; + + if (testmgr_alloc_buf(xbuf)) + goto out_noxbuf; + + if (diff_dst && testmgr_alloc_buf(xoutbuf)) + goto out_nooutbuf; + + if (testmgr_alloc_buf(axbuf)) + goto out_noaxbuf; + + iv = kzalloc(MAX_IVLEN, GFP_KERNEL); + if (!iv) + goto out_noiv; + + key = kzalloc(MAX_KEYLEN, GFP_KERNEL); + if (!key) + goto out_nokey; + + sg = kmalloc(sizeof(*sg) * 8 * (diff_dst ? 2 : 1), GFP_KERNEL); + if (!sg) + goto out_nosg; + + sgout = sg + 8; + + d = diff_dst ? "-ddst" : ""; + e = enc ? "encryption" : "decryption"; + + init_completion(&result.completion); + + req = aead_request_alloc(tfm, GFP_KERNEL); + if (!req) { + pr_err("alg: tls%s: Failed to allocate request for %s\n", + d, algo); + goto out; + } + + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); + + for (i = 0; i < tcount; i++) { + input = xbuf[0]; + assoc = axbuf[0]; + + ret = -EINVAL; + if (WARN_ON(template[i].ilen > PAGE_SIZE || + template[i].alen > PAGE_SIZE)) + goto out; + + memcpy(assoc, template[i].assoc, template[i].alen); + memcpy(input, template[i].input, template[i].ilen); + + if (template[i].iv) + memcpy(iv, template[i].iv, MAX_IVLEN); + else + memset(iv, 0, MAX_IVLEN); + + crypto_aead_clear_flags(tfm, ~0); + + if (template[i].klen > MAX_KEYLEN) { + pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n", + d, i, algo, template[i].klen, MAX_KEYLEN); + ret = -EINVAL; + goto out; + } + memcpy(key, template[i].key, template[i].klen); + + ret = crypto_aead_setkey(tfm, key, template[i].klen); + if (!ret == template[i].fail) { + pr_err("alg: tls%s: setkey failed on test %d for %s: flags=%x\n", + d, i, algo, crypto_aead_get_flags(tfm)); + goto out; + } else if (ret) + continue; + + authsize = 20; + ret = crypto_aead_setauthsize(tfm, authsize); + if (ret) { + pr_err("alg: aead%s: Failed to set authsize to %u on test %d for %s\n", + d, authsize, i, algo); + goto out; + } + + k = !!template[i].alen; + sg_init_table(sg, k + 1); + sg_set_buf(&sg[0], assoc, template[i].alen); + sg_set_buf(&sg[k], input, (enc ? template[i].rlen : + template[i].ilen)); + output = input; + + if (diff_dst) { + sg_init_table(sgout, k + 1); + sg_set_buf(&sgout[0], assoc, template[i].alen); + + output = xoutbuf[0]; + sg_set_buf(&sgout[k], output, + (enc ? template[i].rlen : template[i].ilen)); + } + + aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, + template[i].ilen, iv); + + aead_request_set_ad(req, template[i].alen); + + ret = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); + + switch (ret) { + case 0: + if (template[i].novrfy) { + /* verification was supposed to fail */ + pr_err("alg: tls%s: %s failed on test %d for %s: ret was 0, expected -EBADMSG\n", + d, e, i, algo); + /* so really, we got a bad message */ + ret = -EBADMSG; + goto out; + } + break; + case -EINPROGRESS: + case -EBUSY: + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) + break; + case -EBADMSG: + /* verification failure was expected */ + if (template[i].novrfy) + continue; + /* fall through */ + default: + pr_err("alg: tls%s: %s failed on test %d for %s: ret=%d\n", + d, e, i, algo, -ret); + goto out; + } + + q = output; + if (memcmp(q, template[i].result, template[i].rlen)) { + pr_err("alg: tls%s: Test %d failed on %s for %s\n", + d, i, e, algo); + hexdump(q, template[i].rlen); + pr_err("should be:\n"); + hexdump(template[i].result, template[i].rlen); + ret = -EINVAL; + goto out; + } + } + +out: + aead_request_free(req); + + kfree(sg); +out_nosg: + kfree(key); +out_nokey: + kfree(iv); +out_noiv: + testmgr_free_buf(axbuf); +out_noaxbuf: + if (diff_dst) + testmgr_free_buf(xoutbuf); +out_nooutbuf: + testmgr_free_buf(xbuf); +out_noxbuf: + return ret; +} + +static int test_tls(struct crypto_aead *tfm, int enc, + struct tls_testvec *template, unsigned int tcount) +{ + int ret; + /* test 'dst == src' case */ + ret = __test_tls(tfm, enc, template, tcount, false); + if (ret) + return ret; + /* test 'dst != src' case */ + return __test_tls(tfm, enc, template, tcount, true); +} + +static int alg_test_tls(const struct alg_test_desc *desc, const char *driver, + u32 type, u32 mask) +{ + struct crypto_aead *tfm; + int err = 0; + + tfm = crypto_alloc_aead(driver, type, mask); + if (IS_ERR(tfm)) { + pr_err("alg: aead: Failed to load transform for %s: %ld\n", + driver, PTR_ERR(tfm)); + return PTR_ERR(tfm); + } + + if (desc->suite.tls.enc.vecs) { + err = test_tls(tfm, ENCRYPT, desc->suite.tls.enc.vecs, + desc->suite.tls.enc.count); + if (err) + goto out; + } + + if (!err && desc->suite.tls.dec.vecs) + err = test_tls(tfm, DECRYPT, desc->suite.tls.dec.vecs, + desc->suite.tls.dec.count); + +out: + crypto_free_aead(tfm); + return err; +} + static int test_cipher(struct crypto_cipher *tfm, int enc, - struct cipher_testvec *template, unsigned int tcount) + const struct cipher_testvec *template, + unsigned int tcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm)); unsigned int i, j, k; @@ -1066,7 +1306,8 @@ out_nobuf: } static int __test_skcipher(struct crypto_skcipher *tfm, int enc, - struct cipher_testvec *template, unsigned int tcount, + const struct cipher_testvec *template, + unsigned int tcount, const bool diff_dst, const int align_offset) { const char *algo = @@ -1079,12 +1320,16 @@ static int __test_skcipher(struct crypto const char *e, *d; struct tcrypt_result result; void *data; - char iv[MAX_IVLEN]; + char *iv; char *xbuf[XBUFSIZE]; char *xoutbuf[XBUFSIZE]; int ret = -ENOMEM; unsigned int ivsize = crypto_skcipher_ivsize(tfm); + iv = kmalloc(MAX_IVLEN, GFP_KERNEL); + if (!iv) + return ret; + if (testmgr_alloc_buf(xbuf)) goto out_nobuf; @@ -1325,12 +1570,14 @@ out: testmgr_free_buf(xoutbuf); out_nooutbuf: testmgr_free_buf(xbuf); + kfree(iv); out_nobuf: return ret; } static int test_skcipher(struct crypto_skcipher *tfm, int enc, - struct cipher_testvec *template, unsigned int tcount) + const struct cipher_testvec *template, + unsigned int tcount) { unsigned int alignmask; int ret; @@ -1362,8 +1609,10 @@ static int test_skcipher(struct crypto_s return 0; } -static int test_comp(struct crypto_comp *tfm, struct comp_testvec *ctemplate, - struct comp_testvec *dtemplate, int ctcount, int dtcount) +static int test_comp(struct crypto_comp *tfm, + const struct comp_testvec *ctemplate, + const struct comp_testvec *dtemplate, + int ctcount, int dtcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm)); unsigned int i; @@ -1442,7 +1691,154 @@ out: return ret; } -static int test_cprng(struct crypto_rng *tfm, struct cprng_testvec *template, +static int test_acomp(struct crypto_acomp *tfm, + const struct comp_testvec *ctemplate, + const struct comp_testvec *dtemplate, + int ctcount, int dtcount) +{ + const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); + unsigned int i; + char *output; + int ret; + struct scatterlist src, dst; + struct acomp_req *req; + struct tcrypt_result result; + + output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!output) + return -ENOMEM; + + for (i = 0; i < ctcount; i++) { + unsigned int dlen = COMP_BUF_SIZE; + int ilen = ctemplate[i].inlen; + void *input_vec; + + input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL); + if (!input_vec) { + ret = -ENOMEM; + goto out; + } + + memset(output, 0, dlen); + init_completion(&result.completion); + sg_init_one(&src, input_vec, ilen); + sg_init_one(&dst, output, dlen); + + req = acomp_request_alloc(tfm); + if (!req) { + pr_err("alg: acomp: request alloc failed for %s\n", + algo); + kfree(input_vec); + ret = -ENOMEM; + goto out; + } + + acomp_request_set_params(req, &src, &dst, ilen, dlen); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); + + ret = wait_async_op(&result, crypto_acomp_compress(req)); + if (ret) { + pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + if (req->dlen != ctemplate[i].outlen) { + pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n", + i + 1, algo, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + if (memcmp(output, ctemplate[i].output, req->dlen)) { + pr_err("alg: acomp: Compression test %d failed for %s\n", + i + 1, algo); + hexdump(output, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + kfree(input_vec); + acomp_request_free(req); + } + + for (i = 0; i < dtcount; i++) { + unsigned int dlen = COMP_BUF_SIZE; + int ilen = dtemplate[i].inlen; + void *input_vec; + + input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL); + if (!input_vec) { + ret = -ENOMEM; + goto out; + } + + memset(output, 0, dlen); + init_completion(&result.completion); + sg_init_one(&src, input_vec, ilen); + sg_init_one(&dst, output, dlen); + + req = acomp_request_alloc(tfm); + if (!req) { + pr_err("alg: acomp: request alloc failed for %s\n", + algo); + kfree(input_vec); + ret = -ENOMEM; + goto out; + } + + acomp_request_set_params(req, &src, &dst, ilen, dlen); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &result); + + ret = wait_async_op(&result, crypto_acomp_decompress(req)); + if (ret) { + pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + if (req->dlen != dtemplate[i].outlen) { + pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n", + i + 1, algo, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + if (memcmp(output, dtemplate[i].output, req->dlen)) { + pr_err("alg: acomp: Decompression test %d failed for %s\n", + i + 1, algo); + hexdump(output, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + kfree(input_vec); + acomp_request_free(req); + } + + ret = 0; + +out: + kfree(output); + return ret; +} + +static int test_cprng(struct crypto_rng *tfm, + const struct cprng_testvec *template, unsigned int tcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm)); @@ -1509,7 +1905,7 @@ static int alg_test_aead(const struct al struct crypto_aead *tfm; int err = 0; - tfm = crypto_alloc_aead(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_aead(driver, type, mask); if (IS_ERR(tfm)) { printk(KERN_ERR "alg: aead: Failed to load transform for %s: " "%ld\n", driver, PTR_ERR(tfm)); @@ -1538,7 +1934,7 @@ static int alg_test_cipher(const struct struct crypto_cipher *tfm; int err = 0; - tfm = crypto_alloc_cipher(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_cipher(driver, type, mask); if (IS_ERR(tfm)) { printk(KERN_ERR "alg: cipher: Failed to load transform for " "%s: %ld\n", driver, PTR_ERR(tfm)); @@ -1567,7 +1963,7 @@ static int alg_test_skcipher(const struc struct crypto_skcipher *tfm; int err = 0; - tfm = crypto_alloc_skcipher(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_skcipher(driver, type, mask); if (IS_ERR(tfm)) { printk(KERN_ERR "alg: skcipher: Failed to load transform for " "%s: %ld\n", driver, PTR_ERR(tfm)); @@ -1593,22 +1989,38 @@ out: static int alg_test_comp(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask) { - struct crypto_comp *tfm; + struct crypto_comp *comp; + struct crypto_acomp *acomp; int err; + u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK; - tfm = crypto_alloc_comp(driver, type, mask); - if (IS_ERR(tfm)) { - printk(KERN_ERR "alg: comp: Failed to load transform for %s: " - "%ld\n", driver, PTR_ERR(tfm)); - return PTR_ERR(tfm); - } + if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) { + acomp = crypto_alloc_acomp(driver, type, mask); + if (IS_ERR(acomp)) { + pr_err("alg: acomp: Failed to load transform for %s: %ld\n", + driver, PTR_ERR(acomp)); + return PTR_ERR(acomp); + } + err = test_acomp(acomp, desc->suite.comp.comp.vecs, + desc->suite.comp.decomp.vecs, + desc->suite.comp.comp.count, + desc->suite.comp.decomp.count); + crypto_free_acomp(acomp); + } else { + comp = crypto_alloc_comp(driver, type, mask); + if (IS_ERR(comp)) { + pr_err("alg: comp: Failed to load transform for %s: %ld\n", + driver, PTR_ERR(comp)); + return PTR_ERR(comp); + } - err = test_comp(tfm, desc->suite.comp.comp.vecs, - desc->suite.comp.decomp.vecs, - desc->suite.comp.comp.count, - desc->suite.comp.decomp.count); + err = test_comp(comp, desc->suite.comp.comp.vecs, + desc->suite.comp.decomp.vecs, + desc->suite.comp.comp.count, + desc->suite.comp.decomp.count); - crypto_free_comp(tfm); + crypto_free_comp(comp); + } return err; } @@ -1618,7 +2030,7 @@ static int alg_test_hash(const struct al struct crypto_ahash *tfm; int err; - tfm = crypto_alloc_ahash(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_ahash(driver, type, mask); if (IS_ERR(tfm)) { printk(KERN_ERR "alg: hash: Failed to load transform for %s: " "%ld\n", driver, PTR_ERR(tfm)); @@ -1646,7 +2058,7 @@ static int alg_test_crc32c(const struct if (err) goto out; - tfm = crypto_alloc_shash(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_shash(driver, type, mask); if (IS_ERR(tfm)) { printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: " "%ld\n", driver, PTR_ERR(tfm)); @@ -1688,7 +2100,7 @@ static int alg_test_cprng(const struct a struct crypto_rng *rng; int err; - rng = crypto_alloc_rng(driver, type | CRYPTO_ALG_INTERNAL, mask); + rng = crypto_alloc_rng(driver, type, mask); if (IS_ERR(rng)) { printk(KERN_ERR "alg: cprng: Failed to load transform for %s: " "%ld\n", driver, PTR_ERR(rng)); @@ -1703,7 +2115,7 @@ static int alg_test_cprng(const struct a } -static int drbg_cavs_test(struct drbg_testvec *test, int pr, +static int drbg_cavs_test(const struct drbg_testvec *test, int pr, const char *driver, u32 type, u32 mask) { int ret = -EAGAIN; @@ -1715,7 +2127,7 @@ static int drbg_cavs_test(struct drbg_te if (!buf) return -ENOMEM; - drng = crypto_alloc_rng(driver, type | CRYPTO_ALG_INTERNAL, mask); + drng = crypto_alloc_rng(driver, type, mask); if (IS_ERR(drng)) { printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for " "%s\n", driver); @@ -1777,7 +2189,7 @@ static int alg_test_drbg(const struct al int err = 0; int pr = 0; int i = 0; - struct drbg_testvec *template = desc->suite.drbg.vecs; + const struct drbg_testvec *template = desc->suite.drbg.vecs; unsigned int tcount = desc->suite.drbg.count; if (0 == memcmp(driver, "drbg_pr_", 8)) @@ -1796,7 +2208,7 @@ static int alg_test_drbg(const struct al } -static int do_test_kpp(struct crypto_kpp *tfm, struct kpp_testvec *vec, +static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, const char *alg) { struct kpp_request *req; @@ -1888,7 +2300,7 @@ free_req: } static int test_kpp(struct crypto_kpp *tfm, const char *alg, - struct kpp_testvec *vecs, unsigned int tcount) + const struct kpp_testvec *vecs, unsigned int tcount) { int ret, i; @@ -1909,7 +2321,7 @@ static int alg_test_kpp(const struct alg struct crypto_kpp *tfm; int err = 0; - tfm = crypto_alloc_kpp(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_kpp(driver, type, mask); if (IS_ERR(tfm)) { pr_err("alg: kpp: Failed to load tfm for %s: %ld\n", driver, PTR_ERR(tfm)); @@ -1924,7 +2336,7 @@ static int alg_test_kpp(const struct alg } static int test_akcipher_one(struct crypto_akcipher *tfm, - struct akcipher_testvec *vecs) + const struct akcipher_testvec *vecs) { char *xbuf[XBUFSIZE]; struct akcipher_request *req; @@ -2044,7 +2456,8 @@ free_xbuf: } static int test_akcipher(struct crypto_akcipher *tfm, const char *alg, - struct akcipher_testvec *vecs, unsigned int tcount) + const struct akcipher_testvec *vecs, + unsigned int tcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm)); @@ -2068,7 +2481,7 @@ static int alg_test_akcipher(const struc struct crypto_akcipher *tfm; int err = 0; - tfm = crypto_alloc_akcipher(driver, type | CRYPTO_ALG_INTERNAL, mask); + tfm = crypto_alloc_akcipher(driver, type, mask); if (IS_ERR(tfm)) { pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n", driver, PTR_ERR(tfm)); @@ -2088,112 +2501,23 @@ static int alg_test_null(const struct al return 0; } +#define __VECS(tv) { .vecs = tv, .count = ARRAY_SIZE(tv) } + /* Please keep this list sorted by algorithm name. */ static const struct alg_test_desc alg_test_descs[] = { { - .alg = "__cbc-cast5-avx", - .test = alg_test_null, - }, { - .alg = "__cbc-cast6-avx", - .test = alg_test_null, - }, { - .alg = "__cbc-serpent-avx", - .test = alg_test_null, - }, { - .alg = "__cbc-serpent-avx2", - .test = alg_test_null, - }, { - .alg = "__cbc-serpent-sse2", - .test = alg_test_null, - }, { - .alg = "__cbc-twofish-avx", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-aes-aesni", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "__driver-cbc-camellia-aesni", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-camellia-aesni-avx2", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-cast5-avx", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-cast6-avx", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-serpent-avx", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-serpent-avx2", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-serpent-sse2", - .test = alg_test_null, - }, { - .alg = "__driver-cbc-twofish-avx", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-aes-aesni", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "__driver-ecb-camellia-aesni", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-camellia-aesni-avx2", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-cast5-avx", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-cast6-avx", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-serpent-avx", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-serpent-avx2", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-serpent-sse2", - .test = alg_test_null, - }, { - .alg = "__driver-ecb-twofish-avx", - .test = alg_test_null, - }, { - .alg = "__driver-gcm-aes-aesni", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "__ghash-pclmulqdqni", - .test = alg_test_null, - .fips_allowed = 1, - }, { .alg = "ansi_cprng", .test = alg_test_cprng, .suite = { - .cprng = { - .vecs = ansi_cprng_aes_tv_template, - .count = ANSI_CPRNG_AES_TEST_VECTORS - } + .cprng = __VECS(ansi_cprng_aes_tv_template) } }, { .alg = "authenc(hmac(md5),ecb(cipher_null))", .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = hmac_md5_ecb_cipher_null_enc_tv_template, - .count = HMAC_MD5_ECB_CIPHER_NULL_ENC_TEST_VECTORS - }, - .dec = { - .vecs = hmac_md5_ecb_cipher_null_dec_tv_template, - .count = HMAC_MD5_ECB_CIPHER_NULL_DEC_TEST_VECTORS - } + .enc = __VECS(hmac_md5_ecb_cipher_null_enc_tv_template), + .dec = __VECS(hmac_md5_ecb_cipher_null_dec_tv_template) } } }, { @@ -2201,12 +2525,7 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha1_aes_cbc_enc_tv_temp, - .count = - HMAC_SHA1_AES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha1_aes_cbc_enc_tv_temp) } } }, { @@ -2214,12 +2533,7 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha1_des_cbc_enc_tv_temp, - .count = - HMAC_SHA1_DES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha1_des_cbc_enc_tv_temp) } } }, { @@ -2228,12 +2542,7 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha1_des3_ede_cbc_enc_tv_temp, - .count = - HMAC_SHA1_DES3_EDE_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha1_des3_ede_cbc_enc_tv_temp) } } }, { @@ -2245,18 +2554,8 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha1_ecb_cipher_null_enc_tv_temp, - .count = - HMAC_SHA1_ECB_CIPHER_NULL_ENC_TEST_VEC - }, - .dec = { - .vecs = - hmac_sha1_ecb_cipher_null_dec_tv_temp, - .count = - HMAC_SHA1_ECB_CIPHER_NULL_DEC_TEST_VEC - } + .enc = __VECS(hmac_sha1_ecb_cipher_null_enc_tv_temp), + .dec = __VECS(hmac_sha1_ecb_cipher_null_dec_tv_temp) } } }, { @@ -2268,12 +2567,7 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha224_des_cbc_enc_tv_temp, - .count = - HMAC_SHA224_DES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha224_des_cbc_enc_tv_temp) } } }, { @@ -2282,12 +2576,7 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha224_des3_ede_cbc_enc_tv_temp, - .count = - HMAC_SHA224_DES3_EDE_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha224_des3_ede_cbc_enc_tv_temp) } } }, { @@ -2296,12 +2585,7 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha256_aes_cbc_enc_tv_temp, - .count = - HMAC_SHA256_AES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha256_aes_cbc_enc_tv_temp) } } }, { @@ -2309,12 +2593,7 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha256_des_cbc_enc_tv_temp, - .count = - HMAC_SHA256_DES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha256_des_cbc_enc_tv_temp) } } }, { @@ -2323,12 +2602,7 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha256_des3_ede_cbc_enc_tv_temp, - .count = - HMAC_SHA256_DES3_EDE_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha256_des3_ede_cbc_enc_tv_temp) } } }, { @@ -2344,12 +2618,7 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha384_des_cbc_enc_tv_temp, - .count = - HMAC_SHA384_DES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha384_des_cbc_enc_tv_temp) } } }, { @@ -2358,12 +2627,7 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha384_des3_ede_cbc_enc_tv_temp, - .count = - HMAC_SHA384_DES3_EDE_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha384_des3_ede_cbc_enc_tv_temp) } } }, { @@ -2380,12 +2644,7 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha512_aes_cbc_enc_tv_temp, - .count = - HMAC_SHA512_AES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha512_aes_cbc_enc_tv_temp) } } }, { @@ -2393,12 +2652,7 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha512_des_cbc_enc_tv_temp, - .count = - HMAC_SHA512_DES_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha512_des_cbc_enc_tv_temp) } } }, { @@ -2407,12 +2661,7 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = - hmac_sha512_des3_ede_cbc_enc_tv_temp, - .count = - HMAC_SHA512_DES3_EDE_CBC_ENC_TEST_VEC - } + .enc = __VECS(hmac_sha512_des3_ede_cbc_enc_tv_temp) } } }, { @@ -2429,14 +2678,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = aes_cbc_enc_tv_template, - .count = AES_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_cbc_dec_tv_template, - .count = AES_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(aes_cbc_enc_tv_template), + .dec = __VECS(aes_cbc_dec_tv_template) } } }, { @@ -2444,14 +2687,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = anubis_cbc_enc_tv_template, - .count = ANUBIS_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = anubis_cbc_dec_tv_template, - .count = ANUBIS_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(anubis_cbc_enc_tv_template), + .dec = __VECS(anubis_cbc_dec_tv_template) } } }, { @@ -2459,14 +2696,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = bf_cbc_enc_tv_template, - .count = BF_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = bf_cbc_dec_tv_template, - .count = BF_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(bf_cbc_enc_tv_template), + .dec = __VECS(bf_cbc_dec_tv_template) } } }, { @@ -2474,14 +2705,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = camellia_cbc_enc_tv_template, - .count = CAMELLIA_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = camellia_cbc_dec_tv_template, - .count = CAMELLIA_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(camellia_cbc_enc_tv_template), + .dec = __VECS(camellia_cbc_dec_tv_template) } } }, { @@ -2489,14 +2714,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast5_cbc_enc_tv_template, - .count = CAST5_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast5_cbc_dec_tv_template, - .count = CAST5_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(cast5_cbc_enc_tv_template), + .dec = __VECS(cast5_cbc_dec_tv_template) } } }, { @@ -2504,14 +2723,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast6_cbc_enc_tv_template, - .count = CAST6_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast6_cbc_dec_tv_template, - .count = CAST6_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(cast6_cbc_enc_tv_template), + .dec = __VECS(cast6_cbc_dec_tv_template) } } }, { @@ -2519,14 +2732,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = des_cbc_enc_tv_template, - .count = DES_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = des_cbc_dec_tv_template, - .count = DES_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(des_cbc_enc_tv_template), + .dec = __VECS(des_cbc_dec_tv_template) } } }, { @@ -2535,14 +2742,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = des3_ede_cbc_enc_tv_template, - .count = DES3_EDE_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = des3_ede_cbc_dec_tv_template, - .count = DES3_EDE_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(des3_ede_cbc_enc_tv_template), + .dec = __VECS(des3_ede_cbc_dec_tv_template) } } }, { @@ -2550,14 +2751,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = serpent_cbc_enc_tv_template, - .count = SERPENT_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = serpent_cbc_dec_tv_template, - .count = SERPENT_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(serpent_cbc_enc_tv_template), + .dec = __VECS(serpent_cbc_dec_tv_template) } } }, { @@ -2565,30 +2760,25 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = tf_cbc_enc_tv_template, - .count = TF_CBC_ENC_TEST_VECTORS - }, - .dec = { - .vecs = tf_cbc_dec_tv_template, - .count = TF_CBC_DEC_TEST_VECTORS - } + .enc = __VECS(tf_cbc_enc_tv_template), + .dec = __VECS(tf_cbc_dec_tv_template) } } }, { + .alg = "cbcmac(aes)", + .fips_allowed = 1, + .test = alg_test_hash, + .suite = { + .hash = __VECS(aes_cbcmac_tv_template) + } + }, { .alg = "ccm(aes)", .test = alg_test_aead, .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = aes_ccm_enc_tv_template, - .count = AES_CCM_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_ccm_dec_tv_template, - .count = AES_CCM_DEC_TEST_VECTORS - } + .enc = __VECS(aes_ccm_enc_tv_template), + .dec = __VECS(aes_ccm_dec_tv_template) } } }, { @@ -2596,14 +2786,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = chacha20_enc_tv_template, - .count = CHACHA20_ENC_TEST_VECTORS - }, - .dec = { - .vecs = chacha20_enc_tv_template, - .count = CHACHA20_ENC_TEST_VECTORS - }, + .enc = __VECS(chacha20_enc_tv_template), + .dec = __VECS(chacha20_enc_tv_template), } } }, { @@ -2611,20 +2795,14 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .test = alg_test_hash, .suite = { - .hash = { - .vecs = aes_cmac128_tv_template, - .count = CMAC_AES_TEST_VECTORS - } + .hash = __VECS(aes_cmac128_tv_template) } }, { .alg = "cmac(des3_ede)", .fips_allowed = 1, .test = alg_test_hash, .suite = { - .hash = { - .vecs = des3_ede_cmac64_tv_template, - .count = CMAC_DES3_EDE_TEST_VECTORS - } + .hash = __VECS(des3_ede_cmac64_tv_template) } }, { .alg = "compress_null", @@ -2633,94 +2811,30 @@ static const struct alg_test_desc alg_te .alg = "crc32", .test = alg_test_hash, .suite = { - .hash = { - .vecs = crc32_tv_template, - .count = CRC32_TEST_VECTORS - } + .hash = __VECS(crc32_tv_template) } }, { .alg = "crc32c", .test = alg_test_crc32c, .fips_allowed = 1, .suite = { - .hash = { - .vecs = crc32c_tv_template, - .count = CRC32C_TEST_VECTORS - } + .hash = __VECS(crc32c_tv_template) } }, { .alg = "crct10dif", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = crct10dif_tv_template, - .count = CRCT10DIF_TEST_VECTORS - } + .hash = __VECS(crct10dif_tv_template) } }, { - .alg = "cryptd(__driver-cbc-aes-aesni)", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "cryptd(__driver-cbc-camellia-aesni)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-cbc-camellia-aesni-avx2)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-cbc-serpent-avx2)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-aes-aesni)", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "cryptd(__driver-ecb-camellia-aesni)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-camellia-aesni-avx2)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-cast5-avx)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-cast6-avx)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-serpent-avx)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-serpent-avx2)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-serpent-sse2)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-ecb-twofish-avx)", - .test = alg_test_null, - }, { - .alg = "cryptd(__driver-gcm-aes-aesni)", - .test = alg_test_null, - .fips_allowed = 1, - }, { - .alg = "cryptd(__ghash-pclmulqdqni)", - .test = alg_test_null, - .fips_allowed = 1, - }, { .alg = "ctr(aes)", .test = alg_test_skcipher, .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = aes_ctr_enc_tv_template, - .count = AES_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_ctr_dec_tv_template, - .count = AES_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(aes_ctr_enc_tv_template), + .dec = __VECS(aes_ctr_dec_tv_template) } } }, { @@ -2728,14 +2842,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = bf_ctr_enc_tv_template, - .count = BF_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = bf_ctr_dec_tv_template, - .count = BF_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(bf_ctr_enc_tv_template), + .dec = __VECS(bf_ctr_dec_tv_template) } } }, { @@ -2743,14 +2851,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = camellia_ctr_enc_tv_template, - .count = CAMELLIA_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = camellia_ctr_dec_tv_template, - .count = CAMELLIA_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(camellia_ctr_enc_tv_template), + .dec = __VECS(camellia_ctr_dec_tv_template) } } }, { @@ -2758,14 +2860,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast5_ctr_enc_tv_template, - .count = CAST5_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast5_ctr_dec_tv_template, - .count = CAST5_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(cast5_ctr_enc_tv_template), + .dec = __VECS(cast5_ctr_dec_tv_template) } } }, { @@ -2773,14 +2869,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast6_ctr_enc_tv_template, - .count = CAST6_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast6_ctr_dec_tv_template, - .count = CAST6_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(cast6_ctr_enc_tv_template), + .dec = __VECS(cast6_ctr_dec_tv_template) } } }, { @@ -2788,29 +2878,18 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = des_ctr_enc_tv_template, - .count = DES_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = des_ctr_dec_tv_template, - .count = DES_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(des_ctr_enc_tv_template), + .dec = __VECS(des_ctr_dec_tv_template) } } }, { .alg = "ctr(des3_ede)", .test = alg_test_skcipher, + .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = des3_ede_ctr_enc_tv_template, - .count = DES3_EDE_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = des3_ede_ctr_dec_tv_template, - .count = DES3_EDE_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(des3_ede_ctr_enc_tv_template), + .dec = __VECS(des3_ede_ctr_dec_tv_template) } } }, { @@ -2818,14 +2897,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = serpent_ctr_enc_tv_template, - .count = SERPENT_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = serpent_ctr_dec_tv_template, - .count = SERPENT_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(serpent_ctr_enc_tv_template), + .dec = __VECS(serpent_ctr_dec_tv_template) } } }, { @@ -2833,14 +2906,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = tf_ctr_enc_tv_template, - .count = TF_CTR_ENC_TEST_VECTORS - }, - .dec = { - .vecs = tf_ctr_dec_tv_template, - .count = TF_CTR_DEC_TEST_VECTORS - } + .enc = __VECS(tf_ctr_enc_tv_template), + .dec = __VECS(tf_ctr_dec_tv_template) } } }, { @@ -2848,14 +2915,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cts_mode_enc_tv_template, - .count = CTS_MODE_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cts_mode_dec_tv_template, - .count = CTS_MODE_DEC_TEST_VECTORS - } + .enc = __VECS(cts_mode_enc_tv_template), + .dec = __VECS(cts_mode_dec_tv_template) } } }, { @@ -2864,14 +2925,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .comp = { - .comp = { - .vecs = deflate_comp_tv_template, - .count = DEFLATE_COMP_TEST_VECTORS - }, - .decomp = { - .vecs = deflate_decomp_tv_template, - .count = DEFLATE_DECOMP_TEST_VECTORS - } + .comp = __VECS(deflate_comp_tv_template), + .decomp = __VECS(deflate_decomp_tv_template) } } }, { @@ -2879,10 +2934,7 @@ static const struct alg_test_desc alg_te .test = alg_test_kpp, .fips_allowed = 1, .suite = { - .kpp = { - .vecs = dh_tv_template, - .count = DH_TEST_VECTORS - } + .kpp = __VECS(dh_tv_template) } }, { .alg = "digest_null", @@ -2892,30 +2944,21 @@ static const struct alg_test_desc alg_te .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_nopr_ctr_aes128_tv_template, - .count = ARRAY_SIZE(drbg_nopr_ctr_aes128_tv_template) - } + .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template) } }, { .alg = "drbg_nopr_ctr_aes192", .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_nopr_ctr_aes192_tv_template, - .count = ARRAY_SIZE(drbg_nopr_ctr_aes192_tv_template) - } + .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template) } }, { .alg = "drbg_nopr_ctr_aes256", .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_nopr_ctr_aes256_tv_template, - .count = ARRAY_SIZE(drbg_nopr_ctr_aes256_tv_template) - } + .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template) } }, { /* @@ -2930,11 +2973,7 @@ static const struct alg_test_desc alg_te .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_nopr_hmac_sha256_tv_template, - .count = - ARRAY_SIZE(drbg_nopr_hmac_sha256_tv_template) - } + .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template) } }, { /* covered by drbg_nopr_hmac_sha256 test */ @@ -2954,10 +2993,7 @@ static const struct alg_test_desc alg_te .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_nopr_sha256_tv_template, - .count = ARRAY_SIZE(drbg_nopr_sha256_tv_template) - } + .drbg = __VECS(drbg_nopr_sha256_tv_template) } }, { /* covered by drbg_nopr_sha256 test */ @@ -2973,10 +3009,7 @@ static const struct alg_test_desc alg_te .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_pr_ctr_aes128_tv_template, - .count = ARRAY_SIZE(drbg_pr_ctr_aes128_tv_template) - } + .drbg = __VECS(drbg_pr_ctr_aes128_tv_template) } }, { /* covered by drbg_pr_ctr_aes128 test */ @@ -2996,10 +3029,7 @@ static const struct alg_test_desc alg_te .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_pr_hmac_sha256_tv_template, - .count = ARRAY_SIZE(drbg_pr_hmac_sha256_tv_template) - } + .drbg = __VECS(drbg_pr_hmac_sha256_tv_template) } }, { /* covered by drbg_pr_hmac_sha256 test */ @@ -3019,10 +3049,7 @@ static const struct alg_test_desc alg_te .test = alg_test_drbg, .fips_allowed = 1, .suite = { - .drbg = { - .vecs = drbg_pr_sha256_tv_template, - .count = ARRAY_SIZE(drbg_pr_sha256_tv_template) - } + .drbg = __VECS(drbg_pr_sha256_tv_template) } }, { /* covered by drbg_pr_sha256 test */ @@ -3034,23 +3061,13 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .test = alg_test_null, }, { - .alg = "ecb(__aes-aesni)", - .test = alg_test_null, - .fips_allowed = 1, - }, { .alg = "ecb(aes)", .test = alg_test_skcipher, .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = aes_enc_tv_template, - .count = AES_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_dec_tv_template, - .count = AES_DEC_TEST_VECTORS - } + .enc = __VECS(aes_enc_tv_template), + .dec = __VECS(aes_dec_tv_template) } } }, { @@ -3058,14 +3075,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = anubis_enc_tv_template, - .count = ANUBIS_ENC_TEST_VECTORS - }, - .dec = { - .vecs = anubis_dec_tv_template, - .count = ANUBIS_DEC_TEST_VECTORS - } + .enc = __VECS(anubis_enc_tv_template), + .dec = __VECS(anubis_dec_tv_template) } } }, { @@ -3073,14 +3084,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = arc4_enc_tv_template, - .count = ARC4_ENC_TEST_VECTORS - }, - .dec = { - .vecs = arc4_dec_tv_template, - .count = ARC4_DEC_TEST_VECTORS - } + .enc = __VECS(arc4_enc_tv_template), + .dec = __VECS(arc4_dec_tv_template) } } }, { @@ -3088,14 +3093,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = bf_enc_tv_template, - .count = BF_ENC_TEST_VECTORS - }, - .dec = { - .vecs = bf_dec_tv_template, - .count = BF_DEC_TEST_VECTORS - } + .enc = __VECS(bf_enc_tv_template), + .dec = __VECS(bf_dec_tv_template) } } }, { @@ -3103,14 +3102,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = camellia_enc_tv_template, - .count = CAMELLIA_ENC_TEST_VECTORS - }, - .dec = { - .vecs = camellia_dec_tv_template, - .count = CAMELLIA_DEC_TEST_VECTORS - } + .enc = __VECS(camellia_enc_tv_template), + .dec = __VECS(camellia_dec_tv_template) } } }, { @@ -3118,14 +3111,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast5_enc_tv_template, - .count = CAST5_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast5_dec_tv_template, - .count = CAST5_DEC_TEST_VECTORS - } + .enc = __VECS(cast5_enc_tv_template), + .dec = __VECS(cast5_dec_tv_template) } } }, { @@ -3133,14 +3120,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast6_enc_tv_template, - .count = CAST6_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast6_dec_tv_template, - .count = CAST6_DEC_TEST_VECTORS - } + .enc = __VECS(cast6_enc_tv_template), + .dec = __VECS(cast6_dec_tv_template) } } }, { @@ -3151,14 +3132,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = des_enc_tv_template, - .count = DES_ENC_TEST_VECTORS - }, - .dec = { - .vecs = des_dec_tv_template, - .count = DES_DEC_TEST_VECTORS - } + .enc = __VECS(des_enc_tv_template), + .dec = __VECS(des_dec_tv_template) } } }, { @@ -3167,14 +3142,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = des3_ede_enc_tv_template, - .count = DES3_EDE_ENC_TEST_VECTORS - }, - .dec = { - .vecs = des3_ede_dec_tv_template, - .count = DES3_EDE_DEC_TEST_VECTORS - } + .enc = __VECS(des3_ede_enc_tv_template), + .dec = __VECS(des3_ede_dec_tv_template) } } }, { @@ -3197,14 +3166,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = khazad_enc_tv_template, - .count = KHAZAD_ENC_TEST_VECTORS - }, - .dec = { - .vecs = khazad_dec_tv_template, - .count = KHAZAD_DEC_TEST_VECTORS - } + .enc = __VECS(khazad_enc_tv_template), + .dec = __VECS(khazad_dec_tv_template) } } }, { @@ -3212,14 +3175,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = seed_enc_tv_template, - .count = SEED_ENC_TEST_VECTORS - }, - .dec = { - .vecs = seed_dec_tv_template, - .count = SEED_DEC_TEST_VECTORS - } + .enc = __VECS(seed_enc_tv_template), + .dec = __VECS(seed_dec_tv_template) } } }, { @@ -3227,14 +3184,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = serpent_enc_tv_template, - .count = SERPENT_ENC_TEST_VECTORS - }, - .dec = { - .vecs = serpent_dec_tv_template, - .count = SERPENT_DEC_TEST_VECTORS - } + .enc = __VECS(serpent_enc_tv_template), + .dec = __VECS(serpent_dec_tv_template) } } }, { @@ -3242,14 +3193,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = tea_enc_tv_template, - .count = TEA_ENC_TEST_VECTORS - }, - .dec = { - .vecs = tea_dec_tv_template, - .count = TEA_DEC_TEST_VECTORS - } + .enc = __VECS(tea_enc_tv_template), + .dec = __VECS(tea_dec_tv_template) } } }, { @@ -3257,14 +3202,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = tnepres_enc_tv_template, - .count = TNEPRES_ENC_TEST_VECTORS - }, - .dec = { - .vecs = tnepres_dec_tv_template, - .count = TNEPRES_DEC_TEST_VECTORS - } + .enc = __VECS(tnepres_enc_tv_template), + .dec = __VECS(tnepres_dec_tv_template) } } }, { @@ -3272,14 +3211,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = tf_enc_tv_template, - .count = TF_ENC_TEST_VECTORS - }, - .dec = { - .vecs = tf_dec_tv_template, - .count = TF_DEC_TEST_VECTORS - } + .enc = __VECS(tf_enc_tv_template), + .dec = __VECS(tf_dec_tv_template) } } }, { @@ -3287,14 +3220,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = xeta_enc_tv_template, - .count = XETA_ENC_TEST_VECTORS - }, - .dec = { - .vecs = xeta_dec_tv_template, - .count = XETA_DEC_TEST_VECTORS - } + .enc = __VECS(xeta_enc_tv_template), + .dec = __VECS(xeta_dec_tv_template) } } }, { @@ -3302,14 +3229,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = xtea_enc_tv_template, - .count = XTEA_ENC_TEST_VECTORS - }, - .dec = { - .vecs = xtea_dec_tv_template, - .count = XTEA_DEC_TEST_VECTORS - } + .enc = __VECS(xtea_enc_tv_template), + .dec = __VECS(xtea_dec_tv_template) } } }, { @@ -3317,10 +3238,7 @@ static const struct alg_test_desc alg_te .test = alg_test_kpp, .fips_allowed = 1, .suite = { - .kpp = { - .vecs = ecdh_tv_template, - .count = ECDH_TEST_VECTORS - } + .kpp = __VECS(ecdh_tv_template) } }, { .alg = "gcm(aes)", @@ -3328,14 +3246,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = aes_gcm_enc_tv_template, - .count = AES_GCM_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_gcm_dec_tv_template, - .count = AES_GCM_DEC_TEST_VECTORS - } + .enc = __VECS(aes_gcm_enc_tv_template), + .dec = __VECS(aes_gcm_dec_tv_template) } } }, { @@ -3343,136 +3255,94 @@ static const struct alg_test_desc alg_te .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = ghash_tv_template, - .count = GHASH_TEST_VECTORS - } + .hash = __VECS(ghash_tv_template) } }, { .alg = "hmac(crc32)", .test = alg_test_hash, .suite = { - .hash = { - .vecs = bfin_crc_tv_template, - .count = BFIN_CRC_TEST_VECTORS - } + .hash = __VECS(bfin_crc_tv_template) } }, { .alg = "hmac(md5)", .test = alg_test_hash, .suite = { - .hash = { - .vecs = hmac_md5_tv_template, - .count = HMAC_MD5_TEST_VECTORS - } + .hash = __VECS(hmac_md5_tv_template) } }, { .alg = "hmac(rmd128)", .test = alg_test_hash, .suite = { - .hash = { - .vecs = hmac_rmd128_tv_template, - .count = HMAC_RMD128_TEST_VECTORS - } + .hash = __VECS(hmac_rmd128_tv_template) } }, { .alg = "hmac(rmd160)", .test = alg_test_hash, .suite = { - .hash = { - .vecs = hmac_rmd160_tv_template, - .count = HMAC_RMD160_TEST_VECTORS - } + .hash = __VECS(hmac_rmd160_tv_template) } }, { .alg = "hmac(sha1)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha1_tv_template, - .count = HMAC_SHA1_TEST_VECTORS - } + .hash = __VECS(hmac_sha1_tv_template) } }, { .alg = "hmac(sha224)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha224_tv_template, - .count = HMAC_SHA224_TEST_VECTORS - } + .hash = __VECS(hmac_sha224_tv_template) } }, { .alg = "hmac(sha256)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha256_tv_template, - .count = HMAC_SHA256_TEST_VECTORS - } + .hash = __VECS(hmac_sha256_tv_template) } }, { .alg = "hmac(sha3-224)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha3_224_tv_template, - .count = HMAC_SHA3_224_TEST_VECTORS - } + .hash = __VECS(hmac_sha3_224_tv_template) } }, { .alg = "hmac(sha3-256)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha3_256_tv_template, - .count = HMAC_SHA3_256_TEST_VECTORS - } + .hash = __VECS(hmac_sha3_256_tv_template) } }, { .alg = "hmac(sha3-384)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha3_384_tv_template, - .count = HMAC_SHA3_384_TEST_VECTORS - } + .hash = __VECS(hmac_sha3_384_tv_template) } }, { .alg = "hmac(sha3-512)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha3_512_tv_template, - .count = HMAC_SHA3_512_TEST_VECTORS - } + .hash = __VECS(hmac_sha3_512_tv_template) } }, { .alg = "hmac(sha384)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha384_tv_template, - .count = HMAC_SHA384_TEST_VECTORS - } + .hash = __VECS(hmac_sha384_tv_template) } }, { .alg = "hmac(sha512)", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = hmac_sha512_tv_template, - .count = HMAC_SHA512_TEST_VECTORS - } + .hash = __VECS(hmac_sha512_tv_template) } }, { .alg = "jitterentropy_rng", @@ -3484,14 +3354,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = aes_kw_enc_tv_template, - .count = ARRAY_SIZE(aes_kw_enc_tv_template) - }, - .dec = { - .vecs = aes_kw_dec_tv_template, - .count = ARRAY_SIZE(aes_kw_dec_tv_template) - } + .enc = __VECS(aes_kw_enc_tv_template), + .dec = __VECS(aes_kw_dec_tv_template) } } }, { @@ -3499,14 +3363,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = aes_lrw_enc_tv_template, - .count = AES_LRW_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_lrw_dec_tv_template, - .count = AES_LRW_DEC_TEST_VECTORS - } + .enc = __VECS(aes_lrw_enc_tv_template), + .dec = __VECS(aes_lrw_dec_tv_template) } } }, { @@ -3514,14 +3372,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = camellia_lrw_enc_tv_template, - .count = CAMELLIA_LRW_ENC_TEST_VECTORS - }, - .dec = { - .vecs = camellia_lrw_dec_tv_template, - .count = CAMELLIA_LRW_DEC_TEST_VECTORS - } + .enc = __VECS(camellia_lrw_enc_tv_template), + .dec = __VECS(camellia_lrw_dec_tv_template) } } }, { @@ -3529,14 +3381,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast6_lrw_enc_tv_template, - .count = CAST6_LRW_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast6_lrw_dec_tv_template, - .count = CAST6_LRW_DEC_TEST_VECTORS - } + .enc = __VECS(cast6_lrw_enc_tv_template), + .dec = __VECS(cast6_lrw_dec_tv_template) } } }, { @@ -3544,14 +3390,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = serpent_lrw_enc_tv_template, - .count = SERPENT_LRW_ENC_TEST_VECTORS - }, - .dec = { - .vecs = serpent_lrw_dec_tv_template, - .count = SERPENT_LRW_DEC_TEST_VECTORS - } + .enc = __VECS(serpent_lrw_enc_tv_template), + .dec = __VECS(serpent_lrw_dec_tv_template) } } }, { @@ -3559,14 +3399,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = tf_lrw_enc_tv_template, - .count = TF_LRW_ENC_TEST_VECTORS - }, - .dec = { - .vecs = tf_lrw_dec_tv_template, - .count = TF_LRW_DEC_TEST_VECTORS - } + .enc = __VECS(tf_lrw_enc_tv_template), + .dec = __VECS(tf_lrw_dec_tv_template) } } }, { @@ -3575,14 +3409,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .comp = { - .comp = { - .vecs = lz4_comp_tv_template, - .count = LZ4_COMP_TEST_VECTORS - }, - .decomp = { - .vecs = lz4_decomp_tv_template, - .count = LZ4_DECOMP_TEST_VECTORS - } + .comp = __VECS(lz4_comp_tv_template), + .decomp = __VECS(lz4_decomp_tv_template) } } }, { @@ -3591,14 +3419,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .comp = { - .comp = { - .vecs = lz4hc_comp_tv_template, - .count = LZ4HC_COMP_TEST_VECTORS - }, - .decomp = { - .vecs = lz4hc_decomp_tv_template, - .count = LZ4HC_DECOMP_TEST_VECTORS - } + .comp = __VECS(lz4hc_comp_tv_template), + .decomp = __VECS(lz4hc_decomp_tv_template) } } }, { @@ -3607,42 +3429,27 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .comp = { - .comp = { - .vecs = lzo_comp_tv_template, - .count = LZO_COMP_TEST_VECTORS - }, - .decomp = { - .vecs = lzo_decomp_tv_template, - .count = LZO_DECOMP_TEST_VECTORS - } + .comp = __VECS(lzo_comp_tv_template), + .decomp = __VECS(lzo_decomp_tv_template) } } }, { .alg = "md4", .test = alg_test_hash, .suite = { - .hash = { - .vecs = md4_tv_template, - .count = MD4_TEST_VECTORS - } + .hash = __VECS(md4_tv_template) } }, { .alg = "md5", .test = alg_test_hash, .suite = { - .hash = { - .vecs = md5_tv_template, - .count = MD5_TEST_VECTORS - } + .hash = __VECS(md5_tv_template) } }, { .alg = "michael_mic", .test = alg_test_hash, .suite = { - .hash = { - .vecs = michael_mic_tv_template, - .count = MICHAEL_MIC_TEST_VECTORS - } + .hash = __VECS(michael_mic_tv_template) } }, { .alg = "ofb(aes)", @@ -3650,14 +3457,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = aes_ofb_enc_tv_template, - .count = AES_OFB_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_ofb_dec_tv_template, - .count = AES_OFB_DEC_TEST_VECTORS - } + .enc = __VECS(aes_ofb_enc_tv_template), + .dec = __VECS(aes_ofb_dec_tv_template) } } }, { @@ -3665,24 +3466,15 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = fcrypt_pcbc_enc_tv_template, - .count = FCRYPT_ENC_TEST_VECTORS - }, - .dec = { - .vecs = fcrypt_pcbc_dec_tv_template, - .count = FCRYPT_DEC_TEST_VECTORS - } + .enc = __VECS(fcrypt_pcbc_enc_tv_template), + .dec = __VECS(fcrypt_pcbc_dec_tv_template) } } }, { .alg = "poly1305", .test = alg_test_hash, .suite = { - .hash = { - .vecs = poly1305_tv_template, - .count = POLY1305_TEST_VECTORS - } + .hash = __VECS(poly1305_tv_template) } }, { .alg = "rfc3686(ctr(aes))", @@ -3690,14 +3482,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = aes_ctr_rfc3686_enc_tv_template, - .count = AES_CTR_3686_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_ctr_rfc3686_dec_tv_template, - .count = AES_CTR_3686_DEC_TEST_VECTORS - } + .enc = __VECS(aes_ctr_rfc3686_enc_tv_template), + .dec = __VECS(aes_ctr_rfc3686_dec_tv_template) } } }, { @@ -3706,14 +3492,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = aes_gcm_rfc4106_enc_tv_template, - .count = AES_GCM_4106_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_gcm_rfc4106_dec_tv_template, - .count = AES_GCM_4106_DEC_TEST_VECTORS - } + .enc = __VECS(aes_gcm_rfc4106_enc_tv_template), + .dec = __VECS(aes_gcm_rfc4106_dec_tv_template) } } }, { @@ -3722,14 +3502,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .aead = { - .enc = { - .vecs = aes_ccm_rfc4309_enc_tv_template, - .count = AES_CCM_4309_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_ccm_rfc4309_dec_tv_template, - .count = AES_CCM_4309_DEC_TEST_VECTORS - } + .enc = __VECS(aes_ccm_rfc4309_enc_tv_template), + .dec = __VECS(aes_ccm_rfc4309_dec_tv_template) } } }, { @@ -3737,14 +3511,8 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = aes_gcm_rfc4543_enc_tv_template, - .count = AES_GCM_4543_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_gcm_rfc4543_dec_tv_template, - .count = AES_GCM_4543_DEC_TEST_VECTORS - }, + .enc = __VECS(aes_gcm_rfc4543_enc_tv_template), + .dec = __VECS(aes_gcm_rfc4543_dec_tv_template), } } }, { @@ -3752,14 +3520,8 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = rfc7539_enc_tv_template, - .count = RFC7539_ENC_TEST_VECTORS - }, - .dec = { - .vecs = rfc7539_dec_tv_template, - .count = RFC7539_DEC_TEST_VECTORS - }, + .enc = __VECS(rfc7539_enc_tv_template), + .dec = __VECS(rfc7539_dec_tv_template), } } }, { @@ -3767,71 +3529,47 @@ static const struct alg_test_desc alg_te .test = alg_test_aead, .suite = { .aead = { - .enc = { - .vecs = rfc7539esp_enc_tv_template, - .count = RFC7539ESP_ENC_TEST_VECTORS - }, - .dec = { - .vecs = rfc7539esp_dec_tv_template, - .count = RFC7539ESP_DEC_TEST_VECTORS - }, + .enc = __VECS(rfc7539esp_enc_tv_template), + .dec = __VECS(rfc7539esp_dec_tv_template), } } }, { .alg = "rmd128", .test = alg_test_hash, .suite = { - .hash = { - .vecs = rmd128_tv_template, - .count = RMD128_TEST_VECTORS - } + .hash = __VECS(rmd128_tv_template) } }, { .alg = "rmd160", .test = alg_test_hash, .suite = { - .hash = { - .vecs = rmd160_tv_template, - .count = RMD160_TEST_VECTORS - } + .hash = __VECS(rmd160_tv_template) } }, { .alg = "rmd256", .test = alg_test_hash, .suite = { - .hash = { - .vecs = rmd256_tv_template, - .count = RMD256_TEST_VECTORS - } + .hash = __VECS(rmd256_tv_template) } }, { .alg = "rmd320", .test = alg_test_hash, .suite = { - .hash = { - .vecs = rmd320_tv_template, - .count = RMD320_TEST_VECTORS - } + .hash = __VECS(rmd320_tv_template) } }, { .alg = "rsa", .test = alg_test_akcipher, .fips_allowed = 1, .suite = { - .akcipher = { - .vecs = rsa_tv_template, - .count = RSA_TEST_VECTORS - } + .akcipher = __VECS(rsa_tv_template) } }, { .alg = "salsa20", .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = salsa20_stream_enc_tv_template, - .count = SALSA20_STREAM_ENC_TEST_VECTORS - } + .enc = __VECS(salsa20_stream_enc_tv_template) } } }, { @@ -3839,162 +3577,120 @@ static const struct alg_test_desc alg_te .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha1_tv_template, - .count = SHA1_TEST_VECTORS - } + .hash = __VECS(sha1_tv_template) } }, { .alg = "sha224", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha224_tv_template, - .count = SHA224_TEST_VECTORS - } + .hash = __VECS(sha224_tv_template) } }, { .alg = "sha256", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha256_tv_template, - .count = SHA256_TEST_VECTORS - } + .hash = __VECS(sha256_tv_template) } }, { .alg = "sha3-224", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha3_224_tv_template, - .count = SHA3_224_TEST_VECTORS - } + .hash = __VECS(sha3_224_tv_template) } }, { .alg = "sha3-256", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha3_256_tv_template, - .count = SHA3_256_TEST_VECTORS - } + .hash = __VECS(sha3_256_tv_template) } }, { .alg = "sha3-384", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha3_384_tv_template, - .count = SHA3_384_TEST_VECTORS - } + .hash = __VECS(sha3_384_tv_template) } }, { .alg = "sha3-512", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha3_512_tv_template, - .count = SHA3_512_TEST_VECTORS - } + .hash = __VECS(sha3_512_tv_template) } }, { .alg = "sha384", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha384_tv_template, - .count = SHA384_TEST_VECTORS - } + .hash = __VECS(sha384_tv_template) } }, { .alg = "sha512", .test = alg_test_hash, .fips_allowed = 1, .suite = { - .hash = { - .vecs = sha512_tv_template, - .count = SHA512_TEST_VECTORS - } + .hash = __VECS(sha512_tv_template) } }, { .alg = "tgr128", .test = alg_test_hash, .suite = { - .hash = { - .vecs = tgr128_tv_template, - .count = TGR128_TEST_VECTORS - } + .hash = __VECS(tgr128_tv_template) } }, { .alg = "tgr160", .test = alg_test_hash, .suite = { - .hash = { - .vecs = tgr160_tv_template, - .count = TGR160_TEST_VECTORS - } + .hash = __VECS(tgr160_tv_template) } }, { .alg = "tgr192", .test = alg_test_hash, .suite = { - .hash = { - .vecs = tgr192_tv_template, - .count = TGR192_TEST_VECTORS + .hash = __VECS(tgr192_tv_template) + } + }, { + .alg = "tls10(hmac(sha1),cbc(aes))", + .test = alg_test_tls, + .suite = { + .tls = { + .enc = __VECS(tls_enc_tv_template), + .dec = __VECS(tls_dec_tv_template) } } }, { .alg = "vmac(aes)", .test = alg_test_hash, .suite = { - .hash = { - .vecs = aes_vmac128_tv_template, - .count = VMAC_AES_TEST_VECTORS - } + .hash = __VECS(aes_vmac128_tv_template) } }, { .alg = "wp256", .test = alg_test_hash, .suite = { - .hash = { - .vecs = wp256_tv_template, - .count = WP256_TEST_VECTORS - } + .hash = __VECS(wp256_tv_template) } }, { .alg = "wp384", .test = alg_test_hash, .suite = { - .hash = { - .vecs = wp384_tv_template, - .count = WP384_TEST_VECTORS - } + .hash = __VECS(wp384_tv_template) } }, { .alg = "wp512", .test = alg_test_hash, .suite = { - .hash = { - .vecs = wp512_tv_template, - .count = WP512_TEST_VECTORS - } + .hash = __VECS(wp512_tv_template) } }, { .alg = "xcbc(aes)", .test = alg_test_hash, .suite = { - .hash = { - .vecs = aes_xcbc128_tv_template, - .count = XCBC_AES_TEST_VECTORS - } + .hash = __VECS(aes_xcbc128_tv_template) } }, { .alg = "xts(aes)", @@ -4002,14 +3698,8 @@ static const struct alg_test_desc alg_te .fips_allowed = 1, .suite = { .cipher = { - .enc = { - .vecs = aes_xts_enc_tv_template, - .count = AES_XTS_ENC_TEST_VECTORS - }, - .dec = { - .vecs = aes_xts_dec_tv_template, - .count = AES_XTS_DEC_TEST_VECTORS - } + .enc = __VECS(aes_xts_enc_tv_template), + .dec = __VECS(aes_xts_dec_tv_template) } } }, { @@ -4017,14 +3707,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = camellia_xts_enc_tv_template, - .count = CAMELLIA_XTS_ENC_TEST_VECTORS - }, - .dec = { - .vecs = camellia_xts_dec_tv_template, - .count = CAMELLIA_XTS_DEC_TEST_VECTORS - } + .enc = __VECS(camellia_xts_enc_tv_template), + .dec = __VECS(camellia_xts_dec_tv_template) } } }, { @@ -4032,14 +3716,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = cast6_xts_enc_tv_template, - .count = CAST6_XTS_ENC_TEST_VECTORS - }, - .dec = { - .vecs = cast6_xts_dec_tv_template, - .count = CAST6_XTS_DEC_TEST_VECTORS - } + .enc = __VECS(cast6_xts_enc_tv_template), + .dec = __VECS(cast6_xts_dec_tv_template) } } }, { @@ -4047,14 +3725,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = serpent_xts_enc_tv_template, - .count = SERPENT_XTS_ENC_TEST_VECTORS - }, - .dec = { - .vecs = serpent_xts_dec_tv_template, - .count = SERPENT_XTS_DEC_TEST_VECTORS - } + .enc = __VECS(serpent_xts_enc_tv_template), + .dec = __VECS(serpent_xts_dec_tv_template) } } }, { @@ -4062,14 +3734,8 @@ static const struct alg_test_desc alg_te .test = alg_test_skcipher, .suite = { .cipher = { - .enc = { - .vecs = tf_xts_enc_tv_template, - .count = TF_XTS_ENC_TEST_VECTORS - }, - .dec = { - .vecs = tf_xts_dec_tv_template, - .count = TF_XTS_DEC_TEST_VECTORS - } + .enc = __VECS(tf_xts_enc_tv_template), + .dec = __VECS(tf_xts_dec_tv_template) } } } --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -34,9 +34,9 @@ struct hash_testvec { /* only used with keyed hash algorithms */ - char *key; - char *plaintext; - char *digest; + const char *key; + const char *plaintext; + const char *digest; unsigned char tap[MAX_TAP]; unsigned short psize; unsigned char np; @@ -63,11 +63,11 @@ struct hash_testvec { */ struct cipher_testvec { - char *key; - char *iv; - char *iv_out; - char *input; - char *result; + const char *key; + const char *iv; + const char *iv_out; + const char *input; + const char *result; unsigned short tap[MAX_TAP]; int np; unsigned char also_non_np; @@ -80,11 +80,11 @@ struct cipher_testvec { }; struct aead_testvec { - char *key; - char *iv; - char *input; - char *assoc; - char *result; + const char *key; + const char *iv; + const char *input; + const char *assoc; + const char *result; unsigned char tap[MAX_TAP]; unsigned char atap[MAX_TAP]; int np; @@ -99,10 +99,10 @@ struct aead_testvec { }; struct cprng_testvec { - char *key; - char *dt; - char *v; - char *result; + const char *key; + const char *dt; + const char *v; + const char *result; unsigned char klen; unsigned short dtlen; unsigned short vlen; @@ -111,24 +111,38 @@ struct cprng_testvec { }; struct drbg_testvec { - unsigned char *entropy; + const unsigned char *entropy; size_t entropylen; - unsigned char *entpra; - unsigned char *entprb; + const unsigned char *entpra; + const unsigned char *entprb; size_t entprlen; - unsigned char *addtla; - unsigned char *addtlb; + const unsigned char *addtla; + const unsigned char *addtlb; size_t addtllen; - unsigned char *pers; + const unsigned char *pers; size_t perslen; - unsigned char *expected; + const unsigned char *expected; size_t expectedlen; }; +struct tls_testvec { + char *key; /* wrapped keys for encryption and authentication */ + char *iv; /* initialization vector */ + char *input; /* input data */ + char *assoc; /* associated data: seq num, type, version, input len */ + char *result; /* result data */ + unsigned char fail; /* the test failure is expected */ + unsigned char novrfy; /* dec verification failure expected */ + unsigned char klen; /* key length */ + unsigned short ilen; /* input data length */ + unsigned short alen; /* associated data length */ + unsigned short rlen; /* result length */ +}; + struct akcipher_testvec { - unsigned char *key; - unsigned char *m; - unsigned char *c; + const unsigned char *key; + const unsigned char *m; + const unsigned char *c; unsigned int key_len; unsigned int m_size; unsigned int c_size; @@ -136,27 +150,227 @@ struct akcipher_testvec { }; struct kpp_testvec { - unsigned char *secret; - unsigned char *b_public; - unsigned char *expected_a_public; - unsigned char *expected_ss; + const unsigned char *secret; + const unsigned char *b_public; + const unsigned char *expected_a_public; + const unsigned char *expected_ss; unsigned short secret_size; unsigned short b_public_size; unsigned short expected_a_public_size; unsigned short expected_ss_size; }; -static char zeroed_string[48]; +static const char zeroed_string[48]; /* - * RSA test vectors. Borrowed from openSSL. + * TLS1.0 synthetic test vectors */ -#ifdef CONFIG_CRYPTO_FIPS -#define RSA_TEST_VECTORS 2 +static struct tls_testvec tls_enc_tv_template[] = { + { +#ifdef __LITTLE_ENDIAN + .key = "\x08\x00" /* rta length */ + "\x01\x00" /* rta type */ +#else + .key = "\x00\x08" /* rta length */ + "\x00\x01" /* rta type */ +#endif + "\x00\x00\x00\x10" /* enc key length */ + "authenticationkey20benckeyis16_bytes", + .klen = 8 + 20 + 16, + .iv = "iv0123456789abcd", + .input = "Single block msg", + .ilen = 16, + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x00\x03\x01\x00\x10", + .alen = 13, + .result = "\xd5\xac\xb\xd2\xac\xad\x3f\xb1" + "\x59\x79\x1e\x91\x5f\x52\x14\x9c" + "\xc0\x75\xd8\x4c\x97\x0f\x07\x73" + "\xdc\x89\x47\x49\x49\xcb\x30\x6b" + "\x1b\x45\x23\xa1\xd0\x51\xcf\x02" + "\x2e\xa8\x5d\xa0\xfe\xca\x82\x61", + .rlen = 16 + 20 + 12, + }, { +#ifdef __LITTLE_ENDIAN + .key = "\x08\x00" /* rta length */ + "\x01\x00" /* rta type */ +#else + .key = "\x00\x08" /* rta length */ + "\x00\x01" /* rta type */ +#endif + "\x00\x00\x00\x10" /* enc key length */ + "authenticationkey20benckeyis16_bytes", + .klen = 8 + 20 + 16, + .iv = "iv0123456789abcd", + .input = "", + .ilen = 0, + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x00\x03\x01\x00\x00", + .alen = 13, + .result = "\x58\x2a\x11\xc\x86\x8e\x4b\x67" + "\x2d\x16\x26\x1a\xac\x4b\xe2\x1a" + "\xe9\x6a\xcc\x4d\x6f\x79\x8a\x45" + "\x1f\x4e\x27\xf2\xa7\x59\xb4\x5a", + .rlen = 20 + 12, + }, { +#ifdef __LITTLE_ENDIAN + .key = "\x08\x00" /* rta length */ + "\x01\x00" /* rta type */ +#else + .key = "\x00\x08" /* rta length */ + "\x00\x01" /* rta type */ +#endif + "\x00\x00\x00\x10" /* enc key length */ + "authenticationkey20benckeyis16_bytes", + .klen = 8 + 20 + 16, + .iv = "iv0123456789abcd", + .input = "285 bytes plaintext285 bytes plaintext285 bytes" + " plaintext285 bytes plaintext285 bytes plaintext285" + " bytes plaintext285 bytes plaintext285 bytes" + " plaintext285 bytes plaintext285 bytes plaintext285" + " bytes plaintext285 bytes plaintext285 bytes" + " plaintext285 bytes plaintext285 bytes plaintext285" + " bytes plaintext285 bytes plaintext", + .ilen = 285, + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x00\x03\x01\x01\x1d", + .alen = 13, + .result = "\x80\x23\x82\x44\x14\x2a\x1d\x94\xc\xc2\x1d\xd" + "\x3a\x32\x89\x4c\x57\x30\xa8\x89\x76\x46\xcc\x90" + "\x1d\x88\xb8\xa6\x1a\x58\xe\x2d\xeb\x2c\xc7\x3a" + "\x52\x4e\xdb\xb3\x1e\x83\x11\xf5\x3c\xce\x6e\x94" + "\xd3\x26\x6a\x9a\xd\xbd\xc7\x98\xb9\xb3\x3a\x51" + "\x1e\x4\x84\x8a\x8f\x54\x9a\x51\x69\x9c\xce\x31" + "\x8d\x5d\x8b\xee\x5f\x70\xc\xc9\xb8\x50\x54\xf8" + "\xb2\x4a\x7a\xcd\xeb\x7a\x82\x81\xc6\x41\xc8\x50" + "\x91\x8d\xc8\xed\xcd\x40\x8f\x55\xd1\xec\xc9\xac" + "\x15\x18\xf9\x20\xa0\xed\x18\xa1\xe3\x56\xe3\x14" + "\xe5\xe8\x66\x63\x20\xed\xe4\x62\x9d\xa3\xa4\x1d" + "\x81\x89\x18\xf2\x36\xae\xc8\x8a\x2b\xbc\xc3\xb8" + "\x80\xf\x97\x21\x36\x39\x8\x84\x23\x18\x9e\x9c" + "\x72\x32\x75\x2d\x2e\xf9\x60\xb\xe8\xcc\xd9\x74" + "\x4\x1b\x8e\x99\xc1\x94\xee\xd0\xac\x4e\xfc\x7e" + "\xf1\x96\xb3\xe7\x14\xb8\xf2\xc\x25\x97\x82\x6b" + "\xbd\x0\x65\xab\x5c\xe3\x16\xfb\x68\xef\xea\x9d" + "\xff\x44\x1d\x2a\x44\xf5\xc8\x56\x77\xb7\xbf\x13" + "\xc8\x54\xdb\x92\xfe\x16\x4c\xbe\x18\xe9\xb\x8d" + "\xb\xd4\x43\x58\x43\xaa\xf4\x3\x80\x97\x62\xd5" + "\xdf\x3c\x28\xaa\xee\x48\x4b\x55\x41\x1b\x31\x2" + "\xbe\xa0\x1c\xbd\xb7\x22\x2a\xe5\x53\x72\x73\x20" + "\x44\x4f\xe6\x1\x2b\x34\x33\x11\x7d\xfb\x10\xc1" + "\x66\x7c\xa6\xf4\x48\x36\x5e\x2\xda\x41\x4b\x3e" + "\xe7\x80\x17\x17\xce\xf1\x3e\x6a\x8e\x26\xf3\xb7" + "\x2b\x85\xd\x31\x8d\xba\x6c\x22\xb4\x28\x55\x7e" + "\x2a\x9e\x26\xf1\x3d\x21\xac\x65", + .rlen = 285 + 20 + 15, + } +}; + +static struct tls_testvec tls_dec_tv_template[] = { + { +#ifdef __LITTLE_ENDIAN + .key = "\x08\x00" /* rta length */ + "\x01\x00" /* rta type */ +#else + .key = "\x00\x08" /* rta length */ + "\x00\x01" /* rta type */ +#endif + "\x00\x00\x00\x10" /* enc key length */ + "authenticationkey20benckeyis16_bytes", + .klen = 8 + 20 + 16, + .iv = "iv0123456789abcd", + .input = "\xd5\xac\xb\xd2\xac\xad\x3f\xb1" + "\x59\x79\x1e\x91\x5f\x52\x14\x9c" + "\xc0\x75\xd8\x4c\x97\x0f\x07\x73" + "\xdc\x89\x47\x49\x49\xcb\x30\x6b" + "\x1b\x45\x23\xa1\xd0\x51\xcf\x02" + "\x2e\xa8\x5d\xa0\xfe\xca\x82\x61", + .ilen = 16 + 20 + 12, + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x00\x03\x01\x00\x30", + .alen = 13, + .result = "Single block msg", + .rlen = 16, + }, { +#ifdef __LITTLE_ENDIAN + .key = "\x08\x00" /* rta length */ + "\x01\x00" /* rta type */ #else -#define RSA_TEST_VECTORS 5 + .key = "\x00\x08" /* rta length */ + "\x00\x01" /* rta type */ #endif -static struct akcipher_testvec rsa_tv_template[] = { + "\x00\x00\x00\x10" /* enc key length */ + "authenticationkey20benckeyis16_bytes", + .klen = 8 + 20 + 16, + .iv = "iv0123456789abcd", + .input = "\x58\x2a\x11\xc\x86\x8e\x4b\x67" + "\x2d\x16\x26\x1a\xac\x4b\xe2\x1a" + "\xe9\x6a\xcc\x4d\x6f\x79\x8a\x45" + "\x1f\x4e\x27\xf2\xa7\x59\xb4\x5a", + .ilen = 20 + 12, + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x00\x03\x01\x00\x20", + .alen = 13, + .result = "", + .rlen = 0, + }, { +#ifdef __LITTLE_ENDIAN + .key = "\x08\x00" /* rta length */ + "\x01\x00" /* rta type */ +#else + .key = "\x00\x08" /* rta length */ + "\x00\x01" /* rta type */ +#endif + "\x00\x00\x00\x10" /* enc key length */ + "authenticationkey20benckeyis16_bytes", + .klen = 8 + 20 + 16, + .iv = "iv0123456789abcd", + .input = "\x80\x23\x82\x44\x14\x2a\x1d\x94\xc\xc2\x1d\xd" + "\x3a\x32\x89\x4c\x57\x30\xa8\x89\x76\x46\xcc\x90" + "\x1d\x88\xb8\xa6\x1a\x58\xe\x2d\xeb\x2c\xc7\x3a" + "\x52\x4e\xdb\xb3\x1e\x83\x11\xf5\x3c\xce\x6e\x94" + "\xd3\x26\x6a\x9a\xd\xbd\xc7\x98\xb9\xb3\x3a\x51" + "\x1e\x4\x84\x8a\x8f\x54\x9a\x51\x69\x9c\xce\x31" + "\x8d\x5d\x8b\xee\x5f\x70\xc\xc9\xb8\x50\x54\xf8" + "\xb2\x4a\x7a\xcd\xeb\x7a\x82\x81\xc6\x41\xc8\x50" + "\x91\x8d\xc8\xed\xcd\x40\x8f\x55\xd1\xec\xc9\xac" + "\x15\x18\xf9\x20\xa0\xed\x18\xa1\xe3\x56\xe3\x14" + "\xe5\xe8\x66\x63\x20\xed\xe4\x62\x9d\xa3\xa4\x1d" + "\x81\x89\x18\xf2\x36\xae\xc8\x8a\x2b\xbc\xc3\xb8" + "\x80\xf\x97\x21\x36\x39\x8\x84\x23\x18\x9e\x9c" + "\x72\x32\x75\x2d\x2e\xf9\x60\xb\xe8\xcc\xd9\x74" + "\x4\x1b\x8e\x99\xc1\x94\xee\xd0\xac\x4e\xfc\x7e" + "\xf1\x96\xb3\xe7\x14\xb8\xf2\xc\x25\x97\x82\x6b" + "\xbd\x0\x65\xab\x5c\xe3\x16\xfb\x68\xef\xea\x9d" + "\xff\x44\x1d\x2a\x44\xf5\xc8\x56\x77\xb7\xbf\x13" + "\xc8\x54\xdb\x92\xfe\x16\x4c\xbe\x18\xe9\xb\x8d" + "\xb\xd4\x43\x58\x43\xaa\xf4\x3\x80\x97\x62\xd5" + "\xdf\x3c\x28\xaa\xee\x48\x4b\x55\x41\x1b\x31\x2" + "\xbe\xa0\x1c\xbd\xb7\x22\x2a\xe5\x53\x72\x73\x20" + "\x44\x4f\xe6\x1\x2b\x34\x33\x11\x7d\xfb\x10\xc1" + "\x66\x7c\xa6\xf4\x48\x36\x5e\x2\xda\x41\x4b\x3e" + "\xe7\x80\x17\x17\xce\xf1\x3e\x6a\x8e\x26\xf3\xb7" + "\x2b\x85\xd\x31\x8d\xba\x6c\x22\xb4\x28\x55\x7e" + "\x2a\x9e\x26\xf1\x3d\x21\xac\x65", + + .ilen = 285 + 20 + 15, + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x00\x03\x01\x01\x40", + .alen = 13, + .result = "285 bytes plaintext285 bytes plaintext285 bytes" + " plaintext285 bytes plaintext285 bytes plaintext285" + " bytes plaintext285 bytes plaintext285 bytes" + " plaintext285 bytes plaintext285 bytes plaintext285" + " bytes plaintext285 bytes plaintext285 bytes" + " plaintext285 bytes plaintext285 bytes plaintext", + .rlen = 285, + } +}; + +/* + * RSA test vectors. Borrowed from openSSL. + */ +static const struct akcipher_testvec rsa_tv_template[] = { { #ifndef CONFIG_CRYPTO_FIPS .key = @@ -340,6 +554,7 @@ static struct akcipher_testvec rsa_tv_te .m_size = 8, .c_size = 256, .public_key_vec = true, +#ifndef CONFIG_CRYPTO_FIPS }, { .key = "\x30\x82\x09\x29" /* sequence of 2345 bytes */ @@ -538,12 +753,11 @@ static struct akcipher_testvec rsa_tv_te .key_len = 2349, .m_size = 8, .c_size = 512, +#endif } }; -#define DH_TEST_VECTORS 2 - -struct kpp_testvec dh_tv_template[] = { +static const struct kpp_testvec dh_tv_template[] = { { .secret = #ifdef __LITTLE_ENDIAN @@ -760,12 +974,7 @@ struct kpp_testvec dh_tv_template[] = { } }; -#ifdef CONFIG_CRYPTO_FIPS -#define ECDH_TEST_VECTORS 1 -#else -#define ECDH_TEST_VECTORS 2 -#endif -struct kpp_testvec ecdh_tv_template[] = { +static const struct kpp_testvec ecdh_tv_template[] = { { #ifndef CONFIG_CRYPTO_FIPS .secret = @@ -856,9 +1065,7 @@ struct kpp_testvec ecdh_tv_template[] = /* * MD4 test vectors from RFC1320 */ -#define MD4_TEST_VECTORS 7 - -static struct hash_testvec md4_tv_template [] = { +static const struct hash_testvec md4_tv_template[] = { { .plaintext = "", .digest = "\x31\xd6\xcf\xe0\xd1\x6a\xe9\x31" @@ -899,8 +1106,7 @@ static struct hash_testvec md4_tv_templa }, }; -#define SHA3_224_TEST_VECTORS 3 -static struct hash_testvec sha3_224_tv_template[] = { +static const struct hash_testvec sha3_224_tv_template[] = { { .plaintext = "", .digest = "\x6b\x4e\x03\x42\x36\x67\xdb\xb7" @@ -925,8 +1131,7 @@ static struct hash_testvec sha3_224_tv_t }, }; -#define SHA3_256_TEST_VECTORS 3 -static struct hash_testvec sha3_256_tv_template[] = { +static const struct hash_testvec sha3_256_tv_template[] = { { .plaintext = "", .digest = "\xa7\xff\xc6\xf8\xbf\x1e\xd7\x66" @@ -952,8 +1157,7 @@ static struct hash_testvec sha3_256_tv_t }; -#define SHA3_384_TEST_VECTORS 3 -static struct hash_testvec sha3_384_tv_template[] = { +static const struct hash_testvec sha3_384_tv_template[] = { { .plaintext = "", .digest = "\x0c\x63\xa7\x5b\x84\x5e\x4f\x7d" @@ -985,8 +1189,7 @@ static struct hash_testvec sha3_384_tv_t }; -#define SHA3_512_TEST_VECTORS 3 -static struct hash_testvec sha3_512_tv_template[] = { +static const struct hash_testvec sha3_512_tv_template[] = { { .plaintext = "", .digest = "\xa6\x9f\x73\xcc\xa2\x3a\x9a\xc5" @@ -1027,9 +1230,7 @@ static struct hash_testvec sha3_512_tv_t /* * MD5 test vectors from RFC1321 */ -#define MD5_TEST_VECTORS 7 - -static struct hash_testvec md5_tv_template[] = { +static const struct hash_testvec md5_tv_template[] = { { .digest = "\xd4\x1d\x8c\xd9\x8f\x00\xb2\x04" "\xe9\x80\x09\x98\xec\xf8\x42\x7e", @@ -1073,9 +1274,7 @@ static struct hash_testvec md5_tv_templa /* * RIPEMD-128 test vectors from ISO/IEC 10118-3:2004(E) */ -#define RMD128_TEST_VECTORS 10 - -static struct hash_testvec rmd128_tv_template[] = { +static const struct hash_testvec rmd128_tv_template[] = { { .digest = "\xcd\xf2\x62\x13\xa1\x50\xdc\x3e" "\xcb\x61\x0f\x18\xf6\xb3\x8b\x46", @@ -1137,9 +1336,7 @@ static struct hash_testvec rmd128_tv_tem /* * RIPEMD-160 test vectors from ISO/IEC 10118-3:2004(E) */ -#define RMD160_TEST_VECTORS 10 - -static struct hash_testvec rmd160_tv_template[] = { +static const struct hash_testvec rmd160_tv_template[] = { { .digest = "\x9c\x11\x85\xa5\xc5\xe9\xfc\x54\x61\x28" "\x08\x97\x7e\xe8\xf5\x48\xb2\x25\x8d\x31", @@ -1201,9 +1398,7 @@ static struct hash_testvec rmd160_tv_tem /* * RIPEMD-256 test vectors */ -#define RMD256_TEST_VECTORS 8 - -static struct hash_testvec rmd256_tv_template[] = { +static const struct hash_testvec rmd256_tv_template[] = { { .digest = "\x02\xba\x4c\x4e\x5f\x8e\xcd\x18" "\x77\xfc\x52\xd6\x4d\x30\xe3\x7a" @@ -1269,9 +1464,7 @@ static struct hash_testvec rmd256_tv_tem /* * RIPEMD-320 test vectors */ -#define RMD320_TEST_VECTORS 8 - -static struct hash_testvec rmd320_tv_template[] = { +static const struct hash_testvec rmd320_tv_template[] = { { .digest = "\x22\xd6\x5d\x56\x61\x53\x6c\xdc\x75\xc1" "\xfd\xf5\xc6\xde\x7b\x41\xb9\xf2\x73\x25" @@ -1334,36 +1527,49 @@ static struct hash_testvec rmd320_tv_tem } }; -#define CRCT10DIF_TEST_VECTORS 3 -static struct hash_testvec crct10dif_tv_template[] = { +static const struct hash_testvec crct10dif_tv_template[] = { { - .plaintext = "abc", - .psize = 3, -#ifdef __LITTLE_ENDIAN - .digest = "\x3b\x44", -#else - .digest = "\x44\x3b", -#endif - }, { - .plaintext = "1234567890123456789012345678901234567890" - "123456789012345678901234567890123456789", - .psize = 79, -#ifdef __LITTLE_ENDIAN - .digest = "\x70\x4b", -#else - .digest = "\x4b\x70", -#endif - }, { - .plaintext = - "abcddddddddddddddddddddddddddddddddddddddddddddddddddddd", - .psize = 56, -#ifdef __LITTLE_ENDIAN - .digest = "\xe3\x9c", -#else - .digest = "\x9c\xe3", -#endif - .np = 2, - .tap = { 28, 28 } + .plaintext = "abc", + .psize = 3, + .digest = (u8 *)(u16 []){ 0x443b }, + }, { + .plaintext = "1234567890123456789012345678901234567890" + "123456789012345678901234567890123456789", + .psize = 79, + .digest = (u8 *)(u16 []){ 0x4b70 }, + .np = 2, + .tap = { 63, 16 }, + }, { + .plaintext = "abcdddddddddddddddddddddddddddddddddddddddd" + "ddddddddddddd", + .psize = 56, + .digest = (u8 *)(u16 []){ 0x9ce3 }, + .np = 8, + .tap = { 1, 2, 28, 7, 6, 5, 4, 3 }, + }, { + .plaintext = "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "123456789012345678901234567890123456789", + .psize = 319, + .digest = (u8 *)(u16 []){ 0x44c6 }, + }, { + .plaintext = "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "1234567890123456789012345678901234567890" + "123456789012345678901234567890123456789", + .psize = 319, + .digest = (u8 *)(u16 []){ 0x44c6 }, + .np = 4, + .tap = { 1, 255, 57, 6 }, } }; @@ -1371,9 +1577,7 @@ static struct hash_testvec crct10dif_tv_ * SHA1 test vectors from from FIPS PUB 180-1 * Long vector from CAVS 5.0 */ -#define SHA1_TEST_VECTORS 6 - -static struct hash_testvec sha1_tv_template[] = { +static const struct hash_testvec sha1_tv_template[] = { { .plaintext = "", .psize = 0, @@ -1563,9 +1767,7 @@ static struct hash_testvec sha1_tv_templ /* * SHA224 test vectors from from FIPS PUB 180-2 */ -#define SHA224_TEST_VECTORS 5 - -static struct hash_testvec sha224_tv_template[] = { +static const struct hash_testvec sha224_tv_template[] = { { .plaintext = "", .psize = 0, @@ -1737,9 +1939,7 @@ static struct hash_testvec sha224_tv_tem /* * SHA256 test vectors from from NIST */ -#define SHA256_TEST_VECTORS 5 - -static struct hash_testvec sha256_tv_template[] = { +static const struct hash_testvec sha256_tv_template[] = { { .plaintext = "", .psize = 0, @@ -1910,9 +2110,7 @@ static struct hash_testvec sha256_tv_tem /* * SHA384 test vectors from from NIST and kerneli */ -#define SHA384_TEST_VECTORS 6 - -static struct hash_testvec sha384_tv_template[] = { +static const struct hash_testvec sha384_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2104,9 +2302,7 @@ static struct hash_testvec sha384_tv_tem /* * SHA512 test vectors from from NIST and kerneli */ -#define SHA512_TEST_VECTORS 6 - -static struct hash_testvec sha512_tv_template[] = { +static const struct hash_testvec sha512_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2313,9 +2509,7 @@ static struct hash_testvec sha512_tv_tem * by Vincent Rijmen and Paulo S. L. M. Barreto as part of the NESSIE * submission */ -#define WP512_TEST_VECTORS 8 - -static struct hash_testvec wp512_tv_template[] = { +static const struct hash_testvec wp512_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2411,9 +2605,7 @@ static struct hash_testvec wp512_tv_temp }, }; -#define WP384_TEST_VECTORS 8 - -static struct hash_testvec wp384_tv_template[] = { +static const struct hash_testvec wp384_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2493,9 +2685,7 @@ static struct hash_testvec wp384_tv_temp }, }; -#define WP256_TEST_VECTORS 8 - -static struct hash_testvec wp256_tv_template[] = { +static const struct hash_testvec wp256_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2562,9 +2752,7 @@ static struct hash_testvec wp256_tv_temp /* * TIGER test vectors from Tiger website */ -#define TGR192_TEST_VECTORS 6 - -static struct hash_testvec tgr192_tv_template[] = { +static const struct hash_testvec tgr192_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2607,9 +2795,7 @@ static struct hash_testvec tgr192_tv_tem }, }; -#define TGR160_TEST_VECTORS 6 - -static struct hash_testvec tgr160_tv_template[] = { +static const struct hash_testvec tgr160_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2652,9 +2838,7 @@ static struct hash_testvec tgr160_tv_tem }, }; -#define TGR128_TEST_VECTORS 6 - -static struct hash_testvec tgr128_tv_template[] = { +static const struct hash_testvec tgr128_tv_template[] = { { .plaintext = "", .psize = 0, @@ -2691,9 +2875,7 @@ static struct hash_testvec tgr128_tv_tem }, }; -#define GHASH_TEST_VECTORS 6 - -static struct hash_testvec ghash_tv_template[] = +static const struct hash_testvec ghash_tv_template[] = { { .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03" @@ -2808,9 +2990,7 @@ static struct hash_testvec ghash_tv_temp * HMAC-MD5 test vectors from RFC2202 * (These need to be fixed to not use strlen). */ -#define HMAC_MD5_TEST_VECTORS 7 - -static struct hash_testvec hmac_md5_tv_template[] = +static const struct hash_testvec hmac_md5_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", @@ -2890,9 +3070,7 @@ static struct hash_testvec hmac_md5_tv_t /* * HMAC-RIPEMD128 test vectors from RFC2286 */ -#define HMAC_RMD128_TEST_VECTORS 7 - -static struct hash_testvec hmac_rmd128_tv_template[] = { +static const struct hash_testvec hmac_rmd128_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", .ksize = 16, @@ -2971,9 +3149,7 @@ static struct hash_testvec hmac_rmd128_t /* * HMAC-RIPEMD160 test vectors from RFC2286 */ -#define HMAC_RMD160_TEST_VECTORS 7 - -static struct hash_testvec hmac_rmd160_tv_template[] = { +static const struct hash_testvec hmac_rmd160_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", .ksize = 20, @@ -3052,9 +3228,7 @@ static struct hash_testvec hmac_rmd160_t /* * HMAC-SHA1 test vectors from RFC2202 */ -#define HMAC_SHA1_TEST_VECTORS 7 - -static struct hash_testvec hmac_sha1_tv_template[] = { +static const struct hash_testvec hmac_sha1_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", .ksize = 20, @@ -3135,9 +3309,7 @@ static struct hash_testvec hmac_sha1_tv_ /* * SHA224 HMAC test vectors from RFC4231 */ -#define HMAC_SHA224_TEST_VECTORS 4 - -static struct hash_testvec hmac_sha224_tv_template[] = { +static const struct hash_testvec hmac_sha224_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3250,9 +3422,7 @@ static struct hash_testvec hmac_sha224_t * HMAC-SHA256 test vectors from * draft-ietf-ipsec-ciph-sha-256-01.txt */ -#define HMAC_SHA256_TEST_VECTORS 10 - -static struct hash_testvec hmac_sha256_tv_template[] = { +static const struct hash_testvec hmac_sha256_tv_template[] = { { .key = "\x01\x02\x03\x04\x05\x06\x07\x08" "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" @@ -3387,9 +3557,7 @@ static struct hash_testvec hmac_sha256_t }, }; -#define CMAC_AES_TEST_VECTORS 6 - -static struct hash_testvec aes_cmac128_tv_template[] = { +static const struct hash_testvec aes_cmac128_tv_template[] = { { /* From NIST Special Publication 800-38B, AES-128 */ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", @@ -3464,9 +3632,67 @@ static struct hash_testvec aes_cmac128_t } }; -#define CMAC_DES3_EDE_TEST_VECTORS 4 +static const struct hash_testvec aes_cbcmac_tv_template[] = { + { + .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" + "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a", + .digest = "\x3a\xd7\x7b\xb4\x0d\x7a\x36\x60" + "\xa8\x9e\xca\xf3\x24\x66\xef\x97", + .psize = 16, + .ksize = 16, + }, { + .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" + "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" + "\x30", + .digest = "\x9d\x0d\xd0\x63\xfb\xcb\x24\x43" + "\xf8\xf2\x76\x03\xac\x39\xb0\x9d", + .psize = 33, + .ksize = 16, + .np = 2, + .tap = { 7, 26 }, + }, { + .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" + "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" + "\xad\x2b\x41\x7b\xe6\x6c\x37", + .digest = "\xc0\x71\x73\xb8\xa0\x2c\x11\x7c" + "\xaf\xdc\xb2\xf8\x89\x32\xa3\x3a", + .psize = 63, + .ksize = 16, + }, { + .key = "\x60\x3d\xeb\x10\x15\xca\x71\xbe" + "\x2b\x73\xae\xf0\x85\x7d\x77\x81" + "\x1f\x35\x2c\x07\x3b\x61\x08\xd7" + "\x2d\x98\x10\xa3\x09\x14\xdf\xf4", + .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" + "\xad\x2b\x41\x7b\xe6\x6c\x37\x10" + "\x1c", + .digest = "\x6a\x4e\xdb\x21\x47\x51\xdf\x4f" + "\xa8\x4d\x4c\x10\x3b\x72\x7d\xd6", + .psize = 65, + .ksize = 32, + } +}; -static struct hash_testvec des3_ede_cmac64_tv_template[] = { +static const struct hash_testvec des3_ede_cmac64_tv_template[] = { /* * From NIST Special Publication 800-38B, Three Key TDEA * Corrected test vectors from: @@ -3512,9 +3738,7 @@ static struct hash_testvec des3_ede_cmac } }; -#define XCBC_AES_TEST_VECTORS 6 - -static struct hash_testvec aes_xcbc128_tv_template[] = { +static const struct hash_testvec aes_xcbc128_tv_template[] = { { .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -3580,36 +3804,35 @@ static struct hash_testvec aes_xcbc128_t } }; -#define VMAC_AES_TEST_VECTORS 11 -static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', - '\x02', '\x03', '\x02', '\x02', - '\x02', '\x04', '\x01', '\x07', - '\x04', '\x01', '\x04', '\x03',}; -static char vmac_string2[128] = {'a', 'b', 'c',}; -static char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - 'a', 'b', 'c', 'a', 'b', 'c', - }; - -static char vmac_string4[17] = {'b', 'c', 'e', 'f', - 'i', 'j', 'l', 'm', - 'o', 'p', 'r', 's', - 't', 'u', 'w', 'x', 'z'}; - -static char vmac_string5[127] = {'r', 'm', 'b', 't', 'c', - 'o', 'l', 'k', ']', '%', - '9', '2', '7', '!', 'A'}; - -static char vmac_string6[129] = {'p', 't', '*', '7', 'l', - 'i', '!', '#', 'w', '0', - 'z', '/', '4', 'A', 'n'}; +static const char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', + '\x02', '\x03', '\x02', '\x02', + '\x02', '\x04', '\x01', '\x07', + '\x04', '\x01', '\x04', '\x03',}; +static const char vmac_string2[128] = {'a', 'b', 'c',}; +static const char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + }; + +static const char vmac_string4[17] = {'b', 'c', 'e', 'f', + 'i', 'j', 'l', 'm', + 'o', 'p', 'r', 's', + 't', 'u', 'w', 'x', 'z'}; + +static const char vmac_string5[127] = {'r', 'm', 'b', 't', 'c', + 'o', 'l', 'k', ']', '%', + '9', '2', '7', '!', 'A'}; + +static const char vmac_string6[129] = {'p', 't', '*', '7', 'l', + 'i', '!', '#', 'w', '0', + 'z', '/', '4', 'A', 'n'}; -static struct hash_testvec aes_vmac128_tv_template[] = { +static const struct hash_testvec aes_vmac128_tv_template[] = { { .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -3687,9 +3910,7 @@ static struct hash_testvec aes_vmac128_t * SHA384 HMAC test vectors from RFC4231 */ -#define HMAC_SHA384_TEST_VECTORS 4 - -static struct hash_testvec hmac_sha384_tv_template[] = { +static const struct hash_testvec hmac_sha384_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3787,9 +4008,7 @@ static struct hash_testvec hmac_sha384_t * SHA512 HMAC test vectors from RFC4231 */ -#define HMAC_SHA512_TEST_VECTORS 4 - -static struct hash_testvec hmac_sha512_tv_template[] = { +static const struct hash_testvec hmac_sha512_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3894,9 +4113,7 @@ static struct hash_testvec hmac_sha512_t }, }; -#define HMAC_SHA3_224_TEST_VECTORS 4 - -static struct hash_testvec hmac_sha3_224_tv_template[] = { +static const struct hash_testvec hmac_sha3_224_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -3985,9 +4202,7 @@ static struct hash_testvec hmac_sha3_224 }, }; -#define HMAC_SHA3_256_TEST_VECTORS 4 - -static struct hash_testvec hmac_sha3_256_tv_template[] = { +static const struct hash_testvec hmac_sha3_256_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -4076,9 +4291,7 @@ static struct hash_testvec hmac_sha3_256 }, }; -#define HMAC_SHA3_384_TEST_VECTORS 4 - -static struct hash_testvec hmac_sha3_384_tv_template[] = { +static const struct hash_testvec hmac_sha3_384_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -4175,9 +4388,7 @@ static struct hash_testvec hmac_sha3_384 }, }; -#define HMAC_SHA3_512_TEST_VECTORS 4 - -static struct hash_testvec hmac_sha3_512_tv_template[] = { +static const struct hash_testvec hmac_sha3_512_tv_template[] = { { .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" @@ -4286,9 +4497,7 @@ static struct hash_testvec hmac_sha3_512 * Poly1305 test vectors from RFC7539 A.3. */ -#define POLY1305_TEST_VECTORS 11 - -static struct hash_testvec poly1305_tv_template[] = { +static const struct hash_testvec poly1305_tv_template[] = { { /* Test Vector #1 */ .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -4575,20 +4784,7 @@ static struct hash_testvec poly1305_tv_t /* * DES test vectors. */ -#define DES_ENC_TEST_VECTORS 11 -#define DES_DEC_TEST_VECTORS 5 -#define DES_CBC_ENC_TEST_VECTORS 6 -#define DES_CBC_DEC_TEST_VECTORS 5 -#define DES_CTR_ENC_TEST_VECTORS 2 -#define DES_CTR_DEC_TEST_VECTORS 2 -#define DES3_EDE_ENC_TEST_VECTORS 4 -#define DES3_EDE_DEC_TEST_VECTORS 4 -#define DES3_EDE_CBC_ENC_TEST_VECTORS 2 -#define DES3_EDE_CBC_DEC_TEST_VECTORS 2 -#define DES3_EDE_CTR_ENC_TEST_VECTORS 2 -#define DES3_EDE_CTR_DEC_TEST_VECTORS 2 - -static struct cipher_testvec des_enc_tv_template[] = { +static const struct cipher_testvec des_enc_tv_template[] = { { /* From Applied Cryptography */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -4762,7 +4958,7 @@ static struct cipher_testvec des_enc_tv_ }, }; -static struct cipher_testvec des_dec_tv_template[] = { +static const struct cipher_testvec des_dec_tv_template[] = { { /* From Applied Cryptography */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -4872,7 +5068,7 @@ static struct cipher_testvec des_dec_tv_ }, }; -static struct cipher_testvec des_cbc_enc_tv_template[] = { +static const struct cipher_testvec des_cbc_enc_tv_template[] = { { /* From OpenSSL */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -4998,7 +5194,7 @@ static struct cipher_testvec des_cbc_enc }, }; -static struct cipher_testvec des_cbc_dec_tv_template[] = { +static const struct cipher_testvec des_cbc_dec_tv_template[] = { { /* FIPS Pub 81 */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -5107,7 +5303,7 @@ static struct cipher_testvec des_cbc_dec }, }; -static struct cipher_testvec des_ctr_enc_tv_template[] = { +static const struct cipher_testvec des_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55", .klen = 8, @@ -5253,7 +5449,7 @@ static struct cipher_testvec des_ctr_enc }, }; -static struct cipher_testvec des_ctr_dec_tv_template[] = { +static const struct cipher_testvec des_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55", .klen = 8, @@ -5399,7 +5595,7 @@ static struct cipher_testvec des_ctr_dec }, }; -static struct cipher_testvec des3_ede_enc_tv_template[] = { +static const struct cipher_testvec des3_ede_enc_tv_template[] = { { /* These are from openssl */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\x55\x55\x55\x55\x55\x55\x55\x55" @@ -5564,7 +5760,7 @@ static struct cipher_testvec des3_ede_en }, }; -static struct cipher_testvec des3_ede_dec_tv_template[] = { +static const struct cipher_testvec des3_ede_dec_tv_template[] = { { /* These are from openssl */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\x55\x55\x55\x55\x55\x55\x55\x55" @@ -5729,7 +5925,7 @@ static struct cipher_testvec des3_ede_de }, }; -static struct cipher_testvec des3_ede_cbc_enc_tv_template[] = { +static const struct cipher_testvec des3_ede_cbc_enc_tv_template[] = { { /* Generated from openssl */ .key = "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24" "\x44\x4D\x99\x5A\x12\xD6\x40\xC0" @@ -5909,7 +6105,7 @@ static struct cipher_testvec des3_ede_cb }, }; -static struct cipher_testvec des3_ede_cbc_dec_tv_template[] = { +static const struct cipher_testvec des3_ede_cbc_dec_tv_template[] = { { /* Generated from openssl */ .key = "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24" "\x44\x4D\x99\x5A\x12\xD6\x40\xC0" @@ -6089,7 +6285,7 @@ static struct cipher_testvec des3_ede_cb }, }; -static struct cipher_testvec des3_ede_ctr_enc_tv_template[] = { +static const struct cipher_testvec des3_ede_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00" "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE" @@ -6367,7 +6563,7 @@ static struct cipher_testvec des3_ede_ct }, }; -static struct cipher_testvec des3_ede_ctr_dec_tv_template[] = { +static const struct cipher_testvec des3_ede_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00" "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE" @@ -6648,14 +6844,7 @@ static struct cipher_testvec des3_ede_ct /* * Blowfish test vectors. */ -#define BF_ENC_TEST_VECTORS 7 -#define BF_DEC_TEST_VECTORS 7 -#define BF_CBC_ENC_TEST_VECTORS 2 -#define BF_CBC_DEC_TEST_VECTORS 2 -#define BF_CTR_ENC_TEST_VECTORS 2 -#define BF_CTR_DEC_TEST_VECTORS 2 - -static struct cipher_testvec bf_enc_tv_template[] = { +static const struct cipher_testvec bf_enc_tv_template[] = { { /* DES test vectors from OpenSSL */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -6847,7 +7036,7 @@ static struct cipher_testvec bf_enc_tv_t }, }; -static struct cipher_testvec bf_dec_tv_template[] = { +static const struct cipher_testvec bf_dec_tv_template[] = { { /* DES test vectors from OpenSSL */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -7039,7 +7228,7 @@ static struct cipher_testvec bf_dec_tv_t }, }; -static struct cipher_testvec bf_cbc_enc_tv_template[] = { +static const struct cipher_testvec bf_cbc_enc_tv_template[] = { { /* From OpenSSL */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87", @@ -7196,7 +7385,7 @@ static struct cipher_testvec bf_cbc_enc_ }, }; -static struct cipher_testvec bf_cbc_dec_tv_template[] = { +static const struct cipher_testvec bf_cbc_dec_tv_template[] = { { /* From OpenSSL */ .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87", @@ -7353,7 +7542,7 @@ static struct cipher_testvec bf_cbc_dec_ }, }; -static struct cipher_testvec bf_ctr_enc_tv_template[] = { +static const struct cipher_testvec bf_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -7765,7 +7954,7 @@ static struct cipher_testvec bf_ctr_enc_ }, }; -static struct cipher_testvec bf_ctr_dec_tv_template[] = { +static const struct cipher_testvec bf_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -8180,18 +8369,7 @@ static struct cipher_testvec bf_ctr_dec_ /* * Twofish test vectors. */ -#define TF_ENC_TEST_VECTORS 4 -#define TF_DEC_TEST_VECTORS 4 -#define TF_CBC_ENC_TEST_VECTORS 5 -#define TF_CBC_DEC_TEST_VECTORS 5 -#define TF_CTR_ENC_TEST_VECTORS 2 -#define TF_CTR_DEC_TEST_VECTORS 2 -#define TF_LRW_ENC_TEST_VECTORS 8 -#define TF_LRW_DEC_TEST_VECTORS 8 -#define TF_XTS_ENC_TEST_VECTORS 5 -#define TF_XTS_DEC_TEST_VECTORS 5 - -static struct cipher_testvec tf_enc_tv_template[] = { +static const struct cipher_testvec tf_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -8359,7 +8537,7 @@ static struct cipher_testvec tf_enc_tv_t }, }; -static struct cipher_testvec tf_dec_tv_template[] = { +static const struct cipher_testvec tf_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -8527,7 +8705,7 @@ static struct cipher_testvec tf_dec_tv_t }, }; -static struct cipher_testvec tf_cbc_enc_tv_template[] = { +static const struct cipher_testvec tf_cbc_enc_tv_template[] = { { /* Generated with Nettle */ .key = zeroed_string, .klen = 16, @@ -8710,7 +8888,7 @@ static struct cipher_testvec tf_cbc_enc_ }, }; -static struct cipher_testvec tf_cbc_dec_tv_template[] = { +static const struct cipher_testvec tf_cbc_dec_tv_template[] = { { /* Reverse of the first four above */ .key = zeroed_string, .klen = 16, @@ -8893,7 +9071,7 @@ static struct cipher_testvec tf_cbc_dec_ }, }; -static struct cipher_testvec tf_ctr_enc_tv_template[] = { +static const struct cipher_testvec tf_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -9304,7 +9482,7 @@ static struct cipher_testvec tf_ctr_enc_ }, }; -static struct cipher_testvec tf_ctr_dec_tv_template[] = { +static const struct cipher_testvec tf_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -9715,7 +9893,7 @@ static struct cipher_testvec tf_ctr_dec_ }, }; -static struct cipher_testvec tf_lrw_enc_tv_template[] = { +static const struct cipher_testvec tf_lrw_enc_tv_template[] = { /* Generated from AES-LRW test vectors */ { .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -9967,7 +10145,7 @@ static struct cipher_testvec tf_lrw_enc_ }, }; -static struct cipher_testvec tf_lrw_dec_tv_template[] = { +static const struct cipher_testvec tf_lrw_dec_tv_template[] = { /* Generated from AES-LRW test vectors */ /* same as enc vectors with input and result reversed */ { @@ -10220,7 +10398,7 @@ static struct cipher_testvec tf_lrw_dec_ }, }; -static struct cipher_testvec tf_xts_enc_tv_template[] = { +static const struct cipher_testvec tf_xts_enc_tv_template[] = { /* Generated from AES-XTS test vectors */ { .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -10562,7 +10740,7 @@ static struct cipher_testvec tf_xts_enc_ }, }; -static struct cipher_testvec tf_xts_dec_tv_template[] = { +static const struct cipher_testvec tf_xts_dec_tv_template[] = { /* Generated from AES-XTS test vectors */ /* same as enc vectors with input and result reversed */ { @@ -10909,25 +11087,7 @@ static struct cipher_testvec tf_xts_dec_ * Serpent test vectors. These are backwards because Serpent writes * octet sequences in right-to-left mode. */ -#define SERPENT_ENC_TEST_VECTORS 5 -#define SERPENT_DEC_TEST_VECTORS 5 - -#define TNEPRES_ENC_TEST_VECTORS 4 -#define TNEPRES_DEC_TEST_VECTORS 4 - -#define SERPENT_CBC_ENC_TEST_VECTORS 1 -#define SERPENT_CBC_DEC_TEST_VECTORS 1 - -#define SERPENT_CTR_ENC_TEST_VECTORS 2 -#define SERPENT_CTR_DEC_TEST_VECTORS 2 - -#define SERPENT_LRW_ENC_TEST_VECTORS 8 -#define SERPENT_LRW_DEC_TEST_VECTORS 8 - -#define SERPENT_XTS_ENC_TEST_VECTORS 5 -#define SERPENT_XTS_DEC_TEST_VECTORS 5 - -static struct cipher_testvec serpent_enc_tv_template[] = { +static const struct cipher_testvec serpent_enc_tv_template[] = { { .input = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -11103,7 +11263,7 @@ static struct cipher_testvec serpent_enc }, }; -static struct cipher_testvec tnepres_enc_tv_template[] = { +static const struct cipher_testvec tnepres_enc_tv_template[] = { { /* KeySize=128, PT=0, I=1 */ .input = "\x00\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00", @@ -11153,7 +11313,7 @@ static struct cipher_testvec tnepres_enc }; -static struct cipher_testvec serpent_dec_tv_template[] = { +static const struct cipher_testvec serpent_dec_tv_template[] = { { .input = "\x12\x07\xfc\xce\x9b\xd0\xd6\x47" "\x6a\xe9\x8f\xbe\xd1\x43\xa0\xe2", @@ -11329,7 +11489,7 @@ static struct cipher_testvec serpent_dec }, }; -static struct cipher_testvec tnepres_dec_tv_template[] = { +static const struct cipher_testvec tnepres_dec_tv_template[] = { { .input = "\x41\xcc\x6b\x31\x59\x31\x45\x97" "\x6d\x6f\xbb\x38\x4b\x37\x21\x28", @@ -11370,7 +11530,7 @@ static struct cipher_testvec tnepres_dec }, }; -static struct cipher_testvec serpent_cbc_enc_tv_template[] = { +static const struct cipher_testvec serpent_cbc_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -11511,7 +11671,7 @@ static struct cipher_testvec serpent_cbc }, }; -static struct cipher_testvec serpent_cbc_dec_tv_template[] = { +static const struct cipher_testvec serpent_cbc_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -11652,7 +11812,7 @@ static struct cipher_testvec serpent_cbc }, }; -static struct cipher_testvec serpent_ctr_enc_tv_template[] = { +static const struct cipher_testvec serpent_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -12063,7 +12223,7 @@ static struct cipher_testvec serpent_ctr }, }; -static struct cipher_testvec serpent_ctr_dec_tv_template[] = { +static const struct cipher_testvec serpent_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -12474,7 +12634,7 @@ static struct cipher_testvec serpent_ctr }, }; -static struct cipher_testvec serpent_lrw_enc_tv_template[] = { +static const struct cipher_testvec serpent_lrw_enc_tv_template[] = { /* Generated from AES-LRW test vectors */ { .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -12726,7 +12886,7 @@ static struct cipher_testvec serpent_lrw }, }; -static struct cipher_testvec serpent_lrw_dec_tv_template[] = { +static const struct cipher_testvec serpent_lrw_dec_tv_template[] = { /* Generated from AES-LRW test vectors */ /* same as enc vectors with input and result reversed */ { @@ -12979,7 +13139,7 @@ static struct cipher_testvec serpent_lrw }, }; -static struct cipher_testvec serpent_xts_enc_tv_template[] = { +static const struct cipher_testvec serpent_xts_enc_tv_template[] = { /* Generated from AES-XTS test vectors */ { .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -13321,7 +13481,7 @@ static struct cipher_testvec serpent_xts }, }; -static struct cipher_testvec serpent_xts_dec_tv_template[] = { +static const struct cipher_testvec serpent_xts_dec_tv_template[] = { /* Generated from AES-XTS test vectors */ /* same as enc vectors with input and result reversed */ { @@ -13665,18 +13825,7 @@ static struct cipher_testvec serpent_xts }; /* Cast6 test vectors from RFC 2612 */ -#define CAST6_ENC_TEST_VECTORS 4 -#define CAST6_DEC_TEST_VECTORS 4 -#define CAST6_CBC_ENC_TEST_VECTORS 1 -#define CAST6_CBC_DEC_TEST_VECTORS 1 -#define CAST6_CTR_ENC_TEST_VECTORS 2 -#define CAST6_CTR_DEC_TEST_VECTORS 2 -#define CAST6_LRW_ENC_TEST_VECTORS 1 -#define CAST6_LRW_DEC_TEST_VECTORS 1 -#define CAST6_XTS_ENC_TEST_VECTORS 1 -#define CAST6_XTS_DEC_TEST_VECTORS 1 - -static struct cipher_testvec cast6_enc_tv_template[] = { +static const struct cipher_testvec cast6_enc_tv_template[] = { { .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c" "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d", @@ -13847,7 +13996,7 @@ static struct cipher_testvec cast6_enc_t }, }; -static struct cipher_testvec cast6_dec_tv_template[] = { +static const struct cipher_testvec cast6_dec_tv_template[] = { { .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c" "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d", @@ -14018,7 +14167,7 @@ static struct cipher_testvec cast6_dec_t }, }; -static struct cipher_testvec cast6_cbc_enc_tv_template[] = { +static const struct cipher_testvec cast6_cbc_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14159,7 +14308,7 @@ static struct cipher_testvec cast6_cbc_e }, }; -static struct cipher_testvec cast6_cbc_dec_tv_template[] = { +static const struct cipher_testvec cast6_cbc_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14300,7 +14449,7 @@ static struct cipher_testvec cast6_cbc_d }, }; -static struct cipher_testvec cast6_ctr_enc_tv_template[] = { +static const struct cipher_testvec cast6_ctr_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14457,7 +14606,7 @@ static struct cipher_testvec cast6_ctr_e }, }; -static struct cipher_testvec cast6_ctr_dec_tv_template[] = { +static const struct cipher_testvec cast6_ctr_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -14614,7 +14763,7 @@ static struct cipher_testvec cast6_ctr_d }, }; -static struct cipher_testvec cast6_lrw_enc_tv_template[] = { +static const struct cipher_testvec cast6_lrw_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c" "\x23\x84\xcb\x1c\x77\xd6\x19\x5d" @@ -14761,7 +14910,7 @@ static struct cipher_testvec cast6_lrw_e }, }; -static struct cipher_testvec cast6_lrw_dec_tv_template[] = { +static const struct cipher_testvec cast6_lrw_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c" "\x23\x84\xcb\x1c\x77\xd6\x19\x5d" @@ -14908,7 +15057,7 @@ static struct cipher_testvec cast6_lrw_d }, }; -static struct cipher_testvec cast6_xts_enc_tv_template[] = { +static const struct cipher_testvec cast6_xts_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x27\x18\x28\x18\x28\x45\x90\x45" "\x23\x53\x60\x28\x74\x71\x35\x26" @@ -15057,7 +15206,7 @@ static struct cipher_testvec cast6_xts_e }, }; -static struct cipher_testvec cast6_xts_dec_tv_template[] = { +static const struct cipher_testvec cast6_xts_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x27\x18\x28\x18\x28\x45\x90\x45" "\x23\x53\x60\x28\x74\x71\x35\x26" @@ -15210,39 +15359,7 @@ static struct cipher_testvec cast6_xts_d /* * AES test vectors. */ -#define AES_ENC_TEST_VECTORS 4 -#define AES_DEC_TEST_VECTORS 4 -#define AES_CBC_ENC_TEST_VECTORS 5 -#define AES_CBC_DEC_TEST_VECTORS 5 -#define HMAC_MD5_ECB_CIPHER_NULL_ENC_TEST_VECTORS 2 -#define HMAC_MD5_ECB_CIPHER_NULL_DEC_TEST_VECTORS 2 -#define HMAC_SHA1_ECB_CIPHER_NULL_ENC_TEST_VEC 2 -#define HMAC_SHA1_ECB_CIPHER_NULL_DEC_TEST_VEC 2 -#define HMAC_SHA1_AES_CBC_ENC_TEST_VEC 7 -#define HMAC_SHA256_AES_CBC_ENC_TEST_VEC 7 -#define HMAC_SHA512_AES_CBC_ENC_TEST_VEC 7 -#define AES_LRW_ENC_TEST_VECTORS 8 -#define AES_LRW_DEC_TEST_VECTORS 8 -#define AES_XTS_ENC_TEST_VECTORS 5 -#define AES_XTS_DEC_TEST_VECTORS 5 -#define AES_CTR_ENC_TEST_VECTORS 5 -#define AES_CTR_DEC_TEST_VECTORS 5 -#define AES_OFB_ENC_TEST_VECTORS 1 -#define AES_OFB_DEC_TEST_VECTORS 1 -#define AES_CTR_3686_ENC_TEST_VECTORS 7 -#define AES_CTR_3686_DEC_TEST_VECTORS 6 -#define AES_GCM_ENC_TEST_VECTORS 9 -#define AES_GCM_DEC_TEST_VECTORS 8 -#define AES_GCM_4106_ENC_TEST_VECTORS 23 -#define AES_GCM_4106_DEC_TEST_VECTORS 23 -#define AES_GCM_4543_ENC_TEST_VECTORS 1 -#define AES_GCM_4543_DEC_TEST_VECTORS 2 -#define AES_CCM_ENC_TEST_VECTORS 8 -#define AES_CCM_DEC_TEST_VECTORS 7 -#define AES_CCM_4309_ENC_TEST_VECTORS 7 -#define AES_CCM_4309_DEC_TEST_VECTORS 10 - -static struct cipher_testvec aes_enc_tv_template[] = { +static const struct cipher_testvec aes_enc_tv_template[] = { { /* From FIPS-197 */ .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -15414,7 +15531,7 @@ static struct cipher_testvec aes_enc_tv_ }, }; -static struct cipher_testvec aes_dec_tv_template[] = { +static const struct cipher_testvec aes_dec_tv_template[] = { { /* From FIPS-197 */ .key = "\x00\x01\x02\x03\x04\x05\x06\x07" "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", @@ -15586,7 +15703,7 @@ static struct cipher_testvec aes_dec_tv_ }, }; -static struct cipher_testvec aes_cbc_enc_tv_template[] = { +static const struct cipher_testvec aes_cbc_enc_tv_template[] = { { /* From RFC 3602 */ .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -15808,7 +15925,7 @@ static struct cipher_testvec aes_cbc_enc }, }; -static struct cipher_testvec aes_cbc_dec_tv_template[] = { +static const struct cipher_testvec aes_cbc_dec_tv_template[] = { { /* From RFC 3602 */ .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -16030,7 +16147,7 @@ static struct cipher_testvec aes_cbc_dec }, }; -static struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = { +static const struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = { { /* Input data from RFC 2410 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16072,7 +16189,7 @@ static struct aead_testvec hmac_md5_ecb_ }, }; -static struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = { +static const struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = { { #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16114,7 +16231,7 @@ static struct aead_testvec hmac_md5_ecb_ }, }; -static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { { /* RFC 3602 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16383,7 +16500,7 @@ static struct aead_testvec hmac_sha1_aes }, }; -static struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = { { /* Input data from RFC 2410 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16429,7 +16546,7 @@ static struct aead_testvec hmac_sha1_ecb }, }; -static struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = { +static const struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = { { #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16475,7 +16592,7 @@ static struct aead_testvec hmac_sha1_ecb }, }; -static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { { /* RFC 3602 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -16758,7 +16875,7 @@ static struct aead_testvec hmac_sha256_a }, }; -static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { { /* RFC 3602 Case 1 */ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17097,9 +17214,7 @@ static struct aead_testvec hmac_sha512_a }, }; -#define HMAC_SHA1_DES_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17158,9 +17273,7 @@ static struct aead_testvec hmac_sha1_des }, }; -#define HMAC_SHA224_DES_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17219,9 +17332,7 @@ static struct aead_testvec hmac_sha224_d }, }; -#define HMAC_SHA256_DES_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17282,9 +17393,7 @@ static struct aead_testvec hmac_sha256_d }, }; -#define HMAC_SHA384_DES_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17349,9 +17458,7 @@ static struct aead_testvec hmac_sha384_d }, }; -#define HMAC_SHA512_DES_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17420,9 +17527,7 @@ static struct aead_testvec hmac_sha512_d }, }; -#define HMAC_SHA1_DES3_EDE_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17483,9 +17588,7 @@ static struct aead_testvec hmac_sha1_des }, }; -#define HMAC_SHA224_DES3_EDE_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17546,9 +17649,7 @@ static struct aead_testvec hmac_sha224_d }, }; -#define HMAC_SHA256_DES3_EDE_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17611,9 +17712,7 @@ static struct aead_testvec hmac_sha256_d }, }; -#define HMAC_SHA384_DES3_EDE_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17680,9 +17779,7 @@ static struct aead_testvec hmac_sha384_d }, }; -#define HMAC_SHA512_DES3_EDE_CBC_ENC_TEST_VEC 1 - -static struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { +static const struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { { /*Generated with cryptopp*/ #ifdef __LITTLE_ENDIAN .key = "\x08\x00" /* rta length */ @@ -17753,7 +17850,7 @@ static struct aead_testvec hmac_sha512_d }, }; -static struct cipher_testvec aes_lrw_enc_tv_template[] = { +static const struct cipher_testvec aes_lrw_enc_tv_template[] = { /* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */ { /* LRW-32-AES 1 */ .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -18006,7 +18103,7 @@ static struct cipher_testvec aes_lrw_enc } }; -static struct cipher_testvec aes_lrw_dec_tv_template[] = { +static const struct cipher_testvec aes_lrw_dec_tv_template[] = { /* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */ /* same as enc vectors with input and result reversed */ { /* LRW-32-AES 1 */ @@ -18260,7 +18357,7 @@ static struct cipher_testvec aes_lrw_dec } }; -static struct cipher_testvec aes_xts_enc_tv_template[] = { +static const struct cipher_testvec aes_xts_enc_tv_template[] = { /* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */ { /* XTS-AES 1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -18603,7 +18700,7 @@ static struct cipher_testvec aes_xts_enc } }; -static struct cipher_testvec aes_xts_dec_tv_template[] = { +static const struct cipher_testvec aes_xts_dec_tv_template[] = { /* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */ { /* XTS-AES 1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -18947,7 +19044,7 @@ static struct cipher_testvec aes_xts_dec }; -static struct cipher_testvec aes_ctr_enc_tv_template[] = { +static const struct cipher_testvec aes_ctr_enc_tv_template[] = { { /* From NIST Special Publication 800-38A, Appendix F.5 */ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", @@ -19302,7 +19399,7 @@ static struct cipher_testvec aes_ctr_enc }, }; -static struct cipher_testvec aes_ctr_dec_tv_template[] = { +static const struct cipher_testvec aes_ctr_dec_tv_template[] = { { /* From NIST Special Publication 800-38A, Appendix F.5 */ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", @@ -19657,7 +19754,7 @@ static struct cipher_testvec aes_ctr_dec }, }; -static struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = { +static const struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = { { /* From RFC 3686 */ .key = "\xae\x68\x52\xf8\x12\x10\x67\xcc" "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e" @@ -20789,7 +20886,7 @@ static struct cipher_testvec aes_ctr_rfc }, }; -static struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = { +static const struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = { { /* From RFC 3686 */ .key = "\xae\x68\x52\xf8\x12\x10\x67\xcc" "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e" @@ -20880,7 +20977,7 @@ static struct cipher_testvec aes_ctr_rfc }, }; -static struct cipher_testvec aes_ofb_enc_tv_template[] = { +static const struct cipher_testvec aes_ofb_enc_tv_template[] = { /* From NIST Special Publication 800-38A, Appendix F.5 */ { .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" @@ -20909,7 +21006,7 @@ static struct cipher_testvec aes_ofb_enc } }; -static struct cipher_testvec aes_ofb_dec_tv_template[] = { +static const struct cipher_testvec aes_ofb_dec_tv_template[] = { /* From NIST Special Publication 800-38A, Appendix F.5 */ { .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" @@ -20938,7 +21035,7 @@ static struct cipher_testvec aes_ofb_dec } }; -static struct aead_testvec aes_gcm_enc_tv_template[] = { +static const struct aead_testvec aes_gcm_enc_tv_template[] = { { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ .key = zeroed_string, .klen = 16, @@ -21098,7 +21195,7 @@ static struct aead_testvec aes_gcm_enc_t } }; -static struct aead_testvec aes_gcm_dec_tv_template[] = { +static const struct aead_testvec aes_gcm_dec_tv_template[] = { { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ .key = zeroed_string, .klen = 32, @@ -21300,7 +21397,7 @@ static struct aead_testvec aes_gcm_dec_t } }; -static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 20, @@ -21913,7 +22010,7 @@ static struct aead_testvec aes_gcm_rfc41 } }; -static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 20, @@ -22527,7 +22624,7 @@ static struct aead_testvec aes_gcm_rfc41 } }; -static struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { { /* From draft-mcgrew-gcm-test-01 */ .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" @@ -22558,7 +22655,7 @@ static struct aead_testvec aes_gcm_rfc45 } }; -static struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { +static const struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { { /* From draft-mcgrew-gcm-test-01 */ .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" @@ -22617,7 +22714,7 @@ static struct aead_testvec aes_gcm_rfc45 }, }; -static struct aead_testvec aes_ccm_enc_tv_template[] = { +static const struct aead_testvec aes_ccm_enc_tv_template[] = { { /* From RFC 3610 */ .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", @@ -22901,7 +22998,7 @@ static struct aead_testvec aes_ccm_enc_t } }; -static struct aead_testvec aes_ccm_dec_tv_template[] = { +static const struct aead_testvec aes_ccm_dec_tv_template[] = { { /* From RFC 3610 */ .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", @@ -23233,7 +23330,7 @@ static struct aead_testvec aes_ccm_dec_t * These vectors are copied/generated from the ones for rfc4106 with * the key truncated by one byte.. */ -static struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { +static const struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 19, @@ -23846,7 +23943,7 @@ static struct aead_testvec aes_ccm_rfc43 } }; -static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { +static const struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { { /* Generated using Crypto++ */ .key = zeroed_string, .klen = 19, @@ -24462,9 +24559,7 @@ static struct aead_testvec aes_ccm_rfc43 /* * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5. */ -#define RFC7539_ENC_TEST_VECTORS 2 -#define RFC7539_DEC_TEST_VECTORS 2 -static struct aead_testvec rfc7539_enc_tv_template[] = { +static const struct aead_testvec rfc7539_enc_tv_template[] = { { .key = "\x80\x81\x82\x83\x84\x85\x86\x87" "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" @@ -24596,7 +24691,7 @@ static struct aead_testvec rfc7539_enc_t }, }; -static struct aead_testvec rfc7539_dec_tv_template[] = { +static const struct aead_testvec rfc7539_dec_tv_template[] = { { .key = "\x80\x81\x82\x83\x84\x85\x86\x87" "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" @@ -24731,9 +24826,7 @@ static struct aead_testvec rfc7539_dec_t /* * draft-irtf-cfrg-chacha20-poly1305 */ -#define RFC7539ESP_DEC_TEST_VECTORS 1 -#define RFC7539ESP_ENC_TEST_VECTORS 1 -static struct aead_testvec rfc7539esp_enc_tv_template[] = { +static const struct aead_testvec rfc7539esp_enc_tv_template[] = { { .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" @@ -24821,7 +24914,7 @@ static struct aead_testvec rfc7539esp_en }, }; -static struct aead_testvec rfc7539esp_dec_tv_template[] = { +static const struct aead_testvec rfc7539esp_dec_tv_template[] = { { .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" @@ -24917,7 +25010,7 @@ static struct aead_testvec rfc7539esp_de * semiblock of the ciphertext from the test vector. For decryption, iv is * the first semiblock of the ciphertext. */ -static struct cipher_testvec aes_kw_enc_tv_template[] = { +static const struct cipher_testvec aes_kw_enc_tv_template[] = { { .key = "\x75\x75\xda\x3a\x93\x60\x7c\xc2" "\xbf\xd8\xce\xc7\xaa\xdf\xd9\xa6", @@ -24932,7 +25025,7 @@ static struct cipher_testvec aes_kw_enc_ }, }; -static struct cipher_testvec aes_kw_dec_tv_template[] = { +static const struct cipher_testvec aes_kw_dec_tv_template[] = { { .key = "\x80\xaa\x99\x73\x27\xa4\x80\x6b" "\x6a\x7a\x41\xa5\x2b\x86\xc3\x71" @@ -24955,9 +25048,7 @@ static struct cipher_testvec aes_kw_dec_ * http://csrc.nist.gov/groups/STM/cavp/documents/rng/RNGVS.pdf * Only AES-128 is supported at this time. */ -#define ANSI_CPRNG_AES_TEST_VECTORS 6 - -static struct cprng_testvec ansi_cprng_aes_tv_template[] = { +static const struct cprng_testvec ansi_cprng_aes_tv_template[] = { { .key = "\xf3\xb1\x66\x6d\x13\x60\x72\x42" "\xed\x06\x1c\xab\xb8\xd4\x62\x02", @@ -25053,7 +25144,7 @@ static struct cprng_testvec ansi_cprng_a * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and * w/o personalization string, w/ and w/o additional input string). */ -static struct drbg_testvec drbg_pr_sha256_tv_template[] = { +static const struct drbg_testvec drbg_pr_sha256_tv_template[] = { { .entropy = (unsigned char *) "\x72\x88\x4c\xcd\x6c\x85\x57\x70\xf7\x0b\x8b\x86" @@ -25211,7 +25302,7 @@ static struct drbg_testvec drbg_pr_sha25 }, }; -static struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { +static const struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { { .entropy = (unsigned char *) "\x99\x69\xe5\x4b\x47\x03\xff\x31\x78\x5b\x87\x9a" @@ -25369,7 +25460,7 @@ static struct drbg_testvec drbg_pr_hmac_ }, }; -static struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { +static const struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { { .entropy = (unsigned char *) "\xd1\x44\xc6\x61\x81\x6d\xca\x9d\x15\x28\x8a\x42" @@ -25493,7 +25584,7 @@ static struct drbg_testvec drbg_pr_ctr_a * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and * w/o personalization string, w/ and w/o additional input string). */ -static struct drbg_testvec drbg_nopr_sha256_tv_template[] = { +static const struct drbg_testvec drbg_nopr_sha256_tv_template[] = { { .entropy = (unsigned char *) "\xa6\x5a\xd0\xf3\x45\xdb\x4e\x0e\xff\xe8\x75\xc3" @@ -25615,7 +25706,7 @@ static struct drbg_testvec drbg_nopr_sha }, }; -static struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { +static const struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { { .entropy = (unsigned char *) "\xca\x85\x19\x11\x34\x93\x84\xbf\xfe\x89\xde\x1c" @@ -25737,7 +25828,7 @@ static struct drbg_testvec drbg_nopr_hma }, }; -static struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { +static const struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { { .entropy = (unsigned char *) "\xc3\x5c\x2f\xa2\xa8\x9d\x52\xa1\x1f\xa3\x2a\xa9" @@ -25761,7 +25852,7 @@ static struct drbg_testvec drbg_nopr_ctr }, }; -static struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { +static const struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { { .entropy = (unsigned char *) "\x36\x40\x19\x40\xfa\x8b\x1f\xba\x91\xa1\x66\x1f" @@ -25785,7 +25876,7 @@ static struct drbg_testvec drbg_nopr_ctr }, }; -static struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { +static const struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { { .entropy = (unsigned char *) "\x87\xe1\xc5\x32\x99\x7f\x57\xa3\x5c\x28\x6d\xe8" @@ -25874,14 +25965,7 @@ static struct drbg_testvec drbg_nopr_ctr }; /* Cast5 test vectors from RFC 2144 */ -#define CAST5_ENC_TEST_VECTORS 4 -#define CAST5_DEC_TEST_VECTORS 4 -#define CAST5_CBC_ENC_TEST_VECTORS 1 -#define CAST5_CBC_DEC_TEST_VECTORS 1 -#define CAST5_CTR_ENC_TEST_VECTORS 2 -#define CAST5_CTR_DEC_TEST_VECTORS 2 - -static struct cipher_testvec cast5_enc_tv_template[] = { +static const struct cipher_testvec cast5_enc_tv_template[] = { { .key = "\x01\x23\x45\x67\x12\x34\x56\x78" "\x23\x45\x67\x89\x34\x56\x78\x9a", @@ -26042,7 +26126,7 @@ static struct cipher_testvec cast5_enc_t }, }; -static struct cipher_testvec cast5_dec_tv_template[] = { +static const struct cipher_testvec cast5_dec_tv_template[] = { { .key = "\x01\x23\x45\x67\x12\x34\x56\x78" "\x23\x45\x67\x89\x34\x56\x78\x9a", @@ -26203,7 +26287,7 @@ static struct cipher_testvec cast5_dec_t }, }; -static struct cipher_testvec cast5_cbc_enc_tv_template[] = { +static const struct cipher_testvec cast5_cbc_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26341,7 +26425,7 @@ static struct cipher_testvec cast5_cbc_e }, }; -static struct cipher_testvec cast5_cbc_dec_tv_template[] = { +static const struct cipher_testvec cast5_cbc_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26479,7 +26563,7 @@ static struct cipher_testvec cast5_cbc_d }, }; -static struct cipher_testvec cast5_ctr_enc_tv_template[] = { +static const struct cipher_testvec cast5_ctr_enc_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26630,7 +26714,7 @@ static struct cipher_testvec cast5_ctr_e }, }; -static struct cipher_testvec cast5_ctr_dec_tv_template[] = { +static const struct cipher_testvec cast5_ctr_dec_tv_template[] = { { /* Generated from TF test vectors */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", @@ -26784,10 +26868,7 @@ static struct cipher_testvec cast5_ctr_d /* * ARC4 test vectors from OpenSSL */ -#define ARC4_ENC_TEST_VECTORS 7 -#define ARC4_DEC_TEST_VECTORS 7 - -static struct cipher_testvec arc4_enc_tv_template[] = { +static const struct cipher_testvec arc4_enc_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -26853,7 +26934,7 @@ static struct cipher_testvec arc4_enc_tv }, }; -static struct cipher_testvec arc4_dec_tv_template[] = { +static const struct cipher_testvec arc4_dec_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", .klen = 8, @@ -26922,10 +27003,7 @@ static struct cipher_testvec arc4_dec_tv /* * TEA test vectors */ -#define TEA_ENC_TEST_VECTORS 4 -#define TEA_DEC_TEST_VECTORS 4 - -static struct cipher_testvec tea_enc_tv_template[] = { +static const struct cipher_testvec tea_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -26968,7 +27046,7 @@ static struct cipher_testvec tea_enc_tv_ } }; -static struct cipher_testvec tea_dec_tv_template[] = { +static const struct cipher_testvec tea_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -27014,10 +27092,7 @@ static struct cipher_testvec tea_dec_tv_ /* * XTEA test vectors */ -#define XTEA_ENC_TEST_VECTORS 4 -#define XTEA_DEC_TEST_VECTORS 4 - -static struct cipher_testvec xtea_enc_tv_template[] = { +static const struct cipher_testvec xtea_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -27060,7 +27135,7 @@ static struct cipher_testvec xtea_enc_tv } }; -static struct cipher_testvec xtea_dec_tv_template[] = { +static const struct cipher_testvec xtea_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -27106,10 +27181,7 @@ static struct cipher_testvec xtea_dec_tv /* * KHAZAD test vectors. */ -#define KHAZAD_ENC_TEST_VECTORS 5 -#define KHAZAD_DEC_TEST_VECTORS 5 - -static struct cipher_testvec khazad_enc_tv_template[] = { +static const struct cipher_testvec khazad_enc_tv_template[] = { { .key = "\x80\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00", @@ -27155,7 +27227,7 @@ static struct cipher_testvec khazad_enc_ }, }; -static struct cipher_testvec khazad_dec_tv_template[] = { +static const struct cipher_testvec khazad_dec_tv_template[] = { { .key = "\x80\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00", @@ -27205,12 +27277,7 @@ static struct cipher_testvec khazad_dec_ * Anubis test vectors. */ -#define ANUBIS_ENC_TEST_VECTORS 5 -#define ANUBIS_DEC_TEST_VECTORS 5 -#define ANUBIS_CBC_ENC_TEST_VECTORS 2 -#define ANUBIS_CBC_DEC_TEST_VECTORS 2 - -static struct cipher_testvec anubis_enc_tv_template[] = { +static const struct cipher_testvec anubis_enc_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27273,7 +27340,7 @@ static struct cipher_testvec anubis_enc_ }, }; -static struct cipher_testvec anubis_dec_tv_template[] = { +static const struct cipher_testvec anubis_dec_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27336,7 +27403,7 @@ static struct cipher_testvec anubis_dec_ }, }; -static struct cipher_testvec anubis_cbc_enc_tv_template[] = { +static const struct cipher_testvec anubis_cbc_enc_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27371,7 +27438,7 @@ static struct cipher_testvec anubis_cbc_ }, }; -static struct cipher_testvec anubis_cbc_dec_tv_template[] = { +static const struct cipher_testvec anubis_cbc_dec_tv_template[] = { { .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", @@ -27409,10 +27476,7 @@ static struct cipher_testvec anubis_cbc_ /* * XETA test vectors */ -#define XETA_ENC_TEST_VECTORS 4 -#define XETA_DEC_TEST_VECTORS 4 - -static struct cipher_testvec xeta_enc_tv_template[] = { +static const struct cipher_testvec xeta_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -27455,7 +27519,7 @@ static struct cipher_testvec xeta_enc_tv } }; -static struct cipher_testvec xeta_dec_tv_template[] = { +static const struct cipher_testvec xeta_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -27501,10 +27565,7 @@ static struct cipher_testvec xeta_dec_tv /* * FCrypt test vectors */ -#define FCRYPT_ENC_TEST_VECTORS ARRAY_SIZE(fcrypt_pcbc_enc_tv_template) -#define FCRYPT_DEC_TEST_VECTORS ARRAY_SIZE(fcrypt_pcbc_dec_tv_template) - -static struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { +static const struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -27565,7 +27626,7 @@ static struct cipher_testvec fcrypt_pcbc } }; -static struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = { +static const struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = { { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .klen = 8, @@ -27629,18 +27690,7 @@ static struct cipher_testvec fcrypt_pcbc /* * CAMELLIA test vectors. */ -#define CAMELLIA_ENC_TEST_VECTORS 4 -#define CAMELLIA_DEC_TEST_VECTORS 4 -#define CAMELLIA_CBC_ENC_TEST_VECTORS 3 -#define CAMELLIA_CBC_DEC_TEST_VECTORS 3 -#define CAMELLIA_CTR_ENC_TEST_VECTORS 2 -#define CAMELLIA_CTR_DEC_TEST_VECTORS 2 -#define CAMELLIA_LRW_ENC_TEST_VECTORS 8 -#define CAMELLIA_LRW_DEC_TEST_VECTORS 8 -#define CAMELLIA_XTS_ENC_TEST_VECTORS 5 -#define CAMELLIA_XTS_DEC_TEST_VECTORS 5 - -static struct cipher_testvec camellia_enc_tv_template[] = { +static const struct cipher_testvec camellia_enc_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xfe\xdc\xba\x98\x76\x54\x32\x10", @@ -27940,7 +27990,7 @@ static struct cipher_testvec camellia_en }, }; -static struct cipher_testvec camellia_dec_tv_template[] = { +static const struct cipher_testvec camellia_dec_tv_template[] = { { .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" "\xfe\xdc\xba\x98\x76\x54\x32\x10", @@ -28240,7 +28290,7 @@ static struct cipher_testvec camellia_de }, }; -static struct cipher_testvec camellia_cbc_enc_tv_template[] = { +static const struct cipher_testvec camellia_cbc_enc_tv_template[] = { { .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -28536,7 +28586,7 @@ static struct cipher_testvec camellia_cb }, }; -static struct cipher_testvec camellia_cbc_dec_tv_template[] = { +static const struct cipher_testvec camellia_cbc_dec_tv_template[] = { { .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" "\x51\x2e\x03\xd5\x34\x12\x00\x06", @@ -28832,7 +28882,7 @@ static struct cipher_testvec camellia_cb }, }; -static struct cipher_testvec camellia_ctr_enc_tv_template[] = { +static const struct cipher_testvec camellia_ctr_enc_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -29499,7 +29549,7 @@ static struct cipher_testvec camellia_ct }, }; -static struct cipher_testvec camellia_ctr_dec_tv_template[] = { +static const struct cipher_testvec camellia_ctr_dec_tv_template[] = { { /* Generated with Crypto++ */ .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" @@ -30166,7 +30216,7 @@ static struct cipher_testvec camellia_ct }, }; -static struct cipher_testvec camellia_lrw_enc_tv_template[] = { +static const struct cipher_testvec camellia_lrw_enc_tv_template[] = { /* Generated from AES-LRW test vectors */ { .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" @@ -30418,7 +30468,7 @@ static struct cipher_testvec camellia_lr }, }; -static struct cipher_testvec camellia_lrw_dec_tv_template[] = { +static const struct cipher_testvec camellia_lrw_dec_tv_template[] = { /* Generated from AES-LRW test vectors */ /* same as enc vectors with input and result reversed */ { @@ -30671,7 +30721,7 @@ static struct cipher_testvec camellia_lr }, }; -static struct cipher_testvec camellia_xts_enc_tv_template[] = { +static const struct cipher_testvec camellia_xts_enc_tv_template[] = { /* Generated from AES-XTS test vectors */ { .key = "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -31013,7 +31063,7 @@ static struct cipher_testvec camellia_xt }, }; -static struct cipher_testvec camellia_xts_dec_tv_template[] = { +static const struct cipher_testvec camellia_xts_dec_tv_template[] = { /* Generated from AES-XTS test vectors */ /* same as enc vectors with input and result reversed */ { @@ -31359,10 +31409,7 @@ static struct cipher_testvec camellia_xt /* * SEED test vectors */ -#define SEED_ENC_TEST_VECTORS 4 -#define SEED_DEC_TEST_VECTORS 4 - -static struct cipher_testvec seed_enc_tv_template[] = { +static const struct cipher_testvec seed_enc_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -31404,7 +31451,7 @@ static struct cipher_testvec seed_enc_tv } }; -static struct cipher_testvec seed_dec_tv_template[] = { +static const struct cipher_testvec seed_dec_tv_template[] = { { .key = zeroed_string, .klen = 16, @@ -31446,8 +31493,7 @@ static struct cipher_testvec seed_dec_tv } }; -#define SALSA20_STREAM_ENC_TEST_VECTORS 5 -static struct cipher_testvec salsa20_stream_enc_tv_template[] = { +static const struct cipher_testvec salsa20_stream_enc_tv_template[] = { /* * Testvectors from verified.test-vectors submitted to ECRYPT. * They are truncated to size 39, 64, 111, 129 to test a variety @@ -32616,8 +32662,7 @@ static struct cipher_testvec salsa20_str }, }; -#define CHACHA20_ENC_TEST_VECTORS 4 -static struct cipher_testvec chacha20_enc_tv_template[] = { +static const struct cipher_testvec chacha20_enc_tv_template[] = { { /* RFC7539 A.2. Test Vector #1 */ .key = "\x00\x00\x00\x00\x00\x00\x00\x00" "\x00\x00\x00\x00\x00\x00\x00\x00" @@ -33128,9 +33173,7 @@ static struct cipher_testvec chacha20_en /* * CTS (Cipher Text Stealing) mode tests */ -#define CTS_MODE_ENC_TEST_VECTORS 6 -#define CTS_MODE_DEC_TEST_VECTORS 6 -static struct cipher_testvec cts_mode_enc_tv_template[] = { +static const struct cipher_testvec cts_mode_enc_tv_template[] = { { /* from rfc3962 */ .klen = 16, .key = "\x63\x68\x69\x63\x6b\x65\x6e\x20" @@ -33232,7 +33275,7 @@ static struct cipher_testvec cts_mode_en } }; -static struct cipher_testvec cts_mode_dec_tv_template[] = { +static const struct cipher_testvec cts_mode_dec_tv_template[] = { { /* from rfc3962 */ .klen = 16, .key = "\x63\x68\x69\x63\x6b\x65\x6e\x20" @@ -33350,10 +33393,7 @@ struct comp_testvec { * Params: winbits=-11, Z_DEFAULT_COMPRESSION, MAX_MEM_LEVEL. */ -#define DEFLATE_COMP_TEST_VECTORS 2 -#define DEFLATE_DECOMP_TEST_VECTORS 2 - -static struct comp_testvec deflate_comp_tv_template[] = { +static const struct comp_testvec deflate_comp_tv_template[] = { { .inlen = 70, .outlen = 38, @@ -33389,7 +33429,7 @@ static struct comp_testvec deflate_comp_ }, }; -static struct comp_testvec deflate_decomp_tv_template[] = { +static const struct comp_testvec deflate_decomp_tv_template[] = { { .inlen = 122, .outlen = 191, @@ -33428,10 +33468,7 @@ static struct comp_testvec deflate_decom /* * LZO test vectors (null-terminated strings). */ -#define LZO_COMP_TEST_VECTORS 2 -#define LZO_DECOMP_TEST_VECTORS 2 - -static struct comp_testvec lzo_comp_tv_template[] = { +static const struct comp_testvec lzo_comp_tv_template[] = { { .inlen = 70, .outlen = 57, @@ -33471,7 +33508,7 @@ static struct comp_testvec lzo_comp_tv_t }, }; -static struct comp_testvec lzo_decomp_tv_template[] = { +static const struct comp_testvec lzo_decomp_tv_template[] = { { .inlen = 133, .outlen = 159, @@ -33514,7 +33551,7 @@ static struct comp_testvec lzo_decomp_tv */ #define MICHAEL_MIC_TEST_VECTORS 6 -static struct hash_testvec michael_mic_tv_template[] = { +static const struct hash_testvec michael_mic_tv_template[] = { { .key = "\x00\x00\x00\x00\x00\x00\x00\x00", .ksize = 8, @@ -33562,9 +33599,7 @@ static struct hash_testvec michael_mic_t /* * CRC32 test vectors */ -#define CRC32_TEST_VECTORS 14 - -static struct hash_testvec crc32_tv_template[] = { +static const struct hash_testvec crc32_tv_template[] = { { .key = "\x87\xa9\xcb\xed", .ksize = 4, @@ -33996,9 +34031,7 @@ static struct hash_testvec crc32_tv_temp /* * CRC32C test vectors */ -#define CRC32C_TEST_VECTORS 15 - -static struct hash_testvec crc32c_tv_template[] = { +static const struct hash_testvec crc32c_tv_template[] = { { .psize = 0, .digest = "\x00\x00\x00\x00", @@ -34434,9 +34467,7 @@ static struct hash_testvec crc32c_tv_tem /* * Blakcifn CRC test vectors */ -#define BFIN_CRC_TEST_VECTORS 6 - -static struct hash_testvec bfin_crc_tv_template[] = { +static const struct hash_testvec bfin_crc_tv_template[] = { { .psize = 0, .digest = "\x00\x00\x00\x00", @@ -34521,9 +34552,6 @@ static struct hash_testvec bfin_crc_tv_t }; -#define LZ4_COMP_TEST_VECTORS 1 -#define LZ4_DECOMP_TEST_VECTORS 1 - static struct comp_testvec lz4_comp_tv_template[] = { { .inlen = 70, @@ -34554,9 +34582,6 @@ static struct comp_testvec lz4_decomp_tv }, }; -#define LZ4HC_COMP_TEST_VECTORS 1 -#define LZ4HC_DECOMP_TEST_VECTORS 1 - static struct comp_testvec lz4hc_comp_tv_template[] = { { .inlen = 70, --- /dev/null +++ b/crypto/tls.c @@ -0,0 +1,607 @@ +/* + * Copyright 2013 Freescale Semiconductor, Inc. + * Copyright 2017 NXP Semiconductor, Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct tls_instance_ctx { + struct crypto_ahash_spawn auth; + struct crypto_skcipher_spawn enc; +}; + +struct crypto_tls_ctx { + unsigned int reqoff; + struct crypto_ahash *auth; + struct crypto_skcipher *enc; + struct crypto_skcipher *null; +}; + +struct tls_request_ctx { + /* + * cryptlen holds the payload length in the case of encryption or + * payload_len + icv_len + padding_len in case of decryption + */ + unsigned int cryptlen; + /* working space for partial results */ + struct scatterlist tmp[2]; + struct scatterlist cipher[2]; + struct scatterlist dst[2]; + char tail[]; +}; + +struct async_op { + struct completion completion; + int err; +}; + +static void tls_async_op_done(struct crypto_async_request *req, int err) +{ + struct async_op *areq = req->data; + + if (err == -EINPROGRESS) + return; + + areq->err = err; + complete(&areq->completion); +} + +static int crypto_tls_setkey(struct crypto_aead *tls, const u8 *key, + unsigned int keylen) +{ + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); + struct crypto_ahash *auth = ctx->auth; + struct crypto_skcipher *enc = ctx->enc; + struct crypto_authenc_keys keys; + int err = -EINVAL; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + + crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(auth, crypto_aead_get_flags(tls) & + CRYPTO_TFM_REQ_MASK); + err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen); + crypto_aead_set_flags(tls, crypto_ahash_get_flags(auth) & + CRYPTO_TFM_RES_MASK); + + if (err) + goto out; + + crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(enc, crypto_aead_get_flags(tls) & + CRYPTO_TFM_REQ_MASK); + err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen); + crypto_aead_set_flags(tls, crypto_skcipher_get_flags(enc) & + CRYPTO_TFM_RES_MASK); + +out: + return err; + +badkey: + crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN); + goto out; +} + +/** + * crypto_tls_genicv - Calculate hmac digest for a TLS record + * @hash: (output) buffer to save the digest into + * @src: (input) scatterlist with the assoc and payload data + * @srclen: (input) size of the source buffer (assoclen + cryptlen) + * @req: (input) aead request + **/ +static int crypto_tls_genicv(u8 *hash, struct scatterlist *src, + unsigned int srclen, struct aead_request *req) +{ + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); + struct tls_request_ctx *treq_ctx = aead_request_ctx(req); + struct async_op ahash_op; + struct ahash_request *ahreq = (void *)(treq_ctx->tail + ctx->reqoff); + unsigned int flags = CRYPTO_TFM_REQ_MAY_SLEEP; + int err = -EBADMSG; + + /* Bail out if the request assoc len is 0 */ + if (!req->assoclen) + return err; + + init_completion(&ahash_op.completion); + + /* the hash transform to be executed comes from the original request */ + ahash_request_set_tfm(ahreq, ctx->auth); + /* prepare the hash request with input data and result pointer */ + ahash_request_set_crypt(ahreq, src, hash, srclen); + /* set the notifier for when the async hash function returns */ + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, + tls_async_op_done, &ahash_op); + + /* Calculate the digest on the given data. The result is put in hash */ + err = crypto_ahash_digest(ahreq); + if (err == -EINPROGRESS) { + err = wait_for_completion_interruptible(&ahash_op.completion); + if (!err) + err = ahash_op.err; + } + + return err; +} + +/** + * crypto_tls_gen_padicv - Calculate and pad hmac digest for a TLS record + * @hash: (output) buffer to save the digest and padding into + * @phashlen: (output) the size of digest + padding + * @req: (input) aead request + **/ +static int crypto_tls_gen_padicv(u8 *hash, unsigned int *phashlen, + struct aead_request *req) +{ + struct crypto_aead *tls = crypto_aead_reqtfm(req); + unsigned int hash_size = crypto_aead_authsize(tls); + unsigned int block_size = crypto_aead_blocksize(tls); + unsigned int srclen = req->cryptlen + hash_size; + unsigned int icvlen = req->cryptlen + req->assoclen; + unsigned int padlen; + int err; + + err = crypto_tls_genicv(hash, req->src, icvlen, req); + if (err) + goto out; + + /* add padding after digest */ + padlen = block_size - (srclen % block_size); + memset(hash + hash_size, padlen - 1, padlen); + + *phashlen = hash_size + padlen; +out: + return err; +} + +static int crypto_tls_copy_data(struct aead_request *req, + struct scatterlist *src, + struct scatterlist *dst, + unsigned int len) +{ + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); + SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); + + skcipher_request_set_tfm(skreq, ctx->null); + skcipher_request_set_callback(skreq, aead_request_flags(req), + NULL, NULL); + skcipher_request_set_crypt(skreq, src, dst, len, NULL); + + return crypto_skcipher_encrypt(skreq); +} + +static int crypto_tls_encrypt(struct aead_request *req) +{ + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); + struct tls_request_ctx *treq_ctx = aead_request_ctx(req); + struct skcipher_request *skreq; + struct scatterlist *cipher = treq_ctx->cipher; + struct scatterlist *tmp = treq_ctx->tmp; + struct scatterlist *sg, *src, *dst; + unsigned int cryptlen, phashlen; + u8 *hash = treq_ctx->tail; + int err; + + /* + * The hash result is saved at the beginning of the tls request ctx + * and is aligned as required by the hash transform. Enough space was + * allocated in crypto_tls_init_tfm to accommodate the difference. The + * requests themselves start later at treq_ctx->tail + ctx->reqoff so + * the result is not overwritten by the second (cipher) request. + */ + hash = (u8 *)ALIGN((unsigned long)hash + + crypto_ahash_alignmask(ctx->auth), + crypto_ahash_alignmask(ctx->auth) + 1); + + /* + * STEP 1: create ICV together with necessary padding + */ + err = crypto_tls_gen_padicv(hash, &phashlen, req); + if (err) + return err; + + /* + * STEP 2: Hash and padding are combined with the payload + * depending on the form it arrives. Scatter tables must have at least + * one page of data before chaining with another table and can't have + * an empty data page. The following code addresses these requirements. + * + * If the payload is empty, only the hash is encrypted, otherwise the + * payload scatterlist is merged with the hash. A special merging case + * is when the payload has only one page of data. In that case the + * payload page is moved to another scatterlist and prepared there for + * encryption. + */ + if (req->cryptlen) { + src = scatterwalk_ffwd(tmp, req->src, req->assoclen); + + sg_init_table(cipher, 2); + sg_set_buf(cipher + 1, hash, phashlen); + + if (sg_is_last(src)) { + sg_set_page(cipher, sg_page(src), req->cryptlen, + src->offset); + src = cipher; + } else { + unsigned int rem_len = req->cryptlen; + + for (sg = src; rem_len > sg->length; sg = sg_next(sg)) + rem_len -= min(rem_len, sg->length); + + sg_set_page(cipher, sg_page(sg), rem_len, sg->offset); + sg_chain(sg, 1, cipher); + } + } else { + sg_init_one(cipher, hash, phashlen); + src = cipher; + } + + /** + * If src != dst copy the associated data from source to destination. + * In both cases fast-forward passed the associated data in the dest. + */ + if (req->src != req->dst) { + err = crypto_tls_copy_data(req, req->src, req->dst, + req->assoclen); + if (err) + return err; + } + dst = scatterwalk_ffwd(treq_ctx->dst, req->dst, req->assoclen); + + /* + * STEP 3: encrypt the frame and return the result + */ + cryptlen = req->cryptlen + phashlen; + + /* + * The hash and the cipher are applied at different times and their + * requests can use the same memory space without interference + */ + skreq = (void *)(treq_ctx->tail + ctx->reqoff); + skcipher_request_set_tfm(skreq, ctx->enc); + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); + skcipher_request_set_callback(skreq, aead_request_flags(req), + req->base.complete, req->base.data); + /* + * Apply the cipher transform. The result will be in req->dst when the + * asynchronuous call terminates + */ + err = crypto_skcipher_encrypt(skreq); + + return err; +} + +static int crypto_tls_decrypt(struct aead_request *req) +{ + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); + struct tls_request_ctx *treq_ctx = aead_request_ctx(req); + unsigned int cryptlen = req->cryptlen; + unsigned int hash_size = crypto_aead_authsize(tls); + unsigned int block_size = crypto_aead_blocksize(tls); + struct skcipher_request *skreq = (void *)(treq_ctx->tail + ctx->reqoff); + struct scatterlist *tmp = treq_ctx->tmp; + struct scatterlist *src, *dst; + + u8 padding[255]; /* padding can be 0-255 bytes */ + u8 pad_size; + u16 *len_field; + u8 *ihash, *hash = treq_ctx->tail; + + int paderr = 0; + int err = -EINVAL; + int i; + struct async_op ciph_op; + + /* + * Rule out bad packets. The input packet length must be at least one + * byte more than the hash_size + */ + if (cryptlen <= hash_size || cryptlen % block_size) + goto out; + + /* + * Step 1 - Decrypt the source. Fast-forward past the associated data + * to the encrypted data. The result will be overwritten in place so + * that the decrypted data will be adjacent to the associated data. The + * last step (computing the hash) will have it's input data already + * prepared and ready to be accessed at req->src. + */ + src = scatterwalk_ffwd(tmp, req->src, req->assoclen); + dst = src; + + init_completion(&ciph_op.completion); + skcipher_request_set_tfm(skreq, ctx->enc); + skcipher_request_set_callback(skreq, aead_request_flags(req), + tls_async_op_done, &ciph_op); + skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); + err = crypto_skcipher_decrypt(skreq); + if (err == -EINPROGRESS) { + err = wait_for_completion_interruptible(&ciph_op.completion); + if (!err) + err = ciph_op.err; + } + if (err) + goto out; + + /* + * Step 2 - Verify padding + * Retrieve the last byte of the payload; this is the padding size. + */ + cryptlen -= 1; + scatterwalk_map_and_copy(&pad_size, dst, cryptlen, 1, 0); + + /* RFC recommendation for invalid padding size. */ + if (cryptlen < pad_size + hash_size) { + pad_size = 0; + paderr = -EBADMSG; + } + cryptlen -= pad_size; + scatterwalk_map_and_copy(padding, dst, cryptlen, pad_size, 0); + + /* Padding content must be equal with pad_size. We verify it all */ + for (i = 0; i < pad_size; i++) + if (padding[i] != pad_size) + paderr = -EBADMSG; + + /* + * Step 3 - Verify hash + * Align the digest result as required by the hash transform. Enough + * space was allocated in crypto_tls_init_tfm + */ + hash = (u8 *)ALIGN((unsigned long)hash + + crypto_ahash_alignmask(ctx->auth), + crypto_ahash_alignmask(ctx->auth) + 1); + /* + * Two bytes at the end of the associated data make the length field. + * It must be updated with the length of the cleartext message before + * the hash is calculated. + */ + len_field = sg_virt(req->src) + req->assoclen - 2; + cryptlen -= hash_size; + *len_field = htons(cryptlen); + + /* This is the hash from the decrypted packet. Save it for later */ + ihash = hash + hash_size; + scatterwalk_map_and_copy(ihash, dst, cryptlen, hash_size, 0); + + /* Now compute and compare our ICV with the one from the packet */ + err = crypto_tls_genicv(hash, req->src, cryptlen + req->assoclen, req); + if (!err) + err = memcmp(hash, ihash, hash_size) ? -EBADMSG : 0; + + if (req->src != req->dst) { + err = crypto_tls_copy_data(req, req->src, req->dst, cryptlen + + req->assoclen); + if (err) + goto out; + } + + /* return the first found error */ + if (paderr) + err = paderr; + +out: + aead_request_complete(req, err); + return err; +} + +static int crypto_tls_init_tfm(struct crypto_aead *tfm) +{ + struct aead_instance *inst = aead_alg_instance(tfm); + struct tls_instance_ctx *ictx = aead_instance_ctx(inst); + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_ahash *auth; + struct crypto_skcipher *enc; + struct crypto_skcipher *null; + int err; + + auth = crypto_spawn_ahash(&ictx->auth); + if (IS_ERR(auth)) + return PTR_ERR(auth); + + enc = crypto_spawn_skcipher(&ictx->enc); + err = PTR_ERR(enc); + if (IS_ERR(enc)) + goto err_free_ahash; + + null = crypto_get_default_null_skcipher2(); + err = PTR_ERR(null); + if (IS_ERR(null)) + goto err_free_skcipher; + + ctx->auth = auth; + ctx->enc = enc; + ctx->null = null; + + /* + * Allow enough space for two digests. The two digests will be compared + * during the decryption phase. One will come from the decrypted packet + * and the other will be calculated. For encryption, one digest is + * padded (up to a cipher blocksize) and chained with the payload + */ + ctx->reqoff = ALIGN(crypto_ahash_digestsize(auth) + + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1) + + max(crypto_ahash_digestsize(auth), + crypto_skcipher_blocksize(enc)); + + crypto_aead_set_reqsize(tfm, + sizeof(struct tls_request_ctx) + + ctx->reqoff + + max_t(unsigned int, + crypto_ahash_reqsize(auth) + + sizeof(struct ahash_request), + crypto_skcipher_reqsize(enc) + + sizeof(struct skcipher_request))); + + return 0; + +err_free_skcipher: + crypto_free_skcipher(enc); +err_free_ahash: + crypto_free_ahash(auth); + return err; +} + +static void crypto_tls_exit_tfm(struct crypto_aead *tfm) +{ + struct crypto_tls_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_ahash(ctx->auth); + crypto_free_skcipher(ctx->enc); + crypto_put_default_null_skcipher2(); +} + +static void crypto_tls_free(struct aead_instance *inst) +{ + struct tls_instance_ctx *ctx = aead_instance_ctx(inst); + + crypto_drop_skcipher(&ctx->enc); + crypto_drop_ahash(&ctx->auth); + kfree(inst); +} + +static int crypto_tls_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct crypto_attr_type *algt; + struct aead_instance *inst; + struct hash_alg_common *auth; + struct crypto_alg *auth_base; + struct skcipher_alg *enc; + struct tls_instance_ctx *ctx; + const char *enc_name; + int err; + + algt = crypto_get_attr_type(tb); + if (IS_ERR(algt)) + return PTR_ERR(algt); + + if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) + return -EINVAL; + + auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, + CRYPTO_ALG_TYPE_AHASH_MASK | + crypto_requires_sync(algt->type, algt->mask)); + if (IS_ERR(auth)) + return PTR_ERR(auth); + + auth_base = &auth->base; + + enc_name = crypto_attr_alg_name(tb[2]); + err = PTR_ERR(enc_name); + if (IS_ERR(enc_name)) + goto out_put_auth; + + inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); + err = -ENOMEM; + if (!inst) + goto out_put_auth; + + ctx = aead_instance_ctx(inst); + + err = crypto_init_ahash_spawn(&ctx->auth, auth, + aead_crypto_instance(inst)); + if (err) + goto err_free_inst; + + crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst)); + err = crypto_grab_skcipher(&ctx->enc, enc_name, 0, + crypto_requires_sync(algt->type, + algt->mask)); + if (err) + goto err_drop_auth; + + enc = crypto_spawn_skcipher_alg(&ctx->enc); + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "tls10(%s,%s)", auth_base->cra_name, + enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME) + goto err_drop_enc; + + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "tls10(%s,%s)", auth_base->cra_driver_name, + enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + goto err_drop_enc; + + inst->alg.base.cra_flags = (auth_base->cra_flags | + enc->base.cra_flags) & CRYPTO_ALG_ASYNC; + inst->alg.base.cra_priority = enc->base.cra_priority * 10 + + auth_base->cra_priority; + inst->alg.base.cra_blocksize = enc->base.cra_blocksize; + inst->alg.base.cra_alignmask = auth_base->cra_alignmask | + enc->base.cra_alignmask; + inst->alg.base.cra_ctxsize = sizeof(struct crypto_tls_ctx); + + inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc); + inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc); + inst->alg.maxauthsize = auth->digestsize; + + inst->alg.init = crypto_tls_init_tfm; + inst->alg.exit = crypto_tls_exit_tfm; + + inst->alg.setkey = crypto_tls_setkey; + inst->alg.encrypt = crypto_tls_encrypt; + inst->alg.decrypt = crypto_tls_decrypt; + + inst->free = crypto_tls_free; + + err = aead_register_instance(tmpl, inst); + if (err) + goto err_drop_enc; + +out: + crypto_mod_put(auth_base); + return err; + +err_drop_enc: + crypto_drop_skcipher(&ctx->enc); +err_drop_auth: + crypto_drop_ahash(&ctx->auth); +err_free_inst: + kfree(inst); +out_put_auth: + goto out; +} + +static struct crypto_template crypto_tls_tmpl = { + .name = "tls10", + .create = crypto_tls_create, + .module = THIS_MODULE, +}; + +static int __init crypto_tls_module_init(void) +{ + return crypto_register_template(&crypto_tls_tmpl); +} + +static void __exit crypto_tls_module_exit(void) +{ + crypto_unregister_template(&crypto_tls_tmpl); +} + +module_init(crypto_tls_module_init); +module_exit(crypto_tls_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TLS 1.0 record encryption"); --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -1,6 +1,11 @@ +config CRYPTO_DEV_FSL_CAAM_COMMON + tristate + config CRYPTO_DEV_FSL_CAAM - tristate "Freescale CAAM-Multicore driver backend" + tristate "Freescale CAAM-Multicore platform driver backend" depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE + select CRYPTO_DEV_FSL_CAAM_COMMON + select SOC_BUS help Enables the driver module for Freescale's Cryptographic Accelerator and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). @@ -11,9 +16,16 @@ config CRYPTO_DEV_FSL_CAAM To compile this driver as a module, choose M here: the module will be called caam. +if CRYPTO_DEV_FSL_CAAM + +config CRYPTO_DEV_FSL_CAAM_DEBUG + bool "Enable debug output in CAAM driver" + help + Selecting this will enable printing of various debug + information in the CAAM driver. + config CRYPTO_DEV_FSL_CAAM_JR tristate "Freescale CAAM Job Ring driver backend" - depends on CRYPTO_DEV_FSL_CAAM default y help Enables the driver module for Job Rings which are part of @@ -24,9 +36,10 @@ config CRYPTO_DEV_FSL_CAAM_JR To compile this driver as a module, choose M here: the module will be called caam_jr. +if CRYPTO_DEV_FSL_CAAM_JR + config CRYPTO_DEV_FSL_CAAM_RINGSIZE int "Job Ring size" - depends on CRYPTO_DEV_FSL_CAAM_JR range 2 9 default "9" help @@ -44,7 +57,6 @@ config CRYPTO_DEV_FSL_CAAM_RINGSIZE config CRYPTO_DEV_FSL_CAAM_INTC bool "Job Ring interrupt coalescing" - depends on CRYPTO_DEV_FSL_CAAM_JR help Enable the Job Ring's interrupt coalescing feature. @@ -74,7 +86,6 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THL config CRYPTO_DEV_FSL_CAAM_CRYPTO_API tristate "Register algorithm implementations with the Crypto API" - depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y select CRYPTO_AEAD select CRYPTO_AUTHENC @@ -87,9 +98,25 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API To compile this as a module, choose M here: the module will be called caamalg. +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI + tristate "Queue Interface as Crypto API backend" + depends on FSL_SDK_DPA && NET + default y + select CRYPTO_AUTHENC + select CRYPTO_BLKCIPHER + help + Selecting this will use CAAM Queue Interface (QI) for sending + & receiving crypto jobs to/from CAAM. This gives better performance + than job ring interface when the number of cores are more than the + number of job rings assigned to the kernel. The number of portals + assigned to the kernel should also be more than the number of + job rings. + + To compile this as a module, choose M here: the module + will be called caamalg_qi. + config CRYPTO_DEV_FSL_CAAM_AHASH_API tristate "Register hash algorithm implementations with Crypto API" - depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y select CRYPTO_HASH help @@ -101,7 +128,6 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API config CRYPTO_DEV_FSL_CAAM_PKC_API tristate "Register public key cryptography implementations with Crypto API" - depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y select CRYPTO_RSA help @@ -113,7 +139,6 @@ config CRYPTO_DEV_FSL_CAAM_PKC_API config CRYPTO_DEV_FSL_CAAM_RNG_API tristate "Register caam device for hwrng API" - depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR default y select CRYPTO_RNG select HW_RANDOM @@ -124,13 +149,31 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API To compile this as a module, choose M here: the module will be called caamrng. -config CRYPTO_DEV_FSL_CAAM_IMX - def_bool SOC_IMX6 || SOC_IMX7D - depends on CRYPTO_DEV_FSL_CAAM +endif # CRYPTO_DEV_FSL_CAAM_JR -config CRYPTO_DEV_FSL_CAAM_DEBUG - bool "Enable debug output in CAAM driver" - depends on CRYPTO_DEV_FSL_CAAM - help - Selecting this will enable printing of various debug - information in the CAAM driver. +endif # CRYPTO_DEV_FSL_CAAM + +config CRYPTO_DEV_FSL_DPAA2_CAAM + tristate "QorIQ DPAA2 CAAM (DPSECI) driver" + depends on FSL_MC_DPIO + select CRYPTO_DEV_FSL_CAAM_COMMON + select CRYPTO_BLKCIPHER + select CRYPTO_AUTHENC + select CRYPTO_AEAD + select CRYPTO_HASH + ---help--- + CAAM driver for QorIQ Data Path Acceleration Architecture 2. + It handles DPSECI DPAA2 objects that sit on the Management Complex + (MC) fsl-mc bus. + + To compile this as a module, choose M here: the module + will be called dpaa2_caam. + +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \ + CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \ + CRYPTO_DEV_FSL_DPAA2_CAAM) + +config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC + def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \ + CRYPTO_DEV_FSL_DPAA2_CAAM) --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -5,13 +5,27 @@ ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG ccflags-y := -DDEBUG endif +ccflags-y += -DVERSION=\"\" + +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o caam-objs := ctrl.o -caam_jr-objs := jr.o key_gen.o error.o +caam_jr-objs := jr.o key_gen.o caam_pkc-y := caampkc.o pkc_desc.o +ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) + ccflags-y += -DCONFIG_CAAM_QI + caam-objs += qi.o +endif + +obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o + +dpaa2_caam-y := caamalg_qi2.o dpseci.o --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -2,6 +2,7 @@ * caam - Freescale FSL CAAM support for crypto API * * Copyright 2008-2011 Freescale Semiconductor, Inc. + * Copyright 2016 NXP * * Based on talitos crypto API driver. * @@ -53,6 +54,7 @@ #include "error.h" #include "sg_sw_sec4.h" #include "key_gen.h" +#include "caamalg_desc.h" /* * crypto alg @@ -62,8 +64,6 @@ #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ CTR_RFC3686_NONCE_SIZE + \ SHA512_DIGEST_SIZE * 2) -/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ -#define CAAM_MAX_IV_LENGTH 16 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ @@ -71,37 +71,6 @@ #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ CAAM_CMD_SZ * 5) -/* length of descriptors text */ -#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) -#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) -#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) -#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 10 * CAAM_CMD_SZ) - -/* Note: Nonce is counted in enckeylen */ -#define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) - -#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) -#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) -#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) - -#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) -#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) -#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) - -#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) -#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) -#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) - -#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) -#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) -#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) - -#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) -#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ - 20 * CAAM_CMD_SZ) -#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ - 15 * CAAM_CMD_SZ) - #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) @@ -112,47 +81,11 @@ #define debug(format, arg...) #endif -#ifdef DEBUG -#include - -static void dbg_dump_sg(const char *level, const char *prefix_str, - int prefix_type, int rowsize, int groupsize, - struct scatterlist *sg, size_t tlen, bool ascii, - bool may_sleep) -{ - struct scatterlist *it; - void *it_page; - size_t len; - void *buf; - - for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) { - /* - * make sure the scatterlist's page - * has a valid virtual memory mapping - */ - it_page = kmap_atomic(sg_page(it)); - if (unlikely(!it_page)) { - printk(KERN_ERR "dbg_dump_sg: kmap failed\n"); - return; - } - - buf = it_page + it->offset; - len = min_t(size_t, tlen, it->length); - print_hex_dump(level, prefix_str, prefix_type, rowsize, - groupsize, buf, len, ascii); - tlen -= len; - - kunmap_atomic(it_page); - } -} -#endif - static struct list_head alg_list; struct caam_alg_entry { int class1_alg_type; int class2_alg_type; - int alg_op; bool rfc3686; bool geniv; }; @@ -163,302 +96,70 @@ struct caam_aead_alg { bool registered; }; -/* Set DK bit in class 1 operation if shared */ -static inline void append_dec_op1(u32 *desc, u32 type) -{ - u32 *jump_cmd, *uncond_jump_cmd; - - /* DK bit is valid only for AES */ - if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT); - return; - } - - jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT); - uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); - set_jump_tgt_here(desc, jump_cmd); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT | OP_ALG_AAI_DK); - set_jump_tgt_here(desc, uncond_jump_cmd); -} - -/* - * For aead functions, read payload and write payload, - * both of which are specified in req->src and req->dst - */ -static inline void aead_append_src_dst(u32 *desc, u32 msg_type) -{ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | - KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); -} - -/* - * For ablkcipher encrypt and decrypt, read from req->src and - * write to req->dst - */ -static inline void ablkcipher_append_src_dst(u32 *desc) -{ - append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | - KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); -} - /* * per-session context */ struct caam_ctx { - struct device *jrdev; u32 sh_desc_enc[DESC_MAX_USED_LEN]; u32 sh_desc_dec[DESC_MAX_USED_LEN]; u32 sh_desc_givenc[DESC_MAX_USED_LEN]; + u8 key[CAAM_MAX_KEY_SIZE]; dma_addr_t sh_desc_enc_dma; dma_addr_t sh_desc_dec_dma; dma_addr_t sh_desc_givenc_dma; - u32 class1_alg_type; - u32 class2_alg_type; - u32 alg_op; - u8 key[CAAM_MAX_KEY_SIZE]; dma_addr_t key_dma; - unsigned int enckeylen; - unsigned int split_key_len; - unsigned int split_key_pad_len; + struct device *jrdev; + struct alginfo adata; + struct alginfo cdata; unsigned int authsize; }; -static void append_key_aead(u32 *desc, struct caam_ctx *ctx, - int keys_fit_inline, bool is_rfc3686) -{ - u32 *nonce; - unsigned int enckeylen = ctx->enckeylen; - - /* - * RFC3686 specific: - * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} - * | enckeylen = encryption key size + nonce size - */ - if (is_rfc3686) - enckeylen -= CTR_RFC3686_NONCE_SIZE; - - if (keys_fit_inline) { - append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, - ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - append_key_as_imm(desc, (void *)ctx->key + - ctx->split_key_pad_len, enckeylen, - enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - } else { - append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - append_key(desc, ctx->key_dma + ctx->split_key_pad_len, - enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - } - - /* Load Counter into CONTEXT1 reg */ - if (is_rfc3686) { - nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + - enckeylen); - append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, - LDST_CLASS_IND_CCB | - LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); - append_move(desc, - MOVE_SRC_OUTFIFO | - MOVE_DEST_CLASS1CTX | - (16 << MOVE_OFFSET_SHIFT) | - (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); - } -} - -static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, - int keys_fit_inline, bool is_rfc3686) -{ - u32 *key_jump_cmd; - - /* Note: Context registers are saved. */ - init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); - - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); - - set_jump_tgt_here(desc, key_jump_cmd); -} - static int aead_null_set_sh_desc(struct crypto_aead *aead) { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - bool keys_fit_inline = false; - u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - + ctx->adata.keylen_pad; /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + - ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; + if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { + ctx->adata.key_inline = true; + ctx->adata.key_virt = ctx->key; + } else { + ctx->adata.key_inline = false; + ctx->adata.key_dma = ctx->key_dma; + } /* aead_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, - ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - else - append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - set_jump_tgt_here(desc, key_jump_cmd); - - /* assoclen + cryptlen = seqinlen */ - append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Prepare to read and write cryptlen + assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - - /* - * MOVE_LEN opcode is not available in all SEC HW revisions, - * thus need to do some magic, i.e. self-patch the descriptor - * buffer. - */ - read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | - MOVE_DEST_MATH3 | - (0x6 << MOVE_LEN_SHIFT)); - write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | - MOVE_DEST_DESCBUF | - MOVE_WAITCOMP | - (0x8 << MOVE_LEN_SHIFT)); - - /* Class 2 operation */ - append_operation(desc, ctx->class2_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* Read and write cryptlen bytes */ - aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); - - set_move_tgt_here(desc, read_move_cmd); - set_move_tgt_here(desc, write_move_cmd); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | - MOVE_AUX_LS); - - /* Write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | - LDST_SRCDST_BYTE_CONTEXT); - - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null enc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - keys_fit_inline = false; - if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + - ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - desc = ctx->sh_desc_dec; + if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { + ctx->adata.key_inline = true; + ctx->adata.key_virt = ctx->key; + } else { + ctx->adata.key_inline = false; + ctx->adata.key_dma = ctx->key_dma; + } /* aead_decrypt shared descriptor */ - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, - ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - else - append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); - set_jump_tgt_here(desc, key_jump_cmd); - - /* Class 2 operation */ - append_operation(desc, ctx->class2_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - - /* assoclen + cryptlen = seqoutlen */ - append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* Prepare to read and write cryptlen + assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); - - /* - * MOVE_LEN opcode is not available in all SEC HW revisions, - * thus need to do some magic, i.e. self-patch the descriptor - * buffer. - */ - read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | - MOVE_DEST_MATH2 | - (0x6 << MOVE_LEN_SHIFT)); - write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | - MOVE_DEST_DESCBUF | - MOVE_WAITCOMP | - (0x8 << MOVE_LEN_SHIFT)); - - /* Read and write cryptlen bytes */ - aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); - - /* - * Insert a NOP here, since we need at least 4 instructions between - * code patching the descriptor buffer and the location being patched. - */ - jump_cmd = append_jump(desc, JUMP_TEST_ALL); - set_jump_tgt_here(desc, jump_cmd); - - set_move_tgt_here(desc, read_move_cmd); - set_move_tgt_here(desc, write_move_cmd); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | - MOVE_AUX_LS); - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); - - /* Load ICV */ - append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | - FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); - - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null dec shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + desc = ctx->sh_desc_dec; + cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, + ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), DMA_TO_DEVICE); return 0; } @@ -470,12 +171,12 @@ static int aead_set_sh_desc(struct crypt unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - bool keys_fit_inline; - u32 geniv, moveiv; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 ctx1_iv_off = 0; - u32 *desc; - u32 *wait_cmd; - const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + u32 *desc, *nonce = NULL; + u32 inl_mask; + unsigned int data_len[2]; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; @@ -483,7 +184,7 @@ static int aead_set_sh_desc(struct crypt return 0; /* NULL encryption / decryption */ - if (!ctx->enckeylen) + if (!ctx->cdata.keylen) return aead_null_set_sh_desc(aead); /* @@ -498,8 +199,14 @@ static int aead_set_sh_desc(struct crypt * RFC3686 specific: * CONTEXT1[255:128] = {NONCE, IV, COUNTER} */ - if (is_rfc3686) + if (is_rfc3686) { ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); + } + + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; if (alg->caam.geniv) goto skip_enc; @@ -508,146 +215,64 @@ static int aead_set_sh_desc(struct crypt * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - keys_fit_inline = false; - if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen + - (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= - CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - /* aead_encrypt shared descriptor */ - desc = ctx->sh_desc_enc; - - /* Note: Context registers are saved. */ - init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); - - /* Class 2 operation */ - append_operation(desc, ctx->class2_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* Read and write assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + if (desc_inline_query(DESC_AEAD_ENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; - /* Skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - FIFOLDST_VLF); + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; - /* Load Counter into CONTEXT1 reg */ - if (is_rfc3686) - append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << - LDST_OFFSET_SHIFT)); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* Read and write cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); - - /* Write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | - LDST_SRCDST_BYTE_CONTEXT); + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + /* aead_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, + false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); skip_enc: /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - keys_fit_inline = false; - if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen + - (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= - CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - /* aead_decrypt shared descriptor */ - desc = ctx->sh_desc_dec; - - /* Note: Context registers are saved. */ - init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); - - /* Class 2 operation */ - append_operation(desc, ctx->class2_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + if (desc_inline_query(DESC_AEAD_DEC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; - /* Read and write assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - if (alg->caam.geniv) - append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; else - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + ctx->adata.key_dma = ctx->key_dma; - /* Skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - - if (alg->caam.geniv) { - append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - (ctx1_iv_off << LDST_OFFSET_SHIFT)); - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | - (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); - } - - /* Load Counter into CONTEXT1 reg */ - if (is_rfc3686) - append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << - LDST_OFFSET_SHIFT)); - - /* Choose operation */ - if (ctr_mode) - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; else - append_dec_op1(desc, ctx->class1_alg_type); + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; - /* Read and write cryptlen bytes */ - append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - aead_append_src_dst(desc, FIFOLD_TYPE_MSG); - - /* Load ICV */ - append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | - FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + /* aead_decrypt shared descriptor */ + desc = ctx->sh_desc_dec; + cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, alg->caam.geniv, is_rfc3686, + nonce, ctx1_iv_off, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), DMA_TO_DEVICE); if (!alg->caam.geniv) goto skip_givenc; @@ -656,115 +281,32 @@ skip_enc: * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - keys_fit_inline = false; - if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + - ctx->split_key_pad_len + ctx->enckeylen + - (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= - CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; - - /* aead_givencrypt shared descriptor */ - desc = ctx->sh_desc_enc; - - /* Note: Context registers are saved. */ - init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); - - if (is_rfc3686) - goto copy_iv; - - /* Generate IV */ - geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | - NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | - NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); - append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | - LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - append_move(desc, MOVE_WAITCOMP | - MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | - (ctx1_iv_off << MOVE_OFFSET_SHIFT) | - (ivsize << MOVE_LEN_SHIFT)); - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); - -copy_iv: - /* Copy IV to class 1 context */ - append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | - (ctx1_iv_off << MOVE_OFFSET_SHIFT) | - (ivsize << MOVE_LEN_SHIFT)); - - /* Return to encryption */ - append_operation(desc, ctx->class2_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* Read and write assoclen bytes */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - - /* ivsize + cryptlen = seqoutlen - authsize */ - append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); - - /* Skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - - /* read assoc before reading payload */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | - KEY_VLF); - - /* Copy iv from outfifo to class 2 fifo */ - moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | - NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); - append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | - LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); - append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | - LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); - - /* Load Counter into CONTEXT1 reg */ - if (is_rfc3686) - append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << - LDST_OFFSET_SHIFT)); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* Will write ivsize + cryptlen */ - append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Not need to reload iv */ - append_seq_fifo_load(desc, ivsize, - FIFOLD_CLASS_SKIP); + if (desc_inline_query(DESC_AEAD_GIVENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; - /* Will read cryptlen */ - append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; - /* - * Wait for IV transfer (ofifo -> class2) to finish before starting - * ciphertext transfer (ofifo -> external memory). - */ - wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP); - set_jump_tgt_here(desc, wait_cmd); + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | - FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); - - /* Write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | - LDST_SRCDST_BYTE_CONTEXT); + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + /* aead_givencrypt shared descriptor */ + desc = ctx->sh_desc_enc; + cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, + ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); skip_givenc: return 0; @@ -785,12 +327,12 @@ static int gcm_set_sh_desc(struct crypto { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - bool keys_fit_inline = false; - u32 *key_jump_cmd, *zero_payload_jump_cmd, - *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; + unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - + ctx->cdata.keylen; - if (!ctx->enckeylen || !ctx->authsize) + if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* @@ -798,175 +340,35 @@ static int gcm_set_sh_desc(struct crypto * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ - if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + - ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; + if (rem_bytes >= DESC_GCM_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } desc = ctx->sh_desc_enc; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* skip key loading if they are loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD | JUMP_COND_SELF); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* if assoclen + cryptlen is ZERO, skip to ICV write */ - append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); - - /* if assoclen is ZERO, skip reading the assoc data */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); - - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - - /* skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - - /* cryptlen = seqinlen - assoclen */ - append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); - - /* if cryptlen is ZERO jump to zero-payload commands */ - zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); - - /* read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - set_jump_tgt_here(desc, zero_assoc_jump_cmd1); - - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* write encrypted data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); - - /* read payload data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); - - /* jump the zero-payload commands */ - append_jump(desc, JUMP_TEST_ALL | 2); - - /* zero-payload commands */ - set_jump_tgt_here(desc, zero_payload_jump_cmd); - - /* read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); - - /* There is no input data */ - set_jump_tgt_here(desc, zero_assoc_jump_cmd2); - - /* write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT); - - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - keys_fit_inline = false; - if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + - ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; + if (rem_bytes >= DESC_GCM_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } desc = ctx->sh_desc_dec; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* skip key loading if they are loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | - JUMP_TEST_ALL | JUMP_COND_SHRD | - JUMP_COND_SELF); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - - /* if assoclen is ZERO, skip reading the assoc data */ - append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); - zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); - - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - - /* skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - - /* read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - - set_jump_tgt_here(desc, zero_assoc_jump_cmd1); - - /* cryptlen = seqoutlen - assoclen */ - append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* jump to zero-payload command if cryptlen is zero */ - zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | - JUMP_COND_MATH_Z); - - append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* store encrypted data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); - - /* read payload data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); - - /* zero-payload command */ - set_jump_tgt_here(desc, zero_payload_jump_cmd); - - /* read ICV */ - append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); - - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), DMA_TO_DEVICE); return 0; } @@ -985,11 +387,12 @@ static int rfc4106_set_sh_desc(struct cr { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - bool keys_fit_inline = false; - u32 *key_jump_cmd; + unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - + ctx->cdata.keylen; - if (!ctx->enckeylen || !ctx->authsize) + if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* @@ -997,148 +400,37 @@ static int rfc4106_set_sh_desc(struct cr * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ - if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + - ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; + if (rem_bytes >= DESC_RFC4106_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } desc = ctx->sh_desc_enc; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip key loading if it is loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - - /* Read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - - /* Skip IV */ - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); - - /* Will read cryptlen bytes */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); - - /* Skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - - /* cryptlen = seqoutlen - assoclen */ - append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); - - /* Write encrypted data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); - - /* Read payload data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); - - /* Write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT); - - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - keys_fit_inline = false; - if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + - ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; + if (rem_bytes >= DESC_RFC4106_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } desc = ctx->sh_desc_dec; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip key loading if it is loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | - JUMP_TEST_ALL | JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - - append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); - append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); - - /* Read assoc data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); - - /* Skip IV */ - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); - - /* Will read cryptlen bytes */ - append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); - - /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); - - /* Skip assoc data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); - - /* Will write cryptlen bytes */ - append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* Store payload data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); - - /* Read encrypted data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | - FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); - - /* Read ICV */ - append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); - - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), DMA_TO_DEVICE); return 0; } @@ -1158,12 +450,12 @@ static int rfc4543_set_sh_desc(struct cr { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - bool keys_fit_inline = false; - u32 *key_jump_cmd; - u32 *read_move_cmd, *write_move_cmd; + unsigned int ivsize = crypto_aead_ivsize(aead); u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - + ctx->cdata.keylen; - if (!ctx->enckeylen || !ctx->authsize) + if (!ctx->cdata.keylen || !ctx->authsize) return 0; /* @@ -1171,151 +463,37 @@ static int rfc4543_set_sh_desc(struct cr * Job Descriptor and Shared Descriptor * must fit into the 64-word Descriptor h/w Buffer */ - if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + - ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; + if (rem_bytes >= DESC_RFC4543_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } desc = ctx->sh_desc_enc; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip key loading if it is loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* assoclen + cryptlen = seqinlen */ - append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* - * MOVE_LEN opcode is not available in all SEC HW revisions, - * thus need to do some magic, i.e. self-patch the descriptor - * buffer. - */ - read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | - (0x6 << MOVE_LEN_SHIFT)); - write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | - (0x8 << MOVE_LEN_SHIFT)); - - /* Will read assoclen + cryptlen bytes */ - append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Will write assoclen + cryptlen bytes */ - append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Read and write assoclen + cryptlen bytes */ - aead_append_src_dst(desc, FIFOLD_TYPE_AAD); - - set_move_tgt_here(desc, read_move_cmd); - set_move_tgt_here(desc, write_move_cmd); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - /* Move payload data to OFIFO */ - append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); - - /* Write ICV */ - append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT); - - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ - keys_fit_inline = false; - if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + - ctx->enckeylen <= CAAM_DESC_BYTES_MAX) - keys_fit_inline = true; + if (rem_bytes >= DESC_RFC4543_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } desc = ctx->sh_desc_dec; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip key loading if it is loaded due to sharing */ - key_jump_cmd = append_jump(desc, JUMP_JSL | - JUMP_TEST_ALL | JUMP_COND_SHRD); - if (keys_fit_inline) - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - else - append_key(desc, ctx->key_dma, ctx->enckeylen, - CLASS_1 | KEY_DEST_CLASS_REG); - set_jump_tgt_here(desc, key_jump_cmd); - - /* Class 1 operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - - /* assoclen + cryptlen = seqoutlen */ - append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* - * MOVE_LEN opcode is not available in all SEC HW revisions, - * thus need to do some magic, i.e. self-patch the descriptor - * buffer. - */ - read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | - (0x6 << MOVE_LEN_SHIFT)); - write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | - (0x8 << MOVE_LEN_SHIFT)); - - /* Will read assoclen + cryptlen bytes */ - append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* Will write assoclen + cryptlen bytes */ - append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); - - /* Store payload data */ - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); - - /* In-snoop assoclen + cryptlen data */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | - FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); - - set_move_tgt_here(desc, read_move_cmd); - set_move_tgt_here(desc, write_move_cmd); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - /* Move payload data to OFIFO */ - append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); - - /* Read ICV */ - append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | - FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); - - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + false); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), DMA_TO_DEVICE); return 0; } @@ -1331,74 +509,67 @@ static int rfc4543_setauthsize(struct cr return 0; } -static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, - u32 authkeylen) -{ - return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, - ctx->split_key_pad_len, key_in, authkeylen, - ctx->alg_op); -} - static int aead_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { - /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ - static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); struct crypto_authenc_keys keys; int ret = 0; if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; - /* Pick class 2 key length from algorithm submask */ - ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> - OP_ALG_ALGSEL_SHIFT] * 2; - ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); - - if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) - goto badkey; - #ifdef DEBUG printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", - ctx->split_key_len, ctx->split_key_pad_len); print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif - ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, DMA_TO_DEVICE); + goto skip_split_key; + } + + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, + keys.authkeylen, CAAM_MAX_KEY_SIZE - + keys.enckeylen); if (ret) { goto badkey; } /* postpend encryption key to auth split key */ - memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); - - ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + - keys.enckeylen, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); - return -ENOMEM; - } + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, DMA_TO_DEVICE); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->split_key_pad_len + keys.enckeylen, 1); + ctx->adata.keylen_pad + keys.enckeylen, 1); #endif - ctx->enckeylen = keys.enckeylen; - - ret = aead_set_sh_desc(aead); - if (ret) { - dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + - keys.enckeylen, DMA_TO_DEVICE); - } - - return ret; +skip_split_key: + ctx->cdata.keylen = keys.enckeylen; + return aead_set_sh_desc(aead); badkey: crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; @@ -1409,7 +580,6 @@ static int gcm_setkey(struct crypto_aead { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - int ret = 0; #ifdef DEBUG print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", @@ -1417,21 +587,10 @@ static int gcm_setkey(struct crypto_aead #endif memcpy(ctx->key, key, keylen); - ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); - return -ENOMEM; - } - ctx->enckeylen = keylen; - - ret = gcm_set_sh_desc(aead); - if (ret) { - dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, - DMA_TO_DEVICE); - } + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.keylen = keylen; - return ret; + return gcm_set_sh_desc(aead); } static int rfc4106_setkey(struct crypto_aead *aead, @@ -1439,7 +598,6 @@ static int rfc4106_setkey(struct crypto_ { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - int ret = 0; if (keylen < 4) return -EINVAL; @@ -1455,22 +613,10 @@ static int rfc4106_setkey(struct crypto_ * The last four bytes of the key material are used as the salt value * in the nonce. Update the AES key length. */ - ctx->enckeylen = keylen - 4; - - ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); - return -ENOMEM; - } - - ret = rfc4106_set_sh_desc(aead); - if (ret) { - dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, - DMA_TO_DEVICE); - } - - return ret; + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, + DMA_TO_DEVICE); + return rfc4106_set_sh_desc(aead); } static int rfc4543_setkey(struct crypto_aead *aead, @@ -1478,7 +624,6 @@ static int rfc4543_setkey(struct crypto_ { struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - int ret = 0; if (keylen < 4) return -EINVAL; @@ -1494,43 +639,28 @@ static int rfc4543_setkey(struct crypto_ * The last four bytes of the key material are used as the salt value * in the nonce. Update the AES key length. */ - ctx->enckeylen = keylen - 4; - - ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); - return -ENOMEM; - } - - ret = rfc4543_set_sh_desc(aead); - if (ret) { - dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, - DMA_TO_DEVICE); - } - - return ret; + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, + DMA_TO_DEVICE); + return rfc4543_set_sh_desc(aead); } static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, const u8 *key, unsigned int keylen) { struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); - struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); const char *alg_name = crypto_tfm_alg_name(tfm); struct device *jrdev = ctx->jrdev; - int ret = 0; - u32 *key_jump_cmd; + unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); u32 *desc; - u8 *nonce; - u32 geniv; u32 ctx1_iv_off = 0; - const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = (ctr_mode && (strstr(alg_name, "rfc3686") != NULL)); + memcpy(ctx->key, key, keylen); #ifdef DEBUG print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); @@ -1553,215 +683,33 @@ static int ablkcipher_setkey(struct cryp keylen -= CTR_RFC3686_NONCE_SIZE; } - memcpy(ctx->key, key, keylen); - ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); - return -ENOMEM; - } - ctx->enckeylen = keylen; + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_inline = true; /* ablkcipher_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - /* Load class1 key only */ - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | - KEY_DEST_CLASS_REG); + cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, + ctx1_iv_off); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); - /* Load nonce into CONTEXT1 reg */ - if (is_rfc3686) { - nonce = (u8 *)key + keylen; - append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, - LDST_CLASS_IND_CCB | - LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); - append_move(desc, MOVE_WAITCOMP | - MOVE_SRC_OUTFIFO | - MOVE_DEST_CLASS1CTX | - (16 << MOVE_OFFSET_SHIFT) | - (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); - } - - set_jump_tgt_here(desc, key_jump_cmd); - - /* Load iv */ - append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); - - /* Load counter into CONTEXT1 reg */ - if (is_rfc3686) - append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << - LDST_OFFSET_SHIFT)); - - /* Load operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* Perform operation */ - ablkcipher_append_src_dst(desc); - - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ablkcipher enc shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif /* ablkcipher_decrypt shared descriptor */ desc = ctx->sh_desc_dec; + cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, + ctx1_iv_off); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), DMA_TO_DEVICE); - init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - /* Load class1 key only */ - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | - KEY_DEST_CLASS_REG); - - /* Load nonce into CONTEXT1 reg */ - if (is_rfc3686) { - nonce = (u8 *)key + keylen; - append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, - LDST_CLASS_IND_CCB | - LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); - append_move(desc, MOVE_WAITCOMP | - MOVE_SRC_OUTFIFO | - MOVE_DEST_CLASS1CTX | - (16 << MOVE_OFFSET_SHIFT) | - (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); - } - - set_jump_tgt_here(desc, key_jump_cmd); - - /* load IV */ - append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); - - /* Load counter into CONTEXT1 reg */ - if (is_rfc3686) - append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << - LDST_OFFSET_SHIFT)); - - /* Choose operation */ - if (ctr_mode) - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); - else - append_dec_op1(desc, ctx->class1_alg_type); - - /* Perform operation */ - ablkcipher_append_src_dst(desc); - - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } - -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ablkcipher dec shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif /* ablkcipher_givencrypt shared descriptor */ desc = ctx->sh_desc_givenc; + cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686, + ctx1_iv_off); + dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma, + desc_bytes(desc), DMA_TO_DEVICE); - init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - /* Load class1 key only */ - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | - KEY_DEST_CLASS_REG); - - /* Load Nonce into CONTEXT1 reg */ - if (is_rfc3686) { - nonce = (u8 *)key + keylen; - append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, - LDST_CLASS_IND_CCB | - LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); - append_move(desc, MOVE_WAITCOMP | - MOVE_SRC_OUTFIFO | - MOVE_DEST_CLASS1CTX | - (16 << MOVE_OFFSET_SHIFT) | - (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); - } - set_jump_tgt_here(desc, key_jump_cmd); - - /* Generate IV */ - geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | - NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | - NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); - append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | - LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); - append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); - append_move(desc, MOVE_WAITCOMP | - MOVE_SRC_INFIFO | - MOVE_DEST_CLASS1CTX | - (crt->ivsize << MOVE_LEN_SHIFT) | - (ctx1_iv_off << MOVE_OFFSET_SHIFT)); - append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); - - /* Copy generated IV to memory */ - append_seq_store(desc, crt->ivsize, - LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | - (ctx1_iv_off << LDST_OFFSET_SHIFT)); - - /* Load Counter into CONTEXT1 reg */ - if (is_rfc3686) - append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | - LDST_SRCDST_BYTE_CONTEXT | - ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << - LDST_OFFSET_SHIFT)); - - if (ctx1_iv_off) - append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | - (1 << JUMP_OFFSET_SHIFT)); - - /* Load operation */ - append_operation(desc, ctx->class1_alg_type | - OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); - - /* Perform operation */ - ablkcipher_append_src_dst(desc); - - ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif - - return ret; + return 0; } static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, @@ -1769,8 +717,7 @@ static int xts_ablkcipher_setkey(struct { struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *jrdev = ctx->jrdev; - u32 *key_jump_cmd, *desc; - __be64 sector_size = cpu_to_be64(512); + u32 *desc; if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { crypto_ablkcipher_set_flags(ablkcipher, @@ -1780,126 +727,38 @@ static int xts_ablkcipher_setkey(struct } memcpy(ctx->key, key, keylen); - ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); - return -ENOMEM; - } - ctx->enckeylen = keylen; + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_inline = true; /* xts_ablkcipher_encrypt shared descriptor */ desc = ctx->sh_desc_enc; - init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - /* Load class1 keys only */ - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - - /* Load sector size with index 40 bytes (0x28) */ - append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); - append_data(desc, (void *)§or_size, 8); - - set_jump_tgt_here(desc, key_jump_cmd); - - /* - * create sequence for loading the sector index - * Upper 8B of IV - will be used as sector index - * Lower 8B of IV - will be discarded - */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); - - /* Load operation */ - append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | - OP_ALG_ENCRYPT); - - /* Perform operation */ - ablkcipher_append_src_dst(desc); - - ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); + dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, + desc_bytes(desc), DMA_TO_DEVICE); /* xts_ablkcipher_decrypt shared descriptor */ desc = ctx->sh_desc_dec; - - init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - /* Load class1 key only */ - append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, - ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - - /* Load sector size with index 40 bytes (0x28) */ - append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); - append_data(desc, (void *)§or_size, 8); - - set_jump_tgt_here(desc, key_jump_cmd); - - /* - * create sequence for loading the sector index - * Upper 8B of IV - will be used as sector index - * Lower 8B of IV - will be discarded - */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); - append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); - - /* Load operation */ - append_dec_op1(desc, ctx->class1_alg_type); - - /* Perform operation */ - ablkcipher_append_src_dst(desc); - - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { - dma_unmap_single(jrdev, ctx->sh_desc_enc_dma, - desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); + dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, + desc_bytes(desc), DMA_TO_DEVICE); return 0; } /* * aead_edesc - s/w-extended aead descriptor - * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist - * @src_nents: number of segments in input scatterlist - * @dst_nents: number of segments in output scatterlist - * @iv_dma: dma address of iv for checking continuity and link table - * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) + * @src_nents: number of segments in input s/w scatterlist + * @dst_nents: number of segments in output s/w scatterlist * @sec4_sg_bytes: length of dma mapped sec4_sg space * @sec4_sg_dma: bus physical mapped address of h/w link table + * @sec4_sg: pointer to h/w link table * @hw_desc: the h/w job descriptor followed by any referenced link tables */ struct aead_edesc { - int assoc_nents; int src_nents; int dst_nents; - dma_addr_t iv_dma; int sec4_sg_bytes; dma_addr_t sec4_sg_dma; struct sec4_sg_entry *sec4_sg; @@ -1908,12 +767,12 @@ struct aead_edesc { /* * ablkcipher_edesc - s/w-extended ablkcipher descriptor - * @src_nents: number of segments in input scatterlist - * @dst_nents: number of segments in output scatterlist + * @src_nents: number of segments in input s/w scatterlist + * @dst_nents: number of segments in output s/w scatterlist * @iv_dma: dma address of iv for checking continuity and link table - * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) * @sec4_sg_bytes: length of dma mapped sec4_sg space * @sec4_sg_dma: bus physical mapped address of h/w link table + * @sec4_sg: pointer to h/w link table * @hw_desc: the h/w job descriptor followed by any referenced link tables */ struct ablkcipher_edesc { @@ -1933,10 +792,11 @@ static void caam_unmap(struct device *de int sec4_sg_bytes) { if (dst != src) { - dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE); - dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE); + if (src_nents) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); } else { - dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL); + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); } if (iv_dma) @@ -2031,8 +891,7 @@ static void ablkcipher_encrypt_done(stru dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ablkcipher_edesc *)((char *)desc - - offsetof(struct ablkcipher_edesc, hw_desc)); + edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); @@ -2041,10 +900,10 @@ static void ablkcipher_encrypt_done(stru print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, edesc->src_nents > 1 ? 100 : ivsize, 1); - dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true); #endif + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); ablkcipher_unmap(jrdev, edesc, req); @@ -2074,8 +933,7 @@ static void ablkcipher_decrypt_done(stru dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ablkcipher_edesc *)((char *)desc - - offsetof(struct ablkcipher_edesc, hw_desc)); + edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); @@ -2083,10 +941,10 @@ static void ablkcipher_decrypt_done(stru print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, ivsize, 1); - dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true); #endif + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); ablkcipher_unmap(jrdev, edesc, req); @@ -2127,7 +985,7 @@ static void init_aead_job(struct aead_re init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); if (all_contig) { - src_dma = sg_dma_address(req->src); + src_dma = edesc->src_nents ? sg_dma_address(req->src) : 0; in_options = 0; } else { src_dma = edesc->sec4_sg_dma; @@ -2142,7 +1000,7 @@ static void init_aead_job(struct aead_re out_options = in_options; if (unlikely(req->src != req->dst)) { - if (!edesc->dst_nents) { + if (edesc->dst_nents == 1) { dst_dma = sg_dma_address(req->dst); out_options = 0; } else { @@ -2161,9 +1019,6 @@ static void init_aead_job(struct aead_re append_seq_out_ptr(desc, dst_dma, req->assoclen + req->cryptlen - authsize, out_options); - - /* REG3 = assoclen */ - append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); } static void init_gcm_job(struct aead_request *req, @@ -2178,6 +1033,7 @@ static void init_gcm_job(struct aead_req unsigned int last; init_aead_job(req, edesc, all_contig, encrypt); + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); /* BUG This should not be specific to generic GCM. */ last = 0; @@ -2189,7 +1045,7 @@ static void init_gcm_job(struct aead_req FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); /* Append Salt */ if (!generic_gcm) - append_data(desc, ctx->key + ctx->enckeylen, 4); + append_data(desc, ctx->key + ctx->cdata.keylen, 4); /* Append IV */ append_data(desc, req->iv, ivsize); /* End of blank commands */ @@ -2204,7 +1060,8 @@ static void init_authenc_job(struct aead struct caam_aead_alg, aead); unsigned int ivsize = crypto_aead_ivsize(aead); struct caam_ctx *ctx = crypto_aead_ctx(aead); - const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; u32 *desc = edesc->hw_desc; @@ -2227,6 +1084,15 @@ static void init_authenc_job(struct aead init_aead_job(req, edesc, all_contig, encrypt); + /* + * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports + * having DPOVRD as destination. + */ + if (ctrlpriv->era < 3) + append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); + else + append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); + if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) append_load_as_imm(desc, req->iv, ivsize, LDST_CLASS_1_CCB | @@ -2250,16 +1116,15 @@ static void init_ablkcipher_job(u32 *sh_ int len, sec4_sg_index = 0; #ifdef DEBUG - bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) != 0); print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, ivsize, 1); - printk(KERN_ERR "asked=%d, nbytes%d\n", (int)edesc->src_nents ? 100 : req->nbytes, req->nbytes); - dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->src, - edesc->src_nents ? 100 : req->nbytes, 1, may_sleep); + pr_err("asked=%d, nbytes%d\n", + (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes); #endif + caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->src, + edesc->src_nents > 1 ? 100 : req->nbytes, 1); len = desc_len(sh_desc); init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); @@ -2275,7 +1140,7 @@ static void init_ablkcipher_job(u32 *sh_ append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); if (likely(req->src == req->dst)) { - if (!edesc->src_nents && iv_contig) { + if (edesc->src_nents == 1 && iv_contig) { dst_dma = sg_dma_address(req->src); } else { dst_dma = edesc->sec4_sg_dma + @@ -2283,7 +1148,7 @@ static void init_ablkcipher_job(u32 *sh_ out_options = LDST_SGF; } } else { - if (!edesc->dst_nents) { + if (edesc->dst_nents == 1) { dst_dma = sg_dma_address(req->dst); } else { dst_dma = edesc->sec4_sg_dma + @@ -2310,20 +1175,18 @@ static void init_ablkcipher_giv_job(u32 int len, sec4_sg_index = 0; #ifdef DEBUG - bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) != 0); print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", DUMP_PREFIX_ADDRESS, 16, 4, req->info, ivsize, 1); - dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->src, - edesc->src_nents ? 100 : req->nbytes, 1, may_sleep); #endif + caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->src, + edesc->src_nents > 1 ? 100 : req->nbytes, 1); len = desc_len(sh_desc); init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - if (!edesc->src_nents) { + if (edesc->src_nents == 1) { src_dma = sg_dma_address(req->src); in_options = 0; } else { @@ -2354,87 +1217,100 @@ static struct aead_edesc *aead_edesc_all struct crypto_aead *aead = crypto_aead_reqtfm(req); struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - int src_nents, dst_nents = 0; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; struct aead_edesc *edesc; - int sgc; - bool all_contig = true; - int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; + int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; unsigned int authsize = ctx->authsize; if (unlikely(req->dst != req->src)) { - src_nents = sg_count(req->src, req->assoclen + req->cryptlen); - dst_nents = sg_count(req->dst, - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); - } else { - src_nents = sg_count(req->src, - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); - } - - /* Check if data are contiguous. */ - all_contig = !src_nents; - if (!all_contig) { - src_nents = src_nents ? : 1; - sec4_sg_len = src_nents; - } - - sec4_sg_len += dst_nents; - - sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen); + return ERR_PTR(src_nents); + } - /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, - GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); - return ERR_PTR(-ENOMEM); + dst_nents = sg_nents_for_len(req->dst, req->assoclen + + req->cryptlen + + (encrypt ? authsize : + (-authsize))); + if (unlikely(dst_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : (-authsize))); + return ERR_PTR(dst_nents); + } + } else { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(src_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : 0)); + return ERR_PTR(src_nents); + } } if (likely(req->src == req->dst)) { - sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL); - if (unlikely(!sgc)) { + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { dev_err(jrdev, "unable to map source\n"); - kfree(edesc); return ERR_PTR(-ENOMEM); } } else { - sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE); - if (unlikely(!sgc)) { - dev_err(jrdev, "unable to map source\n"); - kfree(edesc); - return ERR_PTR(-ENOMEM); + /* Cover also the case of null (zero length) input data */ + if (src_nents) { + mapped_src_nents = dma_map_sg(jrdev, req->src, + src_nents, DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; } - sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE); - if (unlikely(!sgc)) { + mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { dev_err(jrdev, "unable to map destination\n"); - dma_unmap_sg(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE); - kfree(edesc); + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); return ERR_PTR(-ENOMEM); } } + sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; + sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, + GFP_DMA | flags); + if (!edesc) { + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + desc_bytes; - *all_contig_ptr = all_contig; + *all_contig_ptr = !(mapped_src_nents > 1); sec4_sg_index = 0; - if (!all_contig) { - sg_to_sec4_sg_last(req->src, src_nents, - edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += src_nents; + if (mapped_src_nents > 1) { + sg_to_sec4_sg_last(req->src, mapped_src_nents, + edesc->sec4_sg + sec4_sg_index, 0); + sec4_sg_index += mapped_src_nents; } - if (dst_nents) { - sg_to_sec4_sg_last(req->dst, dst_nents, + if (mapped_dst_nents > 1) { + sg_to_sec4_sg_last(req->dst, mapped_dst_nents, edesc->sec4_sg + sec4_sg_index, 0); } @@ -2587,13 +1463,9 @@ static int aead_decrypt(struct aead_requ u32 *desc; int ret = 0; -#ifdef DEBUG - bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) != 0); - dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->src, - req->assoclen + req->cryptlen, 1, may_sleep); -#endif + caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->src, + req->assoclen + req->cryptlen, 1); /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, @@ -2633,51 +1505,80 @@ static struct ablkcipher_edesc *ablkciph struct device *jrdev = ctx->jrdev; gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - int src_nents, dst_nents = 0, sec4_sg_bytes; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; struct ablkcipher_edesc *edesc; dma_addr_t iv_dma = 0; - bool iv_contig = false; - int sgc; + bool in_contig; int ivsize = crypto_ablkcipher_ivsize(ablkcipher); - int sec4_sg_index; + int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; - src_nents = sg_count(req->src, req->nbytes); + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } - if (req->dst != req->src) - dst_nents = sg_count(req->dst, req->nbytes); + if (req->dst != req->src) { + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + } if (likely(req->src == req->dst)) { - sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL); + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } } else { - sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE); - sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE); + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(jrdev, "unable to map destination\n"); + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } } iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, iv_dma)) { dev_err(jrdev, "unable to map IV\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0); return ERR_PTR(-ENOMEM); } - /* - * Check if iv can be contiguous with source and destination. - * If so, include it. If not, create scatterlist. - */ - if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) - iv_contig = true; - else - src_nents = src_nents ? : 1; - sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * - sizeof(struct sec4_sg_entry); + if (mapped_src_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->src)) { + in_contig = true; + sec4_sg_ents = 0; + } else { + in_contig = false; + sec4_sg_ents = 1 + mapped_src_nents; + } + dst_sg_idx = sec4_sg_ents; + sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, 0, 0); return ERR_PTR(-ENOMEM); } @@ -2687,23 +1588,24 @@ static struct ablkcipher_edesc *ablkciph edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + desc_bytes; - sec4_sg_index = 0; - if (!iv_contig) { + if (!in_contig) { dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); - sg_to_sec4_sg_last(req->src, src_nents, + sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg + 1, 0); - sec4_sg_index += 1 + src_nents; } - if (dst_nents) { - sg_to_sec4_sg_last(req->dst, dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); + if (mapped_dst_nents > 1) { + sg_to_sec4_sg_last(req->dst, mapped_dst_nents, + edesc->sec4_sg + dst_sg_idx, 0); } edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { dev_err(jrdev, "unable to map S/G table\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, 0, 0); + kfree(edesc); return ERR_PTR(-ENOMEM); } @@ -2715,7 +1617,7 @@ static struct ablkcipher_edesc *ablkciph sec4_sg_bytes, 1); #endif - *iv_contig_out = iv_contig; + *iv_contig_out = in_contig; return edesc; } @@ -2806,30 +1708,54 @@ static struct ablkcipher_edesc *ablkciph struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - int src_nents, dst_nents = 0, sec4_sg_bytes; + int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; struct ablkcipher_edesc *edesc; dma_addr_t iv_dma = 0; - bool iv_contig = false; - int sgc; + bool out_contig; int ivsize = crypto_ablkcipher_ivsize(ablkcipher); - int sec4_sg_index; - - src_nents = sg_count(req->src, req->nbytes); + int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; - if (unlikely(req->dst != req->src)) - dst_nents = sg_count(req->dst, req->nbytes); + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } if (likely(req->src == req->dst)) { - sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, - DMA_BIDIRECTIONAL); + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + dst_nents = src_nents; + mapped_dst_nents = src_nents; } else { - sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, - DMA_TO_DEVICE); - sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, - DMA_FROM_DEVICE); + mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(jrdev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + + mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(jrdev, "unable to map destination\n"); + dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } } /* @@ -2839,21 +1765,29 @@ static struct ablkcipher_edesc *ablkciph iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, iv_dma)) { dev_err(jrdev, "unable to map IV\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0); return ERR_PTR(-ENOMEM); } - if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) - iv_contig = true; - else - dst_nents = dst_nents ? : 1; - sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * - sizeof(struct sec4_sg_entry); + sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; + dst_sg_idx = sec4_sg_ents; + if (mapped_dst_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->dst)) { + out_contig = true; + } else { + out_contig = false; + sec4_sg_ents += 1 + mapped_dst_nents; + } /* allocate space for base edesc and hw desc commands, link tables */ + sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, GFP_DMA | flags); if (!edesc) { dev_err(jrdev, "could not allocate extended descriptor\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, 0, 0); return ERR_PTR(-ENOMEM); } @@ -2863,24 +1797,24 @@ static struct ablkcipher_edesc *ablkciph edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + desc_bytes; - sec4_sg_index = 0; - if (src_nents) { - sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); - sec4_sg_index += src_nents; - } + if (mapped_src_nents > 1) + sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg, + 0); - if (!iv_contig) { - dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, + if (!out_contig) { + dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx, iv_dma, ivsize, 0); - sec4_sg_index += 1; - sg_to_sec4_sg_last(req->dst, dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); + sg_to_sec4_sg_last(req->dst, mapped_dst_nents, + edesc->sec4_sg + dst_sg_idx + 1, 0); } edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { dev_err(jrdev, "unable to map S/G table\n"); + caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, 0, 0); + kfree(edesc); return ERR_PTR(-ENOMEM); } edesc->iv_dma = iv_dma; @@ -2892,7 +1826,7 @@ static struct ablkcipher_edesc *ablkciph sec4_sg_bytes, 1); #endif - *iv_contig_out = iv_contig; + *iv_contig_out = out_contig; return edesc; } @@ -2903,7 +1837,7 @@ static int ablkcipher_givencrypt(struct struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); struct device *jrdev = ctx->jrdev; - bool iv_contig; + bool iv_contig = false; u32 *desc; int ret = 0; @@ -2947,7 +1881,6 @@ struct caam_alg_template { } template_u; u32 class1_alg_type; u32 class2_alg_type; - u32 alg_op; }; static struct caam_alg_template driver_algs[] = { @@ -3132,7 +2065,6 @@ static struct caam_aead_alg driver_aeads .caam = { .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, }, }, { @@ -3154,7 +2086,6 @@ static struct caam_aead_alg driver_aeads .caam = { .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, }, }, { @@ -3176,7 +2107,6 @@ static struct caam_aead_alg driver_aeads .caam = { .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, }, }, { @@ -3198,7 +2128,6 @@ static struct caam_aead_alg driver_aeads .caam = { .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, }, }, { @@ -3220,7 +2149,6 @@ static struct caam_aead_alg driver_aeads .caam = { .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, }, }, { @@ -3242,7 +2170,6 @@ static struct caam_aead_alg driver_aeads .caam = { .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, }, { @@ -3264,7 +2191,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, }, }, { @@ -3287,7 +2213,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3310,7 +2235,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, }, }, { @@ -3333,7 +2257,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3356,7 +2279,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, }, }, { @@ -3379,7 +2301,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3402,7 +2323,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, }, }, { @@ -3425,7 +2345,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3448,7 +2367,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, }, }, { @@ -3471,7 +2389,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3494,7 +2411,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, }, { @@ -3517,7 +2433,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3540,7 +2455,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, } }, { @@ -3563,7 +2477,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, .geniv = true, } }, @@ -3587,7 +2500,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, }, }, { @@ -3611,7 +2523,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3635,7 +2546,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, }, }, { @@ -3659,7 +2569,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3683,7 +2592,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, }, }, { @@ -3707,7 +2615,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3731,7 +2638,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, }, }, { @@ -3755,7 +2661,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3779,7 +2684,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, }, { @@ -3803,7 +2707,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3826,7 +2729,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, }, }, { @@ -3849,7 +2751,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3872,7 +2773,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, }, }, { @@ -3895,7 +2795,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3918,7 +2817,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, }, }, { @@ -3941,7 +2839,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -3964,7 +2861,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, }, }, { @@ -3987,7 +2883,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -4010,7 +2905,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, }, }, { @@ -4033,7 +2927,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -4056,7 +2949,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, }, { @@ -4079,7 +2971,6 @@ static struct caam_aead_alg driver_aeads .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, .geniv = true, }, }, @@ -4104,7 +2995,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, .rfc3686 = true, }, }, @@ -4129,7 +3019,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, .rfc3686 = true, .geniv = true, }, @@ -4155,7 +3044,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, .rfc3686 = true, }, }, @@ -4180,7 +3068,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, .rfc3686 = true, .geniv = true, }, @@ -4206,7 +3093,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, .rfc3686 = true, }, }, @@ -4231,7 +3117,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, .rfc3686 = true, .geniv = true, }, @@ -4257,7 +3142,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, .rfc3686 = true, }, }, @@ -4282,7 +3166,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, .rfc3686 = true, .geniv = true, }, @@ -4308,7 +3191,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, .rfc3686 = true, }, }, @@ -4333,7 +3215,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, .rfc3686 = true, .geniv = true, }, @@ -4359,7 +3240,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, .rfc3686 = true, }, }, @@ -4384,7 +3264,6 @@ static struct caam_aead_alg driver_aeads OP_ALG_AAI_CTR_MOD128, .class2_alg_type = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC_PRECOMP, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, .rfc3686 = true, .geniv = true, }, @@ -4399,16 +3278,34 @@ struct caam_crypto_alg { static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) { + dma_addr_t dma_addr; + ctx->jrdev = caam_jr_alloc(); if (IS_ERR(ctx->jrdev)) { pr_err("Job Ring Device allocation for transform failed\n"); return PTR_ERR(ctx->jrdev); } + dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, + offsetof(struct caam_ctx, + sh_desc_enc_dma), + DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->jrdev, dma_addr)) { + dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + + ctx->sh_desc_enc_dma = dma_addr; + ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, + sh_desc_dec); + ctx->sh_desc_givenc_dma = dma_addr + offsetof(struct caam_ctx, + sh_desc_givenc); + ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key); + /* copy descriptor header template value */ - ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; - ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; - ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; return 0; } @@ -4435,25 +3332,9 @@ static int caam_aead_init(struct crypto_ static void caam_exit_common(struct caam_ctx *ctx) { - if (ctx->sh_desc_enc_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, - desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); - if (ctx->sh_desc_dec_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, - desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); - if (ctx->sh_desc_givenc_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, - desc_bytes(ctx->sh_desc_givenc), - DMA_TO_DEVICE); - if (ctx->key_dma && - !dma_mapping_error(ctx->jrdev, ctx->key_dma)) - dma_unmap_single(ctx->jrdev, ctx->key_dma, - ctx->enckeylen + ctx->split_key_pad_len, - DMA_TO_DEVICE); - + dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, + offsetof(struct caam_ctx, sh_desc_enc_dma), + DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); caam_jr_free(ctx->jrdev); } @@ -4529,7 +3410,6 @@ static struct caam_crypto_alg *caam_alg_ t_alg->caam.class1_alg_type = template->class1_alg_type; t_alg->caam.class2_alg_type = template->class2_alg_type; - t_alg->caam.alg_op = template->alg_op; return t_alg; } --- /dev/null +++ b/drivers/crypto/caam/caamalg_desc.c @@ -0,0 +1,1961 @@ +/* + * Shared descriptors for aead, ablkcipher algorithms + * + * Copyright 2016 NXP + */ + +#include "compat.h" +#include "desc_constr.h" +#include "caamalg_desc.h" + +/* + * For aead functions, read payload and write payload, + * both of which are specified in req->src and req->dst + */ +static inline void aead_append_src_dst(u32 *desc, u32 msg_type) +{ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | + KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); +} + +/* Set DK bit in class 1 operation if shared */ +static inline void append_dec_op1(u32 *desc, u32 type) +{ + u32 *jump_cmd, *uncond_jump_cmd; + + /* DK bit is valid only for AES */ + if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { + append_operation(desc, type | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + return; + } + + jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); + append_operation(desc, type | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, jump_cmd); + append_operation(desc, type | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_AAI_DK); + set_jump_tgt_here(desc, uncond_jump_cmd); +} + +/** + * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor + * (non-protocol) with no (null) encryption. + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @icvsize: integrity check value (ICV) size (truncated or full) + * @era: SEC Era + */ +void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } + set_jump_tgt_here(desc, key_jump_cmd); + + /* assoclen + cryptlen = seqinlen */ + append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Prepare to read and write cryptlen + assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | + MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | + MOVE_DEST_DESCBUF | + MOVE_WAITCOMP | + (0x8 << MOVE_LEN_SHIFT)); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | + MOVE_AUX_LS); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "aead null enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); + +/** + * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor + * (non-protocol) with no (null) decryption. + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @icvsize: integrity check value (ICV) size (truncated or full) + * @era: SEC Era + */ +void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* assoclen + cryptlen = seqoutlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Prepare to read and write cryptlen + assoclen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | + MOVE_DEST_MATH2 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | + MOVE_DEST_DESCBUF | + MOVE_WAITCOMP | + (0x8 << MOVE_LEN_SHIFT)); + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* + * Insert a NOP here, since we need at least 4 instructions between + * code patching the descriptor buffer and the location being patched. + */ + jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, jump_cmd); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | + MOVE_AUX_LS); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Load ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "aead null dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); + +static void init_sh_desc_key_aead(u32 * const desc, + struct alginfo * const cdata, + struct alginfo * const adata, + const bool is_rfc3686, u32 *nonce, int era) +{ + u32 *key_jump_cmd; + unsigned int enckeylen = cdata->keylen; + + /* Note: Context registers are saved. */ + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* + * RFC3686 specific: + * | key = {AUTH_KEY, ENC_KEY, NONCE} + * | enckeylen = encryption key size + nonce size + */ + if (is_rfc3686) + enckeylen -= CTR_RFC3686_NONCE_SIZE; + + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } + + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, enckeylen, + enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, enckeylen, CLASS_1 | + KEY_DEST_CLASS_REG); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) { + append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, + LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, + MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | + (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + + set_jump_tgt_here(desc, key_jump_cmd); +} + +/** + * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @nonce: pointer to rfc3686 nonce + * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi + * @era: SEC Era + */ +void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, const bool is_qi, + int era) +{ + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + } + + /* Read and write assoclen bytes */ + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + } + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + FIFOLDST_VLF); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_aead_encap); + +/** + * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @nonce: pointer to rfc3686 nonce + * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi + * @era: SEC Era + */ +void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool geniv, + const bool is_rfc3686, u32 *nonce, + const u32 ctx1_iv_off, const bool is_qi, int era) +{ + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); + + /* Class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + if (!geniv) + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + } + + /* Read and write assoclen bytes */ + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + if (geniv) + append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, + ivsize); + else + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, + CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + if (geniv) + append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM, + ivsize); + else + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, + CAAM_CMD_SZ); + } + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + if (geniv) { + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); + } + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Choose operation */ + if (ctx1_iv_off) + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + else + append_dec_op1(desc, cdata->algtype); + + /* Read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + aead_append_src_dst(desc, FIFOLD_TYPE_MSG); + + /* Load ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_aead_decap); + +/** + * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor + * (non-protocol) with HW-generated initialization + * vector. + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values - one of + * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed + * with OP_ALG_AAI_HMAC_PRECOMP. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @nonce: pointer to rfc3686 nonce + * @ctx1_iv_off: IV offset in CONTEXT1 register + * @is_qi: true when called from caam/qi + * @era: SEC Era + */ +void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi, int era) +{ + u32 geniv, moveiv; + + /* Note: Context registers are saved. */ + init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + } + + if (is_rfc3686) { + if (is_qi) + append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (ctx1_iv_off << LDST_OFFSET_SHIFT)); + + goto copy_iv; + } + + /* Generate IV */ + geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | + NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | + NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (ivsize << MOVE_LEN_SHIFT)); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + +copy_iv: + /* Copy IV to class 1 context */ + append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | + (ctx1_iv_off << MOVE_OFFSET_SHIFT) | + (ivsize << MOVE_LEN_SHIFT)); + + /* Return to encryption */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Read and write assoclen bytes */ + if (is_qi || era < 3) { + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + } else { + append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); + } + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + /* Copy iv from outfifo to class 2 fifo */ + moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | + NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | + LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Will write ivsize + cryptlen */ + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Not need to reload iv */ + append_seq_fifo_load(desc, ivsize, + FIFOLD_CLASS_SKIP); + + /* Will read cryptlen */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | + FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "aead givenc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); + +/** + * cnstr_shdsc_tls_encap - tls encapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_AES ANDed + * with OP_ALG_AAI_CBC + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values OP_ALG_ALGSEL_SHA1 + * ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @assoclen: associated data length + * @ivsize: initialization vector size + * @authsize: authentication data size + * @blocksize: block cipher size + * @era: SEC Era + */ +void cnstr_shdsc_tls_encap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int assoclen, + unsigned int ivsize, unsigned int authsize, + unsigned int blocksize, int era) +{ + u32 *key_jump_cmd, *zero_payload_jump_cmd; + u32 genpad, idx_ld_datasz, idx_ld_pad, stidx; + + /* + * Compute the index (in bytes) for the LOAD with destination of + * Class 1 Data Size Register and for the LOAD that generates padding + */ + if (adata->key_inline) { + idx_ld_datasz = DESC_TLS10_ENC_LEN + adata->keylen_pad + + cdata->keylen - 4 * CAAM_CMD_SZ; + idx_ld_pad = DESC_TLS10_ENC_LEN + adata->keylen_pad + + cdata->keylen - 2 * CAAM_CMD_SZ; + } else { + idx_ld_datasz = DESC_TLS10_ENC_LEN + 2 * CAAM_PTR_SZ - + 4 * CAAM_CMD_SZ; + idx_ld_pad = DESC_TLS10_ENC_LEN + 2 * CAAM_PTR_SZ - + 2 * CAAM_CMD_SZ; + } + + stidx = 1 << HDR_START_IDX_SHIFT; + init_sh_desc(desc, HDR_SHARE_SERIAL | stidx); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + if (era < 6) { + if (adata->key_inline) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | + KEY_ENC); + else + append_key(desc, adata->key_dma, adata->keylen, + CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); + } else { + append_proto_dkp(desc, adata); + } + + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + /* class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* payloadlen = input data length - (assoclen + ivlen) */ + append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, assoclen + ivsize); + + /* math1 = payloadlen + icvlen */ + append_math_add_imm_u32(desc, REG1, REG0, IMM, authsize); + + /* padlen = block_size - math1 % block_size */ + append_math_and_imm_u32(desc, REG3, REG1, IMM, blocksize - 1); + append_math_sub_imm_u32(desc, REG2, IMM, REG3, blocksize); + + /* cryptlen = payloadlen + icvlen + padlen */ + append_math_add(desc, VARSEQOUTLEN, REG1, REG2, 4); + + /* + * update immediate data with the padding length value + * for the LOAD in the class 1 data size register. + */ + append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH2 | + (idx_ld_datasz << MOVE_OFFSET_SHIFT) | 7); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH2 | MOVE_DEST_DESCBUF | + (idx_ld_datasz << MOVE_OFFSET_SHIFT) | 8); + + /* overwrite PL field for the padding iNFO FIFO entry */ + append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH2 | + (idx_ld_pad << MOVE_OFFSET_SHIFT) | 7); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH2 | MOVE_DEST_DESCBUF | + (idx_ld_pad << MOVE_OFFSET_SHIFT) | 8); + + /* store encrypted payload, icv and padding */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF); + + /* if payload length is zero, jump to zero-payload commands */ + append_math_add(desc, VARSEQINLEN, ZERO, REG0, 4); + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* load iv in context1 */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX | + LDST_CLASS_1_CCB | ivsize); + + /* read assoc for authentication */ + append_seq_fifo_load(desc, assoclen, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_MSG); + /* insnoop payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLD_TYPE_MSG | + FIFOLD_TYPE_LAST2 | FIFOLDST_VLF); + + /* jump the zero-payload commands */ + append_jump(desc, JUMP_TEST_ALL | 3); + + /* zero-payload commands */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* load iv in context1 */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX | + LDST_CLASS_1_CCB | ivsize); + + /* assoc data is the only data for authentication */ + append_seq_fifo_load(desc, assoclen, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2); + + /* send icv to encryption */ + append_move(desc, MOVE_SRC_CLASS2CTX | MOVE_DEST_CLASS1INFIFO | + authsize); + + /* update class 1 data size register with padding length */ + append_load_imm_u32(desc, 0, LDST_CLASS_1_CCB | + LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); + + /* generate padding and send it to encryption */ + genpad = NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_LC1 | NFIFOENTRY_FC1 | + NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_PTYPE_N; + append_load_imm_u32(desc, genpad, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "tls enc shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_tls_encap); + +/** + * cnstr_shdsc_tls_decap - tls decapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_AES ANDed + * with OP_ALG_AAI_CBC + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. Valid algorithm values OP_ALG_ALGSEL_SHA1 + * ANDed with OP_ALG_AAI_HMAC_PRECOMP. + * @assoclen: associated data length + * @ivsize: initialization vector size + * @authsize: authentication data size + * @blocksize: block cipher size + * @era: SEC Era + */ +void cnstr_shdsc_tls_decap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int assoclen, + unsigned int ivsize, unsigned int authsize, + unsigned int blocksize, int era) +{ + u32 stidx, jumpback; + u32 *key_jump_cmd, *zero_payload_jump_cmd, *skip_zero_jump_cmd; + /* + * Pointer Size bool determines the size of address pointers. + * false - Pointers fit in one 32-bit word. + * true - Pointers fit in two 32-bit words. + */ + static const bool ps = (CAAM_PTR_SZ != CAAM_CMD_SZ); + + stidx = 1 << HDR_START_IDX_SHIFT; + init_sh_desc(desc, HDR_SHARE_SERIAL | stidx); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + if (era < 6) + append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + else + append_proto_dkp(desc, adata); + + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 2 operation */ + append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + /* class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + + /* VSIL = input data length - 2 * block_size */ + append_math_sub_imm_u32(desc, VARSEQINLEN, SEQINLEN, IMM, 2 * + blocksize); + + /* + * payloadlen + icvlen + padlen = input data length - (assoclen + + * ivsize) + */ + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, assoclen + ivsize); + + /* skip data to the last but one cipher block */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_SKIP | LDST_VLF); + + /* load iv for the last cipher block */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX | + LDST_CLASS_1_CCB | ivsize); + + /* read last cipher block */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG | + FIFOLD_TYPE_LAST1 | blocksize); + + /* move decrypted block into math0 and math1 */ + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | MOVE_DEST_MATH0 | + blocksize); + + /* reset AES CHA */ + append_load_imm_u32(desc, CCTRL_RESET_CHA_AESA, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_CHACTRL | LDST_IMM); + + /* rewind input sequence */ + append_seq_in_ptr_intlen(desc, 0, 65535, SQIN_RTO); + + /* key1 is in decryption form */ + append_operation(desc, cdata->algtype | OP_ALG_AAI_DK | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); + + /* load iv in context1 */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_CLASS_1_CCB | + LDST_SRCDST_WORD_CLASS_CTX | ivsize); + + /* read sequence number */ + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG); + /* load Type, Version and Len fields in math0 */ + append_cmd(desc, CMD_SEQ_LOAD | LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH0 | (3 << LDST_OFFSET_SHIFT) | 5); + + /* compute (padlen - 1) */ + append_math_and_imm_u64(desc, REG1, REG1, IMM, 255); + + /* math2 = icvlen + (padlen - 1) + 1 */ + append_math_add_imm_u32(desc, REG2, REG1, IMM, authsize + 1); + + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1); + + /* VSOL = payloadlen + icvlen + padlen */ + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, 4); + +#ifdef __LITTLE_ENDIAN + append_moveb(desc, MOVE_WAITCOMP | + MOVE_SRC_MATH0 | MOVE_DEST_MATH0 | 8); +#endif + /* update Len field */ + append_math_sub(desc, REG0, REG0, REG2, 8); + + /* store decrypted payload, icv and padding */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF); + + /* VSIL = (payloadlen + icvlen + padlen) - (icvlen + padlen)*/ + append_math_sub(desc, VARSEQINLEN, REG3, REG2, 4); + + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* send Type, Version and Len(pre ICV) fields to authentication */ + append_move(desc, MOVE_WAITCOMP | + MOVE_SRC_MATH0 | MOVE_DEST_CLASS2INFIFO | + (3 << MOVE_OFFSET_SHIFT) | 5); + + /* outsnooping payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | + FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LAST2 | + FIFOLDST_VLF); + skip_zero_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 2); + + set_jump_tgt_here(desc, zero_payload_jump_cmd); + /* send Type, Version and Len(pre ICV) fields to authentication */ + append_move(desc, MOVE_WAITCOMP | MOVE_AUX_LS | + MOVE_SRC_MATH0 | MOVE_DEST_CLASS2INFIFO | + (3 << MOVE_OFFSET_SHIFT) | 5); + + set_jump_tgt_here(desc, skip_zero_jump_cmd); + append_math_add(desc, VARSEQINLEN, ZERO, REG2, 4); + + /* load icvlen and padlen */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG | + FIFOLD_TYPE_LAST1 | FIFOLDST_VLF); + + /* VSIL = (payloadlen + icvlen + padlen) - icvlen + padlen */ + append_math_sub(desc, VARSEQINLEN, REG3, REG2, 4); + + /* + * Start a new input sequence using the SEQ OUT PTR command options, + * pointer and length used when the current output sequence was defined. + */ + if (ps) { + /* + * Move the lower 32 bits of Shared Descriptor address, the + * SEQ OUT PTR command, Output Pointer (2 words) and + * Output Length into math registers. + */ +#ifdef __LITTLE_ENDIAN + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | + MOVE_DEST_MATH0 | (55 * 4 << MOVE_OFFSET_SHIFT) | + 20); +#else + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | + MOVE_DEST_MATH0 | (54 * 4 << MOVE_OFFSET_SHIFT) | + 20); +#endif + /* Transform SEQ OUT PTR command in SEQ IN PTR command */ + append_math_and_imm_u32(desc, REG0, REG0, IMM, + ~(CMD_SEQ_IN_PTR ^ CMD_SEQ_OUT_PTR)); + /* Append a JUMP command after the copied fields */ + jumpback = CMD_JUMP | (char)-9; + append_load_imm_u32(desc, jumpback, LDST_CLASS_DECO | LDST_IMM | + LDST_SRCDST_WORD_DECO_MATH2 | + (4 << LDST_OFFSET_SHIFT)); + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1); + /* Move the updated fields back to the Job Descriptor */ +#ifdef __LITTLE_ENDIAN + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | + MOVE_DEST_DESCBUF | (55 * 4 << MOVE_OFFSET_SHIFT) | + 24); +#else + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | + MOVE_DEST_DESCBUF | (54 * 4 << MOVE_OFFSET_SHIFT) | + 24); +#endif + /* + * Read the new SEQ IN PTR command, Input Pointer, Input Length + * and then jump back to the next command from the + * Shared Descriptor. + */ + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 6); + } else { + /* + * Move the SEQ OUT PTR command, Output Pointer (1 word) and + * Output Length into math registers. + */ +#ifdef __LITTLE_ENDIAN + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | + MOVE_DEST_MATH0 | (54 * 4 << MOVE_OFFSET_SHIFT) | + 12); +#else + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | + MOVE_DEST_MATH0 | (53 * 4 << MOVE_OFFSET_SHIFT) | + 12); +#endif + /* Transform SEQ OUT PTR command in SEQ IN PTR command */ + append_math_and_imm_u64(desc, REG0, REG0, IMM, + ~(((u64)(CMD_SEQ_IN_PTR ^ + CMD_SEQ_OUT_PTR)) << 32)); + /* Append a JUMP command after the copied fields */ + jumpback = CMD_JUMP | (char)-7; + append_load_imm_u32(desc, jumpback, LDST_CLASS_DECO | LDST_IMM | + LDST_SRCDST_WORD_DECO_MATH1 | + (4 << LDST_OFFSET_SHIFT)); + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1); + /* Move the updated fields back to the Job Descriptor */ +#ifdef __LITTLE_ENDIAN + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | + MOVE_DEST_DESCBUF | (54 * 4 << MOVE_OFFSET_SHIFT) | + 16); +#else + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | + MOVE_DEST_DESCBUF | (53 * 4 << MOVE_OFFSET_SHIFT) | + 16); +#endif + /* + * Read the new SEQ IN PTR command, Input Pointer, Input Length + * and then jump back to the next command from the + * Shared Descriptor. + */ + append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 5); + } + + /* skip payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_SKIP | FIFOLDST_VLF); + /* check icv */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_ICV | + FIFOLD_TYPE_LAST2 | authsize); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "tls dec shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_tls_decap); + +/** + * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, + *zero_assoc_jump_cmd2; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM, + ivsize); + } else { + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, + CAAM_CMD_SZ); + } + + /* if assoclen + cryptlen is ZERO, skip to ICV write */ + zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + if (is_qi) + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* cryptlen = seqinlen - assoclen */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* if cryptlen is ZERO jump to zero-payload commands */ + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* write encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* jump to ICV writing */ + if (is_qi) + append_jump(desc, JUMP_TEST_ALL | 4); + else + append_jump(desc, JUMP_TEST_ALL | 2); + + /* zero-payload commands */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); + if (is_qi) + /* jump to ICV writing */ + append_jump(desc, JUMP_TEST_ALL | 2); + + /* There is no input data */ + set_jump_tgt_here(desc, zero_assoc_jump_cmd2); + + if (is_qi) + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | + FIFOLD_TYPE_LAST1); + + /* write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); + +/** + * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* skip key loading if they are loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | + JUMP_TEST_ALL | JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + /* if assoclen is ZERO, skip reading the assoc data */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + set_jump_tgt_here(desc, zero_assoc_jump_cmd1); + + /* cryptlen = seqoutlen - assoclen */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* jump to zero-payload command if cryptlen is zero */ + zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | + JUMP_COND_MATH_Z); + + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* store encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* zero-payload command */ + set_jump_tgt_here(desc, zero_payload_jump_cmd); + + /* read ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); + +/** + * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Skip IV */ + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); + + /* Will read cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* cryptlen = seqoutlen - assoclen */ + append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); + + /* Write encrypted data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* Read payload data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "rfc4106 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); + +/** + * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + u32 *wait_load_cmd; + + /* REG3 = assoclen */ + append_seq_load(desc, 4, LDST_CLASS_DECO | + LDST_SRCDST_WORD_DECO_MATH3 | + (4 << LDST_OFFSET_SHIFT)); + + wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_CALM | JUMP_COND_NCP | + JUMP_COND_NOP | JUMP_COND_NIP | + JUMP_COND_NIFP); + set_jump_tgt_here(desc, wait_load_cmd); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* Read assoc data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); + + /* Skip IV */ + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); + + /* Will read cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); + + /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); + + /* Skip assoc data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); + + /* Will write cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Store payload data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* Read encrypted data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | + FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* Read ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "rfc4106 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); + +/** + * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + if (is_qi) { + /* assoclen is not needed, skip it */ + append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + /* assoclen + cryptlen = seqinlen */ + append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | + (0x8 << MOVE_LEN_SHIFT)); + + /* Will read assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Will write assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + + /* Read and write assoclen + cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_AAD); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move payload data to OFIFO */ + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); + + /* Write ICV */ + append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "rfc4543 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); + +/** + * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor + * (non-protocol). + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. + * @ivsize: initialization vector size + * @icvsize: integrity check value (ICV) size (truncated or full) + * @is_qi: true when called from caam/qi + */ +void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi) +{ + u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip key loading if it is loaded due to sharing */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (cdata->key_inline) + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + else + append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | + KEY_DEST_CLASS_REG); + set_jump_tgt_here(desc, key_jump_cmd); + + /* Class 1 operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + if (is_qi) { + /* assoclen is not needed, skip it */ + append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); + + /* Read salt and IV */ + append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + + cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV); + append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); + } + + /* assoclen + cryptlen = seqoutlen */ + append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | + (0x8 << MOVE_LEN_SHIFT)); + + /* Will read assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Will write assoclen + cryptlen bytes */ + append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); + + /* Store payload data */ + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); + + /* In-snoop assoclen + cryptlen data */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | + FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + /* Move payload data to OFIFO */ + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Read ICV */ + append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | + FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "rfc4543 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); + +/* + * For ablkcipher encrypt and decrypt, read from req->src and + * write to req->dst + */ +static inline void ablkcipher_append_src_dst(u32 *desc) +{ + append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | + KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); +} + +/** + * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @ivsize: initialization vector size + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @ctx1_iv_off: IV offset in CONTEXT1 register + */ +void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off) +{ + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load nonce into CONTEXT1 reg */ + if (is_rfc3686) { + const u8 *nonce = cdata->key_virt + cdata->keylen; + + append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, + LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + + set_jump_tgt_here(desc, key_jump_cmd); + + /* Load iv */ + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); + + /* Load counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Load operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ablkcipher enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap); + +/** + * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. + * @ivsize: initialization vector size + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @ctx1_iv_off: IV offset in CONTEXT1 register + */ +void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off) +{ + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load nonce into CONTEXT1 reg */ + if (is_rfc3686) { + const u8 *nonce = cdata->key_virt + cdata->keylen; + + append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, + LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + + set_jump_tgt_here(desc, key_jump_cmd); + + /* load IV */ + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); + + /* Load counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + /* Choose operation */ + if (ctx1_iv_off) + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_DECRYPT); + else + append_dec_op1(desc, cdata->algtype); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ablkcipher dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap); + +/** + * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor + * with HW-generated initialization vector. + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed + * with OP_ALG_AAI_CBC. + * @ivsize: initialization vector size + * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template + * @ctx1_iv_off: IV offset in CONTEXT1 register + */ +void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off) +{ + u32 *key_jump_cmd, geniv; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load Nonce into CONTEXT1 reg */ + if (is_rfc3686) { + const u8 *nonce = cdata->key_virt + cdata->keylen; + + append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, + LDST_CLASS_IND_CCB | + LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | + MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | + (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); + } + set_jump_tgt_here(desc, key_jump_cmd); + + /* Generate IV */ + geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | + NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND | + (ivsize << NFIFOENTRY_DLEN_SHIFT); + append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | + LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | + MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) | + (ctx1_iv_off << MOVE_OFFSET_SHIFT)); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Copy generated IV to memory */ + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); + + /* Load Counter into CONTEXT1 reg */ + if (is_rfc3686) + append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << + LDST_OFFSET_SHIFT)); + + if (ctx1_iv_off) + append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | + (1 << JUMP_OFFSET_SHIFT)); + + /* Load operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap); + +/** + * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared + * descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. + */ +void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata) +{ + __be64 sector_size = cpu_to_be64(512); + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 keys only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load sector size with index 40 bytes (0x28) */ + append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (0x28 << LDST_OFFSET_SHIFT)); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* + * create sequence for loading the sector index + * Upper 8B of IV - will be used as sector index + * Lower 8B of IV - will be discarded + */ + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + + /* Load operation */ + append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + OP_ALG_ENCRYPT); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap); + +/** + * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared + * descriptor + * @desc: pointer to buffer used for descriptor construction + * @cdata: pointer to block cipher transform definitions + * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. + */ +void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata) +{ + __be64 sector_size = cpu_to_be64(512); + u32 *key_jump_cmd; + + init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + /* Load class1 key only */ + append_key_as_imm(desc, cdata->key_virt, cdata->keylen, + cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + + /* Load sector size with index 40 bytes (0x28) */ + append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | + LDST_SRCDST_BYTE_CONTEXT | + (0x28 << LDST_OFFSET_SHIFT)); + + set_jump_tgt_here(desc, key_jump_cmd); + + /* + * create sequence for loading the sector index + * Upper 8B of IV - will be used as sector index + * Lower 8B of IV - will be discarded + */ + append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); + + /* Load operation */ + append_dec_op1(desc, cdata->algtype); + + /* Perform operation */ + ablkcipher_append_src_dst(desc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif +} +EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("FSL CAAM descriptor support"); +MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); --- /dev/null +++ b/drivers/crypto/caam/caamalg_desc.h @@ -0,0 +1,127 @@ +/* + * Shared descriptors for aead, ablkcipher algorithms + * + * Copyright 2016 NXP + */ + +#ifndef _CAAMALG_DESC_H_ +#define _CAAMALG_DESC_H_ + +/* length of descriptors text */ +#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) +#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) +#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_ENC_LEN (DESC_AEAD_ENC_LEN + 3 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_DEC_LEN (DESC_AEAD_DEC_LEN + 3 * CAAM_CMD_SZ) +#define DESC_QI_AEAD_GIVENC_LEN (DESC_AEAD_GIVENC_LEN + 3 * CAAM_CMD_SZ) + +#define DESC_TLS_BASE (4 * CAAM_CMD_SZ) +#define DESC_TLS10_ENC_LEN (DESC_TLS_BASE + 29 * CAAM_CMD_SZ) + +/* Note: Nonce is counted in cdata.keylen */ +#define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) + +#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) + +#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) +#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) +#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) +#define DESC_QI_GCM_ENC_LEN (DESC_GCM_ENC_LEN + 6 * CAAM_CMD_SZ) +#define DESC_QI_GCM_DEC_LEN (DESC_GCM_DEC_LEN + 3 * CAAM_CMD_SZ) + +#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) +#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) +#define DESC_QI_RFC4106_ENC_LEN (DESC_RFC4106_ENC_LEN + 5 * CAAM_CMD_SZ) +#define DESC_QI_RFC4106_DEC_LEN (DESC_RFC4106_DEC_LEN + 5 * CAAM_CMD_SZ) + +#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) +#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) +#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) +#define DESC_QI_RFC4543_ENC_LEN (DESC_RFC4543_ENC_LEN + 4 * CAAM_CMD_SZ) +#define DESC_QI_RFC4543_DEC_LEN (DESC_RFC4543_DEC_LEN + 4 * CAAM_CMD_SZ) + +#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) +#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ + 20 * CAAM_CMD_SZ) +#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ + 15 * CAAM_CMD_SZ) + +void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era); + +void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, + unsigned int icvsize, int era); + +void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi, int era); + +void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool geniv, + const bool is_rfc3686, u32 *nonce, + const u32 ctx1_iv_off, const bool is_qi, int era); + +void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int ivsize, + unsigned int icvsize, const bool is_rfc3686, + u32 *nonce, const u32 ctx1_iv_off, + const bool is_qi, int era); + +void cnstr_shdsc_tls_encap(u32 *const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int assoclen, + unsigned int ivsize, unsigned int authsize, + unsigned int blocksize, int era); + +void cnstr_shdsc_tls_decap(u32 *const desc, struct alginfo *cdata, + struct alginfo *adata, unsigned int assoclen, + unsigned int ivsize, unsigned int authsize, + unsigned int blocksize, int era); + +void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, unsigned int icvsize, + const bool is_qi); + +void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off); + +void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off); + +void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, + unsigned int ivsize, const bool is_rfc3686, + const u32 ctx1_iv_off); + +void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata); + +void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata); + +#endif /* _CAAMALG_DESC_H_ */ --- /dev/null +++ b/drivers/crypto/caam/caamalg_qi.c @@ -0,0 +1,2929 @@ +/* + * Freescale FSL CAAM support for crypto API over QI backend. + * Based on caamalg.c + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + */ + +#include "compat.h" +#include "ctrl.h" +#include "regs.h" +#include "intern.h" +#include "desc_constr.h" +#include "error.h" +#include "sg_sw_qm.h" +#include "key_gen.h" +#include "qi.h" +#include "jr.h" +#include "caamalg_desc.h" + +/* + * crypto alg + */ +#define CAAM_CRA_PRIORITY 2000 +/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ +#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ + SHA512_DIGEST_SIZE * 2) + +#define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ + CAAM_MAX_KEY_SIZE) +#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) + +struct caam_alg_entry { + int class1_alg_type; + int class2_alg_type; + bool rfc3686; + bool geniv; +}; + +struct caam_aead_alg { + struct aead_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + +/* + * per-session context + */ +struct caam_ctx { + struct device *jrdev; + u32 sh_desc_enc[DESC_MAX_USED_LEN]; + u32 sh_desc_dec[DESC_MAX_USED_LEN]; + u32 sh_desc_givenc[DESC_MAX_USED_LEN]; + u8 key[CAAM_MAX_KEY_SIZE]; + dma_addr_t key_dma; + struct alginfo adata; + struct alginfo cdata; + unsigned int authsize; + struct device *qidev; + spinlock_t lock; /* Protects multiple init of driver context */ + struct caam_drv_ctx *drv_ctx[NUM_OP]; +}; + +static int aead_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + u32 ctx1_iv_off = 0; + u32 *nonce = NULL; + unsigned int data_len[2]; + u32 inl_mask; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); + } + + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + if (alg->caam.geniv) + goto skip_enc; + + /* aead_encrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true, ctrlpriv->era); + +skip_enc: + /* aead_decrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, alg->caam.geniv, + is_rfc3686, nonce, ctx1_iv_off, true, + ctrlpriv->era); + + if (!alg->caam.geniv) + goto skip_givenc; + + /* aead_givencrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true, ctrlpriv->era); + +skip_givenc: + return 0; +} + +static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + aead_set_sh_desc(authenc); + + return 0; +} + +static int aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + struct crypto_authenc_keys keys; + int ret = 0; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + +#ifdef DEBUG + dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, DMA_TO_DEVICE); + goto skip_split_key; + } + + ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, + keys.authkeylen, CAAM_MAX_KEY_SIZE - + keys.enckeylen); + if (ret) + goto badkey; + + /* postpend encryption key to auth split key */ + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, DMA_TO_DEVICE); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); +#endif + +skip_split_key: + ctx->cdata.keylen = keys.enckeylen; + + ret = aead_set_sh_desc(aead); + if (ret) + goto badkey; + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + return ret; +badkey: + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int tls_set_sh_desc(struct crypto_aead *tls) +{ + struct caam_ctx *ctx = crypto_aead_ctx(tls); + unsigned int ivsize = crypto_aead_ivsize(tls); + unsigned int blocksize = crypto_aead_blocksize(tls); + unsigned int assoclen = 13; /* always 13 bytes for TLS */ + unsigned int data_len[2]; + u32 inl_mask; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * TLS 1.0 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + if (desc_inline_query(DESC_TLS10_ENC_LEN, DESC_JOB_IO_LEN, data_len, + &inl_mask, ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + cnstr_shdsc_tls_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, + assoclen, ivsize, ctx->authsize, blocksize, + ctrlpriv->era); + + /* + * TLS 1.0 decrypt shared descriptor + * Keys do not fit inline, regardless of algorithms used + */ + ctx->adata.key_inline = false; + ctx->adata.key_dma = ctx->key_dma; + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + cnstr_shdsc_tls_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, + assoclen, ivsize, ctx->authsize, blocksize, + ctrlpriv->era); + + return 0; +} + +static int tls_setauthsize(struct crypto_aead *tls, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(tls); + + ctx->authsize = authsize; + tls_set_sh_desc(tls); + + return 0; +} + +static int tls_setkey(struct crypto_aead *tls, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(tls); + struct device *jrdev = ctx->jrdev; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); + struct crypto_authenc_keys keys; + int ret = 0; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + +#ifdef DEBUG + dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, + keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, + ctx->adata.keylen_pad + + keys.enckeylen, DMA_TO_DEVICE); + goto skip_split_key; + } + + ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, + keys.authkeylen, CAAM_MAX_KEY_SIZE - + keys.enckeylen); + if (ret) + goto badkey; + + /* postpend encryption key to auth split key */ + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, DMA_TO_DEVICE); + +#ifdef DEBUG + dev_err(jrdev, "split keylen %d split keylen padded %d\n", + ctx->adata.keylen, ctx->adata.keylen_pad); + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); +#endif + +skip_split_key: + ctx->cdata.keylen = keys.enckeylen; + + ret = tls_set_sh_desc(tls); + if (ret) + goto badkey; + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + return ret; +badkey: + crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); + const char *alg_name = crypto_tfm_alg_name(tfm); + struct device *jrdev = ctx->jrdev; + unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + u32 ctx1_iv_off = 0; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); + int ret = 0; + + memcpy(ctx->key, key, keylen); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} + * | *key = {KEY, NONCE} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + keylen -= CTR_RFC3686_NONCE_SIZE; + } + + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_inline = true; + + /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */ + cnstr_shdsc_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + cnstr_shdsc_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + cnstr_shdsc_ablkcipher_givencap(ctx->sh_desc_givenc, &ctx->cdata, + ivsize, is_rfc3686, ctx1_iv_off); + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[GIVENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[GIVENCRYPT], + ctx->sh_desc_givenc); + if (ret) { + dev_err(jrdev, "driver givenc context update failed\n"); + goto badkey; + } + } + + return ret; +badkey: + crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *jrdev = ctx->jrdev; + int ret = 0; + + if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + crypto_ablkcipher_set_flags(ablkcipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + dev_err(jrdev, "key size mismatch\n"); + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = ctx->key; + ctx->cdata.key_inline = true; + + /* xts ablkcipher encrypt, decrypt shared descriptors */ + cnstr_shdsc_xts_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata); + cnstr_shdsc_xts_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata); + + /* Now update the driver contexts with the new shared descriptor */ + if (ctx->drv_ctx[ENCRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], + ctx->sh_desc_enc); + if (ret) { + dev_err(jrdev, "driver enc context update failed\n"); + goto badkey; + } + } + + if (ctx->drv_ctx[DECRYPT]) { + ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], + ctx->sh_desc_dec); + if (ret) { + dev_err(jrdev, "driver dec context update failed\n"); + goto badkey; + } + } + + return ret; +badkey: + crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return 0; +} + +/* + * aead_edesc - s/w-extended aead descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @assoclen: associated data length, in CAAM endianness + * @assoclen_dma: bus physical mapped address of req->assoclen + * @drv_req: driver-specific request structure + * @sgt: the h/w link table + */ +struct aead_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + unsigned int assoclen; + dma_addr_t assoclen_dma; + struct caam_drv_req drv_req; +#define CAAM_QI_MAX_AEAD_SG \ + ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) / \ + sizeof(struct qm_sg_entry)) + struct qm_sg_entry sgt[0]; +}; + +/* + * tls_edesc - s/w-extended tls descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @tmp: array of scatterlists used by 'scatterwalk_ffwd' + * @qm_sg_dma: bus physical mapped address of h/w link table + * @drv_req: driver-specific request structure + * @sgt: the h/w link table + */ +struct tls_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + struct scatterlist tmp[2]; + struct scatterlist *dst; + struct caam_drv_req drv_req; + struct qm_sg_entry sgt[0]; +}; + +/* + * ablkcipher_edesc - s/w-extended ablkcipher descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @drv_req: driver-specific request structure + * @sgt: the h/w link table + */ +struct ablkcipher_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + struct caam_drv_req drv_req; +#define CAAM_QI_MAX_ABLKCIPHER_SG \ + ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \ + sizeof(struct qm_sg_entry)) + struct qm_sg_entry sgt[0]; +}; + +static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, + enum optype type) +{ + /* + * This function is called on the fast path with values of 'type' + * known at compile time. Invalid arguments are not expected and + * thus no checks are made. + */ + struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; + u32 *desc; + + if (unlikely(!drv_ctx)) { + spin_lock(&ctx->lock); + + /* Read again to check if some other core init drv_ctx */ + drv_ctx = ctx->drv_ctx[type]; + if (!drv_ctx) { + int cpu; + + if (type == ENCRYPT) + desc = ctx->sh_desc_enc; + else if (type == DECRYPT) + desc = ctx->sh_desc_dec; + else /* (type == GIVENCRYPT) */ + desc = ctx->sh_desc_givenc; + + cpu = smp_processor_id(); + drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); + if (likely(!IS_ERR_OR_NULL(drv_ctx))) + drv_ctx->op_type = type; + + ctx->drv_ctx[type] = drv_ctx; + } + + spin_unlock(&ctx->lock); + } + + return drv_ctx; +} + +static void caam_unmap(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, int src_nents, + int dst_nents, dma_addr_t iv_dma, int ivsize, + enum optype op_type, dma_addr_t qm_sg_dma, + int qm_sg_bytes) +{ + if (dst != src) { + if (src_nents) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); + } else { + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } + + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, + op_type == GIVENCRYPT ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); + if (qm_sg_bytes) + dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); +} + +static void aead_unmap(struct device *dev, + struct aead_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(aead); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, + edesc->qm_sg_dma, edesc->qm_sg_bytes); + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); +} + +static void tls_unmap(struct device *dev, + struct tls_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(aead); + + caam_unmap(dev, req->src, edesc->dst, edesc->src_nents, + edesc->dst_nents, edesc->iv_dma, ivsize, + edesc->drv_req.drv_ctx->op_type, edesc->qm_sg_dma, + edesc->qm_sg_bytes); +} + +static void ablkcipher_unmap(struct device *dev, + struct ablkcipher_edesc *edesc, + struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, + edesc->qm_sg_dma, edesc->qm_sg_bytes); +} + +static void aead_done(struct caam_drv_req *drv_req, u32 status) +{ + struct device *qidev; + struct aead_edesc *edesc; + struct aead_request *aead_req = drv_req->app_ctx; + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); + int ecode = 0; + + qidev = caam_ctx->qidev; + + if (unlikely(status)) { + caam_jr_strstatus(qidev, status); + ecode = -EIO; + } + + edesc = container_of(drv_req, typeof(*edesc), drv_req); + aead_unmap(qidev, edesc, aead_req); + + aead_request_complete(aead_req, ecode); + qi_cache_free(edesc); +} + +/* + * allocate and map the aead extended descriptor + */ +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, + bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct aead_edesc *edesc; + dma_addr_t qm_sg_dma, iv_dma = 0; + int ivsize = 0; + unsigned int authsize = ctx->authsize; + int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; + int in_len, out_len; + struct qm_sg_entry *sg_table, *fd_sgt; + struct caam_drv_ctx *drv_ctx; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + + drv_ctx = get_drv_ctx(ctx, op_type); + if (unlikely(IS_ERR_OR_NULL(drv_ctx))) + return (struct aead_edesc *)drv_ctx; + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = qi_cache_alloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(qidev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + if (likely(req->src == req->dst)) { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : 0)); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + dst_nents = sg_nents_for_len(req->dst, req->assoclen + + req->cryptlen + + (encrypt ? authsize : + (-authsize))); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : (-authsize))); + qi_cache_free(edesc); + return ERR_PTR(dst_nents); + } + + if (src_nents) { + mapped_src_nents = dma_map_sg(qidev, req->src, + src_nents, DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) { + ivsize = crypto_aead_ivsize(aead); + iv_dma = dma_map_single(qidev, req->iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, + dst_nents, 0, 0, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + /* + * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. + * Input is not contiguous. + */ + qm_sg_ents = 1 + !!ivsize + mapped_src_nents + + (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) { + dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_AEAD_SG); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + sg_table = &edesc->sgt[0]; + qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = aead_done; + edesc->drv_req.drv_ctx = drv_ctx; + + edesc->assoclen = cpu_to_caam32(req->assoclen); + edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->assoclen_dma)) { + dev_err(qidev, "unable to map assoclen\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); + qm_sg_index++; + if (ivsize) { + dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); + qm_sg_index++; + } + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + qm_sg_index += mapped_src_nents; + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + qm_sg_index, 0); + + qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->qm_sg_dma = qm_sg_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + out_len = req->assoclen + req->cryptlen + + (encrypt ? ctx->authsize : (-ctx->authsize)); + in_len = 4 + ivsize + req->assoclen + req->cryptlen; + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); + + if (req->dst == req->src) { + if (mapped_src_nents == 1) + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), + out_len, 0); + else + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + + (1 + !!ivsize) * sizeof(*sg_table), + out_len, 0); + } else if (mapped_dst_nents == 1) { + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, + 0); + } else { + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * + qm_sg_index, out_len, 0); + } + + return edesc; +} + +static inline int aead_crypt(struct aead_request *req, bool encrypt) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, encrypt); + if (IS_ERR_OR_NULL(edesc)) + return PTR_ERR(edesc); + + /* Create and submit job descriptor */ + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + aead_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int aead_encrypt(struct aead_request *req) +{ + return aead_crypt(req, true); +} + +static int aead_decrypt(struct aead_request *req) +{ + return aead_crypt(req, false); +} + +static void tls_done(struct caam_drv_req *drv_req, u32 status) +{ + struct device *qidev; + struct tls_edesc *edesc; + struct aead_request *aead_req = drv_req->app_ctx; + struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); + struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); + int ecode = 0; + + qidev = caam_ctx->qidev; + + if (unlikely(status)) { + caam_jr_strstatus(qidev, status); + ecode = -EIO; + } + + edesc = container_of(drv_req, typeof(*edesc), drv_req); + tls_unmap(qidev, edesc, aead_req); + + aead_request_complete(aead_req, ecode); + qi_cache_free(edesc); +} + +/* + * allocate and map the tls extended descriptor + */ +static struct tls_edesc *tls_edesc_alloc(struct aead_request *req, bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + unsigned int blocksize = crypto_aead_blocksize(aead); + unsigned int padsize, authsize; + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct tls_edesc *edesc; + dma_addr_t qm_sg_dma, iv_dma = 0; + int ivsize = 0; + int qm_sg_index, qm_sg_ents = 0, qm_sg_bytes; + int in_len, out_len; + struct qm_sg_entry *sg_table, *fd_sgt; + struct caam_drv_ctx *drv_ctx; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + struct scatterlist *dst; + + if (encrypt) { + padsize = blocksize - ((req->cryptlen + ctx->authsize) % + blocksize); + authsize = ctx->authsize + padsize; + } else { + authsize = ctx->authsize; + } + + drv_ctx = get_drv_ctx(ctx, op_type); + if (unlikely(IS_ERR_OR_NULL(drv_ctx))) + return (struct tls_edesc *)drv_ctx; + + /* allocate space for base edesc and hw desc commands, link tables */ + edesc = qi_cache_alloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(qidev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + if (likely(req->src == req->dst)) { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : 0)); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + dst = req->dst; + } else { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + dst = scatterwalk_ffwd(edesc->tmp, req->dst, req->assoclen); + dst_nents = sg_nents_for_len(dst, req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->cryptlen + + (encrypt ? authsize : 0)); + qi_cache_free(edesc); + return ERR_PTR(dst_nents); + } + + if (src_nents) { + mapped_src_nents = dma_map_sg(qidev, req->src, + src_nents, DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + mapped_dst_nents = dma_map_sg(qidev, dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + ivsize = crypto_aead_ivsize(aead); + iv_dma = dma_map_single(qidev, req->iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, dst, src_nents, dst_nents, 0, 0, + op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + /* + * Create S/G table: IV, src, dst. + * Input is not contiguous. + */ + qm_sg_ents = 1 + mapped_src_nents + + (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + sg_table = &edesc->sgt[0]; + qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->dst = dst; + edesc->iv_dma = iv_dma; + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = tls_done; + edesc->drv_req.drv_ctx = drv_ctx; + + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); + qm_sg_index = 1; + + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + qm_sg_index += mapped_src_nents; + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(dst, mapped_dst_nents, sg_table + + qm_sg_index, 0); + + qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + caam_unmap(qidev, req->src, dst, src_nents, dst_nents, iv_dma, + ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->qm_sg_dma = qm_sg_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + out_len = req->cryptlen + (encrypt ? authsize : 0); + in_len = ivsize + req->assoclen + req->cryptlen; + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + + dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); + + if (req->dst == req->src) + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + + (sg_nents_for_len(req->src, req->assoclen) + + 1) * sizeof(*sg_table), out_len, 0); + else if (mapped_dst_nents == 1) + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(dst), out_len, 0); + else + dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * + qm_sg_index, out_len, 0); + + return edesc; +} + +static int tls_crypt(struct aead_request *req, bool encrypt) +{ + struct tls_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + edesc = tls_edesc_alloc(req, encrypt); + if (IS_ERR_OR_NULL(edesc)) + return PTR_ERR(edesc); + + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + tls_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int tls_encrypt(struct aead_request *req) +{ + return tls_crypt(req, true); +} + +static int tls_decrypt(struct aead_request *req) +{ + return tls_crypt(req, false); +} + +static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) +{ + struct ablkcipher_edesc *edesc; + struct ablkcipher_request *req = drv_req->app_ctx; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *qidev = caam_ctx->qidev; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + +#ifdef DEBUG + dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); +#endif + + edesc = container_of(drv_req, typeof(*edesc), drv_req); + + if (status) + caam_jr_strstatus(qidev, status); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->info, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); +#endif + + ablkcipher_unmap(qidev, edesc, req); + qi_cache_free(edesc); + + /* + * The crypto API expects us to set the IV (req->info) to the last + * ciphertext block. This is used e.g. by the CTS mode. + */ + scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, + ivsize, 0); + + ablkcipher_request_complete(req, status); +} + +static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request + *req, bool encrypt) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma; + bool in_contig; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + int dst_sg_idx, qm_sg_ents; + struct qm_sg_entry *sg_table, *fd_sgt; + struct caam_drv_ctx *drv_ctx; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + + drv_ctx = get_drv_ctx(ctx, op_type); + if (unlikely(IS_ERR_OR_NULL(drv_ctx))) + return (struct ablkcipher_edesc *)drv_ctx; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } + + if (unlikely(req->src != req->dst)) { + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } + + iv_dma = dma_map_single(qidev, req->info, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + if (mapped_src_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->src)) { + in_contig = true; + qm_sg_ents = 0; + } else { + in_contig = false; + qm_sg_ents = 1 + mapped_src_nents; + } + dst_sg_idx = qm_sg_ents; + + qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { + dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + return ERR_PTR(-ENOMEM); + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_alloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(qidev, "could not allocate extended descriptor\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + sg_table = &edesc->sgt[0]; + edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = ablkcipher_done; + edesc->drv_req.drv_ctx = drv_ctx; + + if (!in_contig) { + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + } + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + dst_sg_idx, 0); + + edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + + if (!in_contig) + dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, + ivsize + req->nbytes, 0); + else + dma_to_qm_sg_one_last(&fd_sgt[1], iv_dma, ivsize + req->nbytes, + 0); + + if (req->src == req->dst) { + if (!in_contig) + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + + sizeof(*sg_table), req->nbytes, 0); + else + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), + req->nbytes, 0); + } else if (mapped_dst_nents > 1) { + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table), req->nbytes, 0); + } else { + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), + req->nbytes, 0); + } + + return edesc; +} + +static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( + struct skcipher_givcrypt_request *creq) +{ + struct ablkcipher_request *req = &creq->creq; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *qidev = ctx->qidev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma; + bool out_contig; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + struct qm_sg_entry *sg_table, *fd_sgt; + int dst_sg_idx, qm_sg_ents; + struct caam_drv_ctx *drv_ctx; + + drv_ctx = get_drv_ctx(ctx, GIVENCRYPT); + if (unlikely(IS_ERR_OR_NULL(drv_ctx))) + return (struct ablkcipher_edesc *)drv_ctx; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } + + if (unlikely(req->src != req->dst)) { + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(qidev, "unable to map destination\n"); + dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(qidev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + dst_nents = src_nents; + mapped_dst_nents = src_nents; + } + + iv_dma = dma_map_single(qidev, creq->giv, ivsize, DMA_FROM_DEVICE); + if (dma_mapping_error(qidev, iv_dma)) { + dev_err(qidev, "unable to map IV\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; + dst_sg_idx = qm_sg_ents; + if (mapped_dst_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->dst)) { + out_contig = true; + } else { + out_contig = false; + qm_sg_ents += 1 + mapped_dst_nents; + } + + if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { + dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + return ERR_PTR(-ENOMEM); + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_alloc(GFP_DMA | flags); + if (!edesc) { + dev_err(qidev, "could not allocate extended descriptor\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + sg_table = &edesc->sgt[0]; + edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + edesc->drv_req.app_ctx = req; + edesc->drv_req.cbk = ablkcipher_done; + edesc->drv_req.drv_ctx = drv_ctx; + + if (mapped_src_nents > 1) + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); + + if (!out_contig) { + dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + dst_sg_idx + 1, 0); + } + + edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { + dev_err(qidev, "unable to map S/G table\n"); + caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + fd_sgt = &edesc->drv_req.fd_sgt[0]; + + if (mapped_src_nents > 1) + dma_to_qm_sg_one_ext(&fd_sgt[1], edesc->qm_sg_dma, req->nbytes, + 0); + else + dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src), + req->nbytes, 0); + + if (!out_contig) + dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table), ivsize + req->nbytes, + 0); + else + dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), + ivsize + req->nbytes, 0); + + return edesc; +} + +static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt) +{ + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = ablkcipher_edesc_alloc(req, encrypt); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + ablkcipher_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int ablkcipher_encrypt(struct ablkcipher_request *req) +{ + return ablkcipher_crypt(req, true); +} + +static int ablkcipher_decrypt(struct ablkcipher_request *req) +{ + return ablkcipher_crypt(req, false); +} + +static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) +{ + struct ablkcipher_request *req = &creq->creq; + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + int ret; + + if (unlikely(caam_congested)) + return -EAGAIN; + + /* allocate extended descriptor */ + edesc = ablkcipher_giv_edesc_alloc(creq); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); + if (!ret) { + ret = -EINPROGRESS; + } else { + ablkcipher_unmap(ctx->qidev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +#define template_ablkcipher template_u.ablkcipher +struct caam_alg_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + u32 type; + union { + struct ablkcipher_alg ablkcipher; + } template_u; + u32 class1_alg_type; + u32 class2_alg_type; +}; + +static struct caam_alg_template driver_algs[] = { + /* ablkcipher descriptor */ + { + .name = "cbc(aes)", + .driver_name = "cbc-aes-caam-qi", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des3_ede)", + .driver_name = "cbc-3des-caam-qi", + .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des)", + .driver_name = "cbc-des-caam-qi", + .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .name = "ctr(aes)", + .driver_name = "ctr-aes-caam-qi", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "chainiv", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-caam-qi", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "xts(aes)", + .driver_name = "xts-aes-caam-qi", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = xts_ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "eseqiv", + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, + }, +}; + +static struct caam_aead_alg driver_aeads[] = { + /* single-pass ipsec_esp descriptor */ + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-" + "caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam-qi", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-" + "caam-qi", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "tls10(hmac(sha1),cbc(aes))", + .cra_driver_name = "tls10-hmac-sha1-cbc-aes-caam-qi", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = tls_setkey, + .setauthsize = tls_setauthsize, + .encrypt = tls_encrypt, + .decrypt = tls_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + } + } +}; + +struct caam_crypto_alg { + struct list_head entry; + struct crypto_alg crypto_alg; + struct caam_alg_entry caam; +}; + +static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) +{ + struct caam_drv_private *priv; + /* Digest sizes for MD5, SHA1, SHA-224, SHA-256, SHA-384, SHA-512 */ + static const u8 digest_size[] = { + MD5_DIGEST_SIZE, + SHA1_DIGEST_SIZE, + SHA224_DIGEST_SIZE, + SHA256_DIGEST_SIZE, + SHA384_DIGEST_SIZE, + SHA512_DIGEST_SIZE + }; + u8 op_id; + + /* + * distribute tfms across job rings to ensure in-order + * crypto request processing per tfm + */ + ctx->jrdev = caam_jr_alloc(); + if (IS_ERR(ctx->jrdev)) { + pr_err("Job Ring Device allocation for transform failed\n"); + return PTR_ERR(ctx->jrdev); + } + + ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { + dev_err(ctx->jrdev, "unable to map key\n"); + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + + /* copy descriptor header template value */ + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; + + if (ctx->adata.algtype) { + op_id = (ctx->adata.algtype & OP_ALG_ALGSEL_SUBMASK) + >> OP_ALG_ALGSEL_SHIFT; + if (op_id < ARRAY_SIZE(digest_size)) { + ctx->authsize = digest_size[op_id]; + } else { + dev_err(ctx->jrdev, + "incorrect op_id %d; must be less than %zu\n", + op_id, ARRAY_SIZE(digest_size)); + caam_jr_free(ctx->jrdev); + return -EINVAL; + } + } else { + ctx->authsize = 0; + } + + priv = dev_get_drvdata(ctx->jrdev->parent); + ctx->qidev = priv->qidev; + + spin_lock_init(&ctx->lock); + ctx->drv_ctx[ENCRYPT] = NULL; + ctx->drv_ctx[DECRYPT] = NULL; + ctx->drv_ctx[GIVENCRYPT] = NULL; + + return 0; +} + +static int caam_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), + crypto_alg); + struct caam_ctx *ctx = crypto_tfm_ctx(tfm); + + return caam_init_common(ctx, &caam_alg->caam); +} + +static int caam_aead_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), + aead); + struct caam_ctx *ctx = crypto_aead_ctx(tfm); + + return caam_init_common(ctx, &caam_alg->caam); +} + +static void caam_exit_common(struct caam_ctx *ctx) +{ + caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); + caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); + caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]); + + dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), + DMA_TO_DEVICE); + + caam_jr_free(ctx->jrdev); +} + +static void caam_cra_exit(struct crypto_tfm *tfm) +{ + caam_exit_common(crypto_tfm_ctx(tfm)); +} + +static void caam_aead_exit(struct crypto_aead *tfm) +{ + caam_exit_common(crypto_aead_ctx(tfm)); +} + +static struct list_head alg_list; +static void __exit caam_qi_algapi_exit(void) +{ + struct caam_crypto_alg *t_alg, *n; + int i; + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + + if (t_alg->registered) + crypto_unregister_aead(&t_alg->aead); + } + + if (!alg_list.next) + return; + + list_for_each_entry_safe(t_alg, n, &alg_list, entry) { + crypto_unregister_alg(&t_alg->crypto_alg); + list_del(&t_alg->entry); + kfree(t_alg); + } +} + +static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template + *template) +{ + struct caam_crypto_alg *t_alg; + struct crypto_alg *alg; + + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + alg = &t_alg->crypto_alg; + + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + template->driver_name); + alg->cra_module = THIS_MODULE; + alg->cra_init = caam_cra_init; + alg->cra_exit = caam_cra_exit; + alg->cra_priority = CAAM_CRA_PRIORITY; + alg->cra_blocksize = template->blocksize; + alg->cra_alignmask = 0; + alg->cra_ctxsize = sizeof(struct caam_ctx); + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | + template->type; + switch (template->type) { + case CRYPTO_ALG_TYPE_GIVCIPHER: + alg->cra_type = &crypto_givcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; + case CRYPTO_ALG_TYPE_ABLKCIPHER: + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; + } + + t_alg->caam.class1_alg_type = template->class1_alg_type; + t_alg->caam.class2_alg_type = template->class2_alg_type; + + return t_alg; +} + +static void caam_aead_alg_init(struct caam_aead_alg *t_alg) +{ + struct aead_alg *alg = &t_alg->aead; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + + alg->init = caam_aead_init; + alg->exit = caam_aead_exit; +} + +static int __init caam_qi_algapi_init(void) +{ + struct device_node *dev_node; + struct platform_device *pdev; + struct device *ctrldev; + struct caam_drv_private *priv; + int i = 0, err = 0; + u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; + unsigned int md_limit = SHA512_DIGEST_SIZE; + bool registered = false; + + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); + if (!dev_node) { + dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); + if (!dev_node) + return -ENODEV; + } + + pdev = of_find_device_by_node(dev_node); + of_node_put(dev_node); + if (!pdev) + return -ENODEV; + + ctrldev = &pdev->dev; + priv = dev_get_drvdata(ctrldev); + + /* + * If priv is NULL, it's probably because the caam driver wasn't + * properly initialized (e.g. RNG4 init failed). Thus, bail out here. + */ + if (!priv || !priv->qi_present) + return -ENODEV; + + if (caam_dpaa2) { + dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); + return -ENODEV; + } + + INIT_LIST_HEAD(&alg_list); + + /* + * Register crypto algorithms the device supports. + * First, detect presence and attributes of DES, AES, and MD blocks. + */ + cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); + cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); + des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; + aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; + md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; + + /* If MD is present, limit digest size based on LP256 */ + if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) + md_limit = SHA256_DIGEST_SIZE; + + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_crypto_alg *t_alg; + struct caam_alg_template *alg = driver_algs + i; + u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((alg_sel == OP_ALG_ALGSEL_3DES) || + (alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + t_alg = caam_alg_alloc(alg); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + dev_warn(priv->qidev, "%s alg allocation failed\n", + alg->driver_name); + continue; + } + + err = crypto_register_alg(&t_alg->crypto_alg); + if (err) { + dev_warn(priv->qidev, "%s alg registration failed\n", + t_alg->crypto_alg.cra_driver_name); + kfree(t_alg); + continue; + } + + list_add_tail(&t_alg->entry, &alg_list); + registered = true; + } + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!des_inst && + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || + (c1_alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* + * Check support for AES algorithms not available + * on LP devices. + */ + if (((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) && + (alg_aai == OP_ALG_AAI_GCM)) + continue; + + /* + * Skip algorithms requiring message digests + * if MD or MD size is not supported by device. + */ + if (c2_alg_sel && + (!md_inst || (t_alg->aead.maxauthsize > md_limit))) + continue; + + caam_aead_alg_init(t_alg); + + err = crypto_register_aead(&t_alg->aead); + if (err) { + pr_warn("%s alg registration failed\n", + t_alg->aead.base.cra_driver_name); + continue; + } + + t_alg->registered = true; + registered = true; + } + + if (registered) + dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); + + return err; +} + +module_init(caam_qi_algapi_init); +module_exit(caam_qi_algapi_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); +MODULE_AUTHOR("Freescale Semiconductor"); --- /dev/null +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -0,0 +1,5920 @@ +/* + * Copyright 2015-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "compat.h" +#include "regs.h" +#include "caamalg_qi2.h" +#include "dpseci_cmd.h" +#include "desc_constr.h" +#include "error.h" +#include "sg_sw_sec4.h" +#include "sg_sw_qm2.h" +#include "key_gen.h" +#include "caamalg_desc.h" +#include "caamhash_desc.h" +#include "../../../drivers/staging/fsl-mc/include/mc.h" +#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h" +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" + +#define CAAM_CRA_PRIORITY 2000 + +/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ +#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ + SHA512_DIGEST_SIZE * 2) + +#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM +bool caam_little_end; +EXPORT_SYMBOL(caam_little_end); +bool caam_imx; +EXPORT_SYMBOL(caam_imx); +#endif + +/* + * This is a a cache of buffers, from which the users of CAAM QI driver + * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. + * NOTE: A more elegant solution would be to have some headroom in the frames + * being processed. This can be added by the dpaa2-eth driver. This would + * pose a problem for userspace application processing which cannot + * know of this limitation. So for now, this will work. + * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here + */ +static struct kmem_cache *qi_cache; + +struct caam_alg_entry { + struct device *dev; + int class1_alg_type; + int class2_alg_type; + bool rfc3686; + bool geniv; +}; + +struct caam_aead_alg { + struct aead_alg aead; + struct caam_alg_entry caam; + bool registered; +}; + +/** + * caam_ctx - per-session context + * @flc: Flow Contexts array + * @key: virtual address of the key(s): [authentication key], encryption key + * @flc_dma: I/O virtual addresses of the Flow Contexts + * @key_dma: I/O virtual address of the key + * @dev: dpseci device + * @adata: authentication algorithm details + * @cdata: encryption algorithm details + * @authsize: authentication tag (a.k.a. ICV / MAC) size + */ +struct caam_ctx { + struct caam_flc flc[NUM_OP]; + u8 key[CAAM_MAX_KEY_SIZE]; + dma_addr_t flc_dma[NUM_OP]; + dma_addr_t key_dma; + struct device *dev; + struct alginfo adata; + struct alginfo cdata; + unsigned int authsize; +}; + +void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, + dma_addr_t iova_addr) +{ + phys_addr_t phys_addr; + + phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : + iova_addr; + + return phys_to_virt(phys_addr); +} + +/* + * qi_cache_zalloc - Allocate buffers from CAAM-QI cache + * + * Allocate data on the hotpath. Instead of using kzalloc, one can use the + * services of the CAAM QI memory cache (backed by kmem_cache). The buffers + * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for + * hosting 16 SG entries. + * + * @flags - flags that would be used for the equivalent kmalloc(..) call + * + * Returns a pointer to a retrieved buffer on success or NULL on failure. + */ +static inline void *qi_cache_zalloc(gfp_t flags) +{ + return kmem_cache_zalloc(qi_cache, flags); +} + +/* + * qi_cache_free - Frees buffers allocated from CAAM-QI cache + * + * @obj - buffer previously allocated by qi_cache_zalloc + * + * No checking is being done, the call is a passthrough call to + * kmem_cache_free(...) + */ +static inline void qi_cache_free(void *obj) +{ + kmem_cache_free(qi_cache, obj); +} + +static struct caam_request *to_caam_req(struct crypto_async_request *areq) +{ + switch (crypto_tfm_alg_type(areq->tfm)) { + case CRYPTO_ALG_TYPE_ABLKCIPHER: + case CRYPTO_ALG_TYPE_GIVCIPHER: + return ablkcipher_request_ctx(ablkcipher_request_cast(areq)); + case CRYPTO_ALG_TYPE_AEAD: + return aead_request_ctx(container_of(areq, struct aead_request, + base)); + case CRYPTO_ALG_TYPE_AHASH: + return ahash_request_ctx(ahash_request_cast(areq)); + default: + return ERR_PTR(-EINVAL); + } +} + +static void caam_unmap(struct device *dev, struct scatterlist *src, + struct scatterlist *dst, int src_nents, + int dst_nents, dma_addr_t iv_dma, int ivsize, + enum optype op_type, dma_addr_t qm_sg_dma, + int qm_sg_bytes) +{ + if (dst != src) { + if (src_nents) + dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); + } else { + dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); + } + + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, + op_type == GIVENCRYPT ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); + + if (qm_sg_bytes) + dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); +} + +static int aead_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + unsigned int ivsize = crypto_aead_ivsize(aead); + struct device *dev = ctx->dev; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); + struct caam_flc *flc; + u32 *desc; + u32 ctx1_iv_off = 0; + u32 *nonce = NULL; + unsigned int data_len[2]; + u32 inl_mask; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = alg->caam.rfc3686; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * CONTEXT1[255:128] = {NONCE, IV, COUNTER} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + + ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); + } + + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + /* aead_encrypt shared descriptor */ + if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : + DESC_QI_AEAD_ENC_LEN) + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + + if (alg->caam.geniv) + cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, + nonce, ctx1_iv_off, true, + priv->sec_attr.era); + else + cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, is_rfc3686, nonce, + ctx1_iv_off, true, priv->sec_attr.era); + + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* aead_decrypt shared descriptor */ + if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + + (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), + DESC_JOB_IO_LEN, data_len, &inl_mask, + ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, + ivsize, ctx->authsize, alg->caam.geniv, + is_rfc3686, nonce, ctx1_iv_off, true, + priv->sec_attr.era); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + return 0; +} + +static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + aead_set_sh_desc(authenc); + + return 0; +} + +struct split_key_sh_result { + struct completion completion; + int err; + struct device *dev; +}; + +static void split_key_sh_done(void *cbk_ctx, u32 err) +{ + struct split_key_sh_result *res = cbk_ctx; + +#ifdef DEBUG + dev_err(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); +#endif + + if (err) + caam_qi2_strstatus(res->dev, err); + + res->err = err; + complete(&res->completion); +} + +static int aead_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + +#ifdef DEBUG + dev_err(dev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, DMA_BIDIRECTIONAL); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); +#endif + + ctx->cdata.keylen = keys.enckeylen; + + return aead_set_sh_desc(aead); +badkey: + crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, + bool encrypt) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_request *req_ctx = aead_request_ctx(req); + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), + typeof(*alg), aead); + struct device *dev = ctx->dev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct aead_edesc *edesc; + dma_addr_t qm_sg_dma, iv_dma = 0; + int ivsize = 0; + unsigned int authsize = ctx->authsize; + int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; + int in_len, out_len; + struct dpaa2_sg_entry *sg_table; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(dev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + if (unlikely(req->dst != req->src)) { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + dst_nents = sg_nents_for_len(req->dst, req->assoclen + + req->cryptlen + + (encrypt ? authsize : + (-authsize))); + if (unlikely(dst_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : (-authsize))); + qi_cache_free(edesc); + return ERR_PTR(dst_nents); + } + + if (src_nents) { + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(dev, "unable to map destination\n"); + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : 0)); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) { + ivsize = crypto_aead_ivsize(aead); + iv_dma = dma_map_single(dev, req->iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(dev, iv_dma)) { + dev_err(dev, "unable to map IV\n"); + caam_unmap(dev, req->src, req->dst, src_nents, + dst_nents, 0, 0, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + /* + * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. + * Input is not contiguous. + */ + qm_sg_nents = 1 + !!ivsize + mapped_src_nents + + (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + if (unlikely(qm_sg_nents > CAAM_QI_MAX_AEAD_SG)) { + dev_err(dev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_nents, CAAM_QI_MAX_AEAD_SG); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + sg_table = &edesc->sgt[0]; + qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + + edesc->assoclen_dma = dma_map_single(dev, &req->assoclen, 4, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->assoclen_dma)) { + dev_err(dev, "unable to map assoclen\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); + qm_sg_index++; + if (ivsize) { + dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); + qm_sg_index++; + } + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + qm_sg_index += mapped_src_nents; + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + qm_sg_index, 0); + + qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(dev, qm_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->qm_sg_dma = qm_sg_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + out_len = req->assoclen + req->cryptlen + + (encrypt ? ctx->authsize : (-ctx->authsize)); + in_len = 4 + ivsize + req->assoclen + req->cryptlen; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, qm_sg_dma); + dpaa2_fl_set_len(in_fle, in_len); + + if (req->dst == req->src) { + if (mapped_src_nents == 1) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, qm_sg_dma + + (1 + !!ivsize) * sizeof(*sg_table)); + } + } else if (mapped_dst_nents == 1) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * + sizeof(*sg_table)); + } + + dpaa2_fl_set_len(out_fle, out_len); + + return edesc; +} + +static struct tls_edesc *tls_edesc_alloc(struct aead_request *req, + bool encrypt) +{ + struct crypto_aead *tls = crypto_aead_reqtfm(req); + unsigned int blocksize = crypto_aead_blocksize(tls); + unsigned int padsize, authsize; + struct caam_request *req_ctx = aead_request_ctx(req); + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + struct caam_ctx *ctx = crypto_aead_ctx(tls); + struct caam_aead_alg *alg = container_of(crypto_aead_alg(tls), + typeof(*alg), aead); + struct device *dev = ctx->dev; + gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct tls_edesc *edesc; + dma_addr_t qm_sg_dma, iv_dma = 0; + int ivsize = 0; + int qm_sg_index, qm_sg_ents = 0, qm_sg_bytes; + int in_len, out_len; + struct dpaa2_sg_entry *sg_table; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + struct scatterlist *dst; + + if (encrypt) { + padsize = blocksize - ((req->cryptlen + ctx->authsize) % + blocksize); + authsize = ctx->authsize + padsize; + } else { + authsize = ctx->authsize; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(dev, "could not allocate extended descriptor\n"); + return ERR_PTR(-ENOMEM); + } + + if (likely(req->src == req->dst)) { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen + + (encrypt ? authsize : 0)); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + dst = req->dst; + } else { + src_nents = sg_nents_for_len(req->src, req->assoclen + + req->cryptlen); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + req->assoclen + req->cryptlen); + qi_cache_free(edesc); + return ERR_PTR(src_nents); + } + + dst = scatterwalk_ffwd(edesc->tmp, req->dst, req->assoclen); + dst_nents = sg_nents_for_len(dst, req->cryptlen + + (encrypt ? authsize : 0)); + if (unlikely(dst_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", + req->cryptlen + + (encrypt ? authsize : 0)); + qi_cache_free(edesc); + return ERR_PTR(dst_nents); + } + + if (src_nents) { + mapped_src_nents = dma_map_sg(dev, req->src, + src_nents, DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = 0; + } + + mapped_dst_nents = dma_map_sg(dev, dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(dev, "unable to map destination\n"); + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + } + + ivsize = crypto_aead_ivsize(tls); + iv_dma = dma_map_single(dev, req->iv, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(dev, iv_dma)) { + dev_err(dev, "unable to map IV\n"); + caam_unmap(dev, req->src, dst, src_nents, dst_nents, 0, 0, + op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + /* + * Create S/G table: IV, src, dst. + * Input is not contiguous. + */ + qm_sg_ents = 1 + mapped_src_nents + + (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + sg_table = &edesc->sgt[0]; + qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->dst = dst; + edesc->iv_dma = iv_dma; + + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); + qm_sg_index = 1; + + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + qm_sg_index += mapped_src_nents; + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(dst, mapped_dst_nents, sg_table + + qm_sg_index, 0); + + qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(dev, qm_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + caam_unmap(dev, req->src, dst, src_nents, dst_nents, iv_dma, + ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + edesc->qm_sg_dma = qm_sg_dma; + edesc->qm_sg_bytes = qm_sg_bytes; + + out_len = req->cryptlen + (encrypt ? authsize : 0); + in_len = ivsize + req->assoclen + req->cryptlen; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, qm_sg_dma); + dpaa2_fl_set_len(in_fle, in_len); + + if (req->dst == req->src) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, qm_sg_dma + + (sg_nents_for_len(req->src, req->assoclen) + + 1) * sizeof(*sg_table)); + } else if (mapped_dst_nents == 1) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(dst)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * + sizeof(*sg_table)); + } + + dpaa2_fl_set_len(out_fle, out_len); + + return edesc; +} + +static int tls_set_sh_desc(struct crypto_aead *tls) +{ + struct caam_ctx *ctx = crypto_aead_ctx(tls); + unsigned int ivsize = crypto_aead_ivsize(tls); + unsigned int blocksize = crypto_aead_blocksize(tls); + struct device *dev = ctx->dev; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); + struct caam_flc *flc; + u32 *desc; + unsigned int assoclen = 13; /* always 13 bytes for TLS */ + unsigned int data_len[2]; + u32 inl_mask; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * TLS 1.0 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + data_len[0] = ctx->adata.keylen_pad; + data_len[1] = ctx->cdata.keylen; + + if (desc_inline_query(DESC_TLS10_ENC_LEN, DESC_JOB_IO_LEN, data_len, + &inl_mask, ARRAY_SIZE(data_len)) < 0) + return -EINVAL; + + if (inl_mask & 1) + ctx->adata.key_virt = ctx->key; + else + ctx->adata.key_dma = ctx->key_dma; + + if (inl_mask & 2) + ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; + else + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + ctx->adata.key_inline = !!(inl_mask & 1); + ctx->cdata.key_inline = !!(inl_mask & 2); + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_tls_encap(desc, &ctx->cdata, &ctx->adata, + assoclen, ivsize, ctx->authsize, blocksize, + priv->sec_attr.era); + flc->flc[1] = desc_len(desc); + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* + * TLS 1.0 decrypt shared descriptor + * Keys do not fit inline, regardless of algorithms used + */ + ctx->adata.key_inline = false; + ctx->adata.key_dma = ctx->key_dma; + ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_tls_decap(desc, &ctx->cdata, &ctx->adata, assoclen, ivsize, + ctx->authsize, blocksize, priv->sec_attr.era); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + return 0; +} + +static int tls_setkey(struct crypto_aead *tls, const u8 *key, + unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(tls); + struct device *dev = ctx->dev; + struct crypto_authenc_keys keys; + + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) + goto badkey; + +#ifdef DEBUG + dev_err(dev, "keylen %d enckeylen %d authkeylen %d\n", + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + ctx->adata.keylen = keys.authkeylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + + if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); + dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + + keys.enckeylen, DMA_BIDIRECTIONAL); +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); +#endif + + ctx->cdata.keylen = keys.enckeylen; + + return tls_set_sh_desc(tls); +badkey: + crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int tls_setauthsize(struct crypto_aead *tls, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(tls); + + ctx->authsize = authsize; + tls_set_sh_desc(tls); + + return 0; +} + +static int gcm_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_flc *flc; + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + /* + * AES GCM encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { + ctx->cdata.key_inline = true; + ctx->cdata.key_virt = ctx->key; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + return 0; +} + +static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + gcm_set_sh_desc(authenc); + + return 0; +} + +static int gcm_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + memcpy(ctx->key, key, keylen); + dma_sync_single_for_device(dev, ctx->key_dma, keylen, + DMA_BIDIRECTIONAL); + ctx->cdata.keylen = keylen; + + return gcm_set_sh_desc(aead); +} + +static int rfc4106_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_flc *flc; + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + ctx->cdata.key_virt = ctx->key; + + /* + * RFC4106 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + return 0; +} + +static int rfc4106_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + rfc4106_set_sh_desc(authenc); + + return 0; +} + +static int rfc4106_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + + if (keylen < 4) + return -EINVAL; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + memcpy(ctx->key, key, keylen); + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, + DMA_BIDIRECTIONAL); + + return rfc4106_set_sh_desc(aead); +} + +static int rfc4543_set_sh_desc(struct crypto_aead *aead) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + unsigned int ivsize = crypto_aead_ivsize(aead); + struct caam_flc *flc; + u32 *desc; + int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - + ctx->cdata.keylen; + + if (!ctx->cdata.keylen || !ctx->authsize) + return 0; + + ctx->cdata.key_virt = ctx->key; + + /* + * RFC4543 encrypt shared descriptor + * Job Descriptor and Shared Descriptor + * must fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { + ctx->cdata.key_inline = true; + } else { + ctx->cdata.key_inline = false; + ctx->cdata.key_dma = ctx->key_dma; + } + + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, + true); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + return 0; +} + +static int rfc4543_setauthsize(struct crypto_aead *authenc, + unsigned int authsize) +{ + struct caam_ctx *ctx = crypto_aead_ctx(authenc); + + ctx->authsize = authsize; + rfc4543_set_sh_desc(authenc); + + return 0; +} + +static int rfc4543_setkey(struct crypto_aead *aead, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *dev = ctx->dev; + + if (keylen < 4) + return -EINVAL; + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + + memcpy(ctx->key, key, keylen); + /* + * The last four bytes of the key material are used as the salt value + * in the nonce. Update the AES key length. + */ + ctx->cdata.keylen = keylen - 4; + dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, + DMA_BIDIRECTIONAL); + + return rfc4543_set_sh_desc(aead); +} + +static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); + const char *alg_name = crypto_tfm_alg_name(tfm); + struct device *dev = ctx->dev; + struct caam_flc *flc; + unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + u32 *desc; + u32 ctx1_iv_off = 0; + const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == + OP_ALG_AAI_CTR_MOD128); + const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); +#endif + /* + * AES-CTR needs to load IV in CONTEXT1 reg + * at an offset of 128bits (16bytes) + * CONTEXT1[255:128] = IV + */ + if (ctr_mode) + ctx1_iv_off = 16; + + /* + * RFC3686 specific: + * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} + * | *key = {KEY, NONCE} + */ + if (is_rfc3686) { + ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; + keylen -= CTR_RFC3686_NONCE_SIZE; + } + + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + + /* ablkcipher_encrypt shared descriptor */ + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* ablkcipher_decrypt shared descriptor */ + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, + is_rfc3686, ctx1_iv_off); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* ablkcipher_givencrypt shared descriptor */ + flc = &ctx->flc[GIVENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, + ivsize, is_rfc3686, ctx1_iv_off); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[GIVENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + return 0; +} + +static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, + const u8 *key, unsigned int keylen) +{ + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *dev = ctx->dev; + struct caam_flc *flc; + u32 *desc; + + if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { + dev_err(dev, "key size mismatch\n"); + crypto_ablkcipher_set_flags(ablkcipher, + CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + ctx->cdata.keylen = keylen; + ctx->cdata.key_virt = key; + ctx->cdata.key_inline = true; + + /* xts_ablkcipher_encrypt shared descriptor */ + flc = &ctx->flc[ENCRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + /* xts_ablkcipher_decrypt shared descriptor */ + flc = &ctx->flc[DECRYPT]; + desc = flc->sh_desc; + cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], + sizeof(flc->flc) + desc_bytes(desc), + DMA_BIDIRECTIONAL); + + return 0; +} + +static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request + *req, bool encrypt) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_request *req_ctx = ablkcipher_request_ctx(req); + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *dev = ctx->dev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma; + bool in_contig; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + int dst_sg_idx, qm_sg_ents; + struct dpaa2_sg_entry *sg_table; + enum optype op_type = encrypt ? ENCRYPT : DECRYPT; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } + + if (unlikely(req->dst != req->src)) { + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(dev, "unable to map destination\n"); + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + } + + iv_dma = dma_map_single(dev, req->info, ivsize, DMA_TO_DEVICE); + if (dma_mapping_error(dev, iv_dma)) { + dev_err(dev, "unable to map IV\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + if (mapped_src_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->src)) { + in_contig = true; + qm_sg_ents = 0; + } else { + in_contig = false; + qm_sg_ents = 1 + mapped_src_nents; + } + dst_sg_idx = qm_sg_ents; + + qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { + dev_err(dev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + return ERR_PTR(-ENOMEM); + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (unlikely(!edesc)) { + dev_err(dev, "could not allocate extended descriptor\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + sg_table = &edesc->sgt[0]; + edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + + if (!in_contig) { + dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + } + + if (mapped_dst_nents > 1) + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + dst_sg_idx, 0); + + edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->qm_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, op_type, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_len(in_fle, req->nbytes + ivsize); + dpaa2_fl_set_len(out_fle, req->nbytes); + + if (!in_contig) { + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + } else { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, iv_dma); + } + + if (req->src == req->dst) { + if (!in_contig) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + + sizeof(*sg_table)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); + } + } else if (mapped_dst_nents > 1) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); + } + + return edesc; +} + +static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( + struct skcipher_givcrypt_request *greq) +{ + struct ablkcipher_request *req = &greq->creq; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_request *req_ctx = ablkcipher_request_ctx(req); + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct device *dev = ctx->dev; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; + struct ablkcipher_edesc *edesc; + dma_addr_t iv_dma; + bool out_contig; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + struct dpaa2_sg_entry *sg_table; + int dst_sg_idx, qm_sg_ents; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (unlikely(src_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in src S/G\n", + req->nbytes); + return ERR_PTR(src_nents); + } + + if (unlikely(req->dst != req->src)) { + dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (unlikely(dst_nents < 0)) { + dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", + req->nbytes); + return ERR_PTR(dst_nents); + } + + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_TO_DEVICE); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, + DMA_FROM_DEVICE); + if (unlikely(!mapped_dst_nents)) { + dev_err(dev, "unable to map destination\n"); + dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + return ERR_PTR(-ENOMEM); + } + } else { + mapped_src_nents = dma_map_sg(dev, req->src, src_nents, + DMA_BIDIRECTIONAL); + if (unlikely(!mapped_src_nents)) { + dev_err(dev, "unable to map source\n"); + return ERR_PTR(-ENOMEM); + } + + dst_nents = src_nents; + mapped_dst_nents = src_nents; + } + + iv_dma = dma_map_single(dev, greq->giv, ivsize, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, iv_dma)) { + dev_err(dev, "unable to map IV\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, + 0, 0, 0, 0); + return ERR_PTR(-ENOMEM); + } + + qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; + dst_sg_idx = qm_sg_ents; + if (mapped_dst_nents == 1 && + iv_dma + ivsize == sg_dma_address(req->dst)) { + out_contig = true; + } else { + out_contig = false; + qm_sg_ents += 1 + mapped_dst_nents; + } + + if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { + dev_err(dev, "Insufficient S/G entries: %d > %lu\n", + qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + return ERR_PTR(-ENOMEM); + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) { + dev_err(dev, "could not allocate extended descriptor\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + return ERR_PTR(-ENOMEM); + } + + edesc->src_nents = src_nents; + edesc->dst_nents = dst_nents; + edesc->iv_dma = iv_dma; + sg_table = &edesc->sgt[0]; + edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); + + if (mapped_src_nents > 1) + sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); + + if (!out_contig) { + dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); + sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + + dst_sg_idx + 1, 0); + } + + edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, edesc->qm_sg_dma)) { + dev_err(dev, "unable to map S/G table\n"); + caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, + iv_dma, ivsize, GIVENCRYPT, 0, 0); + qi_cache_free(edesc); + return ERR_PTR(-ENOMEM); + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_len(in_fle, req->nbytes); + dpaa2_fl_set_len(out_fle, ivsize + req->nbytes); + + if (mapped_src_nents > 1) { + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + } else { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); + } + + if (!out_contig) { + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * + sizeof(*sg_table)); + } else { + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); + } + + return edesc; +} + +static void aead_unmap(struct device *dev, struct aead_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(aead); + struct caam_request *caam_req = aead_request_ctx(req); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, caam_req->op_type, + edesc->qm_sg_dma, edesc->qm_sg_bytes); + dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); +} + +static void tls_unmap(struct device *dev, struct tls_edesc *edesc, + struct aead_request *req) +{ + struct crypto_aead *tls = crypto_aead_reqtfm(req); + int ivsize = crypto_aead_ivsize(tls); + struct caam_request *caam_req = aead_request_ctx(req); + + caam_unmap(dev, req->src, edesc->dst, edesc->src_nents, + edesc->dst_nents, edesc->iv_dma, ivsize, caam_req->op_type, + edesc->qm_sg_dma, edesc->qm_sg_bytes); +} + +static void ablkcipher_unmap(struct device *dev, + struct ablkcipher_edesc *edesc, + struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + struct caam_request *caam_req = ablkcipher_request_ctx(req); + + caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, + edesc->iv_dma, ivsize, caam_req->op_type, + edesc->qm_sg_dma, edesc->qm_sg_bytes); +} + +static void aead_encrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct caam_request *req_ctx = to_caam_req(areq); + struct aead_edesc *edesc = req_ctx->edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ecode = 0; + +#ifdef DEBUG + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + ecode = -EIO; + } + + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + aead_request_complete(req, ecode); +} + +static void aead_decrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct caam_request *req_ctx = to_caam_req(areq); + struct aead_edesc *edesc = req_ctx->edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + int ecode = 0; + +#ifdef DEBUG + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + /* + * verify hw auth check passed else return -EBADMSG + */ + if ((status & JRSTA_CCBERR_ERRID_MASK) == + JRSTA_CCBERR_ERRID_ICVCHK) + ecode = -EBADMSG; + else + ecode = -EIO; + } + + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + aead_request_complete(req, ecode); +} + +static int aead_encrypt(struct aead_request *req) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct caam_request *caam_req = aead_request_ctx(req); + int ret; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, true); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[ENCRYPT]; + caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; + caam_req->op_type = ENCRYPT; + caam_req->cbk = aead_encrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int aead_decrypt(struct aead_request *req) +{ + struct aead_edesc *edesc; + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct caam_request *caam_req = aead_request_ctx(req); + int ret; + + /* allocate extended descriptor */ + edesc = aead_edesc_alloc(req, false); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[DECRYPT]; + caam_req->flc_dma = ctx->flc_dma[DECRYPT]; + caam_req->op_type = DECRYPT; + caam_req->cbk = aead_decrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + aead_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static void tls_encrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct caam_request *req_ctx = to_caam_req(areq); + struct tls_edesc *edesc = req_ctx->edesc; + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(tls); + int ecode = 0; + +#ifdef DEBUG + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + ecode = -EIO; + } + + tls_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + aead_request_complete(req, ecode); +} + +static void tls_decrypt_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct aead_request *req = container_of(areq, struct aead_request, + base); + struct caam_request *req_ctx = to_caam_req(areq); + struct tls_edesc *edesc = req_ctx->edesc; + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(tls); + int ecode = 0; + +#ifdef DEBUG + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + /* + * verify hw auth check passed else return -EBADMSG + */ + if ((status & JRSTA_CCBERR_ERRID_MASK) == + JRSTA_CCBERR_ERRID_ICVCHK) + ecode = -EBADMSG; + else + ecode = -EIO; + } + + tls_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + aead_request_complete(req, ecode); +} + +static int tls_encrypt(struct aead_request *req) +{ + struct tls_edesc *edesc; + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(tls); + struct caam_request *caam_req = aead_request_ctx(req); + int ret; + + /* allocate extended descriptor */ + edesc = tls_edesc_alloc(req, true); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[ENCRYPT]; + caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; + caam_req->op_type = ENCRYPT; + caam_req->cbk = tls_encrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + tls_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int tls_decrypt(struct aead_request *req) +{ + struct tls_edesc *edesc; + struct crypto_aead *tls = crypto_aead_reqtfm(req); + struct caam_ctx *ctx = crypto_aead_ctx(tls); + struct caam_request *caam_req = aead_request_ctx(req); + int ret; + + /* allocate extended descriptor */ + edesc = tls_edesc_alloc(req, false); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[DECRYPT]; + caam_req->flc_dma = ctx->flc_dma[DECRYPT]; + caam_req->op_type = DECRYPT; + caam_req->cbk = tls_decrypt_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + tls_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int ipsec_gcm_encrypt(struct aead_request *req) +{ + if (req->assoclen < 8) + return -EINVAL; + + return aead_encrypt(req); +} + +static int ipsec_gcm_decrypt(struct aead_request *req) +{ + if (req->assoclen < 8) + return -EINVAL; + + return aead_decrypt(req); +} + +static void ablkcipher_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ablkcipher_request *req = ablkcipher_request_cast(areq); + struct caam_request *req_ctx = to_caam_req(areq); + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct ablkcipher_edesc *edesc = req_ctx->edesc; + int ecode = 0; + int ivsize = crypto_ablkcipher_ivsize(ablkcipher); + +#ifdef DEBUG + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + ecode = -EIO; + } + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->info, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->nbytes, 1); +#endif + + ablkcipher_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + + /* + * The crypto API expects us to set the IV (req->info) to the last + * ciphertext block. This is used e.g. by the CTS mode. + */ + scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, + ivsize, 0); + + ablkcipher_request_complete(req, ecode); +} + +static int ablkcipher_encrypt(struct ablkcipher_request *req) +{ + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct caam_request *caam_req = ablkcipher_request_ctx(req); + int ret; + + /* allocate extended descriptor */ + edesc = ablkcipher_edesc_alloc(req, true); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[ENCRYPT]; + caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; + caam_req->op_type = ENCRYPT; + caam_req->cbk = ablkcipher_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + ablkcipher_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *greq) +{ + struct ablkcipher_request *req = &greq->creq; + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct caam_request *caam_req = ablkcipher_request_ctx(req); + int ret; + + /* allocate extended descriptor */ + edesc = ablkcipher_giv_edesc_alloc(greq); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[GIVENCRYPT]; + caam_req->flc_dma = ctx->flc_dma[GIVENCRYPT]; + caam_req->op_type = GIVENCRYPT; + caam_req->cbk = ablkcipher_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + ablkcipher_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +static int ablkcipher_decrypt(struct ablkcipher_request *req) +{ + struct ablkcipher_edesc *edesc; + struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); + struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); + struct caam_request *caam_req = ablkcipher_request_ctx(req); + int ret; + + /* allocate extended descriptor */ + edesc = ablkcipher_edesc_alloc(req, false); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + caam_req->flc = &ctx->flc[DECRYPT]; + caam_req->flc_dma = ctx->flc_dma[DECRYPT]; + caam_req->op_type = DECRYPT; + caam_req->cbk = ablkcipher_done; + caam_req->ctx = &req->base; + caam_req->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, caam_req); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { + ablkcipher_unmap(ctx->dev, edesc, req); + qi_cache_free(edesc); + } + + return ret; +} + +struct caam_crypto_alg { + struct list_head entry; + struct crypto_alg crypto_alg; + struct caam_alg_entry caam; +}; + +static int caam_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), + crypto_alg); + struct caam_ctx *ctx = crypto_tfm_ctx(tfm); + dma_addr_t dma_addr; + int i; + + /* copy descriptor header template value */ + ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | + caam_alg->caam.class1_alg_type; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | + caam_alg->caam.class2_alg_type; + + ctx->dev = caam_alg->caam.dev; + + dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, + offsetof(struct caam_ctx, flc_dma), + DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->dev, dma_addr)) { + dev_err(ctx->dev, "unable to map key, shared descriptors\n"); + return -ENOMEM; + } + + for (i = 0; i < NUM_OP; i++) + ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); + ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); + + return 0; +} + +static int caam_cra_init_ablkcipher(struct crypto_tfm *tfm) +{ + struct ablkcipher_tfm *ablkcipher_tfm = + crypto_ablkcipher_crt(__crypto_ablkcipher_cast(tfm)); + + ablkcipher_tfm->reqsize = sizeof(struct caam_request); + return caam_cra_init(tfm); +} + +static int caam_cra_init_aead(struct crypto_aead *tfm) +{ + crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); + return caam_cra_init(crypto_aead_tfm(tfm)); +} + +static void caam_exit_common(struct caam_ctx *ctx) +{ + dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], + offsetof(struct caam_ctx, flc_dma), + DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); +} + +static void caam_cra_exit(struct crypto_tfm *tfm) +{ + caam_exit_common(crypto_tfm_ctx(tfm)); +} + +static void caam_cra_exit_aead(struct crypto_aead *tfm) +{ + caam_exit_common(crypto_aead_ctx(tfm)); +} + +#define template_ablkcipher template_u.ablkcipher +struct caam_alg_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + u32 type; + union { + struct ablkcipher_alg ablkcipher; + } template_u; + u32 class1_alg_type; + u32 class2_alg_type; +}; + +static struct caam_alg_template driver_algs[] = { + /* ablkcipher descriptor */ + { + .name = "cbc(aes)", + .driver_name = "cbc-aes-caam-qi2", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des3_ede)", + .driver_name = "cbc-3des-caam-qi2", + .blocksize = DES3_EDE_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + }, + { + .name = "cbc(des)", + .driver_name = "cbc-des-caam-qi2", + .blocksize = DES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + }, + { + .name = "ctr(aes)", + .driver_name = "ctr-aes-caam-qi2", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "chainiv", + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-caam-qi2", + .blocksize = 1, + .type = CRYPTO_ALG_TYPE_GIVCIPHER, + .template_ablkcipher = { + .setkey = ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .givencrypt = ablkcipher_givencrypt, + .geniv = "", + .min_keysize = AES_MIN_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, + }, + { + .name = "xts(aes)", + .driver_name = "xts-aes-caam-qi2", + .blocksize = AES_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .template_ablkcipher = { + .setkey = xts_ablkcipher_setkey, + .encrypt = ablkcipher_encrypt, + .decrypt = ablkcipher_decrypt, + .geniv = "eseqiv", + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, + } +}; + +static struct caam_aead_alg driver_aeads[] = { + { + .aead = { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, + { + .aead = { + .base = { + .cra_name = "rfc4543(gcm(aes))", + .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = rfc4543_setkey, + .setauthsize = rfc4543_setauthsize, + .encrypt = ipsec_gcm_encrypt, + .decrypt = ipsec_gcm_decrypt, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + }, + }, + /* Galois Counter Mode */ + { + .aead = { + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "gcm-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = gcm_setkey, + .setauthsize = gcm_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, + } + }, + /* single-pass ipsec_esp descriptor */ + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(aes))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-aes-" + "caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-aes-" + "caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(aes)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-aes-" + "caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des3_ede)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-" + "cbc-des3_ede-caam-qi2", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5),cbc(des))", + .cra_driver_name = "authenc-hmac-md5-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(md5)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-hmac-md5-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1),cbc(des))", + .cra_driver_name = "authenc-hmac-sha1-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha1)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha1-cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224),cbc(des))", + .cra_driver_name = "authenc-hmac-sha224-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha224)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha224-cbc-des-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256),cbc(des))", + .cra_driver_name = "authenc-hmac-sha256-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha256)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha256-cbc-desi-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384),cbc(des))", + .cra_driver_name = "authenc-hmac-sha384-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha384)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha384-cbc-des-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512),cbc(des))", + .cra_driver_name = "authenc-hmac-sha512-" + "cbc-des-caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + } + }, + { + .aead = { + .base = { + .cra_name = "echainiv(authenc(hmac(sha512)," + "cbc(des)))", + .cra_driver_name = "echainiv-authenc-" + "hmac-sha512-cbc-des-" + "caam-qi2", + .cra_blocksize = DES_BLOCK_SIZE, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .geniv = true, + } + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(md5)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-md5-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(md5),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-md5-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha1)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha1),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha1-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha224)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(" + "hmac(sha224),rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha224-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha256)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha256)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha256-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha384)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha384)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha384-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "authenc(hmac(sha512)," + "rfc3686(ctr(aes)))", + .cra_driver_name = "authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "seqiv(authenc(hmac(sha512)," + "rfc3686(ctr(aes))))", + .cra_driver_name = "seqiv-authenc-hmac-sha512-" + "rfc3686-ctr-aes-caam-qi2", + .cra_blocksize = 1, + }, + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .ivsize = CTR_RFC3686_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | + OP_ALG_AAI_CTR_MOD128, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .rfc3686 = true, + .geniv = true, + }, + }, + { + .aead = { + .base = { + .cra_name = "tls10(hmac(sha1),cbc(aes))", + .cra_driver_name = "tls10-hmac-sha1-cbc-aes-caam-qi2", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = tls_setkey, + .setauthsize = tls_setauthsize, + .encrypt = tls_encrypt, + .decrypt = tls_decrypt, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .caam = { + .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | + OP_ALG_AAI_HMAC_PRECOMP, + }, + }, +}; + +static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template + *template) +{ + struct caam_crypto_alg *t_alg; + struct crypto_alg *alg; + + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + alg = &t_alg->crypto_alg; + + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + template->driver_name); + alg->cra_module = THIS_MODULE; + alg->cra_exit = caam_cra_exit; + alg->cra_priority = CAAM_CRA_PRIORITY; + alg->cra_blocksize = template->blocksize; + alg->cra_alignmask = 0; + alg->cra_ctxsize = sizeof(struct caam_ctx); + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | + template->type; + switch (template->type) { + case CRYPTO_ALG_TYPE_GIVCIPHER: + alg->cra_init = caam_cra_init_ablkcipher; + alg->cra_type = &crypto_givcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; + case CRYPTO_ALG_TYPE_ABLKCIPHER: + alg->cra_init = caam_cra_init_ablkcipher; + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_ablkcipher = template->template_ablkcipher; + break; + } + + t_alg->caam.class1_alg_type = template->class1_alg_type; + t_alg->caam.class2_alg_type = template->class2_alg_type; + + return t_alg; +} + +static void caam_aead_alg_init(struct caam_aead_alg *t_alg) +{ + struct aead_alg *alg = &t_alg->aead; + + alg->base.cra_module = THIS_MODULE; + alg->base.cra_priority = CAAM_CRA_PRIORITY; + alg->base.cra_ctxsize = sizeof(struct caam_ctx); + alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; + + alg->init = caam_cra_init_aead; + alg->exit = caam_cra_exit_aead; +} + +/* max hash key is max split key size */ +#define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) + +#define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE +#define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE + +#define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ + CAAM_MAX_HASH_KEY_SIZE) +#define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) + +/* caam context sizes for hashes: running digest + 8 */ +#define HASH_MSG_LEN 8 +#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) + +enum hash_optype { + UPDATE = 0, + UPDATE_FIRST, + FINALIZE, + DIGEST, + HASH_NUM_OP +}; + +/** + * caam_hash_ctx - ahash per-session context + * @flc: Flow Contexts array + * @flc_dma: I/O virtual addresses of the Flow Contexts + * @key: virtual address of the authentication key + * @dev: dpseci device + * @ctx_len: size of Context Register + * @adata: hashing algorithm details + */ +struct caam_hash_ctx { + struct caam_flc flc[HASH_NUM_OP]; + dma_addr_t flc_dma[HASH_NUM_OP]; + u8 key[CAAM_MAX_HASH_KEY_SIZE]; + struct device *dev; + int ctx_len; + struct alginfo adata; +}; + +/* ahash state */ +struct caam_hash_state { + struct caam_request caam_req; + dma_addr_t buf_dma; + dma_addr_t ctx_dma; + u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; + int buflen_0; + u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; + int buflen_1; + u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); + int current_buf; +}; + +struct caam_export_state { + u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; + u8 caam_ctx[MAX_CTX_LEN]; + int buflen; + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); +}; + +static inline void switch_buf(struct caam_hash_state *state) +{ + state->current_buf ^= 1; +} + +static inline u8 *current_buf(struct caam_hash_state *state) +{ + return state->current_buf ? state->buf_1 : state->buf_0; +} + +static inline u8 *alt_buf(struct caam_hash_state *state) +{ + return state->current_buf ? state->buf_0 : state->buf_1; +} + +static inline int *current_buflen(struct caam_hash_state *state) +{ + return state->current_buf ? &state->buflen_1 : &state->buflen_0; +} + +static inline int *alt_buflen(struct caam_hash_state *state) +{ + return state->current_buf ? &state->buflen_0 : &state->buflen_1; +} + +/* Map current buffer in state (if length > 0) and put it in link table */ +static inline int buf_map_to_qm_sg(struct device *dev, + struct dpaa2_sg_entry *qm_sg, + struct caam_hash_state *state) +{ + int buflen = *current_buflen(state); + + if (!buflen) + return 0; + + state->buf_dma = dma_map_single(dev, current_buf(state), buflen, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, state->buf_dma)) { + dev_err(dev, "unable to map buf\n"); + state->buf_dma = 0; + return -ENOMEM; + } + + dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); + + return 0; +} + +/* Map state->caam_ctx, and add it to link table */ +static inline int ctx_map_to_qm_sg(struct device *dev, + struct caam_hash_state *state, int ctx_len, + struct dpaa2_sg_entry *qm_sg, u32 flag) +{ + state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); + if (dma_mapping_error(dev, state->ctx_dma)) { + dev_err(dev, "unable to map ctx\n"); + state->ctx_dma = 0; + return -ENOMEM; + } + + dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); + + return 0; +} + +static int ahash_set_sh_desc(struct crypto_ahash *ahash) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); + struct caam_flc *flc; + u32 *desc; + + ctx->adata.key_virt = ctx->key; + ctx->adata.key_inline = true; + + /* ahash_update shared descriptor */ + flc = &ctx->flc[UPDATE]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, + ctx->ctx_len, true, priv->sec_attr.era); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], + desc_bytes(desc), DMA_BIDIRECTIONAL); +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ahash update shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif + + /* ahash_update_first shared descriptor */ + flc = &ctx->flc[UPDATE_FIRST]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, + ctx->ctx_len, false, priv->sec_attr.era); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], + desc_bytes(desc), DMA_BIDIRECTIONAL); +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ahash update first shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif + + /* ahash_final shared descriptor */ + flc = &ctx->flc[FINALIZE]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, + ctx->ctx_len, true, priv->sec_attr.era); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], + desc_bytes(desc), DMA_BIDIRECTIONAL); +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ahash final shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif + + /* ahash_digest shared descriptor */ + flc = &ctx->flc[DIGEST]; + desc = flc->sh_desc; + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, + ctx->ctx_len, false, priv->sec_attr.era); + flc->flc[1] = desc_len(desc); /* SDL */ + dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], + desc_bytes(desc), DMA_BIDIRECTIONAL); +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "ahash digest shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif + + return 0; +} + +/* Digest hash size if it is too large */ +static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, + u32 *keylen, u8 *key_out, u32 digestsize) +{ + struct caam_request *req_ctx; + u32 *desc; + struct split_key_sh_result result; + dma_addr_t src_dma, dst_dma; + struct caam_flc *flc; + dma_addr_t flc_dma; + int ret = -ENOMEM; + struct dpaa2_fl_entry *in_fle, *out_fle; + + req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); + if (!req_ctx) + return -ENOMEM; + + in_fle = &req_ctx->fd_flt[1]; + out_fle = &req_ctx->fd_flt[0]; + + flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); + if (!flc) + goto err_flc; + + src_dma = dma_map_single(ctx->dev, (void *)key_in, *keylen, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, src_dma)) { + dev_err(ctx->dev, "unable to map key input memory\n"); + goto err_src_dma; + } + dst_dma = dma_map_single(ctx->dev, (void *)key_out, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, dst_dma)) { + dev_err(ctx->dev, "unable to map key output memory\n"); + goto err_dst_dma; + } + + desc = flc->sh_desc; + + init_sh_desc(desc, 0); + + /* descriptor to perform unkeyed hash on key_in */ + append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | + OP_ALG_AS_INITFINAL); + append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); + append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + flc->flc[1] = desc_len(desc); /* SDL */ + flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + + desc_bytes(desc), DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, flc_dma)) { + dev_err(ctx->dev, "unable to map shared descriptor\n"); + goto err_flc_dma; + } + + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, src_dma); + dpaa2_fl_set_len(in_fle, *keylen); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, dst_dma); + dpaa2_fl_set_len(out_fle, digestsize); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "key_in@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); + print_hex_dump(KERN_ERR, "shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); +#endif + + result.err = 0; + init_completion(&result.completion); + result.dev = ctx->dev; + + req_ctx->flc = flc; + req_ctx->flc_dma = flc_dma; + req_ctx->cbk = split_key_sh_done; + req_ctx->ctx = &result; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS) { + /* in progress */ + wait_for_completion(&result.completion); + ret = result.err; +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "digested key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_in, digestsize, + 1); +#endif + } + + dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), + DMA_TO_DEVICE); +err_flc_dma: + dma_unmap_single(ctx->dev, dst_dma, digestsize, DMA_FROM_DEVICE); +err_dst_dma: + dma_unmap_single(ctx->dev, src_dma, *keylen, DMA_TO_DEVICE); +err_src_dma: + kfree(flc); +err_flc: + kfree(req_ctx); + + *keylen = digestsize; + + return ret; +} + +static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, + unsigned int keylen) +{ + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); + unsigned int digestsize = crypto_ahash_digestsize(ahash); + int ret; + u8 *hashed_key = NULL; + +#ifdef DEBUG + dev_err(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); +#endif + + if (keylen > blocksize) { + hashed_key = kmalloc_array(digestsize, sizeof(*hashed_key), + GFP_KERNEL | GFP_DMA); + if (!hashed_key) + return -ENOMEM; + ret = hash_digest_key(ctx, key, &keylen, hashed_key, + digestsize); + if (ret) + goto bad_free_key; + key = hashed_key; + } + + ctx->adata.keylen = keylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); + if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) + goto bad_free_key; + + memcpy(ctx->key, key, keylen); + + kfree(hashed_key); + return ahash_set_sh_desc(ahash); +bad_free_key: + kfree(hashed_key); + crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, + struct ahash_request *req, int dst_len) +{ + struct caam_hash_state *state = ahash_request_ctx(req); + + if (edesc->src_nents) + dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + if (edesc->dst_dma) + dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); + + if (edesc->qm_sg_bytes) + dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, + DMA_TO_DEVICE); + + if (state->buf_dma) { + dma_unmap_single(dev, state->buf_dma, *current_buflen(state), + DMA_TO_DEVICE); + state->buf_dma = 0; + } +} + +static inline void ahash_unmap_ctx(struct device *dev, + struct ahash_edesc *edesc, + struct ahash_request *req, int dst_len, + u32 flag) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + + if (state->ctx_dma) { + dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); + state->ctx_dma = 0; + } + ahash_unmap(dev, edesc, req, dst_len); +} + +static void ahash_done(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + int ecode = 0; + +#ifdef DEBUG + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + ecode = -EIO; + } + + ahash_unmap(ctx->dev, edesc, req, digestsize); + qi_cache_free(edesc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + if (req->result) + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); +#endif + + req->base.complete(&req->base, ecode); +} + +static void ahash_done_bi(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + int ecode = 0; +#ifdef DEBUG + int digestsize = crypto_ahash_digestsize(ahash); + + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + ecode = -EIO; + } + + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); + switch_buf(state); + qi_cache_free(edesc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + if (req->result) + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); +#endif + + req->base.complete(&req->base, ecode); +} + +static void ahash_done_ctx_src(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + int digestsize = crypto_ahash_digestsize(ahash); + int ecode = 0; + +#ifdef DEBUG + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + ecode = -EIO; + } + + ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_TO_DEVICE); + qi_cache_free(edesc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + if (req->result) + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); +#endif + + req->base.complete(&req->base, ecode); +} + +static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) +{ + struct crypto_async_request *areq = cbk_ctx; + struct ahash_request *req = ahash_request_cast(areq); + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_state *state = ahash_request_ctx(req); + struct ahash_edesc *edesc = state->caam_req.edesc; + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + int ecode = 0; +#ifdef DEBUG + int digestsize = crypto_ahash_digestsize(ahash); + + dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); +#endif + + if (unlikely(status)) { + caam_qi2_strstatus(ctx->dev, status); + ecode = -EIO; + } + + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); + switch_buf(state); + qi_cache_free(edesc); + +#ifdef DEBUG + print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); + if (req->result) + print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); +#endif + + req->base.complete(&req->base, ecode); +} + +static int ahash_update_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = current_buf(state); + int *buflen = current_buflen(state); + u8 *next_buf = alt_buf(state); + int *next_buflen = alt_buflen(state), last_buflen; + int in_len = *buflen + req->nbytes, to_hash; + int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; + struct ahash_edesc *edesc; + int ret = 0; + + last_buflen = *next_buflen; + *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); + to_hash = in_len - *next_buflen; + + if (to_hash) { + struct dpaa2_sg_entry *sg_table; + + src_nents = sg_nents_for_len(req->src, + req->nbytes - (*next_buflen)); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + qm_sg_src_index = 1 + (*buflen ? 1 : 0); + qm_sg_bytes = (qm_sg_src_index + mapped_nents) * + sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, + DMA_BIDIRECTIONAL); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); + if (ret) + goto unmap_ctx; + + if (mapped_nents) { + sg_to_qm_sg_last(req->src, mapped_nents, + sg_table + qm_sg_src_index, 0); + if (*next_buflen) + scatterwalk_map_and_copy(next_buf, req->src, + to_hash - *buflen, + *next_buflen, 0); + } else { + dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, + true); + } + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, ctx->ctx_len); + + req_ctx->flc = &ctx->flc[UPDATE]; + req_ctx->flc_dma = ctx->flc_dma[UPDATE]; + req_ctx->cbk = ahash_done_bi; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && + req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap_ctx; + } else if (*next_buflen) { + scatterwalk_map_and_copy(buf + *buflen, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + *next_buflen = last_buflen; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, + *next_buflen, 1); +#endif + + return ret; +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); + qi_cache_free(edesc); + return ret; +} + +static int ahash_final_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = *current_buflen(state); + int qm_sg_bytes, qm_sg_src_index; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + struct dpaa2_sg_entry *sg_table; + int ret; + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) + return -ENOMEM; + + qm_sg_src_index = 1 + (buflen ? 1 : 0); + qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, + DMA_TO_DEVICE); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); + if (ret) + goto unmap_ctx; + + dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { + dev_err(ctx->dev, "unable to map dst\n"); + edesc->dst_dma = 0; + ret = -ENOMEM; + goto unmap_ctx; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[FINALIZE]; + req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; + req_ctx->cbk = ahash_done_ctx_src; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_finup_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = *current_buflen(state); + int qm_sg_bytes, qm_sg_src_index; + int src_nents, mapped_nents; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + struct dpaa2_sg_entry *sg_table; + int ret; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + qm_sg_src_index = 1 + (buflen ? 1 : 0); + qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, + DMA_TO_DEVICE); + if (ret) + goto unmap_ctx; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); + if (ret) + goto unmap_ctx; + + sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { + dev_err(ctx->dev, "unable to map dst\n"); + edesc->dst_dma = 0; + ret = -ENOMEM; + goto unmap_ctx; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[FINALIZE]; + req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; + req_ctx->cbk = ahash_done_ctx_src; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_digest(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int digestsize = crypto_ahash_digestsize(ahash); + int src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret = -ENOMEM; + + state->buf_dma = 0; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to map source for DMA\n"); + return ret; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); + return ret; + } + + edesc->src_nents = src_nents; + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + + if (mapped_nents > 1) { + int qm_sg_bytes; + struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; + + qm_sg_bytes = mapped_nents * sizeof(*sg_table); + sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + goto unmap; + } + edesc->qm_sg_bytes = qm_sg_bytes; + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + } else { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); + } + + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { + dev_err(ctx->dev, "unable to map dst\n"); + edesc->dst_dma = 0; + goto unmap; + } + + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_len(in_fle, req->nbytes); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[DIGEST]; + req_ctx->flc_dma = ctx->flc_dma[DIGEST]; + req_ctx->cbk = ahash_done; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap: + ahash_unmap(ctx->dev, edesc, req, digestsize); + qi_cache_free(edesc); + return ret; +} + +static int ahash_final_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = current_buf(state); + int buflen = *current_buflen(state); + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + int ret = -ENOMEM; + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) + return ret; + + state->buf_dma = dma_map_single(ctx->dev, buf, buflen, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, state->buf_dma)) { + dev_err(ctx->dev, "unable to map src\n"); + goto unmap; + } + + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { + dev_err(ctx->dev, "unable to map dst\n"); + edesc->dst_dma = 0; + goto unmap; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, state->buf_dma); + dpaa2_fl_set_len(in_fle, buflen); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[DIGEST]; + req_ctx->flc_dma = ctx->flc_dma[DIGEST]; + req_ctx->cbk = ahash_done; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret == -EINPROGRESS || + (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return ret; + +unmap: + ahash_unmap(ctx->dev, edesc, req, digestsize); + qi_cache_free(edesc); + return ret; +} + +static int ahash_update_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = current_buf(state); + int *buflen = current_buflen(state); + u8 *next_buf = alt_buf(state); + int *next_buflen = alt_buflen(state); + int in_len = *buflen + req->nbytes, to_hash; + int qm_sg_bytes, src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret = 0; + + *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); + to_hash = in_len - *next_buflen; + + if (to_hash) { + struct dpaa2_sg_entry *sg_table; + + src_nents = sg_nents_for_len(req->src, + req->nbytes - *next_buflen); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); + if (ret) + goto unmap_ctx; + + sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + + if (*next_buflen) + scatterwalk_map_and_copy(next_buf, req->src, + to_hash - *buflen, + *next_buflen, 0); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, + ctx->ctx_len, DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; + ret = -ENOMEM; + goto unmap_ctx; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, to_hash); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, ctx->ctx_len); + + req_ctx->flc = &ctx->flc[UPDATE_FIRST]; + req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; + req_ctx->cbk = ahash_done_ctx_dst; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && + req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap_ctx; + + state->update = ahash_update_ctx; + state->finup = ahash_finup_ctx; + state->final = ahash_final_ctx; + } else if (*next_buflen) { + scatterwalk_map_and_copy(buf + *buflen, req->src, 0, + req->nbytes, 0); + *buflen = *next_buflen; + *next_buflen = 0; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, + *next_buflen, 1); +#endif + + return ret; +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_finup_no_ctx(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = *current_buflen(state); + int qm_sg_bytes, src_nents, mapped_nents; + int digestsize = crypto_ahash_digestsize(ahash); + struct ahash_edesc *edesc; + struct dpaa2_sg_entry *sg_table; + int ret; + + src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to DMA map source\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table); + sg_table = &edesc->sgt[0]; + + ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); + if (ret) + goto unmap; + + sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap; + } + edesc->qm_sg_bytes = qm_sg_bytes; + + edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, + DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { + dev_err(ctx->dev, "unable to map dst\n"); + edesc->dst_dma = 0; + ret = -ENOMEM; + goto unmap; + } + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + dpaa2_fl_set_len(in_fle, buflen + req->nbytes); + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, edesc->dst_dma); + dpaa2_fl_set_len(out_fle, digestsize); + + req_ctx->flc = &ctx->flc[DIGEST]; + req_ctx->flc_dma = ctx->flc_dma[DIGEST]; + req_ctx->cbk = ahash_done; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap; + + return ret; +unmap: + ahash_unmap(ctx->dev, edesc, req, digestsize); + qi_cache_free(edesc); + return -ENOMEM; +} + +static int ahash_update_first(struct ahash_request *req) +{ + struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); + struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_request *req_ctx = &state->caam_req; + struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; + struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *next_buf = alt_buf(state); + int *next_buflen = alt_buflen(state); + int to_hash; + int src_nents, mapped_nents; + struct ahash_edesc *edesc; + int ret = 0; + + *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - + 1); + to_hash = req->nbytes - *next_buflen; + + if (to_hash) { + struct dpaa2_sg_entry *sg_table; + + src_nents = sg_nents_for_len(req->src, + req->nbytes - (*next_buflen)); + if (src_nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return src_nents; + } + + if (src_nents) { + mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + if (!mapped_nents) { + dev_err(ctx->dev, "unable to map source for DMA\n"); + return -ENOMEM; + } + } else { + mapped_nents = 0; + } + + /* allocate space for base edesc and link tables */ + edesc = qi_cache_zalloc(GFP_DMA | flags); + if (!edesc) { + dma_unmap_sg(ctx->dev, req->src, src_nents, + DMA_TO_DEVICE); + return -ENOMEM; + } + + edesc->src_nents = src_nents; + sg_table = &edesc->sgt[0]; + + memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); + dpaa2_fl_set_final(in_fle, true); + dpaa2_fl_set_len(in_fle, to_hash); + + if (mapped_nents > 1) { + int qm_sg_bytes; + + sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); + qm_sg_bytes = mapped_nents * sizeof(*sg_table); + edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, + qm_sg_bytes, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { + dev_err(ctx->dev, "unable to map S/G table\n"); + ret = -ENOMEM; + goto unmap_ctx; + } + edesc->qm_sg_bytes = qm_sg_bytes; + dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); + dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); + } else { + dpaa2_fl_set_format(in_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); + } + + if (*next_buflen) + scatterwalk_map_and_copy(next_buf, req->src, to_hash, + *next_buflen, 0); + + state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, + ctx->ctx_len, DMA_FROM_DEVICE); + if (dma_mapping_error(ctx->dev, state->ctx_dma)) { + dev_err(ctx->dev, "unable to map ctx\n"); + state->ctx_dma = 0; + ret = -ENOMEM; + goto unmap_ctx; + } + + dpaa2_fl_set_format(out_fle, dpaa2_fl_single); + dpaa2_fl_set_addr(out_fle, state->ctx_dma); + dpaa2_fl_set_len(out_fle, ctx->ctx_len); + + req_ctx->flc = &ctx->flc[UPDATE_FIRST]; + req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; + req_ctx->cbk = ahash_done_ctx_dst; + req_ctx->ctx = &req->base; + req_ctx->edesc = edesc; + + ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); + if (ret != -EINPROGRESS && + !(ret == -EBUSY && req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG)) + goto unmap_ctx; + + state->update = ahash_update_ctx; + state->finup = ahash_finup_ctx; + state->final = ahash_final_ctx; + } else if (*next_buflen) { + state->update = ahash_update_no_ctx; + state->finup = ahash_finup_no_ctx; + state->final = ahash_final_no_ctx; + scatterwalk_map_and_copy(next_buf, req->src, 0, + req->nbytes, 0); + switch_buf(state); + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 1); +#endif + + return ret; +unmap_ctx: + ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); + qi_cache_free(edesc); + return ret; +} + +static int ahash_finup_first(struct ahash_request *req) +{ + return ahash_digest(req); +} + +static int ahash_init(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx(req); + + state->update = ahash_update_first; + state->finup = ahash_finup_first; + state->final = ahash_final_no_ctx; + + state->ctx_dma = 0; + state->current_buf = 0; + state->buf_dma = 0; + state->buflen_0 = 0; + state->buflen_1 = 0; + + return 0; +} + +static int ahash_update(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx(req); + + return state->update(req); +} + +static int ahash_finup(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx(req); + + return state->finup(req); +} + +static int ahash_final(struct ahash_request *req) +{ + struct caam_hash_state *state = ahash_request_ctx(req); + + return state->final(req); +} + +static int ahash_export(struct ahash_request *req, void *out) +{ + struct caam_hash_state *state = ahash_request_ctx(req); + struct caam_export_state *export = out; + int len; + u8 *buf; + + if (state->current_buf) { + buf = state->buf_1; + len = state->buflen_1; + } else { + buf = state->buf_0; + len = state->buflen_0; + } + + memcpy(export->buf, buf, len); + memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); + export->buflen = len; + export->update = state->update; + export->final = state->final; + export->finup = state->finup; + + return 0; +} + +static int ahash_import(struct ahash_request *req, const void *in) +{ + struct caam_hash_state *state = ahash_request_ctx(req); + const struct caam_export_state *export = in; + + memset(state, 0, sizeof(*state)); + memcpy(state->buf_0, export->buf, export->buflen); + memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); + state->buflen_0 = export->buflen; + state->update = export->update; + state->final = export->final; + state->finup = export->finup; + + return 0; +} + +struct caam_hash_template { + char name[CRYPTO_MAX_ALG_NAME]; + char driver_name[CRYPTO_MAX_ALG_NAME]; + char hmac_name[CRYPTO_MAX_ALG_NAME]; + char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; + unsigned int blocksize; + struct ahash_alg template_ahash; + u32 alg_type; +}; + +/* ahash descriptors */ +static struct caam_hash_template driver_hash[] = { + { + .name = "sha1", + .driver_name = "sha1-caam-qi2", + .hmac_name = "hmac(sha1)", + .hmac_driver_name = "hmac-sha1-caam-qi2", + .blocksize = SHA1_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA1, + }, { + .name = "sha224", + .driver_name = "sha224-caam-qi2", + .hmac_name = "hmac(sha224)", + .hmac_driver_name = "hmac-sha224-caam-qi2", + .blocksize = SHA224_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA224, + }, { + .name = "sha256", + .driver_name = "sha256-caam-qi2", + .hmac_name = "hmac(sha256)", + .hmac_driver_name = "hmac-sha256-caam-qi2", + .blocksize = SHA256_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA256, + }, { + .name = "sha384", + .driver_name = "sha384-caam-qi2", + .hmac_name = "hmac(sha384)", + .hmac_driver_name = "hmac-sha384-caam-qi2", + .blocksize = SHA384_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA384, + }, { + .name = "sha512", + .driver_name = "sha512-caam-qi2", + .hmac_name = "hmac(sha512)", + .hmac_driver_name = "hmac-sha512-caam-qi2", + .blocksize = SHA512_BLOCK_SIZE, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_SHA512, + }, { + .name = "md5", + .driver_name = "md5-caam-qi2", + .hmac_name = "hmac(md5)", + .hmac_driver_name = "hmac-md5-caam-qi2", + .blocksize = MD5_BLOCK_WORDS * 4, + .template_ahash = { + .init = ahash_init, + .update = ahash_update, + .final = ahash_final, + .finup = ahash_finup, + .digest = ahash_digest, + .export = ahash_export, + .import = ahash_import, + .setkey = ahash_setkey, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct caam_export_state), + }, + }, + .alg_type = OP_ALG_ALGSEL_MD5, + } +}; + +struct caam_hash_alg { + struct list_head entry; + struct device *dev; + int alg_type; + struct ahash_alg ahash_alg; +}; + +static int caam_hash_cra_init(struct crypto_tfm *tfm) +{ + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct crypto_alg *base = tfm->__crt_alg; + struct hash_alg_common *halg = + container_of(base, struct hash_alg_common, base); + struct ahash_alg *alg = + container_of(halg, struct ahash_alg, halg); + struct caam_hash_alg *caam_hash = + container_of(alg, struct caam_hash_alg, ahash_alg); + struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); + /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ + static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, + HASH_MSG_LEN + SHA1_DIGEST_SIZE, + HASH_MSG_LEN + 32, + HASH_MSG_LEN + SHA256_DIGEST_SIZE, + HASH_MSG_LEN + 64, + HASH_MSG_LEN + SHA512_DIGEST_SIZE }; + dma_addr_t dma_addr; + int i; + + ctx->dev = caam_hash->dev; + + dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), + DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->dev, dma_addr)) { + dev_err(ctx->dev, "unable to map shared descriptors\n"); + return -ENOMEM; + } + + for (i = 0; i < HASH_NUM_OP; i++) + ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); + + /* copy descriptor header template value */ + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; + + ctx->ctx_len = runninglen[(ctx->adata.algtype & + OP_ALG_ALGSEL_SUBMASK) >> + OP_ALG_ALGSEL_SHIFT]; + + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct caam_hash_state)); + + return ahash_set_sh_desc(ahash); +} + +static void caam_hash_cra_exit(struct crypto_tfm *tfm) +{ + struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); + + dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), + DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); +} + +static struct caam_hash_alg *caam_hash_alloc(struct device *dev, + struct caam_hash_template *template, bool keyed) +{ + struct caam_hash_alg *t_alg; + struct ahash_alg *halg; + struct crypto_alg *alg; + + t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); + if (!t_alg) + return ERR_PTR(-ENOMEM); + + t_alg->ahash_alg = template->template_ahash; + halg = &t_alg->ahash_alg; + alg = &halg->halg.base; + + if (keyed) { + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", + template->hmac_name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + template->hmac_driver_name); + } else { + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", + template->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + template->driver_name); + } + alg->cra_module = THIS_MODULE; + alg->cra_init = caam_hash_cra_init; + alg->cra_exit = caam_hash_cra_exit; + alg->cra_ctxsize = sizeof(struct caam_hash_ctx); + alg->cra_priority = CAAM_CRA_PRIORITY; + alg->cra_blocksize = template->blocksize; + alg->cra_alignmask = 0; + alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH; + alg->cra_type = &crypto_ahash_type; + + t_alg->alg_type = template->alg_type; + t_alg->dev = dev; + + return t_alg; +} + +static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) +{ + struct dpaa2_caam_priv_per_cpu *ppriv; + + ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); + napi_schedule_irqoff(&ppriv->napi); +} + +static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct dpaa2_io_notification_ctx *nctx; + struct dpaa2_caam_priv_per_cpu *ppriv; + int err, i = 0, cpu; + + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + ppriv->priv = priv; + nctx = &ppriv->nctx; + nctx->is_cdan = 0; + nctx->id = ppriv->rsp_fqid; + nctx->desired_cpu = cpu; + nctx->cb = dpaa2_caam_fqdan_cb; + + /* Register notification callbacks */ + err = dpaa2_io_service_register(NULL, nctx); + if (unlikely(err)) { + dev_err(dev, "notification register failed\n"); + nctx->cb = NULL; + goto err; + } + + ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, + dev); + if (unlikely(!ppriv->store)) { + dev_err(dev, "dpaa2_io_store_create() failed\n"); + goto err; + } + + if (++i == priv->num_pairs) + break; + } + + return 0; + +err: + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + if (!ppriv->nctx.cb) + break; + dpaa2_io_service_deregister(NULL, &ppriv->nctx); + } + + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + if (!ppriv->store) + break; + dpaa2_io_store_destroy(ppriv->store); + } + + return err; +} + +static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) +{ + struct dpaa2_caam_priv_per_cpu *ppriv; + int i = 0, cpu; + + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + dpaa2_io_service_deregister(NULL, &ppriv->nctx); + dpaa2_io_store_destroy(ppriv->store); + + if (++i == priv->num_pairs) + return; + } +} + +static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) +{ + struct dpseci_rx_queue_cfg rx_queue_cfg; + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + struct dpaa2_caam_priv_per_cpu *ppriv; + int err = 0, i = 0, cpu; + + /* Configure Rx queues */ + for_each_online_cpu(cpu) { + ppriv = per_cpu_ptr(priv->ppriv, cpu); + + rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | + DPSECI_QUEUE_OPT_USER_CTX; + rx_queue_cfg.order_preservation_en = 0; + rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; + rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; + /* + * Rx priority (WQ) doesn't really matter, since we use + * pull mode, i.e. volatile dequeues from specific FQs + */ + rx_queue_cfg.dest_cfg.priority = 0; + rx_queue_cfg.user_ctx = ppriv->nctx.qman64; + + err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, + &rx_queue_cfg); + if (err) { + dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", + err); + return err; + } + + if (++i == priv->num_pairs) + break; + } + + return err; +} + +static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + + if (!priv->cscn_mem) + return; + + dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); + kfree(priv->cscn_mem); +} + +static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + + dpaa2_dpseci_congestion_free(priv); + dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); +} + +static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, + const struct dpaa2_fd *fd) +{ + struct caam_request *req; + u32 fd_err; + + if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { + dev_err(priv->dev, "Only Frame List FD format is supported!\n"); + return; + } + + fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; + if (unlikely(fd_err)) + dev_err(priv->dev, "FD error: %08x\n", fd_err); + + /* + * FD[ADDR] is guaranteed to be valid, irrespective of errors reported + * in FD[ERR] or FD[FRC]. + */ + req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); + dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), + DMA_BIDIRECTIONAL); + req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); +} + +static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) +{ + int err; + + /* Retry while portal is busy */ + do { + err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid, + ppriv->store); + } while (err == -EBUSY); + + if (unlikely(err)) + dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); + + return err; +} + +static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) +{ + struct dpaa2_dq *dq; + int cleaned = 0, is_last; + + do { + dq = dpaa2_io_store_next(ppriv->store, &is_last); + if (unlikely(!dq)) { + if (unlikely(!is_last)) { + dev_dbg(ppriv->priv->dev, + "FQ %d returned no valid frames\n", + ppriv->rsp_fqid); + /* + * MUST retry until we get some sort of + * valid response token (be it "empty dequeue" + * or a valid frame). + */ + continue; + } + break; + } + + /* Process FD */ + dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); + cleaned++; + } while (!is_last); + + return cleaned; +} + +static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) +{ + struct dpaa2_caam_priv_per_cpu *ppriv; + struct dpaa2_caam_priv *priv; + int err, cleaned = 0, store_cleaned; + + ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); + priv = ppriv->priv; + + if (unlikely(dpaa2_caam_pull_fq(ppriv))) + return 0; + + do { + store_cleaned = dpaa2_caam_store_consume(ppriv); + cleaned += store_cleaned; + + if (store_cleaned == 0 || + cleaned > budget - DPAA2_CAAM_STORE_SIZE) + break; + + /* Try to dequeue some more */ + err = dpaa2_caam_pull_fq(ppriv); + if (unlikely(err)) + break; + } while (1); + + if (cleaned < budget) { + napi_complete_done(napi, cleaned); + err = dpaa2_io_service_rearm(NULL, &ppriv->nctx); + if (unlikely(err)) + dev_err(priv->dev, "Notification rearm failed: %d\n", + err); + } + + return cleaned; +} + +static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, + u16 token) +{ + struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; + struct device *dev = priv->dev; + int err; + + /* + * Congestion group feature supported starting with DPSECI API v5.1 + * and only when object has been created with this capability. + */ + if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || + !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) + return 0; + + priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, + GFP_KERNEL | GFP_DMA); + if (!priv->cscn_mem) + return -ENOMEM; + + priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); + priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, + DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, priv->cscn_dma)) { + dev_err(dev, "Error mapping CSCN memory area\n"); + err = -ENOMEM; + goto err_dma_map; + } + + cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; + cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; + cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; + cong_notif_cfg.message_ctx = (u64)priv; + cong_notif_cfg.message_iova = priv->cscn_dma; + cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | + DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | + DPSECI_CGN_MODE_COHERENT_WRITE; + + err = dpseci_set_congestion_notification(priv->mc_io, 0, token, + &cong_notif_cfg); + if (err) { + dev_err(dev, "dpseci_set_congestion_notification failed\n"); + goto err_set_cong; + } + + return 0; + +err_set_cong: + dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); +err_dma_map: + kfree(priv->cscn_mem); + + return err; +} + +static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) +{ + struct device *dev = &ls_dev->dev; + struct dpaa2_caam_priv *priv; + struct dpaa2_caam_priv_per_cpu *ppriv; + int err, cpu; + u8 i; + + priv = dev_get_drvdata(dev); + + priv->dev = dev; + priv->dpsec_id = ls_dev->obj_desc.id; + + /* Get a handle for the DPSECI this interface is associate with */ + err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); + if (err) { + dev_err(dev, "dpsec_open() failed: %d\n", err); + goto err_open; + } + + dev_info(dev, "Opened dpseci object successfully\n"); + + err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, + &priv->minor_ver); + if (err) { + dev_err(dev, "dpseci_get_api_version() failed\n"); + goto err_get_vers; + } + + err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, + &priv->dpseci_attr); + if (err) { + dev_err(dev, "dpseci_get_attributes() failed\n"); + goto err_get_vers; + } + + err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, + &priv->sec_attr); + if (err) { + dev_err(dev, "dpseci_get_sec_attr() failed\n"); + goto err_get_vers; + } + + err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); + if (err) { + dev_err(dev, "setup_congestion() failed\n"); + goto err_get_vers; + } + + priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, + priv->dpseci_attr.num_tx_queues); + if (priv->num_pairs > num_online_cpus()) { + dev_warn(dev, "%d queues won't be used\n", + priv->num_pairs - num_online_cpus()); + priv->num_pairs = num_online_cpus(); + } + + for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { + err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, + &priv->rx_queue_attr[i]); + if (err) { + dev_err(dev, "dpseci_get_rx_queue() failed\n"); + goto err_get_rx_queue; + } + } + + for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { + err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, + &priv->tx_queue_attr[i]); + if (err) { + dev_err(dev, "dpseci_get_tx_queue() failed\n"); + goto err_get_rx_queue; + } + } + + i = 0; + for_each_online_cpu(cpu) { + dev_info(dev, "prio %d: rx queue %d, tx queue %d\n", i, + priv->rx_queue_attr[i].fqid, + priv->tx_queue_attr[i].fqid); + + ppriv = per_cpu_ptr(priv->ppriv, cpu); + ppriv->req_fqid = priv->tx_queue_attr[i].fqid; + ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; + ppriv->prio = i; + + ppriv->net_dev.dev = *dev; + INIT_LIST_HEAD(&ppriv->net_dev.napi_list); + netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, + DPAA2_CAAM_NAPI_WEIGHT); + if (++i == priv->num_pairs) + break; + } + + return 0; + +err_get_rx_queue: + dpaa2_dpseci_congestion_free(priv); +err_get_vers: + dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); +err_open: + return err; +} + +static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + struct dpaa2_caam_priv_per_cpu *ppriv; + int err, i; + + for (i = 0; i < priv->num_pairs; i++) { + ppriv = per_cpu_ptr(priv->ppriv, i); + napi_enable(&ppriv->napi); + } + + err = dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); + if (err) { + dev_err(dev, "dpseci_enable() failed\n"); + return err; + } + + dev_info(dev, "DPSECI version %d.%d\n", + priv->major_ver, + priv->minor_ver); + + return 0; +} + +static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) +{ + struct device *dev = priv->dev; + struct dpaa2_caam_priv_per_cpu *ppriv; + struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); + int i, err = 0, enabled; + + err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); + if (err) { + dev_err(dev, "dpseci_disable() failed\n"); + return err; + } + + err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); + if (err) { + dev_err(dev, "dpseci_is_enabled() failed\n"); + return err; + } + + dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); + + for (i = 0; i < priv->num_pairs; i++) { + ppriv = per_cpu_ptr(priv->ppriv, i); + napi_disable(&ppriv->napi); + netif_napi_del(&ppriv->napi); + } + + return 0; +} + +static struct list_head alg_list; +static struct list_head hash_list; + +static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) +{ + struct device *dev; + struct dpaa2_caam_priv *priv; + int i, err = 0; + bool registered = false; + + /* + * There is no way to get CAAM endianness - there is no direct register + * space access and MC f/w does not provide this attribute. + * All DPAA2-based SoCs have little endian CAAM, thus hard-code this + * property. + */ + caam_little_end = true; + + caam_imx = false; + + dev = &dpseci_dev->dev; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + dev_set_drvdata(dev, priv); + + priv->domain = iommu_get_domain_for_dev(dev); + + qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, + 0, SLAB_CACHE_DMA, NULL); + if (!qi_cache) { + dev_err(dev, "Can't allocate SEC cache\n"); + err = -ENOMEM; + goto err_qicache; + } + + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); + if (err) { + dev_err(dev, "dma_set_mask_and_coherent() failed\n"); + goto err_dma_mask; + } + + /* Obtain a MC portal */ + err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); + if (err) { + dev_err(dev, "MC portal allocation failed\n"); + goto err_dma_mask; + } + + priv->ppriv = alloc_percpu(*priv->ppriv); + if (!priv->ppriv) { + dev_err(dev, "alloc_percpu() failed\n"); + goto err_alloc_ppriv; + } + + /* DPSECI initialization */ + err = dpaa2_dpseci_setup(dpseci_dev); + if (err < 0) { + dev_err(dev, "dpaa2_dpseci_setup() failed\n"); + goto err_dpseci_setup; + } + + /* DPIO */ + err = dpaa2_dpseci_dpio_setup(priv); + if (err) { + dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); + goto err_dpio_setup; + } + + /* DPSECI binding to DPIO */ + err = dpaa2_dpseci_bind(priv); + if (err) { + dev_err(dev, "dpaa2_dpseci_bind() failed\n"); + goto err_bind; + } + + /* DPSECI enable */ + err = dpaa2_dpseci_enable(priv); + if (err) { + dev_err(dev, "dpaa2_dpseci_enable() failed"); + goto err_bind; + } + + /* register crypto algorithms the device supports */ + INIT_LIST_HEAD(&alg_list); + for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { + struct caam_crypto_alg *t_alg; + struct caam_alg_template *alg = driver_algs + i; + u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!priv->sec_attr.des_acc_num && + ((alg_sel == OP_ALG_ALGSEL_3DES) || + (alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!priv->sec_attr.aes_acc_num && + (alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + t_alg = caam_alg_alloc(alg); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + dev_warn(dev, "%s alg allocation failed: %d\n", + alg->driver_name, err); + continue; + } + t_alg->caam.dev = dev; + + err = crypto_register_alg(&t_alg->crypto_alg); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->crypto_alg.cra_driver_name, err); + kfree(t_alg); + continue; + } + + list_add_tail(&t_alg->entry, &alg_list); + registered = true; + } + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + u32 c1_alg_sel = t_alg->caam.class1_alg_type & + OP_ALG_ALGSEL_MASK; + u32 c2_alg_sel = t_alg->caam.class2_alg_type & + OP_ALG_ALGSEL_MASK; + + /* Skip DES algorithms if not supported by device */ + if (!priv->sec_attr.des_acc_num && + ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || + (c1_alg_sel == OP_ALG_ALGSEL_DES))) + continue; + + /* Skip AES algorithms if not supported by device */ + if (!priv->sec_attr.aes_acc_num && + (c1_alg_sel == OP_ALG_ALGSEL_AES)) + continue; + + /* + * Skip algorithms requiring message digests + * if MD not supported by device. + */ + if (!priv->sec_attr.md_acc_num && c2_alg_sel) + continue; + + t_alg->caam.dev = dev; + caam_aead_alg_init(t_alg); + + err = crypto_register_aead(&t_alg->aead); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->aead.base.cra_driver_name, err); + continue; + } + + t_alg->registered = true; + registered = true; + } + if (registered) + dev_info(dev, "algorithms registered in /proc/crypto\n"); + + /* register hash algorithms the device supports */ + INIT_LIST_HEAD(&hash_list); + + /* + * Skip registration of any hashing algorithms if MD block + * is not present. + */ + if (!priv->sec_attr.md_acc_num) + return 0; + + for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { + struct caam_hash_alg *t_alg; + struct caam_hash_template *alg = driver_hash + i; + + /* register hmac version */ + t_alg = caam_hash_alloc(dev, alg, true); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + dev_warn(dev, "%s hash alg allocation failed: %d\n", + alg->driver_name, err); + continue; + } + + err = crypto_register_ahash(&t_alg->ahash_alg); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->ahash_alg.halg.base.cra_driver_name, + err); + kfree(t_alg); + } else { + list_add_tail(&t_alg->entry, &hash_list); + } + + /* register unkeyed version */ + t_alg = caam_hash_alloc(dev, alg, false); + if (IS_ERR(t_alg)) { + err = PTR_ERR(t_alg); + dev_warn(dev, "%s alg allocation failed: %d\n", + alg->driver_name, err); + continue; + } + + err = crypto_register_ahash(&t_alg->ahash_alg); + if (err) { + dev_warn(dev, "%s alg registration failed: %d\n", + t_alg->ahash_alg.halg.base.cra_driver_name, + err); + kfree(t_alg); + } else { + list_add_tail(&t_alg->entry, &hash_list); + } + } + if (!list_empty(&hash_list)) + dev_info(dev, "hash algorithms registered in /proc/crypto\n"); + + return err; + +err_bind: + dpaa2_dpseci_dpio_free(priv); +err_dpio_setup: + dpaa2_dpseci_free(priv); +err_dpseci_setup: + free_percpu(priv->ppriv); +err_alloc_ppriv: + fsl_mc_portal_free(priv->mc_io); +err_dma_mask: + kmem_cache_destroy(qi_cache); +err_qicache: + dev_set_drvdata(dev, NULL); + + return err; +} + +static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) +{ + struct device *dev; + struct dpaa2_caam_priv *priv; + int i; + + dev = &ls_dev->dev; + priv = dev_get_drvdata(dev); + + for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { + struct caam_aead_alg *t_alg = driver_aeads + i; + + if (t_alg->registered) + crypto_unregister_aead(&t_alg->aead); + } + + if (alg_list.next) { + struct caam_crypto_alg *t_alg, *n; + + list_for_each_entry_safe(t_alg, n, &alg_list, entry) { + crypto_unregister_alg(&t_alg->crypto_alg); + list_del(&t_alg->entry); + kfree(t_alg); + } + } + + if (hash_list.next) { + struct caam_hash_alg *t_hash_alg, *p; + + list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { + crypto_unregister_ahash(&t_hash_alg->ahash_alg); + list_del(&t_hash_alg->entry); + kfree(t_hash_alg); + } + } + + dpaa2_dpseci_disable(priv); + dpaa2_dpseci_dpio_free(priv); + dpaa2_dpseci_free(priv); + free_percpu(priv->ppriv); + fsl_mc_portal_free(priv->mc_io); + dev_set_drvdata(dev, NULL); + kmem_cache_destroy(qi_cache); + + return 0; +} + +int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) +{ + struct dpaa2_fd fd; + struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); + int err = 0, i, id; + + if (IS_ERR(req)) + return PTR_ERR(req); + + if (priv->cscn_mem) { + dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, + DPAA2_CSCN_SIZE, + DMA_FROM_DEVICE); + if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { + dev_dbg_ratelimited(dev, "Dropping request\n"); + return -EBUSY; + } + } + + dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); + + req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, req->fd_flt_dma)) { + dev_err(dev, "DMA mapping error for QI enqueue request\n"); + goto err_out; + } + + memset(&fd, 0, sizeof(fd)); + dpaa2_fd_set_format(&fd, dpaa2_fd_list); + dpaa2_fd_set_addr(&fd, req->fd_flt_dma); + dpaa2_fd_set_len(&fd, req->fd_flt[1].len); + dpaa2_fd_set_flc(&fd, req->flc_dma); + + /* + * There is no guarantee that preemption is disabled here, + * thus take action. + */ + preempt_disable(); + id = smp_processor_id() % priv->dpseci_attr.num_tx_queues; + for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { + err = dpaa2_io_service_enqueue_fq(NULL, + priv->tx_queue_attr[id].fqid, + &fd); + if (err != -EBUSY) + break; + } + preempt_enable(); + + if (unlikely(err < 0)) { + dev_err(dev, "Error enqueuing frame: %d\n", err); + goto err_out; + } + + return -EINPROGRESS; + +err_out: + dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), + DMA_BIDIRECTIONAL); + return -EIO; +} +EXPORT_SYMBOL(dpaa2_caam_enqueue); + +const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { + { + .vendor = FSL_MC_VENDOR_FREESCALE, + .obj_type = "dpseci", + }, + { .vendor = 0x0 } +}; + +static struct fsl_mc_driver dpaa2_caam_driver = { + .driver = { + .name = KBUILD_MODNAME, + .owner = THIS_MODULE, + }, + .probe = dpaa2_caam_probe, + .remove = dpaa2_caam_remove, + .match_id_table = dpaa2_caam_match_id_table +}; + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Freescale Semiconductor, Inc"); +MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); + +module_fsl_mc_driver(dpaa2_caam_driver); --- /dev/null +++ b/drivers/crypto/caam/caamalg_qi2.h @@ -0,0 +1,281 @@ +/* + * Copyright 2015-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CAAMALG_QI2_H_ +#define _CAAMALG_QI2_H_ + +#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h" +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" +#include +#include "dpseci.h" +#include "desc_constr.h" + +#define DPAA2_CAAM_STORE_SIZE 16 +/* NAPI weight *must* be a multiple of the store size. */ +#define DPAA2_CAAM_NAPI_WEIGHT 64 + +/* The congestion entrance threshold was chosen so that on LS2088 + * we support the maximum throughput for the available memory + */ +#define DPAA2_SEC_CONG_ENTRY_THRESH (128 * 1024 * 1024) +#define DPAA2_SEC_CONG_EXIT_THRESH (DPAA2_SEC_CONG_ENTRY_THRESH * 9 / 10) + +/** + * dpaa2_caam_priv - driver private data + * @dpseci_id: DPSECI object unique ID + * @major_ver: DPSECI major version + * @minor_ver: DPSECI minor version + * @dpseci_attr: DPSECI attributes + * @sec_attr: SEC engine attributes + * @rx_queue_attr: array of Rx queue attributes + * @tx_queue_attr: array of Tx queue attributes + * @cscn_mem: pointer to memory region containing the + * dpaa2_cscn struct; it's size is larger than + * sizeof(struct dpaa2_cscn) to accommodate alignment + * @cscn_mem_aligned: pointer to struct dpaa2_cscn; it is computed + * as PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN) + * @cscn_dma: dma address used by the QMAN to write CSCN messages + * @dev: device associated with the DPSECI object + * @mc_io: pointer to MC portal's I/O object + * @domain: IOMMU domain + * @ppriv: per CPU pointers to privata data + */ +struct dpaa2_caam_priv { + int dpsec_id; + + u16 major_ver; + u16 minor_ver; + + struct dpseci_attr dpseci_attr; + struct dpseci_sec_attr sec_attr; + struct dpseci_rx_queue_attr rx_queue_attr[DPSECI_PRIO_NUM]; + struct dpseci_tx_queue_attr tx_queue_attr[DPSECI_PRIO_NUM]; + int num_pairs; + + /* congestion */ + void *cscn_mem; + void *cscn_mem_aligned; + dma_addr_t cscn_dma; + + struct device *dev; + struct fsl_mc_io *mc_io; + struct iommu_domain *domain; + + struct dpaa2_caam_priv_per_cpu __percpu *ppriv; +}; + +/** + * dpaa2_caam_priv_per_cpu - per CPU private data + * @napi: napi structure + * @net_dev: netdev used by napi + * @req_fqid: (virtual) request (Tx / enqueue) FQID + * @rsp_fqid: (virtual) response (Rx / dequeue) FQID + * @prio: internal queue number - index for dpaa2_caam_priv.*_queue_attr + * @nctx: notification context of response FQ + * @store: where dequeued frames are stored + * @priv: backpointer to dpaa2_caam_priv + */ +struct dpaa2_caam_priv_per_cpu { + struct napi_struct napi; + struct net_device net_dev; + int req_fqid; + int rsp_fqid; + int prio; + struct dpaa2_io_notification_ctx nctx; + struct dpaa2_io_store *store; + struct dpaa2_caam_priv *priv; +}; + +/* + * The CAAM QI hardware constructs a job descriptor which points + * to shared descriptor (as pointed by context_a of FQ to CAAM). + * When the job descriptor is executed by deco, the whole job + * descriptor together with shared descriptor gets loaded in + * deco buffer which is 64 words long (each 32-bit). + * + * The job descriptor constructed by QI hardware has layout: + * + * HEADER (1 word) + * Shdesc ptr (1 or 2 words) + * SEQ_OUT_PTR (1 word) + * Out ptr (1 or 2 words) + * Out length (1 word) + * SEQ_IN_PTR (1 word) + * In ptr (1 or 2 words) + * In length (1 word) + * + * The shdesc ptr is used to fetch shared descriptor contents + * into deco buffer. + * + * Apart from shdesc contents, the total number of words that + * get loaded in deco buffer are '8' or '11'. The remaining words + * in deco buffer can be used for storing shared descriptor. + */ +#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ) + +/* Length of a single buffer in the QI driver memory cache */ +#define CAAM_QI_MEMCACHE_SIZE 512 + +/* + * aead_edesc - s/w-extended aead descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @assoclen_dma: bus physical mapped address of req->assoclen + * @sgt: the h/w link table + */ +struct aead_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + dma_addr_t assoclen_dma; +#define CAAM_QI_MAX_AEAD_SG \ + ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) / \ + sizeof(struct dpaa2_sg_entry)) + struct dpaa2_sg_entry sgt[0]; +}; + +/* + * tls_edesc - s/w-extended tls descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped h/w link table + * @qm_sg_dma: bus physical mapped address of h/w link table + * @tmp: array of scatterlists used by 'scatterwalk_ffwd' + * @dst: pointer to output scatterlist, usefull for unmapping + * @sgt: the h/w link table + */ +struct tls_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; + struct scatterlist tmp[2]; + struct scatterlist *dst; + struct dpaa2_sg_entry sgt[0]; +}; + +/* + * ablkcipher_edesc - s/w-extended ablkcipher descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @qm_sg_bytes: length of dma mapped qm_sg space + * @qm_sg_dma: I/O virtual address of h/w link table + * @sgt: the h/w link table + */ +struct ablkcipher_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int qm_sg_bytes; + dma_addr_t qm_sg_dma; +#define CAAM_QI_MAX_ABLKCIPHER_SG \ + ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \ + sizeof(struct dpaa2_sg_entry)) + struct dpaa2_sg_entry sgt[0]; +}; + +/* + * ahash_edesc - s/w-extended ahash descriptor + * @dst_dma: I/O virtual address of req->result + * @qm_sg_dma: I/O virtual address of h/w link table + * @src_nents: number of segments in input scatterlist + * @qm_sg_bytes: length of dma mapped qm_sg space + * @sgt: pointer to h/w link table + */ +struct ahash_edesc { + dma_addr_t dst_dma; + dma_addr_t qm_sg_dma; + int src_nents; + int qm_sg_bytes; + struct dpaa2_sg_entry sgt[0]; +}; + +/** + * caam_flc - Flow Context (FLC) + * @flc: Flow Context options + * @sh_desc: Shared Descriptor + */ +struct caam_flc { + u32 flc[16]; + u32 sh_desc[MAX_SDLEN]; +} ____cacheline_aligned; + +enum optype { + ENCRYPT = 0, + DECRYPT, + GIVENCRYPT, + NUM_OP +}; + +/** + * caam_request - the request structure the driver application should fill while + * submitting a job to driver. + * @fd_flt: Frame list table defining input and output + * fd_flt[0] - FLE pointing to output buffer + * fd_flt[1] - FLE pointing to input buffer + * @fd_flt_dma: DMA address for the frame list table + * @flc: Flow Context + * @flc_dma: I/O virtual address of Flow Context + * @op_type: operation type + * @cbk: Callback function to invoke when job is completed + * @ctx: arbit context attached with request by the application + * @edesc: extended descriptor; points to one of {ablkcipher,aead}_edesc + */ +struct caam_request { + struct dpaa2_fl_entry fd_flt[2]; + dma_addr_t fd_flt_dma; + struct caam_flc *flc; + dma_addr_t flc_dma; + enum optype op_type; + void (*cbk)(void *ctx, u32 err); + void *ctx; + void *edesc; +}; + +/** + * dpaa2_caam_enqueue() - enqueue a crypto request + * @dev: device associated with the DPSECI object + * @req: pointer to caam_request + */ +int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req); + +#endif /* _CAAMALG_QI2_H_ */ --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -62,6 +62,7 @@ #include "error.h" #include "sg_sw_sec4.h" #include "key_gen.h" +#include "caamhash_desc.h" #define CAAM_CRA_PRIORITY 3000 @@ -71,14 +72,6 @@ #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE -/* length of descriptors text */ -#define DESC_AHASH_BASE (4 * CAAM_CMD_SZ) -#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) -#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) -#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) -#define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) -#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) - #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ CAAM_MAX_HASH_KEY_SIZE) #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) @@ -103,20 +96,14 @@ struct caam_hash_ctx { u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; - u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; dma_addr_t sh_desc_update_dma ____cacheline_aligned; dma_addr_t sh_desc_update_first_dma; dma_addr_t sh_desc_fin_dma; dma_addr_t sh_desc_digest_dma; - dma_addr_t sh_desc_finup_dma; struct device *jrdev; - u32 alg_type; - u32 alg_op; u8 key[CAAM_MAX_HASH_KEY_SIZE]; - dma_addr_t key_dma; int ctx_len; - unsigned int split_key_len; - unsigned int split_key_pad_len; + struct alginfo adata; }; /* ahash state */ @@ -143,6 +130,31 @@ struct caam_export_state { int (*finup)(struct ahash_request *req); }; +static inline void switch_buf(struct caam_hash_state *state) +{ + state->current_buf ^= 1; +} + +static inline u8 *current_buf(struct caam_hash_state *state) +{ + return state->current_buf ? state->buf_1 : state->buf_0; +} + +static inline u8 *alt_buf(struct caam_hash_state *state) +{ + return state->current_buf ? state->buf_0 : state->buf_1; +} + +static inline int *current_buflen(struct caam_hash_state *state) +{ + return state->current_buf ? &state->buflen_1 : &state->buflen_0; +} + +static inline int *alt_buflen(struct caam_hash_state *state) +{ + return state->current_buf ? &state->buflen_0 : &state->buflen_1; +} + /* Common job descriptor seq in/out ptr routines */ /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ @@ -175,40 +187,31 @@ static inline dma_addr_t map_seq_out_ptr return dst_dma; } -/* Map current buffer in state and put it in link table */ -static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev, - struct sec4_sg_entry *sec4_sg, - u8 *buf, int buflen) -{ - dma_addr_t buf_dma; +/* Map current buffer in state (if length > 0) and put it in link table */ +static inline int buf_map_to_sec4_sg(struct device *jrdev, + struct sec4_sg_entry *sec4_sg, + struct caam_hash_state *state) +{ + int buflen = *current_buflen(state); + + if (!buflen) + return 0; + + state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen, + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, state->buf_dma)) { + dev_err(jrdev, "unable to map buf\n"); + state->buf_dma = 0; + return -ENOMEM; + } - buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); - dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0); + dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0); - return buf_dma; -} - -/* - * Only put buffer in link table if it contains data, which is possible, - * since a buffer has previously been used, and needs to be unmapped, - */ -static inline dma_addr_t -try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg, - u8 *buf, dma_addr_t buf_dma, int buflen, - int last_buflen) -{ - if (buf_dma && !dma_mapping_error(jrdev, buf_dma)) - dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE); - if (buflen) - buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen); - else - buf_dma = 0; - - return buf_dma; + return 0; } /* Map state->caam_ctx, and add it to link table */ -static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, +static inline int ctx_map_to_sec4_sg(struct device *jrdev, struct caam_hash_state *state, int ctx_len, struct sec4_sg_entry *sec4_sg, u32 flag) { @@ -224,124 +227,22 @@ static inline int ctx_map_to_sec4_sg(u32 return 0; } -/* Common shared descriptor commands */ -static inline void append_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) -{ - append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, - ctx->split_key_len, CLASS_2 | - KEY_DEST_MDHA_SPLIT | KEY_ENC); -} - -/* Append key if it has been set */ -static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) -{ - u32 *key_jump_cmd; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - if (ctx->split_key_len) { - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - append_key_ahash(desc, ctx); - - set_jump_tgt_here(desc, key_jump_cmd); - } - - /* Propagate errors from shared to job descriptor */ - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); -} - -/* - * For ahash read data from seqin following state->caam_ctx, - * and write resulting class2 context to seqout, which may be state->caam_ctx - * or req->result - */ -static inline void ahash_append_load_str(u32 *desc, int digestsize) -{ - /* Calculate remaining bytes to read */ - append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); - - /* Read remaining bytes */ - append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | - FIFOLD_TYPE_MSG | KEY_VLF); - - /* Store class2 context bytes */ - append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | - LDST_SRCDST_BYTE_CONTEXT); -} - -/* - * For ahash update, final and finup, import context, read and write to seqout - */ -static inline void ahash_ctx_data_to_out(u32 *desc, u32 op, u32 state, - int digestsize, - struct caam_hash_ctx *ctx) -{ - init_sh_desc_key_ahash(desc, ctx); - - /* Import context from software */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_2_CCB | ctx->ctx_len); - - /* Class 2 operation */ - append_operation(desc, op | state | OP_ALG_ENCRYPT); - - /* - * Load from buf and/or src and write to req->result or state->context - */ - ahash_append_load_str(desc, digestsize); -} - -/* For ahash firsts and digest, read and write to seqout */ -static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state, - int digestsize, struct caam_hash_ctx *ctx) -{ - init_sh_desc_key_ahash(desc, ctx); - - /* Class 2 operation */ - append_operation(desc, op | state | OP_ALG_ENCRYPT); - - /* - * Load from buf and/or src and write to req->result or state->context - */ - ahash_append_load_str(desc, digestsize); -} - static int ahash_set_sh_desc(struct crypto_ahash *ahash) { struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); int digestsize = crypto_ahash_digestsize(ahash); struct device *jrdev = ctx->jrdev; - u32 have_key = 0; + struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; - if (ctx->split_key_len) - have_key = OP_ALG_AAI_HMAC_PRECOMP; + ctx->adata.key_virt = ctx->key; /* ahash_update shared descriptor */ desc = ctx->sh_desc_update; - - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Import context from software */ - append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | - LDST_CLASS_2_CCB | ctx->ctx_len); - - /* Class 2 operation */ - append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE | - OP_ALG_ENCRYPT); - - /* Load data and write to result or context */ - ahash_append_load_str(desc, ctx->ctx_len); - - ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, + ctx->ctx_len, true, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, + desc_bytes(desc), DMA_TO_DEVICE); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash update shdesc@"__stringify(__LINE__)": ", @@ -350,17 +251,10 @@ static int ahash_set_sh_desc(struct cryp /* ahash_update_first shared descriptor */ desc = ctx->sh_desc_update_first; - - ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT, - ctx->ctx_len, ctx); - - ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, + ctx->ctx_len, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, + desc_bytes(desc), DMA_TO_DEVICE); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash update first shdesc@"__stringify(__LINE__)": ", @@ -369,53 +263,22 @@ static int ahash_set_sh_desc(struct cryp /* ahash_final shared descriptor */ desc = ctx->sh_desc_fin; - - ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, - OP_ALG_AS_FINALIZE, digestsize, ctx); - - ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, + ctx->ctx_len, true, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, + desc_bytes(desc), DMA_TO_DEVICE); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); #endif - /* ahash_finup shared descriptor */ - desc = ctx->sh_desc_finup; - - ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, - OP_ALG_AS_FINALIZE, digestsize, ctx); - - ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif - /* ahash_digest shared descriptor */ desc = ctx->sh_desc_digest; - - ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL, - digestsize, ctx); - - ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc, - desc_bytes(desc), - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) { - dev_err(jrdev, "unable to map shared descriptor\n"); - return -ENOMEM; - } + cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, + ctx->ctx_len, false, ctrlpriv->era); + dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, + desc_bytes(desc), DMA_TO_DEVICE); #ifdef DEBUG print_hex_dump(KERN_ERR, "ahash digest shdesc@"__stringify(__LINE__)": ", @@ -426,14 +289,6 @@ static int ahash_set_sh_desc(struct cryp return 0; } -static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, - u32 keylen) -{ - return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, - ctx->split_key_pad_len, key_in, keylen, - ctx->alg_op); -} - /* Digest hash size if it is too large */ static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, u32 *keylen, u8 *key_out, u32 digestsize) @@ -469,7 +324,7 @@ static int hash_digest_key(struct caam_h } /* Job descriptor to perform unkeyed hash on key_in */ - append_operation(desc, ctx->alg_type | OP_ALG_ENCRYPT | + append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | OP_ALG_AS_INITFINAL); append_seq_in_ptr(desc, src_dma, *keylen, 0); append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | @@ -513,12 +368,10 @@ static int hash_digest_key(struct caam_h static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, unsigned int keylen) { - /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ - static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct device *jrdev = ctx->jrdev; int blocksize = crypto_tfm_alg_blocksize(&ahash->base); int digestsize = crypto_ahash_digestsize(ahash); + struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); int ret; u8 *hashed_key = NULL; @@ -539,43 +392,29 @@ static int ahash_setkey(struct crypto_ah key = hashed_key; } - /* Pick class 2 key length from algorithm submask */ - ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> - OP_ALG_ALGSEL_SHIFT] * 2; - ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); - -#ifdef DEBUG - printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", - ctx->split_key_len, ctx->split_key_pad_len); - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + /* + * If DKP is supported, use it in the shared descriptor to generate + * the split key. + */ + if (ctrlpriv->era >= 6) { + ctx->adata.key_inline = true; + ctx->adata.keylen = keylen; + ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & + OP_ALG_ALGSEL_MASK); - ret = gen_split_hash_key(ctx, key, keylen); - if (ret) - goto bad_free_key; + if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) + goto bad_free_key; - ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len, - DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, ctx->key_dma)) { - dev_err(jrdev, "unable to map key i/o memory\n"); - ret = -ENOMEM; - goto error_free_key; + memcpy(ctx->key, key, keylen); + } else { + ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, + keylen, CAAM_MAX_HASH_KEY_SIZE); + if (ret) + goto bad_free_key; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->split_key_pad_len, 1); -#endif - ret = ahash_set_sh_desc(ahash); - if (ret) { - dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len, - DMA_TO_DEVICE); - } - error_free_key: kfree(hashed_key); - return ret; + return ahash_set_sh_desc(ahash); bad_free_key: kfree(hashed_key); crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); @@ -604,6 +443,8 @@ static inline void ahash_unmap(struct de struct ahash_edesc *edesc, struct ahash_request *req, int dst_len) { + struct caam_hash_state *state = ahash_request_ctx(req); + if (edesc->src_nents) dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); if (edesc->dst_dma) @@ -612,6 +453,12 @@ static inline void ahash_unmap(struct de if (edesc->sec4_sg_bytes) dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes, DMA_TO_DEVICE); + + if (state->buf_dma) { + dma_unmap_single(dev, state->buf_dma, *current_buflen(state), + DMA_TO_DEVICE); + state->buf_dma = 0; + } } static inline void ahash_unmap_ctx(struct device *dev, @@ -643,8 +490,7 @@ static void ahash_done(struct device *jr dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); + edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); @@ -671,19 +517,19 @@ static void ahash_done_bi(struct device struct ahash_edesc *edesc; struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -#ifdef DEBUG struct caam_hash_state *state = ahash_request_ctx(req); +#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); + edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); + switch_buf(state); kfree(edesc); #ifdef DEBUG @@ -713,8 +559,7 @@ static void ahash_done_ctx_src(struct de dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); + edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); @@ -741,19 +586,19 @@ static void ahash_done_ctx_dst(struct de struct ahash_edesc *edesc; struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -#ifdef DEBUG struct caam_hash_state *state = ahash_request_ctx(req); +#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - edesc = (struct ahash_edesc *)((char *)desc - - offsetof(struct ahash_edesc, hw_desc)); + edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); + switch_buf(state); kfree(edesc); #ifdef DEBUG @@ -835,13 +680,12 @@ static int ahash_update_ctx(struct ahash struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; - int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; - u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; - int *next_buflen = state->current_buf ? &state->buflen_0 : - &state->buflen_1, last_buflen; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = current_buf(state); + int *buflen = current_buflen(state); + u8 *next_buf = alt_buf(state); + int *next_buflen = alt_buflen(state), last_buflen; int in_len = *buflen + req->nbytes, to_hash; u32 *desc; int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; @@ -890,15 +734,14 @@ static int ahash_update_ctx(struct ahash edesc->src_nents = src_nents; edesc->sec4_sg_bytes = sec4_sg_bytes; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_BIDIRECTIONAL); if (ret) goto unmap_ctx; - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, - edesc->sec4_sg + 1, - buf, state->buf_dma, - *buflen, last_buflen); + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); + if (ret) + goto unmap_ctx; if (mapped_nents) { sg_to_sec4_sg_last(req->src, mapped_nents, @@ -909,12 +752,10 @@ static int ahash_update_ctx(struct ahash to_hash - *buflen, *next_buflen, 0); } else { - (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= - cpu_to_caam32(SEC4_SG_LEN_FIN); + sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - + 1); } - state->current_buf = !state->current_buf; - desc = edesc->hw_desc; edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, @@ -969,12 +810,9 @@ static int ahash_final_ctx(struct ahash_ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; - int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; - int last_buflen = state->current_buf ? state->buflen_0 : - state->buflen_1; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = *current_buflen(state); u32 *desc; int sec4_sg_bytes, sec4_sg_src_index; int digestsize = crypto_ahash_digestsize(ahash); @@ -994,18 +832,17 @@ static int ahash_final_ctx(struct ahash_ desc = edesc->hw_desc; edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->src_nents = 0; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_TO_DEVICE); if (ret) goto unmap_ctx; - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, - buf, state->buf_dma, buflen, - last_buflen); - (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= - cpu_to_caam32(SEC4_SG_LEN_FIN); + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); + if (ret) + goto unmap_ctx; + + sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); @@ -1048,12 +885,9 @@ static int ahash_finup_ctx(struct ahash_ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; - int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; - int last_buflen = state->current_buf ? state->buflen_0 : - state->buflen_1; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = *current_buflen(state); u32 *desc; int sec4_sg_src_index; int src_nents, mapped_nents; @@ -1082,7 +916,7 @@ static int ahash_finup_ctx(struct ahash_ /* allocate space for base edesc and hw desc commands, link tables */ edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, - ctx->sh_desc_finup, ctx->sh_desc_finup_dma, + ctx->sh_desc_fin, ctx->sh_desc_fin_dma, flags); if (!edesc) { dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); @@ -1093,14 +927,14 @@ static int ahash_finup_ctx(struct ahash_ edesc->src_nents = src_nents; - ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, + ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, edesc->sec4_sg, DMA_TO_DEVICE); if (ret) goto unmap_ctx; - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, - buf, state->buf_dma, buflen, - last_buflen); + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); + if (ret) + goto unmap_ctx; ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, sec4_sg_src_index, ctx->ctx_len + buflen, @@ -1136,15 +970,18 @@ static int ahash_digest(struct ahash_req { struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; u32 *desc; int digestsize = crypto_ahash_digestsize(ahash); int src_nents, mapped_nents; struct ahash_edesc *edesc; int ret; + state->buf_dma = 0; + src_nents = sg_nents_for_len(req->src, req->nbytes); if (src_nents < 0) { dev_err(jrdev, "Invalid number of src SG.\n"); @@ -1215,10 +1052,10 @@ static int ahash_final_no_ctx(struct aha struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; - int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = current_buf(state); + int buflen = *current_buflen(state); u32 *desc; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; @@ -1249,7 +1086,6 @@ static int ahash_final_no_ctx(struct aha dev_err(jrdev, "unable to map dst\n"); goto unmap; } - edesc->src_nents = 0; #ifdef DEBUG print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", @@ -1279,13 +1115,12 @@ static int ahash_update_no_ctx(struct ah struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; - int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; - u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; - int *next_buflen = state->current_buf ? &state->buflen_0 : - &state->buflen_1; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *buf = current_buf(state); + int *buflen = current_buflen(state); + u8 *next_buf = alt_buf(state); + int *next_buflen = alt_buflen(state); int in_len = *buflen + req->nbytes, to_hash; int sec4_sg_bytes, src_nents, mapped_nents; struct ahash_edesc *edesc; @@ -1332,10 +1167,11 @@ static int ahash_update_no_ctx(struct ah edesc->src_nents = src_nents; edesc->sec4_sg_bytes = sec4_sg_bytes; - edesc->dst_dma = 0; - state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, - buf, *buflen); + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); + if (ret) + goto unmap_ctx; + sg_to_sec4_sg_last(req->src, mapped_nents, edesc->sec4_sg + 1, 0); @@ -1345,8 +1181,6 @@ static int ahash_update_no_ctx(struct ah *next_buflen, 0); } - state->current_buf = !state->current_buf; - desc = edesc->hw_desc; edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, @@ -1406,12 +1240,9 @@ static int ahash_finup_no_ctx(struct aha struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; - int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; - int last_buflen = state->current_buf ? state->buflen_0 : - state->buflen_1; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + int buflen = *current_buflen(state); u32 *desc; int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; int digestsize = crypto_ahash_digestsize(ahash); @@ -1453,9 +1284,9 @@ static int ahash_finup_no_ctx(struct aha edesc->src_nents = src_nents; edesc->sec4_sg_bytes = sec4_sg_bytes; - state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, - state->buf_dma, buflen, - last_buflen); + ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); + if (ret) + goto unmap; ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, req->nbytes); @@ -1499,11 +1330,10 @@ static int ahash_update_first(struct aha struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; - u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0; - int *next_buflen = state->current_buf ? - &state->buflen_1 : &state->buflen_0; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + u8 *next_buf = alt_buf(state); + int *next_buflen = alt_buflen(state); int to_hash; u32 *desc; int src_nents, mapped_nents; @@ -1548,7 +1378,6 @@ static int ahash_update_first(struct aha } edesc->src_nents = src_nents; - edesc->dst_dma = 0; ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, to_hash); @@ -1585,6 +1414,7 @@ static int ahash_update_first(struct aha state->final = ahash_final_no_ctx; scatterwalk_map_and_copy(next_buf, req->src, 0, req->nbytes, 0); + switch_buf(state); } #ifdef DEBUG print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", @@ -1691,7 +1521,6 @@ struct caam_hash_template { unsigned int blocksize; struct ahash_alg template_ahash; u32 alg_type; - u32 alg_op; }; /* ahash descriptors */ @@ -1717,7 +1546,6 @@ static struct caam_hash_template driver_ }, }, .alg_type = OP_ALG_ALGSEL_SHA1, - .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, }, { .name = "sha224", .driver_name = "sha224-caam", @@ -1739,7 +1567,6 @@ static struct caam_hash_template driver_ }, }, .alg_type = OP_ALG_ALGSEL_SHA224, - .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, }, { .name = "sha256", .driver_name = "sha256-caam", @@ -1761,7 +1588,6 @@ static struct caam_hash_template driver_ }, }, .alg_type = OP_ALG_ALGSEL_SHA256, - .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, }, { .name = "sha384", .driver_name = "sha384-caam", @@ -1783,7 +1609,6 @@ static struct caam_hash_template driver_ }, }, .alg_type = OP_ALG_ALGSEL_SHA384, - .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, }, { .name = "sha512", .driver_name = "sha512-caam", @@ -1805,7 +1630,6 @@ static struct caam_hash_template driver_ }, }, .alg_type = OP_ALG_ALGSEL_SHA512, - .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, }, { .name = "md5", .driver_name = "md5-caam", @@ -1827,14 +1651,12 @@ static struct caam_hash_template driver_ }, }, .alg_type = OP_ALG_ALGSEL_MD5, - .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, }, }; struct caam_hash_alg { struct list_head entry; int alg_type; - int alg_op; struct ahash_alg ahash_alg; }; @@ -1856,6 +1678,7 @@ static int caam_hash_cra_init(struct cry HASH_MSG_LEN + SHA256_DIGEST_SIZE, HASH_MSG_LEN + 64, HASH_MSG_LEN + SHA512_DIGEST_SIZE }; + dma_addr_t dma_addr; /* * Get a Job ring from Job Ring driver to ensure in-order @@ -1866,11 +1689,31 @@ static int caam_hash_cra_init(struct cry pr_err("Job Ring Device allocation for transform failed\n"); return PTR_ERR(ctx->jrdev); } + + dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update, + offsetof(struct caam_hash_ctx, + sh_desc_update_dma), + DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(ctx->jrdev, dma_addr)) { + dev_err(ctx->jrdev, "unable to map shared descriptors\n"); + caam_jr_free(ctx->jrdev); + return -ENOMEM; + } + + ctx->sh_desc_update_dma = dma_addr; + ctx->sh_desc_update_first_dma = dma_addr + + offsetof(struct caam_hash_ctx, + sh_desc_update_first); + ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx, + sh_desc_fin); + ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx, + sh_desc_digest); + /* copy descriptor header template value */ - ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; - ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op; + ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; - ctx->ctx_len = runninglen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> + ctx->ctx_len = runninglen[(ctx->adata.algtype & + OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT]; crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), @@ -1882,30 +1725,10 @@ static void caam_hash_cra_exit(struct cr { struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->sh_desc_update_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma, - desc_bytes(ctx->sh_desc_update), - DMA_TO_DEVICE); - if (ctx->sh_desc_update_first_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma, - desc_bytes(ctx->sh_desc_update_first), - DMA_TO_DEVICE); - if (ctx->sh_desc_fin_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma, - desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE); - if (ctx->sh_desc_digest_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma, - desc_bytes(ctx->sh_desc_digest), - DMA_TO_DEVICE); - if (ctx->sh_desc_finup_dma && - !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma)) - dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma, - desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE); - + dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma, + offsetof(struct caam_hash_ctx, + sh_desc_update_dma), + DMA_TO_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); caam_jr_free(ctx->jrdev); } @@ -1964,7 +1787,6 @@ caam_hash_alloc(struct caam_hash_templat alg->cra_type = &crypto_ahash_type; t_alg->alg_type = template->alg_type; - t_alg->alg_op = template->alg_op; return t_alg; } --- /dev/null +++ b/drivers/crypto/caam/caamhash_desc.c @@ -0,0 +1,108 @@ +/* + * Shared descriptors for ahash algorithms + * + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "compat.h" +#include "desc_constr.h" +#include "caamhash_desc.h" + +/** + * cnstr_shdsc_ahash - ahash shared descriptor + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * A split key is required for SEC Era < 6; the size of the split key + * is specified in this case. + * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224, + * SHA256, SHA384, SHA512}. + * @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE} + * @digestsize: algorithm's digest size + * @ctx_len: size of Context Register + * @import_ctx: true if previous Context Register needs to be restored + * must be true for ahash update and final + * must be false for for ahash first and digest + * @era: SEC Era + */ +void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state, + int digestsize, int ctx_len, bool import_ctx, int era) +{ + u32 op = adata->algtype; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Append key if it has been set; ahash update excluded */ + if (state != OP_ALG_AS_UPDATE && adata->keylen) { + u32 *skip_key_load; + + /* Skip key loading if already shared */ + skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + + if (era < 6) + append_key_as_imm(desc, adata->key_virt, + adata->keylen_pad, + adata->keylen, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + else + append_proto_dkp(desc, adata); + + set_jump_tgt_here(desc, skip_key_load); + + op |= OP_ALG_AAI_HMAC_PRECOMP; + } + + /* If needed, import context from software */ + if (import_ctx) + append_seq_load(desc, ctx_len, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + /* Class 2 operation */ + append_operation(desc, op | state | OP_ALG_ENCRYPT); + + /* + * Load from buf and/or src and write to req->result or state->context + * Calculate remaining bytes to read + */ + append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); + /* Read remaining bytes */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | + FIFOLD_TYPE_MSG | KEY_VLF); + /* Store class2 context bytes */ + append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); +} +EXPORT_SYMBOL(cnstr_shdsc_ahash); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("FSL CAAM ahash descriptors support"); +MODULE_AUTHOR("NXP Semiconductors"); --- /dev/null +++ b/drivers/crypto/caam/caamhash_desc.h @@ -0,0 +1,49 @@ +/* + * Shared descriptors for ahash algorithms + * + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _CAAMHASH_DESC_H_ +#define _CAAMHASH_DESC_H_ + +/* length of descriptors text */ +#define DESC_AHASH_BASE (3 * CAAM_CMD_SZ) +#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) +#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) +#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) +#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) + +void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state, + int digestsize, int ctx_len, bool import_ctx, int era); + +#endif /* _CAAMHASH_DESC_H_ */ --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -18,6 +18,10 @@ #define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb)) #define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \ sizeof(struct rsa_priv_f1_pdb)) +#define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \ + sizeof(struct rsa_priv_f2_pdb)) +#define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \ + sizeof(struct rsa_priv_f3_pdb)) static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, struct akcipher_request *req) @@ -54,6 +58,42 @@ static void rsa_priv_f1_unmap(struct dev dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); } +static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE); +} + +static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc, + struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); + dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE); +} + /* RSA Job Completion handler */ static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context) { @@ -90,6 +130,42 @@ static void rsa_priv_f1_done(struct devi akcipher_request_complete(req, err); } +static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_priv_f2_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + +static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err, + void *context) +{ + struct akcipher_request *req = context; + struct rsa_edesc *edesc; + + if (err) + caam_jr_strstatus(dev, err); + + edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); + + rsa_priv_f3_unmap(dev, edesc, req); + rsa_io_unmap(dev, edesc, req); + kfree(edesc); + + akcipher_request_complete(req, err); +} + static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, size_t desclen) { @@ -97,8 +173,8 @@ static struct rsa_edesc *rsa_edesc_alloc struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct device *dev = ctx->dev; struct rsa_edesc *edesc; - gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; + gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; int sgc; int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; int src_nents, dst_nents; @@ -258,6 +334,172 @@ static int set_rsa_priv_f1_pdb(struct ak return 0; } +static int set_rsa_priv_f2_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; + int sec4_sg_index = 0; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->d_dma)) { + dev_err(dev, "Unable to map RSA private exponent memory\n"); + return -ENOMEM; + } + + pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->p_dma)) { + dev_err(dev, "Unable to map RSA prime factor p memory\n"); + goto unmap_d; + } + + pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->q_dma)) { + dev_err(dev, "Unable to map RSA prime factor q memory\n"); + goto unmap_p; + } + + pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp1_dma)) { + dev_err(dev, "Unable to map RSA tmp1 memory\n"); + goto unmap_q; + } + + pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp2_dma)) { + dev_err(dev, "Unable to map RSA tmp2 memory\n"); + goto unmap_tmp1; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->g_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz; + pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; + + return 0; + +unmap_tmp1: + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); +unmap_q: + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); +unmap_p: + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); +unmap_d: + dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); + + return -ENOMEM; +} + +static int set_rsa_priv_f3_pdb(struct akcipher_request *req, + struct rsa_edesc *edesc) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + struct device *dev = ctx->dev; + struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; + int sec4_sg_index = 0; + size_t p_sz = key->p_sz; + size_t q_sz = key->p_sz; + + pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->p_dma)) { + dev_err(dev, "Unable to map RSA prime factor p memory\n"); + return -ENOMEM; + } + + pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->q_dma)) { + dev_err(dev, "Unable to map RSA prime factor q memory\n"); + goto unmap_p; + } + + pdb->dp_dma = dma_map_single(dev, key->dp, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->dp_dma)) { + dev_err(dev, "Unable to map RSA exponent dp memory\n"); + goto unmap_q; + } + + pdb->dq_dma = dma_map_single(dev, key->dq, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->dq_dma)) { + dev_err(dev, "Unable to map RSA exponent dq memory\n"); + goto unmap_dp; + } + + pdb->c_dma = dma_map_single(dev, key->qinv, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->c_dma)) { + dev_err(dev, "Unable to map RSA CRT coefficient qinv memory\n"); + goto unmap_dq; + } + + pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp1_dma)) { + dev_err(dev, "Unable to map RSA tmp1 memory\n"); + goto unmap_qinv; + } + + pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pdb->tmp2_dma)) { + dev_err(dev, "Unable to map RSA tmp2 memory\n"); + goto unmap_tmp1; + } + + if (edesc->src_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_G; + pdb->g_dma = edesc->sec4_sg_dma; + sec4_sg_index += edesc->src_nents; + } else { + pdb->g_dma = sg_dma_address(req->src); + } + + if (edesc->dst_nents > 1) { + pdb->sgf |= RSA_PRIV_PDB_SGF_F; + pdb->f_dma = edesc->sec4_sg_dma + + sec4_sg_index * sizeof(struct sec4_sg_entry); + } else { + pdb->f_dma = sg_dma_address(req->dst); + } + + pdb->sgf |= key->n_sz; + pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; + + return 0; + +unmap_tmp1: + dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); +unmap_qinv: + dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); +unmap_dq: + dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); +unmap_dp: + dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); +unmap_q: + dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); +unmap_p: + dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); + + return -ENOMEM; +} + static int caam_rsa_enc(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); @@ -301,24 +543,14 @@ init_fail: return ret; } -static int caam_rsa_dec(struct akcipher_request *req) +static int caam_rsa_dec_priv_f1(struct akcipher_request *req) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct caam_rsa_key *key = &ctx->key; struct device *jrdev = ctx->dev; struct rsa_edesc *edesc; int ret; - if (unlikely(!key->n || !key->d)) - return -EINVAL; - - if (req->dst_len < key->n_sz) { - req->dst_len = key->n_sz; - dev_err(jrdev, "Output buffer length less than parameter n\n"); - return -EOVERFLOW; - } - /* Allocate extended descriptor */ edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN); if (IS_ERR(edesc)) @@ -344,17 +576,147 @@ init_fail: return ret; } +static int caam_rsa_dec_priv_f2(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F2_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #2 */ + ret = set_rsa_priv_f2_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f2_desc(edesc->hw_desc, &edesc->pdb.priv_f2); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f2_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_priv_f2_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec_priv_f3(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct device *jrdev = ctx->dev; + struct rsa_edesc *edesc; + int ret; + + /* Allocate extended descriptor */ + edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F3_LEN); + if (IS_ERR(edesc)) + return PTR_ERR(edesc); + + /* Set RSA Decrypt Protocol Data Block - Private Key Form #3 */ + ret = set_rsa_priv_f3_pdb(req, edesc); + if (ret) + goto init_fail; + + /* Initialize Job Descriptor */ + init_rsa_priv_f3_desc(edesc->hw_desc, &edesc->pdb.priv_f3); + + ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f3_done, req); + if (!ret) + return -EINPROGRESS; + + rsa_priv_f3_unmap(jrdev, edesc, req); + +init_fail: + rsa_io_unmap(jrdev, edesc, req); + kfree(edesc); + return ret; +} + +static int caam_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct caam_rsa_key *key = &ctx->key; + int ret; + + if (unlikely(!key->n || !key->d)) + return -EINVAL; + + if (req->dst_len < key->n_sz) { + req->dst_len = key->n_sz; + dev_err(ctx->dev, "Output buffer length less than parameter n\n"); + return -EOVERFLOW; + } + + if (key->priv_form == FORM3) + ret = caam_rsa_dec_priv_f3(req); + else if (key->priv_form == FORM2) + ret = caam_rsa_dec_priv_f2(req); + else + ret = caam_rsa_dec_priv_f1(req); + + return ret; +} + static void caam_rsa_free_key(struct caam_rsa_key *key) { kzfree(key->d); + kzfree(key->p); + kzfree(key->q); + kzfree(key->dp); + kzfree(key->dq); + kzfree(key->qinv); + kzfree(key->tmp1); + kzfree(key->tmp2); kfree(key->e); kfree(key->n); - key->d = NULL; - key->e = NULL; - key->n = NULL; - key->d_sz = 0; - key->e_sz = 0; - key->n_sz = 0; + memset(key, 0, sizeof(*key)); +} + +static void caam_rsa_drop_leading_zeros(const u8 **ptr, size_t *nbytes) +{ + while (!**ptr && *nbytes) { + (*ptr)++; + (*nbytes)--; + } +} + +/** + * caam_read_rsa_crt - Used for reading dP, dQ, qInv CRT members. + * dP, dQ and qInv could decode to less than corresponding p, q length, as the + * BER-encoding requires that the minimum number of bytes be used to encode the + * integer. dP, dQ, qInv decoded values have to be zero-padded to appropriate + * length. + * + * @ptr : pointer to {dP, dQ, qInv} CRT member + * @nbytes: length in bytes of {dP, dQ, qInv} CRT member + * @dstlen: length in bytes of corresponding p or q prime factor + */ +static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen) +{ + u8 *dst; + + caam_rsa_drop_leading_zeros(&ptr, &nbytes); + if (!nbytes) + return NULL; + + dst = kzalloc(dstlen, GFP_DMA | GFP_KERNEL); + if (!dst) + return NULL; + + memcpy(dst + (dstlen - nbytes), ptr, nbytes); + + return dst; } /** @@ -370,10 +732,9 @@ static inline u8 *caam_read_raw_data(con { u8 *val; - while (!*buf && *nbytes) { - buf++; - (*nbytes)--; - } + caam_rsa_drop_leading_zeros(&buf, nbytes); + if (!*nbytes) + return NULL; val = kzalloc(*nbytes, GFP_DMA | GFP_KERNEL); if (!val) @@ -395,7 +756,7 @@ static int caam_rsa_set_pub_key(struct c unsigned int keylen) { struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct rsa_key raw_key = {0}; + struct rsa_key raw_key = {NULL}; struct caam_rsa_key *rsa_key = &ctx->key; int ret; @@ -437,11 +798,69 @@ err: return -ENOMEM; } +static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, + struct rsa_key *raw_key) +{ + struct caam_rsa_key *rsa_key = &ctx->key; + size_t p_sz = raw_key->p_sz; + size_t q_sz = raw_key->q_sz; + + rsa_key->p = caam_read_raw_data(raw_key->p, &p_sz); + if (!rsa_key->p) + return; + rsa_key->p_sz = p_sz; + + rsa_key->q = caam_read_raw_data(raw_key->q, &q_sz); + if (!rsa_key->q) + goto free_p; + rsa_key->q_sz = q_sz; + + rsa_key->tmp1 = kzalloc(raw_key->p_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->tmp1) + goto free_q; + + rsa_key->tmp2 = kzalloc(raw_key->q_sz, GFP_DMA | GFP_KERNEL); + if (!rsa_key->tmp2) + goto free_tmp1; + + rsa_key->priv_form = FORM2; + + rsa_key->dp = caam_read_rsa_crt(raw_key->dp, raw_key->dp_sz, p_sz); + if (!rsa_key->dp) + goto free_tmp2; + + rsa_key->dq = caam_read_rsa_crt(raw_key->dq, raw_key->dq_sz, q_sz); + if (!rsa_key->dq) + goto free_dp; + + rsa_key->qinv = caam_read_rsa_crt(raw_key->qinv, raw_key->qinv_sz, + q_sz); + if (!rsa_key->qinv) + goto free_dq; + + rsa_key->priv_form = FORM3; + + return; + +free_dq: + kzfree(rsa_key->dq); +free_dp: + kzfree(rsa_key->dp); +free_tmp2: + kzfree(rsa_key->tmp2); +free_tmp1: + kzfree(rsa_key->tmp1); +free_q: + kzfree(rsa_key->q); +free_p: + kzfree(rsa_key->p); +} + static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct rsa_key raw_key = {0}; + struct rsa_key raw_key = {NULL}; struct caam_rsa_key *rsa_key = &ctx->key; int ret; @@ -483,6 +902,8 @@ static int caam_rsa_set_priv_key(struct memcpy(rsa_key->d, raw_key.d, raw_key.d_sz); memcpy(rsa_key->e, raw_key.e, raw_key.e_sz); + caam_rsa_set_priv_key_form(ctx, &raw_key); + return 0; err: --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -13,21 +13,75 @@ #include "pdb.h" /** + * caam_priv_key_form - CAAM RSA private key representation + * CAAM RSA private key may have either of three forms. + * + * 1. The first representation consists of the pair (n, d), where the + * components have the following meanings: + * n the RSA modulus + * d the RSA private exponent + * + * 2. The second representation consists of the triplet (p, q, d), where the + * components have the following meanings: + * p the first prime factor of the RSA modulus n + * q the second prime factor of the RSA modulus n + * d the RSA private exponent + * + * 3. The third representation consists of the quintuple (p, q, dP, dQ, qInv), + * where the components have the following meanings: + * p the first prime factor of the RSA modulus n + * q the second prime factor of the RSA modulus n + * dP the first factors's CRT exponent + * dQ the second factors's CRT exponent + * qInv the (first) CRT coefficient + * + * The benefit of using the third or the second key form is lower computational + * cost for the decryption and signature operations. + */ +enum caam_priv_key_form { + FORM1, + FORM2, + FORM3 +}; + +/** * caam_rsa_key - CAAM RSA key structure. Keys are allocated in DMA zone. * @n : RSA modulus raw byte stream * @e : RSA public exponent raw byte stream * @d : RSA private exponent raw byte stream + * @p : RSA prime factor p of RSA modulus n + * @q : RSA prime factor q of RSA modulus n + * @dp : RSA CRT exponent of p + * @dp : RSA CRT exponent of q + * @qinv : RSA CRT coefficient + * @tmp1 : CAAM uses this temporary buffer as internal state buffer. + * It is assumed to be as long as p. + * @tmp2 : CAAM uses this temporary buffer as internal state buffer. + * It is assumed to be as long as q. * @n_sz : length in bytes of RSA modulus n * @e_sz : length in bytes of RSA public exponent * @d_sz : length in bytes of RSA private exponent + * @p_sz : length in bytes of RSA prime factor p of RSA modulus n + * @q_sz : length in bytes of RSA prime factor q of RSA modulus n + * @priv_form : CAAM RSA private key representation */ struct caam_rsa_key { u8 *n; u8 *e; u8 *d; + u8 *p; + u8 *q; + u8 *dp; + u8 *dq; + u8 *qinv; + u8 *tmp1; + u8 *tmp2; size_t n_sz; size_t e_sz; size_t d_sz; + size_t p_sz; + size_t q_sz; + enum caam_priv_key_form priv_form; }; /** @@ -59,6 +113,8 @@ struct rsa_edesc { union { struct rsa_pub_pdb pub; struct rsa_priv_f1_pdb priv_f1; + struct rsa_priv_f2_pdb priv_f2; + struct rsa_priv_f3_pdb priv_f3; } pdb; u32 hw_desc[]; }; @@ -66,5 +122,7 @@ struct rsa_edesc { /* Descriptor construction primitives. */ void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb); void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb); +void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb); +void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb); #endif --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -52,7 +52,7 @@ /* length of descriptors */ #define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2) -#define DESC_RNG_LEN (4 * CAAM_CMD_SZ) +#define DESC_RNG_LEN (3 * CAAM_CMD_SZ) /* Buffer, its dma address and lock */ struct buf_data { @@ -100,8 +100,7 @@ static void rng_done(struct device *jrde { struct buf_data *bd; - bd = (struct buf_data *)((char *)desc - - offsetof(struct buf_data, hw_desc)); + bd = container_of(desc, struct buf_data, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); @@ -196,9 +195,6 @@ static inline int rng_create_sh_desc(str init_sh_desc(desc, HDR_SHARE_SERIAL); - /* Propagate errors from shared to job descriptor */ - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); - /* Generate random bytes */ append_operation(desc, OP_ALG_ALGSEL_RNG | OP_TYPE_CLASS1_ALG); @@ -289,11 +285,7 @@ static int caam_init_rng(struct caam_rng if (err) return err; - err = caam_init_buf(ctx, 1); - if (err) - return err; - - return 0; + return caam_init_buf(ctx, 1); } static struct hwrng caam_rng = { @@ -351,7 +343,7 @@ static int __init caam_rng_init(void) pr_err("Job Ring Device allocation for transform failed\n"); return PTR_ERR(dev); } - rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA); + rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL); if (!rng_ctx) { err = -ENOMEM; goto free_caam_alloc; --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -2,40 +2,41 @@ * Controller-level driver, kernel property detection, initialization * * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2017 NXP */ #include #include #include +#include #include "compat.h" #include "regs.h" #include "intern.h" #include "jr.h" #include "desc_constr.h" -#include "error.h" #include "ctrl.h" bool caam_little_end; EXPORT_SYMBOL(caam_little_end); +bool caam_imx; +EXPORT_SYMBOL(caam_imx); +bool caam_dpaa2; +EXPORT_SYMBOL(caam_dpaa2); + +#ifdef CONFIG_CAAM_QI +#include "qi.h" +#endif /* * i.MX targets tend to have clock control subsystems that can * enable/disable clocking to our device. */ -#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX -static inline struct clk *caam_drv_identify_clk(struct device *dev, - char *clk_name) -{ - return devm_clk_get(dev, clk_name); -} -#else static inline struct clk *caam_drv_identify_clk(struct device *dev, char *clk_name) { - return NULL; + return caam_imx ? devm_clk_get(dev, clk_name) : NULL; } -#endif /* * Descriptor to instantiate RNG State Handle 0 in normal mode and @@ -274,7 +275,7 @@ static int deinstantiate_rng(struct devi /* * If the corresponding bit is set, then it means the state * handle was initialized by us, and thus it needs to be - * deintialized as well + * deinitialized as well */ if ((1 << sh_idx) & state_handle_mask) { /* @@ -307,20 +308,24 @@ static int caam_remove(struct platform_d struct device *ctrldev; struct caam_drv_private *ctrlpriv; struct caam_ctrl __iomem *ctrl; - int ring; ctrldev = &pdev->dev; ctrlpriv = dev_get_drvdata(ctrldev); ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; - /* Remove platform devices for JobRs */ - for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { - if (ctrlpriv->jrpdev[ring]) - of_device_unregister(ctrlpriv->jrpdev[ring]); - } + /* Remove platform devices under the crypto node */ + of_platform_depopulate(ctrldev); + +#ifdef CONFIG_CAAM_QI + if (ctrlpriv->qidev) + caam_qi_shutdown(ctrlpriv->qidev); +#endif - /* De-initialize RNG state handles initialized by this driver. */ - if (ctrlpriv->rng4_sh_init) + /* + * De-initialize RNG state handles initialized by this driver. + * In case of DPAA 2.x, RNG is managed by MC firmware. + */ + if (!caam_dpaa2 && ctrlpriv->rng4_sh_init) deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); /* Shut down debug views */ @@ -335,8 +340,8 @@ static int caam_remove(struct platform_d clk_disable_unprepare(ctrlpriv->caam_ipg); clk_disable_unprepare(ctrlpriv->caam_mem); clk_disable_unprepare(ctrlpriv->caam_aclk); - clk_disable_unprepare(ctrlpriv->caam_emi_slow); - + if (ctrlpriv->caam_emi_slow) + clk_disable_unprepare(ctrlpriv->caam_emi_slow); return 0; } @@ -370,11 +375,8 @@ static void kick_trng(struct platform_de */ val = (rd_reg32(&r4tst->rtsdctl) & RTSDCTL_ENT_DLY_MASK) >> RTSDCTL_ENT_DLY_SHIFT; - if (ent_delay <= val) { - /* put RNG4 into run mode */ - clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, 0); - return; - } + if (ent_delay <= val) + goto start_rng; val = rd_reg32(&r4tst->rtsdctl); val = (val & ~RTSDCTL_ENT_DLY_MASK) | @@ -386,15 +388,12 @@ static void kick_trng(struct platform_de wr_reg32(&r4tst->rtfrqmax, RTFRQMAX_DISABLE); /* read the control register */ val = rd_reg32(&r4tst->rtmctl); +start_rng: /* * select raw sampling in both entropy shifter - * and statistical checker + * and statistical checker; ; put RNG4 into run mode */ - clrsetbits_32(&val, 0, RTMCTL_SAMP_MODE_RAW_ES_SC); - /* put RNG4 into run mode */ - clrsetbits_32(&val, RTMCTL_PRGM, 0); - /* write back the control register */ - wr_reg32(&r4tst->rtmctl, val); + clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, RTMCTL_SAMP_MODE_RAW_ES_SC); } /** @@ -415,28 +414,26 @@ int caam_get_era(void) } EXPORT_SYMBOL(caam_get_era); -#ifdef CONFIG_DEBUG_FS -static int caam_debugfs_u64_get(void *data, u64 *val) -{ - *val = caam64_to_cpu(*(u64 *)data); - return 0; -} - -static int caam_debugfs_u32_get(void *data, u64 *val) -{ - *val = caam32_to_cpu(*(u32 *)data); - return 0; -} - -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); -DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); -#endif +static const struct of_device_id caam_match[] = { + { + .compatible = "fsl,sec-v4.0", + }, + { + .compatible = "fsl,sec4.0", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, caam_match); /* Probe routine for CAAM top (controller) level */ static int caam_probe(struct platform_device *pdev) { - int ret, ring, rspec, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; + int ret, ring, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; u64 caam_id; + static const struct soc_device_attribute imx_soc[] = { + {.family = "Freescale i.MX"}, + {}, + }; struct device *dev; struct device_node *nprop, *np; struct caam_ctrl __iomem *ctrl; @@ -456,9 +453,10 @@ static int caam_probe(struct platform_de dev = &pdev->dev; dev_set_drvdata(dev, ctrlpriv); - ctrlpriv->pdev = pdev; nprop = pdev->dev.of_node; + caam_imx = (bool)soc_device_match(imx_soc); + /* Enable clocking */ clk = caam_drv_identify_clk(&pdev->dev, "ipg"); if (IS_ERR(clk)) { @@ -487,14 +485,16 @@ static int caam_probe(struct platform_de } ctrlpriv->caam_aclk = clk; - clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); - if (IS_ERR(clk)) { - ret = PTR_ERR(clk); - dev_err(&pdev->dev, - "can't identify CAAM emi_slow clk: %d\n", ret); - return ret; + if (!of_machine_is_compatible("fsl,imx6ul")) { + clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(&pdev->dev, + "can't identify CAAM emi_slow clk: %d\n", ret); + return ret; + } + ctrlpriv->caam_emi_slow = clk; } - ctrlpriv->caam_emi_slow = clk; ret = clk_prepare_enable(ctrlpriv->caam_ipg); if (ret < 0) { @@ -515,11 +515,13 @@ static int caam_probe(struct platform_de goto disable_caam_mem; } - ret = clk_prepare_enable(ctrlpriv->caam_emi_slow); - if (ret < 0) { - dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n", - ret); - goto disable_caam_aclk; + if (ctrlpriv->caam_emi_slow) { + ret = clk_prepare_enable(ctrlpriv->caam_emi_slow); + if (ret < 0) { + dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n", + ret); + goto disable_caam_aclk; + } } /* Get configuration properties from device tree */ @@ -546,13 +548,13 @@ static int caam_probe(struct platform_de else BLOCK_OFFSET = PG_SIZE_64K; - ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl; - ctrlpriv->assure = (struct caam_assurance __force *) - ((uint8_t *)ctrl + + ctrlpriv->ctrl = (struct caam_ctrl __iomem __force *)ctrl; + ctrlpriv->assure = (struct caam_assurance __iomem __force *) + ((__force uint8_t *)ctrl + BLOCK_OFFSET * ASSURE_BLOCK_NUMBER ); - ctrlpriv->deco = (struct caam_deco __force *) - ((uint8_t *)ctrl + + ctrlpriv->deco = (struct caam_deco __iomem __force *) + ((__force uint8_t *)ctrl + BLOCK_OFFSET * DECO_BLOCK_NUMBER ); @@ -561,12 +563,17 @@ static int caam_probe(struct platform_de /* * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, - * long pointers in master configuration register + * long pointers in master configuration register. + * In case of DPAA 2.x, Management Complex firmware performs + * the configuration. */ - clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR, - MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | - MCFGR_WDENABLE | MCFGR_LARGE_BURST | - (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); + caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2); + if (!caam_dpaa2) + clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR, + MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | + MCFGR_WDENABLE | MCFGR_LARGE_BURST | + (sizeof(dma_addr_t) == sizeof(u64) ? + MCFGR_LONG_PTR : 0)); /* * Read the Compile Time paramters and SCFGR to determine @@ -594,64 +601,69 @@ static int caam_probe(struct platform_de JRSTART_JR1_START | JRSTART_JR2_START | JRSTART_JR3_START); - if (sizeof(dma_addr_t) == sizeof(u64)) - if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) - dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); + if (sizeof(dma_addr_t) == sizeof(u64)) { + if (caam_dpaa2) + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); + else if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); else - dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); - else - dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); + } else { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); + } + if (ret) { + dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); + goto iounmap_ctrl; + } - /* - * Detect and enable JobRs - * First, find out how many ring spec'ed, allocate references - * for all, then go probe each one. - */ - rspec = 0; - for_each_available_child_of_node(nprop, np) - if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || - of_device_is_compatible(np, "fsl,sec4.0-job-ring")) - rspec++; + ctrlpriv->era = caam_get_era(); - ctrlpriv->jrpdev = devm_kcalloc(&pdev->dev, rspec, - sizeof(*ctrlpriv->jrpdev), GFP_KERNEL); - if (ctrlpriv->jrpdev == NULL) { - ret = -ENOMEM; + ret = of_platform_populate(nprop, caam_match, NULL, dev); + if (ret) { + dev_err(dev, "JR platform devices creation error\n"); goto iounmap_ctrl; } +#ifdef CONFIG_DEBUG_FS + /* + * FIXME: needs better naming distinction, as some amalgamation of + * "caam" and nprop->full_name. The OF name isn't distinctive, + * but does separate instances + */ + perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; + + ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); + ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); +#endif ring = 0; - ctrlpriv->total_jobrs = 0; for_each_available_child_of_node(nprop, np) if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { - ctrlpriv->jrpdev[ring] = - of_platform_device_create(np, NULL, dev); - if (!ctrlpriv->jrpdev[ring]) { - pr_warn("JR%d Platform device creation error\n", - ring); - continue; - } - ctrlpriv->jr[ring] = (struct caam_job_ring __force *) - ((uint8_t *)ctrl + + ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) + ((__force uint8_t *)ctrl + (ring + JR_BLOCK_NUMBER) * BLOCK_OFFSET ); ctrlpriv->total_jobrs++; ring++; - } + } - /* Check to see if QI present. If so, enable */ - ctrlpriv->qi_present = - !!(rd_reg32(&ctrl->perfmon.comp_parms_ms) & - CTPR_MS_QI_MASK); - if (ctrlpriv->qi_present) { - ctrlpriv->qi = (struct caam_queue_if __force *) - ((uint8_t *)ctrl + + /* Check to see if (DPAA 1.x) QI present. If so, enable */ + ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); + if (ctrlpriv->qi_present && !caam_dpaa2) { + ctrlpriv->qi = (struct caam_queue_if __iomem __force *) + ((__force uint8_t *)ctrl + BLOCK_OFFSET * QI_BLOCK_NUMBER ); /* This is all that's required to physically enable QI */ wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN); + + /* If QMAN driver is present, init CAAM-QI backend */ +#ifdef CONFIG_CAAM_QI + ret = caam_qi_init(pdev); + if (ret) + dev_err(dev, "caam qi i/f init failed: %d\n", ret); +#endif } /* If no QI and no rings specified, quit and go home */ @@ -666,8 +678,10 @@ static int caam_probe(struct platform_de /* * If SEC has RNG version >= 4 and RNG state handle has not been * already instantiated, do RNG instantiation + * In case of DPAA 2.x, RNG is managed by MC firmware. */ - if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { + if (!caam_dpaa2 && + (cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { ctrlpriv->rng4_sh_init = rd_reg32(&ctrl->r4tst[0].rdsta); /* @@ -734,78 +748,47 @@ static int caam_probe(struct platform_de /* Report "alive" for developer to see */ dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, - caam_get_era()); - dev_info(dev, "job rings = %d, qi = %d\n", - ctrlpriv->total_jobrs, ctrlpriv->qi_present); + ctrlpriv->era); + dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n", + ctrlpriv->total_jobrs, ctrlpriv->qi_present, + caam_dpaa2 ? "yes" : "no"); #ifdef CONFIG_DEBUG_FS - /* - * FIXME: needs better naming distinction, as some amalgamation of - * "caam" and nprop->full_name. The OF name isn't distinctive, - * but does separate instances - */ - perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; - - ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); - ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); - - /* Controller-level - performance monitor counters */ - - ctrlpriv->ctl_rq_dequeued = - debugfs_create_file("rq_dequeued", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->req_dequeued, - &caam_fops_u64_ro); - ctrlpriv->ctl_ob_enc_req = - debugfs_create_file("ob_rq_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_req, - &caam_fops_u64_ro); - ctrlpriv->ctl_ib_dec_req = - debugfs_create_file("ib_rq_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_req, - &caam_fops_u64_ro); - ctrlpriv->ctl_ob_enc_bytes = - debugfs_create_file("ob_bytes_encrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_enc_bytes, - &caam_fops_u64_ro); - ctrlpriv->ctl_ob_prot_bytes = - debugfs_create_file("ob_bytes_protected", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ob_prot_bytes, - &caam_fops_u64_ro); - ctrlpriv->ctl_ib_dec_bytes = - debugfs_create_file("ib_bytes_decrypted", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_dec_bytes, - &caam_fops_u64_ro); - ctrlpriv->ctl_ib_valid_bytes = - debugfs_create_file("ib_bytes_validated", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->ib_valid_bytes, - &caam_fops_u64_ro); + debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->req_dequeued, + &caam_fops_u64_ro); + debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_req, + &caam_fops_u64_ro); + debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_req, + &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_enc_bytes, + &caam_fops_u64_ro); + debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ob_prot_bytes, + &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_dec_bytes, + &caam_fops_u64_ro); + debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->ib_valid_bytes, + &caam_fops_u64_ro); /* Controller level - global status values */ - ctrlpriv->ctl_faultaddr = - debugfs_create_file("fault_addr", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultaddr, - &caam_fops_u32_ro); - ctrlpriv->ctl_faultdetail = - debugfs_create_file("fault_detail", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->faultdetail, - &caam_fops_u32_ro); - ctrlpriv->ctl_faultstatus = - debugfs_create_file("fault_status", - S_IRUSR | S_IRGRP | S_IROTH, - ctrlpriv->ctl, &perfmon->status, - &caam_fops_u32_ro); + debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultaddr, + &caam_fops_u32_ro); + debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->faultdetail, + &caam_fops_u32_ro); + debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH, + ctrlpriv->ctl, &perfmon->status, + &caam_fops_u32_ro); /* Internal covering keys (useful in non-secure mode only) */ - ctrlpriv->ctl_kek_wrap.data = &ctrlpriv->ctrl->kek[0]; + ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32); ctrlpriv->ctl_kek = debugfs_create_blob("kek", S_IRUSR | @@ -813,7 +796,7 @@ static int caam_probe(struct platform_de ctrlpriv->ctl, &ctrlpriv->ctl_kek_wrap); - ctrlpriv->ctl_tkek_wrap.data = &ctrlpriv->ctrl->tkek[0]; + ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0]; ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32); ctrlpriv->ctl_tkek = debugfs_create_blob("tkek", S_IRUSR | @@ -821,7 +804,7 @@ static int caam_probe(struct platform_de ctrlpriv->ctl, &ctrlpriv->ctl_tkek_wrap); - ctrlpriv->ctl_tdsk_wrap.data = &ctrlpriv->ctrl->tdsk[0]; + ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0]; ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32); ctrlpriv->ctl_tdsk = debugfs_create_blob("tdsk", S_IRUSR | @@ -832,13 +815,17 @@ static int caam_probe(struct platform_de return 0; caam_remove: +#ifdef CONFIG_DEBUG_FS + debugfs_remove_recursive(ctrlpriv->dfs_root); +#endif caam_remove(pdev); return ret; iounmap_ctrl: iounmap(ctrl); disable_caam_emi_slow: - clk_disable_unprepare(ctrlpriv->caam_emi_slow); + if (ctrlpriv->caam_emi_slow) + clk_disable_unprepare(ctrlpriv->caam_emi_slow); disable_caam_aclk: clk_disable_unprepare(ctrlpriv->caam_aclk); disable_caam_mem: @@ -848,17 +835,6 @@ disable_caam_ipg: return ret; } -static struct of_device_id caam_match[] = { - { - .compatible = "fsl,sec-v4.0", - }, - { - .compatible = "fsl,sec4.0", - }, - {}, -}; -MODULE_DEVICE_TABLE(of, caam_match); - static struct platform_driver caam_driver = { .driver = { .name = "caam", --- a/drivers/crypto/caam/ctrl.h +++ b/drivers/crypto/caam/ctrl.h @@ -10,4 +10,6 @@ /* Prototypes for backend-level services exposed to APIs */ int caam_get_era(void); +extern bool caam_dpaa2; + #endif /* CTRL_H */ --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -22,12 +22,6 @@ #define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ #define SEC4_SG_OFFSET_MASK 0x00001fff -struct sec4_sg_entry { - u64 ptr; - u32 len; - u32 bpid_offset; -}; - /* Max size of any CAAM descriptor in 32-bit words, inclusive of header */ #define MAX_CAAM_DESCSIZE 64 @@ -47,6 +41,7 @@ struct sec4_sg_entry { #define CMD_SEQ_LOAD (0x03 << CMD_SHIFT) #define CMD_FIFO_LOAD (0x04 << CMD_SHIFT) #define CMD_SEQ_FIFO_LOAD (0x05 << CMD_SHIFT) +#define CMD_MOVEB (0x07 << CMD_SHIFT) #define CMD_STORE (0x0a << CMD_SHIFT) #define CMD_SEQ_STORE (0x0b << CMD_SHIFT) #define CMD_FIFO_STORE (0x0c << CMD_SHIFT) @@ -90,8 +85,8 @@ struct sec4_sg_entry { #define HDR_ZRO 0x00008000 /* Start Index or SharedDesc Length */ -#define HDR_START_IDX_MASK 0x3f #define HDR_START_IDX_SHIFT 16 +#define HDR_START_IDX_MASK (0x3f << HDR_START_IDX_SHIFT) /* If shared descriptor header, 6-bit length */ #define HDR_DESCLEN_SHR_MASK 0x3f @@ -121,10 +116,10 @@ struct sec4_sg_entry { #define HDR_PROP_DNR 0x00000800 /* JobDesc/SharedDesc share property */ -#define HDR_SD_SHARE_MASK 0x03 #define HDR_SD_SHARE_SHIFT 8 -#define HDR_JD_SHARE_MASK 0x07 +#define HDR_SD_SHARE_MASK (0x03 << HDR_SD_SHARE_SHIFT) #define HDR_JD_SHARE_SHIFT 8 +#define HDR_JD_SHARE_MASK (0x07 << HDR_JD_SHARE_SHIFT) #define HDR_SHARE_NEVER (0x00 << HDR_SD_SHARE_SHIFT) #define HDR_SHARE_WAIT (0x01 << HDR_SD_SHARE_SHIFT) @@ -235,7 +230,7 @@ struct sec4_sg_entry { #define LDST_SRCDST_WORD_DECO_MATH2 (0x0a << LDST_SRCDST_SHIFT) #define LDST_SRCDST_WORD_DECO_AAD_SZ (0x0b << LDST_SRCDST_SHIFT) #define LDST_SRCDST_WORD_DECO_MATH3 (0x0b << LDST_SRCDST_SHIFT) -#define LDST_SRCDST_WORD_CLASS1_ICV_SZ (0x0c << LDST_SRCDST_SHIFT) +#define LDST_SRCDST_WORD_CLASS1_IV_SZ (0x0c << LDST_SRCDST_SHIFT) #define LDST_SRCDST_WORD_ALTDS_CLASS1 (0x0f << LDST_SRCDST_SHIFT) #define LDST_SRCDST_WORD_PKHA_A_SZ (0x10 << LDST_SRCDST_SHIFT) #define LDST_SRCDST_WORD_PKHA_B_SZ (0x11 << LDST_SRCDST_SHIFT) @@ -360,6 +355,7 @@ struct sec4_sg_entry { #define FIFOLD_TYPE_PK_N (0x08 << FIFOLD_TYPE_SHIFT) #define FIFOLD_TYPE_PK_A (0x0c << FIFOLD_TYPE_SHIFT) #define FIFOLD_TYPE_PK_B (0x0d << FIFOLD_TYPE_SHIFT) +#define FIFOLD_TYPE_IFIFO (0x0f << FIFOLD_TYPE_SHIFT) /* Other types. Need to OR in last/flush bits as desired */ #define FIFOLD_TYPE_MSG_MASK (0x38 << FIFOLD_TYPE_SHIFT) @@ -400,7 +396,7 @@ struct sec4_sg_entry { #define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT) -#define FIFOST_TYPE_AF_SBOX_JKEK (0x10 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_E_TKEK (0x23 << FIFOST_TYPE_SHIFT) @@ -413,6 +409,7 @@ struct sec4_sg_entry { #define FIFOST_TYPE_MESSAGE_DATA (0x30 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_RNGSTORE (0x34 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_RNGFIFO (0x35 << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_METADATA (0x3e << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_SKIP (0x3f << FIFOST_TYPE_SHIFT) /* @@ -449,6 +446,18 @@ struct sec4_sg_entry { #define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT) #define OP_PCLID_RSAENC_PUBKEY (0x18 << OP_PCLID_SHIFT) #define OP_PCLID_RSADEC_PRVKEY (0x19 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_MD5 (0x20 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA1 (0x21 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA224 (0x22 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA256 (0x23 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA384 (0x24 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_SHA512 (0x25 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_MD5 (0x60 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA1 (0x61 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA224 (0x62 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA256 (0x63 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA384 (0x64 << OP_PCLID_SHIFT) +#define OP_PCLID_DKP_RIF_SHA512 (0x65 << OP_PCLID_SHIFT) /* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */ #define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT) @@ -1098,6 +1107,22 @@ struct sec4_sg_entry { /* MacSec protinfos */ #define OP_PCL_MACSEC 0x0001 +/* Derived Key Protocol (DKP) Protinfo */ +#define OP_PCL_DKP_SRC_SHIFT 14 +#define OP_PCL_DKP_SRC_MASK (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_IMM (0 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SEQ (1 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_PTR (2 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_SRC_SGF (3 << OP_PCL_DKP_SRC_SHIFT) +#define OP_PCL_DKP_DST_SHIFT 12 +#define OP_PCL_DKP_DST_MASK (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_IMM (0 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SEQ (1 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_PTR (2 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_DST_SGF (3 << OP_PCL_DKP_DST_SHIFT) +#define OP_PCL_DKP_KEY_SHIFT 0 +#define OP_PCL_DKP_KEY_MASK (0xfff << OP_PCL_DKP_KEY_SHIFT) + /* PKI unidirectional protocol protinfo bits */ #define OP_PCL_PKPROT_TEST 0x0008 #define OP_PCL_PKPROT_DECRYPT 0x0004 @@ -1107,8 +1132,8 @@ struct sec4_sg_entry { /* For non-protocol/alg-only op commands */ #define OP_ALG_TYPE_SHIFT 24 #define OP_ALG_TYPE_MASK (0x7 << OP_ALG_TYPE_SHIFT) -#define OP_ALG_TYPE_CLASS1 2 -#define OP_ALG_TYPE_CLASS2 4 +#define OP_ALG_TYPE_CLASS1 (2 << OP_ALG_TYPE_SHIFT) +#define OP_ALG_TYPE_CLASS2 (4 << OP_ALG_TYPE_SHIFT) #define OP_ALG_ALGSEL_SHIFT 16 #define OP_ALG_ALGSEL_MASK (0xff << OP_ALG_ALGSEL_SHIFT) @@ -1249,7 +1274,7 @@ struct sec4_sg_entry { #define OP_ALG_PKMODE_MOD_PRIMALITY 0x00f /* PKHA mode copy-memory functions */ -#define OP_ALG_PKMODE_SRC_REG_SHIFT 13 +#define OP_ALG_PKMODE_SRC_REG_SHIFT 17 #define OP_ALG_PKMODE_SRC_REG_MASK (7 << OP_ALG_PKMODE_SRC_REG_SHIFT) #define OP_ALG_PKMODE_DST_REG_SHIFT 10 #define OP_ALG_PKMODE_DST_REG_MASK (7 << OP_ALG_PKMODE_DST_REG_SHIFT) @@ -1445,10 +1470,11 @@ struct sec4_sg_entry { #define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT) #define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT) #define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT) -#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC0_SHIFT) +#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC1_SHIFT) #define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT) #define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT) #define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT) +#define MATH_SRC1_ZERO (0x0f << MATH_SRC1_SHIFT) /* Destination selectors */ #define MATH_DEST_SHIFT 8 @@ -1457,6 +1483,7 @@ struct sec4_sg_entry { #define MATH_DEST_REG1 (0x01 << MATH_DEST_SHIFT) #define MATH_DEST_REG2 (0x02 << MATH_DEST_SHIFT) #define MATH_DEST_REG3 (0x03 << MATH_DEST_SHIFT) +#define MATH_DEST_DPOVRD (0x07 << MATH_DEST_SHIFT) #define MATH_DEST_SEQINLEN (0x08 << MATH_DEST_SHIFT) #define MATH_DEST_SEQOUTLEN (0x09 << MATH_DEST_SHIFT) #define MATH_DEST_VARSEQINLEN (0x0a << MATH_DEST_SHIFT) @@ -1629,4 +1656,31 @@ struct sec4_sg_entry { /* Frame Descriptor Command for Replacement Job Descriptor */ #define FD_CMD_REPLACE_JOB_DESC 0x20000000 +/* CHA Control Register bits */ +#define CCTRL_RESET_CHA_ALL 0x1 +#define CCTRL_RESET_CHA_AESA 0x2 +#define CCTRL_RESET_CHA_DESA 0x4 +#define CCTRL_RESET_CHA_AFHA 0x8 +#define CCTRL_RESET_CHA_KFHA 0x10 +#define CCTRL_RESET_CHA_SF8A 0x20 +#define CCTRL_RESET_CHA_PKHA 0x40 +#define CCTRL_RESET_CHA_MDHA 0x80 +#define CCTRL_RESET_CHA_CRCA 0x100 +#define CCTRL_RESET_CHA_RNG 0x200 +#define CCTRL_RESET_CHA_SF9A 0x400 +#define CCTRL_RESET_CHA_ZUCE 0x800 +#define CCTRL_RESET_CHA_ZUCA 0x1000 +#define CCTRL_UNLOAD_PK_A0 0x10000 +#define CCTRL_UNLOAD_PK_A1 0x20000 +#define CCTRL_UNLOAD_PK_A2 0x40000 +#define CCTRL_UNLOAD_PK_A3 0x80000 +#define CCTRL_UNLOAD_PK_B0 0x100000 +#define CCTRL_UNLOAD_PK_B1 0x200000 +#define CCTRL_UNLOAD_PK_B2 0x400000 +#define CCTRL_UNLOAD_PK_B3 0x800000 +#define CCTRL_UNLOAD_PK_N 0x1000000 +#define CCTRL_UNLOAD_PK_A 0x4000000 +#define CCTRL_UNLOAD_PK_B 0x8000000 +#define CCTRL_UNLOAD_SBOX 0x10000000 + #endif /* DESC_H */ --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -4,6 +4,9 @@ * Copyright 2008-2012 Freescale Semiconductor, Inc. */ +#ifndef DESC_CONSTR_H +#define DESC_CONSTR_H + #include "desc.h" #include "regs.h" @@ -33,38 +36,39 @@ extern bool caam_little_end; -static inline int desc_len(u32 *desc) +static inline int desc_len(u32 * const desc) { return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; } -static inline int desc_bytes(void *desc) +static inline int desc_bytes(void * const desc) { return desc_len(desc) * CAAM_CMD_SZ; } -static inline u32 *desc_end(u32 *desc) +static inline u32 *desc_end(u32 * const desc) { return desc + desc_len(desc); } -static inline void *sh_desc_pdb(u32 *desc) +static inline void *sh_desc_pdb(u32 * const desc) { return desc + 1; } -static inline void init_desc(u32 *desc, u32 options) +static inline void init_desc(u32 * const desc, u32 options) { *desc = cpu_to_caam32((options | HDR_ONE) + 1); } -static inline void init_sh_desc(u32 *desc, u32 options) +static inline void init_sh_desc(u32 * const desc, u32 options) { PRINT_POS; init_desc(desc, CMD_SHARED_DESC_HDR | options); } -static inline void init_sh_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes) +static inline void init_sh_desc_pdb(u32 * const desc, u32 options, + size_t pdb_bytes) { u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; @@ -72,19 +76,20 @@ static inline void init_sh_desc_pdb(u32 options); } -static inline void init_job_desc(u32 *desc, u32 options) +static inline void init_job_desc(u32 * const desc, u32 options) { init_desc(desc, CMD_DESC_HDR | options); } -static inline void init_job_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes) +static inline void init_job_desc_pdb(u32 * const desc, u32 options, + size_t pdb_bytes) { u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; init_job_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT)) | options); } -static inline void append_ptr(u32 *desc, dma_addr_t ptr) +static inline void append_ptr(u32 * const desc, dma_addr_t ptr) { dma_addr_t *offset = (dma_addr_t *)desc_end(desc); @@ -94,8 +99,8 @@ static inline void append_ptr(u32 *desc, CAAM_PTR_SZ / CAAM_CMD_SZ); } -static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len, - u32 options) +static inline void init_job_desc_shared(u32 * const desc, dma_addr_t ptr, + int len, u32 options) { PRINT_POS; init_job_desc(desc, HDR_SHARED | options | @@ -103,7 +108,7 @@ static inline void init_job_desc_shared( append_ptr(desc, ptr); } -static inline void append_data(u32 *desc, void *data, int len) +static inline void append_data(u32 * const desc, const void *data, int len) { u32 *offset = desc_end(desc); @@ -114,7 +119,7 @@ static inline void append_data(u32 *desc (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ); } -static inline void append_cmd(u32 *desc, u32 command) +static inline void append_cmd(u32 * const desc, u32 command) { u32 *cmd = desc_end(desc); @@ -125,7 +130,7 @@ static inline void append_cmd(u32 *desc, #define append_u32 append_cmd -static inline void append_u64(u32 *desc, u64 data) +static inline void append_u64(u32 * const desc, u64 data) { u32 *offset = desc_end(desc); @@ -142,14 +147,14 @@ static inline void append_u64(u32 *desc, } /* Write command without affecting header, and return pointer to next word */ -static inline u32 *write_cmd(u32 *desc, u32 command) +static inline u32 *write_cmd(u32 * const desc, u32 command) { *desc = cpu_to_caam32(command); return desc + 1; } -static inline void append_cmd_ptr(u32 *desc, dma_addr_t ptr, int len, +static inline void append_cmd_ptr(u32 * const desc, dma_addr_t ptr, int len, u32 command) { append_cmd(desc, command | len); @@ -157,7 +162,7 @@ static inline void append_cmd_ptr(u32 *d } /* Write length after pointer, rather than inside command */ -static inline void append_cmd_ptr_extlen(u32 *desc, dma_addr_t ptr, +static inline void append_cmd_ptr_extlen(u32 * const desc, dma_addr_t ptr, unsigned int len, u32 command) { append_cmd(desc, command); @@ -166,7 +171,7 @@ static inline void append_cmd_ptr_extlen append_cmd(desc, len); } -static inline void append_cmd_data(u32 *desc, void *data, int len, +static inline void append_cmd_data(u32 * const desc, const void *data, int len, u32 command) { append_cmd(desc, command | IMMEDIATE | len); @@ -174,7 +179,7 @@ static inline void append_cmd_data(u32 * } #define APPEND_CMD_RET(cmd, op) \ -static inline u32 *append_##cmd(u32 *desc, u32 options) \ +static inline u32 *append_##cmd(u32 * const desc, u32 options) \ { \ u32 *cmd = desc_end(desc); \ PRINT_POS; \ @@ -183,14 +188,15 @@ static inline u32 *append_##cmd(u32 *des } APPEND_CMD_RET(jump, JUMP) APPEND_CMD_RET(move, MOVE) +APPEND_CMD_RET(moveb, MOVEB) -static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd) +static inline void set_jump_tgt_here(u32 * const desc, u32 *jump_cmd) { *jump_cmd = cpu_to_caam32(caam32_to_cpu(*jump_cmd) | (desc_len(desc) - (jump_cmd - desc))); } -static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd) +static inline void set_move_tgt_here(u32 * const desc, u32 *move_cmd) { u32 val = caam32_to_cpu(*move_cmd); @@ -200,7 +206,7 @@ static inline void set_move_tgt_here(u32 } #define APPEND_CMD(cmd, op) \ -static inline void append_##cmd(u32 *desc, u32 options) \ +static inline void append_##cmd(u32 * const desc, u32 options) \ { \ PRINT_POS; \ append_cmd(desc, CMD_##op | options); \ @@ -208,7 +214,8 @@ static inline void append_##cmd(u32 *des APPEND_CMD(operation, OPERATION) #define APPEND_CMD_LEN(cmd, op) \ -static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \ +static inline void append_##cmd(u32 * const desc, unsigned int len, \ + u32 options) \ { \ PRINT_POS; \ append_cmd(desc, CMD_##op | len | options); \ @@ -220,8 +227,8 @@ APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_L APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE) #define APPEND_CMD_PTR(cmd, op) \ -static inline void append_##cmd(u32 *desc, dma_addr_t ptr, unsigned int len, \ - u32 options) \ +static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \ + unsigned int len, u32 options) \ { \ PRINT_POS; \ append_cmd_ptr(desc, ptr, len, CMD_##op | options); \ @@ -231,8 +238,8 @@ APPEND_CMD_PTR(load, LOAD) APPEND_CMD_PTR(fifo_load, FIFO_LOAD) APPEND_CMD_PTR(fifo_store, FIFO_STORE) -static inline void append_store(u32 *desc, dma_addr_t ptr, unsigned int len, - u32 options) +static inline void append_store(u32 * const desc, dma_addr_t ptr, + unsigned int len, u32 options) { u32 cmd_src; @@ -249,7 +256,8 @@ static inline void append_store(u32 *des } #define APPEND_SEQ_PTR_INTLEN(cmd, op) \ -static inline void append_seq_##cmd##_ptr_intlen(u32 *desc, dma_addr_t ptr, \ +static inline void append_seq_##cmd##_ptr_intlen(u32 * const desc, \ + dma_addr_t ptr, \ unsigned int len, \ u32 options) \ { \ @@ -263,7 +271,7 @@ APPEND_SEQ_PTR_INTLEN(in, IN) APPEND_SEQ_PTR_INTLEN(out, OUT) #define APPEND_CMD_PTR_TO_IMM(cmd, op) \ -static inline void append_##cmd##_as_imm(u32 *desc, void *data, \ +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ unsigned int len, u32 options) \ { \ PRINT_POS; \ @@ -273,7 +281,7 @@ APPEND_CMD_PTR_TO_IMM(load, LOAD); APPEND_CMD_PTR_TO_IMM(fifo_load, FIFO_LOAD); #define APPEND_CMD_PTR_EXTLEN(cmd, op) \ -static inline void append_##cmd##_extlen(u32 *desc, dma_addr_t ptr, \ +static inline void append_##cmd##_extlen(u32 * const desc, dma_addr_t ptr, \ unsigned int len, u32 options) \ { \ PRINT_POS; \ @@ -287,7 +295,7 @@ APPEND_CMD_PTR_EXTLEN(seq_out_ptr, SEQ_O * the size of its type */ #define APPEND_CMD_PTR_LEN(cmd, op, type) \ -static inline void append_##cmd(u32 *desc, dma_addr_t ptr, \ +static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \ type len, u32 options) \ { \ PRINT_POS; \ @@ -304,7 +312,7 @@ APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_ * from length of immediate data provided, e.g., split keys */ #define APPEND_CMD_PTR_TO_IMM2(cmd, op) \ -static inline void append_##cmd##_as_imm(u32 *desc, void *data, \ +static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ unsigned int data_len, \ unsigned int len, u32 options) \ { \ @@ -315,7 +323,7 @@ static inline void append_##cmd##_as_imm APPEND_CMD_PTR_TO_IMM2(key, KEY); #define APPEND_CMD_RAW_IMM(cmd, op, type) \ -static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \ +static inline void append_##cmd##_imm_##type(u32 * const desc, type immediate, \ u32 options) \ { \ PRINT_POS; \ @@ -426,3 +434,107 @@ do { \ APPEND_MATH_IMM_u64(LSHIFT, desc, dest, src0, src1, data) #define append_math_rshift_imm_u64(desc, dest, src0, src1, data) \ APPEND_MATH_IMM_u64(RSHIFT, desc, dest, src0, src1, data) + +/** + * struct alginfo - Container for algorithm details + * @algtype: algorithm selector; for valid values, see documentation of the + * functions where it is used. + * @keylen: length of the provided algorithm key, in bytes + * @keylen_pad: padded length of the provided algorithm key, in bytes + * @key: address where algorithm key resides; virtual address if key_inline + * is true, dma (bus) address if key_inline is false. + * @key_inline: true - key can be inlined in the descriptor; false - key is + * referenced by the descriptor + */ +struct alginfo { + u32 algtype; + unsigned int keylen; + unsigned int keylen_pad; + union { + dma_addr_t key_dma; + const void *key_virt; + }; + bool key_inline; +}; + +/** + * desc_inline_query() - Provide indications on which data items can be inlined + * and which shall be referenced in a shared descriptor. + * @sd_base_len: Shared descriptor base length - bytes consumed by the commands, + * excluding the data items to be inlined (or corresponding + * pointer if an item is not inlined). Each cnstr_* function that + * generates descriptors should have a define mentioning + * corresponding length. + * @jd_len: Maximum length of the job descriptor(s) that will be used + * together with the shared descriptor. + * @data_len: Array of lengths of the data items trying to be inlined + * @inl_mask: 32bit mask with bit x = 1 if data item x can be inlined, 0 + * otherwise. + * @count: Number of data items (size of @data_len array); must be <= 32 + * + * Return: 0 if data can be inlined / referenced, negative value if not. If 0, + * check @inl_mask for details. + */ +static inline int desc_inline_query(unsigned int sd_base_len, + unsigned int jd_len, unsigned int *data_len, + u32 *inl_mask, unsigned int count) +{ + int rem_bytes = (int)(CAAM_DESC_BYTES_MAX - sd_base_len - jd_len); + unsigned int i; + + *inl_mask = 0; + for (i = 0; (i < count) && (rem_bytes > 0); i++) { + if (rem_bytes - (int)(data_len[i] + + (count - i - 1) * CAAM_PTR_SZ) >= 0) { + rem_bytes -= data_len[i]; + *inl_mask |= (1 << i); + } else { + rem_bytes -= CAAM_PTR_SZ; + } + } + + return (rem_bytes >= 0) ? 0 : -1; +} + +/** + * append_proto_dkp - Derived Key Protocol (DKP): key -> split key + * @desc: pointer to buffer used for descriptor construction + * @adata: pointer to authentication transform definitions. + * keylen should be the length of initial key, while keylen_pad + * the length of the derived (split) key. + * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224, + * SHA256, SHA384, SHA512}. + */ +static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata) +{ + u32 protid; + + /* + * Quick & dirty translation from OP_ALG_ALGSEL_{MD5, SHA*} + * to OP_PCLID_DKP_{MD5, SHA*} + */ + protid = (adata->algtype & OP_ALG_ALGSEL_SUBMASK) | + (0x20 << OP_ALG_ALGSEL_SHIFT); + + if (adata->key_inline) { + int words; + + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | + OP_PCL_DKP_SRC_IMM | OP_PCL_DKP_DST_IMM | + adata->keylen); + append_data(desc, adata->key_virt, adata->keylen); + + /* Reserve space in descriptor buffer for the derived key */ + words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) - + ALIGN(adata->keylen, CAAM_CMD_SZ)) / CAAM_CMD_SZ; + if (words) + (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words); + } else { + append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | + OP_PCL_DKP_SRC_PTR | OP_PCL_DKP_DST_PTR | + adata->keylen); + append_ptr(desc, adata->key_dma); + } +} + +#endif /* DESC_CONSTR_H */ --- /dev/null +++ b/drivers/crypto/caam/dpseci.c @@ -0,0 +1,859 @@ +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "../../../drivers/staging/fsl-mc/include/mc-sys.h" +#include "../../../drivers/staging/fsl-mc/include/mc-cmd.h" +#include "../../../drivers/staging/fsl-mc/include/dpopr.h" +#include "dpseci.h" +#include "dpseci_cmd.h" + +/** + * dpseci_open() - Open a control session for the specified object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @dpseci_id: DPSECI unique ID + * @token: Returned token; use in subsequent API calls + * + * This function can be used to open a control session for an already created + * object; an object may have been declared in the DPL or by calling the + * dpseci_create() function. + * This function returns a unique authentication token, associated with the + * specific object ID and the specific MC portal; this token must be used in all + * subsequent commands for this specific object. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id, + u16 *token) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_open *cmd_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_OPEN, + cmd_flags, + 0); + cmd_params = (struct dpseci_cmd_open *)cmd.params; + cmd_params->dpseci_id = cpu_to_le32(dpseci_id); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + *token = mc_cmd_hdr_read_token(&cmd); + + return 0; +} + +/** + * dpseci_close() - Close the control session of the object + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * After this function is called, no further operations are allowed on the + * object without opening a new control session. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLOSE, + cmd_flags, + token); + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_create() - Create the DPSECI object + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @cfg: Configuration structure + * @obj_id: returned object id + * + * Create the DPSECI object, allocate required resources and perform required + * initialization. + * + * The object can be created either by declaring it in the DPL file, or by + * calling this function. + * + * The function accepts an authentication token of a parent container that this + * object should be assigned to. The token can be '0' so the object will be + * assigned to the default container. + * The newly created object can be opened with the returned object id and using + * the container's associated tokens and MC portals. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_create(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, + const struct dpseci_cfg *cfg, u32 *obj_id) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_create *cmd_params; + int i, err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CREATE, + cmd_flags, + dprc_token); + cmd_params = (struct dpseci_cmd_create *)cmd.params; + for (i = 0; i < 8; i++) + cmd_params->priorities[i] = cfg->priorities[i]; + cmd_params->num_tx_queues = cfg->num_tx_queues; + cmd_params->num_rx_queues = cfg->num_rx_queues; + cmd_params->options = cpu_to_le32(cfg->options); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + *obj_id = mc_cmd_read_object_id(&cmd); + + return 0; +} + +/** + * dpseci_destroy() - Destroy the DPSECI object and release all its resources + * @mc_io: Pointer to MC portal's I/O object + * @dprc_token: Parent container token; '0' for default container + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @object_id: The object id; it must be a valid id within the container that + * created this object + * + * The function accepts the authentication token of the parent container that + * created the object (not the one that currently owns the object). The object + * is searched within parent using the provided 'object_id'. + * All tokens to the object must be closed before calling destroy. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_destroy(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, + u32 object_id) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_destroy *cmd_params; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DESTROY, + cmd_flags, + dprc_token); + cmd_params = (struct dpseci_cmd_destroy *)cmd.params; + cmd_params->object_id = cpu_to_le32(object_id); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on success, error code otherwise + */ +int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE, + cmd_flags, + token); + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on success, error code otherwise + */ +int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE, + cmd_flags, + token); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_is_enabled() - Check if the DPSECI is enabled. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @en: Returns '1' if object is enabled; '0' otherwise + * + * Return: '0' on success, error code otherwise + */ +int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + int *en) +{ + struct mc_command cmd = { 0 }; + struct dpseci_rsp_is_enabled *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_is_enabled *)cmd.params; + *en = le32_to_cpu(rsp_params->is_enabled); + + return 0; +} + +/** + * dpseci_reset() - Reset the DPSECI, returns the object to initial state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * + * Return: '0' on success, error code otherwise + */ +int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) +{ + struct mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_RESET, + cmd_flags, + token); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_irq_enable() - Get overall interrupt state + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @irq_index: The interrupt index to configure + * @en: Returned Interrupt state - enable = 1, disable = 0 + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u8 *en) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_irq_enable *cmd_params; + struct dpseci_rsp_get_irq_enable *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_ENABLE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_irq_enable *)cmd.params; + cmd_params->irq_index = irq_index; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_irq_enable *)cmd.params; + *en = rsp_params->enable_state; + + return 0; +} + +/** + * dpseci_set_irq_enable() - Set overall interrupt state. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @irq_index: The interrupt index to configure + * @en: Interrupt state - enable = 1, disable = 0 + * + * Allows GPP software to control when interrupts are generated. + * Each interrupt can have up to 32 causes. The enable/disable control's the + * overall interrupt state. If the interrupt is disabled no causes will cause + * an interrupt. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_set_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u8 en) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_irq_enable *cmd_params; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_ENABLE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_irq_enable *)cmd.params; + cmd_params->irq_index = irq_index; + cmd_params->enable_state = en; + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_irq_mask() - Get interrupt mask. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @irq_index: The interrupt index to configure + * @mask: Returned event mask to trigger interrupt + * + * Every interrupt can have up to 32 causes and the interrupt model supports + * masking/unmasking each cause independently. + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 *mask) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_irq_mask *cmd_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_MASK, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_irq_mask *)cmd.params; + cmd_params->irq_index = irq_index; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + *mask = le32_to_cpu(cmd_params->mask); + + return 0; +} + +/** + * dpseci_set_irq_mask() - Set interrupt mask. + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @irq_index: The interrupt index to configure + * @mask: event mask to trigger interrupt; + * each bit: + * 0 = ignore event + * 1 = consider event for asserting IRQ + * + * Every interrupt can have up to 32 causes and the interrupt model supports + * masking/unmasking each cause independently + * + * Return: '0' on success, error code otherwise + */ +int dpseci_set_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 mask) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_irq_mask *cmd_params; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_MASK, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_irq_mask *)cmd.params; + cmd_params->mask = cpu_to_le32(mask); + cmd_params->irq_index = irq_index; + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_irq_status() - Get the current status of any pending interrupts + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @irq_index: The interrupt index to configure + * @status: Returned interrupts status - one bit per cause: + * 0 = no interrupt pending + * 1 = interrupt pending + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 *status) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_irq_status *cmd_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_STATUS, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_irq_status *)cmd.params; + cmd_params->status = cpu_to_le32(*status); + cmd_params->irq_index = irq_index; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + *status = le32_to_cpu(cmd_params->status); + + return 0; +} + +/** + * dpseci_clear_irq_status() - Clear a pending interrupt's status + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @irq_index: The interrupt index to configure + * @status: bits to clear (W1C) - one bit per cause: + * 0 = don't change + * 1 = clear status bit + * + * Return: '0' on success, error code otherwise + */ +int dpseci_clear_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 status) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_irq_status *cmd_params; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLEAR_IRQ_STATUS, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_irq_status *)cmd.params; + cmd_params->status = cpu_to_le32(status); + cmd_params->irq_index = irq_index; + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_attributes() - Retrieve DPSECI attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned object's attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_attr *attr) +{ + struct mc_command cmd = { 0 }; + struct dpseci_rsp_get_attributes *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_attributes *)cmd.params; + attr->id = le32_to_cpu(rsp_params->id); + attr->num_tx_queues = rsp_params->num_tx_queues; + attr->num_rx_queues = rsp_params->num_rx_queues; + attr->options = le32_to_cpu(rsp_params->options); + + return 0; +} + +/** + * dpseci_set_rx_queue() - Set Rx queue configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of priorities configured at + * DPSECI creation; use DPSECI_ALL_QUEUES to configure all + * Rx queues identically. + * @cfg: Rx queue configuration + * + * Return: '0' on success, error code otherwise + */ +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, const struct dpseci_rx_queue_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_queue *cmd_params; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_queue *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->priority = cfg->dest_cfg.priority; + cmd_params->queue = queue; + cmd_params->dest_type = cfg->dest_cfg.dest_type; + cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); + cmd_params->options = cpu_to_le32(cfg->options); + cmd_params->order_preservation_en = + cpu_to_le32(cfg->order_preservation_en); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_rx_queue() - Retrieve Rx queue attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of priorities configured at + * DPSECI creation + * @attr: Returned Rx queue attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_rx_queue_attr *attr) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_queue *cmd_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_queue *)cmd.params; + cmd_params->queue = queue; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); + attr->dest_cfg.priority = cmd_params->priority; + attr->dest_cfg.dest_type = cmd_params->dest_type; + attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); + attr->fqid = le32_to_cpu(cmd_params->fqid); + attr->order_preservation_en = + le32_to_cpu(cmd_params->order_preservation_en); + + return 0; +} + +/** + * dpseci_get_tx_queue() - Retrieve Tx queue attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @queue: Select the queue relative to number of priorities configured at + * DPSECI creation + * @attr: Returned Tx queue attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_tx_queue_attr *attr) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_queue *cmd_params; + struct dpseci_rsp_get_tx_queue *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_queue *)cmd.params; + cmd_params->queue = queue; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_tx_queue *)cmd.params; + attr->fqid = le32_to_cpu(rsp_params->fqid); + attr->priority = rsp_params->priority; + + return 0; +} + +/** + * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @attr: Returned SEC attributes + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_sec_attr *attr) +{ + struct mc_command cmd = { 0 }; + struct dpseci_rsp_get_sec_attr *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_sec_attr *)cmd.params; + attr->ip_id = le16_to_cpu(rsp_params->ip_id); + attr->major_rev = rsp_params->major_rev; + attr->minor_rev = rsp_params->minor_rev; + attr->era = rsp_params->era; + attr->deco_num = rsp_params->deco_num; + attr->zuc_auth_acc_num = rsp_params->zuc_auth_acc_num; + attr->zuc_enc_acc_num = rsp_params->zuc_enc_acc_num; + attr->snow_f8_acc_num = rsp_params->snow_f8_acc_num; + attr->snow_f9_acc_num = rsp_params->snow_f9_acc_num; + attr->crc_acc_num = rsp_params->crc_acc_num; + attr->pk_acc_num = rsp_params->pk_acc_num; + attr->kasumi_acc_num = rsp_params->kasumi_acc_num; + attr->rng_acc_num = rsp_params->rng_acc_num; + attr->md_acc_num = rsp_params->md_acc_num; + attr->arc4_acc_num = rsp_params->arc4_acc_num; + attr->des_acc_num = rsp_params->des_acc_num; + attr->aes_acc_num = rsp_params->aes_acc_num; + + return 0; +} + +/** + * dpseci_get_sec_counters() - Retrieve SEC accelerator counters + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @counters: Returned SEC counters + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_sec_counters *counters) +{ + struct mc_command cmd = { 0 }; + struct dpseci_rsp_get_sec_counters *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_COUNTERS, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_sec_counters *)cmd.params; + counters->dequeued_requests = + le64_to_cpu(rsp_params->dequeued_requests); + counters->ob_enc_requests = le64_to_cpu(rsp_params->ob_enc_requests); + counters->ib_dec_requests = le64_to_cpu(rsp_params->ib_dec_requests); + counters->ob_enc_bytes = le64_to_cpu(rsp_params->ob_enc_bytes); + counters->ob_prot_bytes = le64_to_cpu(rsp_params->ob_prot_bytes); + counters->ib_dec_bytes = le64_to_cpu(rsp_params->ib_dec_bytes); + counters->ib_valid_bytes = le64_to_cpu(rsp_params->ib_valid_bytes); + + return 0; +} + +/** + * dpseci_get_api_version() - Get Data Path SEC Interface API version + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @major_ver: Major version of data path sec API + * @minor_ver: Minor version of data path sec API + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 *major_ver, u16 *minor_ver) +{ + struct mc_command cmd = { 0 }; + struct dpseci_rsp_get_api_version *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION, + cmd_flags, 0); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_api_version *)cmd.params; + *major_ver = le16_to_cpu(rsp_params->major); + *minor_ver = le16_to_cpu(rsp_params->minor); + + return 0; +} + +/** + * dpseci_set_opr() - Set Order Restoration configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @index: The queue index + * @options: Configuration mode options; can be OPR_OPT_CREATE or + * OPR_OPT_RETIRE + * @cfg: Configuration options for the OPR + * + * Return: '0' on success, error code otherwise + */ +int dpseci_set_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, + u8 options, struct opr_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_opr *cmd_params; + + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_SET_OPR, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_opr *)cmd.params; + cmd_params->index = index; + cmd_params->options = options; + cmd_params->oloe = cfg->oloe; + cmd_params->oeane = cfg->oeane; + cmd_params->olws = cfg->olws; + cmd_params->oa = cfg->oa; + cmd_params->oprrws = cfg->oprrws; + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_opr() - Retrieve Order Restoration config and query + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @index: The queue index + * @cfg: Returned OPR configuration + * @qry: Returned OPR query + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, + struct opr_cfg *cfg, struct opr_qry *qry) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_opr *cmd_params; + struct dpseci_rsp_get_opr *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_OPR, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_opr *)cmd.params; + cmd_params->index = index; + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_rsp_get_opr *)cmd.params; + qry->rip = dpseci_get_field(rsp_params->rip_enable, OPR_RIP); + qry->enable = dpseci_get_field(rsp_params->rip_enable, OPR_ENABLE); + cfg->oloe = rsp_params->oloe; + cfg->oeane = rsp_params->oeane; + cfg->olws = rsp_params->olws; + cfg->oa = rsp_params->oa; + cfg->oprrws = rsp_params->oprrws; + qry->nesn = le16_to_cpu(rsp_params->nesn); + qry->ndsn = le16_to_cpu(rsp_params->ndsn); + qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq); + qry->tseq_nlis = dpseci_get_field(rsp_params->tseq_nlis, OPR_TSEQ_NLIS); + qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq); + qry->hseq_nlis = dpseci_get_field(rsp_params->tseq_nlis, OPR_HSEQ_NLIS); + qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr); + qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr); + qry->opr_vid = le16_to_cpu(rsp_params->opr_vid); + qry->opr_id = le16_to_cpu(rsp_params->opr_id); + + return 0; +} + +/** + * dpseci_set_congestion_notification() - Set congestion group + * notification configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @cfg: congestion notification configuration + * + * Return: '0' on success, error code otherwise + */ +int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, const struct dpseci_congestion_notification_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_congestion_notification *cmd_params; + + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_SET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + cmd_params = (struct dpseci_cmd_congestion_notification *)cmd.params; + cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); + cmd_params->notification_mode = cpu_to_le16(cfg->notification_mode); + cmd_params->priority = cfg->dest_cfg.priority; + dpseci_set_field(cmd_params->options, CGN_DEST_TYPE, + cfg->dest_cfg.dest_type); + dpseci_set_field(cmd_params->options, CGN_UNITS, cfg->units); + cmd_params->message_iova = cpu_to_le64(cfg->message_iova); + cmd_params->message_ctx = cpu_to_le64(cfg->message_ctx); + cmd_params->threshold_entry = cpu_to_le32(cfg->threshold_entry); + cmd_params->threshold_exit = cpu_to_le32(cfg->threshold_exit); + + return mc_send_command(mc_io, &cmd); +} + +/** + * dpseci_get_congestion_notification() - Get congestion group notification + * configuration + * @mc_io: Pointer to MC portal's I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPSECI object + * @cfg: congestion notification configuration + * + * Return: '0' on success, error code otherwise + */ +int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, struct dpseci_congestion_notification_cfg *cfg) +{ + struct mc_command cmd = { 0 }; + struct dpseci_cmd_congestion_notification *rsp_params; + int err; + + cmd.header = mc_encode_cmd_header( + DPSECI_CMDID_GET_CONGESTION_NOTIFICATION, + cmd_flags, + token); + err = mc_send_command(mc_io, &cmd); + if (err) + return err; + + rsp_params = (struct dpseci_cmd_congestion_notification *)cmd.params; + cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); + cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode); + cfg->dest_cfg.priority = rsp_params->priority; + cfg->dest_cfg.dest_type = dpseci_get_field(rsp_params->options, + CGN_DEST_TYPE); + cfg->units = dpseci_get_field(rsp_params->options, CGN_UNITS); + cfg->message_iova = le64_to_cpu(rsp_params->message_iova); + cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx); + cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry); + cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit); + + return 0; +} --- /dev/null +++ b/drivers/crypto/caam/dpseci.h @@ -0,0 +1,395 @@ +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef _DPSECI_H_ +#define _DPSECI_H_ + +/* + * Data Path SEC Interface API + * Contains initialization APIs and runtime control APIs for DPSECI + */ + +struct fsl_mc_io; +struct opr_cfg; +struct opr_qry; + +/** + * General DPSECI macros + */ + +/** + * Maximum number of Tx/Rx priorities per DPSECI object + */ +#define DPSECI_PRIO_NUM 8 + +/** + * All queues considered; see dpseci_set_rx_queue() + */ +#define DPSECI_ALL_QUEUES (u8)(-1) + +int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id, + u16 *token); + +int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +/** + * Enable the Congestion Group support + */ +#define DPSECI_OPT_HAS_CG 0x000020 + +/** + * Enable the Order Restoration support + */ +#define DPSECI_OPT_HAS_OPR 0x000040 + +/** + * Order Point Records are shared for the entire DPSECI + */ +#define DPSECI_OPT_OPR_SHARED 0x000080 + +/** + * struct dpseci_cfg - Structure representing DPSECI configuration + * @options: Any combination of the following options: + * DPSECI_OPT_HAS_CG + * DPSECI_OPT_HAS_OPR + * DPSECI_OPT_OPR_SHARED + * @num_tx_queues: num of queues towards the SEC + * @num_rx_queues: num of queues back from the SEC + * @priorities: Priorities for the SEC hardware processing; + * each place in the array is the priority of the tx queue + * towards the SEC; + * valid priorities are configured with values 1-8; + */ +struct dpseci_cfg { + u32 options; + u8 num_tx_queues; + u8 num_rx_queues; + u8 priorities[DPSECI_PRIO_NUM]; +}; + +int dpseci_create(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, + const struct dpseci_cfg *cfg, u32 *obj_id); + +int dpseci_destroy(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, + u32 object_id); + +int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + int *en); + +int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); + +int dpseci_get_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u8 *en); + +int dpseci_set_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u8 en); + +int dpseci_get_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 *mask); + +int dpseci_set_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 mask); + +int dpseci_get_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 *status); + +int dpseci_clear_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 irq_index, u32 status); + +/** + * struct dpseci_attr - Structure representing DPSECI attributes + * @id: DPSECI object ID + * @num_tx_queues: number of queues towards the SEC + * @num_rx_queues: number of queues back from the SEC + * @options: any combination of the following options: + * DPSECI_OPT_HAS_CG + * DPSECI_OPT_HAS_OPR + * DPSECI_OPT_OPR_SHARED + */ +struct dpseci_attr { + int id; + u8 num_tx_queues; + u8 num_rx_queues; + u32 options; +}; + +int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_attr *attr); + +/** + * enum dpseci_dest - DPSECI destination types + * @DPSECI_DEST_NONE: Unassigned destination; The queue is set in parked mode + * and does not generate FQDAN notifications; user is expected to dequeue + * from the queue based on polling or other user-defined method + * @DPSECI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN + * notifications to the specified DPIO; user is expected to dequeue from + * the queue only after notification is received + * @DPSECI_DEST_DPCON: The queue is set in schedule mode and does not generate + * FQDAN notifications, but is connected to the specified DPCON object; + * user is expected to dequeue from the DPCON channel + */ +enum dpseci_dest { + DPSECI_DEST_NONE = 0, + DPSECI_DEST_DPIO, + DPSECI_DEST_DPCON +}; + +/** + * struct dpseci_dest_cfg - Structure representing DPSECI destination parameters + * @dest_type: Destination type + * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type + * @priority: Priority selection within the DPIO or DPCON channel; valid values + * are 0-1 or 0-7, depending on the number of priorities in that channel; + * not relevant for 'DPSECI_DEST_NONE' option + */ +struct dpseci_dest_cfg { + enum dpseci_dest dest_type; + int dest_id; + u8 priority; +}; + +/** + * DPSECI queue modification options + */ + +/** + * Select to modify the user's context associated with the queue + */ +#define DPSECI_QUEUE_OPT_USER_CTX 0x00000001 + +/** + * Select to modify the queue's destination + */ +#define DPSECI_QUEUE_OPT_DEST 0x00000002 + +/** + * Select to modify the queue's order preservation + */ +#define DPSECI_QUEUE_OPT_ORDER_PRESERVATION 0x00000004 + +/** + * struct dpseci_rx_queue_cfg - DPSECI RX queue configuration + * @options: Flags representing the suggested modifications to the queue; + * Use any combination of 'DPSECI_QUEUE_OPT_' flags + * @order_preservation_en: order preservation configuration for the rx queue + * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in 'options' + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame; valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained + * in 'options' + * @dest_cfg: Queue destination parameters; valid only if + * 'DPSECI_QUEUE_OPT_DEST' is contained in 'options' + */ +struct dpseci_rx_queue_cfg { + u32 options; + int order_preservation_en; + u64 user_ctx; + struct dpseci_dest_cfg dest_cfg; +}; + +int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, const struct dpseci_rx_queue_cfg *cfg); + +/** + * struct dpseci_rx_queue_attr - Structure representing attributes of Rx queues + * @user_ctx: User context value provided in the frame descriptor of each + * dequeued frame + * @order_preservation_en: Status of the order preservation configuration on the + * queue + * @dest_cfg: Queue destination configuration + * @fqid: Virtual FQID value to be used for dequeue operations + */ +struct dpseci_rx_queue_attr { + u64 user_ctx; + int order_preservation_en; + struct dpseci_dest_cfg dest_cfg; + u32 fqid; +}; + +int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_rx_queue_attr *attr); + +/** + * struct dpseci_tx_queue_attr - Structure representing attributes of Tx queues + * @fqid: Virtual FQID to be used for sending frames to SEC hardware + * @priority: SEC hardware processing priority for the queue + */ +struct dpseci_tx_queue_attr { + u32 fqid; + u8 priority; +}; + +int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u8 queue, struct dpseci_tx_queue_attr *attr); + +/** + * struct dpseci_sec_attr - Structure representing attributes of the SEC + * hardware accelerator + * @ip_id: ID for SEC + * @major_rev: Major revision number for SEC + * @minor_rev: Minor revision number for SEC + * @era: SEC Era + * @deco_num: The number of copies of the DECO that are implemented in this + * version of SEC + * @zuc_auth_acc_num: The number of copies of ZUCA that are implemented in this + * version of SEC + * @zuc_enc_acc_num: The number of copies of ZUCE that are implemented in this + * version of SEC + * @snow_f8_acc_num: The number of copies of the SNOW-f8 module that are + * implemented in this version of SEC + * @snow_f9_acc_num: The number of copies of the SNOW-f9 module that are + * implemented in this version of SEC + * @crc_acc_num: The number of copies of the CRC module that are implemented in + * this version of SEC + * @pk_acc_num: The number of copies of the Public Key module that are + * implemented in this version of SEC + * @kasumi_acc_num: The number of copies of the Kasumi module that are + * implemented in this version of SEC + * @rng_acc_num: The number of copies of the Random Number Generator that are + * implemented in this version of SEC + * @md_acc_num: The number of copies of the MDHA (Hashing module) that are + * implemented in this version of SEC + * @arc4_acc_num: The number of copies of the ARC4 module that are implemented + * in this version of SEC + * @des_acc_num: The number of copies of the DES module that are implemented in + * this version of SEC + * @aes_acc_num: The number of copies of the AES module that are implemented in + * this version of SEC + **/ +struct dpseci_sec_attr { + u16 ip_id; + u8 major_rev; + u8 minor_rev; + u8 era; + u8 deco_num; + u8 zuc_auth_acc_num; + u8 zuc_enc_acc_num; + u8 snow_f8_acc_num; + u8 snow_f9_acc_num; + u8 crc_acc_num; + u8 pk_acc_num; + u8 kasumi_acc_num; + u8 rng_acc_num; + u8 md_acc_num; + u8 arc4_acc_num; + u8 des_acc_num; + u8 aes_acc_num; +}; + +int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_sec_attr *attr); + +/** + * struct dpseci_sec_counters - Structure representing global SEC counters and + * not per dpseci counters + * @dequeued_requests: Number of Requests Dequeued + * @ob_enc_requests: Number of Outbound Encrypt Requests + * @ib_dec_requests: Number of Inbound Decrypt Requests + * @ob_enc_bytes: Number of Outbound Bytes Encrypted + * @ob_prot_bytes: Number of Outbound Bytes Protected + * @ib_dec_bytes: Number of Inbound Bytes Decrypted + * @ib_valid_bytes: Number of Inbound Bytes Validated + */ +struct dpseci_sec_counters { + u64 dequeued_requests; + u64 ob_enc_requests; + u64 ib_dec_requests; + u64 ob_enc_bytes; + u64 ob_prot_bytes; + u64 ib_dec_bytes; + u64 ib_valid_bytes; +}; + +int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + struct dpseci_sec_counters *counters); + +int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 *major_ver, u16 *minor_ver); + +int dpseci_set_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, + u8 options, struct opr_cfg *cfg); + +int dpseci_get_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, + struct opr_cfg *cfg, struct opr_qry *qry); + +/** + * enum dpseci_congestion_unit - DPSECI congestion units + * @DPSECI_CONGESTION_UNIT_BYTES: bytes units + * @DPSECI_CONGESTION_UNIT_FRAMES: frames units + */ +enum dpseci_congestion_unit { + DPSECI_CONGESTION_UNIT_BYTES = 0, + DPSECI_CONGESTION_UNIT_FRAMES +}; + +#define DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER 0x00000001 +#define DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT 0x00000002 +#define DPSECI_CGN_MODE_COHERENT_WRITE 0x00000004 +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_ENTER 0x00000008 +#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_EXIT 0x00000010 +#define DPSECI_CGN_MODE_INTR_COALESCING_DISABLED 0x00000020 + +/** + * struct dpseci_congestion_notification_cfg - congestion notification + * configuration + * @units: units type + * @threshold_entry: above this threshold we enter a congestion state. + * set it to '0' to disable it + * @threshold_exit: below this threshold we exit the congestion state. + * @message_ctx: The context that will be part of the CSCN message + * @message_iova: I/O virtual address (must be in DMA-able memory), + * must be 16B aligned; + * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel + * @notification_mode: Mask of available options; use 'DPSECI_CGN_MODE_' + * values + */ +struct dpseci_congestion_notification_cfg { + enum dpseci_congestion_unit units; + u32 threshold_entry; + u32 threshold_exit; + u64 message_ctx; + u64 message_iova; + struct dpseci_dest_cfg dest_cfg; + u16 notification_mode; +}; + +int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, const struct dpseci_congestion_notification_cfg *cfg); + +int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, + u16 token, struct dpseci_congestion_notification_cfg *cfg); + +#endif /* _DPSECI_H_ */ --- /dev/null +++ b/drivers/crypto/caam/dpseci_cmd.h @@ -0,0 +1,261 @@ +/* + * Copyright 2013-2016 Freescale Semiconductor Inc. + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _DPSECI_CMD_H_ +#define _DPSECI_CMD_H_ + +/* DPSECI Version */ +#define DPSECI_VER_MAJOR 5 +#define DPSECI_VER_MINOR 1 + +#define DPSECI_VER(maj, min) (((maj) << 16) | (min)) +#define DPSECI_VERSION DPSECI_VER(DPSECI_VER_MAJOR, DPSECI_VER_MINOR) + +/* Command IDs */ + +#define DPSECI_CMDID_CLOSE 0x8001 +#define DPSECI_CMDID_OPEN 0x8091 +#define DPSECI_CMDID_CREATE 0x9092 +#define DPSECI_CMDID_DESTROY 0x9891 +#define DPSECI_CMDID_GET_API_VERSION 0xa091 + +#define DPSECI_CMDID_ENABLE 0x0021 +#define DPSECI_CMDID_DISABLE 0x0031 +#define DPSECI_CMDID_GET_ATTR 0x0041 +#define DPSECI_CMDID_RESET 0x0051 +#define DPSECI_CMDID_IS_ENABLED 0x0061 + +#define DPSECI_CMDID_SET_IRQ_ENABLE 0x0121 +#define DPSECI_CMDID_GET_IRQ_ENABLE 0x0131 +#define DPSECI_CMDID_SET_IRQ_MASK 0x0141 +#define DPSECI_CMDID_GET_IRQ_MASK 0x0151 +#define DPSECI_CMDID_GET_IRQ_STATUS 0x0161 +#define DPSECI_CMDID_CLEAR_IRQ_STATUS 0x0171 + +#define DPSECI_CMDID_SET_RX_QUEUE 0x1941 +#define DPSECI_CMDID_GET_RX_QUEUE 0x1961 +#define DPSECI_CMDID_GET_TX_QUEUE 0x1971 +#define DPSECI_CMDID_GET_SEC_ATTR 0x1981 +#define DPSECI_CMDID_GET_SEC_COUNTERS 0x1991 +#define DPSECI_CMDID_SET_OPR 0x19A1 +#define DPSECI_CMDID_GET_OPR 0x19B1 + +#define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION 0x1701 +#define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION 0x1711 + +/* Macros for accessing command fields smaller than 1 byte */ +#define DPSECI_MASK(field) \ + GENMASK(DPSECI_##field##_SHIFT + DPSECI_##field##_SIZE - 1, \ + DPSECI_##field##_SHIFT) + +#define dpseci_set_field(var, field, val) \ + ((var) |= (((val) << DPSECI_##field##_SHIFT) & DPSECI_MASK(field))) + +#define dpseci_get_field(var, field) \ + (((var) & DPSECI_MASK(field)) >> DPSECI_##field##_SHIFT) + +struct dpseci_cmd_open { + __le32 dpseci_id; +}; + +struct dpseci_cmd_create { + u8 priorities[8]; + u8 num_tx_queues; + u8 num_rx_queues; + __le16 pad; + __le32 options; +}; + +struct dpseci_cmd_destroy { + __le32 object_id; +}; + +struct dpseci_rsp_is_enabled { + __le32 is_enabled; +}; + +struct dpseci_cmd_irq_enable { + u8 enable_state; + u8 pad[3]; + u8 irq_index; +}; + +struct dpseci_rsp_get_irq_enable { + u8 enable_state; +}; + +struct dpseci_cmd_irq_mask { + __le32 mask; + u8 irq_index; +}; + +struct dpseci_cmd_irq_status { + __le32 status; + u8 irq_index; +}; + +struct dpseci_rsp_get_attributes { + __le32 id; + __le32 pad0; + u8 num_tx_queues; + u8 num_rx_queues; + u8 pad1[6]; + __le32 options; +}; + +struct dpseci_cmd_queue { + __le32 dest_id; + u8 priority; + u8 queue; + u8 dest_type; + u8 pad; + __le64 user_ctx; + union { + __le32 options; + __le32 fqid; + }; + __le32 order_preservation_en; +}; + +struct dpseci_rsp_get_tx_queue { + __le32 pad; + __le32 fqid; + u8 priority; +}; + +struct dpseci_rsp_get_sec_attr { + __le16 ip_id; + u8 major_rev; + u8 minor_rev; + u8 era; + u8 pad0[3]; + u8 deco_num; + u8 zuc_auth_acc_num; + u8 zuc_enc_acc_num; + u8 pad1; + u8 snow_f8_acc_num; + u8 snow_f9_acc_num; + u8 crc_acc_num; + u8 pad2; + u8 pk_acc_num; + u8 kasumi_acc_num; + u8 rng_acc_num; + u8 pad3; + u8 md_acc_num; + u8 arc4_acc_num; + u8 des_acc_num; + u8 aes_acc_num; +}; + +struct dpseci_rsp_get_sec_counters { + __le64 dequeued_requests; + __le64 ob_enc_requests; + __le64 ib_dec_requests; + __le64 ob_enc_bytes; + __le64 ob_prot_bytes; + __le64 ib_dec_bytes; + __le64 ib_valid_bytes; +}; + +struct dpseci_rsp_get_api_version { + __le16 major; + __le16 minor; +}; + +struct dpseci_cmd_opr { + __le16 pad; + u8 index; + u8 options; + u8 pad1[7]; + u8 oloe; + u8 oeane; + u8 olws; + u8 oa; + u8 oprrws; +}; + +#define DPSECI_OPR_RIP_SHIFT 0 +#define DPSECI_OPR_RIP_SIZE 1 +#define DPSECI_OPR_ENABLE_SHIFT 1 +#define DPSECI_OPR_ENABLE_SIZE 1 +#define DPSECI_OPR_TSEQ_NLIS_SHIFT 1 +#define DPSECI_OPR_TSEQ_NLIS_SIZE 1 +#define DPSECI_OPR_HSEQ_NLIS_SHIFT 1 +#define DPSECI_OPR_HSEQ_NLIS_SIZE 1 + +struct dpseci_rsp_get_opr { + __le64 pad; + u8 rip_enable; + u8 pad0[2]; + u8 oloe; + u8 oeane; + u8 olws; + u8 oa; + u8 oprrws; + __le16 nesn; + __le16 pad1; + __le16 ndsn; + __le16 pad2; + __le16 ea_tseq; + u8 tseq_nlis; + u8 pad3; + __le16 ea_hseq; + u8 hseq_nlis; + u8 pad4; + __le16 ea_hptr; + __le16 pad5; + __le16 ea_tptr; + __le16 pad6; + __le16 opr_vid; + __le16 pad7; + __le16 opr_id; +}; + +#define DPSECI_CGN_DEST_TYPE_SHIFT 0 +#define DPSECI_CGN_DEST_TYPE_SIZE 4 +#define DPSECI_CGN_UNITS_SHIFT 4 +#define DPSECI_CGN_UNITS_SIZE 2 + +struct dpseci_cmd_congestion_notification { + __le32 dest_id; + __le16 notification_mode; + u8 priority; + u8 options; + __le64 message_iova; + __le64 message_ctx; + __le32 threshold_entry; + __le32 threshold_exit; +}; + +#endif /* _DPSECI_CMD_H_ */ --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -6,11 +6,54 @@ #include "compat.h" #include "regs.h" -#include "intern.h" #include "desc.h" -#include "jr.h" #include "error.h" +#ifdef DEBUG + +#include + +void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii) +{ + struct scatterlist *it; + void *it_page; + size_t len; + void *buf; + + for (it = sg; it && tlen > 0 ; it = sg_next(sg)) { + /* + * make sure the scatterlist's page + * has a valid virtual memory mapping + */ + it_page = kmap_atomic(sg_page(it)); + if (unlikely(!it_page)) { + pr_err("caam_dump_sg: kmap failed\n"); + return; + } + + buf = it_page + it->offset; + len = min_t(size_t, tlen, it->length); + print_hex_dump(level, prefix_str, prefix_type, rowsize, + groupsize, buf, len, ascii); + tlen -= len; + + kunmap_atomic(it_page); + } +} + +#else + +void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii) +{} + +#endif + +EXPORT_SYMBOL(caam_dump_sg); + static const struct { u8 value; const char *error_text; @@ -69,6 +112,54 @@ static const struct { { 0xF1, "3GPP HFN matches or exceeds the Threshold" }, }; +static const struct { + u8 value; + const char *error_text; +} qi_error_list[] = { + { 0x1F, "Job terminated by FQ or ICID flush" }, + { 0x20, "FD format error"}, + { 0x21, "FD command format error"}, + { 0x23, "FL format error"}, + { 0x25, "CRJD specified in FD, but not enabled in FLC"}, + { 0x30, "Max. buffer size too small"}, + { 0x31, "DHR exceeds max. buffer size (allocate mode, S/G format)"}, + { 0x32, "SGT exceeds max. buffer size (allocate mode, S/G format"}, + { 0x33, "Size over/underflow (allocate mode)"}, + { 0x34, "Size over/underflow (reuse mode)"}, + { 0x35, "Length exceeds max. short length (allocate mode, S/G/ format)"}, + { 0x36, "Memory footprint exceeds max. value (allocate mode, S/G/ format)"}, + { 0x41, "SBC frame format not supported (allocate mode)"}, + { 0x42, "Pool 0 invalid / pool 1 size < pool 0 size (allocate mode)"}, + { 0x43, "Annotation output enabled but ASAR = 0 (allocate mode)"}, + { 0x44, "Unsupported or reserved frame format or SGHR = 1 (reuse mode)"}, + { 0x45, "DHR correction underflow (reuse mode, single buffer format)"}, + { 0x46, "Annotation length exceeds offset (reuse mode)"}, + { 0x48, "Annotation output enabled but ASA limited by ASAR (reuse mode)"}, + { 0x49, "Data offset correction exceeds input frame data length (reuse mode)"}, + { 0x4B, "Annotation output enabled but ASA cannote be expanded (frame list)"}, + { 0x51, "Unsupported IF reuse mode"}, + { 0x52, "Unsupported FL use mode"}, + { 0x53, "Unsupported RJD use mode"}, + { 0x54, "Unsupported inline descriptor use mode"}, + { 0xC0, "Table buffer pool 0 depletion"}, + { 0xC1, "Table buffer pool 1 depletion"}, + { 0xC2, "Data buffer pool 0 depletion, no OF allocated"}, + { 0xC3, "Data buffer pool 1 depletion, no OF allocated"}, + { 0xC4, "Data buffer pool 0 depletion, partial OF allocated"}, + { 0xC5, "Data buffer pool 1 depletion, partial OF allocated"}, + { 0xD0, "FLC read error"}, + { 0xD1, "FL read error"}, + { 0xD2, "FL write error"}, + { 0xD3, "OF SGT write error"}, + { 0xD4, "PTA read error"}, + { 0xD5, "PTA write error"}, + { 0xD6, "OF SGT F-bit write error"}, + { 0xD7, "ASA write error"}, + { 0xE1, "FLC[ICR]=0 ICID error"}, + { 0xE2, "FLC[ICR]=1 ICID error"}, + { 0xE4, "source of ICID flush not trusted (BDI = 0)"}, +}; + static const char * const cha_id_list[] = { "", "AES", @@ -146,10 +237,9 @@ static void report_ccb_status(struct dev strlen(rng_err_id_list[err_id])) { /* RNG-only error */ err_str = rng_err_id_list[err_id]; - } else if (err_id < ARRAY_SIZE(err_id_list)) + } else { err_str = err_id_list[err_id]; - else - snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); + } /* * CCB ICV check failures are part of normal operation life; @@ -198,6 +288,27 @@ static void report_deco_status(struct de status, error, idx_str, idx, err_str, err_err_code); } +static void report_qi_status(struct device *qidev, const u32 status, + const char *error) +{ + u8 err_id = status & JRSTA_QIERR_ERROR_MASK; + const char *err_str = "unidentified error value 0x"; + char err_err_code[3] = { 0 }; + int i; + + for (i = 0; i < ARRAY_SIZE(qi_error_list); i++) + if (qi_error_list[i].value == err_id) + break; + + if (i != ARRAY_SIZE(qi_error_list) && qi_error_list[i].error_text) + err_str = qi_error_list[i].error_text; + else + snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); + + dev_err(qidev, "%08x: %s: %s%s\n", + status, error, err_str, err_err_code); +} + static void report_jr_status(struct device *jrdev, const u32 status, const char *error) { @@ -212,7 +323,7 @@ static void report_cond_code_status(stru status, error, __func__); } -void caam_jr_strstatus(struct device *jrdev, u32 status) +void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2) { static const struct stat_src { void (*report_ssed)(struct device *jrdev, const u32 status, @@ -224,7 +335,7 @@ void caam_jr_strstatus(struct device *jr { report_ccb_status, "CCB" }, { report_jump_status, "Jump" }, { report_deco_status, "DECO" }, - { NULL, "Queue Manager Interface" }, + { report_qi_status, "Queue Manager Interface" }, { report_jr_status, "Job Ring" }, { report_cond_code_status, "Condition Code" }, { NULL, NULL }, @@ -250,4 +361,4 @@ void caam_jr_strstatus(struct device *jr else dev_err(jrdev, "%d: unknown error source\n", ssrc); } -EXPORT_SYMBOL(caam_jr_strstatus); +EXPORT_SYMBOL(caam_strstatus); --- a/drivers/crypto/caam/error.h +++ b/drivers/crypto/caam/error.h @@ -7,5 +7,13 @@ #ifndef CAAM_ERROR_H #define CAAM_ERROR_H #define CAAM_ERROR_STR_MAX 302 -void caam_jr_strstatus(struct device *jrdev, u32 status); + +void caam_strstatus(struct device *dev, u32 status, bool qi_v2); + +#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false) +#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true) + +void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, + int rowsize, int groupsize, struct scatterlist *sg, + size_t tlen, bool ascii); #endif /* CAAM_ERROR_H */ --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -64,10 +64,9 @@ struct caam_drv_private_jr { * Driver-private storage for a single CAAM block instance */ struct caam_drv_private { - - struct device *dev; - struct platform_device **jrpdev; /* Alloc'ed array per sub-device */ - struct platform_device *pdev; +#ifdef CONFIG_CAAM_QI + struct device *qidev; +#endif /* Physical-presence section */ struct caam_ctrl __iomem *ctrl; /* controller region */ @@ -84,6 +83,7 @@ struct caam_drv_private { u8 qi_present; /* Nonzero if QI present in device */ int secvio_irq; /* Security violation interrupt number */ int virt_en; /* Virtualization enabled in CAAM */ + int era; /* CAAM Era (internal HW revision) */ #define RNG4_MAX_HANDLES 2 /* RNG4 block */ @@ -103,11 +103,6 @@ struct caam_drv_private { #ifdef CONFIG_DEBUG_FS struct dentry *dfs_root; struct dentry *ctl; /* controller dir */ - struct dentry *ctl_rq_dequeued, *ctl_ob_enc_req, *ctl_ib_dec_req; - struct dentry *ctl_ob_enc_bytes, *ctl_ob_prot_bytes; - struct dentry *ctl_ib_dec_bytes, *ctl_ib_valid_bytes; - struct dentry *ctl_faultaddr, *ctl_faultdetail, *ctl_faultstatus; - struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap; struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk; #endif @@ -115,4 +110,22 @@ struct caam_drv_private { void caam_jr_algapi_init(struct device *dev); void caam_jr_algapi_remove(struct device *dev); + +#ifdef CONFIG_DEBUG_FS +static int caam_debugfs_u64_get(void *data, u64 *val) +{ + *val = caam64_to_cpu(*(u64 *)data); + return 0; +} + +static int caam_debugfs_u32_get(void *data, u64 *val) +{ + *val = caam32_to_cpu(*(u32 *)data); + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); +#endif + #endif /* INTERN_H */ --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -9,6 +9,7 @@ #include #include "compat.h" +#include "ctrl.h" #include "regs.h" #include "jr.h" #include "desc.h" @@ -22,6 +23,14 @@ struct jr_driver_data { static struct jr_driver_data driver_data; +static int jr_driver_probed; + +int caam_jr_driver_probed(void) +{ + return jr_driver_probed; +} +EXPORT_SYMBOL(caam_jr_driver_probed); + static int caam_reset_hw_jr(struct device *dev) { struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); @@ -118,6 +127,8 @@ static int caam_jr_remove(struct platfor dev_err(jrdev, "Failed to shut down job ring\n"); irq_dispose_mapping(jrpriv->irq); + jr_driver_probed--; + return ret; } @@ -281,6 +292,36 @@ struct device *caam_jr_alloc(void) EXPORT_SYMBOL(caam_jr_alloc); /** + * caam_jridx_alloc() - Alloc a specific job ring based on its index. + * + * returns : pointer to the newly allocated physical + * JobR dev can be written to if successful. + **/ +struct device *caam_jridx_alloc(int idx) +{ + struct caam_drv_private_jr *jrpriv; + struct device *dev = ERR_PTR(-ENODEV); + + spin_lock(&driver_data.jr_alloc_lock); + + if (list_empty(&driver_data.jr_list)) + goto end; + + list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) { + if (jrpriv->ridx == idx) { + atomic_inc(&jrpriv->tfm_count); + dev = jrpriv->dev; + break; + } + } + +end: + spin_unlock(&driver_data.jr_alloc_lock); + return dev; +} +EXPORT_SYMBOL(caam_jridx_alloc); + +/** * caam_jr_free() - Free the Job Ring * @rdev - points to the dev that identifies the Job ring to * be released. @@ -497,15 +538,28 @@ static int caam_jr_probe(struct platform return -ENOMEM; } - jrpriv->rregs = (struct caam_job_ring __force *)ctrl; + jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl; - if (sizeof(dma_addr_t) == sizeof(u64)) - if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) - dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40)); + if (sizeof(dma_addr_t) == sizeof(u64)) { + if (caam_dpaa2) + error = dma_set_mask_and_coherent(jrdev, + DMA_BIT_MASK(49)); + else if (of_device_is_compatible(nprop, + "fsl,sec-v5.0-job-ring")) + error = dma_set_mask_and_coherent(jrdev, + DMA_BIT_MASK(40)); else - dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36)); - else - dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32)); + error = dma_set_mask_and_coherent(jrdev, + DMA_BIT_MASK(36)); + } else { + error = dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32)); + } + if (error) { + dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n", + error); + iounmap(ctrl); + return error; + } /* Identify the interrupt */ jrpriv->irq = irq_of_parse_and_map(nprop, 0); @@ -525,10 +579,12 @@ static int caam_jr_probe(struct platform atomic_set(&jrpriv->tfm_count, 0); + jr_driver_probed++; + return 0; } -static struct of_device_id caam_jr_match[] = { +static const struct of_device_id caam_jr_match[] = { { .compatible = "fsl,sec-v4.0-job-ring", }, --- a/drivers/crypto/caam/jr.h +++ b/drivers/crypto/caam/jr.h @@ -8,7 +8,9 @@ #define JR_H /* Prototypes for backend-level services exposed to APIs */ +int caam_jr_driver_probed(void); struct device *caam_jr_alloc(void); +struct device *caam_jridx_alloc(int idx); void caam_jr_free(struct device *rdev); int caam_jr_enqueue(struct device *dev, u32 *desc, void (*cbk)(struct device *dev, u32 *desc, u32 status, --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -41,15 +41,29 @@ Split key generation-------------------- [06] 0x64260028 fifostr: class2 mdsplit-jdk len=40 @0xffe04000 */ -int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, - int split_key_pad_len, const u8 *key_in, u32 keylen, - u32 alg_op) +int gen_split_key(struct device *jrdev, u8 *key_out, + struct alginfo * const adata, const u8 *key_in, u32 keylen, + int max_keylen) { u32 *desc; struct split_key_result result; dma_addr_t dma_addr_in, dma_addr_out; int ret = -ENOMEM; + adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK); + adata->keylen_pad = split_key_pad_len(adata->algtype & + OP_ALG_ALGSEL_MASK); + +#ifdef DEBUG + dev_err(jrdev, "split keylen %d split keylen padded %d\n", + adata->keylen, adata->keylen_pad); + print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); +#endif + + if (adata->keylen_pad > max_keylen) + return -EINVAL; + desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); if (!desc) { dev_err(jrdev, "unable to allocate key input memory\n"); @@ -63,7 +77,7 @@ int gen_split_key(struct device *jrdev, goto out_free; } - dma_addr_out = dma_map_single(jrdev, key_out, split_key_pad_len, + dma_addr_out = dma_map_single(jrdev, key_out, adata->keylen_pad, DMA_FROM_DEVICE); if (dma_mapping_error(jrdev, dma_addr_out)) { dev_err(jrdev, "unable to map key output memory\n"); @@ -74,7 +88,9 @@ int gen_split_key(struct device *jrdev, append_key(desc, dma_addr_in, keylen, CLASS_2 | KEY_DEST_CLASS_REG); /* Sets MDHA up into an HMAC-INIT */ - append_operation(desc, alg_op | OP_ALG_DECRYPT | OP_ALG_AS_INIT); + append_operation(desc, (adata->algtype & OP_ALG_ALGSEL_MASK) | + OP_ALG_AAI_HMAC | OP_TYPE_CLASS2_ALG | OP_ALG_DECRYPT | + OP_ALG_AS_INIT); /* * do a FIFO_LOAD of zero, this will trigger the internal key expansion @@ -87,7 +103,7 @@ int gen_split_key(struct device *jrdev, * FIFO_STORE with the explicit split-key content store * (0x26 output type) */ - append_fifo_store(desc, dma_addr_out, split_key_len, + append_fifo_store(desc, dma_addr_out, adata->keylen, LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); #ifdef DEBUG @@ -108,11 +124,11 @@ int gen_split_key(struct device *jrdev, #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key_out, - split_key_pad_len, 1); + adata->keylen_pad, 1); #endif } - dma_unmap_single(jrdev, dma_addr_out, split_key_pad_len, + dma_unmap_single(jrdev, dma_addr_out, adata->keylen_pad, DMA_FROM_DEVICE); out_unmap_in: dma_unmap_single(jrdev, dma_addr_in, keylen, DMA_TO_DEVICE); --- a/drivers/crypto/caam/key_gen.h +++ b/drivers/crypto/caam/key_gen.h @@ -5,6 +5,36 @@ * */ +/** + * split_key_len - Compute MDHA split key length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, + * SHA224, SHA384, SHA512. + * + * Return: MDHA split key length + */ +static inline u32 split_key_len(u32 hash) +{ + /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ + static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; + u32 idx; + + idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT; + + return (u32)(mdpadlen[idx] * 2); +} + +/** + * split_key_pad_len - Compute MDHA split key pad length for a given algorithm + * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, + * SHA224, SHA384, SHA512. + * + * Return: MDHA split key pad length + */ +static inline u32 split_key_pad_len(u32 hash) +{ + return ALIGN(split_key_len(hash), 16); +} + struct split_key_result { struct completion completion; int err; @@ -12,6 +42,6 @@ struct split_key_result { void split_key_done(struct device *dev, u32 *desc, u32 err, void *context); -int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, - int split_key_pad_len, const u8 *key_in, u32 keylen, - u32 alg_op); +int gen_split_key(struct device *jrdev, u8 *key_out, + struct alginfo * const adata, const u8 *key_in, u32 keylen, + int max_keylen); --- a/drivers/crypto/caam/pdb.h +++ b/drivers/crypto/caam/pdb.h @@ -483,6 +483,8 @@ struct dsa_verify_pdb { #define RSA_PDB_E_MASK (0xFFF << RSA_PDB_E_SHIFT) #define RSA_PDB_D_SHIFT 12 #define RSA_PDB_D_MASK (0xFFF << RSA_PDB_D_SHIFT) +#define RSA_PDB_Q_SHIFT 12 +#define RSA_PDB_Q_MASK (0xFFF << RSA_PDB_Q_SHIFT) #define RSA_PDB_SGF_F (0x8 << RSA_PDB_SGF_SHIFT) #define RSA_PDB_SGF_G (0x4 << RSA_PDB_SGF_SHIFT) @@ -490,6 +492,8 @@ struct dsa_verify_pdb { #define RSA_PRIV_PDB_SGF_G (0x8 << RSA_PDB_SGF_SHIFT) #define RSA_PRIV_KEY_FRM_1 0 +#define RSA_PRIV_KEY_FRM_2 1 +#define RSA_PRIV_KEY_FRM_3 2 /** * RSA Encrypt Protocol Data Block @@ -525,4 +529,62 @@ struct rsa_priv_f1_pdb { dma_addr_t d_dma; } __packed; +/** + * RSA Decrypt PDB - Private Key Form #2 + * @sgf : scatter-gather field + * @g_dma : dma address of encrypted input data + * @f_dma : dma address of output data + * @d_dma : dma address of RSA private exponent + * @p_dma : dma address of RSA prime factor p of RSA modulus n + * @q_dma : dma address of RSA prime factor q of RSA modulus n + * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as p. + * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as q. + * @p_q_len : length in bytes of first two prime factors of the RSA modulus n + */ +struct rsa_priv_f2_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t d_dma; + dma_addr_t p_dma; + dma_addr_t q_dma; + dma_addr_t tmp1_dma; + dma_addr_t tmp2_dma; + u32 p_q_len; +} __packed; + +/** + * RSA Decrypt PDB - Private Key Form #3 + * This is the RSA Chinese Reminder Theorem (CRT) form for two prime factors of + * the RSA modulus. + * @sgf : scatter-gather field + * @g_dma : dma address of encrypted input data + * @f_dma : dma address of output data + * @c_dma : dma address of RSA CRT coefficient + * @p_dma : dma address of RSA prime factor p of RSA modulus n + * @q_dma : dma address of RSA prime factor q of RSA modulus n + * @dp_dma : dma address of RSA CRT exponent of RSA prime factor p + * @dp_dma : dma address of RSA CRT exponent of RSA prime factor q + * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as p. + * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer + * as internal state buffer. It is assumed to be as long as q. + * @p_q_len : length in bytes of first two prime factors of the RSA modulus n + */ +struct rsa_priv_f3_pdb { + u32 sgf; + dma_addr_t g_dma; + dma_addr_t f_dma; + dma_addr_t c_dma; + dma_addr_t p_dma; + dma_addr_t q_dma; + dma_addr_t dp_dma; + dma_addr_t dq_dma; + dma_addr_t tmp1_dma; + dma_addr_t tmp2_dma; + u32 p_q_len; +} __packed; + #endif --- a/drivers/crypto/caam/pkc_desc.c +++ b/drivers/crypto/caam/pkc_desc.c @@ -34,3 +34,39 @@ void init_rsa_priv_f1_desc(u32 *desc, st append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | RSA_PRIV_KEY_FRM_1); } + +/* Descriptor for RSA Private operation - Private Key Form #2 */ +void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->d_dma); + append_ptr(desc, pdb->p_dma); + append_ptr(desc, pdb->q_dma); + append_ptr(desc, pdb->tmp1_dma); + append_ptr(desc, pdb->tmp2_dma); + append_cmd(desc, pdb->p_q_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_2); +} + +/* Descriptor for RSA Private operation - Private Key Form #3 */ +void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb) +{ + init_job_desc_pdb(desc, 0, sizeof(*pdb)); + append_cmd(desc, pdb->sgf); + append_ptr(desc, pdb->g_dma); + append_ptr(desc, pdb->f_dma); + append_ptr(desc, pdb->c_dma); + append_ptr(desc, pdb->p_dma); + append_ptr(desc, pdb->q_dma); + append_ptr(desc, pdb->dp_dma); + append_ptr(desc, pdb->dq_dma); + append_ptr(desc, pdb->tmp1_dma); + append_ptr(desc, pdb->tmp2_dma); + append_cmd(desc, pdb->p_q_len); + append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | + RSA_PRIV_KEY_FRM_3); +} --- /dev/null +++ b/drivers/crypto/caam/qi.c @@ -0,0 +1,797 @@ +/* + * CAAM/SEC 4.x QI transport/backend driver + * Queue Interface backend functionality + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + */ + +#include +#include +#include + +#include "regs.h" +#include "qi.h" +#include "desc.h" +#include "intern.h" +#include "desc_constr.h" + +#define PREHDR_RSLS_SHIFT 31 + +/* + * Use a reasonable backlog of frames (per CPU) as congestion threshold, + * so that resources used by the in-flight buffers do not become a memory hog. + */ +#define MAX_RSP_FQ_BACKLOG_PER_CPU 256 + +#define CAAM_QI_ENQUEUE_RETRIES 10000 + +#define CAAM_NAPI_WEIGHT 63 + +/* + * caam_napi - struct holding CAAM NAPI-related params + * @irqtask: IRQ task for QI backend + * @p: QMan portal + */ +struct caam_napi { + struct napi_struct irqtask; + struct qman_portal *p; +}; + +/* + * caam_qi_pcpu_priv - percpu private data structure to main list of pending + * responses expected on each cpu. + * @caam_napi: CAAM NAPI params + * @net_dev: netdev used by NAPI + * @rsp_fq: response FQ from CAAM + */ +struct caam_qi_pcpu_priv { + struct caam_napi caam_napi; + struct net_device net_dev; + struct qman_fq *rsp_fq; +} ____cacheline_aligned; + +static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv); +static DEFINE_PER_CPU(int, last_cpu); + +/* + * caam_qi_priv - CAAM QI backend private params + * @cgr: QMan congestion group + * @qi_pdev: platform device for QI backend + */ +struct caam_qi_priv { + struct qman_cgr cgr; + struct platform_device *qi_pdev; +}; + +static struct caam_qi_priv qipriv ____cacheline_aligned; + +/* + * This is written by only one core - the one that initialized the CGR - and + * read by multiple cores (all the others). + */ +bool caam_congested __read_mostly; +EXPORT_SYMBOL(caam_congested); + +#ifdef CONFIG_DEBUG_FS +/* + * This is a counter for the number of times the congestion group (where all + * the request and response queueus are) reached congestion. Incremented + * each time the congestion callback is called with congested == true. + */ +static u64 times_congested; +#endif + +/* + * CPU from where the module initialised. This is required because QMan driver + * requires CGRs to be removed from same CPU from where they were originally + * allocated. + */ +static int mod_init_cpu; + +/* + * This is a a cache of buffers, from which the users of CAAM QI driver + * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than + * doing malloc on the hotpath. + * NOTE: A more elegant solution would be to have some headroom in the frames + * being processed. This could be added by the dpaa-ethernet driver. + * This would pose a problem for userspace application processing which + * cannot know of this limitation. So for now, this will work. + * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here + */ +static struct kmem_cache *qi_cache; + +int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) +{ + struct qm_fd fd; + int ret; + int num_retries = 0; + + fd.cmd = 0; + fd.format = qm_fd_compound; + fd.cong_weight = caam32_to_cpu(req->fd_sgt[1].length); + fd.addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt), + DMA_BIDIRECTIONAL); + if (dma_mapping_error(qidev, fd.addr)) { + dev_err(qidev, "DMA mapping error for QI enqueue request\n"); + return -EIO; + } + + do { + ret = qman_enqueue(req->drv_ctx->req_fq, &fd, 0); + if (likely(!ret)) + return 0; + + if (ret != -EBUSY) + break; + num_retries++; + } while (num_retries < CAAM_QI_ENQUEUE_RETRIES); + + dev_err(qidev, "qman_enqueue failed: %d\n", ret); + + return ret; +} +EXPORT_SYMBOL(caam_qi_enqueue); + +static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, + const struct qm_mr_entry *msg) +{ + const struct qm_fd *fd; + struct caam_drv_req *drv_req; + struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + + fd = &msg->ern.fd; + + if (fd->format != qm_fd_compound) { + dev_err(qidev, "Non-compound FD from CAAM\n"); + return; + } + + drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + if (!drv_req) { + dev_err(qidev, + "Can't find original request for CAAM response\n"); + return; + } + + dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), + sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); + + drv_req->cbk(drv_req, -EIO); +} + +static struct qman_fq *create_caam_req_fq(struct device *qidev, + struct qman_fq *rsp_fq, + dma_addr_t hwdesc, + int fq_sched_flag) +{ + int ret; + struct qman_fq *req_fq; + struct qm_mcc_initfq opts; + + req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC); + if (!req_fq) + return ERR_PTR(-ENOMEM); + + req_fq->cb.ern = caam_fq_ern_cb; + req_fq->cb.fqs = NULL; + + ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID | + QMAN_FQ_FLAG_TO_DCPORTAL | QMAN_FQ_FLAG_LOCKED, + req_fq); + if (ret) { + dev_err(qidev, "Failed to create session req FQ\n"); + goto create_req_fq_fail; + } + + opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | + QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA | + QM_INITFQ_WE_CGID; + opts.fqd.fq_ctrl = QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE; + opts.fqd.dest.channel = qm_channel_caam; + opts.fqd.dest.wq = 2; + opts.fqd.context_b = qman_fq_fqid(rsp_fq); + opts.fqd.context_a.hi = upper_32_bits(hwdesc); + opts.fqd.context_a.lo = lower_32_bits(hwdesc); + opts.fqd.cgid = qipriv.cgr.cgrid; + + ret = qman_init_fq(req_fq, fq_sched_flag, &opts); + if (ret) { + dev_err(qidev, "Failed to init session req FQ\n"); + goto init_req_fq_fail; + } + + dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid, + smp_processor_id()); + return req_fq; + +init_req_fq_fail: + qman_destroy_fq(req_fq, 0); +create_req_fq_fail: + kfree(req_fq); + return ERR_PTR(ret); +} + +static int empty_retired_fq(struct device *qidev, struct qman_fq *fq) +{ + int ret; + + ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT | + QMAN_VOLATILE_FLAG_FINISH, + QM_VDQCR_PRECEDENCE_VDQCR | + QM_VDQCR_NUMFRAMES_TILLEMPTY); + if (ret) { + dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid); + return ret; + } + + do { + struct qman_portal *p; + + p = qman_get_affine_portal(smp_processor_id()); + qman_p_poll_dqrr(p, 16); + } while (fq->flags & QMAN_FQ_STATE_NE); + + return 0; +} + +static int kill_fq(struct device *qidev, struct qman_fq *fq) +{ + u32 flags; + int ret; + + ret = qman_retire_fq(fq, &flags); + if (ret < 0) { + dev_err(qidev, "qman_retire_fq failed: %d\n", ret); + return ret; + } + + if (!ret) + goto empty_fq; + + /* Async FQ retirement condition */ + if (ret == 1) { + /* Retry till FQ gets in retired state */ + do { + msleep(20); + } while (fq->state != qman_fq_state_retired); + + WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS); + WARN_ON(fq->flags & QMAN_FQ_STATE_ORL); + } + +empty_fq: + if (fq->flags & QMAN_FQ_STATE_NE) { + ret = empty_retired_fq(qidev, fq); + if (ret) { + dev_err(qidev, "empty_retired_fq fail for FQ: %u\n", + fq->fqid); + return ret; + } + } + + ret = qman_oos_fq(fq); + if (ret) + dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid); + + qman_destroy_fq(fq, 0); + kfree(fq); + + return ret; +} + +static int empty_caam_fq(struct qman_fq *fq) +{ + int ret; + struct qm_mcr_queryfq_np np; + + /* Wait till the older CAAM FQ get empty */ + do { + ret = qman_query_fq_np(fq, &np); + if (ret) + return ret; + + if (!np.frm_cnt) + break; + + msleep(20); + } while (1); + + /* + * Give extra time for pending jobs from this FQ in holding tanks + * to get processed + */ + msleep(20); + return 0; +} + +int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) +{ + int ret; + u32 num_words; + struct qman_fq *new_fq, *old_fq; + struct device *qidev = drv_ctx->qidev; + + num_words = desc_len(sh_desc); + if (num_words > MAX_SDLEN) { + dev_err(qidev, "Invalid descriptor len: %d words\n", num_words); + return -EINVAL; + } + + /* Note down older req FQ */ + old_fq = drv_ctx->req_fq; + + /* Create a new req FQ in parked state */ + new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq, + drv_ctx->context_a, 0); + if (unlikely(IS_ERR_OR_NULL(new_fq))) { + dev_err(qidev, "FQ allocation for shdesc update failed\n"); + return PTR_ERR(new_fq); + } + + /* Hook up new FQ to context so that new requests keep queuing */ + drv_ctx->req_fq = new_fq; + + /* Empty and remove the older FQ */ + ret = empty_caam_fq(old_fq); + if (ret) { + dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret); + + /* We can revert to older FQ */ + drv_ctx->req_fq = old_fq; + + if (kill_fq(qidev, new_fq)) + dev_warn(qidev, "New CAAM FQ kill failed\n"); + + return ret; + } + + /* + * Re-initialise pre-header. Set RSLS and SDLEN. + * Update the shared descriptor for driver context. + */ + drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | + num_words); + memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); + dma_sync_single_for_device(qidev, drv_ctx->context_a, + sizeof(drv_ctx->sh_desc) + + sizeof(drv_ctx->prehdr), + DMA_BIDIRECTIONAL); + + /* Put the new FQ in scheduled state */ + ret = qman_schedule_fq(new_fq); + if (ret) { + dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret); + + /* + * We can kill new FQ and revert to old FQ. + * Since the desc is already modified, it is success case + */ + + drv_ctx->req_fq = old_fq; + + if (kill_fq(qidev, new_fq)) + dev_warn(qidev, "New CAAM FQ kill failed\n"); + } else if (kill_fq(qidev, old_fq)) { + dev_warn(qidev, "Old CAAM FQ kill failed\n"); + } + + return 0; +} +EXPORT_SYMBOL(caam_drv_ctx_update); + +struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, + int *cpu, + u32 *sh_desc) +{ + size_t size; + u32 num_words; + dma_addr_t hwdesc; + struct caam_drv_ctx *drv_ctx; + const cpumask_t *cpus = qman_affine_cpus(); + + num_words = desc_len(sh_desc); + if (num_words > MAX_SDLEN) { + dev_err(qidev, "Invalid descriptor len: %d words\n", + num_words); + return ERR_PTR(-EINVAL); + } + + drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC); + if (!drv_ctx) + return ERR_PTR(-ENOMEM); + + /* + * Initialise pre-header - set RSLS and SDLEN - and shared descriptor + * and dma-map them. + */ + drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | + num_words); + memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); + size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); + hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(qidev, hwdesc)) { + dev_err(qidev, "DMA map error for preheader + shdesc\n"); + kfree(drv_ctx); + return ERR_PTR(-ENOMEM); + } + drv_ctx->context_a = hwdesc; + + /* If given CPU does not own the portal, choose another one that does */ + if (!cpumask_test_cpu(*cpu, cpus)) { + int *pcpu = &get_cpu_var(last_cpu); + + *pcpu = cpumask_next(*pcpu, cpus); + if (*pcpu >= nr_cpu_ids) + *pcpu = cpumask_first(cpus); + *cpu = *pcpu; + + put_cpu_var(last_cpu); + } + drv_ctx->cpu = *cpu; + + /* Find response FQ hooked with this CPU */ + drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu); + + /* Attach request FQ */ + drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc, + QMAN_INITFQ_FLAG_SCHED); + if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) { + dev_err(qidev, "create_caam_req_fq failed\n"); + dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); + kfree(drv_ctx); + return ERR_PTR(-ENOMEM); + } + + drv_ctx->qidev = qidev; + return drv_ctx; +} +EXPORT_SYMBOL(caam_drv_ctx_init); + +void *qi_cache_alloc(gfp_t flags) +{ + return kmem_cache_alloc(qi_cache, flags); +} +EXPORT_SYMBOL(qi_cache_alloc); + +void qi_cache_free(void *obj) +{ + kmem_cache_free(qi_cache, obj); +} +EXPORT_SYMBOL(qi_cache_free); + +static int caam_qi_poll(struct napi_struct *napi, int budget) +{ + struct caam_napi *np = container_of(napi, struct caam_napi, irqtask); + + int cleaned = qman_p_poll_dqrr(np->p, budget); + + if (cleaned < budget) { + napi_complete(napi); + qman_p_irqsource_add(np->p, QM_PIRQ_DQRI); + } + + return cleaned; +} + +void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx) +{ + if (IS_ERR_OR_NULL(drv_ctx)) + return; + + /* Remove request FQ */ + if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq)) + dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n"); + + dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a, + sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr), + DMA_BIDIRECTIONAL); + kfree(drv_ctx); +} +EXPORT_SYMBOL(caam_drv_ctx_rel); + +int caam_qi_shutdown(struct device *qidev) +{ + int i, ret; + struct caam_qi_priv *priv = dev_get_drvdata(qidev); + const cpumask_t *cpus = qman_affine_cpus(); + struct cpumask old_cpumask = current->cpus_allowed; + + for_each_cpu(i, cpus) { + struct napi_struct *irqtask; + + irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask; + napi_disable(irqtask); + netif_napi_del(irqtask); + + if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i))) + dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i); + } + + /* + * QMan driver requires CGRs to be deleted from same CPU from where they + * were instantiated. Hence we get the module removal execute from the + * same CPU from where it was originally inserted. + */ + set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu)); + + ret = qman_delete_cgr(&priv->cgr); + if (ret) + dev_err(qidev, "Deletion of CGR failed: %d\n", ret); + else + qman_release_cgrid(priv->cgr.cgrid); + + kmem_cache_destroy(qi_cache); + + /* Now that we're done with the CGRs, restore the cpus allowed mask */ + set_cpus_allowed_ptr(current, &old_cpumask); + + platform_device_unregister(priv->qi_pdev); + return ret; +} + +static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) +{ + caam_congested = congested; + + if (congested) { +#ifdef CONFIG_DEBUG_FS + times_congested++; +#endif + pr_debug_ratelimited("CAAM entered congestion\n"); + + } else { + pr_debug_ratelimited("CAAM exited congestion\n"); + } +} + +static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np) +{ + /* + * In case of threaded ISR, for RT kernels in_irq() does not return + * appropriate value, so use in_serving_softirq to distinguish between + * softirq and irq contexts. + */ + if (unlikely(in_irq() || !in_serving_softirq())) { + /* Disable QMan IRQ source and invoke NAPI */ + qman_p_irqsource_remove(p, QM_PIRQ_DQRI); + np->p = p; + napi_schedule(&np->irqtask); + return 1; + } + return 0; +} + +static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, + struct qman_fq *rsp_fq, + const struct qm_dqrr_entry *dqrr) +{ + struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi); + struct caam_drv_req *drv_req; + const struct qm_fd *fd; + struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + + if (caam_qi_napi_schedule(p, caam_napi)) + return qman_cb_dqrr_stop; + + fd = &dqrr->fd; + if (unlikely(fd->status)) + dev_err(qidev, "Error: %#x in CAAM response FD\n", fd->status); + + if (unlikely(fd->format != fd->format)) { + dev_err(qidev, "Non-compound FD from CAAM\n"); + return qman_cb_dqrr_consume; + } + + drv_req = (struct caam_drv_req *)phys_to_virt(fd->addr); + if (unlikely(!drv_req)) { + dev_err(qidev, + "Can't find original request for caam response\n"); + return qman_cb_dqrr_consume; + } + + dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), + sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); + + drv_req->cbk(drv_req, fd->status); + return qman_cb_dqrr_consume; +} + +static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu) +{ + struct qm_mcc_initfq opts; + struct qman_fq *fq; + int ret; + + fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA); + if (!fq) + return -ENOMEM; + + fq->cb.dqrr = caam_rsp_fq_dqrr_cb; + + ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE | + QMAN_FQ_FLAG_DYNAMIC_FQID, fq); + if (ret) { + dev_err(qidev, "Rsp FQ create failed\n"); + kfree(fq); + return -ENODEV; + } + + opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | + QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA | + QM_INITFQ_WE_CGID; + opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING | QM_FQCTRL_CPCSTASH | + QM_FQCTRL_CGE; + opts.fqd.dest.channel = qman_affine_channel(cpu); + opts.fqd.dest.wq = 3; + opts.fqd.cgid = qipriv.cgr.cgrid; + opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX | + QM_STASHING_EXCL_DATA; + opts.fqd.context_a.stashing.data_cl = 1; + opts.fqd.context_a.stashing.context_cl = 1; + + ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts); + if (ret) { + dev_err(qidev, "Rsp FQ init failed\n"); + kfree(fq); + return -ENODEV; + } + + per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq; + + dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu); + return 0; +} + +static int init_cgr(struct device *qidev) +{ + int ret; + struct qm_mcc_initcgr opts; + const u64 cpus = *(u64 *)qman_affine_cpus(); + const int num_cpus = hweight64(cpus); + const u64 val = num_cpus * MAX_RSP_FQ_BACKLOG_PER_CPU; + + ret = qman_alloc_cgrid(&qipriv.cgr.cgrid); + if (ret) { + dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret); + return ret; + } + + qipriv.cgr.cb = cgr_cb; + memset(&opts, 0, sizeof(opts)); + opts.we_mask = QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES | QM_CGR_WE_MODE; + opts.cgr.cscn_en = QM_CGR_EN; + opts.cgr.mode = QMAN_CGR_MODE_FRAME; + qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1); + + ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts); + if (ret) { + dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret, + qipriv.cgr.cgrid); + return ret; + } + + dev_dbg(qidev, "Congestion threshold set to %llu\n", val); + return 0; +} + +static int alloc_rsp_fqs(struct device *qidev) +{ + int ret, i; + const cpumask_t *cpus = qman_affine_cpus(); + + /*Now create response FQs*/ + for_each_cpu(i, cpus) { + ret = alloc_rsp_fq_cpu(qidev, i); + if (ret) { + dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i); + return ret; + } + } + + return 0; +} + +static void free_rsp_fqs(void) +{ + int i; + const cpumask_t *cpus = qman_affine_cpus(); + + for_each_cpu(i, cpus) + kfree(per_cpu(pcpu_qipriv.rsp_fq, i)); +} + +int caam_qi_init(struct platform_device *caam_pdev) +{ + int err, i; + struct platform_device *qi_pdev; + struct device *ctrldev = &caam_pdev->dev, *qidev; + struct caam_drv_private *ctrlpriv; + const cpumask_t *cpus = qman_affine_cpus(); + struct cpumask old_cpumask = current->cpus_allowed; + static struct platform_device_info qi_pdev_info = { + .name = "caam_qi", + .id = PLATFORM_DEVID_NONE + }; + + /* + * QMAN requires CGRs to be removed from same CPU+portal from where it + * was originally allocated. Hence we need to note down the + * initialisation CPU and use the same CPU for module exit. + * We select the first CPU to from the list of portal owning CPUs. + * Then we pin module init to this CPU. + */ + mod_init_cpu = cpumask_first(cpus); + set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu)); + + qi_pdev_info.parent = ctrldev; + qi_pdev_info.dma_mask = dma_get_mask(ctrldev); + qi_pdev = platform_device_register_full(&qi_pdev_info); + if (IS_ERR(qi_pdev)) + return PTR_ERR(qi_pdev); + arch_setup_dma_ops(&qi_pdev->dev, 0, 0, NULL, true); + + ctrlpriv = dev_get_drvdata(ctrldev); + qidev = &qi_pdev->dev; + + qipriv.qi_pdev = qi_pdev; + dev_set_drvdata(qidev, &qipriv); + + /* Initialize the congestion detection */ + err = init_cgr(qidev); + if (err) { + dev_err(qidev, "CGR initialization failed: %d\n", err); + platform_device_unregister(qi_pdev); + return err; + } + + /* Initialise response FQs */ + err = alloc_rsp_fqs(qidev); + if (err) { + dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err); + free_rsp_fqs(); + platform_device_unregister(qi_pdev); + return err; + } + + /* + * Enable the NAPI contexts on each of the core which has an affine + * portal. + */ + for_each_cpu(i, cpus) { + struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i); + struct caam_napi *caam_napi = &priv->caam_napi; + struct napi_struct *irqtask = &caam_napi->irqtask; + struct net_device *net_dev = &priv->net_dev; + + net_dev->dev = *qidev; + INIT_LIST_HEAD(&net_dev->napi_list); + + netif_napi_add(net_dev, irqtask, caam_qi_poll, + CAAM_NAPI_WEIGHT); + + napi_enable(irqtask); + } + + /* Hook up QI device to parent controlling caam device */ + ctrlpriv->qidev = qidev; + + qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0, + SLAB_CACHE_DMA, NULL); + if (!qi_cache) { + dev_err(qidev, "Can't allocate CAAM cache\n"); + free_rsp_fqs(); + platform_device_unregister(qi_pdev); + return -ENOMEM; + } + + /* Done with the CGRs; restore the cpus allowed mask */ + set_cpus_allowed_ptr(current, &old_cpumask); +#ifdef CONFIG_DEBUG_FS + debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, + ×_congested, &caam_fops_u64_ro); +#endif + dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); + return 0; +} --- /dev/null +++ b/drivers/crypto/caam/qi.h @@ -0,0 +1,204 @@ +/* + * Public definitions for the CAAM/QI (Queue Interface) backend. + * + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + */ + +#ifndef __QI_H__ +#define __QI_H__ + +#include +#include "compat.h" +#include "desc.h" +#include "desc_constr.h" + +/* + * CAAM hardware constructs a job descriptor which points to a shared descriptor + * (as pointed by context_a of to-CAAM FQ). + * When the job descriptor is executed by DECO, the whole job descriptor + * together with shared descriptor gets loaded in DECO buffer, which is + * 64 words (each 32-bit) long. + * + * The job descriptor constructed by CAAM hardware has the following layout: + * + * HEADER (1 word) + * Shdesc ptr (1 or 2 words) + * SEQ_OUT_PTR (1 word) + * Out ptr (1 or 2 words) + * Out length (1 word) + * SEQ_IN_PTR (1 word) + * In ptr (1 or 2 words) + * In length (1 word) + * + * The shdesc ptr is used to fetch shared descriptor contents into DECO buffer. + * + * Apart from shdesc contents, the total number of words that get loaded in DECO + * buffer are '8' or '11'. The remaining words in DECO buffer can be used for + * storing shared descriptor. + */ +#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ) + +/* Length of a single buffer in the QI driver memory cache */ +#define CAAM_QI_MEMCACHE_SIZE 768 + +extern bool caam_congested __read_mostly; + +/* + * This is the request structure the driver application should fill while + * submitting a job to driver. + */ +struct caam_drv_req; + +/* + * caam_qi_cbk - application's callback function invoked by the driver when the + * request has been successfully processed. + * @drv_req: original request that was submitted + * @status: completion status of request (0 - success, non-zero - error code) + */ +typedef void (*caam_qi_cbk)(struct caam_drv_req *drv_req, u32 status); + +enum optype { + ENCRYPT, + DECRYPT, + GIVENCRYPT, + NUM_OP +}; + +/** + * caam_drv_ctx - CAAM/QI backend driver context + * + * The jobs are processed by the driver against a driver context. + * With every cryptographic context, a driver context is attached. + * The driver context contains data for private use by driver. + * For the applications, this is an opaque structure. + * + * @prehdr: preheader placed before shrd desc + * @sh_desc: shared descriptor + * @context_a: shared descriptor dma address + * @req_fq: to-CAAM request frame queue + * @rsp_fq: from-CAAM response frame queue + * @cpu: cpu on which to receive CAAM response + * @op_type: operation type + * @qidev: device pointer for CAAM/QI backend + */ +struct caam_drv_ctx { + u32 prehdr[2]; + u32 sh_desc[MAX_SDLEN]; + dma_addr_t context_a; + struct qman_fq *req_fq; + struct qman_fq *rsp_fq; + int cpu; + enum optype op_type; + struct device *qidev; +} ____cacheline_aligned; + +/** + * caam_drv_req - The request structure the driver application should fill while + * submitting a job to driver. + * @fd_sgt: QMan S/G pointing to output (fd_sgt[0]) and input (fd_sgt[1]) + * buffers. + * @cbk: callback function to invoke when job is completed + * @app_ctx: arbitrary context attached with request by the application + * + * The fields mentioned below should not be used by application. + * These are for private use by driver. + * + * @hdr__: linked list header to maintain list of outstanding requests to CAAM + * @hwaddr: DMA address for the S/G table. + */ +struct caam_drv_req { + struct qm_sg_entry fd_sgt[2]; + struct caam_drv_ctx *drv_ctx; + caam_qi_cbk cbk; + void *app_ctx; +} ____cacheline_aligned; + +/** + * caam_drv_ctx_init - Initialise a CAAM/QI driver context + * + * A CAAM/QI driver context must be attached with each cryptographic context. + * This function allocates memory for CAAM/QI context and returns a handle to + * the application. This handle must be submitted along with each enqueue + * request to the driver by the application. + * + * @cpu: CPU where the application prefers to the driver to receive CAAM + * responses. The request completion callback would be issued from this + * CPU. + * @sh_desc: shared descriptor pointer to be attached with CAAM/QI driver + * context. + * + * Returns a driver context on success or negative error code on failure. + */ +struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, int *cpu, + u32 *sh_desc); + +/** + * caam_qi_enqueue - Submit a request to QI backend driver. + * + * The request structure must be properly filled as described above. + * + * @qidev: device pointer for QI backend + * @req: CAAM QI request structure + * + * Returns 0 on success or negative error code on failure. + */ +int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req); + +/** + * caam_drv_ctx_busy - Check if there are too many jobs pending with CAAM + * or too many CAAM responses are pending to be processed. + * @drv_ctx: driver context for which job is to be submitted + * + * Returns caam congestion status 'true/false' + */ +bool caam_drv_ctx_busy(struct caam_drv_ctx *drv_ctx); + +/** + * caam_drv_ctx_update - Update QI driver context + * + * Invoked when shared descriptor is required to be change in driver context. + * + * @drv_ctx: driver context to be updated + * @sh_desc: new shared descriptor pointer to be updated in QI driver context + * + * Returns 0 on success or negative error code on failure. + */ +int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc); + +/** + * caam_drv_ctx_rel - Release a QI driver context + * @drv_ctx: context to be released + */ +void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx); + +int caam_qi_init(struct platform_device *pdev); +int caam_qi_shutdown(struct device *dev); + +/** + * qi_cache_alloc - Allocate buffers from CAAM-QI cache + * + * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) needs data which has + * to be allocated on the hotpath. Instead of using malloc, one can use the + * services of the CAAM QI memory cache (backed by kmem_cache). The buffers + * will have a size of 256B, which is sufficient for hosting 16 SG entries. + * + * @flags: flags that would be used for the equivalent malloc(..) call + * + * Returns a pointer to a retrieved buffer on success or NULL on failure. + */ +void *qi_cache_alloc(gfp_t flags); + +/** + * qi_cache_free - Frees buffers allocated from CAAM-QI cache + * + * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) no longer needs + * the buffer previously allocated by a qi_cache_alloc call. + * No checking is being done, the call is a passthrough call to + * kmem_cache_free(...) + * + * @obj: object previously allocated using qi_cache_alloc() + */ +void qi_cache_free(void *obj); + +#endif /* __QI_H__ */ --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -2,6 +2,7 @@ * CAAM hardware register-level view * * Copyright 2008-2011 Freescale Semiconductor, Inc. + * Copyright 2017 NXP */ #ifndef REGS_H @@ -67,6 +68,7 @@ */ extern bool caam_little_end; +extern bool caam_imx; #define caam_to_cpu(len) \ static inline u##len caam##len ## _to_cpu(u##len val) \ @@ -154,13 +156,10 @@ static inline u64 rd_reg64(void __iomem #else /* CONFIG_64BIT */ static inline void wr_reg64(void __iomem *reg, u64 data) { -#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX - if (caam_little_end) { + if (!caam_imx && caam_little_end) { wr_reg32((u32 __iomem *)(reg) + 1, data >> 32); wr_reg32((u32 __iomem *)(reg), data); - } else -#endif - { + } else { wr_reg32((u32 __iomem *)(reg), data >> 32); wr_reg32((u32 __iomem *)(reg) + 1, data); } @@ -168,41 +167,40 @@ static inline void wr_reg64(void __iomem static inline u64 rd_reg64(void __iomem *reg) { -#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX - if (caam_little_end) + if (!caam_imx && caam_little_end) return ((u64)rd_reg32((u32 __iomem *)(reg) + 1) << 32 | (u64)rd_reg32((u32 __iomem *)(reg))); - else -#endif - return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | - (u64)rd_reg32((u32 __iomem *)(reg) + 1)); + + return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | + (u64)rd_reg32((u32 __iomem *)(reg) + 1)); } #endif /* CONFIG_64BIT */ +static inline u64 cpu_to_caam_dma64(dma_addr_t value) +{ + if (caam_imx) + return (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | + (u64)cpu_to_caam32(upper_32_bits(value))); + + return cpu_to_caam64(value); +} + +static inline u64 caam_dma64_to_cpu(u64 value) +{ + if (caam_imx) + return (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | + (u64)caam32_to_cpu(upper_32_bits(value))); + + return caam64_to_cpu(value); +} + #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT -#ifdef CONFIG_SOC_IMX7D -#define cpu_to_caam_dma(value) \ - (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ - (u64)cpu_to_caam32(upper_32_bits(value))) -#define caam_dma_to_cpu(value) \ - (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | \ - (u64)caam32_to_cpu(upper_32_bits(value))) -#else -#define cpu_to_caam_dma(value) cpu_to_caam64(value) -#define caam_dma_to_cpu(value) caam64_to_cpu(value) -#endif /* CONFIG_SOC_IMX7D */ +#define cpu_to_caam_dma(value) cpu_to_caam_dma64(value) +#define caam_dma_to_cpu(value) caam_dma64_to_cpu(value) #else #define cpu_to_caam_dma(value) cpu_to_caam32(value) #define caam_dma_to_cpu(value) caam32_to_cpu(value) -#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ - -#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX -#define cpu_to_caam_dma64(value) \ - (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ - (u64)cpu_to_caam32(upper_32_bits(value))) -#else -#define cpu_to_caam_dma64(value) cpu_to_caam64(value) -#endif +#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ /* * jr_outentry @@ -293,6 +291,7 @@ struct caam_perfmon { u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/ #define CTPR_MS_QI_SHIFT 25 #define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) +#define CTPR_MS_DPAA2 BIT(13) #define CTPR_MS_VIRT_EN_INCL 0x00000001 #define CTPR_MS_VIRT_EN_POR 0x00000002 #define CTPR_MS_PG_SZ_MASK 0x10 @@ -628,6 +627,8 @@ struct caam_job_ring { #define JRSTA_DECOERR_INVSIGN 0x86 #define JRSTA_DECOERR_DSASIGN 0x87 +#define JRSTA_QIERR_ERROR_MASK 0x00ff + #define JRSTA_CCBERR_JUMP 0x08000000 #define JRSTA_CCBERR_INDEX_MASK 0xff00 #define JRSTA_CCBERR_INDEX_SHIFT 8 --- /dev/null +++ b/drivers/crypto/caam/sg_sw_qm.h @@ -0,0 +1,126 @@ +/* + * Copyright 2013-2016 Freescale Semiconductor, Inc. + * Copyright 2016-2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Freescale Semiconductor nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SG_SW_QM_H +#define __SG_SW_QM_H + +#include +#include "regs.h" + +static inline void cpu_to_hw_sg(struct qm_sg_entry *qm_sg_ptr) +{ + dma_addr_t addr = qm_sg_ptr->opaque; + + qm_sg_ptr->opaque = cpu_to_caam64(addr); + qm_sg_ptr->sgt_efl = cpu_to_caam32(qm_sg_ptr->sgt_efl); +} + +static inline void __dma_to_qm_sg(struct qm_sg_entry *qm_sg_ptr, dma_addr_t dma, + u32 len, u16 offset) +{ + qm_sg_ptr->addr = dma; + qm_sg_ptr->length = len; + qm_sg_ptr->__reserved2 = 0; + qm_sg_ptr->bpid = 0; + qm_sg_ptr->__reserved3 = 0; + qm_sg_ptr->offset = offset & QM_SG_OFFSET_MASK; + + cpu_to_hw_sg(qm_sg_ptr); +} + +static inline void dma_to_qm_sg_one(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + qm_sg_ptr->extension = 0; + qm_sg_ptr->final = 0; + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); +} + +static inline void dma_to_qm_sg_one_last(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + qm_sg_ptr->extension = 0; + qm_sg_ptr->final = 1; + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); +} + +static inline void dma_to_qm_sg_one_ext(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + qm_sg_ptr->extension = 1; + qm_sg_ptr->final = 0; + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); +} + +static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, + u16 offset) +{ + qm_sg_ptr->extension = 1; + qm_sg_ptr->final = 1; + __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); +} + +/* + * convert scatterlist to h/w link table format + * but does not have final bit; instead, returns last entry + */ +static inline struct qm_sg_entry * +sg_to_qm_sg(struct scatterlist *sg, int sg_count, + struct qm_sg_entry *qm_sg_ptr, u16 offset) +{ + while (sg_count && sg) { + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), + sg_dma_len(sg), offset); + qm_sg_ptr++; + sg = sg_next(sg); + sg_count--; + } + return qm_sg_ptr - 1; +} + +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, + struct qm_sg_entry *qm_sg_ptr, u16 offset) +{ + qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + + qm_sg_ptr->sgt_efl = caam32_to_cpu(qm_sg_ptr->sgt_efl); + qm_sg_ptr->final = 1; + qm_sg_ptr->sgt_efl = cpu_to_caam32(qm_sg_ptr->sgt_efl); +} + +#endif /* __SG_SW_QM_H */ --- /dev/null +++ b/drivers/crypto/caam/sg_sw_qm2.h @@ -0,0 +1,81 @@ +/* + * Copyright 2015-2016 Freescale Semiconductor, Inc. + * Copyright 2017 NXP + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the names of the above-listed copyright holders nor the + * names of any contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * + * ALTERNATIVELY, this software may be distributed under the terms of the + * GNU General Public License ("GPL") as published by the Free Software + * Foundation, either version 2 of that License or (at your option) any + * later version. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SG_SW_QM2_H_ +#define _SG_SW_QM2_H_ + +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" + +static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr, + dma_addr_t dma, u32 len, u16 offset) +{ + dpaa2_sg_set_addr(qm_sg_ptr, dma); + dpaa2_sg_set_format(qm_sg_ptr, dpaa2_sg_single); + dpaa2_sg_set_final(qm_sg_ptr, false); + dpaa2_sg_set_len(qm_sg_ptr, len); + dpaa2_sg_set_bpid(qm_sg_ptr, 0); + dpaa2_sg_set_offset(qm_sg_ptr, offset); +} + +/* + * convert scatterlist to h/w link table format + * but does not have final bit; instead, returns last entry + */ +static inline struct dpaa2_sg_entry * +sg_to_qm_sg(struct scatterlist *sg, int sg_count, + struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) +{ + while (sg_count && sg) { + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), + sg_dma_len(sg), offset); + qm_sg_ptr++; + sg = sg_next(sg); + sg_count--; + } + return qm_sg_ptr - 1; +} + +/* + * convert scatterlist to h/w link table format + * scatterlist must have been previously dma mapped + */ +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, + struct dpaa2_sg_entry *qm_sg_ptr, + u16 offset) +{ + qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + dpaa2_sg_set_final(qm_sg_ptr, true); +} + +#endif /* _SG_SW_QM2_H_ */ --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -5,9 +5,19 @@ * */ +#ifndef _SG_SW_SEC4_H_ +#define _SG_SW_SEC4_H_ + +#include "ctrl.h" #include "regs.h" +#include "sg_sw_qm2.h" +#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" -struct sec4_sg_entry; +struct sec4_sg_entry { + u64 ptr; + u32 len; + u32 bpid_offset; +}; /* * convert single dma address to h/w link table format @@ -15,9 +25,15 @@ struct sec4_sg_entry; static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, dma_addr_t dma, u32 len, u16 offset) { - sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); - sec4_sg_ptr->len = cpu_to_caam32(len); - sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); + if (caam_dpaa2) { + dma_to_qm_sg_one((struct dpaa2_sg_entry *)sec4_sg_ptr, dma, len, + offset); + } else { + sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); + sec4_sg_ptr->len = cpu_to_caam32(len); + sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & + SEC4_SG_OFFSET_MASK); + } #ifdef DEBUG print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, @@ -43,6 +59,14 @@ sg_to_sec4_sg(struct scatterlist *sg, in return sec4_sg_ptr - 1; } +static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr) +{ + if (caam_dpaa2) + dpaa2_sg_set_final((struct dpaa2_sg_entry *)sec4_sg_ptr, true); + else + sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); +} + /* * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped @@ -52,31 +76,7 @@ static inline void sg_to_sec4_sg_last(st u16 offset) { sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); - sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); -} - -static inline struct sec4_sg_entry *sg_to_sec4_sg_len( - struct scatterlist *sg, unsigned int total, - struct sec4_sg_entry *sec4_sg_ptr) -{ - do { - unsigned int len = min(sg_dma_len(sg), total); - - dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), len, 0); - sec4_sg_ptr++; - sg = sg_next(sg); - total -= len; - } while (total); - return sec4_sg_ptr - 1; + sg_to_sec4_set_last(sec4_sg_ptr); } -/* derive number of elements in scatterlist, but return 0 for 1 */ -static inline int sg_count(struct scatterlist *sg_list, int nbytes) -{ - int sg_nents = sg_nents_for_len(sg_list, nbytes); - - if (likely(sg_nents == 1)) - return 0; - - return sg_nents; -} +#endif /* _SG_SW_SEC4_H_ */ --- a/drivers/net/wireless/rsi/rsi_91x_usb.c +++ b/drivers/net/wireless/rsi/rsi_91x_usb.c @@ -516,7 +516,7 @@ err: /** * rsi_disconnect() - This function performs the reverse of the probe function, - * it deintialize the driver structure. + * it deinitialize the driver structure. * @pfunction: Pointer to the USB interface structure. * * Return: None. --- a/drivers/staging/wilc1000/linux_wlan.c +++ b/drivers/staging/wilc1000/linux_wlan.c @@ -211,7 +211,7 @@ static void deinit_irq(struct net_device vif = netdev_priv(dev); wilc = vif->wilc; - /* Deintialize IRQ */ + /* Deinitialize IRQ */ if (wilc->dev_irq_num) { free_irq(wilc->dev_irq_num, wilc); gpio_free(wilc->gpio); --- a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c +++ b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c @@ -2359,7 +2359,7 @@ int wilc_deinit_host_int(struct net_devi del_timer_sync(&wilc_during_ip_timer); if (s32Error) - netdev_err(net, "Error while deintializing host interface\n"); + netdev_err(net, "Error while deinitializing host interface\n"); return s32Error; } --- /dev/null +++ b/include/crypto/acompress.h @@ -0,0 +1,269 @@ +/* + * Asynchronous Compression operations + * + * Copyright (c) 2016, Intel Corporation + * Authors: Weigang Li + * Giovanni Cabiddu + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#ifndef _CRYPTO_ACOMP_H +#define _CRYPTO_ACOMP_H +#include + +#define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 + +/** + * struct acomp_req - asynchronous (de)compression request + * + * @base: Common attributes for asynchronous crypto requests + * @src: Source Data + * @dst: Destination data + * @slen: Size of the input buffer + * @dlen: Size of the output buffer and number of bytes produced + * @flags: Internal flags + * @__ctx: Start of private context data + */ +struct acomp_req { + struct crypto_async_request base; + struct scatterlist *src; + struct scatterlist *dst; + unsigned int slen; + unsigned int dlen; + u32 flags; + void *__ctx[] CRYPTO_MINALIGN_ATTR; +}; + +/** + * struct crypto_acomp - user-instantiated objects which encapsulate + * algorithms and core processing logic + * + * @compress: Function performs a compress operation + * @decompress: Function performs a de-compress operation + * @dst_free: Frees destination buffer if allocated inside the + * algorithm + * @reqsize: Context size for (de)compression requests + * @base: Common crypto API algorithm data structure + */ +struct crypto_acomp { + int (*compress)(struct acomp_req *req); + int (*decompress)(struct acomp_req *req); + void (*dst_free)(struct scatterlist *dst); + unsigned int reqsize; + struct crypto_tfm base; +}; + +/** + * struct acomp_alg - asynchronous compression algorithm + * + * @compress: Function performs a compress operation + * @decompress: Function performs a de-compress operation + * @dst_free: Frees destination buffer if allocated inside the algorithm + * @init: Initialize the cryptographic transformation object. + * This function is used to initialize the cryptographic + * transformation object. This function is called only once at + * the instantiation time, right after the transformation context + * was allocated. In case the cryptographic hardware has some + * special requirements which need to be handled by software, this + * function shall check for the precise requirement of the + * transformation and put any software fallbacks in place. + * @exit: Deinitialize the cryptographic transformation object. This is a + * counterpart to @init, used to remove various changes set in + * @init. + * + * @reqsize: Context size for (de)compression requests + * @base: Common crypto API algorithm data structure + */ +struct acomp_alg { + int (*compress)(struct acomp_req *req); + int (*decompress)(struct acomp_req *req); + void (*dst_free)(struct scatterlist *dst); + int (*init)(struct crypto_acomp *tfm); + void (*exit)(struct crypto_acomp *tfm); + unsigned int reqsize; + struct crypto_alg base; +}; + +/** + * DOC: Asynchronous Compression API + * + * The Asynchronous Compression API is used with the algorithms of type + * CRYPTO_ALG_TYPE_ACOMPRESS (listed as type "acomp" in /proc/crypto) + */ + +/** + * crypto_alloc_acomp() -- allocate ACOMPRESS tfm handle + * @alg_name: is the cra_name / name or cra_driver_name / driver name of the + * compression algorithm e.g. "deflate" + * @type: specifies the type of the algorithm + * @mask: specifies the mask for the algorithm + * + * Allocate a handle for a compression algorithm. The returned struct + * crypto_acomp is the handle that is required for any subsequent + * API invocation for the compression operations. + * + * Return: allocated handle in case of success; IS_ERR() is true in case + * of an error, PTR_ERR() returns the error code. + */ +struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type, + u32 mask); + +static inline struct crypto_tfm *crypto_acomp_tfm(struct crypto_acomp *tfm) +{ + return &tfm->base; +} + +static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg) +{ + return container_of(alg, struct acomp_alg, base); +} + +static inline struct crypto_acomp *__crypto_acomp_tfm(struct crypto_tfm *tfm) +{ + return container_of(tfm, struct crypto_acomp, base); +} + +static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm) +{ + return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg); +} + +static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm) +{ + return tfm->reqsize; +} + +static inline void acomp_request_set_tfm(struct acomp_req *req, + struct crypto_acomp *tfm) +{ + req->base.tfm = crypto_acomp_tfm(tfm); +} + +static inline struct crypto_acomp *crypto_acomp_reqtfm(struct acomp_req *req) +{ + return __crypto_acomp_tfm(req->base.tfm); +} + +/** + * crypto_free_acomp() -- free ACOMPRESS tfm handle + * + * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() + */ +static inline void crypto_free_acomp(struct crypto_acomp *tfm) +{ + crypto_destroy_tfm(tfm, crypto_acomp_tfm(tfm)); +} + +static inline int crypto_has_acomp(const char *alg_name, u32 type, u32 mask) +{ + type &= ~CRYPTO_ALG_TYPE_MASK; + type |= CRYPTO_ALG_TYPE_ACOMPRESS; + mask |= CRYPTO_ALG_TYPE_MASK; + + return crypto_has_alg(alg_name, type, mask); +} + +/** + * acomp_request_alloc() -- allocates asynchronous (de)compression request + * + * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() + * + * Return: allocated handle in case of success or NULL in case of an error + */ +struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); + +/** + * acomp_request_free() -- zeroize and free asynchronous (de)compression + * request as well as the output buffer if allocated + * inside the algorithm + * + * @req: request to free + */ +void acomp_request_free(struct acomp_req *req); + +/** + * acomp_request_set_callback() -- Sets an asynchronous callback + * + * Callback will be called when an asynchronous operation on a given + * request is finished. + * + * @req: request that the callback will be set for + * @flgs: specify for instance if the operation may backlog + * @cmlp: callback which will be called + * @data: private data used by the caller + */ +static inline void acomp_request_set_callback(struct acomp_req *req, + u32 flgs, + crypto_completion_t cmpl, + void *data) +{ + req->base.complete = cmpl; + req->base.data = data; + req->base.flags = flgs; +} + +/** + * acomp_request_set_params() -- Sets request parameters + * + * Sets parameters required by an acomp operation + * + * @req: asynchronous compress request + * @src: pointer to input buffer scatterlist + * @dst: pointer to output buffer scatterlist. If this is NULL, the + * acomp layer will allocate the output memory + * @slen: size of the input buffer + * @dlen: size of the output buffer. If dst is NULL, this can be used by + * the user to specify the maximum amount of memory to allocate + */ +static inline void acomp_request_set_params(struct acomp_req *req, + struct scatterlist *src, + struct scatterlist *dst, + unsigned int slen, + unsigned int dlen) +{ + req->src = src; + req->dst = dst; + req->slen = slen; + req->dlen = dlen; + + if (!req->dst) + req->flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; +} + +/** + * crypto_acomp_compress() -- Invoke asynchronous compress operation + * + * Function invokes the asynchronous compress operation + * + * @req: asynchronous compress request + * + * Return: zero on success; error code in case of error + */ +static inline int crypto_acomp_compress(struct acomp_req *req) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + return tfm->compress(req); +} + +/** + * crypto_acomp_decompress() -- Invoke asynchronous decompress operation + * + * Function invokes the asynchronous decompress operation + * + * @req: asynchronous compress request + * + * Return: zero on success; error code in case of error + */ +static inline int crypto_acomp_decompress(struct acomp_req *req) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + return tfm->decompress(req); +} + +#endif --- /dev/null +++ b/include/crypto/internal/acompress.h @@ -0,0 +1,81 @@ +/* + * Asynchronous Compression operations + * + * Copyright (c) 2016, Intel Corporation + * Authors: Weigang Li + * Giovanni Cabiddu + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#ifndef _CRYPTO_ACOMP_INT_H +#define _CRYPTO_ACOMP_INT_H +#include + +/* + * Transform internal helpers. + */ +static inline void *acomp_request_ctx(struct acomp_req *req) +{ + return req->__ctx; +} + +static inline void *acomp_tfm_ctx(struct crypto_acomp *tfm) +{ + return tfm->base.__crt_ctx; +} + +static inline void acomp_request_complete(struct acomp_req *req, + int err) +{ + req->base.complete(&req->base, err); +} + +static inline const char *acomp_alg_name(struct crypto_acomp *tfm) +{ + return crypto_acomp_tfm(tfm)->__crt_alg->cra_name; +} + +static inline struct acomp_req *__acomp_request_alloc(struct crypto_acomp *tfm) +{ + struct acomp_req *req; + + req = kzalloc(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); + if (likely(req)) + acomp_request_set_tfm(req, tfm); + return req; +} + +static inline void __acomp_request_free(struct acomp_req *req) +{ + kzfree(req); +} + +/** + * crypto_register_acomp() -- Register asynchronous compression algorithm + * + * Function registers an implementation of an asynchronous + * compression algorithm + * + * @alg: algorithm definition + * + * Return: zero on success; error code in case of error + */ +int crypto_register_acomp(struct acomp_alg *alg); + +/** + * crypto_unregister_acomp() -- Unregister asynchronous compression algorithm + * + * Function unregisters an implementation of an asynchronous + * compression algorithm + * + * @alg: algorithm definition + * + * Return: zero on success; error code in case of error + */ +int crypto_unregister_acomp(struct acomp_alg *alg); + +#endif --- /dev/null +++ b/include/crypto/internal/scompress.h @@ -0,0 +1,136 @@ +/* + * Synchronous Compression operations + * + * Copyright 2015 LG Electronics Inc. + * Copyright (c) 2016, Intel Corporation + * Author: Giovanni Cabiddu + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + */ +#ifndef _CRYPTO_SCOMP_INT_H +#define _CRYPTO_SCOMP_INT_H +#include + +#define SCOMP_SCRATCH_SIZE 131072 + +struct crypto_scomp { + struct crypto_tfm base; +}; + +/** + * struct scomp_alg - synchronous compression algorithm + * + * @alloc_ctx: Function allocates algorithm specific context + * @free_ctx: Function frees context allocated with alloc_ctx + * @compress: Function performs a compress operation + * @decompress: Function performs a de-compress operation + * @init: Initialize the cryptographic transformation object. + * This function is used to initialize the cryptographic + * transformation object. This function is called only once at + * the instantiation time, right after the transformation context + * was allocated. In case the cryptographic hardware has some + * special requirements which need to be handled by software, this + * function shall check for the precise requirement of the + * transformation and put any software fallbacks in place. + * @exit: Deinitialize the cryptographic transformation object. This is a + * counterpart to @init, used to remove various changes set in + * @init. + * @base: Common crypto API algorithm data structure + */ +struct scomp_alg { + void *(*alloc_ctx)(struct crypto_scomp *tfm); + void (*free_ctx)(struct crypto_scomp *tfm, void *ctx); + int (*compress)(struct crypto_scomp *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int *dlen, + void *ctx); + int (*decompress)(struct crypto_scomp *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int *dlen, + void *ctx); + struct crypto_alg base; +}; + +static inline struct scomp_alg *__crypto_scomp_alg(struct crypto_alg *alg) +{ + return container_of(alg, struct scomp_alg, base); +} + +static inline struct crypto_scomp *__crypto_scomp_tfm(struct crypto_tfm *tfm) +{ + return container_of(tfm, struct crypto_scomp, base); +} + +static inline struct crypto_tfm *crypto_scomp_tfm(struct crypto_scomp *tfm) +{ + return &tfm->base; +} + +static inline void crypto_free_scomp(struct crypto_scomp *tfm) +{ + crypto_destroy_tfm(tfm, crypto_scomp_tfm(tfm)); +} + +static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm) +{ + return __crypto_scomp_alg(crypto_scomp_tfm(tfm)->__crt_alg); +} + +static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm) +{ + return crypto_scomp_alg(tfm)->alloc_ctx(tfm); +} + +static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm, + void *ctx) +{ + return crypto_scomp_alg(tfm)->free_ctx(tfm, ctx); +} + +static inline int crypto_scomp_compress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, void *ctx) +{ + return crypto_scomp_alg(tfm)->compress(tfm, src, slen, dst, dlen, ctx); +} + +static inline int crypto_scomp_decompress(struct crypto_scomp *tfm, + const u8 *src, unsigned int slen, + u8 *dst, unsigned int *dlen, + void *ctx) +{ + return crypto_scomp_alg(tfm)->decompress(tfm, src, slen, dst, dlen, + ctx); +} + +int crypto_init_scomp_ops_async(struct crypto_tfm *tfm); +struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req); +void crypto_acomp_scomp_free_ctx(struct acomp_req *req); + +/** + * crypto_register_scomp() -- Register synchronous compression algorithm + * + * Function registers an implementation of a synchronous + * compression algorithm + * + * @alg: algorithm definition + * + * Return: zero on success; error code in case of error + */ +int crypto_register_scomp(struct scomp_alg *alg); + +/** + * crypto_unregister_scomp() -- Unregister synchronous compression algorithm + * + * Function unregisters an implementation of a synchronous + * compression algorithm + * + * @alg: algorithm definition + * + * Return: zero on success; error code in case of error + */ +int crypto_unregister_scomp(struct scomp_alg *alg); + +#endif --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -50,6 +50,8 @@ #define CRYPTO_ALG_TYPE_SKCIPHER 0x00000005 #define CRYPTO_ALG_TYPE_GIVCIPHER 0x00000006 #define CRYPTO_ALG_TYPE_KPP 0x00000008 +#define CRYPTO_ALG_TYPE_ACOMPRESS 0x0000000a +#define CRYPTO_ALG_TYPE_SCOMPRESS 0x0000000b #define CRYPTO_ALG_TYPE_RNG 0x0000000c #define CRYPTO_ALG_TYPE_AKCIPHER 0x0000000d #define CRYPTO_ALG_TYPE_DIGEST 0x0000000e @@ -60,6 +62,7 @@ #define CRYPTO_ALG_TYPE_HASH_MASK 0x0000000e #define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e #define CRYPTO_ALG_TYPE_BLKCIPHER_MASK 0x0000000c +#define CRYPTO_ALG_TYPE_ACOMPRESS_MASK 0x0000000e #define CRYPTO_ALG_LARVAL 0x00000010 #define CRYPTO_ALG_DEAD 0x00000020 --- a/include/uapi/linux/cryptouser.h +++ b/include/uapi/linux/cryptouser.h @@ -46,6 +46,7 @@ enum crypto_attr_type_t { CRYPTOCFGA_REPORT_CIPHER, /* struct crypto_report_cipher */ CRYPTOCFGA_REPORT_AKCIPHER, /* struct crypto_report_akcipher */ CRYPTOCFGA_REPORT_KPP, /* struct crypto_report_kpp */ + CRYPTOCFGA_REPORT_ACOMP, /* struct crypto_report_acomp */ __CRYPTOCFGA_MAX #define CRYPTOCFGA_MAX (__CRYPTOCFGA_MAX - 1) @@ -112,5 +113,9 @@ struct crypto_report_kpp { char type[CRYPTO_MAX_NAME]; }; +struct crypto_report_acomp { + char type[CRYPTO_MAX_NAME]; +}; + #define CRYPTO_REPORT_MAXSIZE (sizeof(struct crypto_user_alg) + \ sizeof(struct crypto_report_blkcipher)) --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -305,6 +305,9 @@ defintion||definition defintions||definitions defualt||default defult||default +deintializing||deinitializing +deintialize||deinitialize +deintialized||deinitialized deivce||device delared||declared delare||declare --- a/sound/soc/amd/acp-pcm-dma.c +++ b/sound/soc/amd/acp-pcm-dma.c @@ -506,7 +506,7 @@ static int acp_init(void __iomem *acp_mm return 0; } -/* Deintialize ACP */ +/* Deinitialize ACP */ static int acp_deinit(void __iomem *acp_mmio) { u32 val;