From 1b44c5a60c137e5fd0c2c8b86e58fdbc9cd181ce Mon Sep 17 00:00:00 2001 From: Antoine Ténart Date: Wed, 24 May 2017 16:10:34 +0200 Subject: crypto: inside-secure - add SafeXcel EIP197 crypto engine driver Add support for Inside Secure SafeXcel EIP197 cryptographic engine, which can be found on Marvell Armada 7k and 8k boards. This driver currently implements: ecb(aes), cbc(aes), sha1, sha224, sha256 and hmac(sah1) algorithms. Two firmwares are needed for this engine to work. Their are mostly used for more advanced operations than the ones supported (as of now), but we still need them to pass the data to the internal cryptographic engine. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 556 +++++++++++++++++++++++++ 1 file changed, 556 insertions(+) create mode 100644 drivers/crypto/inside-secure/safexcel_cipher.c (limited to 'drivers/crypto/inside-secure/safexcel_cipher.c') diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c new file mode 100644 index 000000000000..59e6081602a2 --- /dev/null +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -0,0 +1,556 @@ +/* + * Copyright (C) 2017 Marvell + * + * Antoine Tenart + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#include +#include +#include + +#include +#include + +#include "safexcel.h" + +enum safexcel_cipher_direction { + SAFEXCEL_ENCRYPT, + SAFEXCEL_DECRYPT, +}; + +struct safexcel_cipher_ctx { + struct safexcel_context base; + struct safexcel_crypto_priv *priv; + + enum safexcel_cipher_direction direction; + u32 mode; + + __le32 key[8]; + unsigned int key_len; +}; + +static void safexcel_cipher_token(struct safexcel_cipher_ctx *ctx, + struct crypto_async_request *async, + struct safexcel_command_desc *cdesc, + u32 length) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_token *token; + unsigned offset = 0; + + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { + offset = AES_BLOCK_SIZE / sizeof(u32); + memcpy(cdesc->control_data.token, req->iv, AES_BLOCK_SIZE); + + cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; + } + + token = (struct safexcel_token *)(cdesc->control_data.token + offset); + + token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION; + token[0].packet_length = length; + token[0].stat = EIP197_TOKEN_STAT_LAST_PACKET; + token[0].instructions = EIP197_TOKEN_INS_LAST | + EIP197_TOKEN_INS_TYPE_CRYTO | + EIP197_TOKEN_INS_TYPE_OUTPUT; +} + +static int safexcel_aes_setkey(struct crypto_skcipher *ctfm, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(ctfm); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aes_ctx aes; + int ret, i; + + ret = crypto_aes_expand_key(&aes, key, len); + if (ret) { + crypto_skcipher_set_flags(ctfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return ret; + } + + for (i = 0; i < len / sizeof(u32); i++) { + if (ctx->key[i] != cpu_to_le32(aes.key_enc[i])) { + ctx->base.needs_inv = true; + break; + } + } + + for (i = 0; i < len / sizeof(u32); i++) + ctx->key[i] = cpu_to_le32(aes.key_enc[i]); + + ctx->key_len = len; + + memzero_explicit(&aes, sizeof(aes)); + return 0; +} + +static int safexcel_context_control(struct safexcel_cipher_ctx *ctx, + struct safexcel_command_desc *cdesc) +{ + struct safexcel_crypto_priv *priv = ctx->priv; + int ctrl_size; + + if (ctx->direction == SAFEXCEL_ENCRYPT) + cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_OUT; + else + cdesc->control_data.control0 |= CONTEXT_CONTROL_TYPE_CRYPTO_IN; + + cdesc->control_data.control0 |= CONTEXT_CONTROL_KEY_EN; + cdesc->control_data.control1 |= ctx->mode; + + switch (ctx->key_len) { + case AES_KEYSIZE_128: + cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES128; + ctrl_size = 4; + break; + case AES_KEYSIZE_192: + cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES192; + ctrl_size = 6; + break; + case AES_KEYSIZE_256: + cdesc->control_data.control0 |= CONTEXT_CONTROL_CRYPTO_ALG_AES256; + ctrl_size = 8; + break; + default: + dev_err(priv->dev, "aes keysize not supported: %u\n", + ctx->key_len); + return -EINVAL; + } + cdesc->control_data.control0 |= CONTEXT_CONTROL_SIZE(ctrl_size); + + return 0; +} + +static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_result_desc *rdesc; + int ndesc = 0; + + *ret = 0; + + spin_lock_bh(&priv->ring[ring].egress_lock); + do { + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "cipher: result: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + break; + } + + if (rdesc->result_data.error_code) { + dev_err(priv->dev, + "cipher: result: result descriptor error (%d)\n", + rdesc->result_data.error_code); + *ret = -EIO; + } + + ndesc++; + } while (!rdesc->last_seg); + + safexcel_complete(priv, ring); + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (req->src == req->dst) { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, req->dst, + sg_nents_for_len(req->dst, req->cryptlen), + DMA_FROM_DEVICE); + } + + *should_complete = true; + + return ndesc; +} + +static int safexcel_aes_send(struct crypto_async_request *async, + int ring, struct safexcel_request *request, + int *commands, int *results) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + struct safexcel_command_desc *cdesc; + struct safexcel_result_desc *rdesc; + struct scatterlist *sg; + int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = req->cryptlen; + int i, ret = 0; + + request->req = &req->base; + + if (req->src == req->dst) { + nr_src = dma_map_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_BIDIRECTIONAL); + nr_dst = nr_src; + if (!nr_src) + return -EINVAL; + } else { + nr_src = dma_map_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + if (!nr_src) + return -EINVAL; + + nr_dst = dma_map_sg(priv->dev, req->dst, + sg_nents_for_len(req->dst, req->cryptlen), + DMA_FROM_DEVICE); + if (!nr_dst) { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + return -EINVAL; + } + } + + memcpy(ctx->base.ctxr->data, ctx->key, ctx->key_len); + + spin_lock_bh(&priv->ring[ring].egress_lock); + + /* command descriptors */ + for_each_sg(req->src, sg, nr_src, i) { + int len = sg_dma_len(sg); + + /* Do not overflow the request */ + if (queued - len < 0) + len = queued; + + cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc, !(queued - len), + sg_dma_address(sg), len, req->cryptlen, + ctx->base.ctxr_dma); + if (IS_ERR(cdesc)) { + /* No space left in the command descriptor ring */ + ret = PTR_ERR(cdesc); + goto cdesc_rollback; + } + n_cdesc++; + + if (n_cdesc == 1) { + safexcel_context_control(ctx, cdesc); + safexcel_cipher_token(ctx, async, cdesc, req->cryptlen); + } + + queued -= len; + if (!queued) + break; + } + + /* result descriptors */ + for_each_sg(req->dst, sg, nr_dst, i) { + bool first = !i, last = (i == nr_dst - 1); + u32 len = sg_dma_len(sg); + + rdesc = safexcel_add_rdesc(priv, ring, first, last, + sg_dma_address(sg), len); + if (IS_ERR(rdesc)) { + /* No space left in the result descriptor ring */ + ret = PTR_ERR(rdesc); + goto rdesc_rollback; + } + n_rdesc++; + } + + ctx->base.handle_result = safexcel_handle_result; + + spin_unlock_bh(&priv->ring[ring].egress_lock); + + *commands = n_cdesc; + *results = nr_dst; + return 0; + +rdesc_rollback: + for (i = 0; i < n_rdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].rdr); +cdesc_rollback: + for (i = 0; i < n_cdesc; i++) + safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); + + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (req->src == req->dst) { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(priv->dev, req->src, + sg_nents_for_len(req->src, req->cryptlen), + DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, req->dst, + sg_nents_for_len(req->dst, req->cryptlen), + DMA_FROM_DEVICE); + } + + return ret; +} + +static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, + int ring, + struct crypto_async_request *async, + bool *should_complete, int *ret) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_result_desc *rdesc; + int ndesc = 0, enq_ret; + + *ret = 0; + + spin_lock_bh(&priv->ring[ring].egress_lock); + do { + rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); + if (IS_ERR(rdesc)) { + dev_err(priv->dev, + "cipher: invalidate: could not retrieve the result descriptor\n"); + *ret = PTR_ERR(rdesc); + break; + } + + if (rdesc->result_data.error_code) { + dev_err(priv->dev, "cipher: invalidate: result descriptor error (%d)\n", + rdesc->result_data.error_code); + *ret = -EIO; + } + + ndesc++; + } while (!rdesc->last_seg); + + safexcel_complete(priv, ring); + spin_unlock_bh(&priv->ring[ring].egress_lock); + + if (ctx->base.exit_inv) { + dma_pool_free(priv->context_pool, ctx->base.ctxr, + ctx->base.ctxr_dma); + + *should_complete = true; + + return ndesc; + } + + ctx->base.needs_inv = false; + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.send = safexcel_aes_send; + + spin_lock_bh(&priv->lock); + enq_ret = crypto_enqueue_request(&priv->queue, async); + spin_unlock_bh(&priv->lock); + + if (enq_ret != -EINPROGRESS) + *ret = enq_ret; + + priv->need_dequeue = true; + *should_complete = false; + + return ndesc; +} + +static int safexcel_cipher_send_inv(struct crypto_async_request *async, + int ring, struct safexcel_request *request, + int *commands, int *results) +{ + struct skcipher_request *req = skcipher_request_cast(async); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + ctx->base.handle_result = safexcel_handle_inv_result; + + ret = safexcel_invalidate_cache(async, &ctx->base, priv, + ctx->base.ctxr_dma, ring, request); + if (unlikely(ret)) + return ret; + + *commands = 1; + *results = 1; + + return 0; +} + +static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + struct skcipher_request req; + struct safexcel_inv_result result = { 0 }; + + memset(&req, 0, sizeof(struct skcipher_request)); + + /* create invalidation request */ + init_completion(&result.completion); + skcipher_request_set_callback(&req, CRYPTO_TFM_REQ_MAY_BACKLOG, + safexcel_inv_complete, &result); + + skcipher_request_set_tfm(&req, __crypto_skcipher_cast(tfm)); + ctx = crypto_tfm_ctx(req.base.tfm); + ctx->base.exit_inv = true; + ctx->base.send = safexcel_cipher_send_inv; + + spin_lock_bh(&priv->lock); + crypto_enqueue_request(&priv->queue, &req.base); + spin_unlock_bh(&priv->lock); + + if (!priv->need_dequeue) + safexcel_dequeue(priv); + + wait_for_completion_interruptible(&result.completion); + + if (result.error) { + dev_warn(priv->dev, + "cipher: sync: invalidate: completion error %d\n", + result.error); + return result.error; + } + + return 0; +} + +static int safexcel_aes(struct skcipher_request *req, + enum safexcel_cipher_direction dir, u32 mode) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + ctx->direction = dir; + ctx->mode = mode; + + if (ctx->base.ctxr) { + if (ctx->base.needs_inv) + ctx->base.send = safexcel_cipher_send_inv; + } else { + ctx->base.ring = safexcel_select_ring(priv); + ctx->base.send = safexcel_aes_send; + + ctx->base.ctxr = dma_pool_zalloc(priv->context_pool, + EIP197_GFP_FLAGS(req->base), + &ctx->base.ctxr_dma); + if (!ctx->base.ctxr) + return -ENOMEM; + } + + spin_lock_bh(&priv->lock); + ret = crypto_enqueue_request(&priv->queue, &req->base); + spin_unlock_bh(&priv->lock); + + if (!priv->need_dequeue) + safexcel_dequeue(priv); + + return ret; +} + +static int safexcel_ecb_aes_encrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_ENCRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_ECB); +} + +static int safexcel_ecb_aes_decrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_DECRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_ECB); +} + +static int safexcel_skcipher_cra_init(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_alg_template *tmpl = + container_of(tfm->__crt_alg, struct safexcel_alg_template, + alg.skcipher.base); + + ctx->priv = tmpl->priv; + + return 0; +} + +static void safexcel_skcipher_cra_exit(struct crypto_tfm *tfm) +{ + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct safexcel_crypto_priv *priv = ctx->priv; + int ret; + + memzero_explicit(ctx->key, 8 * sizeof(u32)); + + /* context not allocated, skip invalidation */ + if (!ctx->base.ctxr) + return; + + memzero_explicit(ctx->base.ctxr->data, 8 * sizeof(u32)); + + ret = safexcel_cipher_exit_inv(tfm); + if (ret) + dev_warn(priv->dev, "cipher: invalidation error %d\n", ret); +} + +struct safexcel_alg_template safexcel_alg_ecb_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .setkey = safexcel_aes_setkey, + .encrypt = safexcel_ecb_aes_encrypt, + .decrypt = safexcel_ecb_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "safexcel-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; + +static int safexcel_cbc_aes_encrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_ENCRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_CBC); +} + +static int safexcel_cbc_aes_decrypt(struct skcipher_request *req) +{ + return safexcel_aes(req, SAFEXCEL_DECRYPT, + CONTEXT_CONTROL_CRYPTO_MODE_CBC); +} + +struct safexcel_alg_template safexcel_alg_cbc_aes = { + .type = SAFEXCEL_ALG_TYPE_SKCIPHER, + .alg.skcipher = { + .setkey = safexcel_aes_setkey, + .encrypt = safexcel_cbc_aes_encrypt, + .decrypt = safexcel_cbc_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "safexcel-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct safexcel_cipher_ctx), + .cra_alignmask = 0, + .cra_init = safexcel_skcipher_cra_init, + .cra_exit = safexcel_skcipher_cra_exit, + .cra_module = THIS_MODULE, + }, + }, +}; -- cgit v1.2.3-70-g09d2 From 152bdf4cdb9c20e1858bc1b754c7bfb984a6ea14 Mon Sep 17 00:00:00 2001 From: Ofer Heifetz Date: Thu, 15 Jun 2017 09:56:22 +0200 Subject: crypto: inside-secure - align the cipher and hash send functions The cipher and hash send functions both set the results variable, to be used later by the caller. Align the way they're setting this variable. Signed-off-by: Ofer Heifetz Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/inside-secure/safexcel_cipher.c') diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 59e6081602a2..8eea4d30db31 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -269,7 +269,7 @@ static int safexcel_aes_send(struct crypto_async_request *async, spin_unlock_bh(&priv->ring[ring].egress_lock); *commands = n_cdesc; - *results = nr_dst; + *results = n_rdesc; return 0; rdesc_rollback: -- cgit v1.2.3-70-g09d2 From 9785843424c803cbe9515c477a6b89181914ea09 Mon Sep 17 00:00:00 2001 From: Antoine Ténart Date: Thu, 15 Jun 2017 09:56:23 +0200 Subject: crypto: inside-secure - update the context and request later This move the context and request updates at the end of the cipher and hash send() functions. This way the context and request fields are set only when everything else was successful in the send() functions. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel_cipher.c | 7 +++---- drivers/crypto/inside-secure/safexcel_hash.c | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel_cipher.c') diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 8eea4d30db31..6037cdfc1f16 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -190,8 +190,6 @@ static int safexcel_aes_send(struct crypto_async_request *async, int nr_src, nr_dst, n_cdesc = 0, n_rdesc = 0, queued = req->cryptlen; int i, ret = 0; - request->req = &req->base; - if (req->src == req->dst) { nr_src = dma_map_sg(priv->dev, req->src, sg_nents_for_len(req->src, req->cryptlen), @@ -264,10 +262,11 @@ static int safexcel_aes_send(struct crypto_async_request *async, n_rdesc++; } - ctx->base.handle_result = safexcel_handle_result; - spin_unlock_bh(&priv->ring[ring].egress_lock); + request->req = &req->base; + ctx->base.handle_result = safexcel_handle_result; + *commands = n_cdesc; *results = n_rdesc; return 0; diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 6eee1a502225..4e526372464f 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -198,9 +198,6 @@ static int safexcel_ahash_send(struct crypto_async_request *async, int ring, len -= extra; } - request->req = &areq->base; - ctx->base.handle_result = safexcel_handle_result; - spin_lock_bh(&priv->ring[ring].egress_lock); /* Add a command descriptor for the cached data, if any */ @@ -291,9 +288,12 @@ send_command: goto cdesc_rollback; } - req->processed += len; spin_unlock_bh(&priv->ring[ring].egress_lock); + req->processed += len; + request->req = &areq->base; + ctx->base.handle_result = safexcel_handle_result; + *commands = n_cdesc; *results = 1; return 0; -- cgit v1.2.3-70-g09d2 From 86671abbbbfc959c4e4cbd2c618d5868a6f8dcf2 Mon Sep 17 00:00:00 2001 From: Antoine Ténart Date: Thu, 15 Jun 2017 09:56:24 +0200 Subject: crypto: inside-secure - use one queue per hw ring Update the inside-secure safexcel driver from using one global queue to one queue per hw ring. This ease the request management and keep the hw in sync with what's done in sw. Signed-off-by: Antoine Tenart Signed-off-by: Herbert Xu --- drivers/crypto/inside-secure/safexcel.c | 86 ++++++++++++-------------- drivers/crypto/inside-secure/safexcel.h | 12 ++-- drivers/crypto/inside-secure/safexcel_cipher.c | 38 +++++++----- drivers/crypto/inside-secure/safexcel_hash.c | 38 +++++++----- 4 files changed, 89 insertions(+), 85 deletions(-) (limited to 'drivers/crypto/inside-secure/safexcel_cipher.c') diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 73f4ef8d71f3..8956b23803a8 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -422,20 +422,18 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) return 0; } -void safexcel_dequeue(struct safexcel_crypto_priv *priv) +void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring) { struct crypto_async_request *req, *backlog; struct safexcel_context *ctx; struct safexcel_request *request; - int i, ret, n = 0, nreq[EIP197_MAX_RINGS] = {0}; - int cdesc[EIP197_MAX_RINGS] = {0}, rdesc[EIP197_MAX_RINGS] = {0}; - int commands, results; + int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results; do { - spin_lock_bh(&priv->lock); - req = crypto_dequeue_request(&priv->queue); - backlog = crypto_get_backlog(&priv->queue); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + req = crypto_dequeue_request(&priv->ring[ring].queue); + backlog = crypto_get_backlog(&priv->ring[ring].queue); + spin_unlock_bh(&priv->ring[ring].queue_lock); if (!req) goto finalize; @@ -445,58 +443,51 @@ void safexcel_dequeue(struct safexcel_crypto_priv *priv) goto requeue; ctx = crypto_tfm_ctx(req->tfm); - ret = ctx->send(req, ctx->ring, request, &commands, &results); + ret = ctx->send(req, ring, request, &commands, &results); if (ret) { kfree(request); requeue: - spin_lock_bh(&priv->lock); - crypto_enqueue_request(&priv->queue, req); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, req); + spin_unlock_bh(&priv->ring[ring].queue_lock); - priv->need_dequeue = true; + priv->ring[ring].need_dequeue = true; continue; } if (backlog) backlog->complete(backlog, -EINPROGRESS); - spin_lock_bh(&priv->ring[ctx->ring].egress_lock); - list_add_tail(&request->list, &priv->ring[ctx->ring].list); - spin_unlock_bh(&priv->ring[ctx->ring].egress_lock); - - cdesc[ctx->ring] += commands; - rdesc[ctx->ring] += results; + spin_lock_bh(&priv->ring[ring].egress_lock); + list_add_tail(&request->list, &priv->ring[ring].list); + spin_unlock_bh(&priv->ring[ring].egress_lock); - nreq[ctx->ring]++; - } while (n++ < EIP197_MAX_BATCH_SZ); + cdesc += commands; + rdesc += results; + } while (nreq++ < EIP197_MAX_BATCH_SZ); finalize: - if (n == EIP197_MAX_BATCH_SZ) - priv->need_dequeue = true; - else if (!n) + if (nreq == EIP197_MAX_BATCH_SZ) + priv->ring[ring].need_dequeue = true; + else if (!nreq) return; - for (i = 0; i < priv->config.rings; i++) { - if (!nreq[i]) - continue; + spin_lock_bh(&priv->ring[ring].lock); - spin_lock_bh(&priv->ring[i].lock); + /* Configure when we want an interrupt */ + writel(EIP197_HIA_RDR_THRESH_PKT_MODE | + EIP197_HIA_RDR_THRESH_PROC_PKT(nreq), + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_THRESH); - /* Configure when we want an interrupt */ - writel(EIP197_HIA_RDR_THRESH_PKT_MODE | - EIP197_HIA_RDR_THRESH_PROC_PKT(nreq[i]), - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_THRESH); + /* let the RDR know we have pending descriptors */ + writel((rdesc * priv->config.rd_offset) << 2, + priv->base + EIP197_HIA_RDR(ring) + EIP197_HIA_xDR_PREP_COUNT); - /* let the RDR know we have pending descriptors */ - writel((rdesc[i] * priv->config.rd_offset) << 2, - priv->base + EIP197_HIA_RDR(i) + EIP197_HIA_xDR_PREP_COUNT); + /* let the CDR know we have pending descriptors */ + writel((cdesc * priv->config.cd_offset) << 2, + priv->base + EIP197_HIA_CDR(ring) + EIP197_HIA_xDR_PREP_COUNT); - /* let the CDR know we have pending descriptors */ - writel((cdesc[i] * priv->config.cd_offset) << 2, - priv->base + EIP197_HIA_CDR(i) + EIP197_HIA_xDR_PREP_COUNT); - - spin_unlock_bh(&priv->ring[i].lock); - } + spin_unlock_bh(&priv->ring[ring].lock); } void safexcel_free_context(struct safexcel_crypto_priv *priv, @@ -638,9 +629,9 @@ static void safexcel_handle_result_work(struct work_struct *work) safexcel_handle_result_descriptor(priv, data->ring); - if (priv->need_dequeue) { - priv->need_dequeue = false; - safexcel_dequeue(data->priv); + if (priv->ring[data->ring].need_dequeue) { + priv->ring[data->ring].need_dequeue = false; + safexcel_dequeue(data->priv, data->ring); } } @@ -864,17 +855,18 @@ static int safexcel_probe(struct platform_device *pdev) goto err_clk; } + crypto_init_queue(&priv->ring[i].queue, + EIP197_DEFAULT_RING_SIZE); + INIT_LIST_HEAD(&priv->ring[i].list); spin_lock_init(&priv->ring[i].lock); spin_lock_init(&priv->ring[i].egress_lock); + spin_lock_init(&priv->ring[i].queue_lock); } platform_set_drvdata(pdev, priv); atomic_set(&priv->ring_used, 0); - spin_lock_init(&priv->lock); - crypto_init_queue(&priv->queue, EIP197_DEFAULT_RING_SIZE); - ret = safexcel_hw_init(priv); if (ret) { dev_err(dev, "EIP h/w init failed (%d)\n", ret); diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 7e3cbb9ac98e..abe0f59d1473 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -469,11 +469,6 @@ struct safexcel_crypto_priv { struct clk *clk; struct safexcel_config config; - spinlock_t lock; - struct crypto_queue queue; - - bool need_dequeue; - /* context DMA pool */ struct dma_pool *context_pool; @@ -490,6 +485,11 @@ struct safexcel_crypto_priv { /* command/result rings */ struct safexcel_ring cdr; struct safexcel_ring rdr; + + /* queue */ + struct crypto_queue queue; + spinlock_t queue_lock; + bool need_dequeue; } ring[EIP197_MAX_RINGS]; }; @@ -533,7 +533,7 @@ struct safexcel_inv_result { int error; }; -void safexcel_dequeue(struct safexcel_crypto_priv *priv); +void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring); void safexcel_complete(struct safexcel_crypto_priv *priv, int ring); void safexcel_free_context(struct safexcel_crypto_priv *priv, struct crypto_async_request *req, diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index 6037cdfc1f16..d2207ac5ba19 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -339,18 +339,21 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, return ndesc; } + ring = safexcel_select_ring(priv); + ctx->base.ring = ring; ctx->base.needs_inv = false; - ctx->base.ring = safexcel_select_ring(priv); ctx->base.send = safexcel_aes_send; - spin_lock_bh(&priv->lock); - enq_ret = crypto_enqueue_request(&priv->queue, async); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); + spin_unlock_bh(&priv->ring[ring].queue_lock); if (enq_ret != -EINPROGRESS) *ret = enq_ret; - priv->need_dequeue = true; + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); + *should_complete = false; return ndesc; @@ -384,6 +387,7 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) struct safexcel_crypto_priv *priv = ctx->priv; struct skcipher_request req; struct safexcel_inv_result result = { 0 }; + int ring = ctx->base.ring; memset(&req, 0, sizeof(struct skcipher_request)); @@ -397,12 +401,12 @@ static int safexcel_cipher_exit_inv(struct crypto_tfm *tfm) ctx->base.exit_inv = true; ctx->base.send = safexcel_cipher_send_inv; - spin_lock_bh(&priv->lock); - crypto_enqueue_request(&priv->queue, &req.base); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, &req.base); + spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->need_dequeue) - safexcel_dequeue(priv); + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); wait_for_completion_interruptible(&result.completion); @@ -421,7 +425,7 @@ static int safexcel_aes(struct skcipher_request *req, { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(req->base.tfm); struct safexcel_crypto_priv *priv = ctx->priv; - int ret; + int ret, ring; ctx->direction = dir; ctx->mode = mode; @@ -440,12 +444,14 @@ static int safexcel_aes(struct skcipher_request *req, return -ENOMEM; } - spin_lock_bh(&priv->lock); - ret = crypto_enqueue_request(&priv->queue, &req->base); - spin_unlock_bh(&priv->lock); + ring = ctx->base.ring; + + spin_lock_bh(&priv->ring[ring].queue_lock); + ret = crypto_enqueue_request(&priv->ring[ring].queue, &req->base); + spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->need_dequeue) - safexcel_dequeue(priv); + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); return ret; } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index 4e526372464f..8527a5899a2f 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -374,18 +374,21 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, return 1; } - ctx->base.ring = safexcel_select_ring(priv); + ring = safexcel_select_ring(priv); + ctx->base.ring = ring; ctx->base.needs_inv = false; ctx->base.send = safexcel_ahash_send; - spin_lock_bh(&priv->lock); - enq_ret = crypto_enqueue_request(&priv->queue, async); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async); + spin_unlock_bh(&priv->ring[ring].queue_lock); if (enq_ret != -EINPROGRESS) *ret = enq_ret; - priv->need_dequeue = true; + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); + *should_complete = false; return 1; @@ -417,6 +420,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) struct safexcel_crypto_priv *priv = ctx->priv; struct ahash_request req; struct safexcel_inv_result result = { 0 }; + int ring = ctx->base.ring; memset(&req, 0, sizeof(struct ahash_request)); @@ -430,12 +434,12 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) ctx->base.exit_inv = true; ctx->base.send = safexcel_ahash_send_inv; - spin_lock_bh(&priv->lock); - crypto_enqueue_request(&priv->queue, &req.base); - spin_unlock_bh(&priv->lock); + spin_lock_bh(&priv->ring[ring].queue_lock); + crypto_enqueue_request(&priv->ring[ring].queue, &req.base); + spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->need_dequeue) - safexcel_dequeue(priv); + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); wait_for_completion_interruptible(&result.completion); @@ -477,7 +481,7 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq)); struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct safexcel_crypto_priv *priv = ctx->priv; - int ret; + int ret, ring; ctx->base.send = safexcel_ahash_send; @@ -496,12 +500,14 @@ static int safexcel_ahash_enqueue(struct ahash_request *areq) return -ENOMEM; } - spin_lock_bh(&priv->lock); - ret = crypto_enqueue_request(&priv->queue, &areq->base); - spin_unlock_bh(&priv->lock); + ring = ctx->base.ring; + + spin_lock_bh(&priv->ring[ring].queue_lock); + ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base); + spin_unlock_bh(&priv->ring[ring].queue_lock); - if (!priv->need_dequeue) - safexcel_dequeue(priv); + if (!priv->ring[ring].need_dequeue) + safexcel_dequeue(priv, ring); return ret; } -- cgit v1.2.3-70-g09d2