From 950a08f48c72bbd8cc53cb698111d3d4e6a19313 Mon Sep 17 00:00:00 2001 From: Lin Jinhan Date: Wed, 6 Apr 2022 09:02:32 +0800 Subject: [PATCH] crypto: rockchip: add multi lli support 256 LLI were pre-allocated for multiple scatter lists to complete in one calculation.If sg nents exceeds 256 in a single calculation, the calculation is divided into multiple calculations. Signed-off-by: Lin Jinhan Change-Id: I39282efc5a2743e12bd33daa94c89a2ed83fb400 --- drivers/crypto/rockchip/rk_crypto_core.c | 25 ++- drivers/crypto/rockchip/rk_crypto_core.h | 3 + .../rockchip/rk_crypto_skcipher_utils.c | 9 +- drivers/crypto/rockchip/rk_crypto_utils.c | 170 ++++++++++++++++++ drivers/crypto/rockchip/rk_crypto_utils.h | 40 +++++ drivers/crypto/rockchip/rk_crypto_v1.h | 1 + drivers/crypto/rockchip/rk_crypto_v2.c | 23 +-- drivers/crypto/rockchip/rk_crypto_v2.h | 15 +- drivers/crypto/rockchip/rk_crypto_v2_ahash.c | 40 +++-- .../crypto/rockchip/rk_crypto_v2_skcipher.c | 39 ++-- drivers/crypto/rockchip/rk_crypto_v3.c | 24 ++- drivers/crypto/rockchip/rk_crypto_v3.h | 6 +- drivers/crypto/rockchip/rk_crypto_v3_ahash.c | 39 ++-- .../crypto/rockchip/rk_crypto_v3_skcipher.c | 39 ++-- 14 files changed, 365 insertions(+), 108 deletions(-) diff --git a/drivers/crypto/rockchip/rk_crypto_core.c b/drivers/crypto/rockchip/rk_crypto_core.c index ab32ef164321..b63d131a09c1 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.c +++ b/drivers/crypto/rockchip/rk_crypto_core.c @@ -78,11 +78,19 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes); if (alg_ctx->aligned) { - count = min_t(unsigned int, alg_ctx->left_bytes, - sg_src->length); + u32 nents; + + if (rk_dev->soc_data->use_lli_chain) { + count = rk_crypto_hw_desc_maxlen(sg_src, alg_ctx->left_bytes, &nents); + } else { + nents = 1; + count = min_t(unsigned int, alg_ctx->left_bytes, sg_src->length); + } + + alg_ctx->map_nents = nents; alg_ctx->left_bytes -= count; - if (!dma_map_sg(dev, sg_src, 1, DMA_TO_DEVICE)) { + if (!dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) { dev_err(dev, "[%s:%d] dma_map_sg(src) error\n", __func__, __LINE__); ret = -EINVAL; @@ -91,7 +99,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, alg_ctx->addr_in = sg_dma_address(sg_src); if (sg_dst) { - if (!dma_map_sg(dev, sg_dst, 1, DMA_FROM_DEVICE)) { + if (!dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) { dev_err(dev, "[%s:%d] dma_map_sg(dst) error\n", __func__, __LINE__); @@ -103,6 +111,8 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev, alg_ctx->addr_out = sg_dma_address(sg_dst); } } else { + alg_ctx->map_nents = 1; + count = (alg_ctx->left_bytes > rk_dev->vir_max) ? rk_dev->vir_max : alg_ctx->left_bytes; @@ -150,6 +160,7 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev) int ret = 0; struct scatterlist *sg_in, *sg_out; struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req); + u32 nents; CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n", alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes); @@ -158,12 +169,14 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev) if (alg_ctx->total == 0 || alg_ctx->count == 0) return 0; + nents = alg_ctx->map_nents; + sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp; - dma_unmap_sg(rk_dev->dev, sg_in, 1, DMA_TO_DEVICE); + dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE); if (alg_ctx->sg_dst) { sg_out = alg_ctx->aligned ? alg_ctx->sg_dst : &alg_ctx->sg_tmp; - dma_unmap_sg(rk_dev->dev, sg_out, 1, DMA_FROM_DEVICE); + dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE); } if (!alg_ctx->aligned && alg_ctx->req_dst) { diff --git a/drivers/crypto/rockchip/rk_crypto_core.h b/drivers/crypto/rockchip/rk_crypto_core.h index 2f8d9aa4c2e6..5ee3f800c4c9 100644 --- a/drivers/crypto/rockchip/rk_crypto_core.h +++ b/drivers/crypto/rockchip/rk_crypto_core.h @@ -93,6 +93,8 @@ struct rk_crypto_soc_data { unsigned int hw_info_size; bool use_soft_aes192; int default_pka_offset; + bool use_lli_chain; + int (*hw_init)(struct device *dev, void *hw_info); void (*hw_deinit)(struct device *dev, void *hw_info); const char * const *(*hw_get_rsts)(uint32_t *num); @@ -123,6 +125,7 @@ struct rk_alg_ctx { struct scatterlist *req_dst; size_t src_nents; size_t dst_nents; + size_t map_nents; unsigned int total; unsigned int count; diff --git a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c index e73895e38be7..15f51b04bd10 100644 --- a/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c +++ b/drivers/crypto/rockchip/rk_crypto_skcipher_utils.c @@ -128,12 +128,14 @@ static uint32_t rk_get_new_iv(struct rk_cipher_ctx *ctx, u32 mode, bool is_enc, case CIPHER_MODE_CBC: case CIPHER_MODE_CFB: if (is_enc) - sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize); + sg_pcopy_to_buffer(sg_dst, alg_ctx->map_nents, + iv, ivsize, alg_ctx->count - ivsize); else memcpy(iv, ctx->lastc, ivsize); break; case CIPHER_MODE_OFB: - sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize); + sg_pcopy_to_buffer(sg_dst, alg_ctx->map_nents, + iv, ivsize, alg_ctx->count - ivsize); crypto_xor(iv, ctx->lastc, ivsize); break; default: @@ -189,7 +191,8 @@ static int rk_set_data_start(struct rk_crypto_dev *rk_dev) ivsize = alg_ctx->count > ivsize ? ivsize : alg_ctx->count; - sg_pcopy_to_buffer(src_sg, 1, ctx->lastc, ivsize, alg_ctx->count - ivsize); + sg_pcopy_to_buffer(src_sg, alg_ctx->map_nents, + ctx->lastc, ivsize, alg_ctx->count - ivsize); alg_ctx->ops.hw_dma_start(rk_dev, true); } diff --git a/drivers/crypto/rockchip/rk_crypto_utils.c b/drivers/crypto/rockchip/rk_crypto_utils.c index 6faa632463cb..0ae2000ed6fb 100644 --- a/drivers/crypto/rockchip/rk_crypto_utils.c +++ b/drivers/crypto/rockchip/rk_crypto_utils.c @@ -8,6 +8,8 @@ * */ +#include + #include "rk_crypto_core.h" #include "rk_crypto_utils.h" @@ -112,3 +114,171 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents, return true; } + +void rk_crypto_dump_hw_desc(struct rk_hw_desc *hw_desc) +{ + struct crypto_lli_desc *cur_lli = NULL; + u32 i; + + cur_lli = hw_desc->lli_head; + + CRYPTO_TRACE("lli_head = %lx, lli_tail = %lx", + (unsigned long)hw_desc->lli_head, (unsigned long)hw_desc->lli_tail); + + for (i = 0; i < hw_desc->total; i++, cur_lli++) { + CRYPTO_TRACE("cur_lli = %lx", (unsigned long)cur_lli); + CRYPTO_TRACE("src_addr = %08x", cur_lli->src_addr); + CRYPTO_TRACE("src_len = %08x", cur_lli->src_len); + CRYPTO_TRACE("dst_addr = %08x", cur_lli->dst_addr); + CRYPTO_TRACE("dst_len = %08x", cur_lli->dst_len); + CRYPTO_TRACE("user_def = %08x", cur_lli->user_define); + CRYPTO_TRACE("dma_ctl = %08x", cur_lli->dma_ctrl); + CRYPTO_TRACE("next = %08x\n", cur_lli->next_addr); + + if (cur_lli == hw_desc->lli_tail) + break; + } +} + +u64 rk_crypto_hw_desc_maxlen(struct scatterlist *sg, u64 len, u32 *max_nents) +{ + int nents; + u64 total; + + if (!len) + return 0; + + for (nents = 0, total = 0; sg; sg = sg_next(sg)) { + if (!sg) + goto exit; + + nents++; + total += sg->length; + + if (nents >= RK_DEFAULT_LLI_CNT || total >= len) + goto exit; + } + +exit: + *max_nents = nents; + return total > len ? len : total; +} + +int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc) +{ + u32 lli_cnt = RK_DEFAULT_LLI_CNT; + u32 lli_len = lli_cnt * sizeof(struct crypto_lli_desc); + + if (!dev || !hw_desc) + return -EINVAL; + + memset(hw_desc, 0x00, sizeof(*hw_desc)); + + ///TODO: cma + hw_desc->lli_head = dma_alloc_coherent(dev, lli_len, &hw_desc->lli_head_dma, GFP_KERNEL); + if (!hw_desc->lli_head) + return -ENOMEM; + + hw_desc->lli_tail = hw_desc->lli_head; + hw_desc->total = lli_cnt; + hw_desc->dev = dev; + + memset(hw_desc->lli_head, 0x00, lli_len); + + CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x", + (unsigned long)hw_desc->dev, lli_len, + (unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma); + + return 0; +} + +void rk_crypto_hw_desc_free(struct rk_hw_desc *hw_desc) +{ + if (!hw_desc || !hw_desc->dev || !hw_desc->lli_head) + return; + + CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x", + (unsigned long)hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc), + (unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma); + + dma_free_coherent(hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc), + hw_desc->lli_head, hw_desc->lli_head_dma); + + memset(hw_desc, 0x00, sizeof(*hw_desc)); +} + +int rk_crypto_hw_desc_init(struct rk_hw_desc *hw_desc, + struct scatterlist *src_sg, + struct scatterlist *dst_sg, + u64 len) +{ + struct crypto_lli_desc *cur_lli = NULL; + struct scatterlist *tmp_src, *tmp_dst; + dma_addr_t tmp_next_dma; + u32 src_nents, dst_nents; + u32 i, data_cnt = 0; + + if (!hw_desc || !hw_desc->dev || !hw_desc->lli_head) + return -EINVAL; + + if (!src_sg || len == 0) + return -EINVAL; + + src_nents = sg_nents_for_len(src_sg, len); + dst_nents = dst_sg ? sg_nents_for_len(dst_sg, len) : src_nents; + + if (src_nents != dst_nents) + return -EINVAL; + + CRYPTO_TRACE("src_nents = %u, total = %u, len = %llu", src_nents, hw_desc->total, len); + + if (src_nents > hw_desc->total) { + pr_err("crypto: nents overflow, %u > %u", src_nents, hw_desc->total); + return -ENOMEM; + } + + memset(hw_desc->lli_head, 0x00, src_nents * sizeof(struct crypto_lli_desc)); + + cur_lli = hw_desc->lli_head; + tmp_src = src_sg; + tmp_dst = dst_sg; + tmp_next_dma = hw_desc->lli_head_dma + sizeof(*cur_lli); + + if (dst_sg) { + for (i = 0; i < src_nents - 1; i++, cur_lli++, tmp_next_dma += sizeof(*cur_lli)) { + cur_lli->src_addr = sg_dma_address(tmp_src); + cur_lli->src_len = sg_dma_len(tmp_src); + cur_lli->dst_addr = sg_dma_address(tmp_dst); + cur_lli->dst_len = sg_dma_len(tmp_dst); + cur_lli->next_addr = tmp_next_dma; + + data_cnt += sg_dma_len(tmp_src); + tmp_src = sg_next(tmp_src); + tmp_dst = sg_next(tmp_dst); + } + } else { + for (i = 0; i < src_nents - 1; i++, cur_lli++, tmp_next_dma += sizeof(*cur_lli)) { + cur_lli->src_addr = sg_dma_address(tmp_src); + cur_lli->src_len = sg_dma_len(tmp_src); + cur_lli->next_addr = tmp_next_dma; + + data_cnt += sg_dma_len(tmp_src); + tmp_src = sg_next(tmp_src); + } + } + + /* for last lli */ + cur_lli->src_addr = sg_dma_address(tmp_src); + cur_lli->src_len = len - data_cnt; + cur_lli->next_addr = 0; + + if (dst_sg) { + cur_lli->dst_addr = sg_dma_address(tmp_dst); + cur_lli->dst_len = len - data_cnt; + } + + hw_desc->lli_tail = cur_lli; + + return 0; +} + diff --git a/drivers/crypto/rockchip/rk_crypto_utils.h b/drivers/crypto/rockchip/rk_crypto_utils.h index 77e444217b9f..c31ba35628d0 100644 --- a/drivers/crypto/rockchip/rk_crypto_utils.h +++ b/drivers/crypto/rockchip/rk_crypto_utils.h @@ -5,6 +5,33 @@ #ifndef __RK_CRYPTO_UTILS_H__ #define __RK_CRYPTO_UTILS_H__ +#include +#include + +#include "rk_crypto_core.h" + +/* Default 256 x 4K = 1MByte */ +#define RK_DEFAULT_LLI_CNT 256 + +struct crypto_lli_desc { + u32 src_addr; + u32 src_len; + u32 dst_addr; + u32 dst_len; + u32 user_define; + u32 reserve; + u32 dma_ctrl; + u32 next_addr; +}; + +struct rk_hw_desc { + struct device *dev; + struct crypto_lli_desc *lli_head; + struct crypto_lli_desc *lli_tail; + dma_addr_t lli_head_dma; + u32 total; +}; + void rk_crypto_write_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, const u8 *data, u32 bytes); void rk_crypto_clear_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, u32 words); @@ -15,5 +42,18 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents, struct scatterlist *dst_sg, size_t dst_nents, int align_mask); +u64 rk_crypto_hw_desc_maxlen(struct scatterlist *sg, u64 len, u32 *max_nents); + +int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc); + +int rk_crypto_hw_desc_init(struct rk_hw_desc *hw_desc, + struct scatterlist *src_sg, + struct scatterlist *dst_sg, + u64 len); + +void rk_crypto_hw_desc_free(struct rk_hw_desc *hw_desc); + +void rk_crypto_dump_hw_desc(struct rk_hw_desc *hw_desc); + #endif diff --git a/drivers/crypto/rockchip/rk_crypto_v1.h b/drivers/crypto/rockchip/rk_crypto_v1.h index 961b13ae6e52..417ead57565e 100644 --- a/drivers/crypto/rockchip/rk_crypto_v1.h +++ b/drivers/crypto/rockchip/rk_crypto_v1.h @@ -23,6 +23,7 @@ struct rk_hw_crypto_v1_info { .hw_is_algo_valid = rk_hw_crypto_v1_algo_valid,\ .hw_info_size = sizeof(struct rk_hw_crypto_v1_info),\ .default_pka_offset = 0,\ + .use_lli_chain = false,\ } #if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V1) diff --git a/drivers/crypto/rockchip/rk_crypto_v2.c b/drivers/crypto/rockchip/rk_crypto_v2.c index 07f4265b12ed..5a37a855fccf 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2.c +++ b/drivers/crypto/rockchip/rk_crypto_v2.c @@ -59,21 +59,15 @@ static struct rk_crypto_algt *crypto_v2_algs[] = { int rk_hw_crypto_v2_init(struct device *dev, void *hw_info) { - int err = 0; struct rk_hw_crypto_v2_info *info = (struct rk_hw_crypto_v2_info *)hw_info; - info->desc = dma_alloc_coherent(dev, - sizeof(struct crypto_lli_desc), - &info->desc_dma, - GFP_KERNEL); - if (!info->desc) { - err = -ENOMEM; - goto end; - } + if (!dev || !hw_info) + return -EINVAL; -end: - return err; + memset(info, 0x00, sizeof(*info)); + + return rk_crypto_hw_desc_alloc(dev, &info->hw_desc); } void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info) @@ -81,9 +75,10 @@ void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info) struct rk_hw_crypto_v2_info *info = (struct rk_hw_crypto_v2_info *)hw_info; - if (info && info->desc) - dma_free_coherent(dev, sizeof(struct crypto_lli_desc), - info->desc, info->desc_dma); + if (!dev || !hw_info) + return; + + rk_crypto_hw_desc_free(&info->hw_desc); } const char * const *rk_hw_crypto_v2_get_rsts(uint32_t *num) diff --git a/drivers/crypto/rockchip/rk_crypto_v2.h b/drivers/crypto/rockchip/rk_crypto_v2.h index 8ad0b3cfeac1..401b166b3359 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2.h +++ b/drivers/crypto/rockchip/rk_crypto_v2.h @@ -7,20 +7,10 @@ #include -struct crypto_lli_desc { - u32 src_addr; - u32 src_len; - u32 dst_addr; - u32 dst_len; - u32 user_define; - u32 reserve; - u32 dma_ctrl; - u32 next_addr; -}; +#include "rk_crypto_utils.h" struct rk_hw_crypto_v2_info { - struct crypto_lli_desc *desc; - dma_addr_t desc_dma; + struct rk_hw_desc hw_desc; }; #define RK_CRYPTO_V2_SOC_DATA_INIT(names, soft_aes_192) {\ @@ -35,6 +25,7 @@ struct rk_hw_crypto_v2_info { .hw_is_algo_valid = rk_hw_crypto_v2_algo_valid,\ .hw_info_size = sizeof(struct rk_hw_crypto_v2_info),\ .default_pka_offset = 0x0480,\ + .use_lli_chain = true,\ } #if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V2) diff --git a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c index 1395cd350f42..16e7cbdb29db 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_ahash.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_ahash.c @@ -16,6 +16,7 @@ #include "rk_crypto_v2.h" #include "rk_crypto_v2_reg.h" #include "rk_crypto_ahash_utils.h" +#include "rk_crypto_utils.h" #define RK_HASH_CTX_MAGIC 0x1A1A1A1A #define RK_POLL_PERIOD_US 100 @@ -76,14 +77,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) interrupt_status &= CRYPTO_LOCKSTEP_MASK; if (interrupt_status != CRYPTO_SRC_ITEM_DONE_INT_ST) { - dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc); + dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head); dev_err(rk_dev->dev, "DMA addr_in = %08x\n", (u32)alg_ctx->addr_in); dev_err(rk_dev->dev, "DMA addr_out = %08x\n", (u32)alg_ctx->addr_out); dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count); dev_err(rk_dev->dev, "DMA desc_dma = %08x\n", - (u32)hw_info->desc_dma); + (u32)hw_info->hw_desc.lli_head_dma); dev_err(rk_dev->dev, "DMA Error status = %08x\n", interrupt_status); dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n", @@ -112,7 +113,7 @@ static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(ctx->rk_dev); struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info; - struct crypto_lli_desc *lli_desc = hw_info->desc; + struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head; if (err) { rk_hash_reset(ctx->rk_dev); @@ -212,8 +213,10 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) (struct rk_hw_crypto_v2_info *)rk_dev->hw_info; struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + struct crypto_lli_desc *lli_head, *lli_tail; u32 dma_ctl = CRYPTO_DMA_RESTART; bool is_final = flag & RK_FLAG_FINAL; + int ret; CRYPTO_TRACE("ctx->calc_cnt = %u, count %u Byte, is_final = %d", ctx->calc_cnt, alg_ctx->count, is_final); @@ -230,32 +233,41 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) return 0; } - memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); + if (alg_ctx->aligned) + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + alg_ctx->sg_src, NULL, alg_ctx->count); + else + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + &alg_ctx->sg_tmp, NULL, alg_ctx->count); + if (ret) + return ret; - hw_info->desc->src_addr = alg_ctx->addr_in; - hw_info->desc->src_len = alg_ctx->count; - hw_info->desc->next_addr = hw_info->desc_dma; + lli_head = hw_info->hw_desc.lli_head; + lli_tail = hw_info->hw_desc.lli_tail; - hw_info->desc->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE; - hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE; + lli_tail->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE; + lli_tail->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE; + lli_tail->next_addr = hw_info->hw_desc.lli_head_dma; if (ctx->calc_cnt == 0) { dma_ctl = CRYPTO_DMA_START; - hw_info->desc->user_define |= LLI_USER_CIPHER_START; - hw_info->desc->user_define |= LLI_USER_STRING_START; + lli_head->user_define |= LLI_USER_CIPHER_START; + lli_head->user_define |= LLI_USER_STRING_START; - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, (CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) | CRYPTO_HASH_ENABLE); } if (is_final && alg_ctx->left_bytes == 0) - hw_info->desc->user_define |= LLI_USER_STRING_LAST; + lli_tail->user_define |= LLI_USER_STRING_LAST; CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u", - hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count); + lli_head->dma_ctrl, lli_head->user_define, alg_ctx->count); + + rk_crypto_dump_hw_desc(&hw_info->hw_desc); dma_wmb(); diff --git a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c index 2c8eb32d6099..e929ff66477d 100644 --- a/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v2_skcipher.c @@ -48,14 +48,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) interrupt_status &= CRYPTO_LOCKSTEP_MASK; if (interrupt_status != CRYPTO_DST_ITEM_DONE_INT_ST) { - dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc); + dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head); dev_err(rk_dev->dev, "DMA addr_in = %08x\n", (u32)alg_ctx->addr_in); dev_err(rk_dev->dev, "DMA addr_out = %08x\n", (u32)alg_ctx->addr_out); dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count); dev_err(rk_dev->dev, "DMA desc_dma = %08x\n", - (u32)hw_info->desc_dma); + (u32)hw_info->hw_desc.lli_head_dma); dev_err(rk_dev->dev, "DMA Error status = %08x\n", interrupt_status); dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n", @@ -153,7 +153,7 @@ static void rk_crypto_complete(struct crypto_async_request *base, int err) struct rk_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info; - struct crypto_lli_desc *lli_desc = hw_info->desc; + struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head; CRYPTO_WRITE(ctx->rk_dev, CRYPTO_BC_CTL, 0xffff0000); if (err) { @@ -293,10 +293,22 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + struct crypto_lli_desc *lli_head, *lli_tail; u32 calc_len = alg_ctx->count; u32 start_flag = CRYPTO_DMA_START; + int ret; - memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); + if (alg_ctx->aligned) + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + alg_ctx->sg_src, alg_ctx->sg_dst, alg_ctx->count); + else + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + &alg_ctx->sg_tmp, &alg_ctx->sg_tmp, alg_ctx->count); + if (ret) + return ret; + + lli_head = hw_info->hw_desc.lli_head; + lli_tail = hw_info->hw_desc.lli_tail; /* * the data length is not aligned will use addr_vir to calculate, @@ -305,19 +317,18 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) if (is_calc_need_round_up(req)) calc_len = round_up(calc_len, alg_ctx->chunk_size); - hw_info->desc->src_addr = alg_ctx->addr_in; - hw_info->desc->src_len = calc_len; - hw_info->desc->dst_addr = alg_ctx->addr_out; - hw_info->desc->dst_len = calc_len; - hw_info->desc->next_addr = 0; - hw_info->desc->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; - hw_info->desc->user_define = LLI_USER_STRING_START | - LLI_USER_CIPHER_START | - LLI_USER_STRING_LAST; + lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START; + + lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; + lli_tail->user_define |= LLI_USER_STRING_LAST; + lli_tail->src_len += (calc_len - alg_ctx->count); + lli_tail->dst_len += (calc_len - alg_ctx->count); + + rk_crypto_dump_hw_desc(&hw_info->hw_desc); dma_wmb(); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK)); return 0; diff --git a/drivers/crypto/rockchip/rk_crypto_v3.c b/drivers/crypto/rockchip/rk_crypto_v3.c index 2b7bd1c24fe9..b4f339019fba 100644 --- a/drivers/crypto/rockchip/rk_crypto_v3.c +++ b/drivers/crypto/rockchip/rk_crypto_v3.c @@ -11,6 +11,7 @@ #include "rk_crypto_core.h" #include "rk_crypto_v3.h" #include "rk_crypto_v3_reg.h" +#include "rk_crypto_utils.h" static const u32 cipher_mode2bit_mask[] = { [CIPHER_MODE_ECB] = CRYPTO_ECB_FLAG, @@ -160,21 +161,15 @@ static bool rk_is_hash_support(struct rk_crypto_dev *rk_dev, u32 algo, u32 type) int rk_hw_crypto_v3_init(struct device *dev, void *hw_info) { - int err = 0; struct rk_hw_crypto_v3_info *info = (struct rk_hw_crypto_v3_info *)hw_info; - info->desc = dma_alloc_coherent(dev, - sizeof(struct crypto_lli_desc), - &info->desc_dma, - GFP_KERNEL); - if (!info->desc) { - err = -ENOMEM; - goto end; - } + if (!dev || !hw_info) + return -EINVAL; -end: - return err; + memset(info, 0x00, sizeof(*info)); + + return rk_crypto_hw_desc_alloc(dev, &info->hw_desc); } void rk_hw_crypto_v3_deinit(struct device *dev, void *hw_info) @@ -182,9 +177,10 @@ void rk_hw_crypto_v3_deinit(struct device *dev, void *hw_info) struct rk_hw_crypto_v3_info *info = (struct rk_hw_crypto_v3_info *)hw_info; - if (info && info->desc) - dma_free_coherent(dev, sizeof(struct crypto_lli_desc), - info->desc, info->desc_dma); + if (!dev || !hw_info) + return; + + rk_crypto_hw_desc_free(&info->hw_desc); } const char * const *rk_hw_crypto_v3_get_rsts(uint32_t *num) diff --git a/drivers/crypto/rockchip/rk_crypto_v3.h b/drivers/crypto/rockchip/rk_crypto_v3.h index 6580ed00bca2..e5ac6d2931ff 100644 --- a/drivers/crypto/rockchip/rk_crypto_v3.h +++ b/drivers/crypto/rockchip/rk_crypto_v3.h @@ -7,11 +7,10 @@ #include -#include "rk_crypto_v2.h" +#include "rk_crypto_utils.h" struct rk_hw_crypto_v3_info { - struct crypto_lli_desc *desc; - dma_addr_t desc_dma; + struct rk_hw_desc hw_desc; }; #define RK_CRYPTO_V3_SOC_DATA_INIT(names) {\ @@ -26,6 +25,7 @@ struct rk_hw_crypto_v3_info { .hw_is_algo_valid = rk_hw_crypto_v3_algo_valid,\ .hw_info_size = sizeof(struct rk_hw_crypto_v3_info),\ .default_pka_offset = 0x0480,\ + .use_lli_chain = true,\ } #if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V3) diff --git a/drivers/crypto/rockchip/rk_crypto_v3_ahash.c b/drivers/crypto/rockchip/rk_crypto_v3_ahash.c index 7fe53ac71a65..213aaf1d084d 100644 --- a/drivers/crypto/rockchip/rk_crypto_v3_ahash.c +++ b/drivers/crypto/rockchip/rk_crypto_v3_ahash.c @@ -15,6 +15,7 @@ #include "rk_crypto_v3.h" #include "rk_crypto_v3_reg.h" #include "rk_crypto_ahash_utils.h" +#include "rk_crypto_utils.h" #define RK_HASH_CTX_MAGIC 0x1A1A1A1A #define RK_POLL_PERIOD_US 100 @@ -141,14 +142,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) interrupt_status &= CRYPTO_LOCKSTEP_MASK; if (interrupt_status != CRYPTO_SRC_ITEM_DONE_INT_ST) { - dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc); + dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head); dev_err(rk_dev->dev, "DMA addr_in = %08x\n", (u32)alg_ctx->addr_in); dev_err(rk_dev->dev, "DMA addr_out = %08x\n", (u32)alg_ctx->addr_out); dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count); dev_err(rk_dev->dev, "DMA desc_dma = %08x\n", - (u32)hw_info->desc_dma); + (u32)hw_info->hw_desc.lli_head_dma); dev_err(rk_dev->dev, "DMA Error status = %08x\n", interrupt_status); dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n", @@ -177,7 +178,7 @@ static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err) struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(ctx->rk_dev); struct rk_hw_crypto_v3_info *hw_info = ctx->rk_dev->hw_info; - struct crypto_lli_desc *lli_desc = hw_info->desc; + struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head; if (err) { rk_hash_reset(ctx->rk_dev); @@ -284,8 +285,10 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) (struct rk_hw_crypto_v3_info *)rk_dev->hw_info; struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev); struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev); + struct crypto_lli_desc *lli_head, *lli_tail; u32 dma_ctl = CRYPTO_DMA_RESTART; bool is_final = flag & RK_FLAG_FINAL; + int ret; CRYPTO_TRACE("ctx->calc_cnt = %u, count %u Byte, is_final = %d", ctx->calc_cnt, alg_ctx->count, is_final); @@ -305,32 +308,40 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) if (alg_ctx->total == alg_ctx->left_bytes + alg_ctx->count) rk_hash_mid_data_restore(rk_dev, (struct rk_hash_mid_data *)ctx->priv); - memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); + if (alg_ctx->aligned) + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + alg_ctx->sg_src, NULL, alg_ctx->count); + else + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + &alg_ctx->sg_tmp, NULL, alg_ctx->count); + if (ret) + return ret; - hw_info->desc->src_addr = alg_ctx->addr_in; - hw_info->desc->src_len = alg_ctx->count; - hw_info->desc->next_addr = hw_info->desc_dma; + lli_head = hw_info->hw_desc.lli_head; + lli_tail = hw_info->hw_desc.lli_tail; - hw_info->desc->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE; - hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE; + lli_tail->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE; + lli_tail->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE; if (ctx->calc_cnt == 0) { dma_ctl = CRYPTO_DMA_START; - hw_info->desc->user_define |= LLI_USER_CIPHER_START; - hw_info->desc->user_define |= LLI_USER_STRING_START; + lli_head->user_define |= LLI_USER_CIPHER_START; + lli_head->user_define |= LLI_USER_STRING_START; - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL, (CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) | CRYPTO_HASH_ENABLE); } if (is_final && alg_ctx->left_bytes == 0) - hw_info->desc->user_define |= LLI_USER_STRING_LAST; + lli_tail->user_define |= LLI_USER_STRING_LAST; CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u", - hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count); + lli_head->dma_ctrl, lli_head->user_define, alg_ctx->count); + + rk_crypto_dump_hw_desc(&hw_info->hw_desc); dma_wmb(); diff --git a/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c b/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c index 9d8cde3e4014..21a71da4082e 100644 --- a/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c +++ b/drivers/crypto/rockchip/rk_crypto_v3_skcipher.c @@ -47,14 +47,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id) interrupt_status &= CRYPTO_LOCKSTEP_MASK; if (interrupt_status != CRYPTO_DST_ITEM_DONE_INT_ST) { - dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc); + dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head); dev_err(rk_dev->dev, "DMA addr_in = %08x\n", (u32)alg_ctx->addr_in); dev_err(rk_dev->dev, "DMA addr_out = %08x\n", (u32)alg_ctx->addr_out); dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count); dev_err(rk_dev->dev, "DMA desc_dma = %08x\n", - (u32)hw_info->desc_dma); + (u32)hw_info->hw_desc.lli_head_dma); dev_err(rk_dev->dev, "DMA Error status = %08x\n", interrupt_status); dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n", @@ -152,7 +152,7 @@ static void rk_crypto_complete(struct crypto_async_request *base, int err) struct rk_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx; struct rk_hw_crypto_v3_info *hw_info = ctx->rk_dev->hw_info; - struct crypto_lli_desc *lli_desc = hw_info->desc; + struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head; CRYPTO_WRITE(ctx->rk_dev, CRYPTO_BC_CTL, 0xffff0000); if (err) { @@ -292,10 +292,22 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) struct skcipher_request *req = skcipher_request_cast(rk_dev->async_req); struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev); + struct crypto_lli_desc *lli_head, *lli_tail; u32 calc_len = alg_ctx->count; u32 start_flag = CRYPTO_DMA_START; + int ret; - memset(hw_info->desc, 0x00, sizeof(*hw_info->desc)); + if (alg_ctx->aligned) + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + alg_ctx->sg_src, alg_ctx->sg_dst, alg_ctx->count); + else + ret = rk_crypto_hw_desc_init(&hw_info->hw_desc, + &alg_ctx->sg_tmp, &alg_ctx->sg_tmp, alg_ctx->count); + if (ret) + return ret; + + lli_head = hw_info->hw_desc.lli_head; + lli_tail = hw_info->hw_desc.lli_tail; /* * the data length is not aligned will use addr_vir to calculate, @@ -304,19 +316,18 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag) if (is_calc_need_round_up(req)) calc_len = round_up(calc_len, alg_ctx->chunk_size); - hw_info->desc->src_addr = alg_ctx->addr_in; - hw_info->desc->src_len = calc_len; - hw_info->desc->dst_addr = alg_ctx->addr_out; - hw_info->desc->dst_len = calc_len; - hw_info->desc->next_addr = 0; - hw_info->desc->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; - hw_info->desc->user_define = LLI_USER_STRING_START | - LLI_USER_CIPHER_START | - LLI_USER_STRING_LAST; + lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START; + + lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST; + lli_tail->user_define |= LLI_USER_STRING_LAST; + lli_tail->src_len += (calc_len - alg_ctx->count); + lli_tail->dst_len += (calc_len - alg_ctx->count); + + rk_crypto_dump_hw_desc(&hw_info->hw_desc); dma_wmb(); - CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma); + CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma); CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK)); return 0;