Lubancat2/kernel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

crypto: rockchip: add multi lli support

Browse Source 
256 LLI were pre-allocated for multiple scatter lists
 to complete in one calculation.If sg nents exceeds
 256 in a single calculation, the calculation is
 divided into multiple calculations.

Signed-off-by: Lin Jinhan <troy.lin@rock-chips.com>
Change-Id: I39282efc5a2743e12bd33daa94c89a2ed83fb400
This commit is contained in:
Lin Jinhan
2022-04-06 09:02:32 +08:00
parent bbc37cfc1f
commit 950a08f48c
14 changed files with 365 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
drivers/crypto/rockchip/rk_crypto_core.c
View File

@ -78,11 +78,19 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes);
if (alg_ctx->aligned) {
count = min_t(unsigned int, alg_ctx->left_bytes,
sg_src->length);
u32 nents;
if (rk_dev->soc_data->use_lli_chain) {
count = rk_crypto_hw_desc_maxlen(sg_src, alg_ctx->left_bytes, &nents);
} else {
nents = 1;
count = min_t(unsigned int, alg_ctx->left_bytes, sg_src->length);
}
alg_ctx->map_nents = nents;
alg_ctx->left_bytes -= count;
if (!dma_map_sg(dev, sg_src, 1, DMA_TO_DEVICE)) {
if (!dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) {
dev_err(dev, "[%s:%d] dma_map_sg(src) error\n",
__func__, __LINE__);
ret = -EINVAL;
@ -91,7 +99,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
alg_ctx->addr_in = sg_dma_address(sg_src);
if (sg_dst) {
if (!dma_map_sg(dev, sg_dst, 1, DMA_FROM_DEVICE)) {
if (!dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) {
dev_err(dev,
"[%s:%d] dma_map_sg(dst) error\n",
__func__, __LINE__);
@ -103,6 +111,8 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
alg_ctx->addr_out = sg_dma_address(sg_dst);
}
} else {
alg_ctx->map_nents = 1;
count = (alg_ctx->left_bytes > rk_dev->vir_max) ?
rk_dev->vir_max : alg_ctx->left_bytes;
@ -150,6 +160,7 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev)
int ret = 0;
struct scatterlist *sg_in, *sg_out;
struct rk_alg_ctx *alg_ctx = rk_alg_ctx_cast(rk_dev->async_req);
u32 nents;
CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u\n",
alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes);
@ -158,12 +169,14 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev)
if (alg_ctx->total == 0 || alg_ctx->count == 0)
return 0;
nents = alg_ctx->map_nents;
sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp;
dma_unmap_sg(rk_dev->dev, sg_in, 1, DMA_TO_DEVICE);
dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE);
if (alg_ctx->sg_dst) {
sg_out = alg_ctx->aligned ? alg_ctx->sg_dst : &alg_ctx->sg_tmp;
dma_unmap_sg(rk_dev->dev, sg_out, 1, DMA_FROM_DEVICE);
dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE);
}
if (!alg_ctx->aligned && alg_ctx->req_dst) {

3
drivers/crypto/rockchip/rk_crypto_core.h
View File

@ -93,6 +93,8 @@ struct rk_crypto_soc_data {
unsigned int hw_info_size;
bool use_soft_aes192;
int default_pka_offset;
bool use_lli_chain;
int (*hw_init)(struct device *dev, void *hw_info);
void (*hw_deinit)(struct device *dev, void *hw_info);
const char * const *(*hw_get_rsts)(uint32_t *num);
@ -123,6 +125,7 @@ struct rk_alg_ctx {
struct scatterlist *req_dst;
size_t src_nents;
size_t dst_nents;
size_t map_nents;
unsigned int total;
unsigned int count;

9
drivers/crypto/rockchip/rk_crypto_skcipher_utils.c
View File

@ -128,12 +128,14 @@ static uint32_t rk_get_new_iv(struct rk_cipher_ctx *ctx, u32 mode, bool is_enc,
case CIPHER_MODE_CBC:
case CIPHER_MODE_CFB:
if (is_enc)
sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize);
sg_pcopy_to_buffer(sg_dst, alg_ctx->map_nents,
iv, ivsize, alg_ctx->count - ivsize);
else
memcpy(iv, ctx->lastc, ivsize);
break;
case CIPHER_MODE_OFB:
sg_pcopy_to_buffer(sg_dst, 1, iv, ivsize, alg_ctx->count - ivsize);
sg_pcopy_to_buffer(sg_dst, alg_ctx->map_nents,
iv, ivsize, alg_ctx->count - ivsize);
crypto_xor(iv, ctx->lastc, ivsize);
break;
default:
@ -189,7 +191,8 @@ static int rk_set_data_start(struct rk_crypto_dev *rk_dev)
ivsize = alg_ctx->count > ivsize ? ivsize : alg_ctx->count;
sg_pcopy_to_buffer(src_sg, 1, ctx->lastc, ivsize, alg_ctx->count - ivsize);
sg_pcopy_to_buffer(src_sg, alg_ctx->map_nents,
ctx->lastc, ivsize, alg_ctx->count - ivsize);
alg_ctx->ops.hw_dma_start(rk_dev, true);
}

170
drivers/crypto/rockchip/rk_crypto_utils.c
View File

@ -8,6 +8,8 @@
*
*/
#include <linux/scatterlist.h>
#include "rk_crypto_core.h"
#include "rk_crypto_utils.h"
@ -112,3 +114,171 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents,
return true;
}
void rk_crypto_dump_hw_desc(struct rk_hw_desc *hw_desc)
{
struct crypto_lli_desc *cur_lli = NULL;
u32 i;
cur_lli = hw_desc->lli_head;
CRYPTO_TRACE("lli_head = %lx, lli_tail = %lx",
(unsigned long)hw_desc->lli_head, (unsigned long)hw_desc->lli_tail);
for (i = 0; i < hw_desc->total; i++, cur_lli++) {
CRYPTO_TRACE("cur_lli = %lx", (unsigned long)cur_lli);
CRYPTO_TRACE("src_addr = %08x", cur_lli->src_addr);
CRYPTO_TRACE("src_len = %08x", cur_lli->src_len);
CRYPTO_TRACE("dst_addr = %08x", cur_lli->dst_addr);
CRYPTO_TRACE("dst_len = %08x", cur_lli->dst_len);
CRYPTO_TRACE("user_def = %08x", cur_lli->user_define);
CRYPTO_TRACE("dma_ctl = %08x", cur_lli->dma_ctrl);
CRYPTO_TRACE("next = %08x\n", cur_lli->next_addr);
if (cur_lli == hw_desc->lli_tail)
break;
}
}
u64 rk_crypto_hw_desc_maxlen(struct scatterlist *sg, u64 len, u32 *max_nents)
{
int nents;
u64 total;
if (!len)
return 0;
for (nents = 0, total = 0; sg; sg = sg_next(sg)) {
if (!sg)
goto exit;
nents++;
total += sg->length;
if (nents >= RK_DEFAULT_LLI_CNT || total >= len)
goto exit;
}
exit:
*max_nents = nents;
return total > len ? len : total;
}
int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc)
{
u32 lli_cnt = RK_DEFAULT_LLI_CNT;
u32 lli_len = lli_cnt * sizeof(struct crypto_lli_desc);
if (!dev || !hw_desc)
return -EINVAL;
memset(hw_desc, 0x00, sizeof(*hw_desc));
///TODO: cma
hw_desc->lli_head = dma_alloc_coherent(dev, lli_len, &hw_desc->lli_head_dma, GFP_KERNEL);
if (!hw_desc->lli_head)
return -ENOMEM;
hw_desc->lli_tail = hw_desc->lli_head;
hw_desc->total = lli_cnt;
hw_desc->dev = dev;
memset(hw_desc->lli_head, 0x00, lli_len);
CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x",
(unsigned long)hw_desc->dev, lli_len,
(unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma);
return 0;
}
void rk_crypto_hw_desc_free(struct rk_hw_desc *hw_desc)
{
if (!hw_desc || !hw_desc->dev || !hw_desc->lli_head)
return;
CRYPTO_TRACE("dev = %lx, buffer_len = %u, lli_head = %lx, lli_head_dma = %x",
(unsigned long)hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc),
(unsigned long)hw_desc->lli_head, hw_desc->lli_head_dma);
dma_free_coherent(hw_desc->dev, hw_desc->total * sizeof(struct crypto_lli_desc),
hw_desc->lli_head, hw_desc->lli_head_dma);
memset(hw_desc, 0x00, sizeof(*hw_desc));
}
int rk_crypto_hw_desc_init(struct rk_hw_desc *hw_desc,
struct scatterlist *src_sg,
struct scatterlist *dst_sg,
u64 len)
{
struct crypto_lli_desc *cur_lli = NULL;
struct scatterlist *tmp_src, *tmp_dst;
dma_addr_t tmp_next_dma;
u32 src_nents, dst_nents;
u32 i, data_cnt = 0;
if (!hw_desc || !hw_desc->dev || !hw_desc->lli_head)
return -EINVAL;
if (!src_sg || len == 0)
return -EINVAL;
src_nents = sg_nents_for_len(src_sg, len);
dst_nents = dst_sg ? sg_nents_for_len(dst_sg, len) : src_nents;
if (src_nents != dst_nents)
return -EINVAL;
CRYPTO_TRACE("src_nents = %u, total = %u, len = %llu", src_nents, hw_desc->total, len);
if (src_nents > hw_desc->total) {
pr_err("crypto: nents overflow, %u > %u", src_nents, hw_desc->total);
return -ENOMEM;
}
memset(hw_desc->lli_head, 0x00, src_nents * sizeof(struct crypto_lli_desc));
cur_lli = hw_desc->lli_head;
tmp_src = src_sg;
tmp_dst = dst_sg;
tmp_next_dma = hw_desc->lli_head_dma + sizeof(*cur_lli);
if (dst_sg) {
for (i = 0; i < src_nents - 1; i++, cur_lli++, tmp_next_dma += sizeof(*cur_lli)) {
cur_lli->src_addr = sg_dma_address(tmp_src);
cur_lli->src_len = sg_dma_len(tmp_src);
cur_lli->dst_addr = sg_dma_address(tmp_dst);
cur_lli->dst_len = sg_dma_len(tmp_dst);
cur_lli->next_addr = tmp_next_dma;
data_cnt += sg_dma_len(tmp_src);
tmp_src = sg_next(tmp_src);
tmp_dst = sg_next(tmp_dst);
}
} else {
for (i = 0; i < src_nents - 1; i++, cur_lli++, tmp_next_dma += sizeof(*cur_lli)) {
cur_lli->src_addr = sg_dma_address(tmp_src);
cur_lli->src_len = sg_dma_len(tmp_src);
cur_lli->next_addr = tmp_next_dma;
data_cnt += sg_dma_len(tmp_src);
tmp_src = sg_next(tmp_src);
}
}
/* for last lli */
cur_lli->src_addr = sg_dma_address(tmp_src);
cur_lli->src_len = len - data_cnt;
cur_lli->next_addr = 0;
if (dst_sg) {
cur_lli->dst_addr = sg_dma_address(tmp_dst);
cur_lli->dst_len = len - data_cnt;
}
hw_desc->lli_tail = cur_lli;
return 0;
}

40
drivers/crypto/rockchip/rk_crypto_utils.h
View File

@ -5,6 +5,33 @@
#ifndef __RK_CRYPTO_UTILS_H__
#define __RK_CRYPTO_UTILS_H__
#include <linux/dma-mapping.h>
#include <linux/scatterlist.h>
#include "rk_crypto_core.h"
/* Default 256 x 4K = 1MByte */
#define RK_DEFAULT_LLI_CNT 256
struct crypto_lli_desc {
u32 src_addr;
u32 src_len;
u32 dst_addr;
u32 dst_len;
u32 user_define;
u32 reserve;
u32 dma_ctrl;
u32 next_addr;
};
struct rk_hw_desc {
struct device *dev;
struct crypto_lli_desc *lli_head;
struct crypto_lli_desc *lli_tail;
dma_addr_t lli_head_dma;
u32 total;
};
void rk_crypto_write_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, const u8 *data, u32 bytes);
void rk_crypto_clear_regs(struct rk_crypto_dev *rk_dev, u32 base_addr, u32 words);
@ -15,5 +42,18 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents,
struct scatterlist *dst_sg, size_t dst_nents,
int align_mask);
u64 rk_crypto_hw_desc_maxlen(struct scatterlist *sg, u64 len, u32 *max_nents);
int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc);
int rk_crypto_hw_desc_init(struct rk_hw_desc *hw_desc,
struct scatterlist *src_sg,
struct scatterlist *dst_sg,
u64 len);
void rk_crypto_hw_desc_free(struct rk_hw_desc *hw_desc);
void rk_crypto_dump_hw_desc(struct rk_hw_desc *hw_desc);
#endif

1
drivers/crypto/rockchip/rk_crypto_v1.h
View File

@ -23,6 +23,7 @@ struct rk_hw_crypto_v1_info {
.hw_is_algo_valid = rk_hw_crypto_v1_algo_valid,\
.hw_info_size = sizeof(struct rk_hw_crypto_v1_info),\
.default_pka_offset = 0,\
.use_lli_chain = false,\
}
#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V1)

23
drivers/crypto/rockchip/rk_crypto_v2.c
View File

@ -59,21 +59,15 @@ static struct rk_crypto_algt *crypto_v2_algs[] = {
int rk_hw_crypto_v2_init(struct device *dev, void *hw_info)
{
int err = 0;
struct rk_hw_crypto_v2_info *info =
(struct rk_hw_crypto_v2_info *)hw_info;
info->desc = dma_alloc_coherent(dev,
sizeof(struct crypto_lli_desc),
&info->desc_dma,
GFP_KERNEL);
if (!info->desc) {
err = -ENOMEM;
goto end;
}
if (!dev || !hw_info)
return -EINVAL;
end:
return err;
memset(info, 0x00, sizeof(*info));
return rk_crypto_hw_desc_alloc(dev, &info->hw_desc);
}
void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info)
@ -81,9 +75,10 @@ void rk_hw_crypto_v2_deinit(struct device *dev, void *hw_info)
struct rk_hw_crypto_v2_info *info =
(struct rk_hw_crypto_v2_info *)hw_info;
if (info && info->desc)
dma_free_coherent(dev, sizeof(struct crypto_lli_desc),
info->desc, info->desc_dma);
if (!dev || !hw_info)
return;
rk_crypto_hw_desc_free(&info->hw_desc);
}
const char * const *rk_hw_crypto_v2_get_rsts(uint32_t *num)

15
drivers/crypto/rockchip/rk_crypto_v2.h
View File

@ -7,20 +7,10 @@
#include <linux/platform_device.h>
struct crypto_lli_desc {
u32 src_addr;
u32 src_len;
u32 dst_addr;
u32 dst_len;
u32 user_define;
u32 reserve;
u32 dma_ctrl;
u32 next_addr;
};
#include "rk_crypto_utils.h"
struct rk_hw_crypto_v2_info {
struct crypto_lli_desc *desc;
dma_addr_t desc_dma;
struct rk_hw_desc hw_desc;
};
#define RK_CRYPTO_V2_SOC_DATA_INIT(names, soft_aes_192) {\
@ -35,6 +25,7 @@ struct rk_hw_crypto_v2_info {
.hw_is_algo_valid = rk_hw_crypto_v2_algo_valid,\
.hw_info_size = sizeof(struct rk_hw_crypto_v2_info),\
.default_pka_offset = 0x0480,\
.use_lli_chain = true,\
}
#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V2)

40
drivers/crypto/rockchip/rk_crypto_v2_ahash.c
View File

@ -16,6 +16,7 @@
#include "rk_crypto_v2.h"
#include "rk_crypto_v2_reg.h"
#include "rk_crypto_ahash_utils.h"
#include "rk_crypto_utils.h"
#define RK_HASH_CTX_MAGIC 0x1A1A1A1A
#define RK_POLL_PERIOD_US 100
@ -76,14 +77,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
interrupt_status &= CRYPTO_LOCKSTEP_MASK;
if (interrupt_status != CRYPTO_SRC_ITEM_DONE_INT_ST) {
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc);
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head);
dev_err(rk_dev->dev, "DMA addr_in = %08x\n",
(u32)alg_ctx->addr_in);
dev_err(rk_dev->dev, "DMA addr_out = %08x\n",
(u32)alg_ctx->addr_out);
dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count);
dev_err(rk_dev->dev, "DMA desc_dma = %08x\n",
(u32)hw_info->desc_dma);
(u32)hw_info->hw_desc.lli_head_dma);
dev_err(rk_dev->dev, "DMA Error status = %08x\n",
interrupt_status);
dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n",
@ -112,7 +113,7 @@ static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err)
struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(ctx->rk_dev);
struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info;
struct crypto_lli_desc *lli_desc = hw_info->desc;
struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head;
if (err) {
rk_hash_reset(ctx->rk_dev);
@ -212,8 +213,10 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
(struct rk_hw_crypto_v2_info *)rk_dev->hw_info;
struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev);
struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev);
struct crypto_lli_desc *lli_head, *lli_tail;
u32 dma_ctl = CRYPTO_DMA_RESTART;
bool is_final = flag & RK_FLAG_FINAL;
int ret;
CRYPTO_TRACE("ctx->calc_cnt = %u, count %u Byte, is_final = %d",
ctx->calc_cnt, alg_ctx->count, is_final);
@ -230,32 +233,41 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
return 0;
}
memset(hw_info->desc, 0x00, sizeof(*hw_info->desc));
if (alg_ctx->aligned)
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
alg_ctx->sg_src, NULL, alg_ctx->count);
else
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
&alg_ctx->sg_tmp, NULL, alg_ctx->count);
if (ret)
return ret;
hw_info->desc->src_addr = alg_ctx->addr_in;
hw_info->desc->src_len = alg_ctx->count;
hw_info->desc->next_addr = hw_info->desc_dma;
lli_head = hw_info->hw_desc.lli_head;
lli_tail = hw_info->hw_desc.lli_tail;
hw_info->desc->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE;
hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE;
lli_tail->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE;
lli_tail->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE;
lli_tail->next_addr = hw_info->hw_desc.lli_head_dma;
if (ctx->calc_cnt == 0) {
dma_ctl = CRYPTO_DMA_START;
hw_info->desc->user_define |= LLI_USER_CIPHER_START;
hw_info->desc->user_define |= LLI_USER_STRING_START;
lli_head->user_define |= LLI_USER_CIPHER_START;
lli_head->user_define |= LLI_USER_STRING_START;
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL,
(CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) |
CRYPTO_HASH_ENABLE);
}
if (is_final && alg_ctx->left_bytes == 0)
hw_info->desc->user_define |= LLI_USER_STRING_LAST;
lli_tail->user_define |= LLI_USER_STRING_LAST;
CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u",
hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count);
lli_head->dma_ctrl, lli_head->user_define, alg_ctx->count);
rk_crypto_dump_hw_desc(&hw_info->hw_desc);
dma_wmb();

39
drivers/crypto/rockchip/rk_crypto_v2_skcipher.c
View File

@ -48,14 +48,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
interrupt_status &= CRYPTO_LOCKSTEP_MASK;
if (interrupt_status != CRYPTO_DST_ITEM_DONE_INT_ST) {
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc);
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head);
dev_err(rk_dev->dev, "DMA addr_in = %08x\n",
(u32)alg_ctx->addr_in);
dev_err(rk_dev->dev, "DMA addr_out = %08x\n",
(u32)alg_ctx->addr_out);
dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count);
dev_err(rk_dev->dev, "DMA desc_dma = %08x\n",
(u32)hw_info->desc_dma);
(u32)hw_info->hw_desc.lli_head_dma);
dev_err(rk_dev->dev, "DMA Error status = %08x\n",
interrupt_status);
dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n",
@ -153,7 +153,7 @@ static void rk_crypto_complete(struct crypto_async_request *base, int err)
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;
struct rk_hw_crypto_v2_info *hw_info = ctx->rk_dev->hw_info;
struct crypto_lli_desc *lli_desc = hw_info->desc;
struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head;
CRYPTO_WRITE(ctx->rk_dev, CRYPTO_BC_CTL, 0xffff0000);
if (err) {
@ -293,10 +293,22 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
struct skcipher_request *req =
skcipher_request_cast(rk_dev->async_req);
struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);
struct crypto_lli_desc *lli_head, *lli_tail;
u32 calc_len = alg_ctx->count;
u32 start_flag = CRYPTO_DMA_START;
int ret;
memset(hw_info->desc, 0x00, sizeof(*hw_info->desc));
if (alg_ctx->aligned)
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
alg_ctx->sg_src, alg_ctx->sg_dst, alg_ctx->count);
else
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
&alg_ctx->sg_tmp, &alg_ctx->sg_tmp, alg_ctx->count);
if (ret)
return ret;
lli_head = hw_info->hw_desc.lli_head;
lli_tail = hw_info->hw_desc.lli_tail;
/*
* the data length is not aligned will use addr_vir to calculate,
@ -305,19 +317,18 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
if (is_calc_need_round_up(req))
calc_len = round_up(calc_len, alg_ctx->chunk_size);
hw_info->desc->src_addr = alg_ctx->addr_in;
hw_info->desc->src_len = calc_len;
hw_info->desc->dst_addr = alg_ctx->addr_out;
hw_info->desc->dst_len = calc_len;
hw_info->desc->next_addr = 0;
hw_info->desc->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
hw_info->desc->user_define = LLI_USER_STRING_START |
LLI_USER_CIPHER_START |
LLI_USER_STRING_LAST;
lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START;
lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
lli_tail->user_define |= LLI_USER_STRING_LAST;
lli_tail->src_len += (calc_len - alg_ctx->count);
lli_tail->dst_len += (calc_len - alg_ctx->count);
rk_crypto_dump_hw_desc(&hw_info->hw_desc);
dma_wmb();
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK));
return 0;

24
drivers/crypto/rockchip/rk_crypto_v3.c
View File

@ -11,6 +11,7 @@
#include "rk_crypto_core.h"
#include "rk_crypto_v3.h"
#include "rk_crypto_v3_reg.h"
#include "rk_crypto_utils.h"
static const u32 cipher_mode2bit_mask[] = {
[CIPHER_MODE_ECB] = CRYPTO_ECB_FLAG,
@ -160,21 +161,15 @@ static bool rk_is_hash_support(struct rk_crypto_dev *rk_dev, u32 algo, u32 type)
int rk_hw_crypto_v3_init(struct device *dev, void *hw_info)
{
int err = 0;
struct rk_hw_crypto_v3_info *info =
(struct rk_hw_crypto_v3_info *)hw_info;
info->desc = dma_alloc_coherent(dev,
sizeof(struct crypto_lli_desc),
&info->desc_dma,
GFP_KERNEL);
if (!info->desc) {
err = -ENOMEM;
goto end;
}
if (!dev || !hw_info)
return -EINVAL;
end:
return err;
memset(info, 0x00, sizeof(*info));
return rk_crypto_hw_desc_alloc(dev, &info->hw_desc);
}
void rk_hw_crypto_v3_deinit(struct device *dev, void *hw_info)
@ -182,9 +177,10 @@ void rk_hw_crypto_v3_deinit(struct device *dev, void *hw_info)
struct rk_hw_crypto_v3_info *info =
(struct rk_hw_crypto_v3_info *)hw_info;
if (info && info->desc)
dma_free_coherent(dev, sizeof(struct crypto_lli_desc),
info->desc, info->desc_dma);
if (!dev || !hw_info)
return;
rk_crypto_hw_desc_free(&info->hw_desc);
}
const char * const *rk_hw_crypto_v3_get_rsts(uint32_t *num)

6
drivers/crypto/rockchip/rk_crypto_v3.h
View File

@ -7,11 +7,10 @@
#include <linux/platform_device.h>
#include "rk_crypto_v2.h"
#include "rk_crypto_utils.h"
struct rk_hw_crypto_v3_info {
struct crypto_lli_desc *desc;
dma_addr_t desc_dma;
struct rk_hw_desc hw_desc;
};
#define RK_CRYPTO_V3_SOC_DATA_INIT(names) {\
@ -26,6 +25,7 @@ struct rk_hw_crypto_v3_info {
.hw_is_algo_valid = rk_hw_crypto_v3_algo_valid,\
.hw_info_size = sizeof(struct rk_hw_crypto_v3_info),\
.default_pka_offset = 0x0480,\
.use_lli_chain = true,\
}
#if IS_ENABLED(CONFIG_CRYPTO_DEV_ROCKCHIP_V3)

39
drivers/crypto/rockchip/rk_crypto_v3_ahash.c
View File

@ -15,6 +15,7 @@
#include "rk_crypto_v3.h"
#include "rk_crypto_v3_reg.h"
#include "rk_crypto_ahash_utils.h"
#include "rk_crypto_utils.h"
#define RK_HASH_CTX_MAGIC 0x1A1A1A1A
#define RK_POLL_PERIOD_US 100
@ -141,14 +142,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
interrupt_status &= CRYPTO_LOCKSTEP_MASK;
if (interrupt_status != CRYPTO_SRC_ITEM_DONE_INT_ST) {
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc);
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head);
dev_err(rk_dev->dev, "DMA addr_in = %08x\n",
(u32)alg_ctx->addr_in);
dev_err(rk_dev->dev, "DMA addr_out = %08x\n",
(u32)alg_ctx->addr_out);
dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count);
dev_err(rk_dev->dev, "DMA desc_dma = %08x\n",
(u32)hw_info->desc_dma);
(u32)hw_info->hw_desc.lli_head_dma);
dev_err(rk_dev->dev, "DMA Error status = %08x\n",
interrupt_status);
dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n",
@ -177,7 +178,7 @@ static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err)
struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(ctx->rk_dev);
struct rk_hw_crypto_v3_info *hw_info = ctx->rk_dev->hw_info;
struct crypto_lli_desc *lli_desc = hw_info->desc;
struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head;
if (err) {
rk_hash_reset(ctx->rk_dev);
@ -284,8 +285,10 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
(struct rk_hw_crypto_v3_info *)rk_dev->hw_info;
struct rk_alg_ctx *alg_ctx = rk_ahash_alg_ctx(rk_dev);
struct rk_ahash_ctx *ctx = rk_ahash_ctx_cast(rk_dev);
struct crypto_lli_desc *lli_head, *lli_tail;
u32 dma_ctl = CRYPTO_DMA_RESTART;
bool is_final = flag & RK_FLAG_FINAL;
int ret;
CRYPTO_TRACE("ctx->calc_cnt = %u, count %u Byte, is_final = %d",
ctx->calc_cnt, alg_ctx->count, is_final);
@ -305,32 +308,40 @@ static int rk_ahash_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
if (alg_ctx->total == alg_ctx->left_bytes + alg_ctx->count)
rk_hash_mid_data_restore(rk_dev, (struct rk_hash_mid_data *)ctx->priv);
memset(hw_info->desc, 0x00, sizeof(*hw_info->desc));
if (alg_ctx->aligned)
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
alg_ctx->sg_src, NULL, alg_ctx->count);
else
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
&alg_ctx->sg_tmp, NULL, alg_ctx->count);
if (ret)
return ret;
hw_info->desc->src_addr = alg_ctx->addr_in;
hw_info->desc->src_len = alg_ctx->count;
hw_info->desc->next_addr = hw_info->desc_dma;
lli_head = hw_info->hw_desc.lli_head;
lli_tail = hw_info->hw_desc.lli_tail;
hw_info->desc->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE;
hw_info->desc->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE;
lli_tail->dma_ctrl = is_final ? LLI_DMA_CTRL_LAST : LLI_DMA_CTRL_PAUSE;
lli_tail->dma_ctrl |= LLI_DMA_CTRL_SRC_DONE;
if (ctx->calc_cnt == 0) {
dma_ctl = CRYPTO_DMA_START;
hw_info->desc->user_define |= LLI_USER_CIPHER_START;
hw_info->desc->user_define |= LLI_USER_STRING_START;
lli_head->user_define |= LLI_USER_CIPHER_START;
lli_head->user_define |= LLI_USER_STRING_START;
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_HASH_CTL,
(CRYPTO_HASH_ENABLE << CRYPTO_WRITE_MASK_SHIFT) |
CRYPTO_HASH_ENABLE);
}
if (is_final && alg_ctx->left_bytes == 0)
hw_info->desc->user_define |= LLI_USER_STRING_LAST;
lli_tail->user_define |= LLI_USER_STRING_LAST;
CRYPTO_TRACE("dma_ctrl = %08x, user_define = %08x, len = %u",
hw_info->desc->dma_ctrl, hw_info->desc->user_define, alg_ctx->count);
lli_head->dma_ctrl, lli_head->user_define, alg_ctx->count);
rk_crypto_dump_hw_desc(&hw_info->hw_desc);
dma_wmb();

39
drivers/crypto/rockchip/rk_crypto_v3_skcipher.c
View File

@ -47,14 +47,14 @@ static int rk_crypto_irq_handle(int irq, void *dev_id)
interrupt_status &= CRYPTO_LOCKSTEP_MASK;
if (interrupt_status != CRYPTO_DST_ITEM_DONE_INT_ST) {
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->desc);
dev_err(rk_dev->dev, "DMA desc = %p\n", hw_info->hw_desc.lli_head);
dev_err(rk_dev->dev, "DMA addr_in = %08x\n",
(u32)alg_ctx->addr_in);
dev_err(rk_dev->dev, "DMA addr_out = %08x\n",
(u32)alg_ctx->addr_out);
dev_err(rk_dev->dev, "DMA count = %08x\n", alg_ctx->count);
dev_err(rk_dev->dev, "DMA desc_dma = %08x\n",
(u32)hw_info->desc_dma);
(u32)hw_info->hw_desc.lli_head_dma);
dev_err(rk_dev->dev, "DMA Error status = %08x\n",
interrupt_status);
dev_err(rk_dev->dev, "DMA CRYPTO_DMA_LLI_ADDR status = %08x\n",
@ -152,7 +152,7 @@ static void rk_crypto_complete(struct crypto_async_request *base, int err)
struct rk_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm);
struct rk_alg_ctx *alg_ctx = &ctx->algs_ctx;
struct rk_hw_crypto_v3_info *hw_info = ctx->rk_dev->hw_info;
struct crypto_lli_desc *lli_desc = hw_info->desc;
struct crypto_lli_desc *lli_desc = hw_info->hw_desc.lli_head;
CRYPTO_WRITE(ctx->rk_dev, CRYPTO_BC_CTL, 0xffff0000);
if (err) {
@ -292,10 +292,22 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
struct skcipher_request *req =
skcipher_request_cast(rk_dev->async_req);
struct rk_alg_ctx *alg_ctx = rk_cipher_alg_ctx(rk_dev);
struct crypto_lli_desc *lli_head, *lli_tail;
u32 calc_len = alg_ctx->count;
u32 start_flag = CRYPTO_DMA_START;
int ret;
memset(hw_info->desc, 0x00, sizeof(*hw_info->desc));
if (alg_ctx->aligned)
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
alg_ctx->sg_src, alg_ctx->sg_dst, alg_ctx->count);
else
ret = rk_crypto_hw_desc_init(&hw_info->hw_desc,
&alg_ctx->sg_tmp, &alg_ctx->sg_tmp, alg_ctx->count);
if (ret)
return ret;
lli_head = hw_info->hw_desc.lli_head;
lli_tail = hw_info->hw_desc.lli_tail;
/*
* the data length is not aligned will use addr_vir to calculate,
@ -304,19 +316,18 @@ static int crypto_dma_start(struct rk_crypto_dev *rk_dev, uint32_t flag)
if (is_calc_need_round_up(req))
calc_len = round_up(calc_len, alg_ctx->chunk_size);
hw_info->desc->src_addr = alg_ctx->addr_in;
hw_info->desc->src_len = calc_len;
hw_info->desc->dst_addr = alg_ctx->addr_out;
hw_info->desc->dst_len = calc_len;
hw_info->desc->next_addr = 0;
hw_info->desc->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
hw_info->desc->user_define = LLI_USER_STRING_START |
LLI_USER_CIPHER_START |
LLI_USER_STRING_LAST;
lli_head->user_define = LLI_USER_STRING_START | LLI_USER_CIPHER_START;
lli_tail->dma_ctrl = LLI_DMA_CTRL_DST_DONE | LLI_DMA_CTRL_LAST;
lli_tail->user_define |= LLI_USER_STRING_LAST;
lli_tail->src_len += (calc_len - alg_ctx->count);
lli_tail->dst_len += (calc_len - alg_ctx->count);
rk_crypto_dump_hw_desc(&hw_info->hw_desc);
dma_wmb();
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->desc_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_LLI_ADDR, hw_info->hw_desc.lli_head_dma);
CRYPTO_WRITE(rk_dev, CRYPTO_DMA_CTL, start_flag | (start_flag << WRITE_MASK));
return 0;