From 21d41cb9dce678d958959bbc0d0f4c66324261ee Mon Sep 17 00:00:00 2001 From: Wang Xiaobin Date: Fri, 8 Apr 2022 18:21:28 +0800 Subject: [PATCH] crypto: rockchip: cryptodev_linux: add support for aead Since the origin COP_FLAG_AEAD_*_TYPE are not applied to both virt and fd API, we add a new type. Change-Id: I1e896c7de90b2a4cd6053dac14bf20ab8d059ffd Signed-off-by: Wang Xiaobin --- .../crypto/rockchip/cryptodev_linux/authenc.c | 257 ++++++++++++ .../rockchip/cryptodev_linux/cryptlib.c | 2 +- .../rockchip/cryptodev_linux/cryptodev.h | 38 +- .../crypto/rockchip/cryptodev_linux/ioctl.c | 15 + .../rockchip/cryptodev_linux/rk_cryptodev.c | 392 ++++++++++++++++++ .../rockchip/cryptodev_linux/rk_cryptodev.h | 11 + include/uapi/linux/cryptodev.h | 16 +- include/uapi/linux/rk_cryptodev.h | 19 + 8 files changed, 742 insertions(+), 8 deletions(-) diff --git a/drivers/crypto/rockchip/cryptodev_linux/authenc.c b/drivers/crypto/rockchip/cryptodev_linux/authenc.c index fceff0eb04db..afca7f76dcd2 100644 --- a/drivers/crypto/rockchip/cryptodev_linux/authenc.c +++ b/drivers/crypto/rockchip/cryptodev_linux/authenc.c @@ -195,6 +195,9 @@ static int cryptodev_get_dst_len(struct crypt_auth_op *caop, struct csession *se if (caop->op == COP_DECRYPT) return dst_len; + if (caop->flags & COP_FLAG_AEAD_RK_TYPE) + return dst_len; + dst_len += caop->tag_len; /* for TLS always add some padding so the total length is rounded to @@ -301,6 +304,85 @@ int cryptodev_kcaop_to_user(struct kernel_crypt_auth_op *kcaop, return 0; } +/* compatibility code for 32bit userlands */ +#ifdef CONFIG_COMPAT + +static inline void +compat_to_crypt_auth_op(struct compat_crypt_auth_op *compat, struct crypt_auth_op *caop) +{ + caop->ses = compat->ses; + caop->op = compat->op; + caop->flags = compat->flags; + caop->len = compat->len; + caop->auth_len = compat->auth_len; + caop->tag_len = compat->tag_len; + caop->iv_len = compat->iv_len; + + caop->auth_src = compat_ptr(compat->auth_src); + caop->src = compat_ptr(compat->src); + caop->dst = compat_ptr(compat->dst); + caop->tag = compat_ptr(compat->tag); + caop->iv = compat_ptr(compat->iv); +} + +static inline void +crypt_auth_op_to_compat(struct crypt_auth_op *caop, struct compat_crypt_auth_op *compat) +{ + compat->ses = caop->ses; + compat->op = caop->op; + compat->flags = caop->flags; + compat->len = caop->len; + compat->auth_len = caop->auth_len; + compat->tag_len = caop->tag_len; + compat->iv_len = caop->iv_len; + + compat->auth_src = ptr_to_compat(caop->auth_src); + compat->src = ptr_to_compat(caop->src); + compat->dst = ptr_to_compat(caop->dst); + compat->tag = ptr_to_compat(caop->tag); + compat->iv = ptr_to_compat(caop->iv); +} + +int compat_kcaop_from_user(struct kernel_crypt_auth_op *kcaop, + struct fcrypt *fcr, void __user *arg) +{ + int ret; + struct compat_crypt_auth_op compat_auth_cop; + + ret = copy_from_user(&compat_auth_cop, arg, sizeof(compat_auth_cop)); + if (unlikely(ret)) { + derr(1, "Error in copying from userspace"); + return -EFAULT; + } + + compat_to_crypt_auth_op(&compat_auth_cop, &kcaop->caop); + + return fill_kcaop_from_caop(kcaop, fcr); +} + +int compat_kcaop_to_user(struct kernel_crypt_auth_op *kcaop, + struct fcrypt *fcr, void __user *arg) +{ + int ret; + struct compat_crypt_auth_op compat_auth_cop; + + ret = fill_caop_from_kcaop(kcaop, fcr); + if (unlikely(ret)) { + derr(1, "fill_caop_from_kcaop"); + return ret; + } + + crypt_auth_op_to_compat(&kcaop->caop, &compat_auth_cop); + + if (unlikely(copy_to_user(arg, &compat_auth_cop, sizeof(compat_auth_cop)))) { + derr(1, "Error in copying to userspace"); + return -EFAULT; + } + return 0; +} + +#endif /* CONFIG_COMPAT */ + static void copy_tls_hash(struct scatterlist *dst_sg, int len, void *hash, int hash_len) { scatterwalk_map_and_copy(hash, dst_sg, len, hash_len, 1); @@ -559,6 +641,45 @@ srtp_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop, return 0; } +static int rk_auth_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop, + struct scatterlist *auth_sg, uint32_t auth_len, + struct scatterlist *src_sg, + struct scatterlist *dst_sg, uint32_t len) +{ + int ret; + struct crypt_auth_op *caop = &kcaop->caop; + int max_tag_len; + + max_tag_len = cryptodev_cipher_get_tag_size(&ses_ptr->cdata); + if (unlikely(caop->tag_len > max_tag_len)) { + derr(0, "Illegal tag length: %d", caop->tag_len); + return -EINVAL; + } + + if (caop->tag_len) + cryptodev_cipher_set_tag_size(&ses_ptr->cdata, caop->tag_len); + else + caop->tag_len = max_tag_len; + + cryptodev_cipher_auth(&ses_ptr->cdata, auth_sg, auth_len); + + if (caop->op == COP_ENCRYPT) { + ret = cryptodev_cipher_encrypt(&ses_ptr->cdata, src_sg, dst_sg, len); + if (unlikely(ret)) { + derr(0, "cryptodev_cipher_encrypt: %d", ret); + return ret; + } + } else { + ret = cryptodev_cipher_decrypt(&ses_ptr->cdata, src_sg, dst_sg, len); + if (unlikely(ret)) { + derr(0, "cryptodev_cipher_decrypt: %d", ret); + return ret; + } + } + + return 0; +} + /* Typical AEAD (i.e. GCM) encryption/decryption. * During decryption the tag is verified. */ @@ -775,6 +896,139 @@ free_auth_buf: return ret; } +/* Chain two sglists together. It will keep the last nent of priv + * and invalidate the first nent of sgl + */ +static struct scatterlist *sg_copy_chain(struct scatterlist *prv, + unsigned int prv_nents, + struct scatterlist *sgl) +{ + struct scatterlist *sg_tmp = sg_last(prv, prv_nents); + + sg_set_page(sgl, sg_page(sg_tmp), sg_tmp->length, sg_tmp->offset); + + if (prv_nents > 1) { + sg_chain(prv, prv_nents, sgl); + return prv; + } else { + return sgl; + } +} + +static int crypto_auth_zc_rk(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop) +{ + struct scatterlist *dst; + struct scatterlist *src; + struct scatterlist *dst_sg; + struct scatterlist *src_sg; + struct crypt_auth_op *caop = &kcaop->caop; + unsigned char *auth_buf = NULL, *tag_buf = NULL; + struct scatterlist auth_src[2], auth_dst[2], tag[3]; + int ret; + + if (unlikely(ses_ptr->cdata.init == 0 || + (ses_ptr->cdata.stream == 0 && ses_ptr->cdata.aead == 0))) { + derr(0, "Only stream and AEAD ciphers are allowed for authenc"); + return -EINVAL; + } + + if (unlikely(caop->auth_len > PAGE_SIZE)) { + derr(1, "auth data len is excessive."); + return -EINVAL; + } + + ret = cryptodev_get_userbuf(ses_ptr, caop->src, caop->len, + caop->dst, kcaop->dst_len, + kcaop->task, kcaop->mm, &src_sg, &dst_sg); + if (unlikely(ret)) { + derr(1, "get_userbuf(): Error getting user pages."); + ret = -EFAULT; + goto exit; + } + + dst = dst_sg; + src = src_sg; + + /* chain tag */ + if (caop->tag && caop->tag_len > 0) { + tag_buf = kcalloc(caop->tag_len, sizeof(*tag_buf), GFP_KERNEL); + if (unlikely(!tag_buf)) { + derr(1, "unable to kcalloc %d.", caop->tag_len); + ret = -EFAULT; + goto free_pages; + } + + if (unlikely(copy_from_user(tag_buf, caop->tag, caop->tag_len))) { + derr(1, "unable to copy tag data from userspace."); + ret = -EFAULT; + goto free_pages; + } + + sg_init_table(tag, ARRAY_SIZE(tag)); + sg_set_buf(&tag[1], tag_buf, caop->tag_len); + + /* Since the sg_chain() requires the last sg in the list is empty and + * used for link information, we can not directly link src/dst_sg to tags + */ + if (caop->op == COP_ENCRYPT) + dst = sg_copy_chain(dst_sg, sg_nents(dst_sg), tag); + else + src = sg_copy_chain(src_sg, sg_nents(src_sg), tag); + } + + /* chain auth */ + auth_buf = (char *)__get_free_page(GFP_KERNEL); + if (unlikely(!auth_buf)) { + derr(1, "unable to get a free page."); + ret = -EFAULT; + goto free_pages; + } + + if (caop->auth_src && caop->auth_len > 0) { + if (unlikely(copy_from_user(auth_buf, caop->auth_src, caop->auth_len))) { + derr(1, "unable to copy auth data from userspace."); + ret = -EFAULT; + goto free_pages; + } + + sg_init_table(auth_src, ARRAY_SIZE(auth_src)); + sg_set_buf(auth_src, auth_buf, caop->auth_len); + sg_init_table(auth_dst, ARRAY_SIZE(auth_dst)); + sg_set_buf(auth_dst, auth_buf, caop->auth_len); + + sg_chain(auth_src, 2, src); + sg_chain(auth_dst, 2, dst); + src = auth_src; + dst = auth_dst; + } + + if (caop->op == COP_ENCRYPT) + ret = rk_auth_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len, + src, dst, caop->len); + else + ret = rk_auth_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len, + src, dst, caop->len + caop->tag_len); + + if (!ret && caop->op == COP_ENCRYPT) { + if (unlikely(copy_to_user(kcaop->caop.tag, tag_buf, caop->tag_len))) { + derr(1, "Error in copying to userspace"); + ret = -EFAULT; + goto free_pages; + } + } + +free_pages: + cryptodev_release_user_pages(ses_ptr); + +exit: + if (auth_buf) + free_page((unsigned long)auth_buf); + + kfree(tag_buf); + + return ret; +} + static int __crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcaop) { @@ -786,6 +1040,9 @@ __crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcao } else if (caop->flags & COP_FLAG_AEAD_TLS_TYPE && ses_ptr->cdata.aead == 0) { ret = crypto_auth_zc_tls(ses_ptr, kcaop); + } else if (caop->flags & COP_FLAG_AEAD_RK_TYPE && + ses_ptr->cdata.aead) { + ret = crypto_auth_zc_rk(ses_ptr, kcaop); } else if (ses_ptr->cdata.aead) { ret = crypto_auth_zc_aead(ses_ptr, kcaop); } else { diff --git a/drivers/crypto/rockchip/cryptodev_linux/cryptlib.c b/drivers/crypto/rockchip/cryptodev_linux/cryptlib.c index 4be9addefd42..8b2a28c7347b 100644 --- a/drivers/crypto/rockchip/cryptodev_linux/cryptlib.c +++ b/drivers/crypto/rockchip/cryptodev_linux/cryptlib.c @@ -197,7 +197,7 @@ int cryptodev_cipher_init(struct cipher_data *out, const char *alg_name, out->async.as = crypto_alloc_aead(alg_name, 0, 0); if (unlikely(IS_ERR(out->async.as))) { ddebug(1, "Failed to load cipher %s", alg_name); - return -EINVAL; + return PTR_ERR(out->async.as); } out->blocksize = crypto_aead_blocksize(out->async.as); diff --git a/drivers/crypto/rockchip/cryptodev_linux/cryptodev.h b/drivers/crypto/rockchip/cryptodev_linux/cryptodev.h index aae99672fd5a..fd8619db6156 100644 --- a/drivers/crypto/rockchip/cryptodev_linux/cryptodev.h +++ b/drivers/crypto/rockchip/cryptodev_linux/cryptodev.h @@ -76,12 +76,43 @@ struct compat_crypt_op { compat_uptr_t iv;/* initialization vector for encryption operations */ }; +/* input of COMPAT_CIOCAUTHCRYPT */ +struct compat_crypt_auth_op { + uint32_t ses; /* session identifier */ + uint16_t op; /* COP_ENCRYPT or COP_DECRYPT */ + uint16_t flags; /* see COP_FLAG_AEAD_* */ + uint32_t len; /* length of source data */ + uint32_t auth_len; /* length of auth data */ + compat_uptr_t auth_src; /* authenticated-only data */ + + /* The current implementation is more efficient if data are + * encrypted in-place (src== dst). + */ + compat_uptr_t src; /* data to be encrypted and authenticated */ + compat_uptr_t dst; /* pointer to output data. Must have + * space for tag. For TLS this should be at least + * len + tag_size + block_size for padding + */ + + compat_uptr_t tag; /* where the tag will be copied to. TLS mode + * doesn't use that as tag is copied to dst. + * SRTP mode copies tag there. + */ + uint32_t tag_len; /* the length of the tag. Use zero for digest size or max tag. */ + + /* initialization vector for encryption operations */ + compat_uptr_t iv; + uint32_t iv_len; +}; + /* compat ioctls, defined for the above structs */ #define COMPAT_CIOCGSESSION _IOWR('c', 102, struct compat_session_op) #define COMPAT_CIOCCRYPT _IOWR('c', 104, struct compat_crypt_op) #define COMPAT_CIOCASYNCCRYPT _IOW('c', 107, struct compat_crypt_op) #define COMPAT_CIOCASYNCFETCH _IOR('c', 108, struct compat_crypt_op) +#define COMPAT_CIOCAUTHCRYPT _IOWR('c', 109, struct compat_crypt_auth_op) + #endif /* CONFIG_COMPAT */ /* kernel-internal extension to struct crypt_op */ @@ -110,7 +141,12 @@ struct kernel_crypt_auth_op { }; /* auth */ - +#ifdef CONFIG_COMPAT +int compat_kcaop_to_user(struct kernel_crypt_auth_op *kcaop, + struct fcrypt *fcr, void __user *arg); +int compat_kcaop_from_user(struct kernel_crypt_auth_op *kcaop, + struct fcrypt *fcr, void __user *arg); +#endif /* CONFIG_COMPAT */ int cryptodev_kcaop_from_user(struct kernel_crypt_auth_op *kcop, struct fcrypt *fcr, void __user *arg); int cryptodev_kcaop_to_user(struct kernel_crypt_auth_op *kcaop, diff --git a/drivers/crypto/rockchip/cryptodev_linux/ioctl.c b/drivers/crypto/rockchip/cryptodev_linux/ioctl.c index e7ad41a1495d..d98535662187 100644 --- a/drivers/crypto/rockchip/cryptodev_linux/ioctl.c +++ b/drivers/crypto/rockchip/cryptodev_linux/ioctl.c @@ -1148,6 +1148,7 @@ cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_) struct session_op sop; struct compat_session_op compat_sop; struct kernel_crypt_op kcop; + struct kernel_crypt_auth_op kcaop; int ret; if (unlikely(!pcr)) @@ -1190,6 +1191,20 @@ cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_) return ret; return compat_kcop_to_user(&kcop, fcr, arg); + + case COMPAT_CIOCAUTHCRYPT: + ret = compat_kcaop_from_user(&kcaop, fcr, arg); + if (unlikely(ret)) { + dwarning(1, "Error copying from user"); + return ret; + } + + ret = crypto_auth_run(fcr, &kcaop); + if (unlikely(ret)) { + dwarning(1, "Error in crypto_auth_run"); + return ret; + } + return compat_kcaop_to_user(&kcaop, fcr, arg); #ifdef ENABLE_ASYNC case COMPAT_CIOCASYNCCRYPT: if (unlikely(ret = compat_kcop_from_user(&kcop, fcr, arg))) diff --git a/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c b/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c index c00a53916edd..26198895fd3e 100644 --- a/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c +++ b/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c @@ -733,12 +733,390 @@ exit: return ret; } +/* Typical AEAD (i.e. GCM) encryption/decryption. + * During decryption the tag is verified. + */ +static int rk_auth_fd_n_crypt(struct csession *ses_ptr, struct kernel_crypt_auth_fd_op *kcaop, + struct scatterlist *auth_sg, uint32_t auth_len, + struct scatterlist *src_sg, + struct scatterlist *dst_sg, uint32_t len) +{ + int ret; + struct crypt_auth_fd_op *caop = &kcaop->caop; + int max_tag_len; + + max_tag_len = cryptodev_cipher_get_tag_size(&ses_ptr->cdata); + if (unlikely(caop->tag_len > max_tag_len)) { + derr(0, "Illegal tag length: %d", caop->tag_len); + return -EINVAL; + } + + if (caop->tag_len) + cryptodev_cipher_set_tag_size(&ses_ptr->cdata, caop->tag_len); + else + caop->tag_len = max_tag_len; + + cryptodev_cipher_auth(&ses_ptr->cdata, auth_sg, auth_len); + + if (caop->op == COP_ENCRYPT) { + ret = cryptodev_cipher_encrypt(&ses_ptr->cdata, + src_sg, dst_sg, len); + if (unlikely(ret)) { + derr(0, "cryptodev_cipher_encrypt: %d", ret); + return ret; + } + } else { + ret = cryptodev_cipher_decrypt(&ses_ptr->cdata, + src_sg, dst_sg, len); + + if (unlikely(ret)) { + derr(0, "cryptodev_cipher_decrypt: %d", ret); + return ret; + } + } + + return 0; +} + +static void sg_init_table_set_page(struct scatterlist *sgl_dst, unsigned int nents_dst, + struct scatterlist *sgl_src, unsigned int len) +{ + sg_init_table(sgl_dst, nents_dst); + sg_set_page(sgl_dst, sg_page(sgl_src), len, sgl_src->offset); + + sg_dma_address(sgl_dst) = sg_dma_address(sgl_src); + sg_dma_len(sgl_dst) = len; +} + +/* This is the main crypto function - zero-copy edition */ +static int crypto_auth_fd_zc_rk(struct fcrypt *fcr, struct csession *ses_ptr, + struct kernel_crypt_auth_fd_op *kcaop) +{ + struct crypt_auth_fd_op *caop = &kcaop->caop; + struct dma_buf *dma_buf_in = NULL, *dma_buf_out = NULL, *dma_buf_auth = NULL; + struct sg_table *sg_tbl_in = NULL, *sg_tbl_out = NULL, *sg_tbl_auth = NULL; + struct dma_buf_attachment *dma_attach_in = NULL, *dma_attach_out = NULL; + struct dma_buf_attachment *dma_attach_auth = NULL; + struct dma_fd_map_node *node_src = NULL, *node_dst = NULL, *node_auth = NULL; + struct scatterlist *dst_sg, *src_sg; + struct scatterlist auth_src[2], auth_dst[2], src[2], dst[2], tag[2]; + unsigned char *tag_buf = NULL; + int ret = 0; + + node_src = dma_fd_find_node(fcr, caop->src_fd); + if (node_src) { + sg_tbl_in = node_src->sgtbl; + } else { + ret = get_dmafd_sgtbl(caop->src_fd, caop->len, DMA_TO_DEVICE, + &sg_tbl_in, &dma_attach_in, &dma_buf_in); + if (unlikely(ret)) { + derr(1, "Error get_dmafd_sgtbl src."); + goto exit; + } + } + + node_dst = dma_fd_find_node(fcr, caop->dst_fd); + if (node_dst) { + sg_tbl_out = node_dst->sgtbl; + } else { + ret = get_dmafd_sgtbl(caop->dst_fd, caop->len, DMA_FROM_DEVICE, + &sg_tbl_out, &dma_attach_out, &dma_buf_out); + if (unlikely(ret)) { + derr(1, "Error get_dmafd_sgtbl dst."); + goto exit; + } + } + + src_sg = sg_tbl_in->sgl; + dst_sg = sg_tbl_out->sgl; + + if (caop->auth_len > 0) { + node_auth = dma_fd_find_node(fcr, caop->auth_fd); + if (node_auth) { + sg_tbl_auth = node_auth->sgtbl; + } else { + ret = get_dmafd_sgtbl(caop->auth_fd, caop->auth_len, DMA_TO_DEVICE, + &sg_tbl_auth, &dma_attach_auth, &dma_buf_auth); + if (unlikely(ret)) { + derr(1, "Error get_dmafd_sgtbl auth."); + goto exit; + } + } + + sg_init_table_set_page(auth_src, ARRAY_SIZE(auth_src), + sg_tbl_auth->sgl, caop->auth_len); + + sg_init_table_set_page(auth_dst, ARRAY_SIZE(auth_dst), + sg_tbl_auth->sgl, caop->auth_len); + + sg_init_table_set_page(src, ARRAY_SIZE(src), + sg_tbl_in->sgl, caop->len); + + sg_init_table_set_page(dst, ARRAY_SIZE(dst), + sg_tbl_out->sgl, caop->len); + + sg_chain(auth_src, 2, src); + sg_chain(auth_dst, 2, dst); + src_sg = auth_src; + dst_sg = auth_dst; + } + + /* get tag */ + if (caop->tag && caop->tag_len > 0) { + tag_buf = kcalloc(caop->tag_len, sizeof(*tag_buf), GFP_KERNEL); + if (unlikely(!tag_buf)) { + derr(1, "unable to kcalloc %d.", caop->tag_len); + ret = -EFAULT; + goto exit; + } + + ret = copy_from_user(tag_buf, u64_to_user_ptr((u64)caop->tag), caop->tag_len); + if (unlikely(ret)) { + derr(1, "unable to copy tag data from userspace."); + ret = -EFAULT; + goto exit; + } + + sg_init_table(tag, 2); + sg_set_buf(tag, tag_buf, caop->tag_len); + + if (caop->op == COP_ENCRYPT) + sg_chain(dst, 2, tag); + else + sg_chain(src, 2, tag); + } + + if (caop->op == COP_ENCRYPT) + ret = rk_auth_fd_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len, + src_sg, dst_sg, caop->len); + else + ret = rk_auth_fd_n_crypt(ses_ptr, kcaop, NULL, caop->auth_len, + src_sg, dst_sg, caop->len + caop->tag_len); + + if (!ret && caop->op == COP_ENCRYPT && tag_buf) { + ret = copy_to_user(u64_to_user_ptr((u64)kcaop->caop.tag), tag_buf, caop->tag_len); + if (unlikely(ret)) { + derr(1, "Error in copying to userspace"); + ret = -EFAULT; + goto exit; + } + } + +exit: + kfree(tag_buf); + + if (dma_buf_in) + put_dmafd_sgtbl(caop->src_fd, DMA_TO_DEVICE, + sg_tbl_in, dma_attach_in, dma_buf_in); + + if (dma_buf_out) + put_dmafd_sgtbl(caop->dst_fd, DMA_FROM_DEVICE, + sg_tbl_out, dma_attach_out, dma_buf_out); + + if (dma_buf_auth) + put_dmafd_sgtbl(caop->auth_fd, DMA_FROM_DEVICE, + sg_tbl_auth, dma_attach_auth, dma_buf_auth); + + return ret; +} + +static int __crypto_auth_fd_run_zc(struct fcrypt *fcr, struct csession *ses_ptr, + struct kernel_crypt_auth_fd_op *kcaop) +{ + struct crypt_auth_fd_op *caop = &kcaop->caop; + int ret; + + if (caop->flags & COP_FLAG_AEAD_RK_TYPE) + ret = crypto_auth_fd_zc_rk(fcr, ses_ptr, kcaop); + else + ret = -EINVAL; /* other types, not implemented */ + + return ret; +} + +static int crypto_auth_fd_run(struct fcrypt *fcr, struct kernel_crypt_auth_fd_op *kcaop) +{ + struct csession *ses_ptr; + struct crypt_auth_fd_op *caop = &kcaop->caop; + int ret = -EINVAL; + + if (unlikely(caop->op != COP_ENCRYPT && caop->op != COP_DECRYPT)) { + ddebug(1, "invalid operation op=%u", caop->op); + return -EINVAL; + } + + /* this also enters ses_ptr->sem */ + ses_ptr = crypto_get_session_by_sid(fcr, caop->ses); + if (unlikely(!ses_ptr)) { + derr(1, "invalid session ID=0x%08X", caop->ses); + return -EINVAL; + } + + if (unlikely(ses_ptr->cdata.init == 0)) { + derr(1, "cipher context not initialized"); + ret = -EINVAL; + goto out_unlock; + } + + /* If we have a hash/mac handle reset its state */ + if (ses_ptr->hdata.init != 0) { + ret = cryptodev_hash_reset(&ses_ptr->hdata); + if (unlikely(ret)) { + derr(1, "error in cryptodev_hash_reset()"); + goto out_unlock; + } + } + + cryptodev_cipher_set_iv(&ses_ptr->cdata, kcaop->iv, + min(ses_ptr->cdata.ivsize, kcaop->ivlen)); + + ret = __crypto_auth_fd_run_zc(fcr, ses_ptr, kcaop); + if (unlikely(ret)) { + derr(1, "error in __crypto_auth_fd_run_zc()"); + goto out_unlock; + } + + ret = 0; + + cryptodev_cipher_get_iv(&ses_ptr->cdata, kcaop->iv, + min(ses_ptr->cdata.ivsize, kcaop->ivlen)); + +out_unlock: + crypto_put_session(ses_ptr); + return ret; +} + +/* + * Return tag (digest) length for authenticated encryption + * If the cipher and digest are separate, hdata.init is set - just return + * digest length. Otherwise return digest length for aead ciphers + */ +static int rk_cryptodev_get_tag_len(struct csession *ses_ptr) +{ + if (ses_ptr->hdata.init) + return ses_ptr->hdata.digestsize; + else + return cryptodev_cipher_get_tag_size(&ses_ptr->cdata); +} + +/* + * Calculate destination buffer length for authenticated encryption. The + * expectation is that user-space code allocates exactly the same space for + * destination buffer before calling cryptodev. The result is cipher-dependent. + */ +static int rk_cryptodev_fd_get_dst_len(struct crypt_auth_fd_op *caop, struct csession *ses_ptr) +{ + int dst_len = caop->len; + + if (caop->op == COP_DECRYPT) + return dst_len; + + dst_len += caop->tag_len; + + /* for TLS always add some padding so the total length is rounded to + * cipher block size + */ + if (caop->flags & COP_FLAG_AEAD_TLS_TYPE) { + int bs = ses_ptr->cdata.blocksize; + + dst_len += bs - (dst_len % bs); + } + + return dst_len; +} + +static int fill_kcaop_fd_from_caop(struct kernel_crypt_auth_fd_op *kcaop, struct fcrypt *fcr) +{ + struct crypt_auth_fd_op *caop = &kcaop->caop; + struct csession *ses_ptr; + int ret; + + /* this also enters ses_ptr->sem */ + ses_ptr = crypto_get_session_by_sid(fcr, caop->ses); + if (unlikely(!ses_ptr)) { + derr(1, "invalid session ID=0x%08X", caop->ses); + return -EINVAL; + } + + if (caop->tag_len == 0) + caop->tag_len = rk_cryptodev_get_tag_len(ses_ptr); + + kcaop->ivlen = caop->iv ? ses_ptr->cdata.ivsize : 0; + kcaop->dst_len = rk_cryptodev_fd_get_dst_len(caop, ses_ptr); + kcaop->task = current; + kcaop->mm = current->mm; + + if (caop->iv) { + ret = copy_from_user(kcaop->iv, u64_to_user_ptr((u64)caop->iv), kcaop->ivlen); + if (unlikely(ret)) { + derr(1, "error copy_from_user IV (%d bytes) returned %d for address %llu", + kcaop->ivlen, ret, caop->iv); + ret = -EFAULT; + goto out_unlock; + } + } + + ret = 0; + +out_unlock: + crypto_put_session(ses_ptr); + return ret; +} + +static int fill_caop_fd_from_kcaop(struct kernel_crypt_auth_fd_op *kcaop, struct fcrypt *fcr) +{ + int ret; + + kcaop->caop.len = kcaop->dst_len; + + if (kcaop->ivlen && kcaop->caop.flags & COP_FLAG_WRITE_IV) { + ret = copy_to_user(u64_to_user_ptr((u64)kcaop->caop.iv), kcaop->iv, kcaop->ivlen); + if (unlikely(ret)) { + derr(1, "Error in copying iv to userspace"); + return -EFAULT; + } + } + + return 0; +} + +static int kcaop_fd_from_user(struct kernel_crypt_auth_fd_op *kcaop, + struct fcrypt *fcr, void __user *arg) +{ + if (unlikely(copy_from_user(&kcaop->caop, arg, sizeof(kcaop->caop)))) { + derr(1, "Error in copying from userspace"); + return -EFAULT; + } + + return fill_kcaop_fd_from_caop(kcaop, fcr); +} + +static int kcaop_fd_to_user(struct kernel_crypt_auth_fd_op *kcaop, + struct fcrypt *fcr, void __user *arg) +{ + int ret; + + ret = fill_caop_fd_from_kcaop(kcaop, fcr); + if (unlikely(ret)) { + derr(1, "Error in fill_caop_from_kcaop"); + return ret; + } + + if (unlikely(copy_to_user(arg, &kcaop->caop, sizeof(kcaop->caop)))) { + derr(1, "Cannot copy to userspace"); + return -EFAULT; + } + + return 0; +} + long rk_cryptodev_ioctl(struct fcrypt *fcr, unsigned int cmd, unsigned long arg_) { struct kernel_crypt_fd_op kcop; struct kernel_crypt_fd_map_op kmop; struct kernel_crypt_rsa_op krop; + struct kernel_crypt_auth_fd_op kcaop; void __user *arg = (void __user *)arg_; int ret; @@ -757,6 +1135,20 @@ rk_cryptodev_ioctl(struct fcrypt *fcr, unsigned int cmd, unsigned long arg_) } return kcop_fd_to_user(&kcop, fcr, arg); + case RIOCAUTHCRYPT_FD: + ret = kcaop_fd_from_user(&kcaop, fcr, arg); + if (unlikely(ret)) { + dwarning(1, "Error copying from user"); + return ret; + } + + ret = crypto_auth_fd_run(fcr, &kcaop); + if (unlikely(ret)) { + dwarning(1, "Error in crypto_run"); + return ret; + } + + return kcaop_fd_to_user(&kcaop, fcr, arg); case RIOCCRYPT_FD_MAP: ret = kcop_map_fd_from_user(&kmop, fcr, arg); if (unlikely(ret)) { diff --git a/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.h b/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.h index fedbcfd61772..dff499be3ccb 100644 --- a/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.h +++ b/drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.h @@ -55,6 +55,17 @@ struct kernel_crypt_fd_op { struct mm_struct *mm; }; +struct kernel_crypt_auth_fd_op { + struct crypt_auth_fd_op caop; + + int dst_len; /* based on src_len */ + __u8 iv[EALG_MAX_BLOCK_LEN]; + int ivlen; + + struct task_struct *task; + struct mm_struct *mm; +}; + /* kernel-internal extension to struct crypt_fd_map_op */ struct kernel_crypt_fd_map_op { struct crypt_fd_map_op mop; diff --git a/include/uapi/linux/cryptodev.h b/include/uapi/linux/cryptodev.h index c634421b963f..0b81c3c2dd13 100644 --- a/include/uapi/linux/cryptodev.h +++ b/include/uapi/linux/cryptodev.h @@ -194,8 +194,9 @@ struct crypt_auth_op { * encrypted in-place (src==dst). */ __u8 __user *src; /* data to be encrypted and authenticated */ __u8 __user *dst; /* pointer to output data. Must have - * space for tag. For TLS this should be at least - * len + tag_size + block_size for padding */ + * space for tag. For TLS this should be at least + * len + tag_size + block_size for padding + */ __u8 __user *tag; /* where the tag will be copied to. TLS mode * doesn't use that as tag is copied to dst. @@ -223,7 +224,7 @@ struct crypt_auth_op { * copies the tag just after data. */ -/* In TLS mode (used for CBC ciphers that required padding) +/* In TLS mode (used for CBC ciphers that required padding) * the following are required: * flags : COP_FLAG_AEAD_TLS_TYPE * iv : the initialization vector @@ -244,7 +245,7 @@ struct crypt_auth_op { * iv : the initialization vector * auth_len: the length of the data to be authenticated. This must * include the SRTP header + SRTP payload (data to be encrypted) + rest - * + * * len : length of data to be encrypted * auth_src: pointer the data to be authenticated. Should point at the same buffer as src. * src : pointer to the data to be encrypted. @@ -262,13 +263,16 @@ struct crypt_auth_op { #define COP_FLAG_FINAL (1 << 1) /* multi-update final hash mode */ #define COP_FLAG_WRITE_IV (1 << 2) /* update the IV during operation */ #define COP_FLAG_NO_ZC (1 << 3) /* do not zero-copy */ -#define COP_FLAG_AEAD_TLS_TYPE (1 << 4) /* authenticate and encrypt using the +#define COP_FLAG_AEAD_TLS_TYPE (1 << 4) /* authenticate and encrypt using the * TLS protocol rules */ -#define COP_FLAG_AEAD_SRTP_TYPE (1 << 5) /* authenticate and encrypt using the +#define COP_FLAG_AEAD_SRTP_TYPE (1 << 5) /* authenticate and encrypt using the * SRTP protocol rules */ #define COP_FLAG_RESET (1 << 6) /* multi-update reset the state. * should be used in combination * with COP_FLAG_UPDATE */ +#define COP_FLAG_AEAD_RK_TYPE (1 << 11) /* authenticate and encrypt using the + * rock-chips define rules + */ /* Stuff for bignum arithmetic and public key diff --git a/include/uapi/linux/rk_cryptodev.h b/include/uapi/linux/rk_cryptodev.h index 3a8be3c89d92..d69d129f2c66 100644 --- a/include/uapi/linux/rk_cryptodev.h +++ b/include/uapi/linux/rk_cryptodev.h @@ -28,6 +28,24 @@ struct crypt_fd_op { __u8 __user *iv; }; +/* input of RIOCAUTHCRYPT_FD */ +struct crypt_auth_fd_op { + __u32 ses; /* session identifier */ + __u16 op; /* COP_ENCRYPT or COP_DECRYPT */ + __u16 flags; /* see COP_FLAG_AEAD_* */ + __u32 len; /* length of source data */ + __u32 auth_len; /* length of auth data */ + int auth_fd; /* authenticated-only data */ + int src_fd; /* source data */ + int dst_fd; /* pointer to output data */ + __u64 tag; + __u32 tag_len; /* the length of the tag. Use zero for digest size or max + * tag. + */ + __u64 iv; /* initialization vector for encryption operations */ + __u32 iv_len; +}; + /* input of RIOCCRYPT_FD_MAP/RIOCCRYPT_FD_UNMAP */ struct crypt_fd_map_op { int dma_fd; /* session identifier */ @@ -63,5 +81,6 @@ struct crypt_rsa_op { #define RIOCCRYPT_CPU_ACCESS _IOW('r', 107, struct crypt_fd_map_op) #define RIOCCRYPT_DEV_ACCESS _IOW('r', 108, struct crypt_fd_map_op) #define RIOCCRYPT_RSA_CRYPT _IOWR('r', 109, struct crypt_rsa_op) +#define RIOCAUTHCRYPT_FD _IOWR('r', 110, struct crypt_auth_fd_op) #endif