github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-07 22:14:04 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

crypto: rockchip: Optimized the DMA_FD cache flushing policy

Browse Source 
Reduce unnecessary DMA_FD flusher to improve performance.

Signed-off-by: Lin Jinhan <troy.lin@rock-chips.com>
Change-Id: I22f9a7c8373419b74e2e250517b5743034043728
This commit is contained in:
Lin Jinhan 2022-04-21 11:29:13 +08:00 committed by Tao Huang
parent 813cc8110a
commit 9044c677ec
6 changed files with 67 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
drivers/crypto/rockchip/cryptodev_linux/rk_cryptodev.c
View File

@ -262,6 +262,10 @@ static int get_dmafd_sgtbl(int dma_fd, unsigned int dma_len, enum dma_data_direc
goto error;
}
/*
* DMA_TO_DEVICE : cache clean for input data
* DMA_FROM_DEVICE: cache invalidate for output data
*/
*sg_tbl = dma_buf_map_attachment(*dma_attach, dir);
if (IS_ERR(*sg_tbl)) {
derr(1, "sg_tbl error! ret = %d", (int)PTR_ERR(*sg_tbl));
@ -269,8 +273,9 @@ static int get_dmafd_sgtbl(int dma_fd, unsigned int dma_len, enum dma_data_direc
goto error;
}
/* insure user data flush to ddr */
dma_sync_sg_for_cpu(crypto_dev, (*sg_tbl)->sgl, (*sg_tbl)->nents, DMA_FROM_DEVICE);
/* cache invalidate for input data */
if (dir == DMA_TO_DEVICE)
dma_sync_sg_for_cpu(crypto_dev, (*sg_tbl)->sgl, (*sg_tbl)->nents, DMA_FROM_DEVICE);
return 0;
error:
@ -298,9 +303,14 @@ static int put_dmafd_sgtbl(int dma_fd, enum dma_data_direction dir,
if (!sg_tbl || !dma_attach || !dmabuf)
return -EINVAL;
/* insure ddr data flush to cache */
dma_sync_sg_for_device(crypto_dev, sg_tbl->sgl, sg_tbl->nents, DMA_TO_DEVICE);
/* cache clean for output data */
if (dir == DMA_FROM_DEVICE)
dma_sync_sg_for_device(crypto_dev, sg_tbl->sgl, sg_tbl->nents, DMA_TO_DEVICE);
/*
* DMA_TO_DEVICE : do nothing for input data
* DMA_FROM_DEVICE: cache invalidate for output data
*/
dma_buf_unmap_attachment(dma_attach, sg_tbl, dir);
dma_buf_detach(dmabuf, dma_attach);
dma_buf_put(dmabuf);
@ -914,7 +924,7 @@ static int crypto_auth_fd_zc_rk(struct fcrypt *fcr, struct csession *ses_ptr,
sg_tbl_out, dma_attach_out, dma_buf_out);
if (dma_buf_auth)
put_dmafd_sgtbl(caop->auth_fd, DMA_FROM_DEVICE,
put_dmafd_sgtbl(caop->auth_fd, DMA_TO_DEVICE,
sg_tbl_auth, dma_attach_auth, dma_buf_auth);
return ret;

15
drivers/crypto/rockchip/rk_crypto_ahash_utils.c
View File

@ -348,6 +348,18 @@ int rk_ahash_start(struct rk_crypto_dev *rk_dev)
/* Concatenate old data to the header */
sg_init_table(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg));
sg_set_buf(ctx->hash_sg, ctx->hash_tmp, ctx->hash_tmp_len);
if (rk_crypto_check_dmafd(req->src, sg_nents_for_len(req->src, req->nbytes))) {
CRYPTO_TRACE("is hash dmafd");
if (!dma_map_sg(rk_dev->dev, &ctx->hash_sg[0], 1, DMA_TO_DEVICE)) {
dev_err(rk_dev->dev, "[%s:%d] dma_map_sg(hash_sg) error\n",
__func__, __LINE__);
ret = -ENOMEM;
goto exit;
}
ctx->hash_tmp_mapped = true;
}
sg_chain(ctx->hash_sg, ARRAY_SIZE(ctx->hash_sg), req->src);
src_sg = &ctx->hash_sg[0];
@ -418,6 +430,9 @@ int rk_ahash_crypto_rx(struct rk_crypto_dev *rk_dev)
*/
struct crypto_ahash *tfm;
if (ctx->hash_tmp_mapped)
dma_unmap_sg(rk_dev->dev, &ctx->hash_sg[0], 1, DMA_TO_DEVICE);
/* only final will get result */
if (!(rctx->flag & RK_FLAG_FINAL))
goto out_rx;

24
drivers/crypto/rockchip/rk_crypto_core.c
View File

@ -117,12 +117,16 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
CRYPTO_TRACE("src_nents = %u, dst_nents = %u", src_nents, dst_nents);
}
if (alg_ctx->left_bytes == alg_ctx->total)
if (alg_ctx->left_bytes == alg_ctx->total) {
alg_ctx->aligned = rk_crypto_check_align(sg_src, src_nents, sg_dst, dst_nents,
alg_ctx->align_size);
alg_ctx->is_dma = rk_crypto_check_dmafd(sg_src, src_nents) &&
rk_crypto_check_dmafd(sg_dst, dst_nents);
}
CRYPTO_TRACE("aligned = %d, total = %u, left_bytes = %u, assoclen = %u\n",
alg_ctx->aligned, alg_ctx->total, alg_ctx->left_bytes, alg_ctx->assoclen);
CRYPTO_TRACE("aligned = %d, is_dma = %d, total = %u, left_bytes = %u, assoclen = %u\n",
alg_ctx->aligned, alg_ctx->is_dma, alg_ctx->total,
alg_ctx->left_bytes, alg_ctx->assoclen);
if (alg_ctx->aligned) {
u32 nents;
@ -137,7 +141,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
alg_ctx->map_nents = nents;
alg_ctx->left_bytes -= count;
if (!dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) {
if (!alg_ctx->is_dma && !dma_map_sg(dev, sg_src, nents, DMA_TO_DEVICE)) {
dev_err(dev, "[%s:%d] dma_map_sg(src) error\n",
__func__, __LINE__);
ret = -EINVAL;
@ -146,7 +150,7 @@ static int rk_load_data(struct rk_crypto_dev *rk_dev,
alg_ctx->addr_in = sg_dma_address(sg_src);
if (sg_dst) {
if (!dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) {
if (!alg_ctx->is_dma && !dma_map_sg(dev, sg_dst, nents, DMA_FROM_DEVICE)) {
dev_err(dev,
"[%s:%d] dma_map_sg(dst) error\n",
__func__, __LINE__);
@ -219,11 +223,17 @@ static int rk_unload_data(struct rk_crypto_dev *rk_dev)
nents = alg_ctx->map_nents;
sg_in = alg_ctx->aligned ? alg_ctx->sg_src : &alg_ctx->sg_tmp;
dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE);
/* only is dma buffer and aligned will skip unmap */
if (!alg_ctx->is_dma || !alg_ctx->aligned)
dma_unmap_sg(rk_dev->dev, sg_in, nents, DMA_TO_DEVICE);
if (alg_ctx->sg_dst) {
sg_out = alg_ctx->aligned ? alg_ctx->sg_dst : &alg_ctx->sg_tmp;
dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE);
/* only is dma buffer and aligned will skip unmap */
if (!alg_ctx->is_dma || !alg_ctx->aligned)
dma_unmap_sg(rk_dev->dev, sg_out, nents, DMA_FROM_DEVICE);
}
if (!alg_ctx->aligned && alg_ctx->req_dst) {

2
drivers/crypto/rockchip/rk_crypto_core.h
View File

@ -140,6 +140,7 @@ struct rk_alg_ctx {
dma_addr_t addr_aad_in;
bool aligned;
bool is_dma;
int align_size;
int chunk_size;
};
@ -153,6 +154,7 @@ struct rk_ahash_ctx {
struct scatterlist hash_sg[2];
u8 *hash_tmp;
u32 hash_tmp_len;
bool hash_tmp_mapped;
u32 calc_cnt;
u8 lastc[RK_DMA_ALIGNMENT];

16
drivers/crypto/rockchip/rk_crypto_utils.c
View File

@ -116,6 +116,22 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents,
return true;
}
bool rk_crypto_check_dmafd(struct scatterlist *sgl, size_t nents)
{
struct scatterlist *src_tmp = NULL;
unsigned int i;
for_each_sg(sgl, src_tmp, nents, i) {
if (!src_tmp)
return false;
if (src_tmp->length && !sg_dma_address(src_tmp))
return false;
}
return true;
}
void rk_crypto_dump_hw_desc(struct rk_hw_desc *hw_desc)
{
struct crypto_lli_desc *cur_lli = NULL;

2
drivers/crypto/rockchip/rk_crypto_utils.h
View File

@ -44,6 +44,8 @@ bool rk_crypto_check_align(struct scatterlist *src_sg, size_t src_nents,
struct scatterlist *dst_sg, size_t dst_nents,
int align_mask);
bool rk_crypto_check_dmafd(struct scatterlist *sgl, size_t nents);
u64 rk_crypto_hw_desc_maxlen(struct scatterlist *sg, u64 len, u32 *max_nents);
int rk_crypto_hw_desc_alloc(struct device *dev, struct rk_hw_desc *hw_desc);