github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-04 20:46:48 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

scsi: iscsi_tcp: Switch to using the crc32c library

Browse Source 
Now that the crc32c() library function directly takes advantage of
architecture-specific optimizations, it is unnecessary to go through the
crypto API.  Just use crc32c().  This is much simpler, and it improves
performance due to eliminating the crypto API overhead.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Link: https://lore.kernel.org/r/20250207041724.70733-1-ebiggers@kernel.org
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
This commit is contained in:
Eric Biggers 2025-02-06 20:17:24 -08:00 committed by Martin K. Petersen
parent 035b9fa023
commit 92186c1455
5 changed files with 56 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/scsi/Kconfig
View File

@ -303,9 +303,9 @@ if SCSI_LOWLEVEL && SCSI
config ISCSI_TCP config ISCSI_TCP
tristate "iSCSI Initiator over TCP/IP" tristate "iSCSI Initiator over TCP/IP"
depends on SCSI && INET depends on SCSI && INET
select CRC32
select CRYPTO select CRYPTO
select CRYPTO_MD5 select CRYPTO_MD5
select CRYPTO_CRC32C
select SCSI_ISCSI_ATTRS select SCSI_ISCSI_ATTRS
help help
The iSCSI Driver provides a host with the ability to access storage The iSCSI Driver provides a host with the ability to access storage

60
drivers/scsi/iscsi_tcp.c
View File

@ -17,7 +17,6 @@
* Zhenyu Wang * Zhenyu Wang
*/ */
#include <crypto/hash.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/inet.h> #include <linux/inet.h>
#include <linux/slab.h> #include <linux/slab.h>
@ -468,8 +467,7 @@ static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr,
* sufficient room. * sufficient room.
*/ */
if (conn->hdrdgst_en) { if (conn->hdrdgst_en) {
iscsi_tcp_dgst_header(tcp_sw_conn->tx_hash, hdr, hdrlen, iscsi_tcp_dgst_header(hdr, hdrlen, hdr + hdrlen);
hdr + hdrlen);
hdrlen += ISCSI_DIGEST_SIZE; hdrlen += ISCSI_DIGEST_SIZE;
} }
@ -494,7 +492,7 @@ iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
{ {
struct iscsi_tcp_conn *tcp_conn = conn->dd_data; struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct ahash_request *tx_hash = NULL; u32 *tx_crcp = NULL;
unsigned int hdr_spec_len; unsigned int hdr_spec_len;
ISCSI_SW_TCP_DBG(conn, "offset=%d, datalen=%d %s\n", offset, len, ISCSI_SW_TCP_DBG(conn, "offset=%d, datalen=%d %s\n", offset, len,
@ -507,11 +505,10 @@ iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
if (conn->datadgst_en) if (conn->datadgst_en)
tx_hash = tcp_sw_conn->tx_hash; tx_crcp = &tcp_sw_conn->tx_crc;
return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment, return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment,
sg, count, offset, len, sg, count, offset, len, NULL, tx_crcp);
NULL, tx_hash);
} }
static void static void
@ -520,7 +517,7 @@ iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data,
{ {
struct iscsi_tcp_conn *tcp_conn = conn->dd_data; struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
struct ahash_request *tx_hash = NULL; u32 *tx_crcp = NULL;
unsigned int hdr_spec_len; unsigned int hdr_spec_len;
ISCSI_SW_TCP_DBG(conn, "datalen=%zd %s\n", len, conn->datadgst_en ? ISCSI_SW_TCP_DBG(conn, "datalen=%zd %s\n", len, conn->datadgst_en ?
@ -532,10 +529,10 @@ iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data,
WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
if (conn->datadgst_en) if (conn->datadgst_en)
tx_hash = tcp_sw_conn->tx_hash; tx_crcp = &tcp_sw_conn->tx_crc;
iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment, iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment,
data, len, NULL, tx_hash); data, len, NULL, tx_crcp);
} }
static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task, static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task,
@ -583,7 +580,6 @@ iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session,
struct iscsi_cls_conn *cls_conn; struct iscsi_cls_conn *cls_conn;
struct iscsi_tcp_conn *tcp_conn; struct iscsi_tcp_conn *tcp_conn;
struct iscsi_sw_tcp_conn *tcp_sw_conn; struct iscsi_sw_tcp_conn *tcp_sw_conn;
struct crypto_ahash *tfm;
cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn), cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn),
conn_idx); conn_idx);
@ -596,37 +592,9 @@ iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session,
tcp_sw_conn->queue_recv = iscsi_recv_from_iscsi_q; tcp_sw_conn->queue_recv = iscsi_recv_from_iscsi_q;
mutex_init(&tcp_sw_conn->sock_lock); mutex_init(&tcp_sw_conn->sock_lock);
tcp_conn->rx_crcp = &tcp_sw_conn->rx_crc;
tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(tfm))
goto free_conn;
tcp_sw_conn->tx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
if (!tcp_sw_conn->tx_hash)
goto free_tfm;
ahash_request_set_callback(tcp_sw_conn->tx_hash, 0, NULL, NULL);
tcp_sw_conn->rx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
if (!tcp_sw_conn->rx_hash)
goto free_tx_hash;
ahash_request_set_callback(tcp_sw_conn->rx_hash, 0, NULL, NULL);
tcp_conn->rx_hash = tcp_sw_conn->rx_hash;
return cls_conn; return cls_conn;
free_tx_hash:
ahash_request_free(tcp_sw_conn->tx_hash);
free_tfm:
crypto_free_ahash(tfm);
free_conn:
iscsi_conn_printk(KERN_ERR, conn,
"Could not create connection due to crc32c "
"loading error. Make sure the crc32c "
"module is built as a module or into the "
"kernel\n");
iscsi_tcp_conn_teardown(cls_conn);
return NULL;
} }
static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn) static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn)
@ -664,20 +632,8 @@ static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn)
static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn) static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
{ {
struct iscsi_conn *conn = cls_conn->dd_data; struct iscsi_conn *conn = cls_conn->dd_data;
struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
iscsi_sw_tcp_release_conn(conn); iscsi_sw_tcp_release_conn(conn);
ahash_request_free(tcp_sw_conn->rx_hash);
if (tcp_sw_conn->tx_hash) {
struct crypto_ahash *tfm;
tfm = crypto_ahash_reqtfm(tcp_sw_conn->tx_hash);
ahash_request_free(tcp_sw_conn->tx_hash);
crypto_free_ahash(tfm);
}
iscsi_tcp_conn_teardown(cls_conn); iscsi_tcp_conn_teardown(cls_conn);
} }

4
drivers/scsi/iscsi_tcp.h
View File

@ -41,8 +41,8 @@ struct iscsi_sw_tcp_conn {
void (*old_write_space)(struct sock *); void (*old_write_space)(struct sock *);
/* data and header digests */ /* data and header digests */
struct ahash_request *tx_hash; /* CRC32C (Tx) */ u32 tx_crc; /* CRC32C (Tx) */
struct ahash_request *rx_hash; /* CRC32C (Rx) */ u32 rx_crc; /* CRC32C (Rx) */
/* MIB custom statistics */ /* MIB custom statistics */
uint32_t sendpage_failures_cnt; uint32_t sendpage_failures_cnt;

89
drivers/scsi/libiscsi_tcp.c
View File

@ -15,7 +15,7 @@
* Zhenyu Wang * Zhenyu Wang
*/ */
#include <crypto/hash.h> #include <linux/crc32c.h>
#include <linux/types.h> #include <linux/types.h>
#include <linux/list.h> #include <linux/list.h>
#include <linux/inet.h> #include <linux/inet.h>
@ -168,7 +168,7 @@ iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)
segment->size = ISCSI_DIGEST_SIZE; segment->size = ISCSI_DIGEST_SIZE;
segment->copied = 0; segment->copied = 0;
segment->sg = NULL; segment->sg = NULL;
segment->hash = NULL; segment->crcp = NULL;
} }
/** /**
@ -191,29 +191,27 @@ int iscsi_tcp_segment_done(struct iscsi_tcp_conn *tcp_conn,
struct iscsi_segment *segment, int recv, struct iscsi_segment *segment, int recv,
unsigned copied) unsigned copied)
{ {
struct scatterlist sg;
unsigned int pad; unsigned int pad;
ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copied %u %u size %u %s\n", ISCSI_DBG_TCP(tcp_conn->iscsi_conn, "copied %u %u size %u %s\n",
segment->copied, copied, segment->size, segment->copied, copied, segment->size,
recv ? "recv" : "xmit"); recv ? "recv" : "xmit");
if (segment->hash && copied) { if (segment->crcp && copied) {
/* if (segment->data) {
* If a segment is kmapd we must unmap it before sending *segment->crcp = crc32c(*segment->crcp,
* to the crypto layer since that will try to kmap it again. segment->data + segment->copied,
*/ copied);
iscsi_tcp_segment_unmap(segment); } else {
const void *data;
if (!segment->data) { data = kmap_local_page(sg_page(segment->sg));
sg_init_table(&sg, 1); *segment->crcp = crc32c(*segment->crcp,
sg_set_page(&sg, sg_page(segment->sg), copied, data + segment->copied +
segment->copied + segment->sg_offset + segment->sg_offset +
segment->sg->offset); segment->sg->offset,
} else copied);
sg_init_one(&sg, segment->data + segment->copied, kunmap_local(data);
copied); }
ahash_request_set_crypt(segment->hash, &sg, NULL, copied);
crypto_ahash_update(segment->hash);
} }
segment->copied += copied; segment->copied += copied;
@ -258,10 +256,8 @@ int iscsi_tcp_segment_done(struct iscsi_tcp_conn *tcp_conn,
* Set us up for transferring the data digest. hdr digest * Set us up for transferring the data digest. hdr digest
* is completely handled in hdr done function. * is completely handled in hdr done function.
*/ */
if (segment->hash) { if (segment->crcp) {
ahash_request_set_crypt(segment->hash, NULL, put_unaligned_le32(~*segment->crcp, segment->digest);
segment->digest, 0);
crypto_ahash_final(segment->hash);
iscsi_tcp_segment_splice_digest(segment, iscsi_tcp_segment_splice_digest(segment,
recv ? segment->recv_digest : segment->digest); recv ? segment->recv_digest : segment->digest);
return 0; return 0;
@ -282,8 +278,7 @@ EXPORT_SYMBOL_GPL(iscsi_tcp_segment_done);
* given buffer, and returns the number of bytes * given buffer, and returns the number of bytes
* consumed, which can actually be less than @len. * consumed, which can actually be less than @len.
* *
* If hash digest is enabled, the function will update the * If CRC is enabled, the function will update the CRC while copying.
* hash while copying.
* Combining these two operations doesn't buy us a lot (yet), * Combining these two operations doesn't buy us a lot (yet),
* but in the future we could implement combined copy+crc, * but in the future we could implement combined copy+crc,
* just way we do for network layer checksums. * just way we do for network layer checksums.
@ -311,14 +306,10 @@ iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn,
} }
inline void inline void
iscsi_tcp_dgst_header(struct ahash_request *hash, const void *hdr, iscsi_tcp_dgst_header(const void *hdr, size_t hdrlen,
size_t hdrlen, unsigned char digest[ISCSI_DIGEST_SIZE]) unsigned char digest[ISCSI_DIGEST_SIZE])
{ {
struct scatterlist sg; put_unaligned_le32(~crc32c(~0, hdr, hdrlen), digest);
sg_init_one(&sg, hdr, hdrlen);
ahash_request_set_crypt(hash, &sg, digest, hdrlen);
crypto_ahash_digest(hash);
} }
EXPORT_SYMBOL_GPL(iscsi_tcp_dgst_header); EXPORT_SYMBOL_GPL(iscsi_tcp_dgst_header);
@ -343,24 +334,23 @@ iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn,
*/ */
static inline void static inline void
__iscsi_segment_init(struct iscsi_segment *segment, size_t size, __iscsi_segment_init(struct iscsi_segment *segment, size_t size,
iscsi_segment_done_fn_t *done, struct ahash_request *hash) iscsi_segment_done_fn_t *done, u32 *crcp)
{ {
memset(segment, 0, sizeof(*segment)); memset(segment, 0, sizeof(*segment));
segment->total_size = size; segment->total_size = size;
segment->done = done; segment->done = done;
if (hash) { if (crcp) {
segment->hash = hash; segment->crcp = crcp;
crypto_ahash_init(hash); *crcp = ~0;
} }
} }
inline void inline void
iscsi_segment_init_linear(struct iscsi_segment *segment, void *data, iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,
size_t size, iscsi_segment_done_fn_t *done, size_t size, iscsi_segment_done_fn_t *done, u32 *crcp)
struct ahash_request *hash)
{ {
__iscsi_segment_init(segment, size, done, hash); __iscsi_segment_init(segment, size, done, crcp);
segment->data = data; segment->data = data;
segment->size = size; segment->size = size;
} }
@ -370,13 +360,12 @@ inline int
iscsi_segment_seek_sg(struct iscsi_segment *segment, iscsi_segment_seek_sg(struct iscsi_segment *segment,
struct scatterlist *sg_list, unsigned int sg_count, struct scatterlist *sg_list, unsigned int sg_count,
unsigned int offset, size_t size, unsigned int offset, size_t size,
iscsi_segment_done_fn_t *done, iscsi_segment_done_fn_t *done, u32 *crcp)
struct ahash_request *hash)
{ {
struct scatterlist *sg; struct scatterlist *sg;
unsigned int i; unsigned int i;
__iscsi_segment_init(segment, size, done, hash); __iscsi_segment_init(segment, size, done, crcp);
for_each_sg(sg_list, sg, sg_count, i) { for_each_sg(sg_list, sg, sg_count, i) {
if (offset < sg->length) { if (offset < sg->length) {
iscsi_tcp_segment_init_sg(segment, sg, offset); iscsi_tcp_segment_init_sg(segment, sg, offset);
@ -393,7 +382,7 @@ EXPORT_SYMBOL_GPL(iscsi_segment_seek_sg);
* iscsi_tcp_hdr_recv_prep - prep segment for hdr reception * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
* @tcp_conn: iscsi connection to prep for * @tcp_conn: iscsi connection to prep for
* *
* This function always passes NULL for the hash argument, because when this * This function always passes NULL for the crcp argument, because when this
* function is called we do not yet know the final size of the header and want * function is called we do not yet know the final size of the header and want
* to delay the digest processing until we know that. * to delay the digest processing until we know that.
*/ */
@ -434,15 +423,15 @@ static void
iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn) iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn)
{ {
struct iscsi_conn *conn = tcp_conn->iscsi_conn; struct iscsi_conn *conn = tcp_conn->iscsi_conn;
struct ahash_request *rx_hash = NULL; u32 *rx_crcp = NULL;
if (conn->datadgst_en && if (conn->datadgst_en &&
!(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
rx_hash = tcp_conn->rx_hash; rx_crcp = tcp_conn->rx_crcp;
iscsi_segment_init_linear(&tcp_conn->in.segment, iscsi_segment_init_linear(&tcp_conn->in.segment,
conn->data, tcp_conn->in.datalen, conn->data, tcp_conn->in.datalen,
iscsi_tcp_data_recv_done, rx_hash); iscsi_tcp_data_recv_done, rx_crcp);
} }
/** /**
@ -730,7 +719,7 @@ iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
if (tcp_conn->in.datalen) { if (tcp_conn->in.datalen) {
struct iscsi_tcp_task *tcp_task = task->dd_data; struct iscsi_tcp_task *tcp_task = task->dd_data;
struct ahash_request *rx_hash = NULL; u32 *rx_crcp = NULL;
struct scsi_data_buffer *sdb = &task->sc->sdb; struct scsi_data_buffer *sdb = &task->sc->sdb;
/* /*
@ -743,7 +732,7 @@ iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
*/ */
if (conn->datadgst_en && if (conn->datadgst_en &&
!(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
rx_hash = tcp_conn->rx_hash; rx_crcp = tcp_conn->rx_crcp;
ISCSI_DBG_TCP(conn, "iscsi_tcp_begin_data_in( " ISCSI_DBG_TCP(conn, "iscsi_tcp_begin_data_in( "
"offset=%d, datalen=%d)\n", "offset=%d, datalen=%d)\n",
@ -756,7 +745,7 @@ iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
tcp_task->data_offset, tcp_task->data_offset,
tcp_conn->in.datalen, tcp_conn->in.datalen,
iscsi_tcp_process_data_in, iscsi_tcp_process_data_in,
rx_hash); rx_crcp);
spin_unlock(&conn->session->back_lock); spin_unlock(&conn->session->back_lock);
return rc; return rc;
} }
@ -878,7 +867,7 @@ iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
return 0; return 0;
} }
iscsi_tcp_dgst_header(tcp_conn->rx_hash, hdr, iscsi_tcp_dgst_header(hdr,
segment->total_copied - ISCSI_DIGEST_SIZE, segment->total_copied - ISCSI_DIGEST_SIZE,
segment->digest); segment->digest);

16
include/scsi/libiscsi_tcp.h
View File

@ -15,7 +15,6 @@
struct iscsi_tcp_conn; struct iscsi_tcp_conn;
struct iscsi_segment; struct iscsi_segment;
struct sk_buff; struct sk_buff;
struct ahash_request;
typedef int iscsi_segment_done_fn_t(struct iscsi_tcp_conn *, typedef int iscsi_segment_done_fn_t(struct iscsi_tcp_conn *,
struct iscsi_segment *); struct iscsi_segment *);
@ -27,7 +26,7 @@ struct iscsi_segment {
unsigned int total_size; unsigned int total_size;
unsigned int total_copied; unsigned int total_copied;
struct ahash_request *hash; u32 *crcp;
unsigned char padbuf[ISCSI_PAD_LEN]; unsigned char padbuf[ISCSI_PAD_LEN];
unsigned char recv_digest[ISCSI_DIGEST_SIZE]; unsigned char recv_digest[ISCSI_DIGEST_SIZE];
unsigned char digest[ISCSI_DIGEST_SIZE]; unsigned char digest[ISCSI_DIGEST_SIZE];
@ -61,8 +60,8 @@ struct iscsi_tcp_conn {
* stop to terminate */ * stop to terminate */
/* control data */ /* control data */
struct iscsi_tcp_recv in; /* TCP receive context */ struct iscsi_tcp_recv in; /* TCP receive context */
/* CRC32C (Rx) LLD should set this is they do not offload */ /* CRC32C (Rx) LLD should set this if they do not offload */
struct ahash_request *rx_hash; u32 *rx_crcp;
}; };
struct iscsi_tcp_task { struct iscsi_tcp_task {
@ -99,18 +98,15 @@ extern void iscsi_tcp_segment_unmap(struct iscsi_segment *segment);
extern void iscsi_segment_init_linear(struct iscsi_segment *segment, extern void iscsi_segment_init_linear(struct iscsi_segment *segment,
void *data, size_t size, void *data, size_t size,
iscsi_segment_done_fn_t *done, iscsi_segment_done_fn_t *done, u32 *crcp);
struct ahash_request *hash);
extern int extern int
iscsi_segment_seek_sg(struct iscsi_segment *segment, iscsi_segment_seek_sg(struct iscsi_segment *segment,
struct scatterlist *sg_list, unsigned int sg_count, struct scatterlist *sg_list, unsigned int sg_count,
unsigned int offset, size_t size, unsigned int offset, size_t size,
iscsi_segment_done_fn_t *done, iscsi_segment_done_fn_t *done, u32 *crcp);
struct ahash_request *hash);
/* digest helpers */ /* digest helpers */
extern void iscsi_tcp_dgst_header(struct ahash_request *hash, const void *hdr, extern void iscsi_tcp_dgst_header(const void *hdr, size_t hdrlen,
size_t hdrlen,
unsigned char digest[ISCSI_DIGEST_SIZE]); unsigned char digest[ISCSI_DIGEST_SIZE]);
extern struct iscsi_cls_conn * extern struct iscsi_cls_conn *
iscsi_tcp_conn_setup(struct iscsi_cls_session *cls_session, int dd_data_size, iscsi_tcp_conn_setup(struct iscsi_cls_session *cls_session, int dd_data_size,