From 7a836a2aba09479c8e71fa43249eecc4af945f61 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 3 Apr 2022 11:11:14 +0200 Subject: [PATCH 01/18] virtio: pci: Fix an error handling path in vp_modern_probe() If an error occurs after a successful pci_request_selected_regions() call, it should be undone by a corresponding pci_release_selected_regions() call, as already done in vp_modern_remove(). Fixes: fd502729fbbf ("virtio-pci: introduce modern device module") Signed-off-by: Christophe JAILLET Message-Id: <237109725aad2c3c03d14549f777b1927c84b045.1648977064.git.christophe.jaillet@wanadoo.fr> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_modern_dev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index 591738ad3d56..4093f9cca7a6 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -347,6 +347,7 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev) err_map_isr: pci_iounmap(pci_dev, mdev->common); err_map_common: + pci_release_selected_regions(pci_dev, mdev->modern_bars); return err; } EXPORT_SYMBOL_GPL(vp_modern_probe); From 7e415282b41bf0d15c6e0fe268f822d9b083f2f7 Mon Sep 17 00:00:00 2001 From: Murilo Opsfelder Araujo Date: Thu, 14 Apr 2022 23:30:02 -0300 Subject: [PATCH 02/18] virtio-pci: Remove wrong address verification in vp_del_vqs() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GCC 12 enhanced -Waddress when comparing array address to null [0], which warns: drivers/virtio/virtio_pci_common.c: In function ‘vp_del_vqs’: drivers/virtio/virtio_pci_common.c:257:29: warning: the comparison will always evaluate as ‘true’ for the pointer operand in ‘vp_dev->msix_affinity_masks + (sizetype)((long unsigned int)i * 256)’ must not be NULL [-Waddress] 257 | if (vp_dev->msix_affinity_masks[i]) | ^~~~~~ In fact, the verification is comparing the result of a pointer arithmetic, the address "msix_affinity_masks + i", which will always evaluate to true. Under the hood, free_cpumask_var() calls kfree(), which is safe to pass NULL, not requiring non-null verification. So remove the verification to make compiler happy (happy compiler, happy life). [0] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102103 Signed-off-by: Murilo Opsfelder Araujo Message-Id: <20220415023002.49805-1-muriloo@linux.ibm.com> Signed-off-by: Michael S. Tsirkin Acked-by: Christophe de Dinechin --- drivers/virtio/virtio_pci_common.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index d724f676608b..5046efcffb4c 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -254,8 +254,7 @@ void vp_del_vqs(struct virtio_device *vdev) if (vp_dev->msix_affinity_masks) { for (i = 0; i < vp_dev->msix_vectors; i++) - if (vp_dev->msix_affinity_masks[i]) - free_cpumask_var(vp_dev->msix_affinity_masks[i]); + free_cpumask_var(vp_dev->msix_affinity_masks[i]); } if (vp_dev->msix_enabled) { From 6fd763d155860eb7ea3a93c8b3bf926940ffa3fb Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Fri, 6 May 2022 21:16:23 +0800 Subject: [PATCH 03/18] virtio-crypto: change code style Use temporary variable to make code easy to read and maintain. /* Pad cipher's parameters */ vcrypto->ctrl.u.sym_create_session.op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); vcrypto->ctrl.u.sym_create_session.u.cipher.para.algo = vcrypto->ctrl.header.algo; vcrypto->ctrl.u.sym_create_session.u.cipher.para.keylen = cpu_to_le32(keylen); vcrypto->ctrl.u.sym_create_session.u.cipher.para.op = cpu_to_le32(op); --> sym_create_session = &ctrl->u.sym_create_session; sym_create_session->op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); sym_create_session->u.cipher.para.algo = ctrl->header.algo; sym_create_session->u.cipher.para.keylen = cpu_to_le32(keylen); sym_create_session->u.cipher.para.op = cpu_to_le32(op); The new style shows more obviously: - the variable we want to operate. - an assignment statement in a single line. Cc: Michael S. Tsirkin Cc: Jason Wang Cc: Gonglei Reviewed-by: Gonglei Signed-off-by: zhenwei pi Message-Id: <20220506131627.180784-2-pizhenwei@bytedance.com> Signed-off-by: Michael S. Tsirkin --- .../virtio/virtio_crypto_akcipher_algs.c | 40 ++++++----- .../virtio/virtio_crypto_skcipher_algs.c | 70 +++++++++---------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index f3ec9420215e..20901a263fc8 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -106,23 +106,27 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher unsigned int inlen; int err; unsigned int num_out = 0, num_in = 0; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_session_input *input; pkey = kmemdup(key, keylen, GFP_ATOMIC); if (!pkey) return -ENOMEM; spin_lock(&vcrypto->ctrl_lock); - memcpy(&vcrypto->ctrl.header, header, sizeof(vcrypto->ctrl.header)); - memcpy(&vcrypto->ctrl.u, para, sizeof(vcrypto->ctrl.u)); - vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR); + ctrl = &vcrypto->ctrl; + memcpy(&ctrl->header, header, sizeof(ctrl->header)); + memcpy(&ctrl->u, para, sizeof(ctrl->u)); + input = &vcrypto->input; + input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); - sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); + sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); sgs[num_out++] = &outhdr_sg; sg_init_one(&key_sg, pkey, keylen); sgs[num_out++] = &key_sg; - sg_init_one(&inhdr_sg, &vcrypto->input, sizeof(vcrypto->input)); + sg_init_one(&inhdr_sg, input, sizeof(*input)); sgs[num_out + num_in++] = &inhdr_sg; err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); @@ -134,12 +138,12 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); - if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) { + if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { err = -EINVAL; goto out; } - ctx->session_id = le64_to_cpu(vcrypto->input.session_id); + ctx->session_id = le64_to_cpu(input->session_id); ctx->session_valid = true; err = 0; @@ -149,7 +153,7 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher if (err < 0) pr_err("virtio_crypto: Create session failed status: %u\n", - le32_to_cpu(vcrypto->input.status)); + le32_to_cpu(input->status)); return err; } @@ -161,23 +165,27 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe struct virtio_crypto *vcrypto = ctx->vcrypto; unsigned int num_out = 0, num_in = 0, inlen; int err; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_inhdr *ctrl_status; spin_lock(&vcrypto->ctrl_lock); if (!ctx->session_valid) { err = 0; goto out; } - vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR; - vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION); - vcrypto->ctrl.header.queue_id = 0; + ctrl_status = &vcrypto->ctrl_status; + ctrl_status->status = VIRTIO_CRYPTO_ERR; + ctrl = &vcrypto->ctrl; + ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION); + ctrl->header.queue_id = 0; - destroy_session = &vcrypto->ctrl.u.destroy_session; + destroy_session = &ctrl->u.destroy_session; destroy_session->session_id = cpu_to_le64(ctx->session_id); - sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); + sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); sgs[num_out++] = &outhdr_sg; - sg_init_one(&inhdr_sg, &vcrypto->ctrl_status.status, sizeof(vcrypto->ctrl_status.status)); + sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status)); sgs[num_out + num_in++] = &inhdr_sg; err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); @@ -189,7 +197,7 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); - if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) { + if (ctrl_status->status != VIRTIO_CRYPTO_OK) { err = -EINVAL; goto out; } @@ -201,7 +209,7 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe spin_unlock(&vcrypto->ctrl_lock); if (err < 0) { pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", - vcrypto->ctrl_status.status, destroy_session->session_id); + ctrl_status->status, destroy_session->session_id); } return err; diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index a618c46a52b8..e3c5bc8d6112 100644 --- a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -123,6 +123,9 @@ static int virtio_crypto_alg_skcipher_init_session( int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT; int err; unsigned int num_out = 0, num_in = 0; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_session_input *input; + struct virtio_crypto_sym_create_session_req *sym_create_session; /* * Avoid to do DMA from the stack, switch to using @@ -135,24 +138,22 @@ static int virtio_crypto_alg_skcipher_init_session( spin_lock(&vcrypto->ctrl_lock); /* Pad ctrl header */ - vcrypto->ctrl.header.opcode = - cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION); - vcrypto->ctrl.header.algo = cpu_to_le32(alg); + ctrl = &vcrypto->ctrl; + ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION); + ctrl->header.algo = cpu_to_le32(alg); /* Set the default dataqueue id to 0 */ - vcrypto->ctrl.header.queue_id = 0; + ctrl->header.queue_id = 0; - vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR); + input = &vcrypto->input; + input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); /* Pad cipher's parameters */ - vcrypto->ctrl.u.sym_create_session.op_type = - cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); - vcrypto->ctrl.u.sym_create_session.u.cipher.para.algo = - vcrypto->ctrl.header.algo; - vcrypto->ctrl.u.sym_create_session.u.cipher.para.keylen = - cpu_to_le32(keylen); - vcrypto->ctrl.u.sym_create_session.u.cipher.para.op = - cpu_to_le32(op); + sym_create_session = &ctrl->u.sym_create_session; + sym_create_session->op_type = cpu_to_le32(VIRTIO_CRYPTO_SYM_OP_CIPHER); + sym_create_session->u.cipher.para.algo = ctrl->header.algo; + sym_create_session->u.cipher.para.keylen = cpu_to_le32(keylen); + sym_create_session->u.cipher.para.op = cpu_to_le32(op); - sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); + sg_init_one(&outhdr, ctrl, sizeof(*ctrl)); sgs[num_out++] = &outhdr; /* Set key */ @@ -160,7 +161,7 @@ static int virtio_crypto_alg_skcipher_init_session( sgs[num_out++] = &key_sg; /* Return status and session id back */ - sg_init_one(&inhdr, &vcrypto->input, sizeof(vcrypto->input)); + sg_init_one(&inhdr, input, sizeof(*input)); sgs[num_out + num_in++] = &inhdr; err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, @@ -180,20 +181,18 @@ static int virtio_crypto_alg_skcipher_init_session( !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); - if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) { + if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { spin_unlock(&vcrypto->ctrl_lock); pr_err("virtio_crypto: Create session failed status: %u\n", - le32_to_cpu(vcrypto->input.status)); + le32_to_cpu(input->status)); kfree_sensitive(cipher_key); return -EINVAL; } if (encrypt) - ctx->enc_sess_info.session_id = - le64_to_cpu(vcrypto->input.session_id); + ctx->enc_sess_info.session_id = le64_to_cpu(input->session_id); else - ctx->dec_sess_info.session_id = - le64_to_cpu(vcrypto->input.session_id); + ctx->dec_sess_info.session_id = le64_to_cpu(input->session_id); spin_unlock(&vcrypto->ctrl_lock); @@ -211,30 +210,30 @@ static int virtio_crypto_alg_skcipher_close_session( struct virtio_crypto *vcrypto = ctx->vcrypto; int err; unsigned int num_out = 0, num_in = 0; + struct virtio_crypto_op_ctrl_req *ctrl; + struct virtio_crypto_inhdr *ctrl_status; spin_lock(&vcrypto->ctrl_lock); - vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR; + ctrl_status = &vcrypto->ctrl_status; + ctrl_status->status = VIRTIO_CRYPTO_ERR; /* Pad ctrl header */ - vcrypto->ctrl.header.opcode = - cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION); + ctrl = &vcrypto->ctrl; + ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION); /* Set the default virtqueue id to 0 */ - vcrypto->ctrl.header.queue_id = 0; + ctrl->header.queue_id = 0; - destroy_session = &vcrypto->ctrl.u.destroy_session; + destroy_session = &ctrl->u.destroy_session; if (encrypt) - destroy_session->session_id = - cpu_to_le64(ctx->enc_sess_info.session_id); + destroy_session->session_id = cpu_to_le64(ctx->enc_sess_info.session_id); else - destroy_session->session_id = - cpu_to_le64(ctx->dec_sess_info.session_id); + destroy_session->session_id = cpu_to_le64(ctx->dec_sess_info.session_id); - sg_init_one(&outhdr, &vcrypto->ctrl, sizeof(vcrypto->ctrl)); + sg_init_one(&outhdr, ctrl, sizeof(*ctrl)); sgs[num_out++] = &outhdr; /* Return status and session id back */ - sg_init_one(&status_sg, &vcrypto->ctrl_status.status, - sizeof(vcrypto->ctrl_status.status)); + sg_init_one(&status_sg, &ctrl_status->status, sizeof(ctrl_status->status)); sgs[num_out + num_in++] = &status_sg; err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, @@ -249,11 +248,10 @@ static int virtio_crypto_alg_skcipher_close_session( !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); - if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) { + if (ctrl_status->status != VIRTIO_CRYPTO_OK) { spin_unlock(&vcrypto->ctrl_lock); pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", - vcrypto->ctrl_status.status, - destroy_session->session_id); + ctrl_status->status, destroy_session->session_id); return -EINVAL; } From 0756ad15b1fef287d4d8fa11bc36ea77a5c42e4a Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Fri, 6 May 2022 21:16:24 +0800 Subject: [PATCH 04/18] virtio-crypto: use private buffer for control request Originally, all of the control requests share a single buffer( ctrl & input & ctrl_status fields in struct virtio_crypto), this allows queue depth 1 only, the performance of control queue gets limited by this design. In this patch, each request allocates request buffer dynamically, and free buffer after request, so the scope protected by ctrl_lock also get optimized here. It's possible to optimize control queue depth in the next step. A necessary comment is already in code, still describe it again: /* * Note: there are padding fields in request, clear them to zero before * sending to host to avoid to divulge any information. * Ex, virtio_crypto_ctrl_request::ctrl::u::destroy_session::padding[48] */ So use kzalloc to allocate buffer of struct virtio_crypto_ctrl_request. Potentially dereferencing uninitialized variables: Reported-by: kernel test robot Reported-by: Dan Carpenter Cc: Michael S. Tsirkin Cc: Jason Wang Cc: Gonglei Reviewed-by: Gonglei Signed-off-by: zhenwei pi Message-Id: <20220506131627.180784-3-pizhenwei@bytedance.com> Signed-off-by: Michael S. Tsirkin --- .../virtio/virtio_crypto_akcipher_algs.c | 57 ++++++++++++------- drivers/crypto/virtio/virtio_crypto_common.h | 17 ++++-- .../virtio/virtio_crypto_skcipher_algs.c | 50 ++++++++++------ 3 files changed, 79 insertions(+), 45 deletions(-) diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 20901a263fc8..698ea57e2649 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -108,16 +108,22 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher unsigned int num_out = 0, num_in = 0; struct virtio_crypto_op_ctrl_req *ctrl; struct virtio_crypto_session_input *input; + struct virtio_crypto_ctrl_request *vc_ctrl_req; pkey = kmemdup(key, keylen, GFP_ATOMIC); if (!pkey) return -ENOMEM; - spin_lock(&vcrypto->ctrl_lock); - ctrl = &vcrypto->ctrl; + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); + if (!vc_ctrl_req) { + err = -ENOMEM; + goto out; + } + + ctrl = &vc_ctrl_req->ctrl; memcpy(&ctrl->header, header, sizeof(ctrl->header)); memcpy(&ctrl->u, para, sizeof(ctrl->u)); - input = &vcrypto->input; + input = &vc_ctrl_req->input; input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); sg_init_one(&outhdr_sg, ctrl, sizeof(*ctrl)); @@ -129,16 +135,22 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher sg_init_one(&inhdr_sg, input, sizeof(*input)); sgs[num_out + num_in++] = &inhdr_sg; + spin_lock(&vcrypto->ctrl_lock); err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); - if (err < 0) + if (err < 0) { + spin_unlock(&vcrypto->ctrl_lock); goto out; + } virtqueue_kick(vcrypto->ctrl_vq); while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); + spin_unlock(&vcrypto->ctrl_lock); if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { + pr_err("virtio_crypto: Create session failed status: %u\n", + le32_to_cpu(input->status)); err = -EINVAL; goto out; } @@ -148,13 +160,9 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher err = 0; out: - spin_unlock(&vcrypto->ctrl_lock); + kfree(vc_ctrl_req); kfree_sensitive(pkey); - if (err < 0) - pr_err("virtio_crypto: Create session failed status: %u\n", - le32_to_cpu(input->status)); - return err; } @@ -167,15 +175,18 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe int err; struct virtio_crypto_op_ctrl_req *ctrl; struct virtio_crypto_inhdr *ctrl_status; + struct virtio_crypto_ctrl_request *vc_ctrl_req; - spin_lock(&vcrypto->ctrl_lock); - if (!ctx->session_valid) { - err = 0; - goto out; - } - ctrl_status = &vcrypto->ctrl_status; + if (!ctx->session_valid) + return 0; + + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); + if (!vc_ctrl_req) + return -ENOMEM; + + ctrl_status = &vc_ctrl_req->ctrl_status; ctrl_status->status = VIRTIO_CRYPTO_ERR; - ctrl = &vcrypto->ctrl; + ctrl = &vc_ctrl_req->ctrl; ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION); ctrl->header.queue_id = 0; @@ -188,16 +199,22 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status)); sgs[num_out + num_in++] = &inhdr_sg; + spin_lock(&vcrypto->ctrl_lock); err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); - if (err < 0) + if (err < 0) { + spin_unlock(&vcrypto->ctrl_lock); goto out; + } virtqueue_kick(vcrypto->ctrl_vq); while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); + spin_unlock(&vcrypto->ctrl_lock); if (ctrl_status->status != VIRTIO_CRYPTO_OK) { + pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", + ctrl_status->status, destroy_session->session_id); err = -EINVAL; goto out; } @@ -206,11 +223,7 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe ctx->session_valid = false; out: - spin_unlock(&vcrypto->ctrl_lock); - if (err < 0) { - pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", - ctrl_status->status, destroy_session->session_id); - } + kfree(vc_ctrl_req); return err; } diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h index e693d4ee83a6..2422237ec4e6 100644 --- a/drivers/crypto/virtio/virtio_crypto_common.h +++ b/drivers/crypto/virtio/virtio_crypto_common.h @@ -13,6 +13,7 @@ #include #include #include +#include /* Internal representation of a data virtqueue */ @@ -65,11 +66,6 @@ struct virtio_crypto { /* Maximum size of per request */ u64 max_size; - /* Control VQ buffers: protected by the ctrl_lock */ - struct virtio_crypto_op_ctrl_req ctrl; - struct virtio_crypto_session_input input; - struct virtio_crypto_inhdr ctrl_status; - unsigned long status; atomic_t ref_count; struct list_head list; @@ -85,6 +81,17 @@ struct virtio_crypto_sym_session_info { __u64 session_id; }; +/* + * Note: there are padding fields in request, clear them to zero before + * sending to host to avoid to divulge any information. + * Ex, virtio_crypto_ctrl_request::ctrl::u::destroy_session::padding[48] + */ +struct virtio_crypto_ctrl_request { + struct virtio_crypto_op_ctrl_req ctrl; + struct virtio_crypto_session_input input; + struct virtio_crypto_inhdr ctrl_status; +}; + struct virtio_crypto_request; typedef void (*virtio_crypto_data_callback) (struct virtio_crypto_request *vc_req, int len); diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index e3c5bc8d6112..6aaf0869b211 100644 --- a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -126,6 +126,7 @@ static int virtio_crypto_alg_skcipher_init_session( struct virtio_crypto_op_ctrl_req *ctrl; struct virtio_crypto_session_input *input; struct virtio_crypto_sym_create_session_req *sym_create_session; + struct virtio_crypto_ctrl_request *vc_ctrl_req; /* * Avoid to do DMA from the stack, switch to using @@ -136,15 +137,20 @@ static int virtio_crypto_alg_skcipher_init_session( if (!cipher_key) return -ENOMEM; - spin_lock(&vcrypto->ctrl_lock); + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); + if (!vc_ctrl_req) { + err = -ENOMEM; + goto out; + } + /* Pad ctrl header */ - ctrl = &vcrypto->ctrl; + ctrl = &vc_ctrl_req->ctrl; ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_CREATE_SESSION); ctrl->header.algo = cpu_to_le32(alg); /* Set the default dataqueue id to 0 */ ctrl->header.queue_id = 0; - input = &vcrypto->input; + input = &vc_ctrl_req->input; input->status = cpu_to_le32(VIRTIO_CRYPTO_ERR); /* Pad cipher's parameters */ sym_create_session = &ctrl->u.sym_create_session; @@ -164,12 +170,12 @@ static int virtio_crypto_alg_skcipher_init_session( sg_init_one(&inhdr, input, sizeof(*input)); sgs[num_out + num_in++] = &inhdr; + spin_lock(&vcrypto->ctrl_lock); err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); if (err < 0) { spin_unlock(&vcrypto->ctrl_lock); - kfree_sensitive(cipher_key); - return err; + goto out; } virtqueue_kick(vcrypto->ctrl_vq); @@ -180,13 +186,13 @@ static int virtio_crypto_alg_skcipher_init_session( while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); + spin_unlock(&vcrypto->ctrl_lock); if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { - spin_unlock(&vcrypto->ctrl_lock); pr_err("virtio_crypto: Create session failed status: %u\n", le32_to_cpu(input->status)); - kfree_sensitive(cipher_key); - return -EINVAL; + err = -EINVAL; + goto out; } if (encrypt) @@ -194,10 +200,11 @@ static int virtio_crypto_alg_skcipher_init_session( else ctx->dec_sess_info.session_id = le64_to_cpu(input->session_id); - spin_unlock(&vcrypto->ctrl_lock); - + err = 0; +out: + kfree(vc_ctrl_req); kfree_sensitive(cipher_key); - return 0; + return err; } static int virtio_crypto_alg_skcipher_close_session( @@ -212,12 +219,16 @@ static int virtio_crypto_alg_skcipher_close_session( unsigned int num_out = 0, num_in = 0; struct virtio_crypto_op_ctrl_req *ctrl; struct virtio_crypto_inhdr *ctrl_status; + struct virtio_crypto_ctrl_request *vc_ctrl_req; - spin_lock(&vcrypto->ctrl_lock); - ctrl_status = &vcrypto->ctrl_status; + vc_ctrl_req = kzalloc(sizeof(*vc_ctrl_req), GFP_KERNEL); + if (!vc_ctrl_req) + return -ENOMEM; + + ctrl_status = &vc_ctrl_req->ctrl_status; ctrl_status->status = VIRTIO_CRYPTO_ERR; /* Pad ctrl header */ - ctrl = &vcrypto->ctrl; + ctrl = &vc_ctrl_req->ctrl; ctrl->header.opcode = cpu_to_le32(VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION); /* Set the default virtqueue id to 0 */ ctrl->header.queue_id = 0; @@ -236,28 +247,31 @@ static int virtio_crypto_alg_skcipher_close_session( sg_init_one(&status_sg, &ctrl_status->status, sizeof(ctrl_status->status)); sgs[num_out + num_in++] = &status_sg; + spin_lock(&vcrypto->ctrl_lock); err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); if (err < 0) { spin_unlock(&vcrypto->ctrl_lock); - return err; + goto out; } virtqueue_kick(vcrypto->ctrl_vq); while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && !virtqueue_is_broken(vcrypto->ctrl_vq)) cpu_relax(); + spin_unlock(&vcrypto->ctrl_lock); if (ctrl_status->status != VIRTIO_CRYPTO_OK) { - spin_unlock(&vcrypto->ctrl_lock); pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", ctrl_status->status, destroy_session->session_id); return -EINVAL; } - spin_unlock(&vcrypto->ctrl_lock); - return 0; + err = 0; +out: + kfree(vc_ctrl_req); + return err; } static int virtio_crypto_alg_skcipher_init_sessions( From 977231e8d45657871a86fe3c7bed94921d04e447 Mon Sep 17 00:00:00 2001 From: zhenwei pi Date: Fri, 6 May 2022 21:16:25 +0800 Subject: [PATCH 05/18] virtio-crypto: wait ctrl queue instead of busy polling Originally, after submitting request into virtio crypto control queue, the guest side polls the result from the virt queue. This works like following: CPU0 CPU1 ... CPUx CPUy | | | | \ \ / / \--------spin_lock(&vcrypto->ctrl_lock)-------/ | virtqueue add & kick | busy poll virtqueue | spin_unlock(&vcrypto->ctrl_lock) ... There are two problems: 1, The queue depth is always 1, the performance of a virtio crypto device gets limited. Multi user processes share a single control queue, and hit spin lock race from control queue. Test on Intel Platinum 8260, a single worker gets ~35K/s create/close session operations, and 8 workers get ~40K/s operations with 800% CPU utilization. 2, The control request is supposed to get handled immediately, but in the current implementation of QEMU(v6.2), the vCPU thread kicks another thread to do this work, the latency also gets unstable. Tracking latency of virtio_crypto_alg_akcipher_close_session in 5s: usecs : count distribution 0 -> 1 : 0 | | 2 -> 3 : 7 | | 4 -> 7 : 72 | | 8 -> 15 : 186485 |************************| 16 -> 31 : 687 | | 32 -> 63 : 5 | | 64 -> 127 : 3 | | 128 -> 255 : 1 | | 256 -> 511 : 0 | | 512 -> 1023 : 0 | | 1024 -> 2047 : 0 | | 2048 -> 4095 : 0 | | 4096 -> 8191 : 0 | | 8192 -> 16383 : 2 | | This means that a CPU may hold vcrypto->ctrl_lock as long as 8192~16383us. To improve the performance of control queue, a request on control queue waits completion instead of busy polling to reduce lock racing, and gets completed by control queue callback. CPU0 CPU1 ... CPUx CPUy | | | | \ \ / / \--------spin_lock(&vcrypto->ctrl_lock)-------/ | virtqueue add & kick | ---------spin_unlock(&vcrypto->ctrl_lock)------ / / \ \ | | | | wait wait wait wait Test this patch, the guest side get ~200K/s operations with 300% CPU utilization. Cc: Michael S. Tsirkin Cc: Jason Wang Cc: Gonglei Reviewed-by: Gonglei Signed-off-by: zhenwei pi Message-Id: <20220506131627.180784-4-pizhenwei@bytedance.com> Signed-off-by: Michael S. Tsirkin --- .../virtio/virtio_crypto_akcipher_algs.c | 29 ++--------- drivers/crypto/virtio/virtio_crypto_common.h | 4 ++ drivers/crypto/virtio/virtio_crypto_core.c | 52 ++++++++++++++++++- .../virtio/virtio_crypto_skcipher_algs.c | 34 ++---------- 4 files changed, 64 insertions(+), 55 deletions(-) diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 698ea57e2649..382ccec9ab12 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -103,7 +103,6 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3]; struct virtio_crypto *vcrypto = ctx->vcrypto; uint8_t *pkey; - unsigned int inlen; int err; unsigned int num_out = 0, num_in = 0; struct virtio_crypto_op_ctrl_req *ctrl; @@ -135,18 +134,9 @@ static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher sg_init_one(&inhdr_sg, input, sizeof(*input)); sgs[num_out + num_in++] = &inhdr_sg; - spin_lock(&vcrypto->ctrl_lock); - err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); - if (err < 0) { - spin_unlock(&vcrypto->ctrl_lock); + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) goto out; - } - - virtqueue_kick(vcrypto->ctrl_vq); - while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && - !virtqueue_is_broken(vcrypto->ctrl_vq)) - cpu_relax(); - spin_unlock(&vcrypto->ctrl_lock); if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { pr_err("virtio_crypto: Create session failed status: %u\n", @@ -171,7 +161,7 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe struct scatterlist outhdr_sg, inhdr_sg, *sgs[2]; struct virtio_crypto_destroy_session_req *destroy_session; struct virtio_crypto *vcrypto = ctx->vcrypto; - unsigned int num_out = 0, num_in = 0, inlen; + unsigned int num_out = 0, num_in = 0; int err; struct virtio_crypto_op_ctrl_req *ctrl; struct virtio_crypto_inhdr *ctrl_status; @@ -199,18 +189,9 @@ static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akciphe sg_init_one(&inhdr_sg, &ctrl_status->status, sizeof(ctrl_status->status)); sgs[num_out + num_in++] = &inhdr_sg; - spin_lock(&vcrypto->ctrl_lock); - err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC); - if (err < 0) { - spin_unlock(&vcrypto->ctrl_lock); + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) goto out; - } - - virtqueue_kick(vcrypto->ctrl_vq); - while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) && - !virtqueue_is_broken(vcrypto->ctrl_vq)) - cpu_relax(); - spin_unlock(&vcrypto->ctrl_lock); if (ctrl_status->status != VIRTIO_CRYPTO_OK) { pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", diff --git a/drivers/crypto/virtio/virtio_crypto_common.h b/drivers/crypto/virtio/virtio_crypto_common.h index 2422237ec4e6..59a4c0259456 100644 --- a/drivers/crypto/virtio/virtio_crypto_common.h +++ b/drivers/crypto/virtio/virtio_crypto_common.h @@ -90,6 +90,7 @@ struct virtio_crypto_ctrl_request { struct virtio_crypto_op_ctrl_req ctrl; struct virtio_crypto_session_input input; struct virtio_crypto_inhdr ctrl_status; + struct completion compl; }; struct virtio_crypto_request; @@ -141,5 +142,8 @@ int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto); void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto); int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto); void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto); +int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[], + unsigned int out_sgs, unsigned int in_sgs, + struct virtio_crypto_ctrl_request *vc_ctrl_req); #endif /* _VIRTIO_CRYPTO_COMMON_H */ diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index c6f482db0bc0..60490ffa3df1 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -22,6 +22,56 @@ virtcrypto_clear_request(struct virtio_crypto_request *vc_req) } } +static void virtio_crypto_ctrlq_callback(struct virtio_crypto_ctrl_request *vc_ctrl_req) +{ + complete(&vc_ctrl_req->compl); +} + +static void virtcrypto_ctrlq_callback(struct virtqueue *vq) +{ + struct virtio_crypto *vcrypto = vq->vdev->priv; + struct virtio_crypto_ctrl_request *vc_ctrl_req; + unsigned long flags; + unsigned int len; + + spin_lock_irqsave(&vcrypto->ctrl_lock, flags); + do { + virtqueue_disable_cb(vq); + while ((vc_ctrl_req = virtqueue_get_buf(vq, &len)) != NULL) { + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); + virtio_crypto_ctrlq_callback(vc_ctrl_req); + spin_lock_irqsave(&vcrypto->ctrl_lock, flags); + } + if (unlikely(virtqueue_is_broken(vq))) + break; + } while (!virtqueue_enable_cb(vq)); + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); +} + +int virtio_crypto_ctrl_vq_request(struct virtio_crypto *vcrypto, struct scatterlist *sgs[], + unsigned int out_sgs, unsigned int in_sgs, + struct virtio_crypto_ctrl_request *vc_ctrl_req) +{ + int err; + unsigned long flags; + + init_completion(&vc_ctrl_req->compl); + + spin_lock_irqsave(&vcrypto->ctrl_lock, flags); + err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, out_sgs, in_sgs, vc_ctrl_req, GFP_ATOMIC); + if (err < 0) { + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); + return err; + } + + virtqueue_kick(vcrypto->ctrl_vq); + spin_unlock_irqrestore(&vcrypto->ctrl_lock, flags); + + wait_for_completion(&vc_ctrl_req->compl); + + return 0; +} + static void virtcrypto_dataq_callback(struct virtqueue *vq) { struct virtio_crypto *vcrypto = vq->vdev->priv; @@ -73,7 +123,7 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) goto err_names; /* Parameters for control virtqueue */ - callbacks[total_vqs - 1] = NULL; + callbacks[total_vqs - 1] = virtcrypto_ctrlq_callback; names[total_vqs - 1] = "controlq"; /* Allocate/initialize parameters for data virtqueues */ diff --git a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c index 6aaf0869b211..e553ccadbcbc 100644 --- a/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_skcipher_algs.c @@ -118,7 +118,6 @@ static int virtio_crypto_alg_skcipher_init_session( int encrypt) { struct scatterlist outhdr, key_sg, inhdr, *sgs[3]; - unsigned int tmp; struct virtio_crypto *vcrypto = ctx->vcrypto; int op = encrypt ? VIRTIO_CRYPTO_OP_ENCRYPT : VIRTIO_CRYPTO_OP_DECRYPT; int err; @@ -170,23 +169,9 @@ static int virtio_crypto_alg_skcipher_init_session( sg_init_one(&inhdr, input, sizeof(*input)); sgs[num_out + num_in++] = &inhdr; - spin_lock(&vcrypto->ctrl_lock); - err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, - num_in, vcrypto, GFP_ATOMIC); - if (err < 0) { - spin_unlock(&vcrypto->ctrl_lock); + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) goto out; - } - virtqueue_kick(vcrypto->ctrl_vq); - - /* - * Trapping into the hypervisor, so the request should be - * handled immediately. - */ - while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && - !virtqueue_is_broken(vcrypto->ctrl_vq)) - cpu_relax(); - spin_unlock(&vcrypto->ctrl_lock); if (le32_to_cpu(input->status) != VIRTIO_CRYPTO_OK) { pr_err("virtio_crypto: Create session failed status: %u\n", @@ -212,7 +197,6 @@ static int virtio_crypto_alg_skcipher_close_session( int encrypt) { struct scatterlist outhdr, status_sg, *sgs[2]; - unsigned int tmp; struct virtio_crypto_destroy_session_req *destroy_session; struct virtio_crypto *vcrypto = ctx->vcrypto; int err; @@ -247,19 +231,9 @@ static int virtio_crypto_alg_skcipher_close_session( sg_init_one(&status_sg, &ctrl_status->status, sizeof(ctrl_status->status)); sgs[num_out + num_in++] = &status_sg; - spin_lock(&vcrypto->ctrl_lock); - err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, - num_in, vcrypto, GFP_ATOMIC); - if (err < 0) { - spin_unlock(&vcrypto->ctrl_lock); + err = virtio_crypto_ctrl_vq_request(vcrypto, sgs, num_out, num_in, vc_ctrl_req); + if (err < 0) goto out; - } - virtqueue_kick(vcrypto->ctrl_vq); - - while (!virtqueue_get_buf(vcrypto->ctrl_vq, &tmp) && - !virtqueue_is_broken(vcrypto->ctrl_vq)) - cpu_relax(); - spin_unlock(&vcrypto->ctrl_lock); if (ctrl_status->status != VIRTIO_CRYPTO_OK) { pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n", From a36bd0ad9fbf69d0d711b1c105954ce8d6cc144a Mon Sep 17 00:00:00 2001 From: lei he Date: Fri, 6 May 2022 21:16:26 +0800 Subject: [PATCH 06/18] virtio-crypto: adjust dst_len at ops callback For some akcipher operations(eg, decryption of pkcs1pad(rsa)), the length of returned result maybe less than akcipher_req->dst_len, we need to recalculate the actual dst_len through the virt-queue protocol. Cc: Michael S. Tsirkin Cc: Jason Wang Cc: Gonglei Reviewed-by: Gonglei Signed-off-by: lei he Signed-off-by: zhenwei pi Message-Id: <20220506131627.180784-5-pizhenwei@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/crypto/virtio/virtio_crypto_akcipher_algs.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 382ccec9ab12..2a60d0525cde 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -90,9 +90,12 @@ static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request * } akcipher_req = vc_akcipher_req->akcipher_req; - if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY) + if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY) { + /* actuall length maybe less than dst buffer */ + akcipher_req->dst_len = len - sizeof(vc_req->status); sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), vc_akcipher_req->dst_buf, akcipher_req->dst_len); + } virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error); } From 4e0d352af04cf4e019d3e45229eaaff9e8ffb33d Mon Sep 17 00:00:00 2001 From: lei he Date: Fri, 6 May 2022 21:16:27 +0800 Subject: [PATCH 07/18] virtio-crypto: enable retry for virtio-crypto-dev Enable retry for virtio-crypto-dev, so that crypto-engine can process cipher-requests parallelly. Cc: Michael S. Tsirkin Cc: Jason Wang Cc: Gonglei Reviewed-by: Gonglei Signed-off-by: lei he Signed-off-by: zhenwei pi Message-Id: <20220506131627.180784-6-pizhenwei@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/crypto/virtio/virtio_crypto_core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/virtio/virtio_crypto_core.c b/drivers/crypto/virtio/virtio_crypto_core.c index 60490ffa3df1..1198bd306365 100644 --- a/drivers/crypto/virtio/virtio_crypto_core.c +++ b/drivers/crypto/virtio/virtio_crypto_core.c @@ -144,7 +144,8 @@ static int virtcrypto_find_vqs(struct virtio_crypto *vi) spin_lock_init(&vi->data_vq[i].lock); vi->data_vq[i].vq = vqs[i]; /* Initialize crypto engine */ - vi->data_vq[i].engine = crypto_engine_alloc_init(dev, 1); + vi->data_vq[i].engine = crypto_engine_alloc_init_and_set(dev, true, NULL, true, + virtqueue_get_vring_size(vqs[i])); if (!vi->data_vq[i].engine) { ret = -ENOMEM; goto err_engine; From 3153234097f6a0d06981565eb3eec3cb37dea8f8 Mon Sep 17 00:00:00 2001 From: Solomon Tan Date: Mon, 18 Apr 2022 02:54:35 +0000 Subject: [PATCH 08/18] virtio: Replace unsigned with unsigned int This patch addresses the checkpatch.pl warning where unsigned int is preferred over unsigned. Signed-off-by: Solomon Tan Message-Id: Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio.c | 2 +- drivers/virtio/virtio_balloon.c | 12 ++++++------ drivers/virtio/virtio_mmio.c | 12 ++++++------ drivers/virtio/virtio_pci_common.c | 12 ++++++------ drivers/virtio/virtio_pci_common.h | 10 +++++----- drivers/virtio/virtio_pci_legacy.c | 10 +++++----- drivers/virtio/virtio_pci_modern.c | 12 ++++++------ drivers/virtio/virtio_ring.c | 12 ++++++------ drivers/virtio/virtio_vdpa.c | 10 +++++----- 9 files changed, 46 insertions(+), 46 deletions(-) diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 22f15f444f75..ce424c16997d 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -169,7 +169,7 @@ EXPORT_SYMBOL_GPL(virtio_add_status); /* Do some validation, then set FEATURES_OK */ static int virtio_features_ok(struct virtio_device *dev) { - unsigned status; + unsigned int status; int ret; might_sleep(); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index f4c34a2a6b8e..b9737da6c4dd 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -27,7 +27,7 @@ * multiple balloon pages. All memory counters in this driver are in balloon * page units. */ -#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) +#define VIRTIO_BALLOON_PAGES_PER_PAGE (unsigned int)(PAGE_SIZE >> VIRTIO_BALLOON_PFN_SHIFT) #define VIRTIO_BALLOON_ARRAY_PFNS_MAX 256 /* Maximum number of (4k) pages to deflate on OOM notifications. */ #define VIRTIO_BALLOON_OOM_NR_PAGES 256 @@ -208,10 +208,10 @@ static void set_page_pfns(struct virtio_balloon *vb, page_to_balloon_pfn(page) + i); } -static unsigned fill_balloon(struct virtio_balloon *vb, size_t num) +static unsigned int fill_balloon(struct virtio_balloon *vb, size_t num) { - unsigned num_allocated_pages; - unsigned num_pfns; + unsigned int num_allocated_pages; + unsigned int num_pfns; struct page *page; LIST_HEAD(pages); @@ -272,9 +272,9 @@ static void release_pages_balloon(struct virtio_balloon *vb, } } -static unsigned leak_balloon(struct virtio_balloon *vb, size_t num) +static unsigned int leak_balloon(struct virtio_balloon *vb, size_t num) { - unsigned num_freed_pages; + unsigned int num_freed_pages; struct page *page; struct balloon_dev_info *vb_dev_info = &vb->vb_dev_info; LIST_HEAD(pages); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 56128b9c46eb..b717302dc4ac 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -144,8 +144,8 @@ static int vm_finalize_features(struct virtio_device *vdev) return 0; } -static void vm_get(struct virtio_device *vdev, unsigned offset, - void *buf, unsigned len) +static void vm_get(struct virtio_device *vdev, unsigned int offset, + void *buf, unsigned int len) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); void __iomem *base = vm_dev->base + VIRTIO_MMIO_CONFIG; @@ -186,8 +186,8 @@ static void vm_get(struct virtio_device *vdev, unsigned offset, } } -static void vm_set(struct virtio_device *vdev, unsigned offset, - const void *buf, unsigned len) +static void vm_set(struct virtio_device *vdev, unsigned int offset, + const void *buf, unsigned int len) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); void __iomem *base = vm_dev->base + VIRTIO_MMIO_CONFIG; @@ -345,7 +345,7 @@ static void vm_del_vqs(struct virtio_device *vdev) free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev); } -static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, +static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, bool ctx) { @@ -455,7 +455,7 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index, return ERR_PTR(err); } -static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs, +static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 5046efcffb4c..ca51fcc9daab 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -104,8 +104,8 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, { struct virtio_pci_device *vp_dev = to_vp_device(vdev); const char *name = dev_name(&vp_dev->vdev.dev); - unsigned flags = PCI_IRQ_MSIX; - unsigned i, v; + unsigned int flags = PCI_IRQ_MSIX; + unsigned int i, v; int err = -ENOMEM; vp_dev->msix_vectors = nvectors; @@ -171,7 +171,7 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, return err; } -static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index, +static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, bool ctx, @@ -275,7 +275,7 @@ void vp_del_vqs(struct virtio_device *vdev) vp_dev->vqs = NULL; } -static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs, +static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], bool per_vq_vectors, const bool *ctx, @@ -349,7 +349,7 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned nvqs, return err; } -static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned nvqs, +static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], const bool *ctx) { @@ -388,7 +388,7 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned nvqs, } /* the config->find_vqs() implementation */ -int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, +int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], const bool *ctx, struct irq_affinity *desc) diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index eb17a29fc7ef..23112d84218f 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -38,7 +38,7 @@ struct virtio_pci_vq_info { struct list_head node; /* MSI-X vector (or none) */ - unsigned msix_vector; + unsigned int msix_vector; }; /* Our device structure */ @@ -68,16 +68,16 @@ struct virtio_pci_device { * and I'm too lazy to allocate each name separately. */ char (*msix_names)[256]; /* Number of available vectors */ - unsigned msix_vectors; + unsigned int msix_vectors; /* Vectors allocated, excluding per-vq vectors if any */ - unsigned msix_used_vectors; + unsigned int msix_used_vectors; /* Whether we have vector per vq */ bool per_vq_vectors; struct virtqueue *(*setup_vq)(struct virtio_pci_device *vp_dev, struct virtio_pci_vq_info *info, - unsigned idx, + unsigned int idx, void (*callback)(struct virtqueue *vq), const char *name, bool ctx, @@ -108,7 +108,7 @@ bool vp_notify(struct virtqueue *vq); /* the config->del_vqs() implementation */ void vp_del_vqs(struct virtio_device *vdev); /* the config->find_vqs() implementation */ -int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, +int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], const bool *ctx, struct irq_affinity *desc); diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index 6f4e34ce96b8..7fe4caa4b519 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -45,8 +45,8 @@ static int vp_finalize_features(struct virtio_device *vdev) } /* virtio config->get() implementation */ -static void vp_get(struct virtio_device *vdev, unsigned offset, - void *buf, unsigned len) +static void vp_get(struct virtio_device *vdev, unsigned int offset, + void *buf, unsigned int len) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); void __iomem *ioaddr = vp_dev->ldev.ioaddr + @@ -61,8 +61,8 @@ static void vp_get(struct virtio_device *vdev, unsigned offset, /* the config->set() implementation. it's symmetric to the config->get() * implementation */ -static void vp_set(struct virtio_device *vdev, unsigned offset, - const void *buf, unsigned len) +static void vp_set(struct virtio_device *vdev, unsigned int offset, + const void *buf, unsigned int len) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); void __iomem *ioaddr = vp_dev->ldev.ioaddr + @@ -109,7 +109,7 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, struct virtio_pci_vq_info *info, - unsigned index, + unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, bool ctx, diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index a2671a20ef77..4acb34409f0b 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -60,8 +60,8 @@ static int vp_finalize_features(struct virtio_device *vdev) } /* virtio config->get() implementation */ -static void vp_get(struct virtio_device *vdev, unsigned offset, - void *buf, unsigned len) +static void vp_get(struct virtio_device *vdev, unsigned int offset, + void *buf, unsigned int len) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtio_pci_modern_device *mdev = &vp_dev->mdev; @@ -98,8 +98,8 @@ static void vp_get(struct virtio_device *vdev, unsigned offset, /* the config->set() implementation. it's symmetric to the config->get() * implementation */ -static void vp_set(struct virtio_device *vdev, unsigned offset, - const void *buf, unsigned len) +static void vp_set(struct virtio_device *vdev, unsigned int offset, + const void *buf, unsigned int len) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtio_pci_modern_device *mdev = &vp_dev->mdev; @@ -183,7 +183,7 @@ static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, struct virtio_pci_vq_info *info, - unsigned index, + unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, bool ctx, @@ -248,7 +248,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, return ERR_PTR(err); } -static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, +static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], const bool *ctx, diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 0dc930de72f5..9d0bae4293be 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -809,7 +809,7 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq) } } -static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq) +static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); u16 last_used_idx; @@ -834,7 +834,7 @@ static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq) return last_used_idx; } -static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned last_used_idx) +static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx) { struct vring_virtqueue *vq = to_vvq(_vq); @@ -1486,7 +1486,7 @@ static void virtqueue_disable_cb_packed(struct virtqueue *_vq) } } -static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq) +static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); @@ -2025,7 +2025,7 @@ EXPORT_SYMBOL_GPL(virtqueue_disable_cb); * Caller must ensure we don't call this with other virtqueue * operations at the same time (except where noted). */ -unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq) +unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); @@ -2046,7 +2046,7 @@ EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare); * * This does not need to be serialized. */ -bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx) +bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx) { struct vring_virtqueue *vq = to_vvq(_vq); @@ -2072,7 +2072,7 @@ EXPORT_SYMBOL_GPL(virtqueue_poll); */ bool virtqueue_enable_cb(struct virtqueue *_vq) { - unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq); + unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq); return !virtqueue_poll(_vq, last_used_idx); } diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 46c71653f508..c40f7deb6b5a 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -53,16 +53,16 @@ static struct vdpa_device *vd_get_vdpa(struct virtio_device *vdev) return to_virtio_vdpa_device(vdev)->vdpa; } -static void virtio_vdpa_get(struct virtio_device *vdev, unsigned offset, - void *buf, unsigned len) +static void virtio_vdpa_get(struct virtio_device *vdev, unsigned int offset, + void *buf, unsigned int len) { struct vdpa_device *vdpa = vd_get_vdpa(vdev); vdpa_get_config(vdpa, offset, buf, len); } -static void virtio_vdpa_set(struct virtio_device *vdev, unsigned offset, - const void *buf, unsigned len) +static void virtio_vdpa_set(struct virtio_device *vdev, unsigned int offset, + const void *buf, unsigned int len) { struct vdpa_device *vdpa = vd_get_vdpa(vdev); @@ -263,7 +263,7 @@ static void virtio_vdpa_del_vqs(struct virtio_device *vdev) virtio_vdpa_del_vq(vq); } -static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned nvqs, +static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], From 0619eda83d905a9e8eedfb5699d6f520f901281c Mon Sep 17 00:00:00 2001 From: Solomon Tan Date: Mon, 18 Apr 2022 02:56:24 +0000 Subject: [PATCH 09/18] virtio: Replace long long int with long long This patch addresses the checkpatch.pl warning that long long is preferred over long long int. Signed-off-by: Solomon Tan Message-Id: Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mmio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index b717302dc4ac..839684d672af 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -657,7 +657,7 @@ static int vm_cmdline_set(const char *device, int err; struct resource resources[2] = {}; char *str; - long long int base, size; + long long base, size; unsigned int irq; int processed, consumed = 0; struct platform_device *pdev; From ffbda8e9df10d1784d5427ec199e7d8308e3763f Mon Sep 17 00:00:00 2001 From: Cindy Lu Date: Fri, 29 Apr 2022 17:10:30 +0800 Subject: [PATCH 10/18] vdpa/vp_vdpa : add vdpa tool support in vp_vdpa this patch is to add the support for vdpa tool in vp_vdpa here is the example steps modprobe vp_vdpa modprobe vhost_vdpa echo 0000:00:06.0>/sys/bus/pci/drivers/virtio-pci/unbind echo 1af4 1041 > /sys/bus/pci/drivers/vp-vdpa/new_id vdpa dev add name vdpa1 mgmtdev pci/0000:00:06.0 Signed-off-by: Cindy Lu Message-Id: <20220429091030.547434-1-lulu@redhat.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/virtio_pci/vp_vdpa.c | 161 ++++++++++++++++++++++++------ include/linux/vdpa.h | 2 +- 2 files changed, 130 insertions(+), 33 deletions(-) diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c index 35acba0e8d6d..04522077735b 100644 --- a/drivers/vdpa/virtio_pci/vp_vdpa.c +++ b/drivers/vdpa/virtio_pci/vp_vdpa.c @@ -32,7 +32,7 @@ struct vp_vring { struct vp_vdpa { struct vdpa_device vdpa; - struct virtio_pci_modern_device mdev; + struct virtio_pci_modern_device *mdev; struct vp_vring *vring; struct vdpa_callback config_cb; char msix_name[VP_VDPA_NAME_SIZE]; @@ -41,6 +41,12 @@ struct vp_vdpa { int vectors; }; +struct vp_vdpa_mgmtdev { + struct vdpa_mgmt_dev mgtdev; + struct virtio_pci_modern_device *mdev; + struct vp_vdpa *vp_vdpa; +}; + static struct vp_vdpa *vdpa_to_vp(struct vdpa_device *vdpa) { return container_of(vdpa, struct vp_vdpa, vdpa); @@ -50,7 +56,12 @@ static struct virtio_pci_modern_device *vdpa_to_mdev(struct vdpa_device *vdpa) { struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); - return &vp_vdpa->mdev; + return vp_vdpa->mdev; +} + +static struct virtio_pci_modern_device *vp_vdpa_to_mdev(struct vp_vdpa *vp_vdpa) +{ + return vp_vdpa->mdev; } static u64 vp_vdpa_get_device_features(struct vdpa_device *vdpa) @@ -96,7 +107,7 @@ static int vp_vdpa_get_vq_irq(struct vdpa_device *vdpa, u16 idx) static void vp_vdpa_free_irq(struct vp_vdpa *vp_vdpa) { - struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; + struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); struct pci_dev *pdev = mdev->pci_dev; int i; @@ -143,7 +154,7 @@ static irqreturn_t vp_vdpa_config_handler(int irq, void *arg) static int vp_vdpa_request_irq(struct vp_vdpa *vp_vdpa) { - struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; + struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); struct pci_dev *pdev = mdev->pci_dev; int i, ret, irq; int queues = vp_vdpa->queues; @@ -198,7 +209,7 @@ static int vp_vdpa_request_irq(struct vp_vdpa *vp_vdpa) static void vp_vdpa_set_status(struct vdpa_device *vdpa, u8 status) { struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); - struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; + struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); u8 s = vp_vdpa_get_status(vdpa); if (status & VIRTIO_CONFIG_S_DRIVER_OK && @@ -212,7 +223,7 @@ static void vp_vdpa_set_status(struct vdpa_device *vdpa, u8 status) static int vp_vdpa_reset(struct vdpa_device *vdpa) { struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); - struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; + struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); u8 s = vp_vdpa_get_status(vdpa); vp_modern_set_status(mdev, 0); @@ -372,7 +383,7 @@ static void vp_vdpa_get_config(struct vdpa_device *vdpa, void *buf, unsigned int len) { struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); - struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; + struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); u8 old, new; u8 *p; int i; @@ -392,7 +403,7 @@ static void vp_vdpa_set_config(struct vdpa_device *vdpa, unsigned int len) { struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); - struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; + struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); const u8 *p = buf; int i; @@ -412,7 +423,7 @@ static struct vdpa_notification_area vp_vdpa_get_vq_notification(struct vdpa_device *vdpa, u16 qid) { struct vp_vdpa *vp_vdpa = vdpa_to_vp(vdpa); - struct virtio_pci_modern_device *mdev = &vp_vdpa->mdev; + struct virtio_pci_modern_device *mdev = vp_vdpa_to_mdev(vp_vdpa); struct vdpa_notification_area notify; notify.addr = vp_vdpa->vring[qid].notify_pa; @@ -454,38 +465,31 @@ static void vp_vdpa_free_irq_vectors(void *data) pci_free_irq_vectors(data); } -static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) +static int vp_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, + const struct vdpa_dev_set_config *add_config) { - struct virtio_pci_modern_device *mdev; + struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = + container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev); + + struct virtio_pci_modern_device *mdev = vp_vdpa_mgtdev->mdev; + struct pci_dev *pdev = mdev->pci_dev; struct device *dev = &pdev->dev; - struct vp_vdpa *vp_vdpa; + struct vp_vdpa *vp_vdpa = NULL; int ret, i; - ret = pcim_enable_device(pdev); - if (ret) - return ret; - vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa, - dev, &vp_vdpa_ops, 1, 1, NULL, false); + dev, &vp_vdpa_ops, 1, 1, name, false); + if (IS_ERR(vp_vdpa)) { dev_err(dev, "vp_vdpa: Failed to allocate vDPA structure\n"); return PTR_ERR(vp_vdpa); } - mdev = &vp_vdpa->mdev; - mdev->pci_dev = pdev; - - ret = vp_modern_probe(mdev); - if (ret) { - dev_err(&pdev->dev, "Failed to probe modern PCI device\n"); - goto err; - } - - pci_set_master(pdev); - pci_set_drvdata(pdev, vp_vdpa); + vp_vdpa_mgtdev->vp_vdpa = vp_vdpa; vp_vdpa->vdpa.dma_dev = &pdev->dev; vp_vdpa->queues = vp_modern_get_num_queues(mdev); + vp_vdpa->mdev = mdev; ret = devm_add_action_or_reset(dev, vp_vdpa_free_irq_vectors, pdev); if (ret) { @@ -516,7 +520,8 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) } vp_vdpa->config_irq = VIRTIO_MSI_NO_VECTOR; - ret = vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues); + vp_vdpa->vdpa.mdev = &vp_vdpa_mgtdev->mgtdev; + ret = _vdpa_register_device(&vp_vdpa->vdpa, vp_vdpa->queues); if (ret) { dev_err(&pdev->dev, "Failed to register to vdpa bus\n"); goto err; @@ -529,12 +534,104 @@ static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) return ret; } +static void vp_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, + struct vdpa_device *dev) +{ + struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = + container_of(v_mdev, struct vp_vdpa_mgmtdev, mgtdev); + + struct vp_vdpa *vp_vdpa = vp_vdpa_mgtdev->vp_vdpa; + + _vdpa_unregister_device(&vp_vdpa->vdpa); + vp_vdpa_mgtdev->vp_vdpa = NULL; +} + +static const struct vdpa_mgmtdev_ops vp_vdpa_mdev_ops = { + .dev_add = vp_vdpa_dev_add, + .dev_del = vp_vdpa_dev_del, +}; + +static int vp_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = NULL; + struct vdpa_mgmt_dev *mgtdev; + struct device *dev = &pdev->dev; + struct virtio_pci_modern_device *mdev = NULL; + struct virtio_device_id *mdev_id = NULL; + int err; + + vp_vdpa_mgtdev = kzalloc(sizeof(*vp_vdpa_mgtdev), GFP_KERNEL); + if (!vp_vdpa_mgtdev) + return -ENOMEM; + + mgtdev = &vp_vdpa_mgtdev->mgtdev; + mgtdev->ops = &vp_vdpa_mdev_ops; + mgtdev->device = dev; + + mdev = kzalloc(sizeof(struct virtio_pci_modern_device), GFP_KERNEL); + if (!mdev) { + err = -ENOMEM; + goto mdev_err; + } + + mdev_id = kzalloc(sizeof(struct virtio_device_id), GFP_KERNEL); + if (!mdev_id) { + err = -ENOMEM; + goto mdev_id_err; + } + + vp_vdpa_mgtdev->mdev = mdev; + mdev->pci_dev = pdev; + + err = pcim_enable_device(pdev); + if (err) { + goto probe_err; + } + + err = vp_modern_probe(mdev); + if (err) { + dev_err(&pdev->dev, "Failed to probe modern PCI device\n"); + goto probe_err; + } + + mdev_id->device = mdev->id.device; + mdev_id->vendor = mdev->id.vendor; + mgtdev->id_table = mdev_id; + mgtdev->max_supported_vqs = vp_modern_get_num_queues(mdev); + mgtdev->supported_features = vp_modern_get_features(mdev); + pci_set_master(pdev); + pci_set_drvdata(pdev, vp_vdpa_mgtdev); + + err = vdpa_mgmtdev_register(mgtdev); + if (err) { + dev_err(&pdev->dev, "Failed to register vdpa mgmtdev device\n"); + goto register_err; + } + + return 0; + +register_err: + vp_modern_remove(vp_vdpa_mgtdev->mdev); +probe_err: + kfree(mdev_id); +mdev_id_err: + kfree(mdev); +mdev_err: + kfree(vp_vdpa_mgtdev); + return err; +} + static void vp_vdpa_remove(struct pci_dev *pdev) { - struct vp_vdpa *vp_vdpa = pci_get_drvdata(pdev); + struct vp_vdpa_mgmtdev *vp_vdpa_mgtdev = pci_get_drvdata(pdev); + struct virtio_pci_modern_device *mdev = NULL; - vp_modern_remove(&vp_vdpa->mdev); - vdpa_unregister_device(&vp_vdpa->vdpa); + mdev = vp_vdpa_mgtdev->mdev; + vp_modern_remove(mdev); + vdpa_mgmtdev_unregister(&vp_vdpa_mgtdev->mgtdev); + kfree(&vp_vdpa_mgtdev->mgtdev.id_table); + kfree(mdev); + kfree(vp_vdpa_mgtdev); } static struct pci_driver vp_vdpa_driver = { diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index f336d253db3d..15af802d41c4 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -492,7 +492,7 @@ struct vdpa_mgmtdev_ops { struct vdpa_mgmt_dev { struct device *device; const struct vdpa_mgmtdev_ops *ops; - const struct virtio_device_id *id_table; + struct virtio_device_id *id_table; u64 config_attr_mask; struct list_head list; u64 supported_features; From ac33f84ba5ffcc6c8a4c3ee98c92f722feb64f43 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Sun, 24 Apr 2022 15:28:06 +0800 Subject: [PATCH 11/18] vDPA/ifcvf: fix uninitialized config_vector warning Static checkers are not informed that config_vector is controlled by vf->msix_vector_status, which can only be MSIX_VECTOR_SHARED_VQ_AND_CONFIG, MSIX_VECTOR_SHARED_VQ_AND_CONFIG and MSIX_VECTOR_DEV_SHARED. This commit uses an "if...elseif...else" code block to tell the checkers that it is a complete set, and config_vector can be initialized anyway Signed-off-by: Zhu Lingshan Reviewed-by: Dan Carpenter Message-Id: <20220424072806.1083189-1-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/ifcvf/ifcvf_main.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index c1767a0ce630..750e5f23406d 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -290,16 +290,16 @@ static int ifcvf_request_config_irq(struct ifcvf_adapter *adapter) struct ifcvf_hw *vf = &adapter->vf; int config_vector, ret; - if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED) - return 0; - if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG) - /* vector 0 ~ vf->nr_vring for vqs, num vf->nr_vring vector for config interrupt */ config_vector = vf->nr_vring; - - if (vf->msix_vector_status == MSIX_VECTOR_SHARED_VQ_AND_CONFIG) + else if (vf->msix_vector_status == MSIX_VECTOR_SHARED_VQ_AND_CONFIG) /* vector 0 for vqs and 1 for config interrupt */ config_vector = 1; + else if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED) + /* re-use the vqs vector */ + return 0; + else + return -EINVAL; snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n", pci_name(pdev)); From 242436973831aa97e8ce19533c6c912ea8def31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eugenio=20P=C3=A9rez?= Date: Thu, 19 May 2022 16:59:19 +0200 Subject: [PATCH 12/18] vdpasim: allow to enable a vq repeatedly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code must be resilient to enable a queue many times. At the moment the queue is resetting so it's definitely not the expected behavior. v2: set vq->ready = 0 at disable. Fixes: 2c53d0f64c06 ("vdpasim: vDPA device simulator") Cc: stable@vger.kernel.org Signed-off-by: Eugenio Pérez Message-Id: <20220519145919.772896-1-eperezma@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Stefano Garzarella --- drivers/vdpa/vdpa_sim/vdpa_sim.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 51bd0bafce06..50d721072beb 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -364,11 +364,14 @@ static void vdpasim_set_vq_ready(struct vdpa_device *vdpa, u16 idx, bool ready) { struct vdpasim *vdpasim = vdpa_to_sim(vdpa); struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx]; + bool old_ready; spin_lock(&vdpasim->lock); + old_ready = vq->ready; vq->ready = ready; - if (vq->ready) + if (vq->ready && !old_ready) { vdpasim_queue_ready(vdpasim, idx); + } spin_unlock(&vdpasim->lock); } From 2536b2ca15418c517e3629cc3dd757f811ce52b2 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Fri, 27 May 2022 14:01:12 +0800 Subject: [PATCH 13/18] virtio: use virtio_device_ready() in virtio_device_restore() It will allow us to do extension on virtio_device_ready() without duplicating code. Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: "Paul E. McKenney" Cc: Marc Zyngier Cc: Halil Pasic Cc: Cornelia Huck Cc: Vineeth Vijayan Cc: Peter Oberparleiter Cc: linux-s390@vger.kernel.org Reviewed-by: Cornelia Huck Signed-off-by: Stefano Garzarella Signed-off-by: Jason Wang Message-Id: <20220527060120.20964-2-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Xuan Zhuo --- drivers/virtio/virtio.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index ce424c16997d..938e975029d4 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -526,8 +526,9 @@ int virtio_device_restore(struct virtio_device *dev) goto err; } - /* Finally, tell the device we're all set */ - virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); + /* If restore didn't do it, mark device DRIVER_OK ourselves. */ + if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK)) + virtio_device_ready(dev); virtio_config_enable(dev); From 0aa96837c3ded5de58782a07b88176f2f1d01404 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 27 May 2022 14:01:13 +0800 Subject: [PATCH 14/18] virtio: use virtio_reset_device() when possible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows us to do common extension without duplicating code. Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: "Paul E. McKenney" Cc: Marc Zyngier Cc: Halil Pasic Cc: Cornelia Huck Cc: Vineeth Vijayan Cc: Peter Oberparleiter Cc: linux-s390@vger.kernel.org Reviewed-by: Cornelia Huck Signed-off-by: Jason Wang Message-Id: <20220527060120.20964-3-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Xuan Zhuo Reviewed-by: Eugenio Pérez --- drivers/virtio/virtio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 938e975029d4..aa1eb5132767 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -430,7 +430,7 @@ int register_virtio_device(struct virtio_device *dev) /* We always start by resetting the device, in case a previous * driver messed it up. This also tests that code path a little. */ - dev->config->reset(dev); + virtio_reset_device(dev); /* Acknowledge that we've seen the device. */ virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE); @@ -496,7 +496,7 @@ int virtio_device_restore(struct virtio_device *dev) /* We always start by resetting the device, in case a previous * driver messed it up. */ - dev->config->reset(dev); + virtio_reset_device(dev); /* Acknowledge that we've seen the device. */ virtio_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE); From 48b69959a8559db5df78661d9c6d5138c2f20871 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 27 May 2022 14:01:14 +0800 Subject: [PATCH 15/18] virtio: introduce config op to synchronize vring callbacks This patch introduces new virtio config op to vring callbacks. Transport specific method is required to make sure the write before this function is visible to the vring_interrupt() that is called after the return of this function. For the transport that doesn't provide synchronize_vqs(), use synchornize_rcu() which synchronize with IRQ implicitly as a fallback. Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: "Paul E. McKenney" Cc: Marc Zyngier Cc: Halil Pasic Cc: Cornelia Huck Cc: Vineeth Vijayan Cc: Peter Oberparleiter Cc: linux-s390@vger.kernel.org Reviewed-by: Cornelia Huck Signed-off-by: Jason Wang Message-Id: <20220527060120.20964-4-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Xuan Zhuo Reviewed-by: Stefano Garzarella --- include/linux/virtio_config.h | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index b341dd62aa4d..25be018810a7 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -57,6 +57,11 @@ struct virtio_shm_region { * include a NULL entry for vqs unused by driver * Returns 0 on success or error status * @del_vqs: free virtqueues found by find_vqs(). + * @synchronize_cbs: synchronize with the virtqueue callbacks (optional) + * The function guarantees that all memory operations on the + * queue before it are visible to the vring_interrupt() that is + * called after it. + * vdev: the virtio_device * @get_features: get the array of feature bits for this device. * vdev: the virtio_device * Returns the first 64 feature bits (all we currently need). @@ -89,6 +94,7 @@ struct virtio_config_ops { const char * const names[], const bool *ctx, struct irq_affinity *desc); void (*del_vqs)(struct virtio_device *); + void (*synchronize_cbs)(struct virtio_device *); u64 (*get_features)(struct virtio_device *vdev); int (*finalize_features)(struct virtio_device *vdev); const char *(*bus_name)(struct virtio_device *vdev); @@ -217,6 +223,25 @@ int virtio_find_vqs_ctx(struct virtio_device *vdev, unsigned nvqs, desc); } +/** + * virtio_synchronize_cbs - synchronize with virtqueue callbacks + * @vdev: the device + */ +static inline +void virtio_synchronize_cbs(struct virtio_device *dev) +{ + if (dev->config->synchronize_cbs) { + dev->config->synchronize_cbs(dev); + } else { + /* + * A best effort fallback to synchronize with + * interrupts, preemption and softirq disabled + * regions. See comment above synchronize_rcu(). + */ + synchronize_rcu(); + } +} + /** * virtio_device_ready - enable vq use in probe function * @vdev: the device From 48b3dd24382139a239a43bca7c1c632438d1dfb2 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 27 May 2022 14:01:15 +0800 Subject: [PATCH 16/18] virtio-pci: implement synchronize_cbs() We can simply reuse vp_synchronize_vectors() for .synchronize_cbs(). Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: "Paul E. McKenney" Cc: Marc Zyngier Cc: Halil Pasic Cc: Cornelia Huck Cc: Vineeth Vijayan Cc: Peter Oberparleiter Cc: linux-s390@vger.kernel.org Reviewed-by: Cornelia Huck Signed-off-by: Jason Wang Message-Id: <20220527060120.20964-5-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Xuan Zhuo --- drivers/virtio/virtio_pci_legacy.c | 1 + drivers/virtio/virtio_pci_modern.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index 7fe4caa4b519..a5e5721145c7 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -192,6 +192,7 @@ static const struct virtio_config_ops virtio_pci_config_ops = { .reset = vp_reset, .find_vqs = vp_find_vqs, .del_vqs = vp_del_vqs, + .synchronize_cbs = vp_synchronize_vectors, .get_features = vp_get_features, .finalize_features = vp_finalize_features, .bus_name = vp_bus_name, diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 4acb34409f0b..623906b4996c 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -394,6 +394,7 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { .reset = vp_reset, .find_vqs = vp_modern_find_vqs, .del_vqs = vp_del_vqs, + .synchronize_cbs = vp_synchronize_vectors, .get_features = vp_get_features, .finalize_features = vp_finalize_features, .bus_name = vp_bus_name, @@ -411,6 +412,7 @@ static const struct virtio_config_ops virtio_pci_config_ops = { .reset = vp_reset, .find_vqs = vp_modern_find_vqs, .del_vqs = vp_del_vqs, + .synchronize_cbs = vp_synchronize_vectors, .get_features = vp_get_features, .finalize_features = vp_finalize_features, .bus_name = vp_bus_name, From 9e9b2893284fa0681ab5c0632d02214385a174c3 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 27 May 2022 14:01:16 +0800 Subject: [PATCH 17/18] virtio-mmio: implement synchronize_cbs() Simply synchronize the platform irq that is used by us. Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: "Paul E. McKenney" Cc: Marc Zyngier Cc: Halil Pasic Cc: Cornelia Huck Cc: Vineeth Vijayan Cc: Peter Oberparleiter Cc: linux-s390@vger.kernel.org Reviewed-by: Cornelia Huck Signed-off-by: Jason Wang Message-Id: <20220527060120.20964-6-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Xuan Zhuo --- drivers/virtio/virtio_mmio.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 839684d672af..c9699a59f93c 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -345,6 +345,13 @@ static void vm_del_vqs(struct virtio_device *vdev) free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev); } +static void vm_synchronize_cbs(struct virtio_device *vdev) +{ + struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); + + synchronize_irq(platform_get_irq(vm_dev->pdev, 0)); +} + static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, bool ctx) @@ -541,6 +548,7 @@ static const struct virtio_config_ops virtio_mmio_config_ops = { .finalize_features = vm_finalize_features, .bus_name = vm_bus_name, .get_shm_region = vm_get_shm_region, + .synchronize_cbs = vm_synchronize_cbs, }; From 3a232277c1ac6df70552129a9722abc6ab250128 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Fri, 27 May 2022 14:01:17 +0800 Subject: [PATCH 18/18] virtio-ccw: implement synchronize_cbs() This patch tries to implement the synchronize_cbs() for ccw. For the vring_interrupt() that is called via virtio_airq_handler(), the synchronization is simply done via the airq_info's lock. For the vring_interrupt() that is called via virtio_ccw_int_handler(), a per device rwlock is introduced and used in the synchronization method. Cc: Thomas Gleixner Cc: Peter Zijlstra Cc: "Paul E. McKenney" Cc: Marc Zyngier Cc: Halil Pasic Cc: Cornelia Huck Cc: Vineeth Vijayan Cc: Peter Oberparleiter Cc: linux-s390@vger.kernel.org Reviewed-by: Halil Pasic Signed-off-by: Jason Wang Message-Id: <20220527060120.20964-7-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/s390/virtio/virtio_ccw.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index d35e7a3f7067..c188e4f20ca3 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -62,6 +62,7 @@ struct virtio_ccw_device { unsigned int revision; /* Transport revision */ wait_queue_head_t wait_q; spinlock_t lock; + rwlock_t irq_lock; struct mutex io_lock; /* Serializes I/O requests */ struct list_head virtqueues; bool is_thinint; @@ -984,6 +985,30 @@ static const char *virtio_ccw_bus_name(struct virtio_device *vdev) return dev_name(&vcdev->cdev->dev); } +static void virtio_ccw_synchronize_cbs(struct virtio_device *vdev) +{ + struct virtio_ccw_device *vcdev = to_vc_device(vdev); + struct airq_info *info = vcdev->airq_info; + + if (info) { + /* + * This device uses adapter interrupts: synchronize with + * vring_interrupt() called by virtio_airq_handler() + * via the indicator area lock. + */ + write_lock_irq(&info->lock); + write_unlock_irq(&info->lock); + } else { + /* This device uses classic interrupts: synchronize + * with vring_interrupt() called by + * virtio_ccw_int_handler() via the per-device + * irq_lock + */ + write_lock_irq(&vcdev->irq_lock); + write_unlock_irq(&vcdev->irq_lock); + } +} + static const struct virtio_config_ops virtio_ccw_config_ops = { .get_features = virtio_ccw_get_features, .finalize_features = virtio_ccw_finalize_features, @@ -995,6 +1020,7 @@ static const struct virtio_config_ops virtio_ccw_config_ops = { .find_vqs = virtio_ccw_find_vqs, .del_vqs = virtio_ccw_del_vqs, .bus_name = virtio_ccw_bus_name, + .synchronize_cbs = virtio_ccw_synchronize_cbs, }; @@ -1106,6 +1132,8 @@ static void virtio_ccw_int_handler(struct ccw_device *cdev, vcdev->err = -EIO; } virtio_ccw_check_activity(vcdev, activity); + /* Interrupts are disabled here */ + read_lock(&vcdev->irq_lock); for_each_set_bit(i, indicators(vcdev), sizeof(*indicators(vcdev)) * BITS_PER_BYTE) { /* The bit clear must happen before the vring kick. */ @@ -1114,6 +1142,7 @@ static void virtio_ccw_int_handler(struct ccw_device *cdev, vq = virtio_ccw_vq_by_ind(vcdev, i); vring_interrupt(0, vq); } + read_unlock(&vcdev->irq_lock); if (test_bit(0, indicators2(vcdev))) { virtio_config_changed(&vcdev->vdev); clear_bit(0, indicators2(vcdev)); @@ -1284,6 +1313,7 @@ static int virtio_ccw_online(struct ccw_device *cdev) init_waitqueue_head(&vcdev->wait_q); INIT_LIST_HEAD(&vcdev->virtqueues); spin_lock_init(&vcdev->lock); + rwlock_init(&vcdev->irq_lock); mutex_init(&vcdev->io_lock); spin_lock_irqsave(get_ccwdev_lock(cdev), flags);