diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 1285f04cdac1..7e5713367291 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -984,12 +984,16 @@ void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt) * @rqst: controlling RPC request * @srcp: points to RPC message payload in receive buffer * @copy_len: remaining length of receive buffer content - * @pad: Write chunk pad bytes needed (zero for pure inline) + * @writelist: bytes conveyed by the Write chunk (zero for pure inline) * * The upper layer has set the maximum number of bytes it can * receive in each component of rq_rcv_buf. These values are set in * the head.iov_len, page_len, tail.iov_len, and buflen fields. * + * For READ replies with a Write chunk, the payload is already in + * the page list. An overestimated head length and buflen can be + * reduced without moving reply data. + * * Unlike the TCP equivalent (xdr_partial_copy_from_skb), in * many cases this function simply updates iov_base pointers in * rq_rcv_buf to point directly to the received reply data, to @@ -998,13 +1002,16 @@ void rpcrdma_reset_cwnd(struct rpcrdma_xprt *r_xprt) * Returns the count of bytes which had to be memcopied. */ static unsigned long -rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad) +rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, + u32 writelist) { unsigned long fixup_copy_count; int i, npages, curlen; char *destp; struct page **ppages; int page_base; + int pad = writelist & 3; + unsigned int delta; /* The head iovec is redirected to the RPC reply message * in the receive buffer, to avoid a memcopy. @@ -1016,8 +1023,20 @@ rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad) * head.iov_len bytes are copied into the page list. */ curlen = rqst->rq_rcv_buf.head[0].iov_len; - if (curlen > copy_len) + if (curlen > copy_len) { + /* These READ replies have no inline fields after the payload. + * Trim the head without moving the directly-placed data. + */ + if (writelist) { + delta = curlen - copy_len; + + rqst->rq_rcv_buf.head[0].iov_len = copy_len; + rqst->rq_private_buf.head[0].iov_len = copy_len; + rqst->rq_rcv_buf.buflen -= delta; + rqst->rq_private_buf.buflen -= delta; + } curlen = copy_len; + } srcp += curlen; copy_len -= curlen; @@ -1245,7 +1264,7 @@ rpcrdma_decode_msg(struct rpcrdma_xprt *r_xprt, struct rpcrdma_rep *rep, base = (char *)xdr_inline_decode(xdr, 0); rpclen = xdr_stream_remaining(xdr); r_xprt->rx_stats.fixup_copy_count += - rpcrdma_inline_fixup(rqst, base, rpclen, writelist & 3); + rpcrdma_inline_fixup(rqst, base, rpclen, writelist); r_xprt->rx_stats.total_rdma_reply += writelist; return rpclen + xdr_align_size(writelist);