From f012e95b377c73c0283f009823c633104dedb337 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Wed, 1 Jun 2022 12:46:52 -0400 Subject: SUNRPC: Trap RDMA segment overflows Prevent svc_rdma_build_writes() from walking off the end of a Write chunk's segment array. Caught with KASAN. The test that this fix replaces is invalid, and might have been left over from an earlier prototype of the PCL work. Fixes: 7a1cbfa18059 ("svcrdma: Use parsed chunk lists to construct RDMA Writes") Signed-off-by: Chuck Lever --- net/sunrpc/xprtrdma/svc_rdma_rw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 5f0155fdefc7..11cf7c646644 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -478,10 +478,10 @@ svc_rdma_build_writes(struct svc_rdma_write_info *info, unsigned int write_len; u64 offset; - seg = &info->wi_chunk->ch_segments[info->wi_seg_no]; - if (!seg) + if (info->wi_seg_no >= info->wi_chunk->ch_segcount) goto out_overflow; + seg = &info->wi_chunk->ch_segments[info->wi_seg_no]; write_len = min(remaining, seg->rs_length - info->wi_seg_off); if (!write_len) goto out_overflow; -- cgit v1.2.3 From 6c254bf3b637dd4ef4f78eb78c7447419c0161d7 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 7 Jun 2022 16:47:52 -0400 Subject: SUNRPC: Fix the calculation of xdr->end in xdr_get_next_encode_buffer() I found that NFSD's new NFSv3 READDIRPLUS XDR encoder was screwing up right at the end of the page array. xdr_get_next_encode_buffer() does not compute the value of xdr->end correctly: * The check to see if we're on the final available page in xdr->buf needs to account for the space consumed by @nbytes. * The new xdr->end value needs to account for the portion of @nbytes that is to be encoded into the previous buffer. Fixes: 2825a7f90753 ("nfsd4: allow encoding across page boundaries") Signed-off-by: Chuck Lever Reviewed-by: NeilBrown Reviewed-by: J. Bruce Fields --- net/sunrpc/xdr.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index df194cc07035..b57cf9df4de8 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -979,7 +979,11 @@ static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, */ xdr->p = (void *)p + frag2bytes; space_left = xdr->buf->buflen - xdr->buf->len; - xdr->end = (void *)p + min_t(int, space_left, PAGE_SIZE); + if (space_left - nbytes >= PAGE_SIZE) + xdr->end = (void *)p + PAGE_SIZE; + else + xdr->end = (void *)p + space_left - frag1bytes; + xdr->buf->page_len += frag2bytes; xdr->buf->len += nbytes; return p; -- cgit v1.2.3 From 62ed448cc53b654036f7d7f3c99f299d79ad14c3 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 7 Jun 2022 16:47:58 -0400 Subject: SUNRPC: Optimize xdr_reserve_space() Transitioning between encode buffers is quite infrequent. It happens about 1 time in 400 calls to xdr_reserve_space(), measured on NFSD with a typical build/test workload. Force the compiler to remove that code from xdr_reserve_space(), which is a hot path on both the server and the client. This change reduces the size of xdr_reserve_space() from 10 cache lines to 2 when compiled with -Os. Signed-off-by: Chuck Lever Reviewed-by: J. Bruce Fields --- include/linux/sunrpc/xdr.h | 16 +++++++++++++++- net/sunrpc/xdr.c | 17 ++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) (limited to 'net/sunrpc') diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 4417f667c757..5860f32e3958 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -243,7 +243,7 @@ extern void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, extern __be32 *xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes); extern int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec, size_t nbytes); -extern void xdr_commit_encode(struct xdr_stream *xdr); +extern void __xdr_commit_encode(struct xdr_stream *xdr); extern void xdr_truncate_encode(struct xdr_stream *xdr, size_t len); extern int xdr_restrict_buflen(struct xdr_stream *xdr, int newbuflen); extern void xdr_write_pages(struct xdr_stream *xdr, struct page **pages, @@ -306,6 +306,20 @@ xdr_reset_scratch_buffer(struct xdr_stream *xdr) xdr_set_scratch_buffer(xdr, NULL, 0); } +/** + * xdr_commit_encode - Ensure all data is written to xdr->buf + * @xdr: pointer to xdr_stream + * + * Handle encoding across page boundaries by giving the caller a + * temporary location to write to, then later copying the data into + * place. __xdr_commit_encode() does that copying. + */ +static inline void xdr_commit_encode(struct xdr_stream *xdr) +{ + if (unlikely(xdr->scratch.iov_len)) + __xdr_commit_encode(xdr); +} + /** * xdr_stream_remaining - Return the number of bytes remaining in the stream * @xdr: pointer to struct xdr_stream diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index b57cf9df4de8..1ad8b4ef14de 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -919,7 +919,7 @@ void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p, EXPORT_SYMBOL_GPL(xdr_init_encode); /** - * xdr_commit_encode - Ensure all data is written to buffer + * __xdr_commit_encode - Ensure all data is written to buffer * @xdr: pointer to xdr_stream * * We handle encoding across page boundaries by giving the caller a @@ -931,22 +931,25 @@ EXPORT_SYMBOL_GPL(xdr_init_encode); * required at the end of encoding, or any other time when the xdr_buf * data might be read. */ -inline void xdr_commit_encode(struct xdr_stream *xdr) +void __xdr_commit_encode(struct xdr_stream *xdr) { int shift = xdr->scratch.iov_len; void *page; - if (shift == 0) - return; page = page_address(*xdr->page_ptr); memcpy(xdr->scratch.iov_base, page, shift); memmove(page, page + shift, (void *)xdr->p - page); xdr_reset_scratch_buffer(xdr); } -EXPORT_SYMBOL_GPL(xdr_commit_encode); +EXPORT_SYMBOL_GPL(__xdr_commit_encode); -static __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, - size_t nbytes) +/* + * The buffer space to be reserved crosses the boundary between + * xdr->buf->head and xdr->buf->pages, or between two pages + * in xdr->buf->pages. + */ +static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, + size_t nbytes) { __be32 *p; int space_left; -- cgit v1.2.3 From 90d871b3b9bb7ef8f835d6b53095f01b9c74b7b3 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 7 Jun 2022 16:48:05 -0400 Subject: SUNRPC: Clean up xdr_commit_encode() Both the kvec::iov_len field and the third parameter of memcpy() and memmove() are size_t. There's no reason for the implicit conversion from size_t to int and back. Change the type of @shift to make the code easier to read and understand. Signed-off-by: Chuck Lever Reviewed-by: NeilBrown Reviewed-by: J. Bruce Fields --- net/sunrpc/xdr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index 1ad8b4ef14de..3c182041e790 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -933,7 +933,7 @@ EXPORT_SYMBOL_GPL(xdr_init_encode); */ void __xdr_commit_encode(struct xdr_stream *xdr) { - int shift = xdr->scratch.iov_len; + size_t shift = xdr->scratch.iov_len; void *page; page = page_address(*xdr->page_ptr); -- cgit v1.2.3 From bd07a64176a2be03f5195c64943063fd119f9f21 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 7 Jun 2022 16:48:11 -0400 Subject: SUNRPC: Clean up xdr_get_next_encode_buffer() The value of @p is not used until the "location of the next item" is computed. Help human readers by moving its initial assignment to the paragraph where that value is used and by clarifying the antecedents in the documenting comment. Signed-off-by: Chuck Lever Reviewed-by: NeilBrown Reviewed-by: J. Bruce Fields --- net/sunrpc/xdr.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index 3c182041e790..eca02d122476 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -967,6 +967,7 @@ static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, xdr->buf->page_len += frag1bytes; xdr->page_ptr++; xdr->iov = NULL; + /* * If the last encode didn't end exactly on a page boundary, the * next one will straddle boundaries. Encode into the next @@ -975,11 +976,12 @@ static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, * space at the end of the previous buffer: */ xdr_set_scratch_buffer(xdr, xdr->p, frag1bytes); - p = page_address(*xdr->page_ptr); + /* - * Note this is where the next encode will start after we've - * shifted this one back: + * xdr->p is where the next encode will start after + * xdr_commit_encode() has shifted this one back: */ + p = page_address(*xdr->page_ptr); xdr->p = (void *)p + frag2bytes; space_left = xdr->buf->buflen - xdr->buf->len; if (space_left - nbytes >= PAGE_SIZE) -- cgit v1.2.3 From da9e94fe000e11f21d3d6f66012fe5c6379bd93c Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Tue, 7 Jun 2022 16:48:18 -0400 Subject: SUNRPC: Remove pointer type casts from xdr_get_next_encode_buffer() To make the code easier to read, remove visual clutter by changing the declared type of @p. Signed-off-by: Chuck Lever Reviewed-by: NeilBrown Reviewed-by: J. Bruce Fields --- net/sunrpc/xdr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'net/sunrpc') diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index eca02d122476..f87a2d8f23a7 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -951,9 +951,9 @@ EXPORT_SYMBOL_GPL(__xdr_commit_encode); static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, size_t nbytes) { - __be32 *p; int space_left; int frag1bytes, frag2bytes; + void *p; if (nbytes > PAGE_SIZE) goto out_overflow; /* Bigger buffers require special handling */ @@ -982,12 +982,12 @@ static noinline __be32 *xdr_get_next_encode_buffer(struct xdr_stream *xdr, * xdr_commit_encode() has shifted this one back: */ p = page_address(*xdr->page_ptr); - xdr->p = (void *)p + frag2bytes; + xdr->p = p + frag2bytes; space_left = xdr->buf->buflen - xdr->buf->len; if (space_left - nbytes >= PAGE_SIZE) - xdr->end = (void *)p + PAGE_SIZE; + xdr->end = p + PAGE_SIZE; else - xdr->end = (void *)p + space_left - frag1bytes; + xdr->end = p + space_left - frag1bytes; xdr->buf->page_len += frag2bytes; xdr->buf->len += nbytes; -- cgit v1.2.3