github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-09-13 06:23:02 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge patch series "netfs, cachefiles: Miscellaneous fixes"

Browse Source 
David Howells <dhowells@redhat.com> says:

Here are some miscellaneous fixes for netfslib and one for cachefiles:

The first six are all in unbuffered/DIO write:

 (1) Fix an uninitialised return value from netfs_unbuffered_write().

 (2) Fix the normal error return, preferring partial transfer size over
     request error over immediate error (e.g. EINTR).

 (3) Fix the async error return, preferring partial transfer size over
     error.

 (4) Fix the update of i_size on partial transfer ending in an error.

 (5) Fix a subrequest leak in an error path.

 (6) Fix the code to handle subrequest allocation failure.

Then there's:

 (7) Fix synchronisation issues with using a progressive rolling buffer for
     readahead by fetching everything into it upfront so that the issues
     don't arise.  Dropping the refs so acquired is deferred until after
     I/O is begun.

 (8) Change the marking of folios to be copied to the cache to be done
     whilst subreqs are being issued rather than at the time they're
     collected.  Whilst this is not strictly a fix, it means that the
     collector thread doesn't need to try and keep track of that, which
     means that delayed progress reporting will not be a problem in patch
     (9).

 (9) Fix read progress reporting to avoid 64-bit tearing on a 32-bit
     machine.  This has been modified from the previous submission as part
     of a different series to take account of a sashiko reported issue[1].

(10) Fix a potential UAF/KASAN warning reported by sashiko[1] in cachefiles
     in which the coherency data buffer is cast to a __be64* and
     dereferenced in a tracepoint - even though it might not be at least
     that large (or aligned).

* patches from https://patch.msgid.link/20260827134304.2075713-1-dhowells@redhat.com:
  cachefiles: Fix potential UAF/KASAN warning
  netfs: Fix read progress reporting
  netfs: Mark folios with COPY_TO_CACHE whilst issuing subreqs
  netfs: Fix readahead synchronisation issues by loading all folios upfront
  netfs: break unbuffered write when netfs_alloc_subrequest() fails
  netfs: Fix subreq ref leak
  netfs: Fix i_size update for partial transfer
  netfs: Fix error vs transferred passed to ->ki_complete()
  netfs: Fix unbuffered/DIO write partial transfer error return
  netfs: Fix uninitialized return value in netfs_unbuffered_write()

Link: https://patch.msgid.link/20260827134304.2075713-1-dhowells@redhat.com
Signed-off-by: Christian Brauner (Amutable) <brauner@kernel.org>
This commit is contained in:
Christian Brauner 2026-08-31 09:54:38 +02:00
commit 3884c9b120
No known key found for this signature in database
GPG Key ID: 91C61BC06578DCA2
16 changed files with 410 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
fs/cachefiles/xattr.c
View File

@ -13,6 +13,7 @@
#include <linux/quotaops.h>
#include <linux/xattr.h>
#include <linux/slab.h>
#include <linux/unaligned.h>
#include "internal.h"
#define CACHEFILES_COOKIE_TYPE_DATA 1
@ -50,7 +51,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object)
_enter("%x,#%d", object->debug_id, len);
buf = kmalloc(sizeof(struct cachefiles_xattr) + len, GFP_KERNEL);
buf = kmalloc(sizeof(struct cachefiles_xattr) + max(len, sizeof(__be64)), GFP_KERNEL);
if (!buf)
return -ENOMEM;
@ -60,6 +61,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object)
buf->content = object->content_info;
if (test_bit(FSCACHE_COOKIE_LOCAL_WRITE, &object->cookie->flags))
buf->content = CACHEFILES_CONTENT_DIRTY;
put_unaligned_be64(0, (__be64 *)buf->data);
if (len > 0)
memcpy(buf->data, fscache_get_aux(object->cookie), len);
@ -77,8 +79,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object)
trace_cachefiles_vfs_error(object, file_inode(file), ret,
cachefiles_trace_setxattr_error);
trace_cachefiles_coherency(object, file_inode(file)->i_ino,
be64_to_cpup((__be64 *)buf->data),
buf->content,
buf->data, buf->content,
cachefiles_coherency_set_fail);
if (ret != -ENOMEM)
cachefiles_io_error_obj(
@ -86,8 +87,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object)
"Failed to set xattr with error %d", ret);
} else {
trace_cachefiles_coherency(object, file_inode(file)->i_ino,
be64_to_cpup((__be64 *)buf->data),
buf->content,
buf->data, buf->content,
cachefiles_coherency_set_ok);
}
@ -110,9 +110,10 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file
int ret = -ESTALE;
tlen = sizeof(struct cachefiles_xattr) + len;
buf = kmalloc(tlen, GFP_KERNEL);
buf = kmalloc(sizeof(struct cachefiles_xattr) + max(len, sizeof(__be64)), GFP_KERNEL);
if (!buf)
return -ENOMEM;
put_unaligned_be64(0, (__be64 *)buf->data);
xlen = cachefiles_inject_read_error();
if (xlen == 0)
@ -148,8 +149,7 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file
out:
trace_cachefiles_coherency(object, file_inode(file)->i_ino,
be64_to_cpup((__be64 *)buf->data),
buf->content, why);
buf->data, buf->content, why);
kfree(buf);
return ret;
}

164
fs/netfs/buffered_read.c
View File

@ -54,6 +54,42 @@ static void netfs_rreq_expand(struct netfs_io_request *rreq,
}
}
/*
* Drop the folio refs acquired from the readahead API.
*/
static void netfs_bulk_drop_ra_refs(struct netfs_io_request *rreq)
{
struct folio_batch fbatch;
struct folio *folio;
pgoff_t nr_pages = DIV_ROUND_UP(rreq->len, PAGE_SIZE);
pgoff_t first = rreq->start / PAGE_SIZE;
XA_STATE(xas, &rreq->mapping->i_pages, first);
folio_batch_init(&fbatch);
rcu_read_lock();
xas_for_each(&xas, folio, first + nr_pages - 1) {
if (xas_retry(&xas, folio))
continue;
if (!folio_batch_add(&fbatch, folio))
folio_batch_release(&fbatch);
}
rcu_read_unlock();
folio_batch_release(&fbatch);
trace_netfs_rreq(rreq, netfs_rreq_trace_ra_put_ref);
clear_bit_unlock(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags);
wake_up(&rreq->waitq);
}
static void netfs_maybe_bulk_drop_ra_refs(struct netfs_io_request *rreq)
{
if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
netfs_bulk_drop_ra_refs(rreq);
}
/*
* Begin an operation, and fetch the stored zero point value from the cookie if
* available.
@ -74,12 +110,8 @@ static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_in
*
* Returns the limited size if successful and -ENOMEM if insufficient memory
* available.
*
* [!] NOTE: This must be run in the same thread as ->issue_read() was called
* in as we access the readahead_control struct.
*/
static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq,
struct readahead_control *ractl)
static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq)
{
struct netfs_io_request *rreq = subreq->rreq;
size_t rsize = subreq->len;
@ -87,30 +119,6 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq,
if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER)
rsize = umin(rsize, rreq->io_streams[0].sreq_max_len);
if (ractl) {
/* If we don't have sufficient folios in the rolling buffer,
* extract a folioq's worth from the readahead region at a time
* into the buffer. Note that this acquires a ref on each page
* that we will need to release later - but we don't want to do
* that until after we've started the I/O.
*/
struct folio_batch put_batch;
folio_batch_init(&put_batch);
while (rreq->submitted < subreq->start + rsize) {
ssize_t added;
added = rolling_buffer_load_from_ra(&rreq->buffer, ractl,
&put_batch);
if (added < 0) {
folio_batch_release(&put_batch);
return added;
}
rreq->submitted += added;
}
folio_batch_release(&put_batch);
}
subreq->len = rsize;
if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize,
@ -203,17 +211,68 @@ static void netfs_issue_read(struct netfs_io_request *rreq,
}
}
/*
* Mark folios that we want to copy to the cache. For filesystems that use
* netfslib fully, we set folio->private to NETFS_FOLIO_COPY_TO_CACHE;
* otherwise we set the deprecated PG_private_2.
*/
static void netfs_mark_copy_to_cache(struct netfs_io_request *rreq,
struct folio_queue **fq,
unsigned int *offset,
int *slot,
size_t len,
bool copy)
{
while (len > 0) {
struct folio *folio;
size_t fsize, overlap;
if (!*fq)
break;
if (*slot >= folioq_count(*fq)) {
*fq = (*fq)->next;
*slot = 0;
*offset = 0;
continue;
}
/* Determine how much the subreq overlaps the folio, if at all. */
fsize = folioq_folio_size(*fq, *slot);
overlap = min(len, fsize - *offset);
if (overlap > 0 && copy) {
folio = folioq_folio(*fq, *slot);
if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags))) {
if (!folio_test_private_2(folio))
folio_start_private_2(folio);
} else {
if (!folio_get_private(folio))
folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
}
trace_netfs_folio(folio, netfs_folio_trace_mark_copy);
}
len -= overlap;
*offset += overlap;
if (*offset >= fsize) {
*slot += 1;
*offset = 0;
}
}
}
/*
* Perform a read to the pagecache from a series of sources of different types,
* slicing up the region to be read according to available cache blocks and
* network rsize.
*/
static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
struct readahead_control *ractl)
static void netfs_read_to_pagecache(struct netfs_io_request *rreq)
{
struct folio_queue *fq = rreq->buffer.tail;
unsigned long long start = rreq->start;
unsigned int offset = 0;
ssize_t size = rreq->len;
int ret = 0;
int ret = 0, slot = 0;
do {
struct netfs_io_subrequest *subreq;
@ -288,7 +347,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
break;
issue:
slice = netfs_prepare_read_iterator(subreq, ractl);
slice = netfs_prepare_read_iterator(subreq);
if (slice < 0) {
ret = slice;
netfs_cancel_read(subreq, ret);
@ -301,7 +360,15 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
}
if (fq) {
/* See if the cache indicated this should be cached. */
bool copy = test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags);
netfs_mark_copy_to_cache(rreq, &fq, &slot, &offset, slice, copy);
}
netfs_issue_read(rreq, subreq);
netfs_maybe_bulk_drop_ra_refs(rreq);
if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
netfs_wait_for_paused_read(rreq);
@ -339,7 +406,8 @@ void netfs_readahead(struct readahead_control *ractl)
{
struct netfs_io_request *rreq;
struct netfs_inode *ictx = netfs_inode(ractl->mapping->host);
unsigned long long start = readahead_pos(ractl);
ssize_t added;
uoff_t start = readahead_pos(ractl);
size_t size = readahead_length(ractl);
int ret;
@ -360,11 +428,24 @@ void netfs_readahead(struct readahead_control *ractl)
netfs_rreq_expand(rreq, ractl);
rreq->submitted = rreq->start;
if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST, rreq->gfp) < 0)
/* Load the folios to be read into a bvecq chain. Note that this
* acquires a ref on each folio that we will need to release later -
* but we don't want to do that until after we've started the I/O.
*/
added = rolling_buffer_bulk_load_from_ra(&rreq->buffer, ractl,
rreq->debug_id, rreq->gfp);
if (added < 0) {
ret = added;
goto cleanup_free;
netfs_read_to_pagecache(rreq, ractl);
}
__set_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags);
rreq->submitted = rreq->start + added;
rreq->cleaned_to = rreq->start;
netfs_read_set_unlock_at(rreq);
netfs_read_to_pagecache(rreq);
netfs_maybe_bulk_drop_ra_refs(rreq);
return netfs_put_request(rreq, netfs_rreq_trace_put_return);
cleanup_free:
@ -387,6 +468,7 @@ static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct fo
if (added < 0)
return added;
rreq->submitted = rreq->start + added;
rreq->progress_at = added;
return 0;
}
@ -457,7 +539,7 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
iov_iter_bvec(&rreq->buffer.iter, ITER_DEST, bvec, i, rreq->len);
rreq->submitted = rreq->start + flen;
netfs_read_to_pagecache(rreq, NULL);
netfs_read_to_pagecache(rreq);
ret = netfs_wait_for_read(rreq);
if (ret >= 0) {
@ -532,7 +614,7 @@ int netfs_read_folio(struct file *file, struct folio *folio)
if (ret < 0)
goto discard;
netfs_read_to_pagecache(rreq, NULL);
netfs_read_to_pagecache(rreq);
ret = netfs_wait_for_read(rreq);
netfs_put_request(rreq, netfs_rreq_trace_put_return);
return ret < 0 ? ret : 0;
@ -689,7 +771,7 @@ int netfs_write_begin(struct netfs_inode *ctx,
if (ret < 0)
goto error_put;
netfs_read_to_pagecache(rreq, NULL);
netfs_read_to_pagecache(rreq);
ret = netfs_wait_for_read(rreq);
netfs_put_request(rreq, netfs_rreq_trace_put_return);
if (ret < 0)
@ -754,7 +836,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio,
if (ret < 0)
goto error_put;
netfs_read_to_pagecache(rreq, NULL);
netfs_read_to_pagecache(rreq);
ret = netfs_wait_for_read(rreq);
netfs_put_request(rreq, netfs_rreq_trace_put_return);
return ret < 0 ? ret : 0;

31
fs/netfs/direct_write.c
View File

@ -21,7 +21,7 @@ static void netfs_unbuffered_write_done(struct netfs_io_request *wreq)
/* Okay, declare that all I/O is complete. */
trace_netfs_rreq(wreq, netfs_rreq_trace_write_done);
if (!wreq->error)
if (wreq->transferred)
netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred);
if (wreq->origin == NETFS_DIO_WRITE &&
@ -51,7 +51,7 @@ static void netfs_unbuffered_write_done(struct netfs_io_request *wreq)
wreq->iocb->ki_pos += written;
if (wreq->iocb->ki_complete) {
trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete);
wreq->iocb->ki_complete(wreq->iocb, wreq->error ?: written);
wreq->iocb->ki_complete(wreq->iocb, written ?: wreq->error);
}
wreq->iocb = VFS_PTR_POISON;
}
@ -95,7 +95,7 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
{
struct netfs_io_subrequest *subreq = NULL;
struct netfs_io_stream *stream = &wreq->io_streams[0];
int ret;
int ret = 0;
_enter("%llx", wreq->len);
@ -110,6 +110,11 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
if (!subreq) {
netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred);
subreq = stream->construct;
if (!subreq) {
wreq->error = -ENOMEM;
ret = -ENOMEM;
break;
}
stream->construct = NULL;
}
@ -121,8 +126,14 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
}
iov_iter_truncate(&subreq->io_iter, wreq->len - wreq->transferred);
if (!iov_iter_count(&subreq->io_iter))
if (!iov_iter_count(&subreq->io_iter)) {
pr_warn("netfs: Unexpected zero-length iterator R=%08x\n",
wreq->debug_id);
__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
netfs_write_subrequest_terminated(subreq, -EIO);
wreq->error = -EIO;
break;
}
subreq->len = netfs_limit_iter(&subreq->io_iter, 0,
stream->sreq_max_len,
@ -139,13 +150,11 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq)
if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
retry = true;
} else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) {
ret = subreq->error;
wreq->error = ret;
wreq->error = subreq->error;
netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed);
subreq = NULL;
break;
}
ret = 0;
if (!retry) {
netfs_unbuffered_write_collect(wreq, stream, subreq);
@ -288,11 +297,11 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *
ret = -EIOCBQUEUED;
} else {
ret = netfs_unbuffered_write(wreq);
if (ret < 0) {
_debug("begin = %zd", ret);
} else {
if (wreq->transferred) {
iocb->ki_pos += wreq->transferred;
ret = wreq->transferred ?: wreq->error;
ret = wreq->transferred;
} else if (wreq->error) {
ret = wreq->error;
}
netfs_put_request(wreq, netfs_rreq_trace_put_complete);

3
fs/netfs/internal.h
View File

@ -79,6 +79,7 @@ ssize_t netfs_wait_for_read(struct netfs_io_request *rreq);
ssize_t netfs_wait_for_write(struct netfs_io_request *rreq);
void netfs_wait_for_paused_read(struct netfs_io_request *rreq);
void netfs_wait_for_paused_write(struct netfs_io_request *rreq);
void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq);
/*
* objects.c
@ -109,6 +110,8 @@ static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq,
/*
* read_collect.c
*/
void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio);
void netfs_read_set_unlock_at(struct netfs_io_request *rreq);
bool netfs_read_collection(struct netfs_io_request *rreq);
void netfs_read_collection_worker(struct work_struct *work);
void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error);

19
fs/netfs/misc.c
View File

@ -563,3 +563,22 @@ void netfs_wait_for_paused_write(struct netfs_io_request *rreq)
{
return netfs_wait_for_pause(rreq, netfs_write_collection);
}
/*
* Wait for the readahead-acquired refs to be put.
*/
void netfs_wait_for_put_ra_refs(struct netfs_io_request *rreq)
{
DEFINE_WAIT(myself);
for (;;) {
trace_netfs_rreq(rreq, netfs_rreq_trace_wait_put_ra_refs);
prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
if (!test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
break;
schedule();
}
trace_netfs_rreq(rreq, netfs_rreq_trace_waited_put_ra_refs);
finish_wait(&rreq->waitq, &myself);
}

32
fs/netfs/objects.c
View File

@ -41,24 +41,32 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping,
memset(rreq, 0, kmem_cache_size(cache));
INIT_WORK(&rreq->cleanup_work, netfs_free_request);
rreq->gfp = gfp;
rreq->start = start;
rreq->len = len;
rreq->origin = origin;
rreq->netfs_ops = ctx->ops;
rreq->mapping = mapping;
rreq->inode = inode;
rreq->i_size = i_size_read(inode);
rreq->debug_id = atomic_inc_return(&debug_ids);
rreq->wsize = INT_MAX;
rreq->gfp = gfp;
rreq->start = start;
rreq->collected_to = start;
rreq->cleaned_to = start;
rreq->len = len;
rreq->progress_at = 0;
rreq->origin = origin;
rreq->netfs_ops = ctx->ops;
rreq->mapping = mapping;
rreq->inode = inode;
rreq->i_size = i_size_read(inode);
rreq->debug_id = atomic_inc_return(&debug_ids);
rreq->wsize = INT_MAX;
rreq->io_streams[0].sreq_max_len = ULONG_MAX;
rreq->io_streams[0].sreq_max_segs = 0;
spin_lock_init(&rreq->lock);
INIT_LIST_HEAD(&rreq->io_streams[0].subrequests);
INIT_LIST_HEAD(&rreq->io_streams[1].subrequests);
init_waitqueue_head(&rreq->waitq);
refcount_set(&rreq->ref, 2);
for (int s = 0; s < NR_IO_STREAMS; s++) {
struct netfs_io_stream *stream = &rreq->io_streams[s];
INIT_LIST_HEAD(&stream->subrequests);
stream->collected_to = rreq->start;
}
if (origin == NETFS_READAHEAD ||
origin == NETFS_READPAGE ||
origin == NETFS_READ_GAPS ||

128
fs/netfs/read_collect.c
View File

@ -19,7 +19,6 @@
#define MADE_PROGRESS 0x04 /* Made progress cleaning up a stream or the folio set */
#define BUFFERED 0x08 /* The pagecache needs cleaning up */
#define NEED_RETRY 0x10 /* A front op requests retrying */
#define COPY_TO_CACHE 0x40 /* Need to copy subrequest to cache */
#define ABANDON_SREQ 0x80 /* Need to abandon untransferred part of subrequest */
/*
@ -34,6 +33,30 @@ static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
__set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
}
/*
* Cancel the copy-to-cache mark on a folio.
*/
void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
{
if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
if (folio_get_private(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
folio_detach_private(folio);
trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
} else if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
struct netfs_folio *finfo = netfs_folio_info(folio);
finfo->netfs_group = NULL;
trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
}
} else {
// TODO: Use of PG_private_2 is deprecated.
if (folio_test_private_2(folio)) {
folio_end_private_2(folio);
trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
}
}
}
/*
* Flush, mark and unlock a folio that's now completely read. If we want to
* cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it
@ -48,37 +71,37 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
if (unlikely(folio_pos(folio) < rreq->abandon_to)) {
trace_netfs_folio(folio, netfs_folio_trace_abandon);
netfs_cancel_copy_to_cache(rreq, folio);
goto just_unlock;
}
flush_dcache_folio(folio);
folio_mark_uptodate(folio);
if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
finfo = netfs_folio_info(folio);
if (finfo) {
trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
if (finfo->netfs_group)
folio_change_private(folio, finfo->netfs_group);
else
folio_detach_private(folio);
kfree(finfo);
}
if (unlikely(test_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags)))
netfs_cancel_copy_to_cache(rreq, folio);
if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) {
if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) {
trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE);
folio_mark_dirty(folio);
}
if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
trace_netfs_folio(folio, netfs_folio_trace_sched_copy);
folio_mark_dirty(folio);
} else {
finfo = netfs_folio_info(folio);
if (finfo) {
trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
if (finfo->netfs_group)
folio_change_private(folio, finfo->netfs_group);
else
folio_detach_private(folio);
kfree(finfo);
}
trace_netfs_folio(folio, netfs_folio_trace_read_done);
}
folioq_clear(folioq, slot);
} else {
// TODO: Use of PG_private_2 is deprecated.
if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags))
if (folio_test_private_2(folio))
netfs_pgpriv2_copy_to_cache(rreq, folio);
}
@ -94,6 +117,35 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
folioq_clear(folioq, slot);
}
/*
* Determine how much to gather before unlocking more folios.
*/
void netfs_read_set_unlock_at(struct netfs_io_request *rreq)
{
struct folio_queue *folioq = rreq->buffer.tail;
unsigned int slot = rreq->buffer.first_tail_slot;
size_t cleaned_to = rreq->cleaned_to - rreq->start;
size_t progress_at = cleaned_to;
size_t minimum = 256 * 1024;
while (progress_at < rreq->len) {
if (slot >= folioq_count(folioq)) {
folioq = folioq->next;
if (!folioq)
break;
slot = 0;
}
progress_at += folioq_folio_size(folioq, slot);
if (progress_at - cleaned_to >= minimum)
break;
slot++;
}
WRITE_ONCE(rreq->progress_at, progress_at);
trace_netfs_read_progress_at(rreq);
}
/*
* Unlock any folios we've finished with.
*/
@ -112,30 +164,31 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
if (slot >= folioq_nr_slots(folioq)) {
folioq = rolling_buffer_delete_spent(&rreq->buffer);
if (!folioq) {
rreq->front_folio_order = 0;
WRITE_ONCE(rreq->progress_at, rreq->len);
return;
}
slot = 0;
}
/* We have to wait for readahead refs to have been released before we
* can unlock any folios as the ref-dropper walks i_pages and the only
* thing preventing these folios from being removed is the folio lock.
*/
if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
netfs_wait_for_put_ra_refs(rreq);
for (;;) {
struct folio *folio;
unsigned long long fpos, fend;
unsigned int order;
size_t fsize;
if (*notes & COPY_TO_CACHE)
set_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags);
folio = folioq_folio(folioq, slot);
if (WARN_ONCE(!folio_test_locked(folio),
"R=%08x: folio %lx is not locked\n",
rreq->debug_id, folio->index))
trace_netfs_folio(folio, netfs_folio_trace_not_locked);
order = folioq_folio_order(folioq, slot);
rreq->front_folio_order = order;
fsize = PAGE_SIZE << order;
fsize = folioq_folio_size(folioq, slot);
fpos = folio_pos(folio);
fend = fpos + fsize;
@ -149,8 +202,6 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
WRITE_ONCE(rreq->cleaned_to, fpos + fsize);
*notes |= MADE_PROGRESS;
clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags);
/* Clean up the head folioq. If we clear an entire folioq, then
* we can get rid of it provided it's not also the tail folioq
* being filled by the issuer.
@ -172,6 +223,8 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
rreq->buffer.tail = folioq;
done:
rreq->buffer.first_tail_slot = slot;
netfs_read_set_unlock_at(rreq);
}
/*
@ -232,7 +285,7 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq)
* subreqs.
*/
if (notes & BUFFERED) {
size_t fsize = PAGE_SIZE << rreq->front_folio_order;
uoff_t unlock_at = rreq->start + rreq->progress_at;
/* Clear the tail of a short read. */
if (!(notes & HIT_PENDING) &&
@ -248,16 +301,13 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq)
stream->collected_to = front->start + transferred;
rreq->collected_to = stream->collected_to;
if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &front->flags))
notes |= COPY_TO_CACHE;
if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {
rreq->abandon_to = front->start + front->len;
front->transferred = front->len;
transferred = front->len;
trace_netfs_rreq(rreq, netfs_rreq_trace_set_abandon);
}
if (front->start + transferred >= rreq->cleaned_to + fsize ||
if (front->start + transferred >= unlock_at ||
test_bit(NETFS_SREQ_HIT_EOF, &front->flags))
netfs_read_unlock_folios(rreq, &notes);
} else {
@ -477,20 +527,22 @@ void netfs_read_collection_worker(struct work_struct *work)
void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq)
{
struct netfs_io_request *rreq = subreq->rreq;
struct netfs_io_stream *stream = &rreq->io_streams[0];
size_t fsize = PAGE_SIZE << rreq->front_folio_order;
trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
size_t progress_at = READ_ONCE(rreq->progress_at);
uoff_t update_at = rreq->start + progress_at;
uoff_t transferred_to = subreq->start + subreq->transferred;
/* If we are at the head of the queue, wake up the collector,
* getting a ref to it if we were the ones to do so.
*/
if (subreq->start + subreq->transferred > rreq->cleaned_to + fsize &&
if (progress_at < rreq->len &&
transferred_to >= update_at &&
(rreq->origin == NETFS_READAHEAD ||
rreq->origin == NETFS_READPAGE ||
rreq->origin == NETFS_READ_FOR_WRITE) &&
list_is_first(&subreq->rreq_link, &stream->subrequests)
) {
trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
__set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
netfs_wake_collector(rreq);
}

15
fs/netfs/read_pgpriv2.c
View File

@ -54,8 +54,8 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio
/* Attach the folio to the rolling buffer. */
if (rolling_buffer_append(&creq->buffer, folio, 0, creq->gfp) < 0) {
set_bit(NETFS_RREQ_CANCEL_CACHING, &creq->flags);
folio_end_private_2(folio);
clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &creq->flags);
return;
}
@ -122,13 +122,14 @@ static struct netfs_io_request *netfs_pgpriv2_begin_copy_to_cache(
netfs_put_failed_request(creq);
cancel:
rreq->copy_to_cache = ERR_PTR(-ENOBUFS);
clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags);
set_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags);
return ERR_PTR(-ENOBUFS);
}
/*
* [DEPRECATED] Mark page as requiring copy-to-cache using PG_private_2 and add
* it to the copy write request.
* it to the copy write request. PG_private_2 should already be set on the
* folio.
*/
void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
{
@ -136,11 +137,13 @@ void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *fo
if (!creq)
creq = netfs_pgpriv2_begin_copy_to_cache(rreq, folio);
if (IS_ERR(creq))
if (IS_ERR(creq)) {
set_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags);
netfs_cancel_copy_to_cache(rreq, folio);
return;
}
trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache);
folio_start_private_2(folio);
trace_netfs_folio(folio, netfs_folio_trace_pgpriv2_copy);
netfs_pgpriv2_copy_folio(creq, folio);
}

13
fs/netfs/read_retry.c
View File

@ -292,11 +292,22 @@ void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
{
struct folio_queue *p;
/* We have to wait for readahead refs to have been released before we
* can unlock any folios as the ref-dropper walks i_pages and the only
* thing preventing these folios from being removed is the folio lock.
*/
if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
netfs_wait_for_put_ra_refs(rreq);
for (p = rreq->buffer.tail; p; p = p->next) {
for (int slot = 0; slot < folioq_count(p); slot++) {
struct folio *folio = folioq_folio(p, slot);
if (folio && !folioq_is_marked2(p, slot)) {
if (!folio)
continue;
netfs_cancel_copy_to_cache(rreq, folio);
if (!folioq_is_marked2(p, slot)) {
if (folio == rreq->no_unlock_folio &&
test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO,
&rreq->flags)) {

2
fs/netfs/read_single.c
View File

@ -170,6 +170,8 @@ ssize_t netfs_read_single(struct inode *inode, struct file *file, struct iov_ite
if (IS_ERR(rreq))
return PTR_ERR(rreq);
rreq->progress_at = rreq->len;
ret = netfs_single_begin_cache_read(rreq, ictx);
if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
goto cleanup_free;

79
fs/netfs/rolling_buffer.c
View File

@ -115,42 +115,65 @@ int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp)
}
/*
* Decant the list of folios to read into a rolling buffer.
* Decant the entire list of folios to read into a rolling buffer.
*/
ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll,
struct readahead_control *ractl,
struct folio_batch *put_batch)
ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll,
struct readahead_control *ractl,
unsigned int rreq_id, gfp_t gfp)
{
struct folio_queue *fq;
struct page **vec;
int nr, ix, to;
ssize_t size = 0;
ssize_t loaded = 0;
if (rolling_buffer_make_space(roll, GFP_KERNEL) < 0)
return -ENOMEM;
while (ractl->_nr_pages - ractl->_batch_count > 0) {
unsigned int nr;
fq = roll->head;
vec = (struct page **)fq->vec.folios;
nr = __readahead_batch(ractl, vec + folio_batch_count(&fq->vec),
folio_batch_space(&fq->vec));
ix = fq->vec.nr;
to = ix + nr;
fq->vec.nr = to;
for (; ix < to; ix++) {
struct folio *folio = folioq_folio(fq, ix);
unsigned int order = folio_order(folio);
/* Allocate a folioq to put some folios into and attach it to
* the rolling buffer.
*/
fq = netfs_folioq_alloc(rreq_id, gfp,
netfs_trace_folioq_make_space);
if (!fq)
goto nomem_unlock;
fq->prev = roll->head;
if (!roll->tail)
roll->tail = fq;
else
roll->head->next = fq;
roll->head = fq;
fq->orders[ix] = order;
size += PAGE_SIZE << order;
trace_netfs_folio(folio, netfs_folio_trace_read);
if (!folio_batch_add(put_batch, folio))
folio_batch_release(put_batch);
/* Get a batch of folios and note their orders. */
nr = __readahead_batch(ractl, (struct page **)fq->vec.folios,
folioq_nr_slots(fq));
if (WARN_ON_ONCE(!nr))
break;
fq->vec.nr = nr;
for (int slot = 0; slot < nr; slot++) {
struct folio *folio = folioq_folio(fq, slot);
unsigned int order;
order = folio_order(folio);
fq->orders[slot] = order;
loaded += PAGE_SIZE << order;
trace_netfs_folio(folio, netfs_folio_trace_read);
}
}
WRITE_ONCE(roll->iter.count, roll->iter.count + size);
/* Store the counter after setting the slot. */
smp_store_release(&roll->next_head_slot, to);
return size;
WRITE_ONCE(roll->iter.count, loaded);
iov_iter_folio_queue(&roll->iter, ITER_DEST, roll->tail, 0, 0, loaded);
return loaded;
nomem_unlock:
for (fq = roll->tail; fq; fq = fq->next) {
for (int slot = 0; slot < folioq_count(fq); slot++) {
folio_unlock(fq->vec.folios[slot]);
folioq_mark(fq, slot);
}
}
rolling_buffer_clear(roll);
roll->head = NULL;
roll->tail = NULL;
return -ENOMEM;
}
/*

2
fs/netfs/write_issue.c
View File

@ -170,6 +170,8 @@ void netfs_prepare_write(struct netfs_io_request *wreq,
rolling_buffer_make_space(&wreq->buffer, wreq->gfp);
subreq = netfs_alloc_subrequest(wreq);
if (!subreq)
return;
subreq->source = stream->source;
subreq->start = start;
subreq->stream_nr = stream->stream_nr;

5
include/linux/netfs.h
View File

@ -246,6 +246,7 @@ struct netfs_io_request {
unsigned long long submitted; /* Amount submitted for I/O so far */
unsigned long long len; /* Length of the request */
size_t transferred; /* Amount to be indicated as transferred */
size_t progress_at; /* Report read progress when hit this much read */
long error; /* 0 or error that occurred */
unsigned long long i_size; /* Size of the file */
unsigned long long start; /* Start position */
@ -262,7 +263,6 @@ struct netfs_io_request {
atomic_t subreq_counter; /* Next subreq->debug_index */
unsigned int nr_group_rel; /* Number of refs to release on ->group */
spinlock_t lock; /* Lock for queuing subreqs */
unsigned char front_folio_order; /* Order (size) of front folio */
enum netfs_io_origin origin; /* Origin of the request */
bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */
refcount_t ref;
@ -275,9 +275,10 @@ struct netfs_io_request {
#define NETFS_RREQ_SHORT_TRANSFER 5 /* Set if we have a short transfer */
#define NETFS_RREQ_OFFLOAD_COLLECTION 8 /* Offload collection to workqueue */
#define NETFS_RREQ_NO_UNLOCK_FOLIO 9 /* Don't unlock no_unlock_folio on completion */
#define NETFS_RREQ_FOLIO_COPY_TO_CACHE 10 /* Copy current folio to cache from read */
#define NETFS_RREQ_CANCEL_CACHING 10 /* Set to cancel caching */
#define NETFS_RREQ_UPLOAD_TO_SERVER 11 /* Need to write to the server */
#define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */
#define NETFS_RREQ_NEED_PUT_RA_REFS 17 /* Need to put the folio refs RA gave us */
#define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark
* write to cache on read */
const struct netfs_request_ops *netfs_ops;

6
include/linux/rolling_buffer.h
View File

@ -45,9 +45,9 @@ struct rolling_buffer_snapshot {
int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id,
unsigned int direction, gfp_t gfp);
int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp);
ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll,
struct readahead_control *ractl,
struct folio_batch *put_batch);
ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll,
struct readahead_control *ractl,
unsigned int rreq_id, gfp_t gfp);
ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio,
unsigned int flags, gfp_t gfp);
struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll);

19
include/trace/events/cachefiles.h
View File

@ -372,7 +372,7 @@ TRACE_EVENT(cachefiles_rename,
TRACE_EVENT(cachefiles_coherency,
TP_PROTO(struct cachefiles_object *obj,
ino_t ino,
u64 disk_aux,
const void *disk_aux,
enum cachefiles_content content,
enum cachefiles_coherency_trace why),
@ -389,12 +389,27 @@ TRACE_EVENT(cachefiles_coherency,
),
TP_fast_assign(
union {
__be16 s[4];
__be64 ll;
} x;
__entry->obj = obj->debug_id;
__entry->why = why;
__entry->content = content;
__entry->ino = ino;
__entry->aux = be64_to_cpup((__be64 *)obj->cookie->inline_aux);
__entry->disk_aux = disk_aux;
/* cachefiles_xattr::data is 2-byte aligned but not 8-byte aligned. */
if (disk_aux) {
x.s[0] = ((__be16 *)disk_aux)[0];
x.s[1] = ((__be16 *)disk_aux)[1];
x.s[2] = ((__be16 *)disk_aux)[2];
x.s[3] = ((__be16 *)disk_aux)[3];
__entry->disk_aux = be64_to_cpu(x.ll);
} else {
__entry->disk_aux = 0;
}
),
TP_printk("o=%08x %s B=%llx c=%u aux=%llx dsk=%llx",

30
include/trace/events/netfs.h
View File

@ -59,6 +59,7 @@
EM(netfs_rreq_trace_free, "FREE ") \
EM(netfs_rreq_trace_intr, "INTR ") \
EM(netfs_rreq_trace_ki_complete, "KI-CMPL") \
EM(netfs_rreq_trace_ra_put_ref, "RA-PUT ") \
EM(netfs_rreq_trace_recollect, "RECLLCT") \
EM(netfs_rreq_trace_redirty, "REDIRTY") \
EM(netfs_rreq_trace_resubmit, "RESUBMT") \
@ -70,9 +71,11 @@
EM(netfs_rreq_trace_unpause, "UNPAUSE") \
EM(netfs_rreq_trace_wait_ip, "WAIT-IP") \
EM(netfs_rreq_trace_wait_pause, "--PAUSED--") \
EM(netfs_rreq_trace_wait_put_ra_refs, "WAIT-P-RA") \
EM(netfs_rreq_trace_wait_quiesce, "WAIT-QUIESCE") \
EM(netfs_rreq_trace_waited_ip, "DONE-IP") \
EM(netfs_rreq_trace_waited_pause, "--UNPAUSED--") \
EM(netfs_rreq_trace_waited_put_ra_refs, "DONE-P-RA") \
EM(netfs_rreq_trace_waited_quiesce, "DONE-QUIESCE") \
EM(netfs_rreq_trace_wake_ip, "WAKE-IP") \
EM(netfs_rreq_trace_wake_queue, "WAKE-Q ") \
@ -195,7 +198,6 @@
EM(netfs_folio_trace_clear_cc, "clear-cc") \
EM(netfs_folio_trace_clear_g, "clear-g") \
EM(netfs_folio_trace_clear_s, "clear-s") \
EM(netfs_folio_trace_copy_to_cache, "mark-copy") \
EM(netfs_folio_trace_end_copy, "end-copy") \
EM(netfs_folio_trace_filled_gaps, "filled-gaps") \
EM(netfs_folio_trace_invalidate_all, "inval-all") \
@ -206,16 +208,19 @@
EM(netfs_folio_trace_kill_cc, "kill-cc") \
EM(netfs_folio_trace_kill_g, "kill-g") \
EM(netfs_folio_trace_kill_s, "kill-s") \
EM(netfs_folio_trace_mark_copy, "mark-copy") \
EM(netfs_folio_trace_mkwrite, "mkwrite") \
EM(netfs_folio_trace_mkwrite_plus, "mkwrite+") \
EM(netfs_folio_trace_not_under_wback, "!wback") \
EM(netfs_folio_trace_not_locked, "!locked") \
EM(netfs_folio_trace_not_under_wback, "!wback") \
EM(netfs_folio_trace_pgpriv2_copy, "pgpriv2-copy") \
EM(netfs_folio_trace_put, "put") \
EM(netfs_folio_trace_read, "read") \
EM(netfs_folio_trace_read_done, "read-done") \
EM(netfs_folio_trace_read_gaps, "read-gaps") \
EM(netfs_folio_trace_read_unlock, "read-unlock") \
EM(netfs_folio_trace_redirtied, "redirtied") \
EM(netfs_folio_trace_sched_copy, "sched-copy") \
EM(netfs_folio_trace_store, "store") \
EM(netfs_folio_trace_store_copy, "store-copy") \
EM(netfs_folio_trace_store_plus, "store+") \
@ -786,6 +791,27 @@ TRACE_EVENT(netfs_folioq,
__print_symbolic(__entry->trace, netfs_folioq_traces))
);
TRACE_EVENT(netfs_read_progress_at,
TP_PROTO(const struct netfs_io_request *rreq),
TP_ARGS(rreq),
TP_STRUCT__entry(
__field(unsigned int, rreq)
__field(size_t, progress_at)
__field(size_t, cleaned_to)
),
TP_fast_assign(
__entry->rreq = rreq->debug_id;
__entry->cleaned_to = rreq->cleaned_to - rreq->start;
__entry->progress_at = rreq->progress_at;
),
TP_printk("R=%08x cln=%zx prg=%zx",
__entry->rreq, __entry->cleaned_to, __entry->progress_at)
);
#undef EM
#undef E_
#endif /* _TRACE_NETFS_H */