diff --git a/Documentation/filesystems/caching/backend-api.rst b/Documentation/filesystems/caching/backend-api.rst index 3a199fc50828..df837531b3fd 100644 --- a/Documentation/filesystems/caching/backend-api.rst +++ b/Documentation/filesystems/caching/backend-api.rst @@ -339,7 +339,7 @@ pointers to methods to allow fscache to request management of the cache: * Change the size of a data storage object [mandatory]:: void (*resize_cookie)(struct netfs_cache_resources *cres, - loff_t new_size); + loff_t new_size, bool shrank); This method is called to inform the cache backend of a change in size of the netfs file due to local truncation. The cache backend should make all diff --git a/Documentation/filesystems/caching/netfs-api.rst b/Documentation/filesystems/caching/netfs-api.rst index 665b27f1556e..3154a1686480 100644 --- a/Documentation/filesystems/caching/netfs-api.rst +++ b/Documentation/filesystems/caching/netfs-api.rst @@ -226,7 +226,7 @@ If a network filesystem file is resized locally by truncation, the following should be called to notify the cache:: void fscache_resize_cookie(struct fscache_cookie *cookie, - loff_t new_size); + loff_t new_size, bool shrank); The caller must have first marked the cookie in-use. The cookie and the new size are passed in and the cache is synchronously resized. This is expected to diff --git a/Documentation/filesystems/netfs_library.rst b/Documentation/filesystems/netfs_library.rst index a9de281db8ee..215a6621f3d3 100644 --- a/Documentation/filesystems/netfs_library.rst +++ b/Documentation/filesystems/netfs_library.rst @@ -195,7 +195,7 @@ structure is defined:: struct inode inode; const struct netfs_request_ops *ops; struct fscache_cookie * cache; - loff_t remote_i_size; + loff_t _remote_i_size; unsigned long flags; ... }; @@ -229,11 +229,14 @@ filesystem: Local caching cookie, or NULL if no caching is enabled. This field does not exist if fscache is disabled. - * ``remote_i_size`` + * ``_remote_i_size`` The size of the file on the server. This differs from inode->i_size if local modifications have been made but not yet written back. + Use netfs_read_remote_i_size() and netfs_write_remote_i_size() to access + this field. Hold inode->i_lock when writing it. + * ``flags`` A set of flags, some of which the filesystem might be interested in: diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c index 13cf87a5f90c..170a2b91c5f0 100644 --- a/fs/9p/vfs_addr.c +++ b/fs/9p/vfs_addr.c @@ -150,7 +150,6 @@ static int v9fs_init_request(struct netfs_io_request *rreq, struct file *file) struct p9_fid *fid; struct dentry *dentry; bool writing = (rreq->origin == NETFS_READ_FOR_WRITE || - rreq->origin == NETFS_WRITETHROUGH || rreq->origin == NETFS_UNBUFFERED_WRITE || rreq->origin == NETFS_DIO_WRITE); diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c index c95e653344f6..537cb22ac48b 100644 --- a/fs/9p/vfs_inode.c +++ b/fs/9p/vfs_inode.c @@ -1022,6 +1022,9 @@ static int v9fs_vfs_setattr(const struct mnt_idmap *idmap, struct v9fs_session_info *v9ses; struct p9_fid *fid = NULL; struct p9_wstat wstat; +#ifdef CONFIG_9P_FSCACHE + uoff_t old_size = i_size_read(inode); +#endif p9_debug(P9_DEBUG_VFS, "\n"); retval = setattr_prepare(&nop_mnt_idmap, dentry, iattr); @@ -1086,7 +1089,8 @@ static int v9fs_vfs_setattr(const struct mnt_idmap *idmap, if (v9ses->cache & CACHE_FSCACHE) { struct v9fs_inode *v9inode = V9FS_I(inode); - fscache_resize_cookie(v9fs_inode_cookie(v9inode), iattr->ia_size); + fscache_resize_cookie(v9fs_inode_cookie(v9inode), + iattr->ia_size, iattr->ia_size < old_size); } #endif } diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c index 2cd2898580a9..c6afc9c1cf58 100644 --- a/fs/9p/vfs_inode_dotl.c +++ b/fs/9p/vfs_inode_dotl.c @@ -519,6 +519,9 @@ int v9fs_vfs_setattr_dotl(const struct mnt_idmap *idmap, .uid = INVALID_UID, .gid = INVALID_GID, }; +#ifdef CONFIG_9P_FSCACHE + uoff_t old_size = i_size_read(inode); +#endif p9_debug(P9_DEBUG_VFS, "\n"); @@ -580,7 +583,7 @@ int v9fs_vfs_setattr_dotl(const struct mnt_idmap *idmap, #ifdef CONFIG_9P_FSCACHE if (v9ses->cache & CACHE_FSCACHE) fscache_resize_cookie(v9fs_inode_cookie(V9FS_I(inode)), - iattr->ia_size); + iattr->ia_size, iattr->ia_size < old_size); #endif } diff --git a/fs/afs/file.c b/fs/afs/file.c index 0467742bfeee..4c78d3441785 100644 --- a/fs/afs/file.c +++ b/fs/afs/file.c @@ -400,7 +400,6 @@ static int afs_init_request(struct netfs_io_request *rreq, struct file *file) } break; case NETFS_WRITEBACK: - case NETFS_WRITETHROUGH: case NETFS_UNBUFFERED_WRITE: case NETFS_DIO_WRITE: if (S_ISREG(rreq->inode->i_mode)) @@ -413,7 +412,7 @@ static int afs_init_request(struct netfs_io_request *rreq, struct file *file) return 0; } -static int afs_check_write_begin(struct file *file, loff_t pos, unsigned len, +static int afs_check_write_begin(struct file *file, uoff_t pos, unsigned len, struct folio **foliop, void **_fsdata) { struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); @@ -434,7 +433,7 @@ static void afs_free_request(struct netfs_io_request *rreq) * Also, estimate the number of 512 bytes blocks used, rounded up to nearest 1K * for consistency with other AFS clients. */ -void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size) +void afs_set_i_size(struct afs_vnode *vnode, uoff_t new_i_size) { struct inode *inode = &vnode->netfs.inode; loff_t i_size; @@ -448,10 +447,9 @@ void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size) } spin_unlock(&inode->i_lock); write_sequnlock(&vnode->cb_lock); - fscache_update_cookie(afs_vnode_cache(vnode), NULL, &new_i_size); } -static void afs_update_i_size(struct inode *inode, loff_t new_i_size) +static void afs_update_i_size(struct inode *inode, uoff_t new_i_size) { afs_set_i_size(AFS_FS_I(inode), new_i_size); } diff --git a/fs/afs/inode.c b/fs/afs/inode.c index 8e6ca6b45c6b..0733abff77a5 100644 --- a/fs/afs/inode.c +++ b/fs/afs/inode.c @@ -744,7 +744,7 @@ static void afs_setattr_edit_file(struct afs_operation *op) if (size != old) { truncate_pagecache(inode, size); netfs_resize_file(&vnode->netfs, size, true); - fscache_resize_cookie(afs_vnode_cache(vnode), size); + fscache_resize_cookie(afs_vnode_cache(vnode), size, size < old); } } } @@ -817,7 +817,7 @@ int afs_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, truncate_setsize(inode, attr->ia_size); netfs_resize_file(&vnode->netfs, size, false); fscache_resize_cookie(afs_vnode_cache(vnode), - attr->ia_size); + attr->ia_size, attr->ia_size < i_size); ret = 0; goto out_unlock; } diff --git a/fs/afs/internal.h b/fs/afs/internal.h index f2033673126b..7cea5ba0d20d 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -1173,7 +1173,7 @@ extern int afs_open(struct inode *, struct file *); extern int afs_release(struct inode *, struct file *); void afs_fetch_data_async_rx(struct work_struct *work); void afs_fetch_data_immediate_cancel(struct afs_call *call); -void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size); +void afs_set_i_size(struct afs_vnode *vnode, uoff_t new_i_size); /* * flock.c diff --git a/fs/cachefiles/Kconfig b/fs/cachefiles/Kconfig index afb25b6af5aa..c9c168c7e072 100644 --- a/fs/cachefiles/Kconfig +++ b/fs/cachefiles/Kconfig @@ -17,7 +17,7 @@ config CACHEFILES_DEBUG help This permits debugging to be dynamically enabled in the filesystem caching on files module. If this is set, the debugging output may be - enabled by setting bits in /sys/modules/cachefiles/parameter/debug or + enabled by setting bits in /sys/module/cachefiles/parameters/debug or by including a debugging specifier in /etc/cachefilesd.conf. config CACHEFILES_ERROR_INJECTION diff --git a/fs/cachefiles/interface.c b/fs/cachefiles/interface.c index 50a000310a8c..4a31904aaf0e 100644 --- a/fs/cachefiles/interface.c +++ b/fs/cachefiles/interface.c @@ -99,73 +99,6 @@ void cachefiles_put_object(struct cachefiles_object *object, _leave(""); } -/* - * Adjust the size of a cache file if necessary to match the DIO size. We keep - * the EOF marker a multiple of DIO blocks so that we don't fall back to doing - * non-DIO for a partial block straddling the EOF, but we also have to be - * careful of someone expanding the file and accidentally accreting the - * padding. - */ -static int cachefiles_adjust_size(struct cachefiles_object *object) -{ - struct iattr newattrs; - struct file *file = object->file; - uint64_t ni_size; - loff_t oi_size; - int ret; - - ni_size = object->cookie->object_size; - ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); - - _enter("{OBJ%x},[%llu]", - object->debug_id, (unsigned long long) ni_size); - - if (!file) - return -ENOBUFS; - - oi_size = i_size_read(file_inode(file)); - if (oi_size == ni_size) - return 0; - - inode_lock(file_inode(file)); - - /* if there's an extension to a partial page at the end of the backing - * file, we need to discard the partial page so that we pick up new - * data after it */ - if (oi_size & ~PAGE_MASK && ni_size > oi_size) { - _debug("discard tail %llx", oi_size); - newattrs.ia_valid = ATTR_SIZE; - newattrs.ia_size = oi_size & PAGE_MASK; - ret = cachefiles_inject_remove_error(); - if (ret == 0) - ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, - &newattrs, NULL); - if (ret < 0) - goto truncate_failed; - } - - newattrs.ia_valid = ATTR_SIZE; - newattrs.ia_size = ni_size; - ret = cachefiles_inject_write_error(); - if (ret == 0) - ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, - &newattrs, NULL); - -truncate_failed: - inode_unlock(file_inode(file)); - - if (ret < 0) - trace_cachefiles_io_error(NULL, file_inode(file), ret, - cachefiles_trace_notify_change_error); - if (ret == -EIO) { - cachefiles_io_error_obj(object, "Size set failed"); - ret = -ENOBUFS; - } - - _leave(" = %d", ret); - return ret; -} - /* * Attempt to look up the nominated node in this cache */ @@ -198,7 +131,6 @@ static bool cachefiles_lookup_cookie(struct fscache_cookie *cookie) spin_lock(&cache->object_list_lock); list_add(&object->cache_link, &cache->object_list); spin_unlock(&cache->object_list_lock); - cachefiles_adjust_size(object); cachefiles_end_secure(cache, saved_cred); _leave(" = t"); @@ -225,14 +157,14 @@ static bool cachefiles_lookup_cookie(struct fscache_cookie *cookie) * any unused granules. */ static bool cachefiles_shorten_object(struct cachefiles_object *object, - struct file *file, loff_t new_size) + struct file *file, uoff_t new_size) { struct cachefiles_cache *cache = object->volume->cache; struct inode *inode = file_inode(file); - loff_t i_size, dio_size; + uoff_t i_size, dio_size; int ret; - dio_size = round_up(new_size, CACHEFILES_DIO_BLOCK_SIZE); + dio_size = round_up(new_size, cache->bsize); i_size = i_size_read(inode); trace_cachefiles_trunc(object, inode, i_size, dio_size, @@ -264,6 +196,7 @@ static bool cachefiles_shorten_object(struct cachefiles_object *object, } } + object->object_size = new_size; return true; } @@ -271,29 +204,38 @@ static bool cachefiles_shorten_object(struct cachefiles_object *object, * Resize the backing object. */ static void cachefiles_resize_cookie(struct netfs_cache_resources *cres, - loff_t new_size) + uoff_t new_size, bool shrank) { struct cachefiles_object *object = cachefiles_cres_object(cres); struct cachefiles_cache *cache = object->volume->cache; struct fscache_cookie *cookie = object->cookie; const struct cred *saved_cred; struct file *file = cachefiles_cres_file(cres); - loff_t old_size = cookie->object_size; + uoff_t i_size = i_size_read(file_inode(file)); - _enter("%llu->%llu", old_size, new_size); + _enter("%llu->%llu", object->object_size, new_size); - if (new_size < old_size) { + /* If the file shrank, we need to downsize the backing file and clear + * the end of the final block. + */ + if (shrank) { + if (new_size >= i_size) + goto out; cachefiles_begin_secure(cache, &saved_cred); cachefiles_shorten_object(object, file, new_size); cachefiles_end_secure(cache, saved_cred); object->cookie->object_size = new_size; + if (new_size == 0) + object->content_info = CACHEFILES_CONTENT_NO_DATA; return; } /* The file is being expanded. We don't need to do anything - * particularly. cookie->initial_size doesn't change and so the point - * at which we have to download before doesn't change. + * particularly. The tail of the last block should have been cleared + * both when it is written and when it is shrunk. */ +out: + object->object_size = new_size; cookie->object_size = new_size; } diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h index c93324e0f98c..664be64ab538 100644 --- a/fs/cachefiles/internal.h +++ b/fs/cachefiles/internal.h @@ -16,8 +16,6 @@ #include #include -#define CACHEFILES_DIO_BLOCK_SIZE 4096 - struct cachefiles_cache; struct cachefiles_object; @@ -51,12 +49,17 @@ struct cachefiles_object { struct list_head cache_link; /* Link in cache->*_list */ struct file *file; /* The file representing this object */ char *d_name; /* Backing file name */ + unsigned long flags; +#define CACHEFILES_OBJECT_USING_TMPFILE 0 /* Have an unlinked tmpfile */ + uoff_t object_size; /* Size of the object stored + * (independent of cookie->object_size for + * coherency reasons) + */ + atomic64_t read_limit; /* Point beyond which uncommitted writes */ int debug_id; spinlock_t lock; refcount_t ref; - enum cachefiles_content content_info:8; /* Info about content presence */ - unsigned long flags; -#define CACHEFILES_OBJECT_USING_TMPFILE 0 /* Have an unlinked tmpfile */ + enum cachefiles_content content_info; /* Info about content presence */ }; /* @@ -203,11 +206,11 @@ extern bool cachefiles_begin_operation(struct netfs_cache_resources *cres, enum fscache_want_state want_state); extern int __cachefiles_prepare_write(struct cachefiles_object *object, struct file *file, - loff_t *_start, size_t *_len, size_t upper_len, + uoff_t *_start, size_t *_len, size_t upper_len, bool no_space_allocated_yet); extern int __cachefiles_write(struct cachefiles_object *object, struct file *file, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv); @@ -280,6 +283,7 @@ void cachefiles_withdraw_volume(struct cachefiles_volume *volume); /* * xattr.c */ +int cachefiles_preset_object_xattr(struct cachefiles_object *object, struct file *file); extern int cachefiles_set_object_xattr(struct cachefiles_object *object); extern int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file); diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c index 9540ec25b3cb..d05059822288 100644 --- a/fs/cachefiles/io.c +++ b/fs/cachefiles/io.c @@ -19,7 +19,7 @@ struct cachefiles_kiocb { struct kiocb iocb; refcount_t ki_refcnt; - loff_t start; + uoff_t start; union { size_t skipped; size_t len; @@ -32,6 +32,8 @@ struct cachefiles_kiocb { u64 b_writing; }; +#define IS_ERR_VALUE_LL(x) unlikely((x) >= (unsigned long long)-MAX_ERRNO) + static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki) { if (refcount_dec_and_test(&ki->ki_refcnt)) { @@ -73,7 +75,7 @@ static void cachefiles_read_complete(struct kiocb *iocb, long ret) * Initiate a read from the cache. */ static int cachefiles_read(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, @@ -193,60 +195,81 @@ static int cachefiles_read(struct netfs_cache_resources *cres, } /* - * Query the occupancy of the cache in a region, returning where the next chunk - * of data starts and how long it is. + * Query the occupancy of the cache in a region, returning the extent of the + * next two chunks of cached data and the next hole. */ static int cachefiles_query_occupancy(struct netfs_cache_resources *cres, - loff_t start, size_t len, size_t granularity, - loff_t *_data_start, size_t *_data_len) + struct fscache_occupancy *occ) { struct cachefiles_object *object; + struct inode *inode; struct file *file; - loff_t off, off2; - - *_data_start = -1; - *_data_len = 0; + uoff_t read_limit; + loff_t ret; + int i; if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) return -ENOBUFS; object = cachefiles_cres_object(cres); file = cachefiles_cres_file(cres); - granularity = max_t(size_t, object->volume->cache->bsize, granularity); + inode = file_inode(file); + occ->granularity = object->volume->cache->bsize; + /* Read read_limit before content_info. */ + read_limit = atomic64_read_acquire(&object->read_limit); - _enter("%pD,%llu,%llx,%zx/%llx", - file, file_inode(file)->i_ino, start, len, - i_size_read(file_inode(file))); + _enter("%pD,%llu,%llx-%llx/%llx", + file, inode->i_ino, occ->query_from, occ->query_to, read_limit); - off = cachefiles_inject_read_error(); - if (off == 0) - off = vfs_llseek(file, start, SEEK_DATA); - if (off == -ENXIO) - return -ENODATA; /* Beyond EOF */ - if (off < 0 && off >= (loff_t)-MAX_ERRNO) - return -ENOBUFS; /* Error. */ - if (round_up(off, granularity) >= start + len) - return -ENODATA; /* No data in range */ + if (read_limit == 0) + goto done; - off2 = cachefiles_inject_read_error(); - if (off2 == 0) - off2 = vfs_llseek(file, off, SEEK_HOLE); - if (off2 == -ENXIO) - return -ENODATA; /* Beyond EOF */ - if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO) - return -ENOBUFS; /* Error. */ + switch (READ_ONCE(object->content_info)) { + case CACHEFILES_CONTENT_ALL: + case CACHEFILES_CONTENT_SINGLE: + if (read_limit > occ->query_from) { + occ->cached_from[0] = 0; + occ->cached_to[0] = read_limit; + occ->cached_type[0] = FSCACHE_EXTENT_DATA; + occ->query_from = ULLONG_MAX; + } + goto done; + default: + break; + } - /* Round away partial blocks */ - off = round_up(off, granularity); - off2 = round_down(off2, granularity); - if (off2 <= off) - return -ENODATA; + for (i = 0; i < ARRAY_SIZE(occ->cached_from); i++) { + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_llseek(file, occ->query_from, SEEK_DATA); + if (IS_ERR_VALUE_LL(ret)) { + if (ret != -ENXIO) + return ret; + occ->query_from = ULLONG_MAX; + goto done; + } + occ->cached_type[i] = FSCACHE_EXTENT_DATA; + occ->cached_from[i] = ret; + occ->query_from = ret; - *_data_start = off; - if (off2 > start + len) - *_data_len = len; - else - *_data_len = off2 - off; + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_llseek(file, occ->query_from, SEEK_HOLE); + if (IS_ERR_VALUE_LL(ret)) { + if (ret != -ENXIO) + return ret; + occ->query_from = ULLONG_MAX; + goto done; + } + occ->cached_to[i] = ret; + occ->query_from = ret; + if (occ->query_from >= occ->query_to) + break; + } + +done: + _debug("query[0] %llx-%llx", occ->cached_from[0], occ->cached_to[0]); + _debug("query[1] %llx-%llx", occ->cached_from[1], occ->cached_to[1]); return 0; } @@ -280,7 +303,7 @@ static void cachefiles_write_complete(struct kiocb *iocb, long ret) */ int __cachefiles_write(struct cachefiles_object *object, struct file *file, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv) @@ -357,7 +380,7 @@ int __cachefiles_write(struct cachefiles_object *object, } static int cachefiles_write(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv) @@ -375,127 +398,12 @@ static int cachefiles_write(struct netfs_cache_resources *cres, term_func, term_func_priv); } -static inline enum netfs_io_source -cachefiles_do_prepare_read(struct netfs_cache_resources *cres, - loff_t start, size_t *_len, loff_t i_size, - unsigned long *_flags, ino_t netfs_ino) -{ - enum cachefiles_prepare_read_trace why; - struct cachefiles_object *object = NULL; - struct cachefiles_cache *cache; - struct fscache_cookie *cookie = fscache_cres_cookie(cres); - const struct cred *saved_cred; - struct file *file = cachefiles_cres_file(cres); - enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER; - size_t len = *_len; - loff_t off, to; - ino_t ino = file ? file_inode(file)->i_ino : 0; - - _enter("%zx @%llx/%llx", len, start, i_size); - - if (start >= i_size) { - ret = NETFS_FILL_WITH_ZEROES; - why = cachefiles_trace_read_after_eof; - goto out_no_object; - } - - if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) { - __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags); - why = cachefiles_trace_read_no_data; - goto out_no_object; - } - - /* The object and the file may be being created in the background. */ - if (!file) { - why = cachefiles_trace_read_no_file; - if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) - goto out_no_object; - file = cachefiles_cres_file(cres); - if (!file) - goto out_no_object; - ino = file_inode(file)->i_ino; - } - - object = cachefiles_cres_object(cres); - cache = object->volume->cache; - cachefiles_begin_secure(cache, &saved_cred); - off = cachefiles_inject_read_error(); - if (off == 0) - off = vfs_llseek(file, start, SEEK_DATA); - if (off < 0 && off >= (loff_t)-MAX_ERRNO) { - if (off == (loff_t)-ENXIO) { - why = cachefiles_trace_read_seek_nxio; - goto download_and_store; - } - trace_cachefiles_io_error(object, file_inode(file), off, - cachefiles_trace_seek_error); - why = cachefiles_trace_read_seek_error; - goto out; - } - - if (off >= start + len) { - why = cachefiles_trace_read_found_hole; - goto download_and_store; - } - - if (off > start) { - off = round_up(off, cache->bsize); - len = off - start; - *_len = len; - why = cachefiles_trace_read_found_part; - goto download_and_store; - } - - to = cachefiles_inject_read_error(); - if (to == 0) - to = vfs_llseek(file, start, SEEK_HOLE); - if (to < 0 && to >= (loff_t)-MAX_ERRNO) { - trace_cachefiles_io_error(object, file_inode(file), to, - cachefiles_trace_seek_error); - why = cachefiles_trace_read_seek_error; - goto out; - } - - if (to < start + len) { - if (start + len >= i_size) - to = round_up(to, cache->bsize); - else - to = round_down(to, cache->bsize); - len = to - start; - *_len = len; - } - - why = cachefiles_trace_read_have_data; - ret = NETFS_READ_FROM_CACHE; - goto out; - -download_and_store: - __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags); -out: - cachefiles_end_secure(cache, saved_cred); -out_no_object: - trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino); - return ret; -} - -/* - * Prepare a read operation, shortening it to a cached/uncached - * boundary as appropriate. - */ -static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq, - unsigned long long i_size) -{ - return cachefiles_do_prepare_read(&subreq->rreq->cache_resources, - subreq->start, &subreq->len, i_size, - &subreq->flags, subreq->rreq->inode->i_ino); -} - /* * Prepare for a write to occur. */ int __cachefiles_prepare_write(struct cachefiles_object *object, struct file *file, - loff_t *_start, size_t *_len, size_t upper_len, + uoff_t *_start, size_t *_len, size_t upper_len, bool no_space_allocated_yet) { struct cachefiles_cache *cache = object->volume->cache; @@ -504,7 +412,7 @@ int __cachefiles_prepare_write(struct cachefiles_object *object, int ret; /* Round to DIO size */ - start = round_down(*_start, PAGE_SIZE); + start = round_down(*_start, cache->bsize); if (start != *_start || *_len > upper_len) { /* Probably asked to cache a streaming write written into the * pagecache when the cookie was temporarily out of service to @@ -514,7 +422,7 @@ int __cachefiles_prepare_write(struct cachefiles_object *object, return -ENOBUFS; } - *_len = round_up(len, PAGE_SIZE); + *_len = round_up(len, cache->bsize); /* We need to work out whether there's sufficient disk space to perform * the write - but we can skip that check if we have space already @@ -540,10 +448,14 @@ int __cachefiles_prepare_write(struct cachefiles_object *object, * space, we need to see if it's fully allocated. If it's not, we may * want to cull it. */ - if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, - cachefiles_has_space_check) == 0) + ret = cachefiles_has_space(cache, 0, *_len / cache->bsize, + cachefiles_has_space_check); + if (ret == 0) return 0; /* Enough space to simply overwrite the whole block */ + if (ret == -ENOBUFS) + trace_cachefiles_no_space(object, cachefiles_trace_write_nospace_2); + pos = cachefiles_inject_read_error(); if (pos == 0) pos = vfs_llseek(file, start, SEEK_HOLE); @@ -572,13 +484,16 @@ int __cachefiles_prepare_write(struct cachefiles_object *object, return ret; check_space: - return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, - cachefiles_has_space_for_write); + ret = cachefiles_has_space(cache, 0, *_len / cache->bsize, + cachefiles_has_space_for_write); + if (ret == -ENOBUFS) + trace_cachefiles_no_space(object, cachefiles_trace_write_nospace); + return ret; } static int cachefiles_prepare_write(struct netfs_cache_resources *cres, - loff_t *_start, size_t *_len, size_t upper_len, - loff_t i_size, bool no_space_allocated_yet) + uoff_t *_start, size_t *_len, size_t upper_len, + uoff_t i_size, bool no_space_allocated_yet) { struct cachefiles_object *object = cachefiles_cres_object(cres); struct cachefiles_cache *cache = object->volume->cache; @@ -612,10 +527,16 @@ static void cachefiles_prepare_write_subreq(struct netfs_io_subrequest *subreq) stream->sreq_max_segs = BIO_MAX_VECS; if (!cachefiles_cres_file(cres)) { - if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) { + subreq->error = -ENOBUFS; + trace_netfs_sreq(subreq, netfs_sreq_trace_cache_waitfail); return netfs_prepare_write_failed(subreq); - if (!cachefiles_cres_file(cres)) + } + if (!cachefiles_cres_file(cres)) { + subreq->error = -ENOBUFS; + trace_netfs_sreq(subreq, netfs_sreq_trace_cache_nofile); return netfs_prepare_write_failed(subreq); + } } } @@ -628,16 +549,16 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq) struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr]; const struct cred *saved_cred; size_t off, pre, post, len = subreq->len; - loff_t start = subreq->start; + uoff_t start = subreq->start; int ret; _enter("W=%x[%x] %llx-%llx", wreq->debug_id, subreq->debug_index, start, start + len - 1); /* We need to start on the cache granularity boundary */ - off = start & (CACHEFILES_DIO_BLOCK_SIZE - 1); + off = start & (cache->bsize - 1); if (off) { - pre = CACHEFILES_DIO_BLOCK_SIZE - off; + pre = cache->bsize - off; if (pre >= len) { fscache_count_dio_misfit(); netfs_write_subrequest_terminated(subreq, len); @@ -651,8 +572,8 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq) /* We also need to end on the cache granularity boundary */ if (start + len == wreq->i_size) { - size_t part = len % CACHEFILES_DIO_BLOCK_SIZE; - size_t need = CACHEFILES_DIO_BLOCK_SIZE - part; + size_t part = len & (cache->bsize - 1); + size_t need = cache->bsize - part; if (part && stream->submit_extendable_to >= need) { len += need; @@ -661,7 +582,7 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq) } } - post = len & (CACHEFILES_DIO_BLOCK_SIZE - 1); + post = len & (cache->bsize - 1); if (post) { len -= post; if (len == 0) { @@ -688,6 +609,198 @@ static void cachefiles_issue_write(struct netfs_io_subrequest *subreq) netfs_write_subrequest_terminated, subreq); } +/* + * Collect the result of buffered writeback to the cache. This includes + * copying a read to the cache. Netfslib collates the results, which might + * occur out of order, and delivers them to the cache so that it can update its + * content record. + * + * block_type is one of: + * - NETFS_CACHE_COLLECT_WRITE_DATA for a contiguous block of data + * - NETFS_CACHE_COLLECT_WRITE_GAP if a discontiguity was skipped + * - NETFS_CACHE_COLLECT_WRITE_CANCEL for a hole due to a failed/cancelled write + * + * The writes we made are all rounded out at both sides to the nearest DIO + * block boundary, so if the final block contains the EOF in the middle of it + * (rather than at the end), padding will have been written to the file. The + * backing file's filesize will have been updated if the write extended the + * file; the filesize may still change due to outstanding subreqs. + * + * The metadata in the cache file xattr records the size of the object we have + * stored, but the cache file EOF only goes up to where we've cached data to + * and, furthermore, is rounded up to the nearest DIO block boundary. + * + * Concurrent updates should be protected against by the caller. Netfslib + * holds NETFS_ICTX_WB_LOCK as a lock on writeback requests. DIO writes + * invalidate the cookie and caching is kept disabled until all users have + * unused the cookie. + */ +static void cachefiles_collect_write(struct netfs_io_request *wreq, + uoff_t start, size_t len, + enum netfs_cache_collect block_type) +{ + struct netfs_cache_resources *cres = &wreq->cache_resources; + struct cachefiles_object *object = cachefiles_cres_object(cres); + struct cachefiles_cache *cache = object->volume->cache; + struct inode *inode; + struct file *file = cachefiles_cres_file(cres); + uoff_t read_limit; + uoff_t old_size = cres->cache_i_size; + uoff_t new_size; + uoff_t data_to = object->object_size; + uoff_t end = start + len; + int ret; + + if (!file) + return; + + inode = file_inode(file); + new_size = i_size_read(inode); + + _enter("%llx,%zx,%x", start, len, cache->bsize); + + if (WARN_ON(old_size & (cache->bsize - 1)) || + WARN_ON(new_size & (cache->bsize - 1)) || + WARN_ON(start & (cache->bsize - 1)) || + WARN_ON(len & (cache->bsize - 1))) { + trace_cachefiles_io_error(object, inode, -EIO, + cachefiles_trace_alignment_error); + cachefiles_remove_object_xattr(cache, object, file->f_path.dentry); + return; + } + + /* If this is recording a gap, due to discontiguous writes or lack of + * cache space, then a hole may have been introduced into the backing + * file. Treat it as a zero-length data block. + */ + if (block_type == NETFS_CACHE_COLLECT_WRITE_GAP || + block_type == NETFS_CACHE_COLLECT_WRITE_CANCEL) { + start = end; + len = 0; + } + + /* Zeroth case: Single monolithic files are handled specially. + */ + if (wreq->origin == NETFS_WRITEBACK_SINGLE) { + if (block_type == NETFS_CACHE_COLLECT_WRITE_GAP || + block_type == NETFS_CACHE_COLLECT_WRITE_CANCEL) { + trace_cachefiles_trunc(object, inode, data_to, 0, + cachefiles_trunc_zap); + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = vfs_truncate(&file->f_path, 0); + if (ret < 0) { + trace_cachefiles_io_error(object, inode, ret, + cachefiles_trace_trunc_error); + cachefiles_io_error_obj(object, "truncate failed %d", ret); + cachefiles_remove_object_xattr(cache, object, file->f_path.dentry); + return; + } + + object->content_info = CACHEFILES_CONTENT_NO_DATA; + read_limit = 0; + } else { + object->content_info = CACHEFILES_CONTENT_SINGLE; + read_limit = len; + } + goto update_sizes_2; + } + + /* First case: The backing file was empty. */ + if (old_size == 0) { + if (start == 0) + object->content_info = CACHEFILES_CONTENT_ALL; + else + object->content_info = CACHEFILES_CONTENT_BACKFS_MAP; + goto update_sizes; + } + + /* Second case: The backing file is entirely within the old object size + * and thus there can be no partial tail block to deal with in the + * cache file. + */ + if (old_size <= data_to) { + if (start > old_size) + goto discontiguous; + goto update_sizes; + } + + /* Third case: The write happened entirely within the bounds of the + * current cache file's size. + */ + if (end <= old_size) + goto update_sizes; + + /* Fourth case: The write overwrote the partial tail block and extended + * the file. We only need to update the object size because netfslib + * rounds out/pads cache writes to whole disk blocks. + */ + if (start < old_size) + goto update_sizes; + + /* Fifth case: The write started from the end of the whole tail block + * and extended the file. Just extend our notion of the filesize. + */ + if (start == old_size && old_size == data_to) + goto update_sizes; + + /* Sixth case: The write continued on from the partial tail block and + * extended the file. Need to clear the gap. + */ + if (start == old_size && old_size > data_to) + goto clear_gap; + +discontiguous: + /* Seventh case: The write was beyond the EOF on the cache file, so now + * there's a hole in the file and we can no longer say in the metadata + * that we can assume we have it all. We may also need to clear the + * end of the partial tail block. + */ + /* TODO: For the moment, we will have to use SEEK_HOLE/SEEK_DATA. */ + if (object->content_info != CACHEFILES_CONTENT_BACKFS_MAP) { + object->content_info = CACHEFILES_CONTENT_BACKFS_MAP; + trace_cachefiles_coherency(object, inode->i_ino, data_to, NULL, + CACHEFILES_CONTENT_BACKFS_MAP, + cachefiles_coherency_discontiguous); + } + +clear_gap: + /* We need to clear any partial padding that got jumped over. It + * *should* be all zeros, but shared-writable mmap exists... + */ + if (old_size > data_to) { + trace_cachefiles_trunc(object, inode, data_to, old_size, + cachefiles_trunc_clear_padding); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_fallocate(file, FALLOC_FL_ZERO_RANGE, + data_to, old_size - data_to); + if (ret < 0) { + trace_cachefiles_io_error(object, inode, ret, + cachefiles_trace_fallocate_error); + cachefiles_io_error_obj(object, "fallocate zero pad failed %d", ret); + cachefiles_remove_object_xattr(cache, object, file->f_path.dentry); + return; + } + } + +update_sizes: + read_limit = umax(old_size, end); +update_sizes_2: + cres->cache_i_size = read_limit; + + /* We need to be careful setting the object_size: we may have written + * more to the cache than to the server (due to cache DIO rounding) and + * the i_size set on the netfs inode may include unwritten data that + * the server doesn't know about yet. + */ + object->object_size = umin(read_limit, wreq->i_size); + + /* Raise the limit at which reads can access the file. */ + /* Update read_limit after content_info */ + atomic64_set_release(&object->read_limit, read_limit); +} + /* * Clean up an operation. */ @@ -705,10 +818,10 @@ static const struct netfs_cache_ops cachefiles_netfs_cache_ops = { .read = cachefiles_read, .write = cachefiles_write, .issue_write = cachefiles_issue_write, - .prepare_read = cachefiles_prepare_read, .prepare_write = cachefiles_prepare_write, .prepare_write_subreq = cachefiles_prepare_write_subreq, .query_occupancy = cachefiles_query_occupancy, + .collect_write = cachefiles_collect_write, }; /* @@ -718,13 +831,20 @@ bool cachefiles_begin_operation(struct netfs_cache_resources *cres, enum fscache_want_state want_state) { struct cachefiles_object *object = cachefiles_cres_object(cres); + struct file *file; + + cres->dio_size = object->volume->cache->bsize; if (!cachefiles_cres_file(cres)) { cres->ops = &cachefiles_netfs_cache_ops; + cres->object_id = object->debug_id; if (object->file) { spin_lock(&object->lock); - if (!cres->cache_priv2 && object->file) - cres->cache_priv2 = get_file(object->file); + file = object->file; + if (!cres->cache_priv2 && file) { + cres->cache_priv2 = get_file(file); + cres->cache_i_size = i_size_read(file_inode(file)); + } spin_unlock(&object->lock); } } diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index 88955249a1a6..ef656a319ede 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -117,8 +117,11 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, if (d_is_negative(subdir)) { ret = cachefiles_has_space(cache, 1, 0, cachefiles_has_space_for_create); - if (ret < 0) + if (ret < 0) { + if (ret == -ENOBUFS) + trace_cachefiles_no_space(NULL, cachefiles_trace_mkdir_nospace); goto mkdir_error; + } _debug("attempt mkdir"); @@ -414,7 +417,6 @@ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; struct file *file; const struct path parentpath = { .mnt = cache->mnt, .dentry = fan }; - uint64_t ni_size; long ret; @@ -442,31 +444,20 @@ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) if (!cachefiles_mark_inode_in_use(object, file_inode(file))) WARN_ON(1); - ni_size = object->cookie->object_size; - ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); - - if (ni_size > 0) { - trace_cachefiles_trunc(object, file_inode(file), 0, ni_size, - cachefiles_trunc_expand_tmpfile); - ret = cachefiles_inject_write_error(); - if (ret == 0) - ret = vfs_truncate(&file->f_path, ni_size); - if (ret < 0) { - trace_cachefiles_vfs_error( - object, file_inode(file), ret, - cachefiles_trace_trunc_error); - goto err_unuse; - } - } - ret = -EINVAL; if (unlikely(!file->f_op->read_iter) || unlikely(!file->f_op->write_iter)) { pr_notice("Cache does not support read_iter and write_iter\n"); goto err_unuse; } + + /* Preallocate space for the xattr. */ + ret = cachefiles_preset_object_xattr(object, file); + if (ret < 0) + goto err_unuse; out: cachefiles_end_secure(cache, saved_cred); + object->content_info = CACHEFILES_CONTENT_ALL; return file; err_unuse: @@ -487,8 +478,11 @@ static bool cachefiles_create_file(struct cachefiles_object *object) ret = cachefiles_has_space(object->volume->cache, 1, 0, cachefiles_has_space_for_create); - if (ret < 0) + if (ret < 0) { + if (ret == -ENOBUFS) + trace_cachefiles_no_space(object, cachefiles_trace_create_nospace); return false; + } file = cachefiles_create_tmpfile(object); if (IS_ERR(file)) diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c index c70bf67e52b0..8ebb713482e3 100644 --- a/fs/cachefiles/xattr.c +++ b/fs/cachefiles/xattr.c @@ -34,6 +34,57 @@ struct cachefiles_vol_xattr { __u8 data[]; /* netfs volume coherency data */ } __packed; +/* + * Preset the state xattr on a cache file to allocate space for it. + */ +int cachefiles_preset_object_xattr(struct cachefiles_object *object, struct file *file) +{ + struct cachefiles_xattr *buf; + struct dentry *dentry = file->f_path.dentry; + unsigned int len = object->cookie->aux_len; + int ret; + + buf = kzalloc(sizeof(struct cachefiles_xattr) + min(len, sizeof(__be64)), GFP_KERNEL); + if (!buf) + return -ENOMEM; + + buf->type = CACHEFILES_COOKIE_TYPE_DATA; + buf->content = CACHEFILES_CONTENT_DIRTY; + + ret = cachefiles_inject_write_error(); + if (ret == 0) { + ret = mnt_want_write_file(file); + if (ret == 0) { + ret = vfs_setxattr(&nop_mnt_idmap, dentry, + cachefiles_xattr_cache, buf, + sizeof(struct cachefiles_xattr) + len, 0); + mnt_drop_write_file(file); + } + } + if (ret < 0) { + trace_cachefiles_vfs_error(object, file_inode(file), ret, + cachefiles_trace_setxattr_error); + trace_cachefiles_coherency(object, file_inode(file)->i_ino, + object->object_size, + buf->data, buf->content, + cachefiles_coherency_set_fail); + switch (ret) { + case -ENOMEM: + case -ENOSPC: + break; + default: + cachefiles_io_error_obj( + object, + "Failed to set xattr with error %d", ret); + break; + } + } + + kfree(buf); + _leave(" = %d", ret); + return ret; +} + /* * set the state xattr on a cache file */ @@ -43,6 +94,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) struct dentry *dentry; struct file *file = object->file; unsigned int len = object->cookie->aux_len; + uoff_t object_size = object->cookie->object_size; int ret; if (!file) @@ -55,7 +107,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) if (!buf) return -ENOMEM; - buf->object_size = cpu_to_be64(object->cookie->object_size); + buf->object_size = cpu_to_be64(object_size); buf->zero_point = 0; buf->type = CACHEFILES_COOKIE_TYPE_DATA; buf->content = object->content_info; @@ -79,15 +131,21 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) trace_cachefiles_vfs_error(object, file_inode(file), ret, cachefiles_trace_setxattr_error); trace_cachefiles_coherency(object, file_inode(file)->i_ino, - buf->data, buf->content, + object_size, buf->data, buf->content, cachefiles_coherency_set_fail); - if (ret != -ENOMEM) + switch (ret) { + case -ENOMEM: + break; + case -ENOSPC: + default: cachefiles_io_error_obj( object, "Failed to set xattr with error %d", ret); + break; + } } else { trace_cachefiles_coherency(object, file_inode(file)->i_ino, - buf->data, buf->content, + object_size, buf->data, buf->content, cachefiles_coherency_set_ok); } @@ -103,10 +161,12 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file { struct cachefiles_xattr *buf; struct dentry *dentry = file->f_path.dentry; + struct inode *inode = file_inode(file); unsigned int len = object->cookie->aux_len, tlen; const void *p = fscache_get_aux(object->cookie); enum cachefiles_coherency_trace why; ssize_t xlen; + uoff_t obj_size; int ret = -ESTALE; tlen = sizeof(struct cachefiles_xattr) + len; @@ -121,34 +181,39 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file if (xlen != tlen) { if (xlen < 0) { ret = xlen; - trace_cachefiles_vfs_error(object, file_inode(file), xlen, + trace_cachefiles_vfs_error(object, inode, xlen, cachefiles_trace_getxattr_error); } if (xlen == -EIO) cachefiles_io_error_obj( object, "Failed to read aux with error %zd", xlen); + obj_size = 0; why = cachefiles_coherency_check_xattr; goto out; } + obj_size = be64_to_cpu(buf->object_size); if (buf->type != CACHEFILES_COOKIE_TYPE_DATA) { why = cachefiles_coherency_check_type; } else if (memcmp(buf->data, p, len) != 0) { why = cachefiles_coherency_check_aux; - } else if (be64_to_cpu(buf->object_size) != object->cookie->object_size) { + } else if (obj_size != object->cookie->object_size) { why = cachefiles_coherency_check_objsize; } else if (buf->content == CACHEFILES_CONTENT_DIRTY) { // TODO: Begin conflict resolution pr_warn("Dirty object in cache\n"); why = cachefiles_coherency_check_dirty; } else { + object->content_info = buf->content; + object->object_size = obj_size; + atomic64_set(&object->read_limit, i_size_read(inode)); why = cachefiles_coherency_check_ok; ret = 0; } out: - trace_cachefiles_coherency(object, file_inode(file)->i_ino, + trace_cachefiles_coherency(object, inode->i_ino, obj_size, buf->data, buf->content, why); kfree(buf); return ret; @@ -163,6 +228,9 @@ int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, { int ret; + trace_cachefiles_coherency(object, d_inode(dentry)->i_ino, 0, NULL, 0, + cachefiles_coherency_remove); + ret = cachefiles_inject_remove_error(); if (ret == 0) { ret = mnt_want_write(cache->mnt); diff --git a/fs/ceph/Kconfig b/fs/ceph/Kconfig index 3d64a316ca31..aa6ccd7794d2 100644 --- a/fs/ceph/Kconfig +++ b/fs/ceph/Kconfig @@ -4,6 +4,7 @@ config CEPH_FS depends on INET select CEPH_LIB select NETFS_SUPPORT + select NETFS_PGPRIV2 select FS_ENCRYPTION_ALGS if FS_ENCRYPTION default n help diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index e598b2d424ec..42b55ce30a32 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -65,7 +65,7 @@ (CONGESTION_ON_THRESH(congestion_kb) - \ (CONGESTION_ON_THRESH(congestion_kb) >> 2)) -static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, +static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len, struct folio **foliop, void **_fsdata); static inline struct ceph_snap_context *page_snap_context(struct page *page) @@ -1868,7 +1868,7 @@ ceph_find_incompatible(struct folio *folio) return NULL; } -static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, +static int ceph_netfs_check_write_begin(struct file *file, uoff_t pos, unsigned int len, struct folio **foliop, void **_fsdata) { struct inode *inode = file_inode(file); diff --git a/fs/ceph/cache.h b/fs/ceph/cache.h index 20efac020394..892b18c31741 100644 --- a/fs/ceph/cache.h +++ b/fs/ceph/cache.h @@ -31,14 +31,14 @@ static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info return netfs_i_cookie(&ci->netfs); } -static inline void ceph_fscache_resize(struct inode *inode, loff_t to) +static inline void ceph_fscache_resize(struct inode *inode, loff_t to, bool shrank) { struct ceph_inode_info *ci = ceph_inode(inode); struct fscache_cookie *cookie = ceph_fscache_cookie(ci); if (cookie) { ceph_fscache_use_cookie(inode, true); - fscache_resize_cookie(cookie, to); + fscache_resize_cookie(cookie, to, shrank); ceph_fscache_unuse_cookie(inode, true); } } @@ -96,7 +96,7 @@ static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info return NULL; } -static inline void ceph_fscache_resize(struct inode *inode, loff_t to) +static inline void ceph_fscache_resize(struct inode *inode, loff_t to, bool shrank) { } diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index a695dba82554..e758d0181fd7 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -680,7 +680,8 @@ struct inode *ceph_alloc_inode(struct super_block *sb) mutex_init(&ci->i_truncate_mutex); ci->i_truncate_seq = 0; ci->i_truncate_size = 0; - ci->i_truncate_pending = 0; + ci->i_truncate_pending = false; + ci->i_truncate_shrank = false; ci->i_truncate_pagecache_size = 0; ci->i_max_size = 0; @@ -847,7 +848,9 @@ int ceph_fill_file_size(struct inode *inode, int issued, CEPH_CAP_FILE_BUFFER)) || mapping_mapped(inode->i_mapping) || __ceph_is_file_opened(ci)) { - ci->i_truncate_pending++; + ci->i_truncate_pending = true; + if (size < isize) + ci->i_truncate_shrank = true; queue_trunc = 1; } } @@ -2304,12 +2307,13 @@ void __ceph_do_pending_vmtruncate(struct inode *inode) struct ceph_client *cl = ceph_inode_to_client(inode); struct ceph_inode_info *ci = ceph_inode(inode); u64 to; + bool shrank; int wrbuffer_refs, finish = 0; mutex_lock(&ci->i_truncate_mutex); retry: spin_lock(&ci->i_ceph_lock); - if (ci->i_truncate_pending == 0) { + if (!ci->i_truncate_pending) { doutc(cl, "%p %llx.%llx none pending\n", inode, ceph_vinop(inode)); spin_unlock(&ci->i_ceph_lock); @@ -2333,18 +2337,20 @@ void __ceph_do_pending_vmtruncate(struct inode *inode) /* there should be no reader or writer */ WARN_ON_ONCE(ci->i_rd_ref || ci->i_wr_ref); + shrank = ci->i_truncate_shrank; to = ci->i_truncate_pagecache_size; wrbuffer_refs = ci->i_wrbuffer_ref; doutc(cl, "%p %llx.%llx (%d) to %lld\n", inode, ceph_vinop(inode), ci->i_truncate_pending, to); spin_unlock(&ci->i_ceph_lock); - ceph_fscache_resize(inode, to); + ceph_fscache_resize(inode, to, shrank); truncate_pagecache(inode, to); spin_lock(&ci->i_ceph_lock); if (to == ci->i_truncate_pagecache_size) { - ci->i_truncate_pending = 0; + ci->i_truncate_shrank = false; + ci->i_truncate_pending = false; finish = 1; } spin_unlock(&ci->i_ceph_lock); diff --git a/fs/ceph/super.h b/fs/ceph/super.h index a033331bb151..533833bbe051 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -481,7 +481,8 @@ struct ceph_inode_info { struct mutex i_truncate_mutex; u32 i_truncate_seq; /* last truncate to smaller size */ u64 i_truncate_size; /* and the size we last truncated down to */ - int i_truncate_pending; /* still need to call vmtruncate */ + bool i_truncate_pending; /* still need to call vmtruncate */ + bool i_truncate_shrank; /* Truncate shrank the file */ /* * For none fscrypt case it equals to i_truncate_size or it will * equals to fscrypt_file_size diff --git a/fs/netfs/Kconfig b/fs/netfs/Kconfig index 7701c037c328..d0e7b0971fa3 100644 --- a/fs/netfs/Kconfig +++ b/fs/netfs/Kconfig @@ -22,6 +22,9 @@ config NETFS_STATS between CPUs. On the other hand, the stats are very useful for debugging purposes. Saying 'Y' here is recommended. +config NETFS_PGPRIV2 + bool + config NETFS_DEBUG bool "Enable dynamic debugging netfslib and FS-Cache" depends on NETFS_SUPPORT diff --git a/fs/netfs/Makefile b/fs/netfs/Makefile index b43188d64bd8..54834cde7e56 100644 --- a/fs/netfs/Makefile +++ b/fs/netfs/Makefile @@ -11,7 +11,6 @@ netfs-y := \ misc.o \ objects.o \ read_collect.o \ - read_pgpriv2.o \ read_retry.o \ read_single.o \ rolling_buffer.o \ @@ -19,6 +18,7 @@ netfs-y := \ write_issue.o \ write_retry.o +netfs-$(CONFIG_NETFS_PGPRIV2) += read_pgpriv2.o netfs-$(CONFIG_NETFS_STATS) += stats.o netfs-$(CONFIG_FSCACHE) += \ diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index e6506942aeda..69e68ef0652a 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -10,9 +10,9 @@ #include "internal.h" static void netfs_cache_expand_readahead(struct netfs_io_request *rreq, - unsigned long long *_start, - unsigned long long *_len, - unsigned long long i_size) + uoff_t *_start, + uoff_t *_len, + uoff_t i_size) { struct netfs_cache_resources *cres = &rreq->cache_resources; @@ -137,21 +137,6 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq) return subreq->len; } -static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_request *rreq, - struct netfs_io_subrequest *subreq, - loff_t i_size) -{ - struct netfs_cache_resources *cres = &rreq->cache_resources; - enum netfs_io_source source; - - if (!cres->ops) - return NETFS_DOWNLOAD_FROM_SERVER; - source = cres->ops->prepare_read(subreq, i_size); - trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); - return source; - -} - /* * Issue a read against the cache. * - Eats the caller's ref on subreq. @@ -166,6 +151,19 @@ static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq, netfs_cache_read_terminated, subreq); } +int netfs_read_query_cache(struct netfs_io_request *rreq, struct fscache_occupancy *occ) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + + occ->granularity = PAGE_SIZE; + if (occ->query_from >= occ->query_to) + return 0; + if (!cres->ops) + return 0; + occ->query_from = round_up(occ->query_from, occ->granularity); + return cres->ops->query_occupancy(cres, occ); +} + void netfs_queue_read(struct netfs_io_request *rreq, struct netfs_io_subrequest *subreq) { @@ -242,7 +240,7 @@ static void netfs_mark_copy_to_cache(struct netfs_io_request *rreq, if (overlap > 0 && copy) { folio = folioq_folio(*fq, *slot); - if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags))) { + if (netfs_using_pgpriv2(rreq)) { if (!folio_test_private_2(folio)) folio_start_private_2(folio); } else { @@ -268,85 +266,137 @@ static void netfs_mark_copy_to_cache(struct netfs_io_request *rreq, */ static void netfs_read_to_pagecache(struct netfs_io_request *rreq) { + struct fscache_occupancy _occ = { + .query_from = rreq->start, + .query_to = rreq->start + rreq->len, + .cached_from[0] = 0, + .cached_to[0] = 0, + .cached_from[1] = ULLONG_MAX, + .cached_to[1] = ULLONG_MAX, + }; + struct fscache_occupancy *occ = &_occ; struct folio_queue *fq = rreq->buffer.tail; - unsigned long long start = rreq->start; unsigned int offset = 0; ssize_t size = rreq->len; + uoff_t start = rreq->start; int ret = 0, slot = 0; do { + int (*prepare_read)(struct netfs_io_subrequest *subreq) = NULL; struct netfs_io_subrequest *subreq; - enum netfs_io_source source = NETFS_SOURCE_UNKNOWN; + enum netfs_io_source source; ssize_t slice; + uoff_t hole_to, cache_to; + size_t len = size; + bool copy = false; - subreq = netfs_alloc_subrequest(rreq); + /* If we don't have any, find out the next couple of data + * extents from the cache, containing of following the + * specified start offset. Holes have to be fetched from the + * server; data regions from the cache. + */ + hole_to = occ->cached_from[0]; + cache_to = occ->cached_to[0]; + if (start >= cache_to) { + /* Extent exhausted; shuffle down. */ + int i; + + for (i = 0; i < ARRAY_SIZE(occ->cached_from) - 1; i++) { + occ->cached_from[i] = occ->cached_from[i + 1]; + occ->cached_to[i] = occ->cached_to[i + 1]; + occ->cached_type[i] = occ->cached_type[i + 1]; + } + occ->cached_from[i] = ULLONG_MAX; + occ->cached_to[i] = ULLONG_MAX; + + if (occ->cached_from[0] != ULLONG_MAX) + continue; + + /* Get new extents */ + ret = netfs_read_query_cache(rreq, occ); + if (ret < 0) + break; + continue; + } + + uoff_t zero_point = netfs_read_zero_point(rreq->inode); + uoff_t zlimit = umin(zero_point, rreq->i_size); + + _debug("rsub %llx %llx-%llx", start, hole_to, cache_to); + + if (start >= hole_to && start < cache_to) { + /* Overlap with a cached region, where the cache may + * record a block of zeroes. + */ + _debug("cached s=%llx c=%llx l=%zx", start, cache_to, size); + len = umin(cache_to - start, size); + len = round_up(len, occ->granularity); + if (occ->cached_type[0] == FSCACHE_EXTENT_ZERO) { + source = NETFS_FILL_WITH_ZEROES; + netfs_stat(&netfs_n_rh_zero); + } else { + source = NETFS_READ_FROM_CACHE; + prepare_read = rreq->cache_resources.ops->prepare_read; + } + } else if (start >= zlimit && size > 0) { + /* If this range lies beyond the zero-point, that part + * can just be cleared locally. + */ + _debug("zero %llx-%llx", start, start + size); + len = size; + source = NETFS_FILL_WITH_ZEROES; + if (rreq->cache_resources.ops) + copy = true; + netfs_stat(&netfs_n_rh_zero); + } else { + /* Read a cache hole from the server. If any part of + * this range lies beyond the zero-point or the EOF, + * that part can just be cleared locally. + */ + uoff_t limit = min3(zlimit, start + size, hole_to); + + _debug("limit %llx %llx", rreq->i_size, zero_point); + _debug("download %llx-%llx", start, start + size); + len = umin(limit - start, ULONG_MAX); + source = NETFS_DOWNLOAD_FROM_SERVER; + prepare_read = rreq->netfs_ops->prepare_read; + if (rreq->cache_resources.ops) + copy = true; + netfs_stat(&netfs_n_rh_download); + } + + if (len == 0) { + pr_err("ZERO-LEN READ: R=%08x l=%zx/%zx s=%llx z=%llx i=%llx", + rreq->debug_id, len, size, + start, zero_point, rreq->i_size); + break; + } + + subreq = netfs_alloc_subrequest(rreq, source); if (!subreq) { ret = -ENOMEM; break; } subreq->start = start; - subreq->len = size; + subreq->len = len; + if (copy) + __set_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); netfs_queue_read(rreq, subreq); - source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size); - subreq->source = source; - if (source == NETFS_DOWNLOAD_FROM_SERVER) { - unsigned long long zero_point = netfs_read_zero_point(rreq->inode); - unsigned long long zp = umin(zero_point, rreq->i_size); - size_t len = subreq->len; + rreq->io_streams[0].sreq_max_len = MAX_RW_COUNT; + rreq->io_streams[0].sreq_max_segs = INT_MAX; - if (unlikely(rreq->origin == NETFS_READ_SINGLE)) - zp = rreq->i_size; - if (subreq->start >= zp) { - subreq->source = source = NETFS_FILL_WITH_ZEROES; - goto fill_with_zeroes; - } - - if (len > zp - subreq->start) - len = zp - subreq->start; - if (len == 0) { - pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx", - rreq->debug_id, subreq->debug_index, - subreq->len, size, - subreq->start, zero_point, rreq->i_size); + if (prepare_read) { + ret = prepare_read(subreq); + if (ret < 0) { netfs_cancel_read(subreq, ret); break; } - subreq->len = len; - - netfs_stat(&netfs_n_rh_download); - if (rreq->netfs_ops->prepare_read) { - ret = rreq->netfs_ops->prepare_read(subreq); - if (ret < 0) { - netfs_cancel_read(subreq, ret); - break; - } - trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); - } - goto issue; + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); } - fill_with_zeroes: - if (source == NETFS_FILL_WITH_ZEROES) { - subreq->source = NETFS_FILL_WITH_ZEROES; - trace_netfs_sreq(subreq, netfs_sreq_trace_submit); - netfs_stat(&netfs_n_rh_zero); - goto issue; - } - - if (source == NETFS_READ_FROM_CACHE) { - trace_netfs_sreq(subreq, netfs_sreq_trace_submit); - goto issue; - } - - pr_err("Unexpected read source %u\n", source); - WARN_ON_ONCE(1); - netfs_cancel_read(subreq, ret); - break; - - issue: slice = netfs_prepare_read_iterator(subreq); if (slice < 0) { ret = slice; @@ -355,18 +405,16 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq) } start += slice; size -= slice; - if (size <= 0) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); - } + if (size <= 0) + netfs_all_subreqs_queued(rreq); if (fq) { /* See if the cache indicated this should be cached. */ - bool copy = test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); - + copy = test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); netfs_mark_copy_to_cache(rreq, &fq, &slot, &offset, slice, copy); } + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); netfs_issue_read(rreq, subreq); netfs_maybe_bulk_drop_ra_refs(rreq); @@ -378,8 +426,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq) } while (size > 0); if (unlikely(size > 0)) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); netfs_wake_collector(rreq); } @@ -653,11 +700,11 @@ EXPORT_SYMBOL(netfs_read_folio); * If any of these criteria are met, then zero out the unwritten parts * of the folio and return true. Otherwise, return false. */ -static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len, +static bool netfs_skip_folio_read(struct folio *folio, uoff_t pos, size_t len, bool always_fill) { struct inode *inode = folio_inode(folio); - loff_t i_size = i_size_read(inode); + uoff_t i_size = i_size_read(inode); size_t offset = offset_in_folio(folio, pos); size_t plen = folio_size(folio); @@ -722,7 +769,7 @@ static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len, */ int netfs_write_begin(struct netfs_inode *ctx, struct file *file, struct address_space *mapping, - loff_t pos, unsigned int len, struct folio **_folio, + uoff_t pos, unsigned int len, struct folio **_folio, void **_fsdata) { struct netfs_io_request *rreq; @@ -818,7 +865,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, struct netfs_io_request *rreq; struct address_space *mapping = folio->mapping; struct netfs_inode *ctx = netfs_inode(mapping->host); - unsigned long long start = folio_pos(folio); + uoff_t start = folio_pos(folio); size_t flen = folio_size(folio); int ret; diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c index ecf119b4f916..42ca521fcad3 100644 --- a/fs/netfs/buffered_write.c +++ b/fs/netfs/buffered_write.c @@ -17,7 +17,7 @@ * as possible to hold as much of the remaining length as possible in one go. */ static struct folio *netfs_grab_folio_for_write(struct address_space *mapping, - loff_t pos, size_t part) + uoff_t pos, size_t part) { pgoff_t index = pos / PAGE_SIZE; fgf_t fgp_flags = FGP_WRITEBEGIN; @@ -35,9 +35,9 @@ static struct folio *netfs_grab_folio_for_write(struct address_space *mapping, * the values actually are. */ void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode, - loff_t pos, size_t copied) + uoff_t pos, size_t copied) { - loff_t i_size, end = pos + copied; + uoff_t i_size, end = pos + copied; blkcnt_t add; size_t gap; @@ -54,9 +54,6 @@ void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode, i_size = i_size_read(inode); if (end > i_size) { i_size_write(inode, end); -#if IS_ENABLED(CONFIG_FSCACHE) - fscache_update_cookie(ctx->cache, NULL, &end); -#endif gap = SECTOR_SIZE - (i_size & (SECTOR_SIZE - 1)); if (copied > gap) { @@ -91,50 +88,20 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, struct inode *inode = file_inode(file); struct address_space *mapping = inode->i_mapping; struct netfs_inode *ctx = netfs_inode(inode); - struct writeback_control wbc = { - .sync_mode = WB_SYNC_NONE, - .for_sync = true, - .nr_to_write = LONG_MAX, - .range_start = iocb->ki_pos, - .range_end = iocb->ki_pos + iter->count, - }; - struct netfs_io_request *wreq = NULL; - struct folio *folio = NULL, *writethrough = NULL; + struct folio *folio = NULL; unsigned int bdp_flags = (iocb->ki_flags & IOCB_NOWAIT) ? BDP_ASYNC : 0; - ssize_t written = 0, ret, ret2; - loff_t pos = iocb->ki_pos; + ssize_t written = 0, ret; + uoff_t pos = iocb->ki_pos; size_t max_chunk = mapping_max_folio_size(mapping); bool maybe_trouble = false; - if (unlikely(iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC)) - ) { - wbc_attach_fdatawrite_inode(&wbc, mapping->host); - - ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count); - if (ret < 0) { - wbc_detach_inode(&wbc); - goto out; - } - - wreq = netfs_begin_writethrough(iocb, iter->count); - if (IS_ERR(wreq)) { - wbc_detach_inode(&wbc); - ret = PTR_ERR(wreq); - wreq = NULL; - goto out; - } - if (!is_sync_kiocb(iocb)) - wreq->iocb = iocb; - netfs_stat(&netfs_n_wh_writethrough); - } else { - netfs_stat(&netfs_n_wh_buffered_write); - } + netfs_stat(&netfs_n_wh_buffered_write); do { enum netfs_folio_trace trace; struct netfs_folio *finfo; struct netfs_group *group; - unsigned long long fpos; + uoff_t fpos; size_t flen; size_t offset; /* Offset into pagecache folio */ size_t part; /* Bytes to write to folio */ @@ -390,15 +357,8 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, pos += copied; written += copied; - if (likely(!wreq)) { - folio_mark_dirty(folio); - folio_unlock(folio); - } else { - netfs_advance_writethrough(wreq, &wbc, folio, copied, - offset + copied == flen, - &writethrough); - /* Folio unlocked */ - } + folio_mark_dirty(folio); + folio_unlock(folio); retry: folio_put(folio); folio = NULL; @@ -420,15 +380,6 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, ctx->ops->post_modify(inode); } - if (unlikely(wreq)) { - ret2 = netfs_end_writethrough(wreq, &wbc, writethrough); - wbc_detach_inode(&wbc); - if (ret2 == -EIOCBQUEUED) - return ret2; - if (ret == 0 && ret2 < 0) - ret = ret2; - } - iocb->ki_pos += written; _leave(" = %zd [%zd]", written, ret); return written ? written : ret; diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c index 6a8fb0d55e04..8c15f3079723 100644 --- a/fs/netfs/direct_read.c +++ b/fs/netfs/direct_read.c @@ -47,15 +47,15 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) */ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) { - unsigned long long start = rreq->start; ssize_t size = rreq->len; + uoff_t start = rreq->start; int ret; do { struct netfs_io_subrequest *subreq; ssize_t slice; - subreq = netfs_alloc_subrequest(rreq); + subreq = netfs_alloc_subrequest(rreq, NETFS_DOWNLOAD_FROM_SERVER); if (!subreq) { /* Stash the error in the request if there's not * already an error set. @@ -64,7 +64,6 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) break; } - subreq->source = NETFS_DOWNLOAD_FROM_SERVER; subreq->start = start; subreq->len = size; @@ -84,10 +83,8 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) size -= slice; start += slice; rreq->submitted += slice; - if (size <= 0) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); - } + if (size <= 0) + netfs_all_subreqs_queued(rreq); rreq->netfs_ops->issue_read(subreq); @@ -99,8 +96,7 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) } while (size > 0); if (unlikely(size > 0)) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); netfs_wake_collector(rreq); } } diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index 8be74706d85e..cc46b7d9321f 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -225,9 +225,9 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * struct netfs_group *netfs_group) { struct netfs_io_request *wreq; - unsigned long long start = iocb->ki_pos; - unsigned long long end = start + iov_iter_count(iter); ssize_t ret, n; + uoff_t start = iocb->ki_pos; + uoff_t end = start + iov_iter_count(iter); size_t len = iov_iter_count(iter); bool async = !is_sync_kiocb(iocb); @@ -336,8 +336,8 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from) struct inode *inode = mapping->host; struct netfs_inode *ictx = netfs_inode(inode); ssize_t ret; - loff_t pos = iocb->ki_pos; - unsigned long long end = pos + iov_iter_count(from) - 1; + uoff_t pos = iocb->ki_pos; + uoff_t end = pos + iov_iter_count(from) - 1; _enter("%llx,%zx,%llx", pos, iov_iter_count(from), i_size_read(inode)); diff --git a/fs/netfs/fscache_cookie.c b/fs/netfs/fscache_cookie.c index 3d56fc73435f..5a226f9cbdea 100644 --- a/fs/netfs/fscache_cookie.c +++ b/fs/netfs/fscache_cookie.c @@ -327,7 +327,7 @@ static struct fscache_cookie *fscache_alloc_cookie( u8 advice, const void *index_key, size_t index_key_len, const void *aux_data, size_t aux_data_len, - loff_t object_size) + uoff_t object_size) { struct fscache_cookie *cookie; @@ -452,7 +452,7 @@ struct fscache_cookie *__fscache_acquire_cookie( u8 advice, const void *index_key, size_t index_key_len, const void *aux_data, size_t aux_data_len, - loff_t object_size) + uoff_t object_size) { struct fscache_cookie *cookie; @@ -663,7 +663,7 @@ static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) * Stop using the cookie for I/O. */ void __fscache_unuse_cookie(struct fscache_cookie *cookie, - const void *aux_data, const loff_t *object_size) + const void *aux_data, const uoff_t *object_size) { unsigned int debug_id = cookie->debug_id; unsigned int r = refcount_read(&cookie->ref); @@ -1049,7 +1049,7 @@ static void fscache_perform_invalidation(struct fscache_cookie *cookie) * Invalidate an object. */ void __fscache_invalidate(struct fscache_cookie *cookie, - const void *aux_data, loff_t new_size, + const void *aux_data, uoff_t new_size, unsigned int flags) { bool is_caching; diff --git a/fs/netfs/fscache_internal.h b/fs/netfs/fscache_internal.h deleted file mode 100644 index a09b948fcef2..000000000000 --- a/fs/netfs/fscache_internal.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* Internal definitions for FS-Cache - * - * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. - * Written by David Howells (dhowells@redhat.com) - */ - -#include "internal.h" - -#ifdef pr_fmt -#undef pr_fmt -#endif - -#define pr_fmt(fmt) "FS-Cache: " fmt diff --git a/fs/netfs/fscache_io.c b/fs/netfs/fscache_io.c index 37f05b4d3469..ea0b11807034 100644 --- a/fs/netfs/fscache_io.c +++ b/fs/netfs/fscache_io.c @@ -79,7 +79,7 @@ static int fscache_begin_operation(struct netfs_cache_resources *cres, cres->ops = NULL; cres->cache_priv = cookie; cres->cache_priv2 = NULL; - cres->debug_id = cookie->debug_id; + cres->cookie_id = cookie->debug_id; cres->inval_counter = cookie->inval_counter; if (!fscache_begin_cookie_access(cookie, why)) { @@ -162,7 +162,7 @@ EXPORT_SYMBOL(__fscache_begin_write_operation); struct fscache_write_request { struct netfs_cache_resources cache_resources; struct address_space *mapping; - loff_t start; + uoff_t start; size_t len; bool set_bits; bool using_pgpriv2; @@ -171,7 +171,7 @@ struct fscache_write_request { }; void __fscache_clear_page_bits(struct address_space *mapping, - loff_t start, size_t len) + uoff_t start, size_t len) { pgoff_t first = start / PAGE_SIZE; pgoff_t last = (start + len - 1) / PAGE_SIZE; @@ -208,7 +208,7 @@ static void fscache_wreq_done(void *priv, ssize_t transferred_or_error) void __fscache_write_to_cache(struct fscache_cookie *cookie, struct address_space *mapping, - loff_t start, size_t len, loff_t i_size, + uoff_t start, size_t len, uoff_t i_size, netfs_io_terminated_t term_func, void *term_func_priv, bool using_pgpriv2, bool cond) @@ -267,7 +267,7 @@ EXPORT_SYMBOL(__fscache_write_to_cache); /* * Change the size of a backing object. */ -void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +void __fscache_resize_cookie(struct fscache_cookie *cookie, uoff_t new_size, bool shrank) { struct netfs_cache_resources cres; @@ -281,7 +281,7 @@ void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) * netfs's inode lock so that we're serialised with respect to * writes. */ - cookie->volume->cache->ops->resize_cookie(&cres, new_size); + cookie->volume->cache->ops->resize_cookie(&cres, new_size, shrank); fscache_end_operation(&cres); } else { fscache_stat(&fscache_n_resizes_null); diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 786af76da98c..d970bd52a0b5 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -23,6 +23,8 @@ /* * buffered_read.c */ +int netfs_read_query_cache(struct netfs_io_request *rreq, + struct fscache_occupancy *occ); void netfs_queue_read(struct netfs_io_request *rreq, struct netfs_io_subrequest *subreq); void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error); @@ -33,7 +35,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio, * buffered_write.c */ void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode, - loff_t pos, size_t copied); + uoff_t pos, size_t copied); /* * main.c @@ -88,13 +90,14 @@ int netfs_clear_stale_pre_isize(struct inode *inode, uoff_t from, */ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, struct file *file, - loff_t start, size_t len, + uoff_t start, size_t len, enum netfs_io_origin origin); void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what); void netfs_clear_subrequests(struct netfs_io_request *rreq); void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what); void netfs_put_failed_request(struct netfs_io_request *rreq); -struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq); +struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq, + enum netfs_io_source source); static inline void netfs_see_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) @@ -122,9 +125,37 @@ void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error); /* * read_pgpriv2.c */ +#ifdef CONFIG_NETFS_PGPRIV2 +int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq, + unsigned int max_segs); void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio); void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq); bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq); +static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq) +{ + return unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)); +} +#else +static inline int netfs_prepare_pgpriv2_write_buffer(struct netfs_io_subrequest *subreq, + unsigned int max_segs) +{ + return -EIO; +} +static inline void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio) +{ +} +static inline void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq) +{ +} +static inline bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq) +{ + return true; +} +static inline bool netfs_using_pgpriv2(const struct netfs_io_request *rreq) +{ + return false; +} +#endif /* * read_retry.c @@ -159,7 +190,6 @@ extern atomic_t netfs_n_rh_write_zskip; extern atomic_t netfs_n_rh_retry_read_req; extern atomic_t netfs_n_rh_retry_read_subreq; extern atomic_t netfs_n_wh_buffered_write; -extern atomic_t netfs_n_wh_writethrough; extern atomic_t netfs_n_wh_dio_write; extern atomic_t netfs_n_wh_writepages; extern atomic_t netfs_n_wh_copy_to_cache; @@ -205,11 +235,11 @@ void netfs_write_collection_worker(struct work_struct *work); */ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, struct file *file, - loff_t start, + uoff_t start, enum netfs_io_origin origin); void netfs_prepare_write(struct netfs_io_request *wreq, struct netfs_io_stream *stream, - loff_t start); + uoff_t start); void netfs_reissue_write(struct netfs_io_stream *stream, struct netfs_io_subrequest *subreq, struct iov_iter *source); @@ -217,13 +247,7 @@ void netfs_issue_write(struct netfs_io_request *wreq, struct netfs_io_stream *stream); size_t netfs_advance_write(struct netfs_io_request *wreq, struct netfs_io_stream *stream, - loff_t start, size_t len, bool to_eof); -struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len); -int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, - struct folio *folio, size_t copied, bool to_page_end, - struct folio **writethrough_cache); -ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, - struct folio *writethrough_cache); + uoff_t start, size_t len, bool to_eof); /* * write_retry.c @@ -322,6 +346,26 @@ static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subreque return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags); } +/* + * Indicate that we've generated and queued all the subrequests we're going to. + */ +static inline void netfs_all_subreqs_queued(struct netfs_io_request *rreq) +{ + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + smp_mb__after_atomic(); + trace_netfs_rreq(rreq, netfs_rreq_trace_all_queued); +} + +/* + * Query if all subrequests are queued. + */ +static inline bool netfs_are_all_subreqs_queued(const struct netfs_io_request *rreq) +{ + /* Read lists after ALL_QUEUED. */ + return test_bit_acquire(NETFS_RREQ_ALL_QUEUED, &rreq->flags); +} + /* * fscache-cache.c */ diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c index b375567e0520..eb1efb17f53a 100644 --- a/fs/netfs/iterator.c +++ b/fs/netfs/iterator.c @@ -209,7 +209,7 @@ static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offse { struct folio *folio; unsigned int nsegs = 0; - loff_t pos = iter->xarray_start + iter->iov_offset; + uoff_t pos = iter->xarray_start + iter->iov_offset; pgoff_t index = pos / PAGE_SIZE; size_t span = 0, n = iter->count; diff --git a/fs/netfs/main.c b/fs/netfs/main.c index 927badf3989d..609e22e8f76a 100644 --- a/fs/netfs/main.c +++ b/fs/netfs/main.c @@ -44,7 +44,6 @@ static const char *netfs_origins[nr__netfs_io_origin] = { [NETFS_DIO_READ] = "DR", [NETFS_WRITEBACK] = "WB", [NETFS_WRITEBACK_SINGLE] = "W1", - [NETFS_WRITETHROUGH] = "WT", [NETFS_UNBUFFERED_WRITE] = "UW", [NETFS_DIO_WRITE] = "DW", [NETFS_PGPRIV2_COPY_TO_CACHE] = "2C", diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index 523057390d6f..5465db834b94 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -194,7 +194,7 @@ void netfs_clear_inode_writeback(struct inode *inode, const void *aux) struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode)); if (inode_state_read_once(inode) & I_PINNING_NETFS_WB) { - loff_t i_size = i_size_read(inode); + uoff_t i_size = i_size_read(inode); fscache_unuse_cookie(cookie, aux, &i_size); } } @@ -219,8 +219,8 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length) _enter("{%lx},%zx,%zx", folio->index, offset, length); if (offset == 0 && length == flen) { - unsigned long long i_size, remote_i_size, zero_point; - unsigned long long fpos = folio_pos(folio), end; + uoff_t i_size, remote_i_size, zero_point; + uoff_t fpos = folio_pos(folio), end; netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); end = umin(fpos + flen, i_size); @@ -306,7 +306,7 @@ bool netfs_release_folio(struct folio *folio, gfp_t gfp) { struct inode *inode = folio_inode(folio); struct netfs_inode *ctx = netfs_inode(inode); - unsigned long long i_size, remote_i_size, zero_point, end; + uoff_t i_size, remote_i_size, zero_point, end; if (folio_test_dirty(folio)) return false; @@ -425,7 +425,7 @@ static int netfs_collect_in_app(struct netfs_io_request *rreq, need_collect = true; break; } - if (subreq || !test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags)) + if (subreq || !netfs_are_all_subreqs_queued(rreq)) done = false; } diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c index ad549daa9c79..4b8d20559b0e 100644 --- a/fs/netfs/objects.c +++ b/fs/netfs/objects.c @@ -16,7 +16,7 @@ static void netfs_free_request(struct work_struct *work); */ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, struct file *file, - loff_t start, size_t len, + uoff_t start, size_t len, enum netfs_io_origin origin) { static atomic_t debug_ids; @@ -44,6 +44,7 @@ struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, rreq->gfp = gfp; rreq->start = start; rreq->collected_to = start; + rreq->cache_coll_to = start; rreq->cleaned_to = start; rreq->len = len; rreq->progress_at = 0; @@ -207,7 +208,8 @@ void netfs_put_failed_request(struct netfs_io_request *rreq) /* * Allocate and partially initialise an I/O request structure. */ -struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq) +struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq, + enum netfs_io_source source) { struct netfs_io_subrequest *subreq; mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool; @@ -224,6 +226,7 @@ struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq INIT_WORK(&subreq->work, NULL); INIT_LIST_HEAD(&subreq->rreq_link); refcount_set(&subreq->ref, 2); + subreq->source = source; subreq->rreq = rreq; subreq->debug_index = atomic_inc_return(&rreq->subreq_counter); netfs_get_request(rreq, netfs_rreq_trace_get_subreq); diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index c5bfaf2c6f25..be0f9d2f7e10 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -54,7 +54,7 @@ static void netfs_clear_unread_dio(struct netfs_io_subrequest *subreq) */ void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio) { - if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { + if (!netfs_using_pgpriv2(rreq)) { if (folio_get_private(folio) == NETFS_FOLIO_COPY_TO_CACHE) { folio_detach_private(folio); trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); @@ -97,7 +97,7 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq, if (unlikely(test_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags))) netfs_cancel_copy_to_cache(rreq, folio); - if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { + if (!netfs_using_pgpriv2(rreq)) { if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) { trace_netfs_folio(folio, netfs_folio_trace_sched_copy); folio_mark_dirty(folio); @@ -169,8 +169,8 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq, unsigned int *notes) { struct folio_queue *folioq = rreq->buffer.tail; - unsigned long long collected_to = rreq->collected_to; unsigned int slot = rreq->buffer.first_tail_slot; + uoff_t collected_to = rreq->collected_to; if (rreq->cleaned_to >= rreq->collected_to) return; @@ -195,7 +195,7 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq, for (;;) { struct folio *folio; - unsigned long long fpos, fend; + uoff_t fpos, fend; size_t fsize; folio = folioq_folio(folioq, slot); @@ -208,7 +208,7 @@ static void netfs_read_unlock_folios(struct netfs_io_request *rreq, fpos = folio_pos(folio); fend = fpos + fsize; - trace_netfs_collect_folio(rreq, folio, fend, collected_to); + trace_netfs_collect_folio(rreq, folio); /* Unlock any folio we've transferred all of. */ if (collected_to < fend) @@ -491,10 +491,8 @@ bool netfs_read_collection(struct netfs_io_request *rreq) /* We're done when the app thread has finished posting subreqs and the * queue is empty. */ - if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags)) + if (!netfs_are_all_subreqs_queued(rreq)) return false; - smp_rmb(); /* Read ALL_QUEUED before subreq lists. */ - if (!list_empty(&stream->subrequests)) return false; diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c index a4b7bb88cbdb..5280b606fda4 100644 --- a/fs/netfs/read_pgpriv2.c +++ b/fs/netfs/read_pgpriv2.c @@ -20,7 +20,7 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio { struct netfs_io_stream *cache = &creq->io_streams[1]; size_t fsize = folio_size(folio), flen = fsize; - loff_t fpos = folio_pos(folio), i_size; + uoff_t fpos = folio_pos(folio), i_size; bool to_eof = false; _enter(""); @@ -158,8 +158,7 @@ void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq) return; netfs_issue_write(creq, &creq->io_streams[1]); - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &creq->flags); + netfs_all_subreqs_queued(creq); trace_netfs_rreq(rreq, netfs_rreq_trace_end_copy_to_cache); if (list_empty_careful(&creq->io_streams[1].subrequests)) netfs_wake_collector(creq); @@ -175,8 +174,8 @@ void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq) bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *creq) { struct folio_queue *folioq = creq->buffer.tail; - unsigned long long collected_to = creq->collected_to; unsigned int slot = creq->buffer.first_tail_slot; + uoff_t collected_to = creq->collected_to; bool made_progress = false; if (slot >= folioq_nr_slots(folioq)) { @@ -186,7 +185,7 @@ bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *creq) for (;;) { struct folio *folio; - unsigned long long fpos, fend; + uoff_t fpos, fend; size_t fsize, flen; folio = folioq_folio(folioq, slot); @@ -199,9 +198,9 @@ bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *creq) fsize = folio_size(folio); flen = fsize; - fend = min_t(unsigned long long, fpos + flen, creq->i_size); + fend = min_t(uoff_t, fpos + flen, creq->i_size); - trace_netfs_collect_folio(creq, folio, fend, collected_to); + trace_netfs_collect_folio(creq, folio); /* Unlock any folio we've transferred all of. */ if (collected_to < fend) diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index 4f6a36c6e214..5bd8dee5a834 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -75,7 +75,7 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) do { struct netfs_io_subrequest *from, *to, *tmp; struct iov_iter source; - unsigned long long start, len; + uoff_t start, len; size_t part; bool boundary = false, subreq_superfluous = false; @@ -195,12 +195,11 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) * and insert them after. */ do { - subreq = netfs_alloc_subrequest(rreq); + subreq = netfs_alloc_subrequest(rreq, NETFS_DOWNLOAD_FROM_SERVER); if (!subreq) { subreq = to; goto abandon_after; } - subreq->source = NETFS_DOWNLOAD_FROM_SERVER; subreq->start = start; subreq->len = len; subreq->stream_nr = stream->stream_nr; @@ -272,6 +271,7 @@ void netfs_retry_reads(struct netfs_io_request *rreq) struct netfs_io_stream *stream = &rreq->io_streams[0]; netfs_stat(&netfs_n_rh_retry_read_req); + trace_netfs_rreq(rreq, netfs_rreq_trace_retry_begin); /* Wait for all outstanding I/O to quiesce before performing retries as * we may need to renegotiate the I/O sizes. @@ -282,6 +282,7 @@ void netfs_retry_reads(struct netfs_io_request *rreq) trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); netfs_retry_read_subrequests(rreq); + trace_netfs_rreq(rreq, netfs_rreq_trace_retry_end); } /* diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c index de67ac41548d..b248e34bd0c8 100644 --- a/fs/netfs/read_single.c +++ b/fs/netfs/read_single.c @@ -58,20 +58,6 @@ static int netfs_single_begin_cache_read(struct netfs_io_request *rreq, struct n return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); } -static void netfs_single_cache_prepare_read(struct netfs_io_request *rreq, - struct netfs_io_subrequest *subreq) -{ - struct netfs_cache_resources *cres = &rreq->cache_resources; - - if (!cres->ops) { - subreq->source = NETFS_DOWNLOAD_FROM_SERVER; - return; - } - subreq->source = cres->ops->prepare_read(subreq, rreq->i_size); - trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); - -} - static void netfs_single_read_cache(struct netfs_io_request *rreq, struct netfs_io_subrequest *subreq) { @@ -89,21 +75,36 @@ static void netfs_single_read_cache(struct netfs_io_request *rreq, */ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) { + struct fscache_occupancy occ = { + .query_from = 0, + .query_to = rreq->len, + .cached_from[0] = ULLONG_MAX, + .cached_to[0] = ULLONG_MAX, + .cached_from[1] = ULLONG_MAX, + .cached_to[1] = ULLONG_MAX, + }; struct netfs_io_subrequest *subreq; + enum netfs_io_source source = NETFS_DOWNLOAD_FROM_SERVER; int ret = 0; - subreq = netfs_alloc_subrequest(rreq); + /* Try to use the cache if the cache content matches the size of the + * remote file. + */ + netfs_read_query_cache(rreq, &occ); + if (occ.cached_from[0] == 0 && + occ.cached_to[0] >= rreq->len) + source = NETFS_READ_FROM_CACHE; + + subreq = netfs_alloc_subrequest(rreq, source); if (!subreq) return -ENOMEM; - subreq->source = NETFS_SOURCE_UNKNOWN; subreq->start = 0; subreq->len = rreq->len; subreq->io_iter = rreq->buffer.iter; netfs_queue_read(rreq, subreq); - netfs_single_cache_prepare_read(rreq, subreq); switch (subreq->source) { case NETFS_DOWNLOAD_FROM_SERVER: netfs_stat(&netfs_n_rh_download); @@ -113,14 +114,18 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) goto cancel; } - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); rreq->netfs_ops->issue_read(subreq); rreq->submitted += subreq->len; break; case NETFS_READ_FROM_CACHE: - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + if (rreq->cache_resources.ops->prepare_read) { + ret = rreq->cache_resources.ops->prepare_read(subreq); + if (ret < 0) + goto cancel; + } + + netfs_all_subreqs_queued(rreq); trace_netfs_sreq(subreq, netfs_sreq_trace_submit); netfs_single_read_cache(rreq, subreq); rreq->submitted += subreq->len; @@ -136,8 +141,7 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) return ret; cancel: netfs_cancel_read(subreq, ret); - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); netfs_wake_collector(rreq); return ret; } diff --git a/fs/netfs/stats.c b/fs/netfs/stats.c index ab6b916addc4..9a607c4e62dd 100644 --- a/fs/netfs/stats.c +++ b/fs/netfs/stats.c @@ -32,7 +32,6 @@ atomic_t netfs_n_rh_write_zskip; atomic_t netfs_n_rh_retry_read_req; atomic_t netfs_n_rh_retry_read_subreq; atomic_t netfs_n_wh_buffered_write; -atomic_t netfs_n_wh_writethrough; atomic_t netfs_n_wh_dio_write; atomic_t netfs_n_wh_writepages; atomic_t netfs_n_wh_copy_to_cache; @@ -58,9 +57,8 @@ int netfs_stats_show(struct seq_file *m, void *v) atomic_read(&netfs_n_rh_read_single), atomic_read(&netfs_n_rh_write_begin), atomic_read(&netfs_n_rh_write_zskip)); - seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u 2C=%u\n", + seq_printf(m, "Writes : BW=%u DW=%u WP=%u 2C=%u\n", atomic_read(&netfs_n_wh_buffered_write), - atomic_read(&netfs_n_wh_writethrough), atomic_read(&netfs_n_wh_dio_write), atomic_read(&netfs_n_wh_writepages), atomic_read(&netfs_n_wh_copy_to_cache)); diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c index 210eb8f3958d..6e8ea534230d 100644 --- a/fs/netfs/write_collect.c +++ b/fs/netfs/write_collect.c @@ -56,7 +56,7 @@ static void netfs_dump_request(const struct netfs_io_request *rreq) */ int netfs_folio_written_back(struct folio *folio) { - enum netfs_folio_trace why = netfs_folio_trace_clear; + enum netfs_folio_trace why = netfs_folio_trace_endwb; struct inode *inode = folio_inode(folio); struct netfs_inode *ictx = netfs_inode(inode); struct netfs_folio *finfo; @@ -67,7 +67,7 @@ int netfs_folio_written_back(struct folio *folio) /* Streaming writes cannot be redirtied whilst under writeback, * so discard the streaming record. */ - unsigned long long fend; + uoff_t fend; fend = folio_pos(folio) + finfo->dirty_offset + finfo->dirty_len; spin_lock(&ictx->inode.i_lock); @@ -79,13 +79,13 @@ int netfs_folio_written_back(struct folio *folio) group = finfo->netfs_group; gcount++; kfree(finfo); - why = netfs_folio_trace_clear_s; + why = netfs_folio_trace_endwb_s; goto end_wb; } if ((group = netfs_folio_group(folio))) { if (group == NETFS_FOLIO_COPY_TO_CACHE) { - why = netfs_folio_trace_clear_cc; + why = netfs_folio_trace_endwb_cc; folio_detach_private(folio); goto end_wb; } @@ -98,7 +98,7 @@ int netfs_folio_written_back(struct folio *folio) if (!folio_test_dirty(folio)) { folio_detach_private(folio); gcount++; - why = netfs_folio_trace_clear_g; + why = netfs_folio_trace_endwb_g; } } @@ -115,8 +115,8 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, unsigned int *notes) { struct folio_queue *folioq = wreq->buffer.tail; - unsigned long long collected_to = wreq->collected_to; unsigned int slot = wreq->buffer.first_tail_slot; + uoff_t collected_to = wreq->collected_to; if (WARN_ON_ONCE(!folioq)) { pr_err("[!] Writeback unlock found empty rolling buffer!\n"); @@ -140,7 +140,7 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, for (;;) { struct folio *folio; struct netfs_folio *finfo; - unsigned long long fpos, fend; + uoff_t fpos, fend; size_t fsize, flen; folio = folioq_folio(folioq, slot); @@ -154,9 +154,9 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, finfo = netfs_folio_info(folio); flen = finfo ? finfo->dirty_offset + finfo->dirty_len : fsize; - fend = min_t(unsigned long long, fpos + flen, wreq->i_size); + fend = min_t(uoff_t, fpos + flen, wreq->i_size); - trace_netfs_collect_folio(wreq, folio, fend, collected_to); + trace_netfs_collect_folio(wreq, folio); /* Unlock any folio we've transferred all of. */ if (collected_to < fend) @@ -188,6 +188,26 @@ static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, wreq->buffer.first_tail_slot = slot; } +/* + * Collect cache results. + */ +static void netfs_cache_collect(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + enum netfs_cache_collect block_type) +{ + struct netfs_cache_resources *cres = &wreq->cache_resources; + + if (stream->source != NETFS_WRITE_TO_CACHE || + wreq->cache_coll_to >= stream->collected_to) + return; + + if (cres->ops && cres->ops->collect_write) + cres->ops->collect_write(wreq, wreq->cache_coll_to, + stream->collected_to - wreq->cache_coll_to, + block_type); + wreq->cache_coll_to = stream->collected_to; +} + /* * Collect and assess the results of various write subrequests. We may need to * retry some of the results - or even do an RMW cycle for content crypto. @@ -201,8 +221,8 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) { struct netfs_io_subrequest *front, *remove; struct netfs_io_stream *stream; - unsigned long long collected_to, issued_to; unsigned int notes; + uoff_t collected_to, issued_to; int s; _enter("%llx-%llx", wreq->start, wreq->start + wreq->len); @@ -214,7 +234,6 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) smp_rmb(); collected_to = ULLONG_MAX; if (wreq->origin == NETFS_WRITEBACK || - wreq->origin == NETFS_WRITETHROUGH || wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) notes = NEED_UNLOCK; else @@ -236,13 +255,19 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) /* Read first subreq pointer before IN_PROGRESS flag. */ while (front) { + enum netfs_cache_collect cache_collect; + trace_netfs_collect_sreq(wreq, front); //_debug("sreq [%x] %llx %zx/%zx", // front->debug_index, front->start, front->transferred, front->len); if (stream->collected_to < front->start) { trace_netfs_collect_gap(wreq, stream, issued_to, 'F'); + if (stream->cache_collect != NETFS_CACHE_COLLECT_WRITE_GAP) + netfs_cache_collect(wreq, stream, stream->cache_collect); stream->collected_to = front->start; + netfs_cache_collect(wreq, stream, NETFS_CACHE_COLLECT_WRITE_GAP); + stream->cache_collect = NETFS_CACHE_COLLECT_WRITE_GAP; } /* Stall if the front is still undergoing I/O. */ @@ -250,7 +275,6 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) notes |= HIT_PENDING; break; } - smp_rmb(); /* Read counters after I-P flag. */ if (stream->failed) { stream->collected_to = front->start + front->len; @@ -263,15 +287,44 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) stream->transferred_valid = true; notes |= MADE_PROGRESS; } - if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { - stream->failed = true; - stream->error = front->error; - if (stream->source == NETFS_UPLOAD_TO_SERVER) - mapping_set_error(wreq->mapping, front->error); - notes |= NEED_REASSESS | SAW_FAILURE; + + /* Handle failed or cancelled subreqs. Failure of + * cache writes are handled differently to upload + * failures. Cache writes aren't fatal, provided we're + * not doing disconnected operation, and so we can kind + * of treat them as if they had succeeded - except that + * we need to log any holes they cause. + */ + switch (stream->source) { + case NETFS_UPLOAD_TO_SERVER: + if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { + if (!stream->failed) { + stream->failed = true; + stream->error = front->error; + mapping_set_error(wreq->mapping, front->error); + break; + } + notes |= NEED_REASSESS | SAW_FAILURE; + } + break; + + case NETFS_WRITE_TO_CACHE: + cache_collect = test_bit(NETFS_SREQ_CANCELLED, &front->flags) ? + NETFS_CACHE_COLLECT_WRITE_CANCEL : + NETFS_CACHE_COLLECT_WRITE_DATA; + if (cache_collect != stream->cache_collect && + stream->cache_collect != NETFS_CACHE_COLLECT_WRITE_GAP) { + trace_netfs_rreq(wreq, netfs_rreq_trace_cache_fail_collect); + netfs_cache_collect(wreq, stream, stream->cache_collect); + } + stream->cache_collect = cache_collect; + break; + + default: + WARN_ON(1); break; } - if (front->transferred < front->len) { + if (test_bit(NETFS_SREQ_NEED_RETRY, &front->flags)) { stream->need_retry = true; notes |= NEED_RETRY | MADE_PROGRESS; break; @@ -360,6 +413,7 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq) */ bool netfs_write_collection(struct netfs_io_request *wreq) { + struct netfs_io_stream *cstream = &wreq->io_streams[1]; struct netfs_inode *ictx = netfs_inode(wreq->inode); size_t transferred; bool transferred_valid = false; @@ -372,9 +426,8 @@ bool netfs_write_collection(struct netfs_io_request *wreq) /* We're done when the app thread has finished posting subreqs and all * the queues in all the streams are empty. */ - if (!test_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags)) + if (!netfs_are_all_subreqs_queued(wreq)) return false; - smp_rmb(); /* Read ALL_QUEUED before lists. */ transferred = LONG_MAX; for (s = 0; s < NR_IO_STREAMS; s++) { @@ -395,13 +448,19 @@ bool netfs_write_collection(struct netfs_io_request *wreq) wreq->transferred = transferred; trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); - if (wreq->io_streams[1].active && - wreq->io_streams[1].failed && - ictx->ops->invalidate_cache) { - /* Cache write failure doesn't prevent writeback completion - * unless we're in disconnected mode. - */ - ictx->ops->invalidate_cache(wreq); + if (cstream->active) { + if (test_bit(NETFS_RREQ_CACHE_ERROR, &wreq->flags)) { + if (ictx->ops->invalidate_cache) { + /* Cache write failure doesn't prevent + * writeback completion unless we're in + * disconnected mode. + */ + trace_netfs_rreq(wreq, netfs_rreq_trace_inval_cache); + ictx->ops->invalidate_cache(wreq); + } + } else if (!cstream->failed) { + netfs_cache_collect(wreq, cstream, cstream->cache_collect); + } } _debug("finished"); @@ -411,7 +470,6 @@ bool netfs_write_collection(struct netfs_io_request *wreq) switch (wreq->origin) { case NETFS_WRITEBACK: case NETFS_WRITEBACK_SINGLE: - case NETFS_WRITETHROUGH: netfs_wb_end(ictx); break; default: @@ -486,24 +544,51 @@ void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error) if (IS_ERR_VALUE(transferred_or_error)) { subreq->error = transferred_or_error; - /* if need retry is set, error should not matter */ - if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { - set_bit(NETFS_SREQ_FAILED, &subreq->flags); - trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); - } switch (subreq->source) { case NETFS_WRITE_TO_CACHE: + /* We don't mark a cache-write subreq as failed. + * Instead we tell the issuer to produce dummy subreqs + * instead and make a note if we need to invalidate the + * cache at the end. We also don't pause the loop that + * grabs pages and launches upload subreqs. + * + * Note that we need to distinguish between -ENOBUFS + * (no space available in the cache) and other errors. + * In the former case, we can keep the data we have, + * though we might have to change the way the on-disk + * data is tracked. + */ netfs_stat(&netfs_n_wh_write_failed); + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) + break; + + trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + __set_bit(NETFS_SREQ_CANCELLED, &subreq->flags); + set_bit(NETFS_RREQ_CACHE_STOP, &wreq->flags); + if (transferred_or_error == -ENOBUFS) + trace_netfs_rreq(wreq, netfs_rreq_trace_cache_no_space); + else if (!test_and_set_bit(NETFS_RREQ_CACHE_ERROR, &wreq->flags)) + trace_netfs_rreq(wreq, netfs_rreq_trace_cache_failed); + subreq->transferred = subreq->len; break; + case NETFS_UPLOAD_TO_SERVER: + /* If need_retry is set, error should not matter */ + if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + set_bit(NETFS_SREQ_FAILED, &subreq->flags); + trace_netfs_failure(wreq, subreq, transferred_or_error, + netfs_fail_upload); + } + + set_bit(NETFS_RREQ_PAUSE, &wreq->flags); + trace_netfs_rreq(wreq, netfs_rreq_trace_set_pause); netfs_stat(&netfs_n_wh_upload_failed); break; + default: break; } - trace_netfs_rreq(wreq, netfs_rreq_trace_set_pause); - set_bit(NETFS_RREQ_PAUSE, &wreq->flags); } else { if (WARN(transferred_or_error > subreq->len - subreq->transferred, "Subreq excess write: R=%x[%x] %zd > %zu - %zu", diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index 851f6f93ad45..b8237e1314d8 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -89,14 +89,13 @@ static void netfs_kill_dirty_pages(struct address_space *mapping, */ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, struct file *file, - loff_t start, + uoff_t start, enum netfs_io_origin origin) { struct netfs_io_request *wreq; struct netfs_inode *ictx; bool is_cacheable = (origin == NETFS_WRITEBACK || origin == NETFS_WRITEBACK_SINGLE || - origin == NETFS_WRITETHROUGH || origin == NETFS_PGPRIV2_COPY_TO_CACHE); wreq = netfs_alloc_request(mapping, file, start, 0, origin); @@ -112,6 +111,8 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, goto nomem; wreq->cleaned_to = wreq->start; + if (wreq->cache_resources.dio_size > 1) + wreq->cache_coll_to = round_down(wreq->start, wreq->cache_resources.dio_size); wreq->io_streams[0].stream_nr = 0; wreq->io_streams[0].source = NETFS_UPLOAD_TO_SERVER; @@ -145,6 +146,7 @@ struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, */ void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq) { + WARN_ON_ONCE(!subreq->error); __set_bit(NETFS_SREQ_FAILED, &subreq->flags); trace_netfs_sreq(subreq, netfs_sreq_trace_prep_failed); } @@ -156,7 +158,7 @@ EXPORT_SYMBOL(netfs_prepare_write_failed); */ void netfs_prepare_write(struct netfs_io_request *wreq, struct netfs_io_stream *stream, - loff_t start) + uoff_t start) { struct netfs_io_subrequest *subreq; struct iov_iter *wreq_iter = &wreq->buffer.iter; @@ -169,10 +171,9 @@ void netfs_prepare_write(struct netfs_io_request *wreq, wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq)) rolling_buffer_make_space(&wreq->buffer, wreq->gfp); - subreq = netfs_alloc_subrequest(wreq); + subreq = netfs_alloc_subrequest(wreq, stream->source); if (!subreq) return; - subreq->source = stream->source; subreq->start = start; subreq->stream_nr = stream->stream_nr; subreq->io_iter = *wreq_iter; @@ -233,6 +234,21 @@ static void netfs_do_issue_write(struct netfs_io_stream *stream, _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len); + if (stream->source == NETFS_WRITE_TO_CACHE && + unlikely(test_bit(NETFS_RREQ_CACHE_STOP, &wreq->flags))) { + size_t dio_size = wreq->cache_resources.dio_size; + size_t len, disp; + + disp = subreq->start & (dio_size - 1); + len = round_up(subreq->len + disp, dio_size); + + subreq->start -= disp; + subreq->len = len; + + __set_bit(NETFS_SREQ_CANCELLED, &subreq->flags); + return netfs_write_subrequest_terminated(subreq, subreq->len); + } + if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) return netfs_write_subrequest_terminated(subreq, subreq->error); @@ -266,6 +282,7 @@ void netfs_issue_write(struct netfs_io_request *wreq, if (!subreq) return; + stream->construct = NULL; subreq->io_iter.count = subreq->len; netfs_do_issue_write(stream, subreq); @@ -279,7 +296,7 @@ void netfs_issue_write(struct netfs_io_request *wreq, */ size_t netfs_advance_write(struct netfs_io_request *wreq, struct netfs_io_stream *stream, - loff_t start, size_t len, bool to_eof) + uoff_t start, size_t len, bool to_eof) { struct netfs_io_subrequest *subreq = stream->construct; size_t part; @@ -330,7 +347,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq, struct netfs_folio *finfo; size_t iter_off = 0; size_t fsize = folio_size(folio), flen = fsize, foff = 0; - loff_t fpos = folio_pos(folio), i_size; + uoff_t fpos = folio_pos(folio), i_size; bool to_eof = false, streamw = false; bool debug = false; @@ -367,11 +384,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq, streamw = true; } - if (wreq->origin == NETFS_WRITETHROUGH) { - to_eof = false; - if (flen > i_size - fpos) - flen = i_size - fpos; - } else if (flen > i_size - fpos) { + if (flen > i_size - fpos) { flen = i_size - fpos; if (!streamw) folio_zero_segment(folio, flen, fsize); @@ -525,8 +538,7 @@ static void netfs_end_issue_write(struct netfs_io_request *wreq) { bool needs_poke = true; - smp_wmb(); /* Write subreq lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); + netfs_all_subreqs_queued(wreq); for (int s = 0; s < NR_IO_STREAMS; s++) { struct netfs_io_stream *stream = &wreq->io_streams[s]; @@ -613,103 +625,6 @@ int netfs_writepages(struct address_space *mapping, } EXPORT_SYMBOL(netfs_writepages); -/* - * Begin a write operation for writing through the pagecache. - */ -struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len) -{ - struct netfs_io_request *wreq = NULL; - struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp)); - - netfs_wb_begin(ictx, false); - - wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp, - iocb->ki_pos, NETFS_WRITETHROUGH); - if (IS_ERR(wreq)) { - netfs_wb_end(ictx); - return wreq; - } - - wreq->io_streams[0].avail = true; - __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); - trace_netfs_write(wreq, netfs_write_trace_writethrough); - return wreq; -} - -/* - * Advance the state of the write operation used when writing through the - * pagecache. Data has been copied into the pagecache that we need to append - * to the request. If we've added more than wsize then we need to create a new - * subrequest. - */ -int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, - struct folio *folio, size_t copied, bool to_page_end, - struct folio **writethrough_cache) -{ - int ret; - - _enter("R=%x ic=%zu ws=%u cp=%zu tp=%u", - wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end); - - /* The folio is locked. */ - - if (*writethrough_cache != folio) { - if (*writethrough_cache) { - /* Did the folio get moved? */ - folio_put(*writethrough_cache); - *writethrough_cache = NULL; - } - /* We can make multiple writes to the folio... */ - if (wreq->len == 0) - trace_netfs_folio(folio, netfs_folio_trace_wthru); - else - trace_netfs_folio(folio, netfs_folio_trace_wthru_plus); - *writethrough_cache = folio; - folio_get(folio); - } - - wreq->len += copied; - - if (!to_page_end) { - folio_mark_dirty(folio); - folio_unlock(folio); - return 0; - } - - ret = netfs_write_folio(wreq, wbc, folio); - folio_put(*writethrough_cache); - *writethrough_cache = NULL; - wreq->submitted = wreq->len; - return ret; -} - -/* - * End a write operation used when writing through the pagecache. - */ -ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, - struct folio *writethrough_cache) -{ - ssize_t ret; - - _enter("R=%x", wreq->debug_id); - - if (writethrough_cache) { - folio_lock(writethrough_cache); - netfs_write_folio(wreq, wbc, writethrough_cache); - folio_put(writethrough_cache); - wreq->submitted = wreq->len; - } - - netfs_end_issue_write(wreq); - - if (wreq->iocb) - ret = -EIOCBQUEUED; - else - ret = netfs_wait_for_write(wreq); - netfs_put_request(wreq, netfs_rreq_trace_put_return); - return ret; -} - /* * Write some of a pending folio data back to the server and/or the cache. */ @@ -721,7 +636,7 @@ static int netfs_write_folio_single(struct netfs_io_request *wreq, struct netfs_io_stream *stream; size_t iter_off = 0; size_t fsize = folio_size(folio), flen; - loff_t fpos = folio_pos(folio); + uoff_t fpos = folio_pos(folio); ssize_t ret; bool to_eof = false; bool no_debug = false; @@ -891,8 +806,7 @@ int netfs_writeback_single(struct address_space *mapping, stop: for (int s = 0; s < NR_IO_STREAMS; s++) netfs_issue_write(wreq, &wreq->io_streams[s]); - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); + netfs_all_subreqs_queued(wreq); netfs_wake_collector(wreq); diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index 058bc7a166a5..2f20577563e1 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -55,7 +55,7 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, do { struct netfs_io_subrequest *subreq = NULL, *from, *to, *tmp; struct iov_iter source; - unsigned long long start, len; + uoff_t start, len; size_t part; bool boundary = false; @@ -149,8 +149,7 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, * and insert them after. */ do { - subreq = netfs_alloc_subrequest(wreq); - subreq->source = to->source; + subreq = netfs_alloc_subrequest(wreq, stream->source); subreq->start = start; subreq->stream_nr = to->stream_nr; subreq->retry_count = 1; @@ -211,6 +210,7 @@ void netfs_retry_writes(struct netfs_io_request *wreq) int s; netfs_stat(&netfs_n_wh_retry_write_req); + trace_netfs_rreq(wreq, netfs_rreq_trace_retry_begin); /* Wait for all outstanding I/O to quiesce before performing retries as * we may need to renegotiate the I/O sizes. @@ -235,4 +235,6 @@ void netfs_retry_writes(struct netfs_io_request *wreq) netfs_retry_write_stream(wreq, stream); } } + + trace_netfs_rreq(wreq, netfs_rreq_trace_retry_end); } diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig index 6bb30543eff0..e7862f35b72c 100644 --- a/fs/nfs/Kconfig +++ b/fs/nfs/Kconfig @@ -174,6 +174,7 @@ config NFS_FSCACHE bool "Provide NFS client caching support" depends on NFS_FS select NETFS_SUPPORT + select NETFS_PGPRIV2 select FSCACHE help Say Y here if you want NFS data to be cached locally on disc through diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index 378916046ee0..f456b42986f6 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -1332,7 +1332,7 @@ static int cifs_precopy_set_eof(struct inode *src_inode, struct cifsInodeInfo *s goto set_failed; netfs_resize_file(&src_cifsi->netfs, src_end, true); - fscache_resize_cookie(cifs_inode_cookie(src_inode), src_end); + fscache_resize_cookie(cifs_inode_cookie(src_inode), src_end, false); return 0; set_failed: @@ -1626,9 +1626,9 @@ ssize_t cifs_file_copychunk_range(unsigned int xid, if (rc > 0 && destoff + rc > i_size_read(target_inode)) { truncate_setsize(target_inode, destoff + rc); netfs_resize_file(&target_cifsi->netfs, - i_size_read(target_inode), true); + destoff + rc, true); fscache_resize_cookie(cifs_inode_cookie(target_inode), - i_size_read(target_inode)); + destoff + rc, false); } if (rc > 0) { spin_lock(&target_inode->i_lock); diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index b0784f5e67e4..b6ff019e9923 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -3097,7 +3097,7 @@ void cifs_resize_file_locked(struct inode *inode, loff_t old_size, return; fscache_use_cookie(cookie, true); - fscache_resize_cookie(cookie, offset); + fscache_resize_cookie(cookie, offset, offset < old_size); cifs_fscache_unuse_inode_cookie(inode, true); } diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 583ab4c4d773..279c41893f8e 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -3611,7 +3611,8 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, if (offset < cifsi->netfs._zero_point) netfs_write_zero_point(inode, offset); spin_unlock(&inode->i_lock); - fscache_resize_cookie(cifs_inode_cookie(inode), new_size); + fscache_resize_cookie(cifs_inode_cookie(inode), + new_size, false); } } @@ -4146,7 +4147,7 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, netfs_resize_file(&cifsi->netfs, new_eof, true); netfs_write_zero_point(inode, new_eof); spin_unlock(&inode->i_lock); - fscache_resize_cookie(cifs_inode_cookie(inode), new_eof); + fscache_resize_cookie(cifs_inode_cookie(inode), new_eof, true); out_2: filemap_invalidate_unlock(inode->i_mapping); out: @@ -4206,9 +4207,9 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, truncate_setsize(inode, new_eof); spin_lock(&inode->i_lock); - netfs_resize_file(&cifsi->netfs, i_size_read(inode), true); + netfs_resize_file(&cifsi->netfs, new_eof, true); spin_unlock(&inode->i_lock); - fscache_resize_cookie(cifs_inode_cookie(inode), i_size_read(inode)); + fscache_resize_cookie(cifs_inode_cookie(inode), new_eof, false); /* * Move [off, old_eof) right by len. The helper copies backwards if the diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 4c91a019972b..2b83ffa4510e 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -67,7 +67,7 @@ struct fscache_cache_ops { /* Change the size of a data object */ void (*resize_cookie)(struct netfs_cache_resources *cres, - loff_t new_size); + uoff_t new_size, bool shrank); /* Invalidate an object */ bool (*invalidate_cookie)(struct fscache_cookie *cookie); diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 58fdb9605425..ecd14d645690 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -112,7 +112,7 @@ struct fscache_cookie { struct list_head proc_link; /* Link in proc list */ struct list_head commit_link; /* Link in commit queue */ struct work_struct work; /* Commit/relinq/withdraw work */ - loff_t object_size; /* Size of the netfs object */ + uoff_t object_size; /* Size of the netfs object */ unsigned long unused_at; /* Time at which unused (jiffies) */ unsigned long flags; #define FSCACHE_COOKIE_RELINQUISHED 0 /* T if cookie has been relinquished */ @@ -147,6 +147,23 @@ struct fscache_cookie { }; }; +enum fscache_extent_type { + FSCACHE_EXTENT_DATA, + FSCACHE_EXTENT_ZERO, +} __mode(byte); + +/* + * Cache occupancy information. + */ +struct fscache_occupancy { + unsigned long long query_from; /* Point to query from */ + unsigned long long query_to; /* Point to query to */ + unsigned long long cached_from[2]; /* Point at which cache extents start */ + unsigned long long cached_to[2]; /* Point at which cache extents end */ + unsigned int granularity; /* Granularity desired */ + enum fscache_extent_type cached_type[2]; /* Type of cache extent */ +}; + /* * slow-path functions for when there is actually caching available, and the * netfs does actually have a valid token @@ -163,22 +180,22 @@ extern struct fscache_cookie *__fscache_acquire_cookie( u8, const void *, size_t, const void *, size_t, - loff_t); + uoff_t); extern void __fscache_use_cookie(struct fscache_cookie *, bool); -extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const loff_t *); +extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const uoff_t *); extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); -extern void __fscache_resize_cookie(struct fscache_cookie *, loff_t); -extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int); +void __fscache_resize_cookie(struct fscache_cookie *cookie, uoff_t new_size, bool shrank); +extern void __fscache_invalidate(struct fscache_cookie *, const void *, uoff_t, unsigned int); extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *); extern int __fscache_begin_write_operation(struct netfs_cache_resources *, struct fscache_cookie *); void __fscache_write_to_cache(struct fscache_cookie *cookie, struct address_space *mapping, - loff_t start, size_t len, loff_t i_size, + uoff_t start, size_t len, uoff_t i_size, netfs_io_terminated_t term_func, void *term_func_priv, bool using_pgpriv2, bool cond); -extern void __fscache_clear_page_bits(struct address_space *, loff_t, size_t); +extern void __fscache_clear_page_bits(struct address_space *, uoff_t, size_t); /** * fscache_acquire_volume - Register a volume as desiring caching services @@ -249,7 +266,7 @@ struct fscache_cookie *fscache_acquire_cookie(struct fscache_volume *volume, size_t index_key_len, const void *aux_data, size_t aux_data_len, - loff_t object_size) + uoff_t object_size) { if (!fscache_volume_valid(volume)) return NULL; @@ -286,7 +303,7 @@ static inline void fscache_use_cookie(struct fscache_cookie *cookie, */ static inline void fscache_unuse_cookie(struct fscache_cookie *cookie, const void *aux_data, - const loff_t *object_size) + const uoff_t *object_size) { if (fscache_cookie_valid(cookie)) __fscache_unuse_cookie(cookie, aux_data, object_size); @@ -327,7 +344,7 @@ static inline void *fscache_get_aux(struct fscache_cookie *cookie) */ static inline void fscache_update_aux(struct fscache_cookie *cookie, - const void *aux_data, const loff_t *object_size) + const void *aux_data, const uoff_t *object_size) { void *p = fscache_get_aux(cookie); @@ -343,7 +360,7 @@ extern atomic_t fscache_n_updates; static inline void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, - const loff_t *object_size) + const uoff_t *object_size) { #ifdef CONFIG_FSCACHE_STATS atomic_inc(&fscache_n_updates); @@ -369,7 +386,7 @@ void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data */ static inline void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, - const loff_t *object_size) + const uoff_t *object_size) { if (fscache_cookie_enabled(cookie)) __fscache_update_cookie(cookie, aux_data, object_size); @@ -377,8 +394,9 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, /** * fscache_resize_cookie - Request that a cache object be resized - * @cookie: The cookie representing the cache object - * @new_size: The new size of the object (may be NULL) + * @cookie: The cookie representing the cache object (may be NULL) + * @new_size: The new size of the object + * @shrank: True if the file shrank; false if it expanded * * Request that the size of an object be changed. * @@ -386,10 +404,10 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, * description. */ static inline -void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +void fscache_resize_cookie(struct fscache_cookie *cookie, uoff_t new_size, bool shrank) { if (fscache_cookie_enabled(cookie)) - __fscache_resize_cookie(cookie, new_size); + __fscache_resize_cookie(cookie, new_size, shrank); } /** @@ -413,7 +431,7 @@ void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) */ static inline void fscache_invalidate(struct fscache_cookie *cookie, - const void *aux_data, loff_t size, unsigned int flags) + const void *aux_data, uoff_t size, unsigned int flags) { if (fscache_cookie_enabled(cookie)) __fscache_invalidate(cookie, aux_data, size, flags); @@ -502,7 +520,7 @@ static inline void fscache_end_operation(struct netfs_cache_resources *cres) */ static inline int fscache_read(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, @@ -561,7 +579,7 @@ int fscache_begin_write_operation(struct netfs_cache_resources *cres, */ static inline int fscache_write(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv) @@ -581,7 +599,7 @@ int fscache_write(struct netfs_cache_resources *cres, * waiting. */ static inline void fscache_clear_page_bits(struct address_space *mapping, - loff_t start, size_t len, + uoff_t start, size_t len, bool caching) { if (caching) @@ -615,7 +633,7 @@ static inline void fscache_clear_page_bits(struct address_space *mapping, */ static inline void fscache_write_to_cache(struct fscache_cookie *cookie, struct address_space *mapping, - loff_t start, size_t len, loff_t i_size, + uoff_t start, size_t len, uoff_t i_size, netfs_io_terminated_t term_func, void *term_func_priv, bool using_pgpriv2, bool caching) diff --git a/include/linux/netfs.h b/include/linux/netfs.h index fe6275e55fc2..e5961450f3f1 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -22,6 +22,7 @@ enum netfs_sreq_ref_trace; typedef struct mempool mempool_t; +struct fscache_occupancy; struct folio_queue; /** @@ -62,8 +63,8 @@ struct netfs_inode { struct fscache_cookie *cache; #endif struct list_head wb_queue; /* Queue of processes wanting to do writeback */ - loff_t _remote_i_size; /* Size of the remote file */ - loff_t _zero_point; /* Size after which we assume there's no data + uoff_t _remote_i_size; /* Size of the remote file */ + uoff_t _zero_point; /* Size after which we assume there's no data * on the server */ spinlock_t lock; /* Lock covering wb_queue */ atomic_t io_count; /* Number of outstanding reqs */ @@ -125,6 +126,12 @@ static inline struct netfs_group *netfs_folio_group(struct folio *folio) return priv; } +enum netfs_cache_collect { + NETFS_CACHE_COLLECT_WRITE_GAP, /* Gap in collection, no state either way */ + NETFS_CACHE_COLLECT_WRITE_DATA, /* Currently collecting good writes */ + NETFS_CACHE_COLLECT_WRITE_CANCEL, /* Currently collecting cancelled writes */ +}; + /* * Stream of I/O subrequests going to a particular destination, such as the * server or the local cache. This is mainly intended for writing where we may @@ -142,7 +149,7 @@ struct netfs_io_stream { void (*issue_write)(struct netfs_io_subrequest *subreq); /* Collection tracking */ struct list_head subrequests; /* Contributory I/O operations */ - unsigned long long collected_to; /* Position we've collected results to */ + uoff_t collected_to; /* Position we've collected results to */ size_t transferred; /* The amount transferred from this stream */ unsigned short error; /* Aggregate error for the stream */ enum netfs_io_source source; /* Where to read from/write to */ @@ -152,6 +159,7 @@ struct netfs_io_stream { bool need_retry; /* T if this stream needs retrying */ bool failed; /* T if this stream failed */ bool transferred_valid; /* T is ->transferred is valid */ + enum netfs_cache_collect cache_collect; /* Current writeback cache collect state */ }; /* @@ -161,8 +169,11 @@ struct netfs_cache_resources { const struct netfs_cache_ops *ops; void *cache_priv; void *cache_priv2; - unsigned int debug_id; /* Cookie debug ID */ + uoff_t cache_i_size; /* Initial size of cache file */ + unsigned int cookie_id; /* Cache cookie debug ID */ + unsigned int object_id; /* Cache object debug ID */ unsigned int inval_counter; /* object->inval_counter at begin_op */ + unsigned int dio_size; /* DIO block size */ }; /* @@ -177,7 +188,7 @@ struct netfs_io_subrequest { struct work_struct work; struct list_head rreq_link; /* Link in rreq->subrequests */ struct iov_iter io_iter; /* Iterator for this subrequest */ - unsigned long long start; /* Where to start the I/O */ + uoff_t start; /* Where to start the I/O */ size_t len; /* Size of the I/O */ size_t transferred; /* Amount of data transferred */ refcount_t ref; @@ -196,6 +207,7 @@ struct netfs_io_subrequest { #define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */ #define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */ #define NETFS_SREQ_FAILED 10 /* Set if the subreq failed unretryably */ +#define NETFS_SREQ_CANCELLED 11 /* Set if the subreq was cancelled by netfslib */ }; enum netfs_io_origin { @@ -208,7 +220,6 @@ enum netfs_io_origin { NETFS_DIO_READ, /* This is a direct I/O read */ NETFS_WRITEBACK, /* This write was triggered by writepages */ NETFS_WRITEBACK_SINGLE, /* This monolithic write was triggered by writepages */ - NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */ NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */ NETFS_DIO_WRITE, /* This is a direct I/O write */ NETFS_PGPRIV2_COPY_TO_CACHE, /* [DEPRECATED] This is writing read data to the cache */ @@ -243,17 +254,18 @@ struct netfs_io_request { void *netfs_priv; /* Private data for the netfs */ void *netfs_priv2; /* Private data for the netfs */ struct bio_vec *direct_bv; /* DIO buffer list (when handling iovec-iter) */ - unsigned long long submitted; /* Amount submitted for I/O so far */ - unsigned long long len; /* Length of the request */ + uoff_t submitted; /* Amount submitted for I/O so far */ + uoff_t len; /* Length of the request */ size_t transferred; /* Amount to be indicated as transferred */ size_t progress_at; /* Report read progress when hit this much read */ long error; /* 0 or error that occurred */ - unsigned long long i_size; /* Size of the file */ - unsigned long long start; /* Start position */ + uoff_t i_size; /* Size of the file */ + uoff_t start; /* Start position */ atomic64_t issued_to; /* Write issuer folio cursor */ - unsigned long long collected_to; /* Point we've collected to */ - unsigned long long cleaned_to; /* Position we've cleaned folios to */ - unsigned long long abandon_to; /* Position to abandon folios to */ + uoff_t collected_to; /* Point we've collected to */ + uoff_t cache_coll_to; /* Point the cache has collected to */ + uoff_t cleaned_to; /* Position we've cleaned folios to */ + uoff_t abandon_to; /* Position to abandon folios to */ const struct folio *no_unlock_folio; /* Don't unlock this folio after read */ gfp_t gfp; /* GFP flags to use */ unsigned int direct_bv_count; /* Number of elements in direct_bv[] */ @@ -273,14 +285,18 @@ struct netfs_io_request { #define NETFS_RREQ_FAILED 3 /* The request failed */ #define NETFS_RREQ_RETRYING 4 /* Set if we're in the retry path */ #define NETFS_RREQ_SHORT_TRANSFER 5 /* Set if we have a short transfer */ -#define NETFS_RREQ_OFFLOAD_COLLECTION 8 /* Offload collection to workqueue */ -#define NETFS_RREQ_NO_UNLOCK_FOLIO 9 /* Don't unlock no_unlock_folio on completion */ +#define NETFS_RREQ_CACHE_STOP 8 /* Set to stop caching (ENOBUFS or error) */ +#define NETFS_RREQ_CACHE_ERROR 9 /* Set if we got an error from the cache */ #define NETFS_RREQ_CANCEL_CACHING 10 /* Set to cancel caching */ -#define NETFS_RREQ_UPLOAD_TO_SERVER 11 /* Need to write to the server */ -#define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */ +#define NETFS_RREQ_OFFLOAD_COLLECTION 12 /* Offload collection to workqueue */ +#define NETFS_RREQ_NO_UNLOCK_FOLIO 13 /* Don't unlock no_unlock_folio on completion */ +#define NETFS_RREQ_UPLOAD_TO_SERVER 14 /* Need to write to the server */ +#define NETFS_RREQ_USE_IO_ITER 15 /* Use ->io_iter rather than ->i_pages */ #define NETFS_RREQ_NEED_PUT_RA_REFS 17 /* Need to put the folio refs RA gave us */ +#ifdef CONFIG_NETFS_PGPRIV2 #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark * write to cache on read */ +#endif const struct netfs_request_ops *netfs_ops; }; @@ -299,12 +315,12 @@ struct netfs_request_ops { int (*prepare_read)(struct netfs_io_subrequest *subreq); void (*issue_read)(struct netfs_io_subrequest *subreq); bool (*is_still_valid)(struct netfs_io_request *rreq); - int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, + int (*check_write_begin)(struct file *file, uoff_t pos, unsigned len, struct folio **foliop, void **_fsdata); void (*done)(struct netfs_io_request *rreq); /* Modification handling */ - void (*update_i_size)(struct inode *inode, loff_t i_size); + void (*update_i_size)(struct inode *inode, uoff_t i_size); void (*post_modify)(struct inode *inode); /* Write request handling */ @@ -332,7 +348,7 @@ struct netfs_cache_ops { /* Read data from the cache */ int (*read)(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, @@ -340,7 +356,7 @@ struct netfs_cache_ops { /* Write data to the cache */ int (*write)(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv); @@ -350,15 +366,14 @@ struct netfs_cache_ops { /* Expand readahead request */ void (*expand_readahead)(struct netfs_cache_resources *cres, - unsigned long long *_start, - unsigned long long *_len, - unsigned long long i_size); + uoff_t *_start, + uoff_t *_len, + uoff_t i_size); /* Prepare a read operation, shortening it to a cached/uncached * boundary as appropriate. */ - enum netfs_io_source (*prepare_read)(struct netfs_io_subrequest *subreq, - unsigned long long i_size); + int (*prepare_read)(struct netfs_io_subrequest *subreq); /* Prepare a write subrequest, working out if we're allowed to do it * and finding out the maximum amount of data to gather before @@ -371,15 +386,24 @@ struct netfs_cache_ops { * actually do. */ int (*prepare_write)(struct netfs_cache_resources *cres, - loff_t *_start, size_t *_len, size_t upper_len, - loff_t i_size, bool no_space_allocated_yet); + uoff_t *_start, size_t *_len, size_t upper_len, + uoff_t i_size, bool no_space_allocated_yet); /* Query the occupancy of the cache in a region, returning where the * next chunk of data starts and how long it is. */ int (*query_occupancy)(struct netfs_cache_resources *cres, - loff_t start, size_t len, size_t granularity, - loff_t *_data_start, size_t *_data_len); + struct fscache_occupancy *occ); + + /* Collect the result of buffered writeback to the cache. This + * includes copying a read to the cache. block_type is one of: + * - NETFS_CACHE_COLLECT_WRITE_DATA for a block of data + * - NETFS_CACHE_COLLECT_WRITE_GAP if a discontiguity was skipped + * - NETFS_CACHE_COLLECT_WRITE_CANCEL for a cancellation gap + */ + void (*collect_write)(struct netfs_io_request *wreq, + uoff_t start, size_t len, + enum netfs_cache_collect block_type); }; /* High-level read API. */ @@ -412,7 +436,7 @@ struct readahead_control; void netfs_readahead(struct readahead_control *); int netfs_read_folio(struct file *, struct folio *); int netfs_write_begin(struct netfs_inode *, struct file *, - struct address_space *, loff_t pos, unsigned int len, + struct address_space *, uoff_t pos, unsigned int len, struct folio **, void **fsdata); int netfs_writepages(struct address_space *mapping, struct writeback_control *wbc); @@ -490,10 +514,10 @@ static inline struct netfs_inode *netfs_inode(struct inode *inode) * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit * archs it makes no difference if preempt is enabled or not. */ -static inline unsigned long long netfs_read_remote_i_size(const struct inode *inode) +static inline uoff_t netfs_read_remote_i_size(const struct inode *inode) { const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); - unsigned long long remote_i_size; + uoff_t remote_i_size; #if BITS_PER_LONG==32 && defined(CONFIG_SMP) unsigned int seq; @@ -528,7 +552,7 @@ static inline unsigned long long netfs_read_remote_i_size(const struct inode *in * spinning forever. */ static inline void netfs_write_remote_i_size(struct inode *inode, - unsigned long long remote_i_size) + uoff_t remote_i_size) { struct netfs_inode *ictx = netfs_inode(inode); @@ -565,10 +589,10 @@ static inline void netfs_write_remote_i_size(struct inode *inode, * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit * archs it makes no difference if preempt is enabled or not. */ -static inline unsigned long long netfs_read_zero_point(const struct inode *inode) +static inline uoff_t netfs_read_zero_point(const struct inode *inode) { struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); - unsigned long long zero_point; + uoff_t zero_point; #if BITS_PER_LONG==32 && defined(CONFIG_SMP) unsigned int seq; @@ -603,7 +627,7 @@ static inline unsigned long long netfs_read_zero_point(const struct inode *inode * forever. */ static inline void netfs_write_zero_point(struct inode *inode, - unsigned long long zero_point) + uoff_t zero_point) { struct netfs_inode *ictx = netfs_inode(inode); @@ -644,9 +668,9 @@ static inline void netfs_write_zero_point(struct inode *inode, * archs it makes no difference if preempt is enabled or not. */ static inline void netfs_read_sizes(const struct inode *inode, - unsigned long long *i_size, - unsigned long long *remote_i_size, - unsigned long long *zero_point) + uoff_t *i_size, + uoff_t *remote_i_size, + uoff_t *zero_point) { const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); #if BITS_PER_LONG==32 && defined(CONFIG_SMP) @@ -692,9 +716,9 @@ static inline void netfs_read_sizes(const struct inode *inode, * forever. */ static inline void netfs_write_sizes(struct inode *inode, - unsigned long long i_size, - unsigned long long remote_i_size, - unsigned long long zero_point) + uoff_t i_size, + uoff_t remote_i_size, + uoff_t zero_point) { struct netfs_inode *ictx = netfs_inode(inode); @@ -762,7 +786,7 @@ static inline void netfs_inode_init(struct netfs_inode *ctx, * Inform the netfs lib that a file got resized so that it can adjust its state. */ static inline void netfs_resize_file(struct netfs_inode *ictx, - unsigned long long new_i_size, + uoff_t new_i_size, bool changed_on_server) { #if BITS_PER_LONG==32 && defined(CONFIG_SMP) diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h index e3101410e8b2..a19233e8ae78 100644 --- a/include/trace/events/cachefiles.h +++ b/include/trace/events/cachefiles.h @@ -52,6 +52,8 @@ enum cachefiles_coherency_trace { cachefiles_coherency_check_ok, cachefiles_coherency_check_type, cachefiles_coherency_check_xattr, + cachefiles_coherency_discontiguous, + cachefiles_coherency_remove, cachefiles_coherency_set_fail, cachefiles_coherency_set_ok, cachefiles_coherency_vol_check_cmp, @@ -63,9 +65,11 @@ enum cachefiles_coherency_trace { }; enum cachefiles_trunc_trace { + cachefiles_trunc_clear_padding, cachefiles_trunc_dio_adjust, cachefiles_trunc_expand_tmpfile, cachefiles_trunc_shrink, + cachefiles_trunc_zap, }; enum cachefiles_prepare_read_trace { @@ -80,11 +84,14 @@ enum cachefiles_prepare_read_trace { }; enum cachefiles_error_trace { + cachefiles_trace_alignment_error, + cachefiles_trace_create_nospace, cachefiles_trace_fallocate_error, cachefiles_trace_getxattr_error, cachefiles_trace_link_error, cachefiles_trace_lookup_error, cachefiles_trace_mkdir_error, + cachefiles_trace_mkdir_nospace, cachefiles_trace_notify_change_error, cachefiles_trace_open_error, cachefiles_trace_read_error, @@ -97,6 +104,8 @@ enum cachefiles_error_trace { cachefiles_trace_trunc_error, cachefiles_trace_unlink_error, cachefiles_trace_write_error, + cachefiles_trace_write_nospace, + cachefiles_trace_write_nospace_2, }; #endif @@ -136,6 +145,8 @@ enum cachefiles_error_trace { EM(cachefiles_coherency_check_ok, "OK ") \ EM(cachefiles_coherency_check_type, "BAD type") \ EM(cachefiles_coherency_check_xattr, "BAD xatt") \ + EM(cachefiles_coherency_discontiguous, "--- gap ") \ + EM(cachefiles_coherency_remove, "REMOVE ") \ EM(cachefiles_coherency_set_fail, "SET fail") \ EM(cachefiles_coherency_set_ok, "SET ok ") \ EM(cachefiles_coherency_vol_check_cmp, "VOL BAD cmp ") \ @@ -146,9 +157,11 @@ enum cachefiles_error_trace { E_(cachefiles_coherency_vol_set_ok, "VOL SET ok ") #define cachefiles_trunc_traces \ + EM(cachefiles_trunc_clear_padding, "CLRPAD") \ EM(cachefiles_trunc_dio_adjust, "DIOADJ") \ EM(cachefiles_trunc_expand_tmpfile, "EXPTMP") \ - E_(cachefiles_trunc_shrink, "SHRINK") + EM(cachefiles_trunc_shrink, "SHRINK") \ + E_(cachefiles_trunc_zap, "ZAP ") #define cachefiles_prepare_read_traces \ EM(cachefiles_trace_read_after_eof, "after-eof ") \ @@ -161,11 +174,14 @@ enum cachefiles_error_trace { E_(cachefiles_trace_read_seek_nxio, "seek-enxio") #define cachefiles_error_traces \ + EM(cachefiles_trace_alignment_error, "align") \ + EM(cachefiles_trace_create_nospace, "create-nospace") \ EM(cachefiles_trace_fallocate_error, "fallocate") \ EM(cachefiles_trace_getxattr_error, "getxattr") \ EM(cachefiles_trace_link_error, "link") \ EM(cachefiles_trace_lookup_error, "lookup") \ EM(cachefiles_trace_mkdir_error, "mkdir") \ + EM(cachefiles_trace_mkdir_nospace, "mkdir-nospace") \ EM(cachefiles_trace_notify_change_error, "notify_change") \ EM(cachefiles_trace_open_error, "open") \ EM(cachefiles_trace_read_error, "read") \ @@ -177,7 +193,9 @@ enum cachefiles_error_trace { EM(cachefiles_trace_tmpfile_error, "tmpfile") \ EM(cachefiles_trace_trunc_error, "trunc") \ EM(cachefiles_trace_unlink_error, "unlink") \ - E_(cachefiles_trace_write_error, "write") + EM(cachefiles_trace_write_error, "write") \ + EM(cachefiles_trace_write_nospace, "write-nospace") \ + E_(cachefiles_trace_write_nospace_2, "write-nospace-2") /* @@ -371,12 +389,12 @@ TRACE_EVENT(cachefiles_rename, TRACE_EVENT(cachefiles_coherency, TP_PROTO(struct cachefiles_object *obj, - ino_t ino, + ino_t ino, uoff_t obj_size, const void *disk_aux, enum cachefiles_content content, enum cachefiles_coherency_trace why), - TP_ARGS(obj, ino, disk_aux, content, why), + TP_ARGS(obj, ino, obj_size, disk_aux, content, why), /* Note that obj may be NULL */ TP_STRUCT__entry( @@ -384,6 +402,7 @@ TRACE_EVENT(cachefiles_coherency, __field(enum cachefiles_coherency_trace, why) __field(enum cachefiles_content, content) __field(u64, ino) + __field(u64, obj_size) __field(u64, aux) __field(u64, disk_aux) ), @@ -398,6 +417,7 @@ TRACE_EVENT(cachefiles_coherency, __entry->why = why; __entry->content = content; __entry->ino = ino; + __entry->obj_size = obj_size; __entry->aux = be64_to_cpup((__be64 *)obj->cookie->inline_aux); /* cachefiles_xattr::data is 2-byte aligned but not 8-byte aligned. */ @@ -412,10 +432,11 @@ TRACE_EVENT(cachefiles_coherency, } ), - TP_printk("o=%08x %s B=%llx c=%u aux=%llx dsk=%llx", + TP_printk("o=%08x %s B=%llx oz=%llx c=%u aux=%llx dsk=%llx", __entry->obj, __print_symbolic(__entry->why, cachefiles_coherency_traces), __entry->ino, + __entry->obj_size, __entry->content, __entry->aux, __entry->disk_aux) @@ -449,7 +470,7 @@ TRACE_EVENT(cachefiles_vol_coherency, TRACE_EVENT(cachefiles_prep_read, TP_PROTO(struct cachefiles_object *obj, - loff_t start, + uoff_t start, size_t len, unsigned short flags, enum netfs_io_source source, @@ -464,7 +485,7 @@ TRACE_EVENT(cachefiles_prep_read, __field(enum netfs_io_source, source) __field(enum cachefiles_prepare_read_trace, why) __field(size_t, len) - __field(loff_t, start) + __field(uoff_t, start) __field(unsigned int, netfs_inode) __field(unsigned int, cache_inode) ), @@ -492,16 +513,16 @@ TRACE_EVENT(cachefiles_prep_read, TRACE_EVENT(cachefiles_read, TP_PROTO(struct cachefiles_object *obj, struct inode *backer, - loff_t start, + uoff_t start, size_t len), TP_ARGS(obj, backer, start, len), TP_STRUCT__entry( - __field(unsigned int, obj) - __field(unsigned int, backer) - __field(size_t, len) - __field(loff_t, start) + __field(unsigned int, obj) + __field(unsigned int, backer) + __field(size_t, len) + __field(uoff_t, start) ), TP_fast_assign( @@ -521,16 +542,16 @@ TRACE_EVENT(cachefiles_read, TRACE_EVENT(cachefiles_write, TP_PROTO(struct cachefiles_object *obj, struct inode *backer, - loff_t start, + uoff_t start, size_t len), TP_ARGS(obj, backer, start, len), TP_STRUCT__entry( - __field(unsigned int, obj) - __field(unsigned int, backer) - __field(size_t, len) - __field(loff_t, start) + __field(unsigned int, obj) + __field(unsigned int, backer) + __field(size_t, len) + __field(uoff_t, start) ), TP_fast_assign( @@ -549,7 +570,7 @@ TRACE_EVENT(cachefiles_write, TRACE_EVENT(cachefiles_trunc, TP_PROTO(struct cachefiles_object *obj, struct inode *backer, - loff_t from, loff_t to, enum cachefiles_trunc_trace why), + uoff_t from, uoff_t to, enum cachefiles_trunc_trace why), TP_ARGS(obj, backer, from, to, why), @@ -557,8 +578,8 @@ TRACE_EVENT(cachefiles_trunc, __field(unsigned int, obj) __field(unsigned int, backer) __field(enum cachefiles_trunc_trace, why) - __field(loff_t, from) - __field(loff_t, to) + __field(uoff_t, from) + __field(uoff_t, to) ), TP_fast_assign( @@ -694,6 +715,26 @@ TRACE_EVENT(cachefiles_io_error, __entry->error) ); +TRACE_EVENT(cachefiles_no_space, + TP_PROTO(struct cachefiles_object *obj, enum cachefiles_error_trace trace), + + TP_ARGS(obj, trace), + + TP_STRUCT__entry( + __field(unsigned int, obj) + __field(enum cachefiles_error_trace, trace) + ), + + TP_fast_assign( + __entry->obj = obj ? obj->debug_id : 0; + __entry->trace = trace; + ), + + TP_printk("o=%08x %s", + __entry->obj, + __print_symbolic(__entry->trace, cachefiles_error_traces)) + ); + #endif /* _TRACE_CACHEFILES_H */ /* This part must be outside protection */ diff --git a/include/trace/events/fscache.h b/include/trace/events/fscache.h index f1a73aa83fbb..8735d428ebd9 100644 --- a/include/trace/events/fscache.h +++ b/include/trace/events/fscache.h @@ -460,13 +460,13 @@ TRACE_EVENT(fscache_relinquish, ); TRACE_EVENT(fscache_invalidate, - TP_PROTO(struct fscache_cookie *cookie, loff_t new_size), + TP_PROTO(struct fscache_cookie *cookie, uoff_t new_size), TP_ARGS(cookie, new_size), TP_STRUCT__entry( __field(unsigned int, cookie ) - __field(loff_t, new_size ) + __field(uoff_t, new_size ) ), TP_fast_assign( @@ -479,14 +479,14 @@ TRACE_EVENT(fscache_invalidate, ); TRACE_EVENT(fscache_resize, - TP_PROTO(struct fscache_cookie *cookie, loff_t new_size), + TP_PROTO(struct fscache_cookie *cookie, uoff_t new_size), TP_ARGS(cookie, new_size), TP_STRUCT__entry( __field(unsigned int, cookie ) - __field(loff_t, old_size ) - __field(loff_t, new_size ) + __field(uoff_t, old_size ) + __field(uoff_t, new_size ) ), TP_fast_assign( diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index 3fec3e8f91c8..bf1e1f185b05 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -30,8 +30,7 @@ EM(netfs_write_trace_dio_write, "DIO-WRITE") \ EM(netfs_write_trace_unbuffered_write, "UNB-WRITE") \ EM(netfs_write_trace_writeback, "WRITEBACK") \ - EM(netfs_write_trace_writeback_single, "WB-SINGLE") \ - E_(netfs_write_trace_writethrough, "WRITETHRU") + E_(netfs_write_trace_writeback_single, "WB-SINGLE") #define netfs_rreq_origins \ EM(NETFS_READAHEAD, "RA") \ @@ -43,13 +42,17 @@ EM(NETFS_DIO_READ, "DR") \ EM(NETFS_WRITEBACK, "WB") \ EM(NETFS_WRITEBACK_SINGLE, "W1") \ - EM(NETFS_WRITETHROUGH, "WT") \ EM(NETFS_UNBUFFERED_WRITE, "UW") \ EM(NETFS_DIO_WRITE, "DW") \ E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C") #define netfs_rreq_traces \ + EM(netfs_rreq_trace_all_queued, "ALL-Q ") \ EM(netfs_rreq_trace_assess, "ASSESS ") \ + EM(netfs_rreq_trace_cache_cancelled, "CA-CNCL") \ + EM(netfs_rreq_trace_cache_failed, "CA-FAIL") \ + EM(netfs_rreq_trace_cache_fail_collect, "CA-F-CO") \ + EM(netfs_rreq_trace_cache_no_space, "CA-NOSP") \ EM(netfs_rreq_trace_collect, "COLLECT") \ EM(netfs_rreq_trace_complete, "COMPLET") \ EM(netfs_rreq_trace_copy, "COPY ") \ @@ -58,11 +61,14 @@ EM(netfs_rreq_trace_end_copy_to_cache, "END-C2C") \ EM(netfs_rreq_trace_free, "FREE ") \ EM(netfs_rreq_trace_intr, "INTR ") \ + EM(netfs_rreq_trace_inval_cache, "INVL-CA") \ EM(netfs_rreq_trace_ki_complete, "KI-CMPL") \ EM(netfs_rreq_trace_ra_put_ref, "RA-PUT ") \ EM(netfs_rreq_trace_recollect, "RECLLCT") \ EM(netfs_rreq_trace_redirty, "REDIRTY") \ EM(netfs_rreq_trace_resubmit, "RESUBMT") \ + EM(netfs_rreq_trace_retry_begin, "RETRY-BEGIN") \ + EM(netfs_rreq_trace_retry_end, "RETRY-END") \ EM(netfs_rreq_trace_set_abandon, "S-ABNDN") \ EM(netfs_rreq_trace_set_pause, "PAUSE ") \ EM(netfs_rreq_trace_unlock, "UNLOCK ") \ @@ -94,8 +100,10 @@ EM(netfs_sreq_trace_abandoned, "ABNDN") \ EM(netfs_sreq_trace_add_donations, "+DON ") \ EM(netfs_sreq_trace_added, "ADD ") \ + EM(netfs_sreq_trace_cache_nofile, "CA-!F") \ EM(netfs_sreq_trace_cache_nowrite, "CA-NW") \ EM(netfs_sreq_trace_cache_prepare, "CA-PR") \ + EM(netfs_sreq_trace_cache_waitfail, "CA-!W") \ EM(netfs_sreq_trace_cache_write, "CA-WR") \ EM(netfs_sreq_trace_cancel, "CANCL") \ EM(netfs_sreq_trace_clear, "CLEAR") \ @@ -134,12 +142,12 @@ #define netfs_failures \ EM(netfs_fail_check_write_begin, "check-write-begin") \ - EM(netfs_fail_copy_to_cache, "copy-to-cache") \ EM(netfs_fail_dio_read_short, "dio-read-short") \ EM(netfs_fail_dio_read_zero, "dio-read-zero") \ EM(netfs_fail_read, "read") \ EM(netfs_fail_short_read, "short-read") \ EM(netfs_fail_prepare_write, "prep-write") \ + EM(netfs_fail_upload, "upload") \ E_(netfs_fail_write, "write") #define netfs_rreq_ref_traces \ @@ -194,11 +202,11 @@ EM(netfs_folio_trace_alloc_buffer, "alloc-buf") \ EM(netfs_folio_trace_cancel_copy, "cancel-copy") \ EM(netfs_folio_trace_cancel_store, "cancel-store") \ - EM(netfs_folio_trace_clear, "clear") \ - EM(netfs_folio_trace_clear_cc, "clear-cc") \ - EM(netfs_folio_trace_clear_g, "clear-g") \ - EM(netfs_folio_trace_clear_s, "clear-s") \ EM(netfs_folio_trace_end_copy, "end-copy") \ + EM(netfs_folio_trace_endwb, "endwb") \ + EM(netfs_folio_trace_endwb_cc, "endwb-cc") \ + EM(netfs_folio_trace_endwb_g, "endwb-g") \ + EM(netfs_folio_trace_endwb_s, "endwb-s") \ EM(netfs_folio_trace_filled_gaps, "filled-gaps") \ EM(netfs_folio_trace_invalidate_all, "inval-all") \ EM(netfs_folio_trace_invalidate_front, "inval-front") \ @@ -223,9 +231,7 @@ EM(netfs_folio_trace_sched_copy, "sched-copy") \ EM(netfs_folio_trace_store, "store") \ EM(netfs_folio_trace_store_copy, "store-copy") \ - EM(netfs_folio_trace_store_plus, "store+") \ - EM(netfs_folio_trace_wthru, "wthru") \ - E_(netfs_folio_trace_wthru_plus, "wthru+") + E_(netfs_folio_trace_store_plus, "store+") #define netfs_collect_contig_traces \ EM(netfs_contig_trace_collect, "Collect") \ @@ -301,7 +307,7 @@ netfs_folioq_traces; TRACE_EVENT(netfs_read, TP_PROTO(struct netfs_io_request *rreq, - loff_t start, size_t len, + uoff_t start, size_t len, enum netfs_read_trace what), TP_ARGS(rreq, start, len, what), @@ -309,8 +315,9 @@ TRACE_EVENT(netfs_read, TP_STRUCT__entry( __field(unsigned int, rreq) __field(unsigned int, cookie) - __field(loff_t, i_size) - __field(loff_t, start) + __field(unsigned int, object) + __field(uoff_t, i_size) + __field(uoff_t, start) __field(size_t, len) __field(enum netfs_read_trace, what) __field(u64, netfs_inode) @@ -318,7 +325,8 @@ TRACE_EVENT(netfs_read, TP_fast_assign( __entry->rreq = rreq->debug_id; - __entry->cookie = rreq->cache_resources.debug_id; + __entry->cookie = rreq->cache_resources.cookie_id; + __entry->object = rreq->cache_resources.object_id; __entry->i_size = rreq->i_size; __entry->start = start; __entry->len = len; @@ -326,10 +334,10 @@ TRACE_EVENT(netfs_read, __entry->netfs_inode = rreq->inode->i_ino; ), - TP_printk("R=%08x %s c=%08x ni=%llx s=%llx l=%zx sz=%llx", + TP_printk("R=%08x %s c=%08x o=%08x ni=%llx s=%llx l=%zx sz=%llx", __entry->rreq, __print_symbolic(__entry->what, netfs_read_traces), - __entry->cookie, + __entry->cookie, __entry->object, __entry->netfs_inode, __entry->start, __entry->len, __entry->i_size) ); @@ -377,7 +385,7 @@ TRACE_EVENT(netfs_sreq, __field(u8, slot) __field(size_t, len) __field(size_t, transferred) - __field(loff_t, start) + __field(uoff_t, start) ), TP_fast_assign( @@ -418,7 +426,7 @@ TRACE_EVENT(netfs_failure, __field(enum netfs_failure, what) __field(size_t, len) __field(size_t, transferred) - __field(loff_t, start) + __field(uoff_t, start) ), TP_fast_assign( @@ -501,6 +509,7 @@ TRACE_EVENT(netfs_folio, TP_STRUCT__entry( __field(u64, ino) __field(pgoff_t, index) + __field(unsigned long, pfn) __field(unsigned int, nr) __field(enum netfs_folio_trace, why) ), @@ -511,9 +520,11 @@ TRACE_EVENT(netfs_folio, __entry->why = why; __entry->index = folio->index; __entry->nr = folio_nr_pages(folio); + __entry->pfn = folio_pfn(folio); ), - TP_printk("i=%05llx ix=%05lx-%05lx %s", + TP_printk("p=%lx i=%05llx ix=%05lx-%05lx %s", + __entry->pfn, __entry->ino, __entry->index, __entry->index + __entry->nr - 1, __print_symbolic(__entry->why, netfs_folio_traces)) ); @@ -524,10 +535,10 @@ TRACE_EVENT(netfs_write_iter, TP_ARGS(iocb, from), TP_STRUCT__entry( - __field(unsigned long long, start) - __field(size_t, len) - __field(unsigned int, flags) - __field(unsigned int, ino) + __field(uoff_t, start) + __field(size_t, len) + __field(unsigned int, flags) + __field(unsigned int, ino) ), TP_fast_assign( @@ -550,27 +561,27 @@ TRACE_EVENT(netfs_write, TP_STRUCT__entry( __field(unsigned int, wreq) __field(unsigned int, cookie) + __field(unsigned int, object) __field(unsigned int, ino) __field(enum netfs_write_trace, what) - __field(unsigned long long, start) - __field(unsigned long long, len) + __field(uoff_t, start) + __field(uoff_t, len) ), TP_fast_assign( - struct netfs_inode *__ctx = netfs_inode(wreq->inode); - struct fscache_cookie *__cookie = netfs_i_cookie(__ctx); __entry->wreq = wreq->debug_id; - __entry->cookie = __cookie ? __cookie->debug_id : 0; + __entry->cookie = wreq->cache_resources.cookie_id; + __entry->object = wreq->cache_resources.object_id; __entry->ino = wreq->inode->i_ino; __entry->what = what; __entry->start = wreq->start; __entry->len = wreq->len; ), - TP_printk("R=%08x %s c=%08x i=%x by=%llx-%llx", + TP_printk("R=%08x %s c=%08x o=%08x i=%x by=%llx-%llx", __entry->wreq, __print_symbolic(__entry->what, netfs_write_traces), - __entry->cookie, + __entry->cookie, __entry->object, __entry->ino, __entry->start, __entry->start + __entry->len - 1) ); @@ -582,25 +593,26 @@ TRACE_EVENT(netfs_copy2cache, TP_ARGS(rreq, creq), TP_STRUCT__entry( - __field(unsigned int, rreq) - __field(unsigned int, creq) - __field(unsigned int, cookie) - __field(unsigned int, ino) + __field(unsigned int, rreq) + __field(unsigned int, creq) + __field(unsigned int, cookie) + __field(unsigned int, object) + __field(unsigned int, ino) ), TP_fast_assign( - struct netfs_inode *__ctx = netfs_inode(rreq->inode); - struct fscache_cookie *__cookie = netfs_i_cookie(__ctx); __entry->rreq = rreq->debug_id; __entry->creq = creq->debug_id; - __entry->cookie = __cookie ? __cookie->debug_id : 0; + __entry->cookie = rreq->cache_resources.cookie_id; + __entry->object = rreq->cache_resources.object_id; __entry->ino = rreq->inode->i_ino; ), - TP_printk("R=%08x CR=%08x c=%08x i=%x ", + TP_printk("R=%08x CR=%08x c=%08x o=%08x i=%x ", __entry->rreq, __entry->creq, __entry->cookie, + __entry->object, __entry->ino) ); @@ -610,10 +622,10 @@ TRACE_EVENT(netfs_collect, TP_ARGS(wreq), TP_STRUCT__entry( - __field(unsigned int, wreq) - __field(unsigned int, len) - __field(unsigned long long, transferred) - __field(unsigned long long, start) + __field(unsigned int, wreq) + __field(unsigned int, len) + __field(uoff_t, transferred) + __field(uoff_t, start) ), TP_fast_assign( @@ -636,12 +648,12 @@ TRACE_EVENT(netfs_collect_sreq, TP_ARGS(wreq, subreq), TP_STRUCT__entry( - __field(unsigned int, wreq) - __field(unsigned int, subreq) - __field(unsigned int, stream) - __field(unsigned int, len) - __field(unsigned int, transferred) - __field(unsigned long long, start) + __field(unsigned int, wreq) + __field(unsigned int, subreq) + __field(unsigned int, stream) + __field(unsigned int, len) + __field(unsigned int, transferred) + __field(uoff_t, start) ), TP_fast_assign( @@ -660,37 +672,30 @@ TRACE_EVENT(netfs_collect_sreq, TRACE_EVENT(netfs_collect_folio, TP_PROTO(const struct netfs_io_request *wreq, - const struct folio *folio, - unsigned long long fend, - unsigned long long collected_to), + const struct folio *folio), - TP_ARGS(wreq, folio, fend, collected_to), + TP_ARGS(wreq, folio), TP_STRUCT__entry( __field(unsigned int, wreq) __field(unsigned long, index) - __field(unsigned long long, fend) - __field(unsigned long long, cleaned_to) - __field(unsigned long long, collected_to) + __field(unsigned int, nr) ), TP_fast_assign( __entry->wreq = wreq->debug_id; __entry->index = folio->index; - __entry->fend = fend; - __entry->cleaned_to = wreq->cleaned_to; - __entry->collected_to = collected_to; + __entry->nr = folio_nr_pages(folio); ), - TP_printk("R=%08x ix=%05lx r=%llx-%llx t=%llx/%llx", + TP_printk("R=%08x ix=%05lx-%05lx", __entry->wreq, __entry->index, - (unsigned long long)__entry->index * PAGE_SIZE, __entry->fend, - __entry->cleaned_to, __entry->collected_to) + __entry->index + __entry->nr - 1) ); TRACE_EVENT(netfs_collect_state, TP_PROTO(const struct netfs_io_request *wreq, - unsigned long long collected_to, + uoff_t collected_to, unsigned int notes), TP_ARGS(wreq, collected_to, notes), @@ -698,8 +703,8 @@ TRACE_EVENT(netfs_collect_state, TP_STRUCT__entry( __field(unsigned int, wreq) __field(unsigned int, notes) - __field(unsigned long long, collected_to) - __field(unsigned long long, cleaned_to) + __field(uoff_t, collected_to) + __field(uoff_t, cleaned_to) ), TP_fast_assign( @@ -718,7 +723,7 @@ TRACE_EVENT(netfs_collect_state, TRACE_EVENT(netfs_collect_gap, TP_PROTO(const struct netfs_io_request *wreq, const struct netfs_io_stream *stream, - unsigned long long jump_to, char type), + uoff_t jump_to, char type), TP_ARGS(wreq, stream, jump_to, type), @@ -726,8 +731,8 @@ TRACE_EVENT(netfs_collect_gap, __field(unsigned int, wreq) __field(unsigned char, stream) __field(unsigned char, type) - __field(unsigned long long, from) - __field(unsigned long long, to) + __field(uoff_t, from) + __field(uoff_t, to) ), TP_fast_assign( @@ -752,8 +757,8 @@ TRACE_EVENT(netfs_collect_stream, TP_STRUCT__entry( __field(unsigned int, wreq) __field(unsigned char, stream) - __field(unsigned long long, collected_to) - __field(unsigned long long, issued_to) + __field(uoff_t, collected_to) + __field(uoff_t, issued_to) ), TP_fast_assign(