github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-07-27 09:36:22 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge patch series "assorted ->i_count changes + extension of lockless handling"

Browse Source 
Mateusz Guzik <mjguzik@gmail.com> says:

The stock kernel support partial lockless in handling in that iput() can
decrement any value > 1. Any ref acquire however requires the spinlock.

With this patchset ref acquires when the value was already at least 1
also become lockless. That is, only transitions 0->1 and 1->0 take the
lock.

I verified when nfs calls into the hash taking the lock is typically
avoided. Similarly, btrfs likes to igrab() and avoids the lock.
However, I have to fully admit I did not perform any benchmarks. While
cleaning stuff up I noticed lockless operation is almost readily
available so I went for it.

Clean-up wise, the icount_read_once() stuff lines up with
inode_state_read_once(). The prefix is different but I opted to not
change it due to igrab(), ihold() et al.

* patches from https://patch.msgid.link/20260421182538.1215894-1-mjguzik@gmail.com:
  fs: allow lockless ->i_count bumps as long as it does not transition 0->1
  fs: relocate and tidy up ihold()
  fs: add icount_read_once() and stop open-coding ->i_count loads

Link: https://patch.msgid.link/20260421182538.1215894-1-mjguzik@gmail.com
Signed-off-by: Christian Brauner <brauner@kernel.org>
This commit is contained in:
Christian Brauner 2026-04-22 15:00:32 +02:00
commit 62c251278f
No known key found for this signature in database
GPG Key ID: 91C61BC06578DCA2
15 changed files with 112 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/powerpc/platforms/cell/spufs/file.c
View File

@ -1430,7 +1430,7 @@ static int spufs_mfc_open(struct inode *inode, struct file *file)
if (ctx->owner != current->mm)
return -EINVAL;
if (icount_read(inode) != 1)
if (icount_read_once(inode) != 1)
return -EBUSY;
mutex_lock(&ctx->mapping_lock);

2
fs/btrfs/inode.c
View File

@ -4745,7 +4745,7 @@ static void btrfs_prune_dentries(struct btrfs_root *root)
inode = btrfs_find_first_inode(root, min_ino);
while (inode) {
if (icount_read(&inode->vfs_inode) > 1)
if (icount_read_once(&inode->vfs_inode) > 1)
d_prune_aliases(&inode->vfs_inode);
min_ino = btrfs_ino(inode) + 1;

2
fs/ceph/mds_client.c
View File

@ -2267,7 +2267,7 @@ static int trim_caps_cb(struct inode *inode, int mds, void *arg)
int count;
dput(dentry);
d_prune_aliases(inode);
count = icount_read(inode);
count = icount_read_once(inode);
if (count == 1)
(*remaining)--;
doutc(cl, "%p %llx.%llx cap %p pruned, count now %d\n",

4
fs/dcache.c
View File

@ -2100,6 +2100,10 @@ void d_instantiate_new(struct dentry *entry, struct inode *inode)
__d_instantiate(entry, inode);
spin_unlock(&entry->d_lock);
WARN_ON(!(inode_state_read(inode) & I_NEW));
/*
* Paired with igrab_from_hash()
*/
smp_wmb();
inode_state_clear(inode, I_NEW | I_CREATING);
inode_wake_up_bit(inode, __I_NEW);
spin_unlock(&inode->i_lock);

4
fs/ext4/ialloc.c
View File

@ -252,10 +252,10 @@ void ext4_free_inode(handle_t *handle, struct inode *inode)
"nonexistent device\n", __func__, __LINE__);
return;
}
if (icount_read(inode) > 1) {
if (icount_read_once(inode) > 1) {
ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: count=%d",
__func__, __LINE__, inode->i_ino,
icount_read(inode));
icount_read_once(inode));
return;
}
if (inode->i_nlink) {

2
fs/hpfs/inode.c
View File

@ -184,7 +184,7 @@ void hpfs_write_inode(struct inode *i)
struct hpfs_inode_info *hpfs_inode = hpfs_i(i);
struct inode *parent;
if (i->i_ino == hpfs_sb(i->i_sb)->sb_root) return;
if (hpfs_inode->i_rddir_off && !icount_read(i)) {
if (hpfs_inode->i_rddir_off && !icount_read_once(i)) {
if (*hpfs_inode->i_rddir_off)
pr_err("write_inode: some position still there\n");
kfree(hpfs_inode->i_rddir_off);

96
fs/inode.c
View File

@ -518,15 +518,6 @@ static void init_once(void *foo)
inode_init_once(inode);
}
/*
* get additional reference to inode; caller must already hold one.
*/
void ihold(struct inode *inode)
{
WARN_ON(atomic_inc_return(&inode->i_count) < 2);
}
EXPORT_SYMBOL(ihold);
struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe,
struct inode *inode, u32 bit)
{
@ -902,7 +893,7 @@ void evict_inodes(struct super_block *sb)
again:
spin_lock(&sb->s_inode_list_lock);
list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
if (icount_read(inode))
if (icount_read_once(inode))
continue;
spin_lock(&inode->i_lock);
@ -1032,6 +1023,7 @@ long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
}
static void __wait_on_freeing_inode(struct inode *inode, bool hash_locked, bool rcu_locked);
static bool igrab_from_hash(struct inode *inode);
/*
* Called with the inode lock held.
@ -1056,6 +1048,11 @@ static struct inode *find_inode(struct super_block *sb,
continue;
if (!test(inode, data))
continue;
if (igrab_from_hash(inode)) {
rcu_read_unlock();
*isnew = false;
return inode;
}
spin_lock(&inode->i_lock);
if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) {
__wait_on_freeing_inode(inode, hash_locked, true);
@ -1098,6 +1095,11 @@ static struct inode *find_inode_fast(struct super_block *sb,
continue;
if (inode->i_sb != sb)
continue;
if (igrab_from_hash(inode)) {
rcu_read_unlock();
*isnew = false;
return inode;
}
spin_lock(&inode->i_lock);
if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) {
__wait_on_freeing_inode(inode, hash_locked, true);
@ -1215,6 +1217,10 @@ void unlock_new_inode(struct inode *inode)
lockdep_annotate_inode_mutex_key(inode);
spin_lock(&inode->i_lock);
WARN_ON(!(inode_state_read(inode) & I_NEW));
/*
* Paired with igrab_from_hash()
*/
smp_wmb();
inode_state_clear(inode, I_NEW | I_CREATING);
inode_wake_up_bit(inode, __I_NEW);
spin_unlock(&inode->i_lock);
@ -1226,6 +1232,10 @@ void discard_new_inode(struct inode *inode)
lockdep_annotate_inode_mutex_key(inode);
spin_lock(&inode->i_lock);
WARN_ON(!(inode_state_read(inode) & I_NEW));
/*
* Paired with igrab_from_hash()
*/
smp_wmb();
inode_state_clear(inode, I_NEW);
inode_wake_up_bit(inode, __I_NEW);
spin_unlock(&inode->i_lock);
@ -1572,8 +1582,27 @@ ino_t iunique(struct super_block *sb, ino_t max_reserved)
}
EXPORT_SYMBOL(iunique);
/**
* ihold - get a reference on the inode, provided you already have one
* @inode: inode to operate on
*/
void ihold(struct inode *inode)
{
VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode);
WARN_ON(atomic_inc_return(&inode->i_count) < 2);
}
EXPORT_SYMBOL(ihold);
struct inode *igrab(struct inode *inode)
{
/*
* Read commentary above igrab_from_hash() for an explanation why this works.
*/
if (atomic_add_unless(&inode->i_count, 1, 0)) {
VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode);
return inode;
}
spin_lock(&inode->i_lock);
if (!(inode_state_read(inode) & (I_FREEING | I_WILL_FREE))) {
__iget(inode);
@ -1591,6 +1620,43 @@ struct inode *igrab(struct inode *inode)
}
EXPORT_SYMBOL(igrab);
/*
* igrab_from_hash - special inode refcount acquire primitive for the inode hash
*
* It provides lockless refcount acquire in the common case of no problematic
* flags being set and the count being > 0.
*
* There are 4 state flags to worry about and the routine makes sure to not bump the
* ref if any of them is present.
*
* I_NEW and I_CREATING can only legally get set *before* the inode becomes visible
* during lookup. Thus if the flags are not spotted, they are guaranteed to not be
* a factor. However, we need an acquire fence before returning the inode just
* in case we raced against clearing the state to make sure our consumer picks up
* any other changes made prior. atomic_add_unless provides a full fence, which
* takes care of it.
*
* I_FREEING and I_WILL_FREE can only legally get set if ->i_count == 0 and it is
* illegal to bump the ref if either is present. Consequently if atomic_add_unless
* managed to replace a non-0 value with a bigger one, we have a guarantee neither
* of these flags is set. Note this means explicitly checking of these flags below
* is not necessary, it is only done because it does not cost anything on top of the
* load which already needs to be done to handle the other flags.
*/
static bool igrab_from_hash(struct inode *inode)
{
if (inode_state_read_once(inode) & (I_NEW | I_CREATING | I_FREEING | I_WILL_FREE))
return false;
/*
* Paired with routines clearing I_NEW
*/
if (atomic_add_unless(&inode->i_count, 1, 0)) {
VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode);
return true;
}
return false;
}
/**
* ilookup5_nowait - search for an inode in the inode cache
* @sb: super block of file system to search
@ -1920,7 +1986,7 @@ static void iput_final(struct inode *inode)
int drop;
WARN_ON(inode_state_read(inode) & I_NEW);
VFS_BUG_ON_INODE(atomic_read(&inode->i_count) != 0, inode);
VFS_BUG_ON_INODE(icount_read(inode) != 0, inode);
if (op->drop_inode)
drop = op->drop_inode(inode);
@ -1939,7 +2005,7 @@ static void iput_final(struct inode *inode)
* Re-check ->i_count in case the ->drop_inode() hooks played games.
* Note we only execute this if the verdict was to drop the inode.
*/
VFS_BUG_ON_INODE(atomic_read(&inode->i_count) != 0, inode);
VFS_BUG_ON_INODE(icount_read(inode) != 0, inode);
if (drop) {
inode_state_set(inode, I_FREEING);
@ -1983,7 +2049,7 @@ void iput(struct inode *inode)
* equal to one, then two CPUs racing to further drop it can both
* conclude it's fine.
*/
VFS_BUG_ON_INODE(atomic_read(&inode->i_count) < 1, inode);
VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode);
if (atomic_add_unless(&inode->i_count, -1, 1))
return;
@ -2017,7 +2083,7 @@ EXPORT_SYMBOL(iput);
void iput_not_last(struct inode *inode)
{
VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode);
VFS_BUG_ON_INODE(atomic_read(&inode->i_count) < 2, inode);
VFS_BUG_ON_INODE(icount_read_once(inode) < 2, inode);
WARN_ON(atomic_sub_return(1, &inode->i_count) == 0);
}
@ -3040,7 +3106,7 @@ void dump_inode(struct inode *inode, const char *reason)
}
state = inode_state_read_once(inode);
count = atomic_read(&inode->i_count);
count = icount_read_once(inode);
if (!sb ||
get_kernel_nofault(s_type, &sb->s_type) || !s_type ||

4
fs/nfs/inode.c
View File

@ -608,7 +608,7 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr)
inode->i_sb->s_id,
(unsigned long long)NFS_FILEID(inode),
nfs_display_fhandle_hash(fh),
icount_read(inode));
icount_read_once(inode));
out:
return inode;
@ -2255,7 +2255,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr)
dfprintk(VFS, "NFS: %s(%s/%llu fh_crc=0x%08x ct=%d info=0x%llx)\n",
__func__, inode->i_sb->s_id, inode->i_ino,
nfs_display_fhandle_hash(NFS_FH(inode)),
icount_read(inode), fattr->valid);
icount_read_once(inode), fattr->valid);
if (!(fattr->valid & NFS_ATTR_FATTR_FILEID)) {
/* Only a mounted-on-fileid? Just exit */

2
fs/smb/client/inode.c
View File

@ -2843,7 +2843,7 @@ int cifs_revalidate_dentry_attr(struct dentry *dentry)
}
cifs_dbg(FYI, "Update attributes: %s inode 0x%p count %d dentry: 0x%p d_time %ld jiffies %ld\n",
full_path, inode, icount_read(inode),
full_path, inode, icount_read_once(inode),
dentry, cifs_get_time(dentry), jiffies);
again:

2
fs/ubifs/super.c
View File

@ -358,7 +358,7 @@ static void ubifs_evict_inode(struct inode *inode)
goto out;
dbg_gen("inode %llu, mode %#x", inode->i_ino, (int)inode->i_mode);
ubifs_assert(c, !icount_read(inode));
ubifs_assert(c, !icount_read_once(inode));
truncate_inode_pages_final(&inode->i_data);

2
fs/xfs/xfs_inode.c
View File

@ -1046,7 +1046,7 @@ xfs_itruncate_extents_flags(
int error = 0;
xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
if (icount_read(VFS_I(ip)))
if (icount_read_once(VFS_I(ip)))
xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL);
if (whichfork == XFS_DATA_FORK)
ASSERT(new_size <= XFS_ISIZE(ip));

2
fs/xfs/xfs_trace.h
View File

@ -1158,7 +1158,7 @@ DECLARE_EVENT_CLASS(xfs_iref_class,
TP_fast_assign(
__entry->dev = VFS_I(ip)->i_sb->s_dev;
__entry->ino = ip->i_ino;
__entry->count = icount_read(VFS_I(ip));
__entry->count = icount_read_once(VFS_I(ip));
__entry->pincount = atomic_read(&ip->i_pincount);
__entry->iflags = ip->i_flags;
__entry->caller_ip = caller_ip;

13
include/linux/fs.h
View File

@ -2218,8 +2218,21 @@ static inline void mark_inode_dirty_sync(struct inode *inode)
__mark_inode_dirty(inode, I_DIRTY_SYNC);
}
/*
* returns the refcount on the inode. it can change arbitrarily.
*/
static inline int icount_read_once(const struct inode *inode)
{
return atomic_read(&inode->i_count);
}
/*
* returns the refcount on the inode. The lock guarantees no 0->1 or 1->0 transitions
* of the count are going to take place, otherwise it changes arbitrarily.
*/
static inline int icount_read(const struct inode *inode)
{
lockdep_assert_held(&inode->i_lock);
return atomic_read(&inode->i_count);
}

2
include/trace/events/filelock.h
View File

@ -190,7 +190,7 @@ TRACE_EVENT(generic_add_lease,
__entry->i_ino = inode->i_ino;
__entry->wcount = atomic_read(&inode->i_writecount);
__entry->rcount = atomic_read(&inode->i_readcount);
__entry->icount = icount_read(inode);
__entry->icount = icount_read_once(inode);
__entry->owner = fl->c.flc_owner;
__entry->flags = fl->c.flc_flags;
__entry->type = fl->c.flc_type;

2
security/landlock/fs.c
View File

@ -1278,7 +1278,7 @@ static void hook_sb_delete(struct super_block *const sb)
struct landlock_object *object;
/* Only handles referenced inodes. */
if (!icount_read(inode))
if (!icount_read_once(inode))
continue;
/*