mirror of
https://github.com/torvalds/linux.git
synced 2026-06-07 05:55:44 +02:00
ANDROID: Incremental fs: RCU locks instead of mutex for pending_reads.
Use RCU locks instead of pending_reads_mutex. Current mutex is taking lock on entire mount_info structure which seems a heavy operation. Following fields of mount_info structure are protected through spinlocks for multiple writers and are RCU safe for readers: - reads_list_head - mi_pending_reads_count - mi_last_pending_read_number - data_file_segment.reads_list_head We could probably use atomic_inc/atomic_dec for mi_pending_reads_count and mi_last_pending_read_number which can futher cut down spin_locks at couple of more places, thereby only the list addition and removal can protected by spinlock. This CL doesn't address it. Bug: 161565969 Test: kernel selftest incfs_test and incfs_perf Signed-off-by: Akilesh Kailash <akailash@google.com> Change-Id: Iad7439657016764dce25d64c8b3df69b930452bc
This commit is contained in:
Akilesh Kailash
parent
839511af20
commit
0f406eaea5
|
|
@ -40,11 +40,11 @@ struct mount_info *incfs_alloc_mount_info(struct super_block *sb,
|
|||
mi->mi_owner = get_current_cred();
|
||||
path_get(&mi->mi_backing_dir_path);
|
||||
mutex_init(&mi->mi_dir_struct_mutex);
|
||||
mutex_init(&mi->mi_pending_reads_mutex);
|
||||
init_waitqueue_head(&mi->mi_pending_reads_notif_wq);
|
||||
init_waitqueue_head(&mi->mi_log.ml_notif_wq);
|
||||
INIT_DELAYED_WORK(&mi->mi_log.ml_wakeup_work, log_wake_up_all);
|
||||
spin_lock_init(&mi->mi_log.rl_lock);
|
||||
spin_lock_init(&mi->pending_read_lock);
|
||||
INIT_LIST_HEAD(&mi->mi_reads_list_head);
|
||||
|
||||
error = incfs_realloc_mount_info(mi, options);
|
||||
|
|
@ -109,7 +109,6 @@ void incfs_free_mount_info(struct mount_info *mi)
|
|||
dput(mi->mi_index_dir);
|
||||
path_put(&mi->mi_backing_dir_path);
|
||||
mutex_destroy(&mi->mi_dir_struct_mutex);
|
||||
mutex_destroy(&mi->mi_pending_reads_mutex);
|
||||
put_cred(mi->mi_owner);
|
||||
kfree(mi->mi_log.rl_ring_buf);
|
||||
kfree(mi->log_xattr);
|
||||
|
|
@ -756,19 +755,29 @@ static struct pending_read *add_pending_read(struct data_file *df,
|
|||
result->block_index = block_index;
|
||||
result->timestamp_us = ktime_to_us(ktime_get());
|
||||
|
||||
mutex_lock(&mi->mi_pending_reads_mutex);
|
||||
spin_lock(&mi->pending_read_lock);
|
||||
|
||||
result->serial_number = ++mi->mi_last_pending_read_number;
|
||||
mi->mi_pending_reads_count++;
|
||||
|
||||
list_add(&result->mi_reads_list, &mi->mi_reads_list_head);
|
||||
list_add(&result->segment_reads_list, &segment->reads_list_head);
|
||||
mutex_unlock(&mi->mi_pending_reads_mutex);
|
||||
list_add_rcu(&result->mi_reads_list, &mi->mi_reads_list_head);
|
||||
list_add_rcu(&result->segment_reads_list, &segment->reads_list_head);
|
||||
|
||||
spin_unlock(&mi->pending_read_lock);
|
||||
|
||||
wake_up_all(&mi->mi_pending_reads_notif_wq);
|
||||
return result;
|
||||
}
|
||||
|
||||
static void free_pending_read_entry(struct rcu_head *entry)
|
||||
{
|
||||
struct pending_read *read;
|
||||
|
||||
read = container_of(entry, struct pending_read, rcu);
|
||||
|
||||
kfree(read);
|
||||
}
|
||||
|
||||
/* Notifies a given data file that pending read is completed. */
|
||||
static void remove_pending_read(struct data_file *df, struct pending_read *read)
|
||||
{
|
||||
|
|
@ -782,14 +791,17 @@ static void remove_pending_read(struct data_file *df, struct pending_read *read)
|
|||
|
||||
mi = df->df_mount_info;
|
||||
|
||||
mutex_lock(&mi->mi_pending_reads_mutex);
|
||||
list_del(&read->mi_reads_list);
|
||||
list_del(&read->segment_reads_list);
|
||||
spin_lock(&mi->pending_read_lock);
|
||||
|
||||
list_del_rcu(&read->mi_reads_list);
|
||||
list_del_rcu(&read->segment_reads_list);
|
||||
|
||||
mi->mi_pending_reads_count--;
|
||||
mutex_unlock(&mi->mi_pending_reads_mutex);
|
||||
|
||||
kfree(read);
|
||||
spin_unlock(&mi->pending_read_lock);
|
||||
|
||||
/* Don't free. Wait for readers */
|
||||
call_rcu(&read->rcu, free_pending_read_entry);
|
||||
}
|
||||
|
||||
static void notify_pending_reads(struct mount_info *mi,
|
||||
|
|
@ -799,13 +811,13 @@ static void notify_pending_reads(struct mount_info *mi,
|
|||
struct pending_read *entry = NULL;
|
||||
|
||||
/* Notify pending reads waiting for this block. */
|
||||
mutex_lock(&mi->mi_pending_reads_mutex);
|
||||
list_for_each_entry(entry, &segment->reads_list_head,
|
||||
rcu_read_lock();
|
||||
list_for_each_entry_rcu(entry, &segment->reads_list_head,
|
||||
segment_reads_list) {
|
||||
if (entry->block_index == index)
|
||||
set_read_done(entry);
|
||||
}
|
||||
mutex_unlock(&mi->mi_pending_reads_mutex);
|
||||
rcu_read_unlock();
|
||||
wake_up_all(&segment->new_data_arrival_wq);
|
||||
}
|
||||
|
||||
|
|
@ -869,7 +881,6 @@ static int wait_for_data_block(struct data_file *df, int block_index,
|
|||
|
||||
/* Woke up, the pending read is no longer needed. */
|
||||
remove_pending_read(df, read);
|
||||
read = NULL;
|
||||
|
||||
if (wait_res == 0) {
|
||||
/* Wait has timed out */
|
||||
|
|
@ -1292,10 +1303,14 @@ bool incfs_fresh_pending_reads_exist(struct mount_info *mi, int last_number)
|
|||
{
|
||||
bool result = false;
|
||||
|
||||
mutex_lock(&mi->mi_pending_reads_mutex);
|
||||
/*
|
||||
* We could probably get away with spin locks here;
|
||||
* if we use atomic_read() on both these variables.
|
||||
*/
|
||||
spin_lock(&mi->pending_read_lock);
|
||||
result = (mi->mi_last_pending_read_number > last_number) &&
|
||||
(mi->mi_pending_reads_count > 0);
|
||||
mutex_unlock(&mi->mi_pending_reads_mutex);
|
||||
(mi->mi_pending_reads_count > 0);
|
||||
spin_unlock(&mi->pending_read_lock);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
|
@ -1305,6 +1320,7 @@ int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
|
|||
{
|
||||
int reported_reads = 0;
|
||||
struct pending_read *entry = NULL;
|
||||
bool result = false;
|
||||
|
||||
if (!mi)
|
||||
return -EFAULT;
|
||||
|
|
@ -1312,13 +1328,19 @@ int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
|
|||
if (reads_size <= 0)
|
||||
return 0;
|
||||
|
||||
mutex_lock(&mi->mi_pending_reads_mutex);
|
||||
spin_lock(&mi->pending_read_lock);
|
||||
|
||||
if (mi->mi_last_pending_read_number <= sn_lowerbound
|
||||
|| mi->mi_pending_reads_count == 0)
|
||||
goto unlock;
|
||||
result = ((mi->mi_last_pending_read_number <= sn_lowerbound)
|
||||
|| (mi->mi_pending_reads_count == 0));
|
||||
|
||||
list_for_each_entry(entry, &mi->mi_reads_list_head, mi_reads_list) {
|
||||
spin_unlock(&mi->pending_read_lock);
|
||||
|
||||
if (result)
|
||||
return reported_reads;
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
list_for_each_entry_rcu(entry, &mi->mi_reads_list_head, mi_reads_list) {
|
||||
if (entry->serial_number <= sn_lowerbound)
|
||||
continue;
|
||||
|
||||
|
|
@ -1333,8 +1355,7 @@ int incfs_collect_pending_reads(struct mount_info *mi, int sn_lowerbound,
|
|||
break;
|
||||
}
|
||||
|
||||
unlock:
|
||||
mutex_unlock(&mi->mi_pending_reads_mutex);
|
||||
rcu_read_unlock();
|
||||
|
||||
return reported_reads;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include <linux/types.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/rcupdate.h>
|
||||
#include <linux/completion.h>
|
||||
#include <linux/wait.h>
|
||||
#include <crypto/hash.h>
|
||||
|
|
@ -123,13 +124,13 @@ struct mount_info {
|
|||
wait_queue_head_t mi_pending_reads_notif_wq;
|
||||
|
||||
/*
|
||||
* Protects:
|
||||
* Protects - RCU safe:
|
||||
* - reads_list_head
|
||||
* - mi_pending_reads_count
|
||||
* - mi_last_pending_read_number
|
||||
* - data_file_segment.reads_list_head
|
||||
*/
|
||||
struct mutex mi_pending_reads_mutex;
|
||||
spinlock_t pending_read_lock;
|
||||
|
||||
/* List of active pending_read objects */
|
||||
struct list_head mi_reads_list_head;
|
||||
|
|
@ -175,6 +176,8 @@ struct pending_read {
|
|||
struct list_head mi_reads_list;
|
||||
|
||||
struct list_head segment_reads_list;
|
||||
|
||||
struct rcu_head rcu;
|
||||
};
|
||||
|
||||
struct data_file_segment {
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user