fs: fs_struct use seqlock
Use a seqlock in the fs_struct to enable us to take an atomic copy of the
complete cwd and root paths. Use this in the RCU lookup path to avoid a
thread-shared spinlock in RCU lookup operations.
Multi-threaded apps may now perform path lookups with scalability matching
multi-process apps. Operations such as stat(2) become very scalable for
multi-threaded workload.
Signed-off-by: Nick Piggin <npiggin@kernel.dk>
diff --git a/fs/fs_struct.c b/fs/fs_struct.c
index ed45a9c..60b8531 100644
--- a/fs/fs_struct.c
+++ b/fs/fs_struct.c
@@ -14,9 +14,11 @@
struct path old_root;
spin_lock(&fs->lock);
+ write_seqcount_begin(&fs->seq);
old_root = fs->root;
fs->root = *path;
path_get(path);
+ write_seqcount_end(&fs->seq);
spin_unlock(&fs->lock);
if (old_root.dentry)
path_put(&old_root);
@@ -31,9 +33,11 @@
struct path old_pwd;
spin_lock(&fs->lock);
+ write_seqcount_begin(&fs->seq);
old_pwd = fs->pwd;
fs->pwd = *path;
path_get(path);
+ write_seqcount_end(&fs->seq);
spin_unlock(&fs->lock);
if (old_pwd.dentry)
@@ -52,6 +56,7 @@
fs = p->fs;
if (fs) {
spin_lock(&fs->lock);
+ write_seqcount_begin(&fs->seq);
if (fs->root.dentry == old_root->dentry
&& fs->root.mnt == old_root->mnt) {
path_get(new_root);
@@ -64,6 +69,7 @@
fs->pwd = *new_root;
count++;
}
+ write_seqcount_end(&fs->seq);
spin_unlock(&fs->lock);
}
task_unlock(p);
@@ -88,8 +94,10 @@
int kill;
task_lock(tsk);
spin_lock(&fs->lock);
+ write_seqcount_begin(&fs->seq);
tsk->fs = NULL;
kill = !--fs->users;
+ write_seqcount_end(&fs->seq);
spin_unlock(&fs->lock);
task_unlock(tsk);
if (kill)
@@ -105,6 +113,7 @@
fs->users = 1;
fs->in_exec = 0;
spin_lock_init(&fs->lock);
+ seqcount_init(&fs->seq);
fs->umask = old->umask;
get_fs_root_and_pwd(old, &fs->root, &fs->pwd);
}
@@ -144,6 +153,7 @@
struct fs_struct init_fs = {
.users = 1,
.lock = __SPIN_LOCK_UNLOCKED(init_fs.lock),
+ .seq = SEQCNT_ZERO,
.umask = 0022,
};
