diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index db32bb40a867..e4bba9b6fecb 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -1875,7 +1875,7 @@ static int fill_note_info(struct elfhdr *elf, int phdrs, return 0; info->thread->task = dump_task; - for (ct = dump_task->signal->core_state->dumper.next; ct; ct = ct->next) { + for (ct = dump_task->signal->core_state->tasks; ct; ct = ct->next) { t = kzalloc_flex(*t, notes, info->thread_notes); if (unlikely(!t)) return 0; diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index 068c46875c74..a2633330c59b 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -1504,7 +1504,7 @@ static int elf_fdpic_core_dump(struct coredump_params *cprm) if (!psinfo) goto end_coredump; - for (ct = current->signal->core_state->dumper.next; + for (ct = current->signal->core_state->tasks; ct; ct = ct->next) { tmp = elf_dump_thread_status(cprm->siginfo->si_signo, ct->task, &thread_status_size); diff --git a/fs/coredump.c b/fs/coredump.c index 3e78941f281e..6f4c6a900dc5 100644 --- a/fs/coredump.c +++ b/fs/coredump.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -514,38 +515,38 @@ static int zap_threads(struct task_struct *tsk, nr = zap_process(signal, exit_code); clear_tsk_thread_flag(tsk, TIF_SIGPENDING); tsk->flags |= PF_DUMPCORE; - atomic_set(&core_state->nr_threads, nr); + atomic_set(&core_state->threads_remaining, nr); } spin_unlock_irq(&tsk->sighand->siglock); return nr; } +static void coredump_wait_inactive(struct core_state *core_state) +{ + struct core_thread *ptr; + + wait_var_event_state(&core_state->threads_remaining, + !atomic_read_acquire(&core_state->threads_remaining), + TASK_UNINTERRUPTIBLE | TASK_FREEZABLE); + /* + * Wait for all the threads to become inactive, so that + * all the thread context (extended register state, like + * fpu etc) gets copied to the memory. + */ + for (ptr = core_state->tasks; ptr; ptr = ptr->next) + wait_task_inactive(ptr->task, TASK_ANY); +} + static int coredump_wait(int exit_code, struct core_state *core_state) { struct task_struct *tsk = current; int core_waiters = -EBUSY; - init_completion(&core_state->startup); - core_state->dumper.task = tsk; - core_state->dumper.next = NULL; + core_state->tasks = NULL; core_waiters = zap_threads(tsk, core_state, exit_code); - if (core_waiters > 0) { - struct core_thread *ptr; - - wait_for_completion_state(&core_state->startup, - TASK_UNINTERRUPTIBLE|TASK_FREEZABLE); - /* - * Wait for all the threads to become inactive, so that - * all the thread context (extended register state, like - * fpu etc) gets copied to the memory. - */ - ptr = core_state->dumper.next; - while (ptr != NULL) { - wait_task_inactive(ptr->task, TASK_ANY); - ptr = ptr->next; - } - } + if (core_waiters > 0) + coredump_wait_inactive(core_state); return core_waiters; } @@ -558,7 +559,7 @@ static void coredump_finish(bool core_dumped) spin_lock_irq(¤t->sighand->siglock); if (core_dumped && !__fatal_signal_pending(current)) current->signal->group_exit_code |= 0x80; - next = current->signal->core_state->dumper.next; + next = current->signal->core_state->tasks; current->signal->core_state = NULL; spin_unlock_irq(¤t->sighand->siglock); diff --git a/fs/exec.c b/fs/exec.c index d3081c8f7c10..075a744421e1 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1124,6 +1124,7 @@ static struct file *bprm_identity_file(const struct linux_binprm *bprm) int begin_new_exec(struct linux_binprm * bprm) { struct task_struct *me = current; + struct files_struct *files = NULL; int retval; /* A pending PT_INTERP substitution this format cannot consume. */ @@ -1160,9 +1161,11 @@ int begin_new_exec(struct linux_binprm * bprm) io_uring_task_cancel(); /* Ensure the files table is not shared. */ - retval = unshare_files(); + retval = unshare_fd(CLONE_FILES, &files); if (retval) goto out; + if (files) + switch_files_struct(me, files); /* * We have to apply CLOEXEC before we change whether the process is @@ -1170,13 +1173,13 @@ int begin_new_exec(struct linux_binprm * bprm) * trying to access the should-be-closed file descriptors of a process * undergoing exec(2). * - * This can block on filesystem ->flush() handlers, including waiting - * for FUSE daemons, so do it before exec_mmap takes the - * exec_update_lock. + * This can block on filesystem ->flush() and ->release() handlers, + * including waiting for FUSE daemons, so do it before exec_mmap + * takes the exec_update_lock. * This must happen after the point of no return, and after unsharing * the FD table. */ - do_close_on_exec(me->files); + close_cloexec_files(me->files); /* * Must be called _before_ exec_mmap() as bprm->mm is diff --git a/fs/file.c b/fs/file.c index 37f0ba740c39..49a6e308f561 100644 --- a/fs/file.c +++ b/fs/file.c @@ -531,7 +531,25 @@ struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *range) return newf; } -static struct fdtable *close_files(struct files_struct * files) +/* + * Unshare file descriptor table if it is being shared + */ +int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp) +{ + struct files_struct *fd = current->files; + + if ((unshare_flags & CLONE_FILES) && + (fd && atomic_read(&fd->count) > 1)) { + fd = dup_fd(fd, NULL); + if (IS_ERR(fd)) + return PTR_ERR(fd); + *new_fdp = fd; + } + + return 0; +} + +static struct fdtable *close_files(struct files_struct *files) { /* * It is safe to dereference the fd table without RCU or @@ -551,7 +569,7 @@ static struct fdtable *close_files(struct files_struct * files) if (set & 1) { struct file *file = fdt->fd[i]; if (file) { - filp_close(file, files); + filp_close_sync(file, files); cond_resched(); } } @@ -575,16 +593,18 @@ void put_files_struct(struct files_struct *files) } } +/* Install @files on @tsk, consuming the reference, and put the old table. */ +void switch_files_struct(struct task_struct *tsk, struct files_struct *files) +{ + scoped_guard(task_lock, tsk) + swap(tsk->files, files); + put_files_struct(files); +} + void exit_files(struct task_struct *tsk) { - struct files_struct * files = tsk->files; - - if (files) { - task_lock(tsk); - tsk->files = NULL; - task_unlock(tsk); - put_files_struct(files); - } + if (tsk->files) + switch_files_struct(tsk, NULL); } struct files_struct init_files = { @@ -792,16 +812,13 @@ struct file *file_close_fd_locked(struct files_struct *files, unsigned fd) int close_fd(unsigned fd) { - struct files_struct *files = current->files; struct file *file; - spin_lock(&files->file_lock); - file = file_close_fd_locked(files, fd); - spin_unlock(&files->file_lock); + file = file_close_fd(fd); if (!file) return -EBADF; - return filp_close(file, files); + return filp_close(file, current->files); } EXPORT_SYMBOL(close_fd); @@ -894,7 +911,7 @@ static inline void __range_close(struct files_struct *files, file = file_close_fd_locked(files, fd); if (file) { spin_unlock(&files->file_lock); - filp_close(file, files); + filp_close_sync(file, files); cond_resched(); spin_lock(&files->file_lock); fdt = files_fdtable(files); @@ -986,10 +1003,7 @@ SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd, * We're done closing the files we were supposed to. Time to install * the new file descriptor table and drop the old one. */ - task_lock(me); - me->files = cur_fds; - task_unlock(me); - put_files_struct(fds); + switch_files_struct(me, cur_fds); } return 0; @@ -1015,7 +1029,7 @@ struct file *file_close_fd(unsigned int fd) return file; } -void do_close_on_exec(struct files_struct *files) +void close_cloexec_files(struct files_struct *files) { unsigned i; struct fdtable *fdt; @@ -1042,7 +1056,7 @@ void do_close_on_exec(struct files_struct *files) rcu_assign_pointer(fdt->fd[fd], NULL); __put_unused_fd(files, fd); spin_unlock(&files->file_lock); - filp_close(file, files); + filp_close_sync(file, files); cond_resched(); spin_lock(&files->file_lock); } diff --git a/fs/internal.h b/fs/internal.h index c658c8a5ebd5..8812de210d3f 100644 --- a/fs/internal.h +++ b/fs/internal.h @@ -198,6 +198,7 @@ extern struct file *do_file_open_root(const struct path *, extern struct open_how build_open_how(int flags, umode_t mode); extern int build_open_flags(const struct open_how *how, struct open_flags *op); struct file *file_close_fd_locked(struct files_struct *files, unsigned fd); +int filp_close_sync(struct file *filp, fl_owner_t id); int do_ftruncate(struct file *file, loff_t length, unsigned int flags); int chmod_common(const struct path *path, umode_t mode); diff --git a/fs/open.c b/fs/open.c index 6b1c14e684a9..998e42ac319a 100644 --- a/fs/open.c +++ b/fs/open.c @@ -1537,6 +1537,19 @@ int filp_close(struct file *filp, fl_owner_t id) } EXPORT_SYMBOL(filp_close); +/* Like filp_close() but the last reference is put right here. */ +int filp_close_sync(struct file *filp, fl_owner_t id) +{ + int retval; + + /* Kernel threads must never put their final reference here. */ + VFS_WARN_ON_ONCE(current->flags & PF_KTHREAD); + retval = filp_flush(filp, id); + fput_close_sync(filp); + + return retval; +} + /* * Careful here! We test whether the file pointer is NULL before * releasing the fd. This ensures that one clone task can't release @@ -1551,13 +1564,11 @@ SYSCALL_DEFINE1(close, unsigned int, fd) if (!file) return -EBADF; - retval = filp_flush(file, current->files); - /* * We're returning to user space. Don't bother * with any delayed fput() cases. */ - fput_close_sync(file); + retval = filp_close_sync(file, current->files); if (likely(retval == 0)) return 0; diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h index afdfaad381df..8c28fdd15f31 100644 --- a/include/linux/fdtable.h +++ b/include/linux/fdtable.h @@ -100,7 +100,8 @@ static inline bool close_on_exec(unsigned int fd, const struct files_struct *fil struct task_struct; void put_files_struct(struct files_struct *fs); -int unshare_files(void); +void switch_files_struct(struct task_struct *tsk, struct files_struct *files); +int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp); enum fd_range_flags { /* Leave behind all descriptors outside of the specified range. */ FD_RANGE_EXCEPT = (1U << 0), @@ -114,7 +115,7 @@ struct fd_range { enum fd_range_flags flags; }; struct files_struct *dup_fd(struct files_struct *, struct fd_range *) __latent_entropy; -void do_close_on_exec(struct files_struct *); +void close_cloexec_files(struct files_struct *); int iterate_fd(struct files_struct *, unsigned, int (*)(const void *, struct file *, unsigned), const void *); diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index d45a5476b97d..e039e29cd8c5 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -79,9 +79,9 @@ struct core_thread { }; struct core_state { - atomic_t nr_threads; - struct core_thread dumper; - struct completion startup; + /* Threads the dumper still waits for. */ + atomic_t threads_remaining; + struct core_thread *tasks; }; /* diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 553d7b23e3ad..af077ed4caf6 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -432,6 +432,32 @@ do { \ __ret; \ }) +/** + * wait_var_event_state - wait for a variable to be updated and notified + * @var: the address of variable being waited on + * @condition: the condition to wait for + * @state: the task state to sleep in, %TASK_UNINTERRUPTIBLE etc. + * + * Wait for a @condition to be true, only re-checking when a wake up is + * received for the given @var (an arbitrary kernel address which need + * not be directly related to the given condition, but usually is). + * + * Returns 0 if the condition became true, or %-ERESTARTSYS if a signal + * arrived which @state allows to interrupt. + * + * The condition should normally use smp_load_acquire() or a similarly + * ordered access to ensure that any changes to memory made before the + * condition became true will be visible after the wait completes. + */ +#define wait_var_event_state(var, condition, state) \ +({ \ + int __ret = 0; \ + might_sleep(); \ + if (!(condition)) \ + __ret = ___wait_var_event(var, condition, (state), 0, 0, schedule()); \ + __ret; \ +}) + /** * wait_var_event_any_lock - wait for a variable to be updated under a lock * @var: the address of the variable being waited on diff --git a/kernel/exit.c b/kernel/exit.c index 4e028f157597..55dbea3b242e 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -435,15 +436,14 @@ static void coredump_task_exit(struct task_struct *tsk, self.task = tsk; if (self.task->flags & PF_SIGNALED) - self.next = xchg(&core_state->dumper.next, &self); + self.next = xchg(&core_state->tasks, &self); else self.task = NULL; /* * Implies mb(), the result of xchg() must be visible - * to core_state->dumper. + * to the dumper. */ - if (atomic_dec_and_test(&core_state->nr_threads)) - complete(&core_state->startup); + atomic_dec_and_wake_up(&core_state->threads_remaining); for (;;) { set_current_state(TASK_IDLE|TASK_FREEZABLE); @@ -917,7 +917,7 @@ static void synchronize_group_exit(struct task_struct *tsk, long code) * Serialize with any possible pending coredump. * We must hold siglock around checking core_state * and setting PF_POSTCOREDUMP. The core-inducing thread - * will increment ->nr_threads for each thread in the + * will increment ->threads_remaining for each thread in the * group without PF_POSTCOREDUMP set. */ tsk->flags |= PF_POSTCOREDUMP; diff --git a/kernel/fork.c b/kernel/fork.c index a5934a317634..10be4a0ecb3f 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -3210,24 +3210,6 @@ static int unshare_fs(unsigned long unshare_flags, struct fs_struct **new_fsp) return 0; } -/* - * Unshare file descriptor table if it is being shared - */ -static int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp) -{ - struct files_struct *fd = current->files; - - if ((unshare_flags & CLONE_FILES) && - (fd && atomic_read(&fd->count) > 1)) { - fd = dup_fd(fd, NULL); - if (IS_ERR(fd)) - return PTR_ERR(fd); - *new_fdp = fd; - } - - return 0; -} - /* * unshare allows a process to 'unshare' part of the process * context which was originally shared using clone. copy_* @@ -3323,10 +3305,8 @@ int ksys_unshare(unsigned long unshare_flags) if (new_fs) new_fs = switch_fs_struct(new_fs); - if (new_fd) { - guard(task_lock)(current); - swap(current->files, new_fd); - } + if (new_fd) + switch_files_struct(current, no_free_ptr(new_fd)); if (new_cred) { /* Install the new user namespace */ @@ -3359,30 +3339,6 @@ SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) return ksys_unshare(unshare_flags); } -/* - * Helper to unshare the files of the current task. - * We don't want to expose copy_files internals to - * the exec layer of the kernel. - */ - -int unshare_files(void) -{ - struct task_struct *task = current; - struct files_struct *old, *copy = NULL; - int error; - - error = unshare_fd(CLONE_FILES, ©); - if (error || !copy) - return error; - - old = task->files; - task_lock(task); - task->files = copy; - task_unlock(task); - put_files_struct(old); - return 0; -} - static int sysctl_max_threads(const struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos) {