From: Jens Axboe axboe@kernel.dk
mainline inclusion from mainline-5.9-rc1 commit 4c6e277c4cc4a6b3b2b9c66a7b014787ae757cc1 category: feature bugzilla: https://bugzilla.openeuler.org/show_bug.cgi?id=27 CVE: NA ---------------------------
Provide a helper to run task_work instead of checking and running manually in a bunch of different spots. While doing so, also move the task run state setting where we run the task work. Then we can move it out of the callback helpers. This also helps ensure we only do this once per task_work list run, not per task_work item.
Suggested-by: Oleg Nesterov oleg@redhat.com Signed-off-by: Jens Axboe axboe@kernel.dk
Conflicts: fs/io_uring.c [28cea78af449 ("io_uring: allow non-fixed files with SQPOLL") include first, b63534c41e20 ("io_uring: re-issue block requests that failed because of resources") not include, 23b3628e4592 ("io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works") include first, 37c54f9bd486 ("kernel: set USER_DS in kthread_use_mm") not include]
Signed-off-by: yangerkun yangerkun@huawei.com Reviewed-by: zhangyi (F) yi.zhang@huawei.com Signed-off-by: Cheng Jian cj.chengjian@huawei.com --- fs/io_uring.c | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c index e2ddfdc48bc7..affde6571d6c 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -1781,7 +1781,6 @@ static void __io_req_task_submit(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx;
- __set_current_state(TASK_RUNNING); if (!__io_sq_thread_acquire_mm(ctx)) { __io_sq_thread_acquire_files(ctx); mutex_lock(&ctx->uring_lock); @@ -1970,6 +1969,17 @@ static int io_put_kbuf(struct io_kiocb *req) return cflags; }
+static inline bool io_run_task_work(void) +{ + if (current->task_works) { + __set_current_state(TASK_RUNNING); + task_work_run(); + return true; + } + + return false; +} + static void io_iopoll_queue(struct list_head *again) { struct io_kiocb *req; @@ -2164,8 +2174,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned *nr_events, */ if (!(++iters & 7)) { mutex_unlock(&ctx->uring_lock); - if (current->task_works) - task_work_run(); + io_run_task_work(); mutex_lock(&ctx->uring_lock); }
@@ -6271,8 +6280,7 @@ static int io_sq_thread(void *data) if (!list_empty(&ctx->poll_list) || need_resched() || (!time_after(jiffies, timeout) && ret != -EBUSY && !percpu_ref_is_dying(&ctx->refs))) { - if (current->task_works) - task_work_run(); + io_run_task_work(); cond_resched(); continue; } @@ -6301,8 +6309,7 @@ static int io_sq_thread(void *data) finish_wait(&ctx->sqo_wait, &wait); break; } - if (current->task_works) { - task_work_run(); + if (io_run_task_work()) { finish_wait(&ctx->sqo_wait, &wait); io_ring_clear_wakeup_flag(ctx); continue; @@ -6328,8 +6335,7 @@ static int io_sq_thread(void *data) timeout = jiffies + ctx->sq_thread_idle; }
- if (current->task_works) - task_work_run(); + io_run_task_work();
set_fs(old_fs); io_sq_thread_drop_mm_files(); @@ -6400,9 +6406,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, do { if (io_cqring_events(ctx, false) >= min_events) return 0; - if (!current->task_works) + if (!io_run_task_work()) break; - task_work_run(); } while (1);
if (sig) { @@ -6424,8 +6429,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, prepare_to_wait_exclusive(&ctx->wait, &iowq.wq, TASK_INTERRUPTIBLE); /* make sure we run task_work before checking for signals */ - if (current->task_works) - task_work_run(); + if (io_run_task_work()) + continue; if (signal_pending(current)) { if (current->jobctl & JOBCTL_TASK_WORK) { spin_lock_irq(¤t->sighand->siglock); @@ -7861,8 +7866,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit, int submitted = 0; struct fd f;
- if (current->task_works) - task_work_run(); + io_run_task_work();
if (flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP)) return -EINVAL;