From: Jens Axboe axboe@kernel.dk
mainline inclusion from mainline-v6.9-rc1 commit 06b23f92af87a84d70881b2ecaa72e00f7838264 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IAGRKP
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...
--------------------------------
Mark the task as having a cached timestamp when set assign it, so we can efficiently check if it needs updating post being scheduled back in. This covers both the actual schedule out case, which would've flushed the plug, and the preemption case which doesn't touch the plugged requests (for many reasons, one of them being then we'd need to have preemption disabled around plug state manipulation).
Reviewed-by: Johannes Thumshirn johannes.thumshirn@wdc.com Signed-off-by: Jens Axboe axboe@kernel.dk
Conflicts: block/blk.h block/blk-core.c include/linux/blkdev.h include/linux/sched.h [Lots of conficts in context, reimplement the patch for current context] Signed-off-by: Yu Kuai yukuai3@huawei.com --- block/blk-core.c | 2 ++ block/blk.h | 4 +++- include/linux/blkdev.h | 16 ++++++++++++++++ include/linux/sched.h | 1 + kernel/sched/core.c | 6 ++++-- 5 files changed, 26 insertions(+), 3 deletions(-)
diff --git a/block/blk-core.c b/block/blk-core.c index d63d82a628ac..c8a033facf4d 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -4061,6 +4061,8 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) */ if (q) queue_unplugged(q, depth, from_schedule); + + current->flags &= ~PF_BLOCK_TS; }
void blk_finish_plug(struct blk_plug *plug) diff --git a/block/blk.h b/block/blk.h index 075afe1cdfe2..d02b515fc448 100644 --- a/block/blk.h +++ b/block/blk.h @@ -491,8 +491,10 @@ static inline u64 blk_time_get_ns(void) * a valid timestamp" separately, just accept that we'll do an extra * ktime_get_ns() if we just happen to get 0 as the current time. */ - if (!plug->cur_ktime) + if (!plug->cur_ktime) { plug->cur_ktime = ktime_get_ns(); + current->flags |= PF_BLOCK_TS; + } return plug->cur_ktime; }
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 961bc66a0dd1..8fe611439db5 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -1436,6 +1436,18 @@ static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, return bqt->tag_index[tag]; }
+/* + * tsk == current here + */ +static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +{ + struct blk_plug *plug = tsk->plug; + + if (plug) + plug->cur_ktime = 0; + current->flags &= ~PF_BLOCK_TS; +} + extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *); extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct page *page); @@ -2151,6 +2163,10 @@ static inline bool blk_needs_flush_plug(struct task_struct *tsk) return false; }
+static inline void blk_plug_invalidate_ts(struct task_struct *tsk) +{ +} + static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, sector_t *error_sector) { diff --git a/include/linux/sched.h b/include/linux/sched.h index 26255b76ca52..14eba475138a 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1495,6 +1495,7 @@ extern struct pid *cad_pid; #define PF_UCE_KERNEL_RECOVERY 0x02000000 /* Task in uce kernel recovery state */ #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ +#define PF_BLOCK_TS 0x10000000 /* plug has ts that needs updating */ #define PF_IO_WORKER 0x20000000 /* Task is an IO worker */ #define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */ #define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */ diff --git a/kernel/sched/core.c b/kernel/sched/core.c index fe9f91f39e2f..e37428598155 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -3581,10 +3581,12 @@ static inline void sched_submit_work(struct task_struct *tsk)
static void sched_update_worker(struct task_struct *tsk) { - if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER)) { + if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) { + if (tsk->flags & PF_BLOCK_TS) + blk_plug_invalidate_ts(tsk); if (tsk->flags & PF_WQ_WORKER) wq_worker_running(tsk); - else + else if (tsk->flags & PF_IO_WORKER) io_wq_worker_running(tsk); } }