From: Tejun Heo <tj@kernel.org> mainline inclusion from mainline-v6.12-rc1 commit 6462dd53a26088a433f90a5c15822196f201037c category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IDC9YK Reference: https://web.git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commi... -------------------------------- struct scx_iter_scx_dsq is defined as 6 u64's and scx_dsq_iter_kern was using 5 of them. We want to add two more u64 fields but it's better if we do so while staying within scx_iter_scx_dsq to maintain binary compatibility. The way scx_iter_scx_dsq_kern is laid out is rather inefficient - the node field takes up three u64's but only one bit of the last u64 is used. Turn the bool into u32 flags and only use the lower 16 bits freeing up 48 bits - 16 bits for flags, 32 bits for a u32 - for use by struct bpf_iter_scx_dsq_kern. This allows moving the dsq_seq and flags fields of bpf_iter_scx_dsq_kern into the cursor field reducing the struct size by a full u64. No behavior changes intended. Signed-off-by: Tejun Heo <tj@kernel.org> Signed-off-by: Zicheng Qu <quzicheng@huawei.com> --- include/linux/sched/ext.h | 10 +++++++++- kernel/sched/ext.c | 23 ++++++++++++----------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index dc646cca4fcf..1ddbde64a31b 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -119,9 +119,17 @@ enum scx_kf_mask { __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST, }; +enum scx_dsq_lnode_flags { + SCX_DSQ_LNODE_ITER_CURSOR = 1 << 0, + + /* high 16 bits can be for iter cursor flags */ + __SCX_DSQ_LNODE_PRIV_SHIFT = 16, +}; + struct scx_dsq_list_node { struct list_head node; - bool is_bpf_iter_cursor; + u32 flags; + u32 priv; /* can be used by iter cursor */ }; /* diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 50a7b7f5cb4f..a833767c8189 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -1195,7 +1195,7 @@ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, dsq_lnode = container_of(list_node, struct scx_dsq_list_node, node); - } while (dsq_lnode->is_bpf_iter_cursor); + } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); return container_of(dsq_lnode, struct task_struct, scx.dsq_list); } @@ -1213,16 +1213,15 @@ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, */ enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ - SCX_DSQ_ITER_REV = 1U << 0, + SCX_DSQ_ITER_REV = 1U << 16, - __SCX_DSQ_ITER_ALL_FLAGS = SCX_DSQ_ITER_REV, + __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, + __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS, }; struct bpf_iter_scx_dsq_kern { struct scx_dsq_list_node cursor; struct scx_dispatch_q *dsq; - u32 dsq_seq; - u32 flags; } __attribute__((aligned(8))); struct bpf_iter_scx_dsq { @@ -1260,6 +1259,9 @@ static void scx_task_iter_init(struct scx_task_iter *iter) { lockdep_assert_held(&scx_tasks_lock); + BUILD_BUG_ON(__SCX_DSQ_ITER_ALL_FLAGS & + ((1U << __SCX_DSQ_LNODE_PRIV_SHIFT) - 1)); + iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR }; list_add(&iter->cursor.tasks_node, &scx_tasks); iter->locked = NULL; @@ -6293,7 +6295,7 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) != __alignof__(struct bpf_iter_scx_dsq)); - if (flags & ~__SCX_DSQ_ITER_ALL_FLAGS) + if (flags & ~__SCX_DSQ_ITER_USER_FLAGS) return -EINVAL; kit->dsq = find_non_local_dsq(dsq_id); @@ -6301,9 +6303,8 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, return -ENOENT; INIT_LIST_HEAD(&kit->cursor.node); - kit->cursor.is_bpf_iter_cursor = true; - kit->dsq_seq = READ_ONCE(kit->dsq->seq); - kit->flags = flags; + kit->cursor.flags |= SCX_DSQ_LNODE_ITER_CURSOR | flags; + kit->cursor.priv = READ_ONCE(kit->dsq->seq); return 0; } @@ -6317,7 +6318,7 @@ __bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) { struct bpf_iter_scx_dsq_kern *kit = (void *)it; - bool rev = kit->flags & SCX_DSQ_ITER_REV; + bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV; struct task_struct *p; unsigned long flags; @@ -6338,7 +6339,7 @@ __bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *i */ do { p = nldsq_next_task(kit->dsq, p, rev); - } while (p && unlikely(u32_before(kit->dsq_seq, p->scx.dsq_seq))); + } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq))); if (p) { if (rev) -- 2.34.1