[PATCH OLK-6.6 0/6] CVE-2026-89799
Andrii Nakryiko (1): bpf: decouple stack_map_get_build_id_offset() from perf_callchain_entry Jiri Olsa (5): bpf: Factor stackid_init function from __bpf_get_stackid bpf: Factor stackid_fastpath function from __bpf_get_stackid bpf: Factor stackid_new_bucket from __bpf_get_stackid bpf: Use stack id functions instead of __bpf_get_stackid bpf: Disable preemption in bpf_get_stackid kernel/bpf/stackmap.c | 214 ++++++++++++++++++++++++++++-------------- 1 file changed, 146 insertions(+), 68 deletions(-) -- 2.34.1
From: Andrii Nakryiko <andrii@kernel.org> mainline inclusion from mainline-v6.12-rc1 commit 4f4c4fc0153fb11ac40b16c24a24543dc9689d8c category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/19054 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=... -------------------------------- Change stack_map_get_build_id_offset() which is used to convert stack trace IP addresses into build ID+offset pairs. Right now this function accepts an array of u64s as an input, and uses array of struct bpf_stack_build_id as an output. This is problematic because u64 array is coming from perf_callchain_entry, which is (non-sleepable) RCU protected, so once we allows sleepable build ID fetching, this all breaks down. But its actually pretty easy to make stack_map_get_build_id_offset() works with array of struct bpf_stack_build_id as both input and output. Which is what this patch is doing, eliminating the dependency on perf_callchain_entry. We require caller to fill out bpf_stack_build_id.ip fields (all other can be left uninitialized), and update in place as we do build ID resolution. We make sure to READ_ONCE() and cache locally current IP value as we used it in a few places to find matching VMA and so on. Given this data is directly accessible and modifiable by user's BPF code, we should make sure to have a consistent view of it. Reviewed-by: Eduard Zingerman <eddyz87@gmail.com> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/r/20240829174232.3133883-9-andrii@kernel.org Signed-off-by: Alexei Starovoitov <ast@kernel.org> Conflicts: kernel/bpf/stackmap.c [ctx conflicts] Signed-off-by: Pu Lehui <pulehui@huawei.com> --- kernel/bpf/stackmap.c | 49 +++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 2a4070d0b594..4e4a98a3b8e2 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -146,8 +146,18 @@ static struct bpf_map *stack_map_alloc(union bpf_attr *attr) return ERR_PTR(err); } +/* + * Expects all id_offs[i].ip values to be set to correct initial IPs. + * They will be subsequently: + * - either adjusted in place to a file offset, if build ID fetching + * succeeds; in this case id_offs[i].build_id is set to correct build ID, + * and id_offs[i].status is set to BPF_STACK_BUILD_ID_VALID; + * - or IP will be kept intact, if build ID fetching failed; in this case + * id_offs[i].build_id is zeroed out and id_offs[i].status is set to + * BPF_STACK_BUILD_ID_IP. + */ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, - u64 *ips, u32 trace_nr, bool user) + u32 trace_nr, bool user) { int i; struct mmap_unlock_irq_work *work = NULL; @@ -164,30 +174,28 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, /* cannot access current->mm, fall back to ips */ for (i = 0; i < trace_nr; i++) { id_offs[i].status = BPF_STACK_BUILD_ID_IP; - id_offs[i].ip = ips[i]; memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); } return; } for (i = 0; i < trace_nr; i++) { - if (range_in_vma(prev_vma, ips[i], ips[i])) { + u64 ip = READ_ONCE(id_offs[i].ip); + + if (range_in_vma(prev_vma, ip, ip)) { vma = prev_vma; - memcpy(id_offs[i].build_id, prev_build_id, - BUILD_ID_SIZE_MAX); + memcpy(id_offs[i].build_id, prev_build_id, BUILD_ID_SIZE_MAX); goto build_id_valid; } - vma = find_vma(current->mm, ips[i]); + vma = find_vma(current->mm, ip); if (!vma || build_id_parse(vma, id_offs[i].build_id, NULL)) { /* per entry fall back to ips */ id_offs[i].status = BPF_STACK_BUILD_ID_IP; - id_offs[i].ip = ips[i]; memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); continue; } build_id_valid: - id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ips[i] - - vma->vm_start; + id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ip - vma->vm_start; id_offs[i].status = BPF_STACK_BUILD_ID_VALID; prev_vma = vma; prev_build_id = id_offs[i].build_id; @@ -237,7 +245,7 @@ static long __bpf_get_stackid(struct bpf_map *map, { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); struct stack_map_bucket *bucket, *new_bucket, *old_bucket; - u32 hash, id, trace_nr, trace_len, max_depth; + u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; bool user = flags & BPF_F_USER_STACK; u64 *ips; @@ -261,15 +269,18 @@ static long __bpf_get_stackid(struct bpf_map *map, return id; if (stack_map_use_build_id(map)) { + struct bpf_stack_build_id *id_offs; + /* for build_id+offset, pop a bucket before slow cmp */ new_bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); if (unlikely(!new_bucket)) return -ENOMEM; new_bucket->nr = trace_nr; - stack_map_get_build_id_offset( - (struct bpf_stack_build_id *)new_bucket->data, - ips, trace_nr, user); + id_offs = (struct bpf_stack_build_id *)new_bucket->data; + for (i = 0; i < trace_nr; i++) + id_offs[i].ip = ips[i]; + stack_map_get_build_id_offset(id_offs, trace_nr, user); trace_len = trace_nr * sizeof(struct bpf_stack_build_id); if (hash_matches && bucket->nr == trace_nr && memcmp(bucket->data, new_bucket->data, trace_len) == 0) { @@ -463,10 +474,16 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, copy_len = trace_nr * elem_size; ips = trace->ip + skip; - if (user && user_build_id) - stack_map_get_build_id_offset(buf, ips, trace_nr, user); - else + if (user && user_build_id) { + struct bpf_stack_build_id *id_offs = buf; + u32 i; + + for (i = 0; i < trace_nr; i++) + id_offs[i].ip = ips[i]; + stack_map_get_build_id_offset(buf, trace_nr, user); + } else { memcpy(buf, ips, copy_len); + } if (size > copy_len) memset(buf + copy_len, 0, size - copy_len); -- 2.34.1
From: Jiri Olsa <jolsa@kernel.org> mainline inclusion from mainline-v7.3-rc1 commit 15b837759a97237d647962f9943afe0d55af615a category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/19054 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- The new stackid_init function stores all the necessary bits for stackid trace and it will be used by other functions in following changes. Signed-off-by: Jiri Olsa <jolsa@kernel.org> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/bpf/20260803210149.296496-2-jolsa@kernel.org Conflicts: kernel/bpf/stackmap.c [ctx conflicts] Signed-off-by: Pu Lehui <pulehui@huawei.com> --- kernel/bpf/stackmap.c | 95 +++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 4e4a98a3b8e2..dc22bee0aeba 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -240,33 +240,54 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) #endif } -static long __bpf_get_stackid(struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +struct stackid { + struct stack_map_bucket *bucket; + u64 *ips; + u32 nr; + u32 len; + u32 hash; + u32 id; +}; + +static int stackid_init(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *bucket, *new_bucket, *old_bucket; - u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - bool user = flags & BPF_F_USER_STACK; - u64 *ips; - bool hash_matches; + u32 max_depth; if (trace->nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - trace_nr = min_t(u32, trace->nr - skip, max_depth - skip); - trace_len = trace_nr * sizeof(u64); - ips = trace->ip + skip; - hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); - id = hash & (smap->n_buckets - 1); - bucket = READ_ONCE(smap->buckets[id]); + stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip); + stackid->len = stackid->nr * sizeof(u64); + stackid->ips = trace->ip + skip; + stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); + stackid->id = stackid->hash & (smap->n_buckets - 1); + stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + return 0; +} - hash_matches = bucket && bucket->hash == hash; +static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct stack_map_bucket *new_bucket, *old_bucket; + bool user = flags & BPF_F_USER_STACK; + bool hash_matches; + u32 trace_len, i; + int err; + + err = stackid_init(stackid, map, trace, flags); + if (err) + return err; + + hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; /* fast cmp */ if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return id; + return stackid->id; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; @@ -276,42 +297,42 @@ static long __bpf_get_stackid(struct bpf_map *map, pcpu_freelist_pop(&smap->freelist); if (unlikely(!new_bucket)) return -ENOMEM; - new_bucket->nr = trace_nr; + new_bucket->nr = stackid->nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - stack_map_get_build_id_offset(id_offs, trace_nr, user); - trace_len = trace_nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, new_bucket->data, trace_len) == 0) { + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + stack_map_get_build_id_offset(id_offs, stackid->nr, user); + trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); + if (hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); - return id; + return stackid->id; } - if (bucket && !(flags & BPF_F_REUSE_STACKID)) { + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } } else { - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, ips, trace_len) == 0) - return id; - if (bucket && !(flags & BPF_F_REUSE_STACKID)) + if (hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) return -EEXIST; new_bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); if (unlikely(!new_bucket)) return -ENOMEM; - memcpy(new_bucket->data, ips, trace_len); + memcpy(new_bucket->data, stackid->ips, stackid->len); } - new_bucket->hash = hash; - new_bucket->nr = trace_nr; + new_bucket->hash = stackid->hash; + new_bucket->nr = stackid->nr; - old_bucket = xchg(&smap->buckets[id], new_bucket); + old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); - return id; + return stackid->id; } BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, @@ -320,6 +341,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel = !user; u32 max_depth; @@ -335,7 +357,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, /* couldn't fetch the stack trace */ return -EFAULT; - return __bpf_get_stackid(map, trace, flags); + return __bpf_get_stackid(&stackid, map, trace, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -364,6 +386,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, { struct perf_event *event = ctx->event; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel, user; __u64 nr_kernel, nr; int ret; @@ -389,7 +412,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, if (kernel) { trace->nr = nr_kernel; - ret = __bpf_get_stackid(map, trace, flags); + ret = __bpf_get_stackid(&stackid, map, trace, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -398,7 +421,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(map, trace, flags); + ret = __bpf_get_stackid(&stackid, map, trace, flags); } /* restore nr */ -- 2.34.1
From: Jiri Olsa <jolsa@kernel.org> mainline inclusion from mainline-v7.3-rc1 commit 0ca56befcffec3a6c9d1842eae06c74e1cf41f11 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/19054 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- The new stackid_fastpath does the fast stack hash and trace check, that does not need new bucket allocation. It covers both just-ip and buildid code paths. Signed-off-by: Jiri Olsa <jolsa@kernel.org> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/bpf/20260803210149.296496-3-jolsa@kernel.org Conflicts: kernel/bpf/stackmap.c [ctx conflicts] Signed-off-by: Pu Lehui <pulehui@huawei.com> --- kernel/bpf/stackmap.c | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index dc22bee0aeba..6d565f35e13f 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -247,6 +247,7 @@ struct stackid { u32 len; u32 hash; u32 id; + bool hash_matches; }; static int stackid_init(struct stackid *stackid, struct bpf_map *map, @@ -267,28 +268,46 @@ static int stackid_init(struct stackid *stackid, struct bpf_map *map, stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); stackid->id = stackid->hash & (smap->n_buckets - 1); stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; return 0; } +static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) +{ + int err; + + err = stackid_init(stackid, map, trace, flags); + if (err) + return err; + + /* fast cmp */ + if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP) + return stackid->id; + + if (stack_map_use_build_id(map)) + return -ENOENT; + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) + return -EEXIST; + return -ENOENT; +} + static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); struct stack_map_bucket *new_bucket, *old_bucket; bool user = flags & BPF_F_USER_STACK; - bool hash_matches; u32 trace_len, i; int err; - err = stackid_init(stackid, map, trace, flags); - if (err) + err = stackid_fastpath(stackid, map, trace, flags); + if (err != -ENOENT) return err; - hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; - /* fast cmp */ - if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return stackid->id; - if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; @@ -303,7 +322,7 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, id_offs[i].ip = stackid->ips[i]; stack_map_get_build_id_offset(id_offs, stackid->nr, user); trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && stackid->bucket->nr == stackid->nr && + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return stackid->id; @@ -313,12 +332,6 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, return -EEXIST; } } else { - if (hash_matches && stackid->bucket->nr == stackid->nr && - memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) - return stackid->id; - if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) - return -EEXIST; - new_bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); if (unlikely(!new_bucket)) -- 2.34.1
From: Jiri Olsa <jolsa@kernel.org> mainline inclusion from mainline-v7.3-rc1 commit bb4e6f4e1b68fe60c04ca04c564c6624e837dbf4 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/19054 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- The new stackid_new_bucket allocates the new bucket and initializes it with the trace data. Signed-off-by: Jiri Olsa <jolsa@kernel.org> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/bpf/20260803210149.296496-4-jolsa@kernel.org Conflicts: kernel/bpf/stackmap.c [ctx conflicts] Signed-off-by: Pu Lehui <pulehui@huawei.com> --- kernel/bpf/stackmap.c | 48 +++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 6d565f35e13f..58167405918b 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -295,31 +295,52 @@ static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, return -ENOENT; } +static struct stack_map_bucket * +stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct bpf_stack_build_id *id_offs; + struct stack_map_bucket *bucket; + u32 i; + + bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); + if (unlikely(!bucket)) + return NULL; + + if (stack_map_use_build_id(map)) { + id_offs = (struct bpf_stack_build_id *)bucket->data; + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + } else { + memcpy(bucket->data, stackid->ips, stackid->len); + } + + bucket->hash = stackid->hash; + bucket->nr = stackid->nr; + return bucket; +} + static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); struct stack_map_bucket *new_bucket, *old_bucket; bool user = flags & BPF_F_USER_STACK; - u32 trace_len, i; + u32 trace_len; int err; err = stackid_fastpath(stackid, map, trace, flags); if (err != -ENOENT) return err; + new_bucket = stackid_new_bucket(stackid, map); + if (!new_bucket) + return -ENOMEM; + if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; - /* for build_id+offset, pop a bucket before slow cmp */ - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - new_bucket->nr = stackid->nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < stackid->nr; i++) - id_offs[i].ip = stackid->ips[i]; stack_map_get_build_id_offset(id_offs, stackid->nr, user); trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && @@ -331,17 +352,8 @@ static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } - } else { - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - memcpy(new_bucket->data, stackid->ips, stackid->len); } - new_bucket->hash = stackid->hash; - new_bucket->nr = stackid->nr; - old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); -- 2.34.1
From: Jiri Olsa <jolsa@kernel.org> mainline inclusion from mainline-v7.3-rc1 commit 09b3fd6caa0b57f8a39254ee5db3af30bdd53c18 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/19054 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- Replacing __bpf_get_stackid calls with sequence of following functions: stackid_fastpath stackid_new_bucket stackid_install This makes code more structured and allows us to easily disable preemption only in bpf_get_stackid in following changes. Signed-off-by: Jiri Olsa <jolsa@kernel.org> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/bpf/20260803210149.296496-5-jolsa@kernel.org Signed-off-by: Pu Lehui <pulehui@huawei.com> --- kernel/bpf/stackmap.c | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 58167405918b..ffb19791f632 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -320,22 +320,13 @@ stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) return bucket; } -static long __bpf_get_stackid(struct stackid *stackid, struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +static long stackid_install(struct stackid *stackid, struct bpf_map *map, + struct stack_map_bucket *new_bucket, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *new_bucket, *old_bucket; bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *old_bucket; u32 trace_len; - int err; - - err = stackid_fastpath(stackid, map, trace, flags); - if (err != -ENOENT) - return err; - - new_bucket = stackid_new_bucket(stackid, map); - if (!new_bucket) - return -ENOMEM; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; @@ -365,10 +356,12 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, { u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; struct stackid stackid; bool kernel = !user; u32 max_depth; + int err; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) @@ -382,7 +375,15 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, /* couldn't fetch the stack trace */ return -EFAULT; - return __bpf_get_stackid(&stackid, map, trace, flags); + err = stackid_fastpath(&stackid, map, trace, flags); + if (err != -ENOENT) + return err; + + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + + return stackid_install(&stackid, map, new_bucket, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -410,6 +411,7 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, struct bpf_map *, map, u64, flags) { struct perf_event *event = ctx->event; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; struct stackid stackid; bool kernel, user; @@ -437,7 +439,6 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, if (kernel) { trace->nr = nr_kernel; - ret = __bpf_get_stackid(&stackid, map, trace, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -446,12 +447,22 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(&stackid, map, trace, flags); } + ret = stackid_fastpath(&stackid, map, trace, flags); + if (ret != -ENOENT) + goto out; + + new_bucket = stackid_new_bucket(&stackid, map); + if (new_bucket) { + trace->nr = nr; + return stackid_install(&stackid, map, new_bucket, flags); + } + ret = -ENOMEM; + +out: /* restore nr */ trace->nr = nr; - return ret; } -- 2.34.1
From: Jiri Olsa <jolsa@kernel.org> mainline inclusion from mainline-v7.3-rc1 commit 15f1bd8574662f1b7b26aaa2e23ebf4066f0117d category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/19054 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- The get_perf_callchain call needs disabled preemption plus we need it disabled as long as we access its returned trace entries buffer. Note the bpf_get_stackid_pe function is executed already with preemption disabled. Fixes: d5a3b1f69186 ("bpf: introduce BPF_MAP_TYPE_STACK_TRACE") Reported-by: Tao Chen <chen.dylane@linux.dev> Signed-off-by: Jiri Olsa <jolsa@kernel.org> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Cc: stable@vger.kernel.org Link: https://lore.kernel.org/bpf/20260803210149.296496-6-jolsa@kernel.org Closes: https://lore.kernel.org/bpf/20260206090653.1336687-2-chen.dylane@linux.dev/ Conflicts: kernel/bpf/stackmap.c [ctx conflicts] Signed-off-by: Pu Lehui <pulehui@huawei.com> --- kernel/bpf/stackmap.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index ffb19791f632..12bb2f66f03a 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -368,20 +368,22 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, return -EINVAL; max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags); - trace = get_perf_callchain(regs, 0, kernel, user, max_depth, - false, false); - if (unlikely(!trace)) - /* couldn't fetch the stack trace */ - return -EFAULT; + scoped_guard(preempt) { + trace = get_perf_callchain(regs, 0, kernel, user, max_depth, + false, false); + if (unlikely(!trace)) + /* couldn't fetch the stack trace */ + return -EFAULT; - err = stackid_fastpath(&stackid, map, trace, flags); - if (err != -ENOENT) - return err; + err = stackid_fastpath(&stackid, map, trace, flags); + if (err != -ENOENT) + return err; - new_bucket = stackid_new_bucket(&stackid, map); - if (!new_bucket) - return -ENOMEM; + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + } return stackid_install(&stackid, map, new_bucket, flags); } -- 2.34.1
反馈： 您发送到kernel@openeuler.org的补丁/补丁集，已成功转换为PR！ PR链接地址： https://atomgit.com/openeuler/kernel/merge_requests/28526 邮件列表地址：https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/DVE... FeedBack: The patch(es) which you have sent to kernel@openeuler.org mailing list has been converted to a pull request successfully! Pull request link: https://atomgit.com/openeuler/kernel/merge_requests/28526 Mailing list address: https://mailweb.openeuler.org/archives/list/kernel@openeuler.org/message/DVE...
participants (2)
-
patchwork bot -
Pu Lehui