From: Wanpeng Li wanpengli@tencent.com

mainline inclusion from mainline-v5.13-rc6 commit e898da784aed0ea65f7672d941c01dc9b79e6299 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4NP0K CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

-------------------

According to the SDM 10.5.4.1:

A write of 0 to the initial-count register effectively stops the local APIC timer, in both one-shot and periodic mode.

However, the lapic timer oneshot/periodic mode which is emulated by vmx-preemption timer doesn't stop by writing 0 to TMICT since vmx->hv_deadline_tsc is still programmed and the guest will receive the spurious timer interrupt later. This patch fixes it by also cancelling the vmx-preemption timer when writing 0 to the initial-count register.

Reviewed-by: Sean Christopherson seanjc@google.com Signed-off-by: Wanpeng Li wanpengli@tencent.com Message-Id: 1623050385-100988-1-git-send-email-wanpengli@tencent.com Signed-off-by: Paolo Bonzini pbonzini@redhat.com Signed-off-by: Jingyi Wang wangjingyi11@huawei.com Reviewed-by: Zenghui Yu yuzenghui@huawei.com Reviewed-by: Wei Li liwei391@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/x86/kvm/lapic.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 677d21082454..94e75c941b47 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1492,6 +1492,15 @@ static void limit_periodic_timer_frequency(struct kvm_lapic *apic)

static void cancel_hv_timer(struct kvm_lapic *apic);

+static void cancel_apic_timer(struct kvm_lapic *apic) +{ + hrtimer_cancel(&apic->lapic_timer.timer); + preempt_disable(); + if (apic->lapic_timer.hv_timer_in_use) + cancel_hv_timer(apic); + preempt_enable(); +} + static void apic_update_lvtt(struct kvm_lapic *apic) { u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) & @@ -1500,11 +1509,7 @@ static void apic_update_lvtt(struct kvm_lapic *apic) if (apic->lapic_timer.timer_mode != timer_mode) { if (apic_lvtt_tscdeadline(apic) != (timer_mode == APIC_LVT_TIMER_TSCDEADLINE)) { - hrtimer_cancel(&apic->lapic_timer.timer); - preempt_disable(); - if (apic->lapic_timer.hv_timer_in_use) - cancel_hv_timer(apic); - preempt_enable(); + cancel_apic_timer(apic); kvm_lapic_set_reg(apic, APIC_TMICT, 0); apic->lapic_timer.period = 0; apic->lapic_timer.tscdeadline = 0; @@ -2082,7 +2087,7 @@ int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val) if (apic_lvtt_tscdeadline(apic)) break;

- hrtimer_cancel(&apic->lapic_timer.timer); + cancel_apic_timer(apic); kvm_lapic_set_reg(apic, APIC_TMICT, val); start_apic_timer(apic); break;

From: Wanpeng Li wanpengli@tencent.com

mainline inclusion from mainline-v5.14-rc1 commit 2735886c9ef115fc7b40d27bfe73605c38e9d56b category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4NP0K CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

-------------------

KVM_GET_LAPIC stores the current value of TMCCT and KVM_SET_LAPIC's memcpy stores it in vcpu->arch.apic->regs, KVM_SET_LAPIC could store zero in vcpu->arch.apic->regs after it uses it, and then the stored value would always be zero. In addition, the TMCCT is always computed on-demand and never directly readable.

Suggested-by: Paolo Bonzini pbonzini@redhat.com Signed-off-by: Wanpeng Li wanpengli@tencent.com Message-Id: 1623223000-18116-1-git-send-email-wanpengli@tencent.com Signed-off-by: Paolo Bonzini pbonzini@redhat.com Signed-off-by: Jingyi Wang wangjingyi11@huawei.com Reviewed-by: Zenghui Yu yuzenghui@huawei.com Reviewed-by: Wei Li liwei391@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/x86/kvm/lapic.c | 1 + 1 file changed, 1 insertion(+)

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 94e75c941b47..70dcb723a0f9 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2615,6 +2615,7 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); update_divide_count(apic); __start_apic_timer(apic, APIC_TMCCT); + kvm_lapic_set_reg(apic, APIC_TMCCT, 0); kvm_apic_update_apicv(vcpu); apic->highest_isr_cache = -1; if (vcpu->arch.apicv_active) {

From: Alexandru Elisei alexandru.elisei@arm.com

mainline inclusion from mainline-v5.11-rc1 commit 9bbfa4b565379eeb2fb8fdbcc9979549ae0e48d9 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4NP0K CVE: NA

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...

-------------------

When enabling the PMU in kvm_arm_pmu_v3_enable(), KVM returns early if the PMU flag created is false and skips any other checks. Because PMU emulation is gated only on the VCPU feature being set, this makes it possible for userspace to get away with setting the VCPU feature but not doing any initialization for the PMU. Fix it by returning an error when trying to run the VCPU if the PMU hasn't been initialized correctly.

The PMU is marked as created only if the interrupt ID has been set when using an in-kernel irqchip. This means the same check in kvm_arm_pmu_v3_enable() is redundant, remove it.

Signed-off-by: Alexandru Elisei alexandru.elisei@arm.com Signed-off-by: Marc Zyngier maz@kernel.org Link: https://lore.kernel.org/r/20201126144916.164075-1-alexandru.elisei@arm.com Signed-off-by: Jingyi Wang wangjingyi11@huawei.com Reviewed-by: Zenghui Yu yuzenghui@huawei.com Reviewed-by: Wei Li liwei391@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/kvm/pmu-emul.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index a2a20f358f13..00bf8733e718 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -830,9 +830,12 @@ bool kvm_arm_support_pmu_v3(void)

int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) { - if (!vcpu->arch.pmu.created) + if (!kvm_vcpu_has_pmu(vcpu)) return 0;

+ if (!vcpu->arch.pmu.created) + return -EINVAL; + /* * A valid interrupt configuration for the PMU is either to have a * properly configured interrupt number and using an in-kernel @@ -840,9 +843,6 @@ int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) */ if (irqchip_in_kernel(vcpu->kvm)) { int irq = vcpu->arch.pmu.irq_num; - if (!kvm_arm_pmu_irq_initialized(vcpu)) - return -EINVAL; - /* * If we are using an in-kernel vgic, at this point we know * the vgic will be initialized, so we can check the PMU irq

From: Xingang Wang wangxingang5@huawei.com

stable inclusion category: feature from stable-5.13-rc1 commit 18d731242d5c67c0783126c42d3f85870cec2df5 bugzilla: https://gitee.com/openeuler/kernel/issues/I4NR4D

Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=...

-------------------------------------------------

This can fail, and seems to be a popular target for syzkaller error injection. Check the error return and unwind with put_device().

Fixes: 7b96953bc640 ("vfio: Mediated device Core driver") Reviewed-by: Christoph Hellwig hch@lst.de Reviewed-by: Kevin Tian kevin.tian@intel.com Reviewed-by: Max Gurtovoy mgurtovoy@nvidia.com Reviewed-by: Cornelia Huck cohuck@redhat.com Signed-off-by: Jason Gunthorpe jgg@nvidia.com Message-Id: 9-v2-d36939638fc6+d54-vfio2_jgg@nvidia.com Signed-off-by: Alex Williamson alex.williamson@redhat.com Signed-off-by: Xingang Wang wangxingang5@huawei.com Reviewed-by: Xu Xiaoyang xuxiaoyang2@huawei.com Reviewed-by: Christoph Hellwig hch@lst.de Reviewed-by: Kevin Tian kevin.tian@intel.com Reviewed-by: Max Gurtovoy mgurtovoy@nvidia.com Reviewed-by: Cornelia Huck cohuck@redhat.com Signed-off-by: Jason Gunthorpe jgg@nvidia.com Signed-off-by: Alex Williamson alex.williamson@redhat.com Signed-off-by: Xingang Wang wangxingang5@huawei.com Reviewed-by: Xu Xiaoyang xuxiaoyang2@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/vfio/mdev/mdev_core.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/drivers/vfio/mdev/mdev_core.c b/drivers/vfio/mdev/mdev_core.c index b558d4cfd082..5199e6c0467e 100644 --- a/drivers/vfio/mdev/mdev_core.c +++ b/drivers/vfio/mdev/mdev_core.c @@ -316,7 +316,10 @@ int mdev_device_create(struct kobject *kobj, mdev->dev.parent = dev; mdev->dev.bus = &mdev_bus_type; mdev->dev.release = mdev_device_release; - dev_set_name(&mdev->dev, "%pUl", uuid); + ret = dev_set_name(&mdev->dev, "%pUl", uuid); + if (ret) + goto ops_create_fail; + mdev->dev.groups = parent->ops->mdev_attr_groups; mdev->type_kobj = kobj;

From: Wang ShaoBo bobo.shaobowang@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4LMMF CVE: NA

-------------------------------------------------

This adds tips when rmid modification failed.

Fixes: a85aba6a1d67 ("mpam: Add support for group rmid modify") Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Reviewed-by: Xiongfeng Wang wangxiongfeng2@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/kernel/mpam/mpam_resctrl.c | 3 +++ 1 file changed, 3 insertions(+)

diff --git a/arch/arm64/kernel/mpam/mpam_resctrl.c b/arch/arm64/kernel/mpam/mpam_resctrl.c index a9b99a0f347f..27b59c0fbf5a 100644 --- a/arch/arm64/kernel/mpam/mpam_resctrl.c +++ b/arch/arm64/kernel/mpam/mpam_resctrl.c @@ -1848,12 +1848,14 @@ static ssize_t resctrl_group_rmid_write(struct kernfs_open_file *of,

if (rmid == 0 || rdtgrp->mon.rmid == 0) { ret = -EINVAL; + rdt_last_cmd_puts("default rmid 0 is always kept\n"); goto unlock; }

ret = rmid_to_partid_pmg(rmid, &partid, &pmg); if (ret < 0) { ret = -EINVAL; + rdt_last_cmd_puts("invalid rmid\n"); goto unlock; }

@@ -1862,6 +1864,7 @@ static ssize_t resctrl_group_rmid_write(struct kernfs_open_file *of,

if (rdtgrp->type != RDTCTRL_GROUP || !list_empty(&rdtgrp->mon.crdtgrp_list)) { + ret = -EINVAL; rdt_last_cmd_puts("unsupported operation\n"); goto unlock; }

From: Li Zefan lizefan@huawei.com

euler inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4OPKC CVE: NA

-------------------------------------------------

Signed-off-by: Pavel Emelyanov xemul@parallels.com Signed-off-by: Li Zefan lizefan@huawei.com Signed-off-by: luojiajun luojiajun3@huawei.com Reviewed-by: Li Zefan lizefan@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com Signed-off-by: Cui GaoSheng cuigaosheng1@huawei.com Reviewed-by: weiyang wang wangweiyang2@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/pid.h | 3 --- include/linux/pid_namespace.h | 1 + kernel/pid.c | 45 ++++++++++++++++++++++++++++++----- kernel/pid_namespace.c | 6 +++-- kernel/sysctl.c | 9 ------- kernel/trace/trace.c | 4 ++-- kernel/trace/trace.h | 2 -- 7 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/include/linux/pid.h b/include/linux/pid.h index fa10acb8d6a4..34afff2dc888 100644 --- a/include/linux/pid.h +++ b/include/linux/pid.h @@ -110,9 +110,6 @@ extern void transfer_pid(struct task_struct *old, struct task_struct *new, struct pid_namespace; extern struct pid_namespace init_pid_ns;

-extern int pid_max; -extern int pid_max_min, pid_max_max; - /* * look up a PID in the hash table. Must be called with the tasklist_lock * or rcu_read_lock() held. diff --git a/include/linux/pid_namespace.h b/include/linux/pid_namespace.h index 5a5cb45ac57e..b202733aa6cc 100644 --- a/include/linux/pid_namespace.h +++ b/include/linux/pid_namespace.h @@ -31,6 +31,7 @@ struct pid_namespace { #endif struct user_namespace *user_ns; struct ucounts *ucounts; + int pid_max; int reboot; /* group exit code if this pidns was rebooted */ struct ns_common ns; } __randomize_layout; diff --git a/kernel/pid.c b/kernel/pid.c index be2ec1d26896..28fdf3dc1005 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -43,6 +43,7 @@ #include <linux/sched/task.h> #include <linux/idr.h> #include <net/sock.h> +#include <linux/kmemleak.h> #include <uapi/linux/pidfd.h> #ifdef CONFIG_PID_RESERVE #include <linux/pin_mem.h> @@ -62,12 +63,10 @@ struct pid init_struct_pid = { }, } };

-int pid_max = PID_MAX_DEFAULT; - #define RESERVED_PIDS 300

-int pid_max_min = RESERVED_PIDS + 1; -int pid_max_max = PID_MAX_LIMIT; +static int pid_max_min = RESERVED_PIDS + 1; +static int pid_max_max = PID_MAX_LIMIT;

/* * PID-map pages start out as NULL, they get allocated upon @@ -83,6 +82,7 @@ struct pid_namespace init_pid_ns = { .child_reaper = &init_task, .user_ns = &init_user_ns, .ns.inum = PROC_PID_INIT_INO, + .pid_max = PID_MAX_DEFAULT, #ifdef CONFIG_PID_NS .ns.ops = &pidns_operations, #endif @@ -194,7 +194,7 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid, tid = set_tid[ns->level - i];

retval = -EINVAL; - if (tid < 1 || tid >= pid_max) + if (tid < 1 || tid >= task_active_pid_ns(current)->pid_max) goto out_free; /* * Also fail if a PID != 1 is requested and @@ -237,7 +237,7 @@ struct pid *alloc_pid(struct pid_namespace *ns, pid_t *set_tid, * a partially initialized PID (see below). */ nr = idr_alloc_cyclic(&tmp->idr, NULL, pid_min, - pid_max, GFP_ATOMIC); + tmp->pid_max, GFP_ATOMIC); } spin_unlock_irq(&pidmap_lock); idr_preload_end(); @@ -612,8 +612,37 @@ SYSCALL_DEFINE2(pidfd_open, pid_t, pid, unsigned int, flags) return fd; }

+static int proc_dointvec_pidmax(struct ctl_table *table, int write, + void __user *buffer, size_t *lenp, loff_t *ppos) +{ + struct ctl_table tmp; + + tmp = *table; + tmp.data = &task_active_pid_ns(current)->pid_max; + + return proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); +} + +static struct ctl_table pid_ctl_table[] = { + { + .procname = "pid_max", + .data = &init_pid_ns.pid_max, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_pidmax, + .extra1 = &pid_max_min, + .extra2 = &pid_max_max, + }, + {} +}; + +static struct ctl_path pid_kern_path[] = { { .procname = "kernel" }, {} }; + void __init pid_idr_init(void) { + struct ctl_table_header *hdr; + int pid_max = init_pid_ns.pid_max; + /* Verify no one has done anything silly: */ BUILD_BUG_ON(PID_MAX_LIMIT >= PIDNS_ADDING);

@@ -624,6 +653,8 @@ void __init pid_idr_init(void) PIDS_PER_CPU_MIN * num_possible_cpus()); pr_info("pid_max: default: %u minimum: %u\n", pid_max, pid_max_min);

+ init_pid_ns.pid_max = pid_max; + idr_init(&init_pid_ns.idr);

init_pid_ns.pid_cachep = KMEM_CACHE(pid, @@ -632,6 +663,8 @@ void __init pid_idr_init(void) if (is_need_reserve_pids()) reserve_pids(&init_pid_ns.idr, pid_max); #endif + hdr = register_sysctl_paths(pid_kern_path, pid_ctl_table); + kmemleak_not_leak(hdr); }

static struct file *__pidfd_fget(struct task_struct *task, int fd) diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c index 52c017feabcb..c290d21b6c24 100644 --- a/kernel/pid_namespace.c +++ b/kernel/pid_namespace.c @@ -109,6 +109,7 @@ static struct pid_namespace *create_pid_namespace(struct user_namespace *user_ns ns->user_ns = get_user_ns(user_ns); ns->ucounts = ucounts; ns->pid_allocated = PIDNS_ADDING; + ns->pid_max = parent_pid_ns->pid_max;

return ns;

@@ -282,6 +283,8 @@ static int pid_ns_ctl_handler(struct ctl_table *table, int write, next = idr_get_cursor(&pid_ns->idr) - 1;

tmp.data = &next; + tmp.extra2 = &pid_ns->pid_max; + ret = proc_dointvec_minmax(&tmp, write, buffer, lenp, ppos); if (!ret && write) idr_set_cursor(&pid_ns->idr, next + 1); @@ -289,7 +292,6 @@ static int pid_ns_ctl_handler(struct ctl_table *table, int write, return ret; }

-extern int pid_max; static struct ctl_table pid_ns_ctl_table[] = { { .procname = "ns_last_pid", @@ -297,7 +299,7 @@ static struct ctl_table pid_ns_ctl_table[] = { .mode = 0666, /* permissions are checked in the handler */ .proc_handler = pid_ns_ctl_handler, .extra1 = SYSCTL_ZERO, - .extra2 = &pid_max, + .extra2 = &init_pid_ns.pid_max, }, { } }; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 749ef59224e2..6eb2b1e88f0c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2231,15 +2231,6 @@ static struct ctl_table kern_table[] = { .extra2 = SYSCTL_ONE, }, #endif /* CONFIG_SMP */ - { - .procname = "pid_max", - .data = &pid_max, - .maxlen = sizeof (int), - .mode = 0644, - .proc_handler = proc_dointvec_minmax, - .extra1 = &pid_max_min, - .extra2 = &pid_max_max, - }, { .procname = "panic_on_oops", .data = &panic_on_oops, diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index e4f154119e52..2ce366687ce4 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -693,7 +693,7 @@ int trace_pid_write(struct trace_pid_list *filtered_pids, return -ENOMEM; }

- pid_list->pid_max = READ_ONCE(pid_max); + pid_list->pid_max = READ_ONCE(init_pid_ns.pid_max);

/* Only truncating will shrink pid_max */ if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max) @@ -4896,7 +4896,7 @@ int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)

if (mask == TRACE_ITER_RECORD_TGID) { if (!tgid_map) { - tgid_map_max = pid_max; + tgid_map_max = init_pid_ns.pid_max; map = kvcalloc(tgid_map_max + 1, sizeof(*tgid_map), GFP_KERNEL);

diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 15a811d34cd8..94f8087a3c22 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -806,8 +806,6 @@ extern unsigned long tracing_thresh;

/* PID filtering */

-extern int pid_max; - bool trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid); bool trace_ignore_this_task(struct trace_pid_list *filtered_pids,

From: Kefeng Wang wangkefeng.wang@huawei.com

maillist inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

Reference: https://lore.kernel.org/lkml/20211226083912.166512-4-wangkefeng.wang@huawei....

-------------------

This patch select HAVE_ARCH_HUGE_VMALLOC to let arm64 support huge vmalloc mappings, it is disabled by default, use hugevmalloc=on to enable it in some scenarios.

Cc: Catalin Marinas catalin.marinas@arm.com Cc: Will Deacon will@kernel.org Signed-off-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- Documentation/admin-guide/kernel-parameters.txt | 4 ++-- arch/arm64/Kconfig | 1 + arch/arm64/kernel/module.c | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a84f62b61dbc..5eaa1b8a6c6e 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1587,7 +1587,7 @@ on: enable the feature off: disable the feature

- hugevmalloc [PPC] Requires CONFIG_HAVE_ARCH_HUGE_VMALLOC + hugevmalloc [KNL,PPC,ARM64] Requires CONFIG_HAVE_ARCH_HUGE_VMALLOC Format: { on | off } Default set by CONFIG_HUGE_VMALLOC_DEFAULT_ENABLED.

@@ -3312,7 +3312,7 @@

nohugeiomap [KNL,X86,PPC,ARM64] Disable kernel huge I/O mappings.

- nohugevmalloc [PPC] Disable kernel huge vmalloc mappings. + nohugevmalloc [KNL,PPC,ARM64] Disable kernel huge vmalloc mappings.

nosmt [KNL,S390] Disable symmetric multithreading (SMT). Equivalent to smt=1. diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 44f1bf1a5b08..df6176b55fc8 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -133,6 +133,7 @@ config ARM64 select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_BITREVERSE select HAVE_ARCH_COMPILER_H + select HAVE_ARCH_HUGE_VMALLOC select HAVE_ARCH_HUGE_VMAP select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c index 2a1ad95d9b2c..031be3c6a9d5 100644 --- a/arch/arm64/kernel/module.c +++ b/arch/arm64/kernel/module.c @@ -35,7 +35,7 @@ void *module_alloc(unsigned long size) module_alloc_end = MODULES_END;

p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, - module_alloc_end, gfp_mask, PAGE_KERNEL, 0, + module_alloc_end, gfp_mask, PAGE_KERNEL, VM_NO_HUGE_VMAP, NUMA_NO_NODE, __builtin_return_address(0));

if (!p && IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && @@ -51,7 +51,7 @@ void *module_alloc(unsigned long size) */ p = __vmalloc_node_range(size, MODULE_ALIGN, module_alloc_base, module_alloc_base + SZ_2G, GFP_KERNEL, - PAGE_KERNEL, 0, NUMA_NO_NODE, + PAGE_KERNEL, VM_NO_HUGE_VMAP, NUMA_NO_NODE, __builtin_return_address(0));

if (p && (kasan_module_alloc(p, size) < 0)) {

From: Weilong Chen chenweilong@huawei.com

ascend inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4K2U5 CVE: NA

-------------------------------------------------

Support disable oom-killer, and report oom events to bbox vm.enable_oom_killer: 0: disable oom killer 1: enable oom killer (default,compatible with mainline)

Signed-off-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zhang Jian zhangjian210@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/Kconfig | 12 ++++++++++++ include/linux/oom.h | 26 ++++++++++++++++++++++++++ kernel/sysctl.c | 11 +++++++++++ mm/memcontrol.c | 5 ++++- mm/oom_kill.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ mm/util.c | 2 ++ 6 files changed, 100 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index df6176b55fc8..3afabc81551c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2010,6 +2010,18 @@ config ASCEND_CLEAN_CDM cause the kernel crash if the essential structures went wrong. So move the management structures for hbm nodes to the ddr nodes of the same partion to reduce the probability of kernel crashes. + +config ASCEND_OOM + bool "Enable support for disable oom killer" + default y + help + In some cases we hopes that the oom will not kill the process when it occurs, + be able to notify the black box to report the event, and be able to trigger + the panic to locate the problem. + vm.enable_oom_killer: + 0: disable oom killer + 1: enable oom killer (default,compatible with mainline) + endif

endmenu diff --git a/include/linux/oom.h b/include/linux/oom.h index 2db9a1432511..d57de7b1e75a 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -127,4 +127,30 @@ extern struct task_struct *find_lock_task_mm(struct task_struct *p); extern int sysctl_oom_dump_tasks; extern int sysctl_oom_kill_allocating_task; extern int sysctl_panic_on_oom; +extern int sysctl_enable_oom_killer; + +#define OOM_TYPE_NOMEM 0 +#define OOM_TYPE_OVERCOMMIT 1 +#define OOM_TYPE_CGROUP 2 + +#ifdef CONFIG_ASCEND_OOM +extern int register_hisi_oom_notifier(struct notifier_block *nb); +extern int unregister_hisi_oom_notifier(struct notifier_block *nb); +int oom_type_notifier_call(unsigned int type, struct oom_control *oc); +#else +static inline int register_hisi_oom_notifier(struct notifier_block *nb) +{ + return -EINVAL; +} + +static inline int unregister_hisi_oom_notifier(struct notifier_block *nb) +{ + return -EINVAL; +} + +static inline int oom_type_notifier_call(unsigned int type, struct oom_control *oc) +{ + return -EINVAL; +} +#endif #endif /* _INCLUDE_LINUX_OOM_H */ diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 6eb2b1e88f0c..ce200213ccbd 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -2761,6 +2761,17 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &two, }, +#ifdef CONFIG_ASCEND_OOM + { + .procname = "enable_oom_killer", + .data = &sysctl_enable_oom_killer, + .maxlen = sizeof(sysctl_enable_oom_killer), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif { .procname = "oom_kill_allocating_task", .data = &sysctl_oom_kill_allocating_task, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b5ba2dd2d468..99cfd840e3bd 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1895,6 +1895,7 @@ static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int current->memcg_in_oom = memcg; current->memcg_oom_gfp_mask = mask; current->memcg_oom_order = order; + oom_type_notifier_call(OOM_TYPE_CGROUP, NULL);

return OOM_ASYNC; } @@ -1962,12 +1963,14 @@ bool mem_cgroup_oom_synchronize(bool handle) if (locked) mem_cgroup_oom_notify(memcg);

- if (locked && !memcg->oom_kill_disable) { + if (locked && !memcg->oom_kill_disable && sysctl_enable_oom_killer) { mem_cgroup_unmark_under_oom(memcg); finish_wait(&memcg_oom_waitq, &owait.wait); mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask, current->memcg_oom_order); } else { + oom_type_notifier_call(OOM_TYPE_CGROUP, NULL); + schedule(); mem_cgroup_unmark_under_oom(memcg); finish_wait(&memcg_oom_waitq, &owait.wait); diff --git a/mm/oom_kill.c b/mm/oom_kill.c index 57300369efe3..ffbe8fe2bbf6 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -54,6 +54,7 @@ int sysctl_panic_on_oom; int sysctl_oom_kill_allocating_task; int sysctl_oom_dump_tasks = 1; +int sysctl_enable_oom_killer = 1;

/* * Serializes oom killer invocations (out_of_memory()) from all contexts to @@ -1081,6 +1082,45 @@ int unregister_oom_notifier(struct notifier_block *nb) } EXPORT_SYMBOL_GPL(unregister_oom_notifier);

+#ifdef CONFIG_ASCEND_OOM +static BLOCKING_NOTIFIER_HEAD(oom_type_notify_list); + +int register_hisi_oom_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_register(&oom_type_notify_list, nb); +} +EXPORT_SYMBOL_GPL(register_hisi_oom_notifier); + +int unregister_hisi_oom_notifier(struct notifier_block *nb) +{ + return blocking_notifier_chain_unregister(&oom_type_notify_list, nb); +} +EXPORT_SYMBOL_GPL(unregister_hisi_oom_notifier); + +int oom_type_notifier_call(unsigned int type, struct oom_control *oc) +{ + struct oom_control oc_tmp = { 0 }; + static unsigned long caller_jiffies; + + if (sysctl_enable_oom_killer) + return -EINVAL; + + if (oc) + type = is_memcg_oom(oc) ? OOM_TYPE_CGROUP : OOM_TYPE_NOMEM; + else + oc = &oc_tmp; + + if (printk_timed_ratelimit(&caller_jiffies, 10000)) { + pr_err("OOM_NOTIFIER: oom type %u\n", type); + dump_stack(); + show_mem(SHOW_MEM_FILTER_NODES, NULL); + dump_tasks(oc); + } + + return blocking_notifier_call_chain(&oom_type_notify_list, type, NULL); +} +#endif + /** * out_of_memory - kill the "best" process when we run out of memory * @oc: pointer to struct oom_control @@ -1097,6 +1137,11 @@ bool out_of_memory(struct oom_control *oc) if (oom_killer_disabled) return false;

+ if (!sysctl_enable_oom_killer) { + oom_type_notifier_call(0, oc); + return false; + } + if (!is_memcg_oom(oc)) { blocking_notifier_call_chain(&oom_notify_list, 0, &freed); if (freed > 0) diff --git a/mm/util.c b/mm/util.c index 47d074912630..2350c064abc6 100644 --- a/mm/util.c +++ b/mm/util.c @@ -25,6 +25,7 @@ #include <linux/compat.h>

#include <linux/uaccess.h> +#include <linux/oom.h>

#include "internal.h"

@@ -913,6 +914,7 @@ int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin) if (percpu_counter_read_positive(&vm_committed_as) < allowed) return 0; error: + oom_type_notifier_call(OOM_TYPE_OVERCOMMIT, NULL); vm_unacct_memory(pages);

return -ENOMEM;

From: Lijun Fang fanglijun3@huawei.com

ascend inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4JMM0 CVE: NA -------------------

Change svm to modules by default. Remove get mem info functions, users can get the meminfo from procfs.

Signed-off-by: Lijun Fang fanglijun3@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/char/Kconfig | 2 +- drivers/char/svm.c | 215 +++---------------------------------------- 2 files changed, 16 insertions(+), 201 deletions(-)

diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index c80a5c641634..f75f9e3ea324 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -479,7 +479,7 @@ config PIN_MEMORY_DEV pin memory driver

config HISI_SVM - bool "Hisilicon svm driver" + tristate "Hisilicon svm driver" depends on ARM64 && ARM_SMMU_V3 && MMU_NOTIFIER default m help diff --git a/drivers/char/svm.c b/drivers/char/svm.c index b85283118417..3b591f197af6 100644 --- a/drivers/char/svm.c +++ b/drivers/char/svm.c @@ -38,9 +38,7 @@

#define SVM_IOCTL_REMAP_PROC 0xfff4 #define SVM_IOCTL_UNPIN_MEMORY 0xfff5 -#define SVM_IOCTL_GETHUGEINFO 0xfff6 #define SVM_IOCTL_PIN_MEMORY 0xfff7 -#define SVM_IOCTL_GET_PHYMEMINFO 0xfff8 #define SVM_IOCTL_GET_PHYS 0xfff9 #define SVM_IOCTL_LOAD_FLAG 0xfffa #define SVM_IOCTL_SET_RC 0xfffc @@ -120,23 +118,6 @@ struct svm_proc_mem { u64 buf; };

-struct meminfo { - unsigned long hugetlbfree; - unsigned long hugetlbtotal; -}; - -struct phymeminfo { - unsigned long normal_total; - unsigned long normal_free; - unsigned long huge_total; - unsigned long huge_free; -}; - -struct phymeminfo_ioctl { - struct phymeminfo *info; - unsigned long nodemask; -}; - static char *svm_cmd_to_string(unsigned int cmd) { switch (cmd) { @@ -150,10 +131,6 @@ static char *svm_cmd_to_string(unsigned int cmd) return "pin memory"; case SVM_IOCTL_UNPIN_MEMORY: return "unpin memory"; - case SVM_IOCTL_GETHUGEINFO: - return "get hugeinfo"; - case SVM_IOCTL_GET_PHYMEMINFO: - return "get physical memory info"; case SVM_IOCTL_REMAP_PROC: return "remap proc"; case SVM_IOCTL_LOAD_FLAG: @@ -853,11 +830,7 @@ static struct task_struct *svm_get_task(struct svm_bind_process params) if (params.flags & SVM_BIND_PID) { struct mm_struct *mm = NULL;

- rcu_read_lock(); - task = find_task_by_vpid(params.vpid); - if (task) - get_task_struct(task); - rcu_read_unlock(); + task = find_get_task_by_vpid(params.vpid); if (task == NULL) return ERR_PTR(-ESRCH);

@@ -967,7 +940,7 @@ static pte_t *svm_get_pte(struct vm_area_struct *vma,

if (is_vm_hugetlb_page(vma)) { if (pud_present(*pud)) { - if (pud_huge(*pud)) { + if (pud_val(*pud) && !(pud_val(*pud) & PUD_TABLE_BIT)) { pte = (pte_t *)pud; *offset = addr & (PUD_SIZE - 1); size = PUD_SIZE; @@ -989,8 +962,6 @@ static pte_t *svm_get_pte(struct vm_area_struct *vma, pte = (pte_t *)pmd; *offset = addr & (PMD_SIZE - 1); size = PMD_SIZE; - } else if (pmd_trans_unstable(pmd)) { - pr_warn("%s: thp unstable\n", __func__); } else { pte = pte_offset_map(pmd, addr); *offset = addr & (PAGE_SIZE - 1); @@ -1019,15 +990,15 @@ static pte_t *svm_walk_pt(unsigned long addr, unsigned long *page_size, return NULL;

pgd = pgd_offset(mm, addr); - if (pgd_none_or_clear_bad(pgd)) + if (pgd_none(*pgd)) return NULL;

p4d = p4d_offset(pgd, addr); - if (p4d_none_or_clear_bad(p4d)) + if (p4d_none(*p4d)) return NULL;

pud = pud_offset(p4d, addr); - if (pud_none_or_clear_bad(pud)) + if (pud_none(*pud)) return NULL;

return svm_get_pte(vma, pud, addr, page_size, offset); @@ -1155,95 +1126,6 @@ static int svm_set_rc(unsigned long __user *arg) return 0; }

-static long svm_get_hugeinfo(unsigned long __user *arg) -{ - struct hstate *h = &default_hstate; - struct meminfo info; - - if (!acpi_disabled) - return -EPERM; - - if (arg == NULL) - return -EINVAL; - - if (!hugepages_supported()) - return -ENOTSUPP; - - info.hugetlbfree = h->free_huge_pages; - info.hugetlbtotal = h->nr_huge_pages; - - if (copy_to_user((void __user *)arg, &info, sizeof(info))) - return -EFAULT; - - pr_info("svm get hugetlb info: order(%u), max_huge_pages(%lu)," - "nr_huge_pages(%lu), free_huge_pages(%lu), resv_huge_pages(%lu)", - h->order, - h->max_huge_pages, - h->nr_huge_pages, - h->free_huge_pages, - h->resv_huge_pages); - - return 0; -} - -static void svm_get_node_memory_info_inc(unsigned long nid, struct phymeminfo *info) -{ - struct sysinfo i; - struct hstate *h = &default_hstate; - unsigned long huge_free = 0; - unsigned long huge_total = 0; - - if (hugepages_supported()) { - huge_free = h->free_huge_pages_node[nid] * (PAGE_SIZE << huge_page_order(h)); - huge_total = h->nr_huge_pages_node[nid] * (PAGE_SIZE << huge_page_order(h)); - } - -#ifdef CONFIG_NUMA - si_meminfo_node(&i, nid); -#else - si_meminfo(&i); -#endif - info->normal_free += i.freeram * PAGE_SIZE; - info->normal_total += i.totalram * PAGE_SIZE - huge_total; - info->huge_total += huge_total; - info->huge_free += huge_free; -} - -static void __svm_get_memory_info(unsigned long nodemask, struct phymeminfo *info) -{ - memset(info, 0x0, sizeof(struct phymeminfo)); - - nodemask = nodemask & ((1UL << MAX_NUMNODES) - 1); - - while (nodemask) { - unsigned long nid = find_first_bit(&nodemask, BITS_PER_LONG); - - if (node_isset(nid, node_online_map)) - (void)svm_get_node_memory_info_inc(nid, info); - - nodemask &= ~(1UL << nid); - } -} - -static long svm_get_phy_memory_info(unsigned long __user *arg) -{ - struct phymeminfo info; - struct phymeminfo_ioctl para; - - if (arg == NULL) - return -EINVAL; - - if (copy_from_user(&para, (void __user *)arg, sizeof(para))) - return -EFAULT; - - __svm_get_memory_info(para.nodemask, &info); - - if (copy_to_user((void __user *)para.info, &info, sizeof(info))) - return -EFAULT; - - return 0; -} - static long svm_remap_get_phys(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, unsigned long *phys, unsigned long *page_size, unsigned long *offset) @@ -1259,15 +1141,15 @@ static long svm_remap_get_phys(struct mm_struct *mm, struct vm_area_struct *vma, return err;

pgd = pgd_offset(mm, addr); - if (pgd_none_or_clear_bad(pgd)) + if (pgd_none(*pgd)) return err;

p4d = p4d_offset(pgd, addr); - if (p4d_none_or_clear_bad(p4d)) + if (p4d_none(*p4d)) return err;

pud = pud_offset(p4d, addr); - if (pud_none_or_clear_bad(pud)) + if (pud_none(*pud)) return err;

pte = svm_get_pte(vma, pud, addr, page_size, offset); @@ -1308,11 +1190,9 @@ static long svm_remap_proc(unsigned long __user *arg) return -EINVAL; }

- rcu_read_lock(); if (pmem.pid) { - ptask = find_task_by_vpid(pmem.pid); + ptask = find_get_task_by_vpid(pmem.pid); if (!ptask) { - rcu_read_unlock(); pr_err("No task for this pid\n"); return -EINVAL; } @@ -1320,8 +1200,6 @@ static long svm_remap_proc(unsigned long __user *arg) ptask = current; }

- get_task_struct(ptask); - rcu_read_unlock(); pmm = ptask->mm;

down_read(&mm->mmap_lock); @@ -1408,65 +1286,6 @@ static int svm_proc_load_flag(int __user *arg) return put_user(flag, arg); }

-static unsigned long svm_get_unmapped_area(struct file *file, - unsigned long addr0, unsigned long len, - unsigned long pgoff, unsigned long flags) -{ - unsigned long addr = addr0; - struct mm_struct *mm = current->mm; - struct vm_unmapped_area_info info; - struct svm_device *sdev = file_to_sdev(file); - - if (!acpi_disabled) - return -EPERM; - - if (flags & MAP_FIXED) { - if (IS_ALIGNED(addr, len)) - return addr; - - dev_err(sdev->dev, "MAP_FIXED but not aligned\n"); - return -EINVAL; //lint !e570 - } - - if (addr) { - struct vm_area_struct *vma = NULL; - - addr = ALIGN(addr, len); - - if (dvpp_mmap_check(addr, len, flags)) - return -ENOMEM; - - vma = find_vma(mm, addr); - if (TASK_SIZE - len >= addr && addr >= mmap_min_addr && - (vma == NULL || addr + len <= vm_start_gap(vma))) - return addr; - } - - info.flags = VM_UNMAPPED_AREA_TOPDOWN; - info.length = len; - info.low_limit = max(PAGE_SIZE, mmap_min_addr); - info.high_limit = ((mm->mmap_base <= DVPP_MMAP_BASE) ? - mm->mmap_base : DVPP_MMAP_BASE); - info.align_mask = ((len >> PAGE_SHIFT) - 1) << PAGE_SHIFT; - info.align_offset = pgoff << PAGE_SHIFT; - - addr = vm_unmapped_area(&info); - - if (offset_in_page(addr)) { - VM_BUG_ON(addr != -ENOMEM); - info.flags = 0; - info.low_limit = TASK_UNMAPPED_BASE; - info.high_limit = DVPP_MMAP_BASE; - - if (enable_mmap_dvpp) - dvpp_mmap_get_area(&info, flags); - - addr = vm_unmapped_area(&info); - } - - return addr; -} - static int svm_mmap(struct file *file, struct vm_area_struct *vma) { int err; @@ -1622,12 +1441,6 @@ static long svm_ioctl(struct file *file, unsigned int cmd, case SVM_IOCTL_UNPIN_MEMORY: err = svm_unpin_memory((unsigned long __user *)arg); break; - case SVM_IOCTL_GETHUGEINFO: - err = svm_get_hugeinfo((unsigned long __user *)arg); - break; - case SVM_IOCTL_GET_PHYMEMINFO: - err = svm_get_phy_memory_info((unsigned long __user *)arg); - break; case SVM_IOCTL_REMAP_PROC: err = svm_remap_proc((unsigned long __user *)arg); break; @@ -1652,7 +1465,6 @@ static const struct file_operations svm_fops = { .owner = THIS_MODULE, .open = svm_open, .mmap = svm_mmap, - .get_unmapped_area = svm_get_unmapped_area, .unlocked_ioctl = svm_ioctl, };

@@ -1690,6 +1502,7 @@ static int svm_acpi_add_core(struct svm_device *sdev, struct core_device *cdev = NULL; char *name = NULL; enum dev_dma_attr attr; + const union acpi_object *obj;

name = devm_kasprintf(sdev->dev, GFP_KERNEL, "svm_child_dev%d", id); if (name == NULL) @@ -1714,7 +1527,7 @@ static int svm_acpi_add_core(struct svm_device *sdev, return err; }

- attr = acpi_get_dma_attr(children); + attr = device_get_dma_attr(&children->dev); if (attr != DEV_DMA_NOT_SUPPORTED) { err = acpi_dma_configure(&cdev->dev, attr); if (err) { @@ -1723,11 +1536,13 @@ static int svm_acpi_add_core(struct svm_device *sdev, } }

- err = acpi_dev_prop_read_single(children, "hisi,smmu-bypass", - DEV_PROP_U8, &cdev->smmu_bypass); + err = acpi_dev_get_property(children, "hisi,smmu-bypass", + DEV_PROP_U8, &obj); if (err) dev_info(&children->dev, "read smmu bypass failed\n");

+ cdev->smmu_bypass = *(u8 *)obj->integer.value; + cdev->group = iommu_group_get(&cdev->dev); if (IS_ERR_OR_NULL(cdev->group)) { dev_err(&cdev->dev, "smmu is not right configured\n");

From: Dave Ertman david.m.ertman@intel.com

mainline inclusion from mainline-v5.11-rc1 commit 7de3697e9cbd4bd3d62bafa249d57990e1b8f294 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4O662 CVE: NA

-------------------------------------------------

Add support for the Auxiliary Bus, auxiliary_device and auxiliary_driver. It enables drivers to create an auxiliary_device and bind an auxiliary_driver to it.

The bus supports probe/remove shutdown and suspend/resume callbacks. Each auxiliary_device has a unique string based id; driver binds to an auxiliary_device based on this id through the bus.

Co-developed-by: Kiran Patil kiran.patil@intel.com Co-developed-by: Ranjani Sridharan ranjani.sridharan@linux.intel.com Co-developed-by: Fred Oh fred.oh@linux.intel.com Co-developed-by: Leon Romanovsky leonro@nvidia.com Signed-off-by: Kiran Patil kiran.patil@intel.com Signed-off-by: Ranjani Sridharan ranjani.sridharan@linux.intel.com Signed-off-by: Fred Oh fred.oh@linux.intel.com Signed-off-by: Leon Romanovsky leonro@nvidia.com Signed-off-by: Dave Ertman david.m.ertman@intel.com Reviewed-by: Pierre-Louis Bossart pierre-louis.bossart@linux.intel.com Reviewed-by: Shiraz Saleem shiraz.saleem@intel.com Reviewed-by: Parav Pandit parav@mellanox.com Reviewed-by: Dan Williams dan.j.williams@intel.com Reviewed-by: Martin Habets mhabets@solarflare.com Link: https://lore.kernel.org/r/20201113161859.1775473-2-david.m.ertman@intel.com Signed-off-by: Dan Williams dan.j.williams@intel.com Link: https://lore.kernel.org/r/160695681289.505290.8978295443574440604.stgit@dwil... Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Yuanzheng Song songyuanzheng@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Hanjun Guo guohanjun@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- Documentation/driver-api/auxiliary_bus.rst | 234 ++++++++++++++++++ Documentation/driver-api/index.rst | 1 + drivers/base/Kconfig | 3 + drivers/base/Makefile | 1 + drivers/base/auxiliary.c | 268 +++++++++++++++++++++ include/linux/auxiliary_bus.h | 78 ++++++ include/linux/mod_devicetable.h | 8 + scripts/mod/devicetable-offsets.c | 3 + scripts/mod/file2alias.c | 8 + 9 files changed, 604 insertions(+) create mode 100644 Documentation/driver-api/auxiliary_bus.rst create mode 100644 drivers/base/auxiliary.c create mode 100644 include/linux/auxiliary_bus.h

diff --git a/Documentation/driver-api/auxiliary_bus.rst b/Documentation/driver-api/auxiliary_bus.rst new file mode 100644 index 000000000000..5dd7804631ef --- /dev/null +++ b/Documentation/driver-api/auxiliary_bus.rst @@ -0,0 +1,234 @@ +.. SPDX-License-Identifier: GPL-2.0-only + +============= +Auxiliary Bus +============= + +In some subsystems, the functionality of the core device (PCI/ACPI/other) is +too complex for a single device to be managed by a monolithic driver +(e.g. Sound Open Firmware), multiple devices might implement a common +intersection of functionality (e.g. NICs + RDMA), or a driver may want to +export an interface for another subsystem to drive (e.g. SIOV Physical Function +export Virtual Function management). A split of the functinoality into child- +devices representing sub-domains of functionality makes it possible to +compartmentalize, layer, and distribute domain-specific concerns via a Linux +device-driver model. + +An example for this kind of requirement is the audio subsystem where a single +IP is handling multiple entities such as HDMI, Soundwire, local devices such as +mics/speakers etc. The split for the core's functionality can be arbitrary or +be defined by the DSP firmware topology and include hooks for test/debug. This +allows for the audio core device to be minimal and focused on hardware-specific +control and communication. + +Each auxiliary_device represents a part of its parent functionality. The +generic behavior can be extended and specialized as needed by encapsulating an +auxiliary_device within other domain-specific structures and the use of .ops +callbacks. Devices on the auxiliary bus do not share any structures and the use +of a communication channel with the parent is domain-specific. + +Note that ops are intended as a way to augment instance behavior within a class +of auxiliary devices, it is not the mechanism for exporting common +infrastructure from the parent. Consider EXPORT_SYMBOL_NS() to convey +infrastructure from the parent module to the auxiliary module(s). + + +When Should the Auxiliary Bus Be Used +===================================== + +The auxiliary bus is to be used when a driver and one or more kernel modules, +who share a common header file with the driver, need a mechanism to connect and +provide access to a shared object allocated by the auxiliary_device's +registering driver. The registering driver for the auxiliary_device(s) and the +kernel module(s) registering auxiliary_drivers can be from the same subsystem, +or from multiple subsystems. + +The emphasis here is on a common generic interface that keeps subsystem +customization out of the bus infrastructure. + +One example is a PCI network device that is RDMA-capable and exports a child +device to be driven by an auxiliary_driver in the RDMA subsystem. The PCI +driver allocates and registers an auxiliary_device for each physical +function on the NIC. The RDMA driver registers an auxiliary_driver that claims +each of these auxiliary_devices. This conveys data/ops published by the parent +PCI device/driver to the RDMA auxiliary_driver. + +Another use case is for the PCI device to be split out into multiple sub +functions. For each sub function an auxiliary_device is created. A PCI sub +function driver binds to such devices that creates its own one or more class +devices. A PCI sub function auxiliary device is likely to be contained in a +struct with additional attributes such as user defined sub function number and +optional attributes such as resources and a link to the parent device. These +attributes could be used by systemd/udev; and hence should be initialized +before a driver binds to an auxiliary_device. + +A key requirement for utilizing the auxiliary bus is that there is no +dependency on a physical bus, device, register accesses or regmap support. +These individual devices split from the core cannot live on the platform bus as +they are not physical devices that are controlled by DT/ACPI. The same +argument applies for not using MFD in this scenario as MFD relies on individual +function devices being physical devices. + +Auxiliary Device +================ + +An auxiliary_device represents a part of its parent device's functionality. It +is given a name that, combined with the registering drivers KBUILD_MODNAME, +creates a match_name that is used for driver binding, and an id that combined +with the match_name provide a unique name to register with the bus subsystem. + +Registering an auxiliary_device is a two-step process. First call +auxiliary_device_init(), which checks several aspects of the auxiliary_device +struct and performs a device_initialize(). After this step completes, any +error state must have a call to auxiliary_device_uninit() in its resolution path. +The second step in registering an auxiliary_device is to perform a call to +auxiliary_device_add(), which sets the name of the device and add the device to +the bus. + +Unregistering an auxiliary_device is also a two-step process to mirror the +register process. First call auxiliary_device_delete(), then call +auxiliary_device_uninit(). + +.. code-block:: c + + struct auxiliary_device { + struct device dev; + const char *name; + u32 id; + }; + +If two auxiliary_devices both with a match_name "mod.foo" are registered onto +the bus, they must have unique id values (e.g. "x" and "y") so that the +registered devices names are "mod.foo.x" and "mod.foo.y". If match_name + id +are not unique, then the device_add fails and generates an error message. + +The auxiliary_device.dev.type.release or auxiliary_device.dev.release must be +populated with a non-NULL pointer to successfully register the auxiliary_device. + +The auxiliary_device.dev.parent must also be populated. + +Auxiliary Device Memory Model and Lifespan +------------------------------------------ + +The registering driver is the entity that allocates memory for the +auxiliary_device and register it on the auxiliary bus. It is important to note +that, as opposed to the platform bus, the registering driver is wholly +responsible for the management for the memory used for the driver object. + +A parent object, defined in the shared header file, contains the +auxiliary_device. It also contains a pointer to the shared object(s), which +also is defined in the shared header. Both the parent object and the shared +object(s) are allocated by the registering driver. This layout allows the +auxiliary_driver's registering module to perform a container_of() call to go +from the pointer to the auxiliary_device, that is passed during the call to the +auxiliary_driver's probe function, up to the parent object, and then have +access to the shared object(s). + +The memory for the auxiliary_device is freed only in its release() callback +flow as defined by its registering driver. + +The memory for the shared object(s) must have a lifespan equal to, or greater +than, the lifespan of the memory for the auxiliary_device. The auxiliary_driver +should only consider that this shared object is valid as long as the +auxiliary_device is still registered on the auxiliary bus. It is up to the +registering driver to manage (e.g. free or keep available) the memory for the +shared object beyond the life of the auxiliary_device. + +The registering driver must unregister all auxiliary devices before its own +driver.remove() is completed. + +Auxiliary Drivers +================= + +Auxiliary drivers follow the standard driver model convention, where +discovery/enumeration is handled by the core, and drivers +provide probe() and remove() methods. They support power management +and shutdown notifications using the standard conventions. + +.. code-block:: c + + struct auxiliary_driver { + int (*probe)(struct auxiliary_device *, + const struct auxiliary_device_id *id); + int (*remove)(struct auxiliary_device *); + void (*shutdown)(struct auxiliary_device *); + int (*suspend)(struct auxiliary_device *, pm_message_t); + int (*resume)(struct auxiliary_device *); + struct device_driver driver; + const struct auxiliary_device_id *id_table; + }; + +Auxiliary drivers register themselves with the bus by calling +auxiliary_driver_register(). The id_table contains the match_names of auxiliary +devices that a driver can bind with. + +Example Usage +============= + +Auxiliary devices are created and registered by a subsystem-level core device +that needs to break up its functionality into smaller fragments. One way to +extend the scope of an auxiliary_device is to encapsulate it within a domain- +pecific structure defined by the parent device. This structure contains the +auxiliary_device and any associated shared data/callbacks needed to establish +the connection with the parent. + +An example is: + +.. code-block:: c + + struct foo { + struct auxiliary_device auxdev; + void (*connect)(struct auxiliary_device *auxdev); + void (*disconnect)(struct auxiliary_device *auxdev); + void *data; + }; + +The parent device then registers the auxiliary_device by calling +auxiliary_device_init(), and then auxiliary_device_add(), with the pointer to +the auxdev member of the above structure. The parent provides a name for the +auxiliary_device that, combined with the parent's KBUILD_MODNAME, creates a +match_name that is be used for matching and binding with a driver. + +Whenever an auxiliary_driver is registered, based on the match_name, the +auxiliary_driver's probe() is invoked for the matching devices. The +auxiliary_driver can also be encapsulated inside custom drivers that make the +core device's functionality extensible by adding additional domain-specific ops +as follows: + +.. code-block:: c + + struct my_ops { + void (*send)(struct auxiliary_device *auxdev); + void (*receive)(struct auxiliary_device *auxdev); + }; + + + struct my_driver { + struct auxiliary_driver auxiliary_drv; + const struct my_ops ops; + }; + +An example of this type of usage is: + +.. code-block:: c + + const struct auxiliary_device_id my_auxiliary_id_table[] = { + { .name = "foo_mod.foo_dev" }, + { }, + }; + + const struct my_ops my_custom_ops = { + .send = my_tx, + .receive = my_rx, + }; + + const struct my_driver my_drv = { + .auxiliary_drv = { + .name = "myauxiliarydrv", + .id_table = my_auxiliary_id_table, + .probe = my_probe, + .remove = my_remove, + .shutdown = my_shutdown, + }, + .ops = my_custom_ops, + }; diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index f357f3eb400c..86759a74b7f1 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -72,6 +72,7 @@ available subsections can be seen below. thermal/index fpga/index acpi/index + auxiliary_bus backlight/lp855x-driver.rst connector console diff --git a/drivers/base/Kconfig b/drivers/base/Kconfig index 8d7001712062..040be48ce046 100644 --- a/drivers/base/Kconfig +++ b/drivers/base/Kconfig @@ -1,6 +1,9 @@ # SPDX-License-Identifier: GPL-2.0 menu "Generic Driver Options"

+config AUXILIARY_BUS + bool + config UEVENT_HELPER bool "Support for uevent helper" help diff --git a/drivers/base/Makefile b/drivers/base/Makefile index 41369fc7004f..5e7bf9669a81 100644 --- a/drivers/base/Makefile +++ b/drivers/base/Makefile @@ -7,6 +7,7 @@ obj-y := component.o core.o bus.o dd.o syscore.o \ attribute_container.o transport_class.o \ topology.o container.o property.o cacheinfo.o \ swnode.o +obj-$(CONFIG_AUXILIARY_BUS) += auxiliary.o obj-$(CONFIG_DEVTMPFS) += devtmpfs.o obj-y += power/ obj-$(CONFIG_ISA_BUS_API) += isa.o diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c new file mode 100644 index 000000000000..ef2af417438b --- /dev/null +++ b/drivers/base/auxiliary.c @@ -0,0 +1,268 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2019-2020 Intel Corporation + * + * Please see Documentation/driver-api/auxiliary_bus.rst for more information. + */ + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include <linux/device.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> +#include <linux/string.h> +#include <linux/auxiliary_bus.h> + +static const struct auxiliary_device_id *auxiliary_match_id(const struct auxiliary_device_id *id, + const struct auxiliary_device *auxdev) +{ + for (; id->name[0]; id++) { + const char *p = strrchr(dev_name(&auxdev->dev), '.'); + int match_size; + + if (!p) + continue; + match_size = p - dev_name(&auxdev->dev); + + /* use dev_name(&auxdev->dev) prefix before last '.' char to match to */ + if (strlen(id->name) == match_size && + !strncmp(dev_name(&auxdev->dev), id->name, match_size)) + return id; + } + return NULL; +} + +static int auxiliary_match(struct device *dev, struct device_driver *drv) +{ + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + struct auxiliary_driver *auxdrv = to_auxiliary_drv(drv); + + return !!auxiliary_match_id(auxdrv->id_table, auxdev); +} + +static int auxiliary_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + const char *name, *p; + + name = dev_name(dev); + p = strrchr(name, '.'); + + return add_uevent_var(env, "MODALIAS=%s%.*s", AUXILIARY_MODULE_PREFIX, (int)(p - name), + name); +} + +static const struct dev_pm_ops auxiliary_dev_pm_ops = { + SET_RUNTIME_PM_OPS(pm_generic_runtime_suspend, pm_generic_runtime_resume, NULL) + SET_SYSTEM_SLEEP_PM_OPS(pm_generic_suspend, pm_generic_resume) +}; + +static int auxiliary_bus_probe(struct device *dev) +{ + struct auxiliary_driver *auxdrv = to_auxiliary_drv(dev->driver); + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + int ret; + + ret = dev_pm_domain_attach(dev, true); + if (ret) { + dev_warn(dev, "Failed to attach to PM Domain : %d\n", ret); + return ret; + } + + ret = auxdrv->probe(auxdev, auxiliary_match_id(auxdrv->id_table, auxdev)); + if (ret) + dev_pm_domain_detach(dev, true); + + return ret; +} + +static int auxiliary_bus_remove(struct device *dev) +{ + struct auxiliary_driver *auxdrv = to_auxiliary_drv(dev->driver); + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + int ret = 0; + + if (auxdrv->remove) + ret = auxdrv->remove(auxdev); + dev_pm_domain_detach(dev, true); + + return ret; +} + +static void auxiliary_bus_shutdown(struct device *dev) +{ + struct auxiliary_driver *auxdrv = to_auxiliary_drv(dev->driver); + struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + + if (auxdrv->shutdown) + auxdrv->shutdown(auxdev); +} + +static struct bus_type auxiliary_bus_type = { + .name = "auxiliary", + .probe = auxiliary_bus_probe, + .remove = auxiliary_bus_remove, + .shutdown = auxiliary_bus_shutdown, + .match = auxiliary_match, + .uevent = auxiliary_uevent, + .pm = &auxiliary_dev_pm_ops, +}; + +/** + * auxiliary_device_init - check auxiliary_device and initialize + * @auxdev: auxiliary device struct + * + * This is the first step in the two-step process to register an auxiliary_device. + * + * When this function returns an error code, then the device_initialize will *not* have + * been performed, and the caller will be responsible to free any memory allocated for the + * auxiliary_device in the error path directly. + * + * It returns 0 on success. On success, the device_initialize has been performed. After this + * point any error unwinding will need to include a call to auxiliary_device_uninit(). + * In this post-initialize error scenario, a call to the device's .release callback will be + * triggered, and all memory clean-up is expected to be handled there. + */ +int auxiliary_device_init(struct auxiliary_device *auxdev) +{ + struct device *dev = &auxdev->dev; + + if (!dev->parent) { + pr_err("auxiliary_device has a NULL dev->parent\n"); + return -EINVAL; + } + + if (!auxdev->name) { + pr_err("auxiliary_device has a NULL name\n"); + return -EINVAL; + } + + dev->bus = &auxiliary_bus_type; + device_initialize(&auxdev->dev); + return 0; +} +EXPORT_SYMBOL_GPL(auxiliary_device_init); + +/** + * __auxiliary_device_add - add an auxiliary bus device + * @auxdev: auxiliary bus device to add to the bus + * @modname: name of the parent device's driver module + * + * This is the second step in the two-step process to register an auxiliary_device. + * + * This function must be called after a successful call to auxiliary_device_init(), which + * will perform the device_initialize. This means that if this returns an error code, then a + * call to auxiliary_device_uninit() must be performed so that the .release callback will + * be triggered to free the memory associated with the auxiliary_device. + * + * The expectation is that users will call the "auxiliary_device_add" macro so that the caller's + * KBUILD_MODNAME is automatically inserted for the modname parameter. Only if a user requires + * a custom name would this version be called directly. + */ +int __auxiliary_device_add(struct auxiliary_device *auxdev, const char *modname) +{ + struct device *dev = &auxdev->dev; + int ret; + + if (!modname) { + pr_err("auxiliary device modname is NULL\n"); + return -EINVAL; + } + + ret = dev_set_name(dev, "%s.%s.%d", modname, auxdev->name, auxdev->id); + if (ret) { + pr_err("auxiliary device dev_set_name failed: %d\n", ret); + return ret; + } + + ret = device_add(dev); + if (ret) + dev_err(dev, "adding auxiliary device failed!: %d\n", ret); + + return ret; +} +EXPORT_SYMBOL_GPL(__auxiliary_device_add); + +/** + * auxiliary_find_device - auxiliary device iterator for locating a particular device. + * @start: Device to begin with + * @data: Data to pass to match function + * @match: Callback function to check device + * + * This function returns a reference to a device that is 'found' + * for later use, as determined by the @match callback. + * + * The callback should return 0 if the device doesn't match and non-zero + * if it does. If the callback returns non-zero, this function will + * return to the caller and not iterate over any more devices. + */ +struct auxiliary_device * +auxiliary_find_device(struct device *start, const void *data, + int (*match)(struct device *dev, const void *data)) +{ + struct device *dev; + + dev = bus_find_device(&auxiliary_bus_type, start, data, match); + if (!dev) + return NULL; + + return to_auxiliary_dev(dev); +} +EXPORT_SYMBOL_GPL(auxiliary_find_device); + +/** + * __auxiliary_driver_register - register a driver for auxiliary bus devices + * @auxdrv: auxiliary_driver structure + * @owner: owning module/driver + * @modname: KBUILD_MODNAME for parent driver + */ +int __auxiliary_driver_register(struct auxiliary_driver *auxdrv, struct module *owner, + const char *modname) +{ + if (WARN_ON(!auxdrv->probe) || WARN_ON(!auxdrv->id_table)) + return -EINVAL; + + if (auxdrv->name) + auxdrv->driver.name = kasprintf(GFP_KERNEL, "%s.%s", modname, auxdrv->name); + else + auxdrv->driver.name = kasprintf(GFP_KERNEL, "%s", modname); + if (!auxdrv->driver.name) + return -ENOMEM; + + auxdrv->driver.owner = owner; + auxdrv->driver.bus = &auxiliary_bus_type; + auxdrv->driver.mod_name = modname; + + return driver_register(&auxdrv->driver); +} +EXPORT_SYMBOL_GPL(__auxiliary_driver_register); + +/** + * auxiliary_driver_unregister - unregister a driver + * @auxdrv: auxiliary_driver structure + */ +void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv) +{ + driver_unregister(&auxdrv->driver); + kfree(auxdrv->driver.name); +} +EXPORT_SYMBOL_GPL(auxiliary_driver_unregister); + +static int __init auxiliary_bus_init(void) +{ + return bus_register(&auxiliary_bus_type); +} + +static void __exit auxiliary_bus_exit(void) +{ + bus_unregister(&auxiliary_bus_type); +} + +module_init(auxiliary_bus_init); +module_exit(auxiliary_bus_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Auxiliary Bus"); +MODULE_AUTHOR("David Ertman david.m.ertman@intel.com"); +MODULE_AUTHOR("Kiran Patil kiran.patil@intel.com"); diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h new file mode 100644 index 000000000000..282fbf7bf9af --- /dev/null +++ b/include/linux/auxiliary_bus.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2019-2020 Intel Corporation + * + * Please see Documentation/driver-api/auxiliary_bus.rst for more information. + */ + +#ifndef _AUXILIARY_BUS_H_ +#define _AUXILIARY_BUS_H_ + +#include <linux/device.h> +#include <linux/mod_devicetable.h> +#include <linux/slab.h> + +struct auxiliary_device { + struct device dev; + const char *name; + u32 id; +}; + +struct auxiliary_driver { + int (*probe)(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id); + int (*remove)(struct auxiliary_device *auxdev); + void (*shutdown)(struct auxiliary_device *auxdev); + int (*suspend)(struct auxiliary_device *auxdev, pm_message_t state); + int (*resume)(struct auxiliary_device *auxdev); + const char *name; + struct device_driver driver; + const struct auxiliary_device_id *id_table; +}; + +static inline struct auxiliary_device *to_auxiliary_dev(struct device *dev) +{ + return container_of(dev, struct auxiliary_device, dev); +} + +static inline struct auxiliary_driver *to_auxiliary_drv(struct device_driver *drv) +{ + return container_of(drv, struct auxiliary_driver, driver); +} + +int auxiliary_device_init(struct auxiliary_device *auxdev); +int __auxiliary_device_add(struct auxiliary_device *auxdev, const char *modname); +#define auxiliary_device_add(auxdev) __auxiliary_device_add(auxdev, KBUILD_MODNAME) + +static inline void auxiliary_device_uninit(struct auxiliary_device *auxdev) +{ + put_device(&auxdev->dev); +} + +static inline void auxiliary_device_delete(struct auxiliary_device *auxdev) +{ + device_del(&auxdev->dev); +} + +int __auxiliary_driver_register(struct auxiliary_driver *auxdrv, struct module *owner, + const char *modname); +#define auxiliary_driver_register(auxdrv) \ + __auxiliary_driver_register(auxdrv, THIS_MODULE, KBUILD_MODNAME) + +void auxiliary_driver_unregister(struct auxiliary_driver *auxdrv); + +/** + * module_auxiliary_driver() - Helper macro for registering an auxiliary driver + * @__auxiliary_driver: auxiliary driver struct + * + * Helper macro for auxiliary drivers which do not do anything special in + * module init/exit. This eliminates a lot of boilerplate. Each module may only + * use this macro once, and calling it replaces module_init() and module_exit() + */ +#define module_auxiliary_driver(__auxiliary_driver) \ + module_driver(__auxiliary_driver, auxiliary_driver_register, auxiliary_driver_unregister) + +struct auxiliary_device * +auxiliary_find_device(struct device *start, const void *data, + int (*match)(struct device *dev, const void *data)); + +#endif /* _AUXILIARY_BUS_H_ */ diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 5b08a473cdba..c425290b21e2 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -838,4 +838,12 @@ struct mhi_device_id { kernel_ulong_t driver_data; };

+#define AUXILIARY_NAME_SIZE 32 +#define AUXILIARY_MODULE_PREFIX "auxiliary:" + +struct auxiliary_device_id { + char name[AUXILIARY_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + #endif /* LINUX_MOD_DEVICETABLE_H */ diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index 27007c18e754..e377f52dbfa3 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -243,5 +243,8 @@ int main(void) DEVID(mhi_device_id); DEVID_FIELD(mhi_device_id, chan);

+ DEVID(auxiliary_device_id); + DEVID_FIELD(auxiliary_device_id, name); + return 0; } diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c index 2417dd1dee33..fb4827027536 100644 --- a/scripts/mod/file2alias.c +++ b/scripts/mod/file2alias.c @@ -1364,6 +1364,13 @@ static int do_mhi_entry(const char *filename, void *symval, char *alias) { DEF_FIELD_ADDR(symval, mhi_device_id, chan); sprintf(alias, MHI_DEVICE_MODALIAS_FMT, *chan); + return 1; +} + +static int do_auxiliary_entry(const char *filename, void *symval, char *alias) +{ + DEF_FIELD_ADDR(symval, auxiliary_device_id, name); + sprintf(alias, AUXILIARY_MODULE_PREFIX "%s", *name);

return 1; } @@ -1442,6 +1449,7 @@ static const struct devtable devtable[] = { {"tee", SIZE_tee_client_device_id, do_tee_entry}, {"wmi", SIZE_wmi_device_id, do_wmi_entry}, {"mhi", SIZE_mhi_device_id, do_mhi_entry}, + {"auxiliary", SIZE_auxiliary_device_id, do_auxiliary_entry}, };

/* Create MODULE_ALIAS() statements.

From: Greg Kroah-Hartman gregkh@linuxfoundation.org

mainline inclusion from mainline-v5.11-rc1 commit 8142a46c50d2dd8160c42284e1044eed3bec0d18 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4O662 CVE: NA

-------------------------------------------------

There's an effort to move the remove() callback in the driver core to not return an int, as nothing can be done if this function fails. To make that effort easier, make the aux bus remove function void to start with so that no users have to be changed sometime in the future.

Cc: Dan Williams dan.j.williams@intel.com Cc: Dave Ertman david.m.ertman@intel.com Cc: Fred Oh fred.oh@linux.intel.com Cc: Kiran Patil kiran.patil@intel.com Cc: Leon Romanovsky leonro@nvidia.com Cc: Martin Habets mhabets@solarflare.com Cc: Parav Pandit parav@mellanox.com Cc: Pierre-Louis Bossart pierre-louis.bossart@linux.intel.com Cc: Ranjani Sridharan ranjani.sridharan@linux.intel.com Cc: Shiraz Saleem shiraz.saleem@intel.com Link: https://lore.kernel.org/r/X8ohB1ks1NK7kPop@kroah.com Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Yuanzheng Song songyuanzheng@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Hanjun Guo guohanjun@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- Documentation/driver-api/auxiliary_bus.rst | 2 +- drivers/base/auxiliary.c | 5 ++--- include/linux/auxiliary_bus.h | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/Documentation/driver-api/auxiliary_bus.rst b/Documentation/driver-api/auxiliary_bus.rst index 5dd7804631ef..2312506b0674 100644 --- a/Documentation/driver-api/auxiliary_bus.rst +++ b/Documentation/driver-api/auxiliary_bus.rst @@ -150,7 +150,7 @@ and shutdown notifications using the standard conventions. struct auxiliary_driver { int (*probe)(struct auxiliary_device *, const struct auxiliary_device_id *id); - int (*remove)(struct auxiliary_device *); + void (*remove)(struct auxiliary_device *); void (*shutdown)(struct auxiliary_device *); int (*suspend)(struct auxiliary_device *, pm_message_t); int (*resume)(struct auxiliary_device *); diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c index eca36d6284d0..c44e85802b43 100644 --- a/drivers/base/auxiliary.c +++ b/drivers/base/auxiliary.c @@ -82,13 +82,12 @@ static int auxiliary_bus_remove(struct device *dev) { struct auxiliary_driver *auxdrv = to_auxiliary_drv(dev->driver); struct auxiliary_device *auxdev = to_auxiliary_dev(dev); - int ret = 0;

if (auxdrv->remove) - ret = auxdrv->remove(auxdev); + auxdrv->remove(auxdev); dev_pm_domain_detach(dev, true);

- return ret; + return 0; }

static void auxiliary_bus_shutdown(struct device *dev) diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h index 3580743d0e8d..d67b17606210 100644 --- a/include/linux/auxiliary_bus.h +++ b/include/linux/auxiliary_bus.h @@ -19,7 +19,7 @@ struct auxiliary_device {

struct auxiliary_driver { int (*probe)(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id); - int (*remove)(struct auxiliary_device *auxdev); + void (*remove)(struct auxiliary_device *auxdev); void (*shutdown)(struct auxiliary_device *auxdev); int (*suspend)(struct auxiliary_device *auxdev, pm_message_t state); int (*resume)(struct auxiliary_device *auxdev);

From: Dave Jiang dave.jiang@intel.com

mainline inclusion from mainline-v5.11-rc1 commit 784b2c48ac12dcee27db001fb1a3c58c39380cb6 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4O662 CVE: NA

-------------------------------------------------

If the probe of the auxdrv failed, the device->driver is set to NULL. During kernel shutdown, the bus shutdown will call auxdrv->shutdown and cause an invalid ptr dereference. Add check to make sure device->driver is not NULL before we proceed.

Fixes: 7de3697e9cbd ("Add auxiliary bus support") Cc: Dave Ertman david.m.ertman@intel.com Signed-off-by: Dave Jiang dave.jiang@intel.com Reviewed-by: Dan Williams dan.j.williams@intel.com Link: https://lore.kernel.org/r/160710040926.1889434.8840329810698403478.stgit@dji... Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Yuanzheng Song songyuanzheng@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Hanjun Guo guohanjun@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/base/auxiliary.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c index f303daadf843..8336535f1e11 100644 --- a/drivers/base/auxiliary.c +++ b/drivers/base/auxiliary.c @@ -92,10 +92,15 @@ static int auxiliary_bus_remove(struct device *dev)

static void auxiliary_bus_shutdown(struct device *dev) { - struct auxiliary_driver *auxdrv = to_auxiliary_drv(dev->driver); - struct auxiliary_device *auxdev = to_auxiliary_dev(dev); + struct auxiliary_driver *auxdrv = NULL; + struct auxiliary_device *auxdev; + + if (dev->driver) { + auxdrv = to_auxiliary_drv(dev->driver); + auxdev = to_auxiliary_dev(dev); + }

- if (auxdrv->shutdown) + if (auxdrv && auxdrv->shutdown) auxdrv->shutdown(auxdev); }

From: Dave Jiang dave.jiang@intel.com

mainline inclusion from mainline-v5.13-rc1 commit bbf44abeeabfe05a124535e6c3a9fd7d682d42bf category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4O662 CVE: NA

-------------------------------------------------

Remove module bits in the auxiliary bus code since the auxiliary bus cannot be built as a module and the relevant code is not needed.

Cc: Dave Ertman david.m.ertman@intel.com Suggested-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Dave Jiang dave.jiang@intel.com Link: https://lore.kernel.org/r/161307488980.1896017.15627190714413338196.stgit@dj... Signed-off-by: Greg Kroah-Hartman gregkh@linuxfoundation.org Signed-off-by: Yuanzheng Song songyuanzheng@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Hanjun Guo guohanjun@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- drivers/base/auxiliary.c | 5 ----- 1 file changed, 5 deletions(-)

diff --git a/drivers/base/auxiliary.c b/drivers/base/auxiliary.c index d8b314e7d0fd..adc199dfba3c 100644 --- a/drivers/base/auxiliary.c +++ b/drivers/base/auxiliary.c @@ -265,8 +265,3 @@ void __init auxiliary_bus_init(void) { WARN_ON(bus_register(&auxiliary_bus_type)); } - -MODULE_LICENSE("GPL v2"); -MODULE_DESCRIPTION("Auxiliary Bus"); -MODULE_AUTHOR("David Ertman david.m.ertman@intel.com"); -MODULE_AUTHOR("Kiran Patil kiran.patil@intel.com");

From: Daniel Bristot de Oliveira bristot@redhat.com

mainline inclusion from mainline-v5.14-rc1 commit bc87cf0a08d437ea192b15f0918cb581a8698f15 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

The hwlat detector and (in preparation for) the osnoise/timerlat tracers have a set of u64 parameters that the user can read/write via tracefs. For instance, we have hwlat_detector's window and width.

To reduce the code duplication, hwlat's window and width share the same read function. However, they do not share the write functions because they do different parameter checks. For instance, the width needs to be smaller than the window, while the window needs to be larger than the window. The same pattern repeats on osnoise/timerlat, and a large portion of the code was devoted to the write function.

Despite having different checks, the write functions have the same structure:

read a user-space buffer take the lock that protects the value check for minimum and maximum acceptable values save the value release the lock return success or error

To reduce the code duplication also in the write functions, this patch provides a generic read and write implementation for u64 values that need to be within some minimum and/or maximum parameters, while (potentially) being protected by a lock.

To use this interface, the structure trace_min_max_param needs to be filled:

struct trace_min_max_param { struct mutex *lock; u64 *val; u64 *min; u64 *max; };

The desired value is stored on the variable pointed by *val. If *min points to a minimum acceptable value, it will be checked during the write operation. Likewise, if *max points to a maximum allowable value, it will be checked during the write operation. Finally, if *lock points to a mutex, it will be taken at the beginning of the operation and released at the end.

The definition of a trace_min_max_param needs to passed as the (private) *data for tracefs_create_file(), and the trace_min_max_fops (added by this patch) as the *fops file_operations.

Link: https://lkml.kernel.org/r/3e35760a7c8b5c55f16ae5ad5fc54a0e71cbe647.162437231...

Cc: Phil Auld pauld@redhat.com Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Kate Carcia kcarcia@redhat.com Cc: Jonathan Corbet corbet@lwn.net Cc: Ingo Molnar mingo@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Thomas Gleixner tglx@linutronix.de Cc: Alexandre Chartre alexandre.chartre@oracle.com Cc: Clark Willaims williams@redhat.com Cc: John Kacur jkacur@redhat.com Cc: Juri Lelli juri.lelli@redhat.com Cc: Borislav Petkov bp@alien8.de Cc: "H. Peter Anvin" hpa@zytor.com Cc: x86@kernel.org Cc: linux-doc@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Daniel Bristot de Oliveira bristot@redhat.com Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace.c | 85 ++++++++++++++++++++++++++++++++++++++++++++ kernel/trace/trace.h | 18 ++++++++++ 2 files changed, 103 insertions(+)

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 2ce366687ce4..a29a722f1982 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7228,6 +7228,91 @@ static const struct file_operations snapshot_raw_fops = {

#endif /* CONFIG_TRACER_SNAPSHOT */

+/* + * trace_min_max_write - Write a u64 value to a trace_min_max_param struct + * @filp: The active open file structure + * @ubuf: The userspace provided buffer to read value into + * @cnt: The maximum number of bytes to read + * @ppos: The current "file" position + * + * This function implements the write interface for a struct trace_min_max_param. + * The filp->private_data must point to a trace_min_max_param structure that + * defines where to write the value, the min and the max acceptable values, + * and a lock to protect the write. + */ +static ssize_t +trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ + struct trace_min_max_param *param = filp->private_data; + u64 val; + int err; + + if (!param) + return -EFAULT; + + err = kstrtoull_from_user(ubuf, cnt, 10, &val); + if (err) + return err; + + if (param->lock) + mutex_lock(param->lock); + + if (param->min && val < *param->min) + err = -EINVAL; + + if (param->max && val > *param->max) + err = -EINVAL; + + if (!err) + *param->val = val; + + if (param->lock) + mutex_unlock(param->lock); + + if (err) + return err; + + return cnt; +} + +/* + * trace_min_max_read - Read a u64 value from a trace_min_max_param struct + * @filp: The active open file structure + * @ubuf: The userspace provided buffer to read value into + * @cnt: The maximum number of bytes to read + * @ppos: The current "file" position + * + * This function implements the read interface for a struct trace_min_max_param. + * The filp->private_data must point to a trace_min_max_param struct with valid + * data. + */ +static ssize_t +trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos) +{ + struct trace_min_max_param *param = filp->private_data; + char buf[U64_STR_SIZE]; + int len; + u64 val; + + if (!param) + return -EFAULT; + + val = *param->val; + + if (cnt > sizeof(buf)) + cnt = sizeof(buf); + + len = snprintf(buf, sizeof(buf), "%llu\n", val); + + return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); +} + +const struct file_operations trace_min_max_fops = { + .open = tracing_open_generic, + .read = trace_min_max_read, + .write = trace_min_max_write, +}; + #define TRACING_LOG_ERRS_MAX 8 #define TRACING_LOG_LOC_MAX 128

diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 94f8087a3c22..419456b3f52f 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -2105,4 +2105,22 @@ static inline bool is_good_name(const char *name) return true; }

+/* + * This is a generic way to read and write a u64 value from a file in tracefs. + * + * The value is stored on the variable pointed by *val. The value needs + * to be at least *min and at most *max. The write is protected by an + * existing *lock. + */ +struct trace_min_max_param { + struct mutex *lock; + u64 *val; + u64 *min; + u64 *max; +}; + +#define U64_STR_SIZE 24 /* 20 digits max */ + +extern const struct file_operations trace_min_max_fops; + #endif /* _LINUX_KERNEL_TRACE_H */

From: Sebastian Andrzej Siewior bigeasy@linutronix.de

mainline inclusion from mainline-v5.12-rc1 commit 36590c50b2d0729952511129916beeea30d31d81 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

The state of the interrupts (irqflags) and the preemption counter are both passed down to tracing_generic_entry_update(). Only one bit of irqflags is actually required: The on/off state. The complete 32bit of the preemption counter isn't needed. Just whether of the upper bits (softirq, hardirq and NMI) are set and the preemption depth is needed.

The irqflags and the preemption counter could be evaluated early and the information stored in an integer `trace_ctx'. tracing_generic_entry_update() would use the upper bits as the TRACE_FLAG_* and the lower 8bit as the disabled-preemption depth (considering that one must be substracted from the counter in one special cases).

The actual preemption value is not used except for the tracing record. The `irqflags' variable is mostly used only for the tracing record. An exception here is for instance wakeup_tracer_call() or probe_wakeup_sched_switch() which explicilty disable interrupts and use that `irqflags' to save (and restore) the IRQ state and to record the state.

Struct trace_event_buffer has also the `pc' and flags' members which can be replaced with `trace_ctx' since their actual value is not used outside of trace recording.

This will reduce tracing_generic_entry_update() to simply assign values to struct trace_entry. The evaluation of the TRACE_FLAG_* bits is moved to _tracing_gen_ctx_flags() which replaces preempt_count() and local_save_flags() invocations.

As an example, ftrace_syscall_enter() may invoke: - trace_buffer_lock_reserve() -> … -> tracing_generic_entry_update() - event_trigger_unlock_commit() -> ftrace_trace_stack() -> … -> tracing_generic_entry_update() -> ftrace_trace_userstack() -> … -> tracing_generic_entry_update()

In this case the TRACE_FLAG_* bits were evaluated three times. By using the `trace_ctx' they are evaluated once and assigned three times.

A build with all tracers enabled on x86-64 with and without the patch:

text data bss dec hex filename 21970669 17084168 7639260 46694097 2c87ed1 vmlinux.old 21970293 17084168 7639260 46693721 2c87d59 vmlinux.new

text shrank by 379 bytes, data remained constant.

Link: https://lkml.kernel.org/r/20210125194511.3924915-2-bigeasy@linutronix.de

Signed-off-by: Sebastian Andrzej Siewior bigeasy@linutronix.de Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/trace_events.h | 25 +++- kernel/trace/blktrace.c | 17 +-- kernel/trace/trace.c | 208 ++++++++++++++------------- kernel/trace/trace.h | 38 +++-- kernel/trace/trace_branch.c | 6 +- kernel/trace/trace_event_perf.c | 5 +- kernel/trace/trace_events.c | 18 +-- kernel/trace/trace_events_inject.c | 6 +- kernel/trace/trace_functions.c | 28 ++-- kernel/trace/trace_functions_graph.c | 32 ++--- kernel/trace/trace_hwlat.c | 7 +- kernel/trace/trace_irqsoff.c | 86 +++++------ kernel/trace/trace_kprobe.c | 10 +- kernel/trace/trace_mmiotrace.c | 14 +- kernel/trace/trace_sched_wakeup.c | 71 +++++---- kernel/trace/trace_syscalls.c | 20 ++- kernel/trace/trace_uprobe.c | 4 +- 17 files changed, 287 insertions(+), 308 deletions(-)

diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index d321fe5ad1a1..091250b0895a 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -148,17 +148,29 @@ enum print_line_t {

enum print_line_t trace_handle_return(struct trace_seq *s);

-void tracing_generic_entry_update(struct trace_entry *entry, - unsigned short type, - unsigned long flags, - int pc); +static inline void tracing_generic_entry_update(struct trace_entry *entry, + unsigned short type, + unsigned int trace_ctx) +{ + struct task_struct *tsk = current; + + entry->preempt_count = trace_ctx & 0xff; + entry->pid = (tsk) ? tsk->pid : 0; + entry->type = type; + entry->flags = trace_ctx >> 16; +} + +unsigned int tracing_gen_ctx_flags(unsigned long irqflags); +unsigned int tracing_gen_ctx(void); +unsigned int tracing_gen_ctx_dec(void); + struct trace_event_file;

struct ring_buffer_event * trace_event_buffer_lock_reserve(struct trace_buffer **current_buffer, struct trace_event_file *trace_file, int type, unsigned long len, - unsigned long flags, int pc); + unsigned int trace_ctx);

#define TRACE_RECORD_CMDLINE BIT(0) #define TRACE_RECORD_TGID BIT(1) @@ -232,8 +244,7 @@ struct trace_event_buffer { struct ring_buffer_event *event; struct trace_event_file *trace_file; void *entry; - unsigned long flags; - int pc; + unsigned int trace_ctx; struct pt_regs *regs; };

diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index b89ff188a618..b3afd103b3aa 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -72,17 +72,17 @@ static void trace_note(struct blk_trace *bt, pid_t pid, int action, struct blk_io_trace *t; struct ring_buffer_event *event = NULL; struct trace_buffer *buffer = NULL; - int pc = 0; + unsigned int trace_ctx = 0; int cpu = smp_processor_id(); bool blk_tracer = blk_tracer_enabled; ssize_t cgid_len = cgid ? sizeof(cgid) : 0;

if (blk_tracer) { buffer = blk_tr->array_buffer.buffer; - pc = preempt_count(); + trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_BLK, sizeof(*t) + len + cgid_len, - 0, pc); + trace_ctx); if (!event) return; t = ring_buffer_event_data(event); @@ -107,7 +107,7 @@ static void trace_note(struct blk_trace *bt, pid_t pid, int action, memcpy((void *) t + sizeof(*t) + cgid_len, data, len);

if (blk_tracer) - trace_buffer_unlock_commit(blk_tr, buffer, event, 0, pc); + trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); } }

@@ -222,8 +222,9 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, struct blk_io_trace *t; unsigned long flags = 0; unsigned long *sequence; + unsigned int trace_ctx = 0; pid_t pid; - int cpu, pc = 0; + int cpu; bool blk_tracer = blk_tracer_enabled; ssize_t cgid_len = cgid ? sizeof(cgid) : 0;

@@ -252,10 +253,10 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, tracing_record_cmdline(current);

buffer = blk_tr->array_buffer.buffer; - pc = preempt_count(); + trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_BLK, sizeof(*t) + pdu_len + cgid_len, - 0, pc); + trace_ctx); if (!event) return; t = ring_buffer_event_data(event); @@ -301,7 +302,7 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len);

if (blk_tracer) { - trace_buffer_unlock_commit(blk_tr, buffer, event, 0, pc); + trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); return; } } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index a29a722f1982..2573a42e270a 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -176,7 +176,7 @@ static union trace_eval_map_item *trace_eval_maps; int tracing_set_tracer(struct trace_array *tr, const char *buf); static void ftrace_trace_userstack(struct trace_array *tr, struct trace_buffer *buffer, - unsigned long flags, int pc); + unsigned int trace_ctx);

#define MAX_TRACER_SIZE 100 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata; @@ -905,23 +905,23 @@ static inline void trace_access_lock_init(void)

#ifdef CONFIG_STACKTRACE static void __ftrace_trace_stack(struct trace_buffer *buffer, - unsigned long flags, - int skip, int pc, struct pt_regs *regs); + unsigned int trace_ctx, + int skip, struct pt_regs *regs); static inline void ftrace_trace_stack(struct trace_array *tr, struct trace_buffer *buffer, - unsigned long flags, - int skip, int pc, struct pt_regs *regs); + unsigned int trace_ctx, + int skip, struct pt_regs *regs);

#else static inline void __ftrace_trace_stack(struct trace_buffer *buffer, - unsigned long flags, - int skip, int pc, struct pt_regs *regs) + unsigned int trace_ctx, + int skip, struct pt_regs *regs) { } static inline void ftrace_trace_stack(struct trace_array *tr, struct trace_buffer *buffer, - unsigned long flags, - int skip, int pc, struct pt_regs *regs) + unsigned long trace_ctx, + int skip, struct pt_regs *regs) { }

@@ -929,24 +929,24 @@ static inline void ftrace_trace_stack(struct trace_array *tr,

static __always_inline void trace_event_setup(struct ring_buffer_event *event, - int type, unsigned long flags, int pc) + int type, unsigned int trace_ctx) { struct trace_entry *ent = ring_buffer_event_data(event);

- tracing_generic_entry_update(ent, type, flags, pc); + tracing_generic_entry_update(ent, type, trace_ctx); }

static __always_inline struct ring_buffer_event * __trace_buffer_lock_reserve(struct trace_buffer *buffer, int type, unsigned long len, - unsigned long flags, int pc) + unsigned int trace_ctx) { struct ring_buffer_event *event;

event = ring_buffer_lock_reserve(buffer, len); if (event != NULL) - trace_event_setup(event, type, flags, pc); + trace_event_setup(event, type, trace_ctx);

return event; } @@ -1007,25 +1007,22 @@ int __trace_puts(unsigned long ip, const char *str, int size) struct ring_buffer_event *event; struct trace_buffer *buffer; struct print_entry *entry; - unsigned long irq_flags; + unsigned int trace_ctx; int alloc; - int pc;

if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) return 0;

- pc = preempt_count(); - if (unlikely(tracing_selftest_running || tracing_disabled)) return 0;

alloc = sizeof(*entry) + size + 2; /* possible \n added */

- local_save_flags(irq_flags); + trace_ctx = tracing_gen_ctx(); buffer = global_trace.array_buffer.buffer; ring_buffer_nest_start(buffer); - event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, - irq_flags, pc); + event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, + trace_ctx); if (!event) { size = 0; goto out; @@ -1044,7 +1041,7 @@ int __trace_puts(unsigned long ip, const char *str, int size) entry->buf[size] = '\0';

__buffer_unlock_commit(buffer, event); - ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); + ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL); out: ring_buffer_nest_end(buffer); return size; @@ -1061,25 +1058,22 @@ int __trace_bputs(unsigned long ip, const char *str) struct ring_buffer_event *event; struct trace_buffer *buffer; struct bputs_entry *entry; - unsigned long irq_flags; + unsigned int trace_ctx; int size = sizeof(struct bputs_entry); int ret = 0; - int pc;

if (!(global_trace.trace_flags & TRACE_ITER_PRINTK)) return 0;

- pc = preempt_count(); - if (unlikely(tracing_selftest_running || tracing_disabled)) return 0;

- local_save_flags(irq_flags); + trace_ctx = tracing_gen_ctx(); buffer = global_trace.array_buffer.buffer;

ring_buffer_nest_start(buffer); event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size, - irq_flags, pc); + trace_ctx); if (!event) goto out;

@@ -1088,7 +1082,7 @@ int __trace_bputs(unsigned long ip, const char *str) entry->str = str;

__buffer_unlock_commit(buffer, event); - ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL); + ftrace_trace_stack(&global_trace, buffer, trace_ctx, 4, NULL);

ret = 1; out: @@ -2592,36 +2586,69 @@ enum print_line_t trace_handle_return(struct trace_seq *s) } EXPORT_SYMBOL_GPL(trace_handle_return);

-void -tracing_generic_entry_update(struct trace_entry *entry, unsigned short type, - unsigned long flags, int pc) +unsigned int tracing_gen_ctx_flags(unsigned long irqflags) { - struct task_struct *tsk = current; + unsigned int trace_flags = 0; + unsigned int pc; + + pc = preempt_count();

- entry->preempt_count = pc & 0xff; - entry->pid = (tsk) ? tsk->pid : 0; - entry->type = type; - entry->flags = #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT - (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | + if (irqs_disabled_flags(irqflags)) + trace_flags |= TRACE_FLAG_IRQS_OFF; #else - TRACE_FLAG_IRQS_NOSUPPORT | + trace_flags |= TRACE_FLAG_IRQS_NOSUPPORT; #endif - ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) | - ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | - ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) | - (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | - (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); + + if (pc & NMI_MASK) + trace_flags |= TRACE_FLAG_NMI; + if (pc & HARDIRQ_MASK) + trace_flags |= TRACE_FLAG_HARDIRQ; + + if (pc & SOFTIRQ_OFFSET) + trace_flags |= TRACE_FLAG_SOFTIRQ; + + if (tif_need_resched()) + trace_flags |= TRACE_FLAG_NEED_RESCHED; + if (test_preempt_need_resched()) + trace_flags |= TRACE_FLAG_PREEMPT_RESCHED; + return (trace_flags << 16) | (pc & 0xff); +} + +unsigned int tracing_gen_ctx(void) +{ + unsigned long irqflags; + +#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT + local_save_flags(irqflags); +#else + irqflags = 0; +#endif + return tracing_gen_ctx_flags(irqflags); +} + +unsigned int tracing_gen_ctx_dec(void) +{ + unsigned int trace_ctx; + + trace_ctx = tracing_gen_ctx(); + + /* + * Subtract one from the preeption counter if preemption is enabled, + * see trace_event_buffer_reserve()for details. + */ + if (IS_ENABLED(CONFIG_PREEMPTION)) + trace_ctx--; + return trace_ctx; } -EXPORT_SYMBOL_GPL(tracing_generic_entry_update);

struct ring_buffer_event * trace_buffer_lock_reserve(struct trace_buffer *buffer, int type, unsigned long len, - unsigned long flags, int pc) + unsigned int trace_ctx) { - return __trace_buffer_lock_reserve(buffer, type, len, flags, pc); + return __trace_buffer_lock_reserve(buffer, type, len, trace_ctx); }

DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); @@ -2741,7 +2768,7 @@ struct ring_buffer_event * trace_event_buffer_lock_reserve(struct trace_buffer **current_rb, struct trace_event_file *trace_file, int type, unsigned long len, - unsigned long flags, int pc) + unsigned int trace_ctx) { struct ring_buffer_event *entry; int val; @@ -2754,15 +2781,15 @@ trace_event_buffer_lock_reserve(struct trace_buffer **current_rb, /* Try to use the per cpu buffer first */ val = this_cpu_inc_return(trace_buffered_event_cnt); if ((len < (PAGE_SIZE - sizeof(*entry) - sizeof(entry->array[0]))) && val == 1) { - trace_event_setup(entry, type, flags, pc); + trace_event_setup(entry, type, trace_ctx); entry->array[0] = len; return entry; } this_cpu_dec(trace_buffered_event_cnt); }

- entry = __trace_buffer_lock_reserve(*current_rb, - type, len, flags, pc); + entry = __trace_buffer_lock_reserve(*current_rb, type, len, + trace_ctx); /* * If tracing is off, but we have triggers enabled * we still need to look at the event data. Use the temp_buffer @@ -2771,8 +2798,8 @@ trace_event_buffer_lock_reserve(struct trace_buffer **current_rb, */ if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) { *current_rb = temp_buffer; - entry = __trace_buffer_lock_reserve(*current_rb, - type, len, flags, pc); + entry = __trace_buffer_lock_reserve(*current_rb, type, len, + trace_ctx); } return entry; } @@ -2858,7 +2885,7 @@ void trace_event_buffer_commit(struct trace_event_buffer *fbuffer) ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT); event_trigger_unlock_commit_regs(fbuffer->trace_file, fbuffer->buffer, fbuffer->event, fbuffer->entry, - fbuffer->flags, fbuffer->pc, fbuffer->regs); + fbuffer->trace_ctx, fbuffer->regs); } EXPORT_SYMBOL_GPL(trace_event_buffer_commit);

@@ -2874,7 +2901,7 @@ EXPORT_SYMBOL_GPL(trace_event_buffer_commit); void trace_buffer_unlock_commit_regs(struct trace_array *tr, struct trace_buffer *buffer, struct ring_buffer_event *event, - unsigned long flags, int pc, + unsigned int trace_ctx, struct pt_regs *regs) { __buffer_unlock_commit(buffer, event); @@ -2885,8 +2912,8 @@ void trace_buffer_unlock_commit_regs(struct trace_array *tr, * and mmiotrace, but that's ok if they lose a function or * two. They are not that meaningful. */ - ftrace_trace_stack(tr, buffer, flags, regs ? 0 : STACK_SKIP, pc, regs); - ftrace_trace_userstack(tr, buffer, flags, pc); + ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs); + ftrace_trace_userstack(tr, buffer, trace_ctx); }

/* @@ -2900,9 +2927,8 @@ trace_buffer_unlock_commit_nostack(struct trace_buffer *buffer, }

void -trace_function(struct trace_array *tr, - unsigned long ip, unsigned long parent_ip, unsigned long flags, - int pc) +trace_function(struct trace_array *tr, unsigned long ip, unsigned long + parent_ip, unsigned int trace_ctx) { struct trace_event_call *call = &event_function; struct trace_buffer *buffer = tr->array_buffer.buffer; @@ -2910,7 +2936,7 @@ trace_function(struct trace_array *tr, struct ftrace_entry *entry;

event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry), - flags, pc); + trace_ctx); if (!event) return; entry = ring_buffer_event_data(event); @@ -2944,8 +2970,8 @@ static DEFINE_PER_CPU(struct ftrace_stacks, ftrace_stacks); static DEFINE_PER_CPU(int, ftrace_stack_reserve);

static void __ftrace_trace_stack(struct trace_buffer *buffer, - unsigned long flags, - int skip, int pc, struct pt_regs *regs) + unsigned int trace_ctx, + int skip, struct pt_regs *regs) { struct trace_event_call *call = &event_kernel_stack; struct ring_buffer_event *event; @@ -2993,7 +3019,7 @@ static void __ftrace_trace_stack(struct trace_buffer *buffer, size = nr_entries * sizeof(unsigned long); event = __trace_buffer_lock_reserve(buffer, TRACE_STACK, (sizeof(*entry) - sizeof(entry->caller)) + size, - flags, pc); + trace_ctx); if (!event) goto out; entry = ring_buffer_event_data(event); @@ -3014,22 +3040,22 @@ static void __ftrace_trace_stack(struct trace_buffer *buffer,

static inline void ftrace_trace_stack(struct trace_array *tr, struct trace_buffer *buffer, - unsigned long flags, - int skip, int pc, struct pt_regs *regs) + unsigned int trace_ctx, + int skip, struct pt_regs *regs) { if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) return;

- __ftrace_trace_stack(buffer, flags, skip, pc, regs); + __ftrace_trace_stack(buffer, trace_ctx, skip, regs); }

-void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, - int pc) +void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, + int skip) { struct trace_buffer *buffer = tr->array_buffer.buffer;

if (rcu_is_watching()) { - __ftrace_trace_stack(buffer, flags, skip, pc, NULL); + __ftrace_trace_stack(buffer, trace_ctx, skip, NULL); return; }

@@ -3043,7 +3069,7 @@ void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, return;

rcu_irq_enter_irqson(); - __ftrace_trace_stack(buffer, flags, skip, pc, NULL); + __ftrace_trace_stack(buffer, trace_ctx, skip, NULL); rcu_irq_exit_irqson(); }

@@ -3053,19 +3079,15 @@ void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, */ void trace_dump_stack(int skip) { - unsigned long flags; - if (tracing_disabled || tracing_selftest_running) return;

- local_save_flags(flags); - #ifndef CONFIG_UNWINDER_ORC /* Skip 1 to skip this function. */ skip++; #endif __ftrace_trace_stack(global_trace.array_buffer.buffer, - flags, skip, preempt_count(), NULL); + tracing_gen_ctx(), skip, NULL); } EXPORT_SYMBOL_GPL(trace_dump_stack);

@@ -3074,7 +3096,7 @@ static DEFINE_PER_CPU(int, user_stack_count);

static void ftrace_trace_userstack(struct trace_array *tr, - struct trace_buffer *buffer, unsigned long flags, int pc) + struct trace_buffer *buffer, unsigned int trace_ctx) { struct trace_event_call *call = &event_user_stack; struct ring_buffer_event *event; @@ -3101,7 +3123,7 @@ ftrace_trace_userstack(struct trace_array *tr, __this_cpu_inc(user_stack_count);

event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK, - sizeof(*entry), flags, pc); + sizeof(*entry), trace_ctx); if (!event) goto out_drop_count; entry = ring_buffer_event_data(event); @@ -3121,7 +3143,7 @@ ftrace_trace_userstack(struct trace_array *tr, #else /* CONFIG_USER_STACKTRACE_SUPPORT */ static void ftrace_trace_userstack(struct trace_array *tr, struct trace_buffer *buffer, - unsigned long flags, int pc) + unsigned int trace_ctx) { } #endif /* !CONFIG_USER_STACKTRACE_SUPPORT */ @@ -3251,9 +3273,9 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) struct trace_buffer *buffer; struct trace_array *tr = &global_trace; struct bprint_entry *entry; - unsigned long flags; + unsigned int trace_ctx; char *tbuffer; - int len = 0, size, pc; + int len = 0, size;

if (unlikely(tracing_selftest_running || tracing_disabled)) return 0; @@ -3261,7 +3283,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) /* Don't pollute graph traces with trace_vprintk internals */ pause_graph_tracing();

- pc = preempt_count(); + trace_ctx = tracing_gen_ctx(); preempt_disable_notrace();

tbuffer = get_trace_buf(); @@ -3275,12 +3297,11 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) goto out_put;

- local_save_flags(flags); size = sizeof(*entry) + sizeof(u32) * len; buffer = tr->array_buffer.buffer; ring_buffer_nest_start(buffer); event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, - flags, pc); + trace_ctx); if (!event) goto out; entry = ring_buffer_event_data(event); @@ -3290,7 +3311,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) memcpy(entry->buf, tbuffer, sizeof(u32) * len); if (!call_filter_check_discard(call, entry, buffer, event)) { __buffer_unlock_commit(buffer, event); - ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL); + ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL); }

out: @@ -3313,9 +3334,9 @@ __trace_array_vprintk(struct trace_buffer *buffer, { struct trace_event_call *call = &event_print; struct ring_buffer_event *event; - int len = 0, size, pc; + int len = 0, size; struct print_entry *entry; - unsigned long flags; + unsigned int trace_ctx; char *tbuffer;

if (tracing_disabled || tracing_selftest_running) @@ -3324,7 +3345,7 @@ __trace_array_vprintk(struct trace_buffer *buffer, /* Don't pollute graph traces with trace_vprintk internals */ pause_graph_tracing();

- pc = preempt_count(); + trace_ctx = tracing_gen_ctx(); preempt_disable_notrace();

@@ -3336,11 +3357,10 @@ __trace_array_vprintk(struct trace_buffer *buffer,

len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);

- local_save_flags(flags); size = sizeof(*entry) + len + 1; ring_buffer_nest_start(buffer); event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, - flags, pc); + trace_ctx); if (!event) goto out; entry = ring_buffer_event_data(event); @@ -3349,7 +3369,7 @@ __trace_array_vprintk(struct trace_buffer *buffer, memcpy(&entry->buf, tbuffer, len + 1); if (!call_filter_check_discard(call, entry, buffer, event)) { __buffer_unlock_commit(buffer, event); - ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL); + ftrace_trace_stack(&global_trace, buffer, trace_ctx, 6, NULL); }

out: @@ -6655,7 +6675,6 @@ tracing_mark_write(struct file *filp, const char __user *ubuf, enum event_trigger_type tt = ETT_NONE; struct trace_buffer *buffer; struct print_entry *entry; - unsigned long irq_flags; ssize_t written; int size; int len; @@ -6675,7 +6694,6 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,

BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);

- local_save_flags(irq_flags); size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */

/* If less than "<faulted>", then make sure we can still add that */ @@ -6684,7 +6702,7 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,

buffer = tr->array_buffer.buffer; event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, - irq_flags, preempt_count()); + tracing_gen_ctx()); if (unlikely(!event)) /* Ring buffer disabled, return as if not open for write */ return -EBADF; @@ -6736,7 +6754,6 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf, struct ring_buffer_event *event; struct trace_buffer *buffer; struct raw_data_entry *entry; - unsigned long irq_flags; ssize_t written; int size; int len; @@ -6758,14 +6775,13 @@ tracing_mark_raw_write(struct file *filp, const char __user *ubuf,

BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);

- local_save_flags(irq_flags); size = sizeof(*entry) + cnt; if (cnt < FAULT_SIZE_ID) size += FAULT_SIZE_ID - cnt;

buffer = tr->array_buffer.buffer; event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size, - irq_flags, preempt_count()); + tracing_gen_ctx()); if (!event) /* Ring buffer disabled, return as if not open for write */ return -EBADF; diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 419456b3f52f..3261691b5756 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -745,8 +745,7 @@ struct ring_buffer_event * trace_buffer_lock_reserve(struct trace_buffer *buffer, int type, unsigned long len, - unsigned long flags, - int pc); + unsigned int trace_ctx);

struct trace_entry *tracing_get_trace_entry(struct trace_array *tr, struct trace_array_cpu *data); @@ -771,11 +770,11 @@ unsigned long trace_total_entries(struct trace_array *tr); void trace_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, - unsigned long flags, int pc); + unsigned int trace_ctx); void trace_graph_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, - unsigned long flags, int pc); + unsigned int trace_ctx); void trace_latency_header(struct seq_file *m); void trace_default_header(struct seq_file *m); void print_trace_header(struct seq_file *m, struct trace_iterator *iter); @@ -841,11 +840,10 @@ static inline void latency_fsnotify(struct trace_array *tr) { } #endif

#ifdef CONFIG_STACKTRACE -void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, - int pc); +void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, int skip); #else -static inline void __trace_stack(struct trace_array *tr, unsigned long flags, - int skip, int pc) +static inline void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, + int skip) { } #endif /* CONFIG_STACKTRACE */ @@ -985,10 +983,10 @@ extern void graph_trace_open(struct trace_iterator *iter); extern void graph_trace_close(struct trace_iterator *iter); extern int __trace_graph_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, - unsigned long flags, int pc); + unsigned int trace_ctx); extern void __trace_graph_return(struct trace_array *tr, struct ftrace_graph_ret *trace, - unsigned long flags, int pc); + unsigned int trace_ctx);

#ifdef CONFIG_DYNAMIC_FTRACE extern struct ftrace_hash __rcu *ftrace_graph_hash; @@ -1451,15 +1449,15 @@ extern int call_filter_check_discard(struct trace_event_call *call, void *rec, void trace_buffer_unlock_commit_regs(struct trace_array *tr, struct trace_buffer *buffer, struct ring_buffer_event *event, - unsigned long flags, int pc, + unsigned int trcace_ctx, struct pt_regs *regs);

static inline void trace_buffer_unlock_commit(struct trace_array *tr, struct trace_buffer *buffer, struct ring_buffer_event *event, - unsigned long flags, int pc) + unsigned int trace_ctx) { - trace_buffer_unlock_commit_regs(tr, buffer, event, flags, pc, NULL); + trace_buffer_unlock_commit_regs(tr, buffer, event, trace_ctx, NULL); }

DECLARE_PER_CPU(struct ring_buffer_event *, trace_buffered_event); @@ -1520,8 +1518,7 @@ __event_trigger_test_discard(struct trace_event_file *file, * @buffer: The ring buffer that the event is being written to * @event: The event meta data in the ring buffer * @entry: The event itself - * @irq_flags: The state of the interrupts at the start of the event - * @pc: The state of the preempt count at the start of the event. + * @trace_ctx: The tracing context flags. * * This is a helper function to handle triggers that require data * from the event itself. It also tests the event against filters and @@ -1531,12 +1528,12 @@ static inline void event_trigger_unlock_commit(struct trace_event_file *file, struct trace_buffer *buffer, struct ring_buffer_event *event, - void *entry, unsigned long irq_flags, int pc) + void *entry, unsigned int trace_ctx) { enum event_trigger_type tt = ETT_NONE;

if (!__event_trigger_test_discard(file, buffer, event, entry, &tt)) - trace_buffer_unlock_commit(file->tr, buffer, event, irq_flags, pc); + trace_buffer_unlock_commit(file->tr, buffer, event, trace_ctx);

if (tt) event_triggers_post_call(file, tt); @@ -1548,8 +1545,7 @@ event_trigger_unlock_commit(struct trace_event_file *file, * @buffer: The ring buffer that the event is being written to * @event: The event meta data in the ring buffer * @entry: The event itself - * @irq_flags: The state of the interrupts at the start of the event - * @pc: The state of the preempt count at the start of the event. + * @trace_ctx: The tracing context flags. * * This is a helper function to handle triggers that require data * from the event itself. It also tests the event against filters and @@ -1562,14 +1558,14 @@ static inline void event_trigger_unlock_commit_regs(struct trace_event_file *file, struct trace_buffer *buffer, struct ring_buffer_event *event, - void *entry, unsigned long irq_flags, int pc, + void *entry, unsigned int trace_ctx, struct pt_regs *regs) { enum event_trigger_type tt = ETT_NONE;

if (!__event_trigger_test_discard(file, buffer, event, entry, &tt)) trace_buffer_unlock_commit_regs(file->tr, buffer, event, - irq_flags, pc, regs); + trace_ctx, regs);

if (tt) event_triggers_post_call(file, tt); diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c index eff099123aa2..e47fdb4c92fb 100644 --- a/kernel/trace/trace_branch.c +++ b/kernel/trace/trace_branch.c @@ -37,7 +37,7 @@ probe_likely_condition(struct ftrace_likely_data *f, int val, int expect) struct ring_buffer_event *event; struct trace_branch *entry; unsigned long flags; - int pc; + unsigned int trace_ctx; const char *p;

if (current->trace_recursion & TRACE_BRANCH_BIT) @@ -59,10 +59,10 @@ probe_likely_condition(struct ftrace_likely_data *f, int val, int expect) if (atomic_read(&data->disabled)) goto out;

- pc = preempt_count(); + trace_ctx = tracing_gen_ctx_flags(flags); buffer = tr->array_buffer.buffer; event = trace_buffer_lock_reserve(buffer, TRACE_BRANCH, - sizeof(*entry), flags, pc); + sizeof(*entry), trace_ctx); if (!event) goto out;

diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index 643e0b19920d..0443dd61667b 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c @@ -421,11 +421,8 @@ NOKPROBE_SYMBOL(perf_trace_buf_alloc); void perf_trace_buf_update(void *record, u16 type) { struct trace_entry *entry = record; - int pc = preempt_count(); - unsigned long flags;

- local_save_flags(flags); - tracing_generic_entry_update(entry, type, flags, pc); + tracing_generic_entry_update(entry, type, tracing_gen_ctx()); } NOKPROBE_SYMBOL(perf_trace_buf_update);

diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index ab3cb67b869e..546a535f1490 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -258,22 +258,19 @@ void *trace_event_buffer_reserve(struct trace_event_buffer *fbuffer, trace_event_ignore_this_pid(trace_file)) return NULL;

- local_save_flags(fbuffer->flags); - fbuffer->pc = preempt_count(); /* * If CONFIG_PREEMPTION is enabled, then the tracepoint itself disables * preemption (adding one to the preempt_count). Since we are * interested in the preempt_count at the time the tracepoint was * hit, we need to subtract one to offset the increment. */ - if (IS_ENABLED(CONFIG_PREEMPTION)) - fbuffer->pc--; + fbuffer->trace_ctx = tracing_gen_ctx_dec(); fbuffer->trace_file = trace_file;

fbuffer->event = trace_event_buffer_lock_reserve(&fbuffer->buffer, trace_file, event_call->event.type, len, - fbuffer->flags, fbuffer->pc); + fbuffer->trace_ctx); if (!fbuffer->event) return NULL;

@@ -3679,12 +3676,11 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip, struct trace_buffer *buffer; struct ring_buffer_event *event; struct ftrace_entry *entry; - unsigned long flags; + unsigned int trace_ctx; long disabled; int cpu; - int pc;

- pc = preempt_count(); + trace_ctx = tracing_gen_ctx(); preempt_disable_notrace(); cpu = raw_smp_processor_id(); disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu)); @@ -3692,11 +3688,9 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip, if (disabled != 1) goto out;

- local_save_flags(flags); - event = trace_event_buffer_lock_reserve(&buffer, &event_trace_file, TRACE_FN, sizeof(*entry), - flags, pc); + trace_ctx); if (!event) goto out; entry = ring_buffer_event_data(event); @@ -3704,7 +3698,7 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip, entry->parent_ip = parent_ip;

event_trigger_unlock_commit(&event_trace_file, buffer, event, - entry, flags, pc); + entry, trace_ctx); out: atomic_dec(&per_cpu(ftrace_test_event_disable, cpu)); preempt_enable_notrace(); diff --git a/kernel/trace/trace_events_inject.c b/kernel/trace/trace_events_inject.c index 22bcf7c51d1e..c188045c5f97 100644 --- a/kernel/trace/trace_events_inject.c +++ b/kernel/trace/trace_events_inject.c @@ -192,7 +192,6 @@ static void *trace_alloc_entry(struct trace_event_call *call, int *size) static int parse_entry(char *str, struct trace_event_call *call, void **pentry) { struct ftrace_event_field *field; - unsigned long irq_flags; void *entry = NULL; int entry_size; u64 val = 0; @@ -203,9 +202,8 @@ static int parse_entry(char *str, struct trace_event_call *call, void **pentry) if (!entry) return -ENOMEM;

- local_save_flags(irq_flags); - tracing_generic_entry_update(entry, call->event.type, irq_flags, - preempt_count()); + tracing_generic_entry_update(entry, call->event.type, + tracing_gen_ctx());

while ((len = parse_field(str, call, &field, &val)) > 0) { if (is_function_field(field)) diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index 93e20ed642e5..8606cb73341e 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -133,15 +133,14 @@ function_trace_call(unsigned long ip, unsigned long parent_ip, { struct trace_array *tr = op->private; struct trace_array_cpu *data; - unsigned long flags; + unsigned int trace_ctx; int bit; int cpu; - int pc;

if (unlikely(!tr->function_enabled)) return;

- pc = preempt_count(); + trace_ctx = tracing_gen_ctx(); preempt_disable_notrace();

bit = trace_test_and_set_recursion(TRACE_FTRACE_START); @@ -150,10 +149,9 @@ function_trace_call(unsigned long ip, unsigned long parent_ip,

cpu = smp_processor_id(); data = per_cpu_ptr(tr->array_buffer.data, cpu); - if (!atomic_read(&data->disabled)) { - local_save_flags(flags); - trace_function(tr, ip, parent_ip, flags, pc); - } + if (!atomic_read(&data->disabled)) + trace_function(tr, ip, parent_ip, trace_ctx); + trace_clear_recursion(bit);

out: @@ -187,7 +185,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, unsigned long flags; long disabled; int cpu; - int pc; + unsigned int trace_ctx;

if (unlikely(!tr->function_enabled)) return; @@ -202,9 +200,9 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, disabled = atomic_inc_return(&data->disabled);

if (likely(disabled == 1)) { - pc = preempt_count(); - trace_function(tr, ip, parent_ip, flags, pc); - __trace_stack(tr, flags, STACK_SKIP, pc); + trace_ctx = tracing_gen_ctx_flags(flags); + trace_function(tr, ip, parent_ip, trace_ctx); + __trace_stack(tr, trace_ctx, STACK_SKIP); }

atomic_dec(&data->disabled); @@ -407,13 +405,11 @@ ftrace_traceoff(unsigned long ip, unsigned long parent_ip,

static __always_inline void trace_stack(struct trace_array *tr) { - unsigned long flags; - int pc; + unsigned int trace_ctx;

- local_save_flags(flags); - pc = preempt_count(); + trace_ctx = tracing_gen_ctx();

- __trace_stack(tr, flags, FTRACE_STACK_SKIP, pc); + __trace_stack(tr, trace_ctx, FTRACE_STACK_SKIP); }

static void diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 60d66278aa0d..b086ba8bb3d6 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -96,8 +96,7 @@ print_graph_duration(struct trace_array *tr, unsigned long long duration,

int __trace_graph_entry(struct trace_array *tr, struct ftrace_graph_ent *trace, - unsigned long flags, - int pc) + unsigned int trace_ctx) { struct trace_event_call *call = &event_funcgraph_entry; struct ring_buffer_event *event; @@ -105,7 +104,7 @@ int __trace_graph_entry(struct trace_array *tr, struct ftrace_graph_ent_entry *entry;

event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT, - sizeof(*entry), flags, pc); + sizeof(*entry), trace_ctx); if (!event) return 0; entry = ring_buffer_event_data(event); @@ -129,10 +128,10 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) struct trace_array *tr = graph_array; struct trace_array_cpu *data; unsigned long flags; + unsigned int trace_ctx; long disabled; int ret; int cpu; - int pc;

if (trace_recursion_test(TRACE_GRAPH_NOTRACE_BIT)) return 0; @@ -174,8 +173,8 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) data = per_cpu_ptr(tr->array_buffer.data, cpu); disabled = atomic_inc_return(&data->disabled); if (likely(disabled == 1)) { - pc = preempt_count(); - ret = __trace_graph_entry(tr, trace, flags, pc); + trace_ctx = tracing_gen_ctx_flags(flags); + ret = __trace_graph_entry(tr, trace, trace_ctx); } else { ret = 0; } @@ -188,7 +187,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace)

static void __trace_graph_function(struct trace_array *tr, - unsigned long ip, unsigned long flags, int pc) + unsigned long ip, unsigned int trace_ctx) { u64 time = trace_clock_local(); struct ftrace_graph_ent ent = { @@ -202,22 +201,21 @@ __trace_graph_function(struct trace_array *tr, .rettime = time, };

- __trace_graph_entry(tr, &ent, flags, pc); - __trace_graph_return(tr, &ret, flags, pc); + __trace_graph_entry(tr, &ent, trace_ctx); + __trace_graph_return(tr, &ret, trace_ctx); }

void trace_graph_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, - unsigned long flags, int pc) + unsigned int trace_ctx) { - __trace_graph_function(tr, ip, flags, pc); + __trace_graph_function(tr, ip, trace_ctx); }

void __trace_graph_return(struct trace_array *tr, struct ftrace_graph_ret *trace, - unsigned long flags, - int pc) + unsigned int trace_ctx) { struct trace_event_call *call = &event_funcgraph_exit; struct ring_buffer_event *event; @@ -225,7 +223,7 @@ void __trace_graph_return(struct trace_array *tr, struct ftrace_graph_ret_entry *entry;

event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET, - sizeof(*entry), flags, pc); + sizeof(*entry), trace_ctx); if (!event) return; entry = ring_buffer_event_data(event); @@ -239,9 +237,9 @@ void trace_graph_return(struct ftrace_graph_ret *trace) struct trace_array *tr = graph_array; struct trace_array_cpu *data; unsigned long flags; + unsigned int trace_ctx; long disabled; int cpu; - int pc;

ftrace_graph_addr_finish(trace);

@@ -255,8 +253,8 @@ void trace_graph_return(struct ftrace_graph_ret *trace) data = per_cpu_ptr(tr->array_buffer.data, cpu); disabled = atomic_inc_return(&data->disabled); if (likely(disabled == 1)) { - pc = preempt_count(); - __trace_graph_return(tr, trace, flags, pc); + trace_ctx = tracing_gen_ctx_flags(flags); + __trace_graph_return(tr, trace, trace_ctx); } atomic_dec(&data->disabled); local_irq_restore(flags); diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index d071fc271eef..4c01c5d8b9a7 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -108,14 +108,9 @@ static void trace_hwlat_sample(struct hwlat_sample *sample) struct trace_buffer *buffer = tr->array_buffer.buffer; struct ring_buffer_event *event; struct hwlat_entry *entry; - unsigned long flags; - int pc; - - pc = preempt_count(); - local_save_flags(flags);

event = trace_buffer_lock_reserve(buffer, TRACE_HWLAT, sizeof(*entry), - flags, pc); + tracing_gen_ctx()); if (!event) return; entry = ring_buffer_event_data(event); diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c index ee4571b624bc..f11add83c108 100644 --- a/kernel/trace/trace_irqsoff.c +++ b/kernel/trace/trace_irqsoff.c @@ -143,11 +143,14 @@ irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip, struct trace_array *tr = irqsoff_trace; struct trace_array_cpu *data; unsigned long flags; + unsigned int trace_ctx;

if (!func_prolog_dec(tr, &data, &flags)) return;

- trace_function(tr, ip, parent_ip, flags, preempt_count()); + trace_ctx = tracing_gen_ctx_flags(flags); + + trace_function(tr, ip, parent_ip, trace_ctx);

atomic_dec(&data->disabled); } @@ -177,8 +180,8 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) struct trace_array *tr = irqsoff_trace; struct trace_array_cpu *data; unsigned long flags; + unsigned int trace_ctx; int ret; - int pc;

if (ftrace_graph_ignore_func(trace)) return 0; @@ -195,8 +198,8 @@ static int irqsoff_graph_entry(struct ftrace_graph_ent *trace) if (!func_prolog_dec(tr, &data, &flags)) return 0;

- pc = preempt_count(); - ret = __trace_graph_entry(tr, trace, flags, pc); + trace_ctx = tracing_gen_ctx_flags(flags); + ret = __trace_graph_entry(tr, trace, trace_ctx); atomic_dec(&data->disabled);

return ret; @@ -207,15 +210,15 @@ static void irqsoff_graph_return(struct ftrace_graph_ret *trace) struct trace_array *tr = irqsoff_trace; struct trace_array_cpu *data; unsigned long flags; - int pc; + unsigned int trace_ctx;

ftrace_graph_addr_finish(trace);

if (!func_prolog_dec(tr, &data, &flags)) return;

- pc = preempt_count(); - __trace_graph_return(tr, trace, flags, pc); + trace_ctx = tracing_gen_ctx_flags(flags); + __trace_graph_return(tr, trace, trace_ctx); atomic_dec(&data->disabled); }

@@ -267,12 +270,12 @@ static void irqsoff_print_header(struct seq_file *s) static void __trace_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, - unsigned long flags, int pc) + unsigned int trace_ctx) { if (is_graph(tr)) - trace_graph_function(tr, ip, parent_ip, flags, pc); + trace_graph_function(tr, ip, parent_ip, trace_ctx); else - trace_function(tr, ip, parent_ip, flags, pc); + trace_function(tr, ip, parent_ip, trace_ctx); }

#else @@ -322,15 +325,13 @@ check_critical_timing(struct trace_array *tr, { u64 T0, T1, delta; unsigned long flags; - int pc; + unsigned int trace_ctx;

T0 = data->preempt_timestamp; T1 = ftrace_now(cpu); delta = T1-T0;

- local_save_flags(flags); - - pc = preempt_count(); + trace_ctx = tracing_gen_ctx();

if (!report_latency(tr, delta)) goto out; @@ -341,9 +342,9 @@ check_critical_timing(struct trace_array *tr, if (!report_latency(tr, delta)) goto out_unlock;

- __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); + __trace_function(tr, CALLER_ADDR0, parent_ip, trace_ctx); /* Skip 5 functions to get to the irq/preempt enable function */ - __trace_stack(tr, flags, 5, pc); + __trace_stack(tr, trace_ctx, 5);

if (data->critical_sequence != max_sequence) goto out_unlock; @@ -363,16 +364,15 @@ check_critical_timing(struct trace_array *tr, out: data->critical_sequence = max_sequence; data->preempt_timestamp = ftrace_now(cpu); - __trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); + __trace_function(tr, CALLER_ADDR0, parent_ip, trace_ctx); }

static nokprobe_inline void -start_critical_timing(unsigned long ip, unsigned long parent_ip, int pc) +start_critical_timing(unsigned long ip, unsigned long parent_ip) { int cpu; struct trace_array *tr = irqsoff_trace; struct trace_array_cpu *data; - unsigned long flags;

if (!tracer_enabled || !tracing_is_enabled()) return; @@ -393,9 +393,7 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip, int pc) data->preempt_timestamp = ftrace_now(cpu); data->critical_start = parent_ip ? : ip;

- local_save_flags(flags); - - __trace_function(tr, ip, parent_ip, flags, pc); + __trace_function(tr, ip, parent_ip, tracing_gen_ctx());

per_cpu(tracing_cpu, cpu) = 1;

@@ -403,12 +401,12 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip, int pc) }

static nokprobe_inline void -stop_critical_timing(unsigned long ip, unsigned long parent_ip, int pc) +stop_critical_timing(unsigned long ip, unsigned long parent_ip) { int cpu; struct trace_array *tr = irqsoff_trace; struct trace_array_cpu *data; - unsigned long flags; + unsigned int trace_ctx;

cpu = raw_smp_processor_id(); /* Always clear the tracing cpu on stopping the trace */ @@ -428,8 +426,8 @@ stop_critical_timing(unsigned long ip, unsigned long parent_ip, int pc)

atomic_inc(&data->disabled);

- local_save_flags(flags); - __trace_function(tr, ip, parent_ip, flags, pc); + trace_ctx = tracing_gen_ctx(); + __trace_function(tr, ip, parent_ip, trace_ctx); check_critical_timing(tr, data, parent_ip ? : ip, cpu); data->critical_start = 0; atomic_dec(&data->disabled); @@ -438,20 +436,16 @@ stop_critical_timing(unsigned long ip, unsigned long parent_ip, int pc) /* start and stop critical timings used to for stoppage (in idle) */ void start_critical_timings(void) { - int pc = preempt_count(); - - if (preempt_trace(pc) || irq_trace()) - start_critical_timing(CALLER_ADDR0, CALLER_ADDR1, pc); + if (preempt_trace(preempt_count()) || irq_trace()) + start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); } EXPORT_SYMBOL_GPL(start_critical_timings); NOKPROBE_SYMBOL(start_critical_timings);

void stop_critical_timings(void) { - int pc = preempt_count(); - - if (preempt_trace(pc) || irq_trace()) - stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1, pc); + if (preempt_trace(preempt_count()) || irq_trace()) + stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); } EXPORT_SYMBOL_GPL(stop_critical_timings); NOKPROBE_SYMBOL(stop_critical_timings); @@ -613,19 +607,15 @@ static void irqsoff_tracer_stop(struct trace_array *tr) */ void tracer_hardirqs_on(unsigned long a0, unsigned long a1) { - unsigned int pc = preempt_count(); - - if (!preempt_trace(pc) && irq_trace()) - stop_critical_timing(a0, a1, pc); + if (!preempt_trace(preempt_count()) && irq_trace()) + stop_critical_timing(a0, a1); } NOKPROBE_SYMBOL(tracer_hardirqs_on);

void tracer_hardirqs_off(unsigned long a0, unsigned long a1) { - unsigned int pc = preempt_count(); - - if (!preempt_trace(pc) && irq_trace()) - start_critical_timing(a0, a1, pc); + if (!preempt_trace(preempt_count()) && irq_trace()) + start_critical_timing(a0, a1); } NOKPROBE_SYMBOL(tracer_hardirqs_off);

@@ -665,18 +655,14 @@ static struct tracer irqsoff_tracer __read_mostly = #ifdef CONFIG_PREEMPT_TRACER void tracer_preempt_on(unsigned long a0, unsigned long a1) { - int pc = preempt_count(); - - if (preempt_trace(pc) && !irq_trace()) - stop_critical_timing(a0, a1, pc); + if (preempt_trace(preempt_count()) && !irq_trace()) + stop_critical_timing(a0, a1); }

void tracer_preempt_off(unsigned long a0, unsigned long a1) { - int pc = preempt_count(); - - if (preempt_trace(pc) && !irq_trace()) - start_critical_timing(a0, a1, pc); + if (preempt_trace(preempt_count()) && !irq_trace()) + start_critical_timing(a0, a1); }

static int preemptoff_tracer_init(struct trace_array *tr) diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 552dbc9d5226..11242ff53663 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -1390,8 +1390,7 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, if (trace_trigger_soft_disabled(trace_file)) return;

- local_save_flags(fbuffer.flags); - fbuffer.pc = preempt_count(); + fbuffer.trace_ctx = tracing_gen_ctx(); fbuffer.trace_file = trace_file;

dsize = __get_data_size(&tk->tp, regs); @@ -1400,7 +1399,7 @@ __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs, trace_event_buffer_lock_reserve(&fbuffer.buffer, trace_file, call->event.type, sizeof(*entry) + tk->tp.size + dsize, - fbuffer.flags, fbuffer.pc); + fbuffer.trace_ctx); if (!fbuffer.event) return;

@@ -1438,8 +1437,7 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, if (trace_trigger_soft_disabled(trace_file)) return;

- local_save_flags(fbuffer.flags); - fbuffer.pc = preempt_count(); + fbuffer.trace_ctx = tracing_gen_ctx(); fbuffer.trace_file = trace_file;

dsize = __get_data_size(&tk->tp, regs); @@ -1447,7 +1445,7 @@ __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri, trace_event_buffer_lock_reserve(&fbuffer.buffer, trace_file, call->event.type, sizeof(*entry) + tk->tp.size + dsize, - fbuffer.flags, fbuffer.pc); + fbuffer.trace_ctx); if (!fbuffer.event) return;

diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c index 84582bf1ed5f..7221ae0b4c47 100644 --- a/kernel/trace/trace_mmiotrace.c +++ b/kernel/trace/trace_mmiotrace.c @@ -300,10 +300,11 @@ static void __trace_mmiotrace_rw(struct trace_array *tr, struct trace_buffer *buffer = tr->array_buffer.buffer; struct ring_buffer_event *event; struct trace_mmiotrace_rw *entry; - int pc = preempt_count(); + unsigned int trace_ctx;

+ trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW, - sizeof(*entry), 0, pc); + sizeof(*entry), trace_ctx); if (!event) { atomic_inc(&dropped_count); return; @@ -312,7 +313,7 @@ static void __trace_mmiotrace_rw(struct trace_array *tr, entry->rw = *rw;

if (!call_filter_check_discard(call, entry, buffer, event)) - trace_buffer_unlock_commit(tr, buffer, event, 0, pc); + trace_buffer_unlock_commit(tr, buffer, event, trace_ctx); }

void mmio_trace_rw(struct mmiotrace_rw *rw) @@ -330,10 +331,11 @@ static void __trace_mmiotrace_map(struct trace_array *tr, struct trace_buffer *buffer = tr->array_buffer.buffer; struct ring_buffer_event *event; struct trace_mmiotrace_map *entry; - int pc = preempt_count(); + unsigned int trace_ctx;

+ trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP, - sizeof(*entry), 0, pc); + sizeof(*entry), trace_ctx); if (!event) { atomic_inc(&dropped_count); return; @@ -342,7 +344,7 @@ static void __trace_mmiotrace_map(struct trace_array *tr, entry->map = *map;

if (!call_filter_check_discard(call, entry, buffer, event)) - trace_buffer_unlock_commit(tr, buffer, event, 0, pc); + trace_buffer_unlock_commit(tr, buffer, event, trace_ctx); }

void mmio_trace_mapping(struct mmiotrace_map *map) diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 97b10bb31a1f..f1c603358ff3 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -67,7 +67,7 @@ static bool function_enabled; static int func_prolog_preempt_disable(struct trace_array *tr, struct trace_array_cpu **data, - int *pc) + unsigned int *trace_ctx) { long disabled; int cpu; @@ -75,7 +75,7 @@ func_prolog_preempt_disable(struct trace_array *tr, if (likely(!wakeup_task)) return 0;

- *pc = preempt_count(); + *trace_ctx = tracing_gen_ctx(); preempt_disable_notrace();

cpu = raw_smp_processor_id(); @@ -116,8 +116,8 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace) { struct trace_array *tr = wakeup_trace; struct trace_array_cpu *data; - unsigned long flags; - int pc, ret = 0; + unsigned int trace_ctx; + int ret = 0;

if (ftrace_graph_ignore_func(trace)) return 0; @@ -131,11 +131,10 @@ static int wakeup_graph_entry(struct ftrace_graph_ent *trace) if (ftrace_graph_notrace_addr(trace->func)) return 1;

- if (!func_prolog_preempt_disable(tr, &data, &pc)) + if (!func_prolog_preempt_disable(tr, &data, &trace_ctx)) return 0;

- local_save_flags(flags); - ret = __trace_graph_entry(tr, trace, flags, pc); + ret = __trace_graph_entry(tr, trace, trace_ctx); atomic_dec(&data->disabled); preempt_enable_notrace();

@@ -146,16 +145,14 @@ static void wakeup_graph_return(struct ftrace_graph_ret *trace) { struct trace_array *tr = wakeup_trace; struct trace_array_cpu *data; - unsigned long flags; - int pc; + unsigned int trace_ctx;

ftrace_graph_addr_finish(trace);

- if (!func_prolog_preempt_disable(tr, &data, &pc)) + if (!func_prolog_preempt_disable(tr, &data, &trace_ctx)) return;

- local_save_flags(flags); - __trace_graph_return(tr, trace, flags, pc); + __trace_graph_return(tr, trace, trace_ctx); atomic_dec(&data->disabled);

preempt_enable_notrace(); @@ -217,13 +214,13 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip, struct trace_array *tr = wakeup_trace; struct trace_array_cpu *data; unsigned long flags; - int pc; + unsigned int trace_ctx;

- if (!func_prolog_preempt_disable(tr, &data, &pc)) + if (!func_prolog_preempt_disable(tr, &data, &trace_ctx)) return;

local_irq_save(flags); - trace_function(tr, ip, parent_ip, flags, pc); + trace_function(tr, ip, parent_ip, trace_ctx); local_irq_restore(flags);

atomic_dec(&data->disabled); @@ -303,12 +300,12 @@ static void wakeup_print_header(struct seq_file *s) static void __trace_function(struct trace_array *tr, unsigned long ip, unsigned long parent_ip, - unsigned long flags, int pc) + unsigned int trace_ctx) { if (is_graph(tr)) - trace_graph_function(tr, ip, parent_ip, flags, pc); + trace_graph_function(tr, ip, parent_ip, trace_ctx); else - trace_function(tr, ip, parent_ip, flags, pc); + trace_function(tr, ip, parent_ip, trace_ctx); }

static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set) @@ -375,7 +372,7 @@ static void tracing_sched_switch_trace(struct trace_array *tr, struct task_struct *prev, struct task_struct *next, - unsigned long flags, int pc) + unsigned int trace_ctx) { struct trace_event_call *call = &event_context_switch; struct trace_buffer *buffer = tr->array_buffer.buffer; @@ -383,7 +380,7 @@ tracing_sched_switch_trace(struct trace_array *tr, struct ctx_switch_entry *entry;

event = trace_buffer_lock_reserve(buffer, TRACE_CTX, - sizeof(*entry), flags, pc); + sizeof(*entry), trace_ctx); if (!event) return; entry = ring_buffer_event_data(event); @@ -396,14 +393,14 @@ tracing_sched_switch_trace(struct trace_array *tr, entry->next_cpu = task_cpu(next);

if (!call_filter_check_discard(call, entry, buffer, event)) - trace_buffer_unlock_commit(tr, buffer, event, flags, pc); + trace_buffer_unlock_commit(tr, buffer, event, trace_ctx); }

static void tracing_sched_wakeup_trace(struct trace_array *tr, struct task_struct *wakee, struct task_struct *curr, - unsigned long flags, int pc) + unsigned int trace_ctx) { struct trace_event_call *call = &event_wakeup; struct ring_buffer_event *event; @@ -411,7 +408,7 @@ tracing_sched_wakeup_trace(struct trace_array *tr, struct trace_buffer *buffer = tr->array_buffer.buffer;

event = trace_buffer_lock_reserve(buffer, TRACE_WAKE, - sizeof(*entry), flags, pc); + sizeof(*entry), trace_ctx); if (!event) return; entry = ring_buffer_event_data(event); @@ -424,7 +421,7 @@ tracing_sched_wakeup_trace(struct trace_array *tr, entry->next_cpu = task_cpu(wakee);

if (!call_filter_check_discard(call, entry, buffer, event)) - trace_buffer_unlock_commit(tr, buffer, event, flags, pc); + trace_buffer_unlock_commit(tr, buffer, event, trace_ctx); }

static void notrace @@ -436,7 +433,7 @@ probe_wakeup_sched_switch(void *ignore, bool preempt, unsigned long flags; long disabled; int cpu; - int pc; + unsigned int trace_ctx;

tracing_record_cmdline(prev);

@@ -455,8 +452,6 @@ probe_wakeup_sched_switch(void *ignore, bool preempt, if (next != wakeup_task) return;

- pc = preempt_count(); - /* disable local data, not wakeup_cpu data */ cpu = raw_smp_processor_id(); disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); @@ -464,6 +459,8 @@ probe_wakeup_sched_switch(void *ignore, bool preempt, goto out;

local_irq_save(flags); + trace_ctx = tracing_gen_ctx_flags(flags); + arch_spin_lock(&wakeup_lock);

/* We could race with grabbing wakeup_lock */ @@ -473,9 +470,9 @@ probe_wakeup_sched_switch(void *ignore, bool preempt, /* The task we are waiting for is waking up */ data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);

- __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, flags, pc); - tracing_sched_switch_trace(wakeup_trace, prev, next, flags, pc); - __trace_stack(wakeup_trace, flags, 0, pc); + __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, trace_ctx); + tracing_sched_switch_trace(wakeup_trace, prev, next, trace_ctx); + __trace_stack(wakeup_trace, trace_ctx, 0);

T0 = data->preempt_timestamp; T1 = ftrace_now(cpu); @@ -527,9 +524,8 @@ probe_wakeup(void *ignore, struct task_struct *p) { struct trace_array_cpu *data; int cpu = smp_processor_id(); - unsigned long flags; long disabled; - int pc; + unsigned int trace_ctx;

if (likely(!tracer_enabled)) return; @@ -550,11 +546,12 @@ probe_wakeup(void *ignore, struct task_struct *p) (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio))) return;

- pc = preempt_count(); disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled); if (unlikely(disabled != 1)) goto out;

+ trace_ctx = tracing_gen_ctx(); + /* interrupts should be off from try_to_wake_up */ arch_spin_lock(&wakeup_lock);

@@ -581,19 +578,17 @@ probe_wakeup(void *ignore, struct task_struct *p)

wakeup_task = get_task_struct(p);

- local_save_flags(flags); - data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu); data->preempt_timestamp = ftrace_now(cpu); - tracing_sched_wakeup_trace(wakeup_trace, p, current, flags, pc); - __trace_stack(wakeup_trace, flags, 0, pc); + tracing_sched_wakeup_trace(wakeup_trace, p, current, trace_ctx); + __trace_stack(wakeup_trace, trace_ctx, 0);

/* * We must be careful in using CALLER_ADDR2. But since wake_up * is not called by an assembly function (where as schedule is) * it should be safe to use it here. */ - __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, flags, pc); + __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, trace_ctx);

out_locked: arch_spin_unlock(&wakeup_lock); diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c index d85a2f0f316b..8bfcd3b09422 100644 --- a/kernel/trace/trace_syscalls.c +++ b/kernel/trace/trace_syscalls.c @@ -298,9 +298,8 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id) struct syscall_metadata *sys_data; struct ring_buffer_event *event; struct trace_buffer *buffer; - unsigned long irq_flags; + unsigned int trace_ctx; unsigned long args[6]; - int pc; int syscall_nr; int size;

@@ -322,12 +321,11 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id)

size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;

- local_save_flags(irq_flags); - pc = preempt_count(); + trace_ctx = tracing_gen_ctx();

buffer = tr->array_buffer.buffer; event = trace_buffer_lock_reserve(buffer, - sys_data->enter_event->event.type, size, irq_flags, pc); + sys_data->enter_event->event.type, size, trace_ctx); if (!event) return;

@@ -337,7 +335,7 @@ static void ftrace_syscall_enter(void *data, struct pt_regs *regs, long id) memcpy(entry->args, args, sizeof(unsigned long) * sys_data->nb_args);

event_trigger_unlock_commit(trace_file, buffer, event, entry, - irq_flags, pc); + trace_ctx); }

static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) @@ -348,8 +346,7 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) struct syscall_metadata *sys_data; struct ring_buffer_event *event; struct trace_buffer *buffer; - unsigned long irq_flags; - int pc; + unsigned int trace_ctx; int syscall_nr;

syscall_nr = trace_get_syscall_nr(current, regs); @@ -368,13 +365,12 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) if (!sys_data) return;

- local_save_flags(irq_flags); - pc = preempt_count(); + trace_ctx = tracing_gen_ctx();

buffer = tr->array_buffer.buffer; event = trace_buffer_lock_reserve(buffer, sys_data->exit_event->event.type, sizeof(*entry), - irq_flags, pc); + trace_ctx); if (!event) return;

@@ -383,7 +379,7 @@ static void ftrace_syscall_exit(void *data, struct pt_regs *regs, long ret) entry->ret = syscall_get_return_value(current, regs);

event_trigger_unlock_commit(trace_file, buffer, event, entry, - irq_flags, pc); + trace_ctx); }

static int reg_event_syscall_enter(struct trace_event_file *file, diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index 0dd6e286e519..a1bc68de1b29 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -965,7 +965,7 @@ static void __uprobe_trace_func(struct trace_uprobe *tu, esize = SIZEOF_TRACE_ENTRY(is_ret_probe(tu)); size = esize + tu->tp.size + dsize; event = trace_event_buffer_lock_reserve(&buffer, trace_file, - call->event.type, size, 0, 0); + call->event.type, size, 0); if (!event) return;

@@ -981,7 +981,7 @@ static void __uprobe_trace_func(struct trace_uprobe *tu,

memcpy(data, ucb->buf, tu->tp.size + dsize);

- event_trigger_unlock_commit(trace_file, buffer, event, entry, 0, 0); + event_trigger_unlock_commit(trace_file, buffer, event, entry, 0); }

/* uprobe handler */

From: Daniel Bristot de Oliveira bristot@redhat.com

mainline inclusion from mainline-v5.14-rc1 commit a955d7eac1779b437ceb24fc352026a2cbcec140 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

The timerlat tracer aims to help the preemptive kernel developers to found souces of wakeup latencies of real-time threads. Like cyclictest, the tracer sets a periodic timer that wakes up a thread. The thread then computes a *wakeup latency* value as the difference between the *current time* and the *absolute time* that the timer was set to expire. The main goal of timerlat is tracing in such a way to help kernel developers.

Usage

Write the ASCII text "timerlat" into the current_tracer file of the tracing system (generally mounted at /sys/kernel/tracing).

For example:

[root@f32 ~]# cd /sys/kernel/tracing/ [root@f32 tracing]# echo timerlat > current_tracer

It is possible to follow the trace by reading the trace trace file:

[root@f32 tracing]# cat trace # tracer: timerlat # # _-----=> irqs-off # / _----=> need-resched # | / _---=> hardirq/softirq # || / _--=> preempt-depth # || / # |||| ACTIVATION # TASK-PID CPU# |||| TIMESTAMP ID CONTEXT LATENCY # | | | |||| | | | | <idle>-0 [000] d.h1 54.029328: #1 context irq timer_latency 932 ns <...>-867 [000] .... 54.029339: #1 context thread timer_latency 11700 ns <idle>-0 [001] dNh1 54.029346: #1 context irq timer_latency 2833 ns <...>-868 [001] .... 54.029353: #1 context thread timer_latency 9820 ns <idle>-0 [000] d.h1 54.030328: #2 context irq timer_latency 769 ns <...>-867 [000] .... 54.030330: #2 context thread timer_latency 3070 ns <idle>-0 [001] d.h1 54.030344: #2 context irq timer_latency 935 ns <...>-868 [001] .... 54.030347: #2 context thread timer_latency 4351 ns

The tracer creates a per-cpu kernel thread with real-time priority that prints two lines at every activation. The first is the *timer latency* observed at the *hardirq* context before the activation of the thread. The second is the *timer latency* observed by the thread, which is the same level that cyclictest reports. The ACTIVATION ID field serves to relate the *irq* execution to its respective *thread* execution.

The irq/thread splitting is important to clarify at which context the unexpected high value is coming from. The *irq* context can be delayed by hardware related actions, such as SMIs, NMIs, IRQs or by a thread masking interrupts. Once the timer happens, the delay can also be influenced by blocking caused by threads. For example, by postponing the scheduler execution via preempt_disable(), by the scheduler execution, or by masking interrupts. Threads can also be delayed by the interference from other threads and IRQs.

The timerlat can also take advantage of the osnoise: traceevents. For example:

[root@f32 ~]# cd /sys/kernel/tracing/ [root@f32 tracing]# echo timerlat > current_tracer [root@f32 tracing]# echo osnoise > set_event [root@f32 tracing]# echo 25 > osnoise/stop_tracing_total_us [root@f32 tracing]# tail -10 trace cc1-87882 [005] d..h... 548.771078: #402268 context irq timer_latency 1585 ns cc1-87882 [005] dNLh1.. 548.771082: irq_noise: local_timer:236 start 548.771077442 duration 4597 ns cc1-87882 [005] dNLh2.. 548.771083: irq_noise: reschedule:253 start 548.771083017 duration 56 ns cc1-87882 [005] dNLh2.. 548.771086: irq_noise: call_function_single:251 start 548.771083811 duration 2048 ns cc1-87882 [005] dNLh2.. 548.771088: irq_noise: call_function_single:251 start 548.771086814 duration 1495 ns cc1-87882 [005] dNLh2.. 548.771091: irq_noise: call_function_single:251 start 548.771089194 duration 1558 ns cc1-87882 [005] dNLh2.. 548.771094: irq_noise: call_function_single:251 start 548.771091719 duration 1932 ns cc1-87882 [005] dNLh2.. 548.771096: irq_noise: call_function_single:251 start 548.771094696 duration 1050 ns cc1-87882 [005] d...3.. 548.771101: thread_noise: cc1:87882 start 548.771078243 duration 10909 ns timerlat/5-1035 [005] ....... 548.771103: #402268 context thread timer_latency 25960 ns

For further information see: Documentation/trace/timerlat-tracer.rst

Link: https://lkml.kernel.org/r/71f18efc013e1194bcaea1e54db957de2b19ba62.162437231...

Cc: Phil Auld pauld@redhat.com Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Kate Carcia kcarcia@redhat.com Cc: Jonathan Corbet corbet@lwn.net Cc: Ingo Molnar mingo@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Thomas Gleixner tglx@linutronix.de Cc: Alexandre Chartre alexandre.chartre@oracle.com Cc: Clark Willaims williams@redhat.com Cc: John Kacur jkacur@redhat.com Cc: Juri Lelli juri.lelli@redhat.com Cc: Borislav Petkov bp@alien8.de Cc: "H. Peter Anvin" hpa@zytor.com Cc: x86@kernel.org Cc: linux-doc@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Daniel Bristot de Oliveira bristot@redhat.com Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- Documentation/trace/index.rst | 1 + Documentation/trace/timerlat-tracer.rst | 181 +++++++ kernel/trace/Kconfig | 28 ++ kernel/trace/trace.h | 2 + kernel/trace/trace_entries.h | 16 + kernel/trace/trace_osnoise.c | 626 ++++++++++++++++++++++-- kernel/trace/trace_output.c | 47 ++ 7 files changed, 870 insertions(+), 31 deletions(-) create mode 100644 Documentation/trace/timerlat-tracer.rst

diff --git a/Documentation/trace/index.rst b/Documentation/trace/index.rst index 608107b27cc0..3769b9b7aed8 100644 --- a/Documentation/trace/index.rst +++ b/Documentation/trace/index.rst @@ -24,6 +24,7 @@ Linux Tracing Technologies boottime-trace hwlat_detector osnoise-tracer + timerlat-tracer intel_th ring-buffer-design stm diff --git a/Documentation/trace/timerlat-tracer.rst b/Documentation/trace/timerlat-tracer.rst new file mode 100644 index 000000000000..c7cbb557aee7 --- /dev/null +++ b/Documentation/trace/timerlat-tracer.rst @@ -0,0 +1,181 @@ +############### +Timerlat tracer +############### + +The timerlat tracer aims to help the preemptive kernel developers to +find souces of wakeup latencies of real-time threads. Like cyclictest, +the tracer sets a periodic timer that wakes up a thread. The thread then +computes a *wakeup latency* value as the difference between the *current +time* and the *absolute time* that the timer was set to expire. The main +goal of timerlat is tracing in such a way to help kernel developers. + +Usage +----- + +Write the ASCII text "timerlat" into the current_tracer file of the +tracing system (generally mounted at /sys/kernel/tracing). + +For example:: + + [root@f32 ~]# cd /sys/kernel/tracing/ + [root@f32 tracing]# echo timerlat > current_tracer + +It is possible to follow the trace by reading the trace trace file:: + + [root@f32 tracing]# cat trace + # tracer: timerlat + # + # _-----=> irqs-off + # / _----=> need-resched + # | / _---=> hardirq/softirq + # || / _--=> preempt-depth + # || / + # |||| ACTIVATION + # TASK-PID CPU# |||| TIMESTAMP ID CONTEXT LATENCY + # | | | |||| | | | | + <idle>-0 [000] d.h1 54.029328: #1 context irq timer_latency 932 ns + <...>-867 [000] .... 54.029339: #1 context thread timer_latency 11700 ns + <idle>-0 [001] dNh1 54.029346: #1 context irq timer_latency 2833 ns + <...>-868 [001] .... 54.029353: #1 context thread timer_latency 9820 ns + <idle>-0 [000] d.h1 54.030328: #2 context irq timer_latency 769 ns + <...>-867 [000] .... 54.030330: #2 context thread timer_latency 3070 ns + <idle>-0 [001] d.h1 54.030344: #2 context irq timer_latency 935 ns + <...>-868 [001] .... 54.030347: #2 context thread timer_latency 4351 ns + + +The tracer creates a per-cpu kernel thread with real-time priority that +prints two lines at every activation. The first is the *timer latency* +observed at the *hardirq* context before the activation of the thread. +The second is the *timer latency* observed by the thread. The ACTIVATION +ID field serves to relate the *irq* execution to its respective *thread* +execution. + +The *irq*/*thread* splitting is important to clarify at which context +the unexpected high value is coming from. The *irq* context can be +delayed by hardware related actions, such as SMIs, NMIs, IRQs +or by a thread masking interrupts. Once the timer happens, the delay +can also be influenced by blocking caused by threads. For example, by +postponing the scheduler execution via preempt_disable(), by the +scheduler execution, or by masking interrupts. Threads can +also be delayed by the interference from other threads and IRQs. + +Tracer options +--------------------- + +The timerlat tracer is built on top of osnoise tracer. +So its configuration is also done in the osnoise/ config +directory. The timerlat configs are: + + - cpus: CPUs at which a timerlat thread will execute. + - timerlat_period_us: the period of the timerlat thread. + - osnoise/stop_tracing_us: stop the system tracing if a + timer latency at the *irq* context higher than the configured + value happens. Writing 0 disables this option. + - stop_tracing_total_us: stop the system tracing if a + timer latency at the *thread* context higher than the configured + value happens. Writing 0 disables this option. + - print_stack: save the stack of the IRQ ocurrence, and print + it afte the *thread context* event". + +timerlat and osnoise +---------------------------- + +The timerlat can also take advantage of the osnoise: traceevents. +For example:: + + [root@f32 ~]# cd /sys/kernel/tracing/ + [root@f32 tracing]# echo timerlat > current_tracer + [root@f32 tracing]# echo 1 > events/osnoise/enable + [root@f32 tracing]# echo 25 > osnoise/stop_tracing_total_us + [root@f32 tracing]# tail -10 trace + cc1-87882 [005] d..h... 548.771078: #402268 context irq timer_latency 13585 ns + cc1-87882 [005] dNLh1.. 548.771082: irq_noise: local_timer:236 start 548.771077442 duration 7597 ns + cc1-87882 [005] dNLh2.. 548.771099: irq_noise: qxl:21 start 548.771085017 duration 7139 ns + cc1-87882 [005] d...3.. 548.771102: thread_noise: cc1:87882 start 548.771078243 duration 9909 ns + timerlat/5-1035 [005] ....... 548.771104: #402268 context thread timer_latency 39960 ns + +In this case, the root cause of the timer latency does not point to a +single cause, but to multiple ones. Firstly, the timer IRQ was delayed +for 13 us, which may point to a long IRQ disabled section (see IRQ +stacktrace section). Then the timer interrupt that wakes up the timerlat +thread took 7597 ns, and the qxl:21 device IRQ took 7139 ns. Finally, +the cc1 thread noise took 9909 ns of time before the context switch. +Such pieces of evidence are useful for the developer to use other +tracing methods to figure out how to debug and optimize the system. + +It is worth mentioning that the *duration* values reported +by the osnoise: events are *net* values. For example, the +thread_noise does not include the duration of the overhead caused +by the IRQ execution (which indeed accounted for 12736 ns). But +the values reported by the timerlat tracer (timerlat_latency) +are *gross* values. + +The art below illustrates a CPU timeline and how the timerlat tracer +observes it at the top and the osnoise: events at the bottom. Each "-" +in the timelines means circa 1 us, and the time moves ==>:: + + External timer irq thread + clock latency latency + event 13585 ns 39960 ns + | ^ ^ + v | | + |-------------| | + |-------------+-------------------------| + ^ ^ + ======================================================================== + [tmr irq] [dev irq] + [another thread...^ v..^ v.......][timerlat/ thread] <-- CPU timeline + ========================================================================= + |-------| |-------| + |--^ v-------| + | | | + | | + thread_noise: 9909 ns + | +-> irq_noise: 6139 ns + +-> irq_noise: 7597 ns + +IRQ stacktrace +--------------------------- + +The osnoise/print_stack option is helpful for the cases in which a thread +noise causes the major factor for the timer latency, because of preempt or +irq disabled. For example:: + + [root@f32 tracing]# echo 500 > osnoise/stop_tracing_total_us + [root@f32 tracing]# echo 500 > osnoise/print_stack + [root@f32 tracing]# echo timerlat > current_tracer + [root@f32 tracing]# tail -21 per_cpu/cpu7/trace + insmod-1026 [007] dN.h1.. 200.201948: irq_noise: local_timer:236 start 200.201939376 duration 7872 ns + insmod-1026 [007] d..h1.. 200.202587: #29800 context irq timer_latency 1616 ns + insmod-1026 [007] dN.h2.. 200.202598: irq_noise: local_timer:236 start 200.202586162 duration 11855 ns + insmod-1026 [007] dN.h3.. 200.202947: irq_noise: local_timer:236 start 200.202939174 duration 7318 ns + insmod-1026 [007] d...3.. 200.203444: thread_noise: insmod:1026 start 200.202586933 duration 838681 ns + timerlat/7-1001 [007] ....... 200.203445: #29800 context thread timer_latency 859978 ns + timerlat/7-1001 [007] ....1.. 200.203446: <stack trace> + => timerlat_irq + => __hrtimer_run_queues + => hrtimer_interrupt + => __sysvec_apic_timer_interrupt + => asm_call_irq_on_stack + => sysvec_apic_timer_interrupt + => asm_sysvec_apic_timer_interrupt + => delay_tsc + => dummy_load_1ms_pd_init + => do_one_initcall + => do_init_module + => __do_sys_finit_module + => do_syscall_64 + => entry_SYSCALL_64_after_hwframe + +In this case, it is possible to see that the thread added the highest +contribution to the *timer latency* and the stack trace, saved during +the timerlat IRQ handler, points to a function named +dummy_load_1ms_pd_init, which had the following code (on purpose):: + + static int __init dummy_load_1ms_pd_init(void) + { + preempt_disable(); + mdelay(1); + preempt_enable(); + return 0; + + } diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 146ad9eec221..9682ceb1f3df 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -378,6 +378,34 @@ config OSNOISE_TRACER To enable this tracer, echo in "osnoise" into the current_tracer file.

+config TIMERLAT_TRACER + bool "Timerlat tracer" + select OSNOISE_TRACER + select GENERIC_TRACER + help + The timerlat tracer aims to help the preemptive kernel developers + to find sources of wakeup latencies of real-time threads. + + The tracer creates a per-cpu kernel thread with real-time priority. + The tracer thread sets a periodic timer to wakeup itself, and goes + to sleep waiting for the timer to fire. At the wakeup, the thread + then computes a wakeup latency value as the difference between + the current time and the absolute time that the timer was set + to expire. + + The tracer prints two lines at every activation. The first is the + timer latency observed at the hardirq context before the + activation of the thread. The second is the timer latency observed + by the thread, which is the same level that cyclictest reports. The + ACTIVATION ID field serves to relate the irq execution to its + respective thread execution. + + The tracer is build on top of osnoise tracer, and the osnoise: + events can be used to trace the source of interference from NMI, + IRQs and other threads. It also enables the capture of the + stacktrace at the IRQ context, which helps to identify the code + path that can cause thread delay. + config MMIOTRACE bool "Memory mapped IO tracing" depends on HAVE_MMIOTRACE_SUPPORT && PCI diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 75fd887496a1..46c8a6ac9e3c 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -45,6 +45,7 @@ enum trace_type { TRACE_BPUTS, TRACE_HWLAT, TRACE_OSNOISE, + TRACE_TIMERLAT, TRACE_RAW_DATA,

__TRACE_LAST_TYPE, @@ -453,6 +454,7 @@ extern void __ftrace_bad_type(void); IF_ASSIGN(var, ent, struct bputs_entry, TRACE_BPUTS); \ IF_ASSIGN(var, ent, struct hwlat_entry, TRACE_HWLAT); \ IF_ASSIGN(var, ent, struct osnoise_entry, TRACE_OSNOISE);\ + IF_ASSIGN(var, ent, struct timerlat_entry, TRACE_TIMERLAT);\ IF_ASSIGN(var, ent, struct raw_data_entry, TRACE_RAW_DATA);\ IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \ TRACE_MMIO_RW); \ diff --git a/kernel/trace/trace_entries.h b/kernel/trace/trace_entries.h index f689b7e5c29d..89bc02efe058 100644 --- a/kernel/trace/trace_entries.h +++ b/kernel/trace/trace_entries.h @@ -363,3 +363,19 @@ FTRACE_ENTRY(osnoise, osnoise_entry, __entry->softirq_count, __entry->thread_count) ); + +FTRACE_ENTRY(timerlat, timerlat_entry, + + TRACE_TIMERLAT, + + F_STRUCT( + __field( unsigned int, seqnum ) + __field( int, context ) + __field( u64, timer_latency ) + ), + + F_printk("seq:%u\tcontext:%d\ttimer_latency:%llu\n", + __entry->seqnum, + __entry->context, + __entry->timer_latency) +); diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 4e2c47dc4f19..8546e66bafcb 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* * OS Noise Tracer: computes the OS Noise suffered by a running thread. + * Timerlat Tracer: measures the wakeup latency of a timer triggered IRQ and thread. * * Based on "hwlat_detector" tracer by: * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. jcm@redhat.com @@ -21,6 +22,7 @@ #include <linux/cpumask.h> #include <linux/delay.h> #include <linux/sched/clock.h> +#include <uapi/linux/sched/types.h> #include <linux/sched.h> #include "trace.h"

@@ -45,6 +47,9 @@ static struct trace_array *osnoise_trace; #define DEFAULT_SAMPLE_PERIOD 1000000 /* 1s */ #define DEFAULT_SAMPLE_RUNTIME 1000000 /* 1s */

+#define DEFAULT_TIMERLAT_PERIOD 1000 /* 1ms */ +#define DEFAULT_TIMERLAT_PRIO 95 /* FIFO 95 */ + /* * NMI runtime info. */ @@ -62,6 +67,8 @@ struct osn_irq { u64 delta_start; };

+#define IRQ_CONTEXT 0 +#define THREAD_CONTEXT 1 /* * sofirq runtime info. */ @@ -108,32 +115,76 @@ static inline struct osnoise_variables *this_cpu_osn_var(void) return this_cpu_ptr(&per_cpu_osnoise_var); }

+#ifdef CONFIG_TIMERLAT_TRACER /* - * osn_var_reset - Reset the values of the given osnoise_variables + * Runtime information for the timer mode. + */ +struct timerlat_variables { + struct task_struct *kthread; + struct hrtimer timer; + u64 rel_period; + u64 abs_period; + bool tracing_thread; + u64 count; +}; + +DEFINE_PER_CPU(struct timerlat_variables, per_cpu_timerlat_var); + +/* + * this_cpu_tmr_var - Return the per-cpu timerlat_variables on its relative CPU + */ +static inline struct timerlat_variables *this_cpu_tmr_var(void) +{ + return this_cpu_ptr(&per_cpu_timerlat_var); +} + +/* + * tlat_var_reset - Reset the values of the given timerlat_variables */ -static inline void osn_var_reset(struct osnoise_variables *osn_var) +static inline void tlat_var_reset(void) { + struct timerlat_variables *tlat_var; + int cpu; /* * So far, all the values are initialized as 0, so * zeroing the structure is perfect. */ - memset(osn_var, 0, sizeof(*osn_var)); + for_each_cpu(cpu, cpu_online_mask) { + tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu); + memset(tlat_var, 0, sizeof(*tlat_var)); + } } +#else /* CONFIG_TIMERLAT_TRACER */ +#define tlat_var_reset() do {} while (0) +#endif /* CONFIG_TIMERLAT_TRACER */

/* - * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables + * osn_var_reset - Reset the values of the given osnoise_variables */ -static inline void osn_var_reset_all(void) +static inline void osn_var_reset(void) { struct osnoise_variables *osn_var; int cpu;

+ /* + * So far, all the values are initialized as 0, so + * zeroing the structure is perfect. + */ for_each_cpu(cpu, cpu_online_mask) { osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu); - osn_var_reset(osn_var); + memset(osn_var, 0, sizeof(*osn_var)); } }

+/* + * osn_var_reset_all - Reset the value of all per-cpu osnoise_variables + */ +static inline void osn_var_reset_all(void) +{ + osn_var_reset(); + tlat_var_reset(); +} + /* * Tells NMIs to call back to the osnoise tracer to record timestamps. */ @@ -154,6 +205,18 @@ struct osnoise_sample { int thread_count; /* # threads during this sample */ };

+#ifdef CONFIG_TIMERLAT_TRACER +/* + * timerlat sample structure definition. Used to store the statistics of + * a sample run. + */ +struct timerlat_sample { + u64 timer_latency; /* timer_latency */ + unsigned int seqnum; /* unique sequence */ + int context; /* timer context */ +}; +#endif + /* * Protect the interface. */ @@ -165,14 +228,24 @@ struct mutex interface_lock; static struct osnoise_data { u64 sample_period; /* total sampling period */ u64 sample_runtime; /* active sampling portion of period */ - u64 stop_tracing; /* stop trace in the inside operation (loop) */ - u64 stop_tracing_total; /* stop trace in the outside operation (report) */ + u64 stop_tracing; /* stop trace in the internal operation (loop/irq) */ + u64 stop_tracing_total; /* stop trace in the final operation (report/thread) */ +#ifdef CONFIG_TIMERLAT_TRACER + u64 timerlat_period; /* timerlat period */ + u64 print_stack; /* print IRQ stack if total > */ + int timerlat_tracer; /* timerlat tracer */ +#endif bool tainted; /* infor users and developers about a problem */ } osnoise_data = { .sample_period = DEFAULT_SAMPLE_PERIOD, .sample_runtime = DEFAULT_SAMPLE_RUNTIME, .stop_tracing = 0, .stop_tracing_total = 0, +#ifdef CONFIG_TIMERLAT_TRACER + .print_stack = 0, + .timerlat_period = DEFAULT_TIMERLAT_PERIOD, + .timerlat_tracer = 0, +#endif };

/* @@ -246,6 +319,128 @@ static void trace_osnoise_sample(struct osnoise_sample *sample) trace_buffer_unlock_commit_nostack(buffer, event); }

+#ifdef CONFIG_TIMERLAT_TRACER +/* + * Print the timerlat header info. + */ +static void print_timerlat_headers(struct seq_file *s) +{ + seq_puts(s, "# _-----=> irqs-off\n"); + seq_puts(s, "# / _----=> need-resched\n"); + seq_puts(s, "# | / _---=> hardirq/softirq\n"); + seq_puts(s, "# || / _--=> preempt-depth\n"); + seq_puts(s, "# || /\n"); + seq_puts(s, "# |||| ACTIVATION\n"); + seq_puts(s, "# TASK-PID CPU# |||| TIMESTAMP ID "); + seq_puts(s, " CONTEXT LATENCY\n"); + seq_puts(s, "# | | | |||| | | "); + seq_puts(s, " | |\n"); +} + +/* + * Record an timerlat_sample into the tracer buffer. + */ +static void trace_timerlat_sample(struct timerlat_sample *sample) +{ + struct trace_array *tr = osnoise_trace; + struct trace_event_call *call = &event_osnoise; + struct trace_buffer *buffer = tr->array_buffer.buffer; + struct ring_buffer_event *event; + struct timerlat_entry *entry; + + event = trace_buffer_lock_reserve(buffer, TRACE_TIMERLAT, sizeof(*entry), + tracing_gen_ctx()); + if (!event) + return; + entry = ring_buffer_event_data(event); + entry->seqnum = sample->seqnum; + entry->context = sample->context; + entry->timer_latency = sample->timer_latency; + + if (!call_filter_check_discard(call, entry, buffer, event)) + trace_buffer_unlock_commit_nostack(buffer, event); +} + +#ifdef CONFIG_STACKTRACE + +#define MAX_CALLS 256 + +/* + * Stack trace will take place only at IRQ level, so, no need + * to control nesting here. + */ +struct trace_stack { + int stack_size; + int nr_entries; + unsigned long calls[MAX_CALLS]; +}; + +static DEFINE_PER_CPU(struct trace_stack, trace_stack); + +/* + * timerlat_save_stack - save a stack trace without printing + * + * Save the current stack trace without printing. The + * stack will be printed later, after the end of the measurement. + */ +static void timerlat_save_stack(int skip) +{ + unsigned int size, nr_entries; + struct trace_stack *fstack; + + fstack = this_cpu_ptr(&trace_stack); + + size = ARRAY_SIZE(fstack->calls); + + nr_entries = stack_trace_save(fstack->calls, size, skip); + + fstack->stack_size = nr_entries * sizeof(unsigned long); + fstack->nr_entries = nr_entries; + + return; + +} +/* + * timerlat_dump_stack - dump a stack trace previously saved + * + * Dump a saved stack trace into the trace buffer. + */ +static void timerlat_dump_stack(void) +{ + struct trace_event_call *call = &event_osnoise; + struct trace_array *tr = osnoise_trace; + struct trace_buffer *buffer = tr->array_buffer.buffer; + struct ring_buffer_event *event; + struct trace_stack *fstack; + struct stack_entry *entry; + unsigned int size; + + preempt_disable_notrace(); + fstack = this_cpu_ptr(&trace_stack); + size = fstack->stack_size; + + event = trace_buffer_lock_reserve(buffer, TRACE_STACK, sizeof(*entry) + size, + tracing_gen_ctx()); + if (!event) + goto out; + + entry = ring_buffer_event_data(event); + + memcpy(&entry->caller, fstack->calls, size); + entry->size = fstack->nr_entries; + + if (!call_filter_check_discard(call, entry, buffer, event)) + trace_buffer_unlock_commit_nostack(buffer, event); + +out: + preempt_enable_notrace(); +} +#else +#define timerlat_dump_stack() do {} while (0) +#define timerlat_save_stack(a) do {} while (0) +#endif /* CONFIG_STACKTRACE */ +#endif /* CONFIG_TIMERLAT_TRACER */ + /* * Macros to encapsulate the time capturing infrastructure. */ @@ -387,6 +582,30 @@ set_int_safe_time(struct osnoise_variables *osn_var, u64 *time) return int_counter; }

+#ifdef CONFIG_TIMERLAT_TRACER +/* + * copy_int_safe_time - Copy *src into *desc aware of interference + */ +static u64 +copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src) +{ + u64 int_counter; + + do { + int_counter = local_read(&osn_var->int_counter); + /* synchronize with interrupts */ + barrier(); + + *dst = *src; + + /* synchronize with interrupts */ + barrier(); + } while (int_counter != local_read(&osn_var->int_counter)); + + return int_counter; +} +#endif /* CONFIG_TIMERLAT_TRACER */ + /* * trace_osnoise_callback - NMI entry/exit callback * @@ -597,6 +816,22 @@ void trace_softirq_exit_callback(void *data, unsigned int vec_nr) if (!osn_var->sampling) return;

+#ifdef CONFIG_TIMERLAT_TRACER + /* + * If the timerlat is enabled, but the irq handler did + * not run yet enabling timerlat_tracer, do not trace. + */ + if (unlikely(osnoise_data.timerlat_tracer)) { + struct timerlat_variables *tlat_var; + tlat_var = this_cpu_tmr_var(); + if (!tlat_var->tracing_thread) { + osn_var->softirq.arrival_time = 0; + osn_var->softirq.delta_start = 0; + return; + } + } +#endif + duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start); trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration); cond_move_thread_delta_start(osn_var, duration); @@ -689,6 +924,18 @@ thread_exit(struct osnoise_variables *osn_var, struct task_struct *t) if (!osn_var->sampling) return;

+#ifdef CONFIG_TIMERLAT_TRACER + if (osnoise_data.timerlat_tracer) { + struct timerlat_variables *tlat_var; + tlat_var = this_cpu_tmr_var(); + if (!tlat_var->tracing_thread) { + osn_var->thread.delta_start = 0; + osn_var->thread.arrival_time = 0; + return; + } + } +#endif + duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);

trace_thread_noise(t, osn_var->thread.arrival_time, duration); @@ -979,6 +1226,195 @@ static int osnoise_main(void *data) return 0; }

+#ifdef CONFIG_TIMERLAT_TRACER +/* + * timerlat_irq - hrtimer handler for timerlat. + */ +static enum hrtimer_restart timerlat_irq(struct hrtimer *timer) +{ + struct osnoise_variables *osn_var = this_cpu_osn_var(); + struct trace_array *tr = osnoise_trace; + struct timerlat_variables *tlat; + struct timerlat_sample s; + u64 now; + u64 diff; + + /* + * I am not sure if the timer was armed for this CPU. So, get + * the timerlat struct from the timer itself, not from this + * CPU. + */ + tlat = container_of(timer, struct timerlat_variables, timer); + + now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer)); + + /* + * Enable the osnoise: events for thread an softirq. + */ + tlat->tracing_thread = true; + + osn_var->thread.arrival_time = time_get(); + + /* + * A hardirq is running: the timer IRQ. It is for sure preempting + * a thread, and potentially preempting a softirq. + * + * At this point, it is not interesting to know the duration of the + * preempted thread (and maybe softirq), but how much time they will + * delay the beginning of the execution of the timer thread. + * + * To get the correct (net) delay added by the softirq, its delta_start + * is set as the IRQ one. In this way, at the return of the IRQ, the delta + * start of the sofitrq will be zeroed, accounting then only the time + * after that. + * + * The thread follows the same principle. However, if a softirq is + * running, the thread needs to receive the softirq delta_start. The + * reason being is that the softirq will be the last to be unfolded, + * resseting the thread delay to zero. + */ +#ifndef CONFIG_PREEMPT_RT + if (osn_var->softirq.delta_start) { + copy_int_safe_time(osn_var, &osn_var->thread.delta_start, + &osn_var->softirq.delta_start); + + copy_int_safe_time(osn_var, &osn_var->softirq.delta_start, + &osn_var->irq.delta_start); + } else { + copy_int_safe_time(osn_var, &osn_var->thread.delta_start, + &osn_var->irq.delta_start); + } +#else /* CONFIG_PREEMPT_RT */ + /* + * The sofirqs run as threads on RT, so there is not need + * to keep track of it. + */ + copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->irq.delta_start); +#endif /* CONFIG_PREEMPT_RT */ + + /* + * Compute the current time with the expected time. + */ + diff = now - tlat->abs_period; + + tlat->count++; + s.seqnum = tlat->count; + s.timer_latency = diff; + s.context = IRQ_CONTEXT; + + trace_timerlat_sample(&s); + + /* Keep a running maximum ever recorded os noise "latency" */ + if (diff > tr->max_latency) { + tr->max_latency = diff; + latency_fsnotify(tr); + } + + if (osnoise_data.stop_tracing) + if (time_to_us(diff) >= osnoise_data.stop_tracing) + osnoise_stop_tracing(); + + wake_up_process(tlat->kthread); + + if (osnoise_data.print_stack) + timerlat_save_stack(0); + + return HRTIMER_NORESTART; +} + +/* + * wait_next_period - Wait for the next period for timerlat + */ +static int wait_next_period(struct timerlat_variables *tlat) +{ + ktime_t next_abs_period, now; + u64 rel_period = osnoise_data.timerlat_period * 1000; + + now = hrtimer_cb_get_time(&tlat->timer); + next_abs_period = ns_to_ktime(tlat->abs_period + rel_period); + + /* + * Save the next abs_period. + */ + tlat->abs_period = (u64) ktime_to_ns(next_abs_period); + + /* + * If the new abs_period is in the past, skip the activation. + */ + while (ktime_compare(now, next_abs_period) > 0) { + next_abs_period = ns_to_ktime(tlat->abs_period + rel_period); + tlat->abs_period = (u64) ktime_to_ns(next_abs_period); + } + + set_current_state(TASK_INTERRUPTIBLE); + + hrtimer_start(&tlat->timer, next_abs_period, HRTIMER_MODE_ABS_PINNED_HARD); + schedule(); + return 1; +} + +/* + * timerlat_main- Timerlat main + */ +static int timerlat_main(void *data) +{ + struct osnoise_variables *osn_var = this_cpu_osn_var(); + struct timerlat_variables *tlat = this_cpu_tmr_var(); + struct timerlat_sample s; + struct sched_param sp; + u64 now, diff; + + /* + * Make the thread RT, that is how cyclictest is usually used. + */ + sp.sched_priority = DEFAULT_TIMERLAT_PRIO; + sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); + + tlat->count = 0; + tlat->tracing_thread = false; + + hrtimer_init(&tlat->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD); + tlat->timer.function = timerlat_irq; + tlat->kthread = current; + osn_var->pid = current->pid; + /* + * Anotate the arrival time. + */ + tlat->abs_period = hrtimer_cb_get_time(&tlat->timer); + + wait_next_period(tlat); + + osn_var->sampling = 1; + + while (!kthread_should_stop()) { + now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer)); + diff = now - tlat->abs_period; + + s.seqnum = tlat->count; + s.timer_latency = diff; + s.context = THREAD_CONTEXT; + + trace_timerlat_sample(&s); + +#ifdef CONFIG_STACKTRACE + if (osnoise_data.print_stack) + if (osnoise_data.print_stack <= time_to_us(diff)) + timerlat_dump_stack(); +#endif /* CONFIG_STACKTRACE */ + + tlat->tracing_thread = false; + if (osnoise_data.stop_tracing_total) + if (time_to_us(diff) >= osnoise_data.stop_tracing_total) + osnoise_stop_tracing(); + + wait_next_period(tlat); + } + + hrtimer_cancel(&tlat->timer); + return 0; +} +#endif /* CONFIG_TIMERLAT_TRACER */ + /* * stop_per_cpu_kthread - stop per-cpu threads * @@ -1009,6 +1445,7 @@ static int start_per_cpu_kthreads(struct trace_array *tr) struct cpumask *current_mask = &save_cpumask; struct task_struct *kthread; char comm[24]; + void *main = osnoise_main; int cpu;

get_online_cpus(); @@ -1026,9 +1463,17 @@ static int start_per_cpu_kthreads(struct trace_array *tr) per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;

for_each_cpu(cpu, current_mask) { +#ifdef CONFIG_TIMERLAT_TRACER + if (osnoise_data.timerlat_tracer) { + snprintf(comm, 24, "timerlat/%d", cpu); + main = timerlat_main; + } else { + snprintf(comm, 24, "osnoise/%d", cpu); + } +#else snprintf(comm, 24, "osnoise/%d", cpu); - - kthread = kthread_create_on_cpu(osnoise_main, NULL, cpu, comm); +#endif + kthread = kthread_create_on_cpu(main, NULL, cpu, comm);

if (IS_ERR(kthread)) { pr_err(BANNER "could not start sampling thread\n"); @@ -1194,6 +1639,31 @@ static struct trace_min_max_param osnoise_stop_tracing_total = { .min = NULL, };

+#ifdef CONFIG_TIMERLAT_TRACER +/* + * osnoise/print_stack: print the stacktrace of the IRQ handler if the total + * latency is higher than val. + */ +static struct trace_min_max_param osnoise_print_stack = { + .lock = &interface_lock, + .val = &osnoise_data.print_stack, + .max = NULL, + .min = NULL, +}; + +/* + * osnoise/timerlat_period: min 100 us, max 1 s + */ +u64 timerlat_min_period = 100; +u64 timerlat_max_period = 1000000; +static struct trace_min_max_param timerlat_period = { + .lock = &interface_lock, + .val = &osnoise_data.timerlat_period, + .max = &timerlat_max_period, + .min = &timerlat_min_period, +}; +#endif + static const struct file_operations cpus_fops = { .open = tracing_open_generic, .read = osnoise_cpus_read, @@ -1204,10 +1674,9 @@ static const struct file_operations cpus_fops = { /* * init_tracefs - A function to initialize the tracefs interface files * - * This function creates entries in tracefs for "osnoise". It creates the - * "osnoise" directory in the tracing directory, and within that - * directory is the count, runtime and period files to change and view - * those values. + * This function creates entries in tracefs for "osnoise" and "timerlat". + * It creates these directories in the tracing directory, and within that + * directory the use can change and view the configs. */ static int init_tracefs(void) { @@ -1221,7 +1690,7 @@ static int init_tracefs(void)

top_dir = tracefs_create_dir("osnoise", NULL); if (!top_dir) - return -ENOMEM; + return 0;

tmp = tracefs_create_file("period_us", 0640, top_dir, &osnoise_period, &trace_min_max_fops); @@ -1246,6 +1715,19 @@ static int init_tracefs(void) tmp = trace_create_file("cpus", 0644, top_dir, NULL, &cpus_fops); if (!tmp) goto err; +#ifdef CONFIG_TIMERLAT_TRACER +#ifdef CONFIG_STACKTRACE + tmp = tracefs_create_file("print_stack", 0640, top_dir, + &osnoise_print_stack, &trace_min_max_fops); + if (!tmp) + goto err; +#endif + + tmp = tracefs_create_file("timerlat_period_us", 0640, top_dir, + &timerlat_period, &trace_min_max_fops); + if (!tmp) + goto err; +#endif

return 0;

@@ -1286,18 +1768,15 @@ static int osnoise_hook_events(void) return -EINVAL; }

-static void osnoise_tracer_start(struct trace_array *tr) +static int __osnoise_tracer_start(struct trace_array *tr) { int retval;

- if (osnoise_busy) - return; - osn_var_reset_all();

retval = osnoise_hook_events(); if (retval) - goto out_err; + return retval; /* * Make sure NMIs see reseted values. */ @@ -1305,15 +1784,27 @@ static void osnoise_tracer_start(struct trace_array *tr) trace_osnoise_callback_enabled = true;

retval = start_per_cpu_kthreads(tr); - /* - * all fine! - */ - if (!retval) + if (retval) { + unhook_irq_events(); + return retval; + } + + osnoise_busy = true; + + return 0; +} + +static void osnoise_tracer_start(struct trace_array *tr) +{ + int retval; + + if (osnoise_busy) return;

-out_err: - unhook_irq_events(); - pr_err(BANNER "Error starting osnoise tracer\n"); + retval = __osnoise_tracer_start(tr); + if (retval) + pr_err(BANNER "Error starting osnoise tracer\n"); + }

static void osnoise_tracer_stop(struct trace_array *tr) @@ -1335,18 +1826,16 @@ static void osnoise_tracer_stop(struct trace_array *tr)

static int osnoise_tracer_init(struct trace_array *tr) { + /* Only allow one instance to enable this */ if (osnoise_busy) return -EBUSY;

osnoise_trace = tr; - tr->max_latency = 0;

osnoise_tracer_start(tr);

- osnoise_busy = true; - return 0; }

@@ -1365,6 +1854,71 @@ static struct tracer osnoise_tracer __read_mostly = { .allow_instances = true, };

+#ifdef CONFIG_TIMERLAT_TRACER +static void timerlat_tracer_start(struct trace_array *tr) +{ + int retval; + + if (osnoise_busy) + return; + + osnoise_data.timerlat_tracer = 1; + + retval = __osnoise_tracer_start(tr); + if (retval) + goto out_err; + + return; +out_err: + pr_err(BANNER "Error starting timerlat tracer\n"); +} + +static void timerlat_tracer_stop(struct trace_array *tr) +{ + int cpu; + + if (!osnoise_busy) + return; + + for_each_online_cpu(cpu) + per_cpu(per_cpu_osnoise_var, cpu).sampling = 0; + + osnoise_tracer_stop(tr); + + osnoise_data.timerlat_tracer = 0; +} + +static int timerlat_tracer_init(struct trace_array *tr) +{ + /* Only allow one instance to enable this */ + if (osnoise_busy) + return -EBUSY; + + osnoise_trace = tr; + + tr->max_latency = 0; + + timerlat_tracer_start(tr); + + return 0; +} + +static void timerlat_tracer_reset(struct trace_array *tr) +{ + timerlat_tracer_stop(tr); +} + +static struct tracer timerlat_tracer __read_mostly = { + .name = "timerlat", + .init = timerlat_tracer_init, + .reset = timerlat_tracer_reset, + .start = timerlat_tracer_start, + .stop = timerlat_tracer_stop, + .print_header = print_timerlat_headers, + .allow_instances = true, +}; +#endif /* CONFIG_TIMERLAT_TRACER */ + __init static int init_osnoise_tracer(void) { int ret; @@ -1374,8 +1928,18 @@ __init static int init_osnoise_tracer(void) cpumask_copy(&osnoise_cpumask, cpu_all_mask);

ret = register_tracer(&osnoise_tracer); - if (ret) + if (ret) { + pr_err(BANNER "Error registering osnoise!\n"); return ret; + } + +#ifdef CONFIG_TIMERLAT_TRACER + ret = register_tracer(&timerlat_tracer); + if (ret) { + pr_err(BANNER "Error registering timerlat\n"); + return ret; + } +#endif

init_tracefs();

diff --git a/kernel/trace/trace_output.c b/kernel/trace/trace_output.c index f1dce388121a..7042544c5bde 100644 --- a/kernel/trace/trace_output.c +++ b/kernel/trace/trace_output.c @@ -1278,6 +1278,52 @@ static struct trace_event trace_osnoise_event = { .funcs = &trace_osnoise_funcs, };

+/* TRACE_TIMERLAT */ +static enum print_line_t +trace_timerlat_print(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct trace_entry *entry = iter->ent; + struct trace_seq *s = &iter->seq; + struct timerlat_entry *field; + + trace_assign_type(field, entry); + + trace_seq_printf(s, "#%-5u context %6s timer_latency %9llu ns\n", + field->seqnum, + field->context ? "thread" : "irq", + field->timer_latency); + + return trace_handle_return(s); +} + +static enum print_line_t +trace_timerlat_raw(struct trace_iterator *iter, int flags, + struct trace_event *event) +{ + struct timerlat_entry *field; + struct trace_seq *s = &iter->seq; + + trace_assign_type(field, iter->ent); + + trace_seq_printf(s, "%u %d %llu\n", + field->seqnum, + field->context, + field->timer_latency); + + return trace_handle_return(s); +} + +static struct trace_event_functions trace_timerlat_funcs = { + .trace = trace_timerlat_print, + .raw = trace_timerlat_raw, +}; + +static struct trace_event trace_timerlat_event = { + .type = TRACE_TIMERLAT, + .funcs = &trace_timerlat_funcs, +}; + /* TRACE_BPUTS */ static enum print_line_t trace_bputs_print(struct trace_iterator *iter, int flags, @@ -1444,6 +1490,7 @@ static struct trace_event *events[] __initdata = { &trace_print_event, &trace_hwlat_event, &trace_osnoise_event, + &trace_timerlat_event, &trace_raw_data_event, NULL };

From: Daniel Bristot de Oliveira bristot@kernel.org

mainline inclusion from mainline-v5.16-rc1 commit 9bd985766a43ac0115f13f67783d381ebcba70c6 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

s/CONFIG_OSNOISE_TRAECR/CONFIG_OSNOISE_TRACER/

No functional changes.

Link: https://lkml.kernel.org/r/33924a16f6e5559ce24952ca7d62561604bfd94a.163430838...

Cc: Daniel Bristot de Oliveira bristot@kernel.org Cc: Jonathan Corbet corbet@lwn.net Cc: Thomas Gleixner tglx@linutronix.de Cc: Ingo Molnar mingo@redhat.com Cc: Borislav Petkov bp@alien8.de Cc: "H. Peter Anvin" hpa@zytor.com Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: x86@kernel.org Cc: linux-doc@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Daniel Bristot de Oliveira bristot@kernel.org Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/x86/kernel/trace.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/kernel/trace.c b/arch/x86/kernel/trace.c index 6912672c33a7..ebcde0f86769 100644 --- a/arch/x86/kernel/trace.c +++ b/arch/x86/kernel/trace.c @@ -234,4 +234,4 @@ void osnoise_arch_unregister(void) unregister_trace_local_timer_exit(trace_intel_irq_exit, "local_timer"); unregister_trace_local_timer_entry(trace_intel_irq_entry, NULL); } -#endif /* CONFIG_OSNOISE_TRAECR && CONFIG_X86_LOCAL_APIC */ +#endif /* CONFIG_OSNOISE_TRACER && CONFIG_X86_LOCAL_APIC */

From: Daniel Bristot de Oliveira bristot@redhat.com

mainline inclusion from mainline-v5.14-rc1 commit c8895e271f7994a3ecb13b8a280e39aa53879545 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

Enable and disable osnoise/timerlat thread during on CPU hotplug online and offline operations respectivelly.

Link: https://lore.kernel.org/linux-doc/20210621134636.5b332226@oasis.local.home/ Link: https://lkml.kernel.org/r/39f98590b3caeb3c32f09526214058efe0e9272a.162437231...

Cc: Phil Auld pauld@redhat.com Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Kate Carcia kcarcia@redhat.com Cc: Jonathan Corbet corbet@lwn.net Cc: Ingo Molnar mingo@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Thomas Gleixner tglx@linutronix.de Cc: Alexandre Chartre alexandre.chartre@oracle.com Cc: Clark Willaims williams@redhat.com Cc: John Kacur jkacur@redhat.com Cc: Juri Lelli juri.lelli@redhat.com Cc: Borislav Petkov bp@alien8.de Cc: "H. Peter Anvin" hpa@zytor.com Cc: x86@kernel.org Cc: linux-doc@vger.kernel.org Cc: linux-kernel@vger.kernel.org Suggested-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Daniel Bristot de Oliveira bristot@redhat.com Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 165 ++++++++++++++++++++++++++++------- 1 file changed, 135 insertions(+), 30 deletions(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 5ea4c7e6f888..35e9fb9edfb5 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1449,22 +1449,67 @@ static int timerlat_main(void *data) #endif /* CONFIG_TIMERLAT_TRACER */

/* - * stop_per_cpu_kthread - stop per-cpu threads + * stop_kthread - stop a workload thread + */ +static void stop_kthread(unsigned int cpu) +{ + struct task_struct *kthread; + + kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread; + if (kthread) + kthread_stop(kthread); + per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL; +} + +/* + * stop_per_cpu_kthread - Stop per-cpu threads * * Stop the osnoise sampling htread. Use this on unload and at system * shutdown. */ static void stop_per_cpu_kthreads(void) { - struct task_struct *kthread; int cpu;

- for_each_online_cpu(cpu) { - kthread = per_cpu(per_cpu_osnoise_var, cpu).kthread; - if (kthread) - kthread_stop(kthread); - per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL; + get_online_cpus(); + + for_each_online_cpu(cpu) + stop_kthread(cpu); + + put_online_cpus(); +} + +/* + * start_kthread - Start a workload tread + */ +static int start_kthread(unsigned int cpu) +{ + struct task_struct *kthread; + void *main = osnoise_main; + char comm[24]; + +#ifdef CONFIG_TIMERLAT_TRACER + if (osnoise_data.timerlat_tracer) { + snprintf(comm, 24, "timerlat/%d", cpu); + main = timerlat_main; + } else { + snprintf(comm, 24, "osnoise/%d", cpu); } +#else + snprintf(comm, 24, "osnoise/%d", cpu); +#endif + kthread = kthread_create_on_cpu(main, NULL, cpu, comm); + + if (IS_ERR(kthread)) { + pr_err(BANNER "could not start sampling thread\n"); + stop_per_cpu_kthreads(); + return -ENOMEM; + } + + per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread; + wake_up_process(kthread); + + return 0; }

/* @@ -1476,9 +1521,7 @@ static void stop_per_cpu_kthreads(void) static int start_per_cpu_kthreads(struct trace_array *tr) { struct cpumask *current_mask = &save_cpumask; - struct task_struct *kthread; - char comm[24]; - void *main = osnoise_main; + int retval; int cpu;

get_online_cpus(); @@ -1490,37 +1533,91 @@ static int start_per_cpu_kthreads(struct trace_array *tr) * And the CPU is online. */ cpumask_and(current_mask, cpu_online_mask, current_mask); - put_online_cpus();

- for_each_online_cpu(cpu) + for_each_possible_cpu(cpu) per_cpu(per_cpu_osnoise_var, cpu).kthread = NULL;

for_each_cpu(cpu, current_mask) { -#ifdef CONFIG_TIMERLAT_TRACER - if (osnoise_data.timerlat_tracer) { - snprintf(comm, 24, "timerlat/%d", cpu); - main = timerlat_main; - } else { - snprintf(comm, 24, "osnoise/%d", cpu); - } -#else - snprintf(comm, 24, "osnoise/%d", cpu); -#endif - kthread = kthread_create_on_cpu(main, NULL, cpu, comm); - - if (IS_ERR(kthread)) { - pr_err(BANNER "could not start sampling thread\n"); + retval = start_kthread(cpu); + if (retval) { stop_per_cpu_kthreads(); - return -ENOMEM; + return retval; } - - per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread; - wake_up_process(kthread); }

+ put_online_cpus(); + return 0; }

+#ifdef CONFIG_HOTPLUG_CPU +static void osnoise_hotplug_workfn(struct work_struct *dummy) +{ + struct trace_array *tr = osnoise_trace; + unsigned int cpu = smp_processor_id(); + + + mutex_lock(&trace_types_lock); + + if (!osnoise_busy) + goto out_unlock_trace; + + mutex_lock(&interface_lock); + get_online_cpus(); + + if (!cpumask_test_cpu(cpu, &osnoise_cpumask)) + goto out_unlock; + + if (!cpumask_test_cpu(cpu, tr->tracing_cpumask)) + goto out_unlock; + + start_kthread(cpu); + +out_unlock: + put_online_cpus(); + mutex_unlock(&interface_lock); +out_unlock_trace: + mutex_unlock(&trace_types_lock); +} + +static DECLARE_WORK(osnoise_hotplug_work, osnoise_hotplug_workfn); + +/* + * osnoise_cpu_init - CPU hotplug online callback function + */ +static int osnoise_cpu_init(unsigned int cpu) +{ + schedule_work_on(cpu, &osnoise_hotplug_work); + return 0; +} + +/* + * osnoise_cpu_die - CPU hotplug offline callback function + */ +static int osnoise_cpu_die(unsigned int cpu) +{ + stop_kthread(cpu); + return 0; +} + +static void osnoise_init_hotplug_support(void) +{ + int ret; + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "trace/osnoise:online", + osnoise_cpu_init, osnoise_cpu_die); + if (ret < 0) + pr_warn(BANNER "Error to init cpu hotplug support\n"); + + return; +} +#else /* CONFIG_HOTPLUG_CPU */ +static void osnoise_init_hotplug_support(void) +{ + return 0; +} +#endif /* CONFIG_HOTPLUG_CPU */ + /* * osnoise_cpus_read - Read function for reading the "cpus" file * @filp: The active open file structure @@ -1616,7 +1713,14 @@ osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, osnoise_tracer_stop(tr);

mutex_lock(&interface_lock); + /* + * osnoise_cpumask is read by CPU hotplug operations. + */ + get_online_cpus(); + cpumask_copy(&osnoise_cpumask, osnoise_cpumask_new); + + put_online_cpus(); mutex_unlock(&interface_lock);

if (running) @@ -1973,6 +2077,7 @@ __init static int init_osnoise_tracer(void) return ret; } #endif + osnoise_init_hotplug_support();

init_tracefs();

From: Daniel Bristot de Oliveira bristot@redhat.com

mainline inclusion from mainline-v5.14-rc1 commit 498627b4ac85780b9962ed9b5c5abbefd884ef8e category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

kernel test robot reported:

...

kernel/trace/trace_osnoise.c:1584:2: error: void function

'osnoise_init_hotplug_support' should not return a value [-Wreturn-type] return 0;

When !CONFIG_HOTPLUG_CPU.

Fix it problem by removing the return value.

Link: https://lkml.kernel.org/r/c7fc67f1a117cc88bab2e508c898634872795341.162487260...

Fixes: c8895e271f79 ("trace/osnoise: Support hotplug operations") Reported-by: kernel test robot lkp@intel.com Signed-off-by: Daniel Bristot de Oliveira bristot@redhat.com Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index e1ed353dfdc6..93be0b918eb2 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1614,7 +1614,7 @@ static void osnoise_init_hotplug_support(void) #else /* CONFIG_HOTPLUG_CPU */ static void osnoise_init_hotplug_support(void) { - return 0; + return; } #endif /* CONFIG_HOTPLUG_CPU */

From: Colin Ian King colin.king@canonical.com

mainline inclusion from mainline-v5.14-rc1 commit b62613b431bdababc90bf1440b2c7427172d94f4 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

There is a spelling mistake in a TP_printk message, the word interferences is not the plural of interference. Fix this.

Link: https://lkml.kernel.org/r/20210628125522.56361-1-colin.king@canonical.com

Reviewed-by: Daniel Bristot de Oliveira bristot@redhat.com Signed-off-by: Colin Ian King colin.king@canonical.com Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/trace/events/osnoise.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/include/trace/events/osnoise.h b/include/trace/events/osnoise.h index 28762c69f6c9..82f741ec0f57 100644 --- a/include/trace/events/osnoise.h +++ b/include/trace/events/osnoise.h @@ -129,7 +129,7 @@ TRACE_EVENT(sample_threshold, __entry->interference = interference; ),

- TP_printk("start %llu.%09u duration %llu ns interferences %llu", + TP_printk("start %llu.%09u duration %llu ns interference %llu", __print_ns_to_secs(__entry->start), __print_ns_without_secs(__entry->start), __entry->duration,

From: Daniel Bristot de Oliveira bristot@kernel.org

mainline inclusion from mainline-v5.14-rc7 commit 0e05ba498dd0a19fc12868a9506be0f86cf36912 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

When using osnoise/timerlat with stop tracing, sometimes it is not clear in which CPU the stop condition was hit, mainly when using some extra events.

Print a message informing in which CPU the trace stopped, like in the example below:

<idle>-0 [006] d.h. 2932.676616: #1672599 context irq timer_latency 34689 ns <idle>-0 [006] dNh. 2932.676618: irq_noise: local_timer:236 start 2932.676615639 duration 2391 ns <idle>-0 [006] dNh. 2932.676620: irq_noise: virtio0-output.0:47 start 2932.676620180 duration 86 ns <idle>-0 [003] d.h. 2932.676621: #1673374 context irq timer_latency 1200 ns <idle>-0 [006] d... 2932.676623: thread_noise: swapper/6:0 start 2932.676615964 duration 4339 ns <idle>-0 [003] dNh. 2932.676623: irq_noise: local_timer:236 start 2932.676620597 duration 1881 ns <idle>-0 [006] d... 2932.676623: sched_switch: prev_comm=swapper/6 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=timerlat/6 next_pid=852 next_prio=4 timerlat/6-852 [006] .... 2932.676623: #1672599 context thread timer_latency 41931 ns <idle>-0 [003] d... 2932.676623: thread_noise: swapper/3:0 start 2932.676620854 duration 880 ns <idle>-0 [003] d... 2932.676624: sched_switch: prev_comm=swapper/3 prev_pid=0 prev_prio=120 prev_state=R ==> next_comm=timerlat/3 next_pid=849 next_prio=4 timerlat/6-852 [006] .... 2932.676624: timerlat_main: stop tracing hit on cpu 6 timerlat/3-849 [003] .... 2932.676624: #1673374 context thread timer_latency 4310 ns

Link: https://lkml.kernel.org/r/b30a0d7542adba019185f44ee648e60e14923b11.162659884...

Cc: Tom Zanussi zanussi@kernel.org Cc: Namhyung Kim namhyung@kernel.org Cc: Masami Hiramatsu mhiramat@kernel.org Signed-off-by: Daniel Bristot de Oliveira bristot@kernel.org Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index b4b07af1acbc..ff25bb827796 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1056,9 +1056,13 @@ diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample * /* * osnoise_stop_tracing - Stop tracing and the tracer. */ -static void osnoise_stop_tracing(void) +static __always_inline void osnoise_stop_tracing(void) { struct trace_array *tr = osnoise_trace; + + trace_array_printk_buf(tr->array_buffer.buffer, _THIS_IP_, + "stop tracing hit on cpu %d\n", smp_processor_id()); + tracer_tracing_off(tr); }

From: Jackie Liu liuyun01@kylinos.cn

mainline inclusion from mainline-v5.15-rc6 commit 424b650f35c77defbb3cbd6e5221d3697af42250 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

The compiler warns when the data are actually unused:

kernel/trace/trace.c:1712:13: error: ‘trace_create_maxlat_file’ defined but not used [-Werror=unused-function] 1712 | static void trace_create_maxlat_file(struct trace_array *tr, | ^~~~~~~~~~~~~~~~~~~~~~~~

[Why] CONFIG_HWLAT_TRACER=n, CONFIG_TRACER_MAX_TRACE=n, CONFIG_OSNOISE_TRACER=y gcc report warns.

[How] Now trace_create_maxlat_file will only take effect when CONFIG_HWLAT_TRACER=y or CONFIG_TRACER_MAX_TRACE=y. In fact, after adding osnoise trace, it also needs to take effect.

Link: https://lore.kernel.org/all/c1d9e328-ad7c-920b-6c24-9e1598a6421c@infradead.o... Link: https://lkml.kernel.org/r/20210922025122.3268022-1-liu.yun@linux.dev

Fixes: bce29ac9ce0b ("trace: Add osnoise tracer") Cc: Daniel Bristot de Oliveira bristot@redhat.com Suggested-by: Steven Rostedt rostedt@goodmis.org Reviewed-by: Daniel Bristot de Oliveira bristot@kernel.org Tested-by: Randy Dunlap rdunlap@infradead.org # build-tested Signed-off-by: Jackie Liu liuyun01@kylinos.cn Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 2573a42e270a..3499324efe20 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1733,16 +1733,15 @@ void latency_fsnotify(struct trace_array *tr) irq_work_queue(&tr->fsnotify_irqwork); }

-/* - * (defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)) && \ - * defined(CONFIG_FSNOTIFY) - */ -#else +#elif defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) \ + || defined(CONFIG_OSNOISE_TRACER)

#define trace_create_maxlat_file(tr, d_tracer) \ trace_create_file("tracing_max_latency", 0644, d_tracer, \ &tr->max_latency, &tracing_max_lat_fops)

+#else +#define trace_create_maxlat_file(tr, d_tracer) do { } while (0) #endif

#ifdef CONFIG_TRACER_MAX_TRACE @@ -9093,9 +9092,7 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)

create_trace_options_dir(tr);

-#if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER) trace_create_maxlat_file(tr, d_tracer); -#endif

if (ftrace_create_function_files(tr, d_tracer)) MEM_FAIL(1, "Could not allocate function filter files");

From: Daniel Bristot de Oliveira bristot@kernel.org

mainline inclusion from mainline-v5.16-rc1 commit c3b6343c0dc4a76f838e25391f6f1cdb25cfbb8c category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

trace_osnoise_callback_enabled is used by ftrace_nmi_enter/exit() to know when to call the NMI callback. The barrier is used to avoid having callbacks enabled before the resetting date during the start or to touch the values after stopping the tracer.

Link: https://lkml.kernel.org/r/a413b8f14aa9312fbd1ba99f96225a8aed831053.163570289...

Cc: Ingo Molnar mingo@redhat.com Cc: Tom Zanussi zanussi@kernel.org Cc: Masami Hiramatsu mhiramat@kernel.org Cc: Juri Lelli juri.lelli@redhat.com Cc: Clark Williams williams@redhat.com Cc: John Kacur jkacur@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Thomas Gleixner tglx@linutronix.de Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Daniel Bristot de Oliveira bristot@kernel.org Cc: linux-rt-users@vger.kernel.org Cc: linux-trace-devel@vger.kernel.org Cc: linux-kernel@vger.kernel.org Suggested-by: Steven Rostedt rostedt@goodmis.org Signed-off-by: Daniel Bristot de Oliveira bristot@kernel.org Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index b3f1c09870ca..2bfc4f10f7d6 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1907,8 +1907,10 @@ static int __osnoise_tracer_start(struct trace_array *tr) retval = osnoise_hook_events(); if (retval) return retval; + /* - * Make sure NMIs see reseted values. + * Make sure that ftrace_nmi_enter/exit() see reset values + * before enabling trace_osnoise_callback_enabled. */ barrier(); trace_osnoise_callback_enabled = true; @@ -1943,6 +1945,10 @@ static void osnoise_tracer_stop(struct trace_array *tr) return;

trace_osnoise_callback_enabled = false; + /* + * Make sure that ftrace_nmi_enter/exit() see + * trace_osnoise_callback_enabled as false before continuing. + */ barrier();

stop_per_cpu_kthreads();

From: Daniel Bristot de Oliveira bristot@kernel.org

mainline inclusion from mainline-v5.16-rc1 commit 2bd1bdf01fb25906f18cd8ebfac81c2217d1478a category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

When writing a new CPU mask via osnoise/cpus, if the tracer is running, the workload is restarted to follow the new cpumask. The restart is currently done using osnoise_workload_start/stop(), which disables the workload *and* the instrumentation. However, disabling the instrumentation is not necessary.

Calling start/stop_per_cpu_kthreads() is enough to apply the new osnoise/cpus config.

Link: https://lkml.kernel.org/r/ee633e82867c5b88851aa6040522a799c0034486.163570289...

Cc: Ingo Molnar mingo@redhat.com Cc: Tom Zanussi zanussi@kernel.org Cc: Masami Hiramatsu mhiramat@kernel.org Cc: Juri Lelli juri.lelli@redhat.com Cc: Clark Williams williams@redhat.com Cc: John Kacur jkacur@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Thomas Gleixner tglx@linutronix.de Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Daniel Bristot de Oliveira bristot@kernel.org Cc: linux-rt-users@vger.kernel.org Cc: linux-trace-devel@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Daniel Bristot de Oliveira bristot@kernel.org Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 225888c9164b..d6cedd2e039b 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1655,9 +1655,6 @@ osnoise_cpus_read(struct file *filp, char __user *ubuf, size_t count, return count; }

-static int osnoise_workload_start(void); -static void osnoise_workload_stop(void); - /* * osnoise_cpus_write - Write function for "cpus" entry * @filp: The active open file structure @@ -1702,7 +1699,7 @@ osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, mutex_lock(&trace_types_lock); running = osnoise_busy; if (running) - osnoise_workload_stop(); + stop_per_cpu_kthreads();

mutex_lock(&interface_lock); /* @@ -1716,7 +1713,7 @@ osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, mutex_unlock(&interface_lock);

if (running) - osnoise_workload_start(); + start_per_cpu_kthreads(); mutex_unlock(&trace_types_lock);

free_cpumask_var(osnoise_cpumask_new);

From: "Steven Rostedt (VMware)" rostedt@goodmis.org

mainline inclusion from mainline-v5.16-rc1 commit 21ccc9cd72116289469e5519b6159c675a2fa58f category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

When building the files in the tracefs file system, do not by default set any permissions for OTH (other). This will make it easier for admins who want to define a group for accessing tracefs and not having to first disable all the permission bits for "other" in the file system.

As tracing can leak sensitive information, it should never by default allowing all users access. An admin can still set the permission bits for others to have access, which may be useful for creating a honeypot and seeing who takes advantage of it and roots the machine.

Link: https://lkml.kernel.org/r/20210818153038.864149276@goodmis.org

Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/ftrace.c | 23 ++++----- kernel/trace/trace.c | 73 ++++++++++++++-------------- kernel/trace/trace.h | 3 ++ kernel/trace/trace_dynevent.c | 2 +- kernel/trace/trace_events.c | 37 +++++++------- kernel/trace/trace_events_synth.c | 4 +- kernel/trace/trace_functions_graph.c | 2 +- kernel/trace/trace_hwlat.c | 4 +- kernel/trace/trace_kprobe.c | 8 +-- kernel/trace/trace_osnoise.c | 10 ++-- kernel/trace/trace_printk.c | 2 +- kernel/trace/trace_stack.c | 6 +-- kernel/trace/trace_stat.c | 6 +-- kernel/trace/trace_uprobe.c | 4 +- 14 files changed, 95 insertions(+), 89 deletions(-)

diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 4a5d35dc490b..a1c517ed769b 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -988,8 +988,9 @@ static __init void ftrace_profile_tracefs(struct dentry *d_tracer) } }

- entry = tracefs_create_file("function_profile_enabled", 0644, - d_tracer, NULL, &ftrace_profile_fops); + entry = tracefs_create_file("function_profile_enabled", + TRACE_MODE_WRITE, d_tracer, NULL, + &ftrace_profile_fops); if (!entry) pr_warn("Could not create tracefs 'function_profile_enabled' entry\n"); } @@ -6117,10 +6118,10 @@ void ftrace_create_filter_files(struct ftrace_ops *ops, struct dentry *parent) {

- trace_create_file("set_ftrace_filter", 0644, parent, + trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent, ops, &ftrace_filter_fops);

- trace_create_file("set_ftrace_notrace", 0644, parent, + trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent, ops, &ftrace_notrace_fops); }

@@ -6147,19 +6148,19 @@ void ftrace_destroy_filter_files(struct ftrace_ops *ops) static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) {

- trace_create_file("available_filter_functions", 0444, + trace_create_file("available_filter_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_avail_fops);

- trace_create_file("enabled_functions", 0444, + trace_create_file("enabled_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_enabled_fops);

ftrace_create_filter_files(&global_ops, d_tracer);

#ifdef CONFIG_FUNCTION_GRAPH_TRACER - trace_create_file("set_graph_function", 0644, d_tracer, + trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer, NULL, &ftrace_graph_fops); - trace_create_file("set_graph_notrace", 0644, d_tracer, + trace_create_file("set_graph_notrace", TRACE_MODE_WRITE, d_tracer, NULL, &ftrace_graph_notrace_fops); #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ @@ -7502,10 +7503,10 @@ static const struct file_operations ftrace_no_pid_fops = {

void ftrace_init_tracefs(struct trace_array *tr, struct dentry *d_tracer) { - trace_create_file("set_ftrace_pid", 0644, d_tracer, + trace_create_file("set_ftrace_pid", TRACE_MODE_WRITE, d_tracer, tr, &ftrace_pid_fops); - trace_create_file("set_ftrace_notrace_pid", 0644, d_tracer, - tr, &ftrace_no_pid_fops); + trace_create_file("set_ftrace_notrace_pid", TRACE_MODE_WRITE, + d_tracer, tr, &ftrace_no_pid_fops); }

void __init ftrace_init_tracefs_toplevel(struct trace_array *tr, diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 3499324efe20..e2df88b691e8 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -1703,7 +1703,8 @@ static void trace_create_maxlat_file(struct trace_array *tr, { INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); - tr->d_max_latency = trace_create_file("tracing_max_latency", 0644, + tr->d_max_latency = trace_create_file("tracing_max_latency", + TRACE_MODE_WRITE, d_tracer, &tr->max_latency, &tracing_max_lat_fops); } @@ -1737,8 +1738,8 @@ void latency_fsnotify(struct trace_array *tr) || defined(CONFIG_OSNOISE_TRACER)

#define trace_create_maxlat_file(tr, d_tracer) \ - trace_create_file("tracing_max_latency", 0644, d_tracer, \ - &tr->max_latency, &tracing_max_lat_fops) + trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \ + d_tracer, &tr->max_latency, &tracing_max_lat_fops)

#else #define trace_create_maxlat_file(tr, d_tracer) do { } while (0) @@ -5701,7 +5702,7 @@ trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,

static void trace_create_eval_file(struct dentry *d_tracer) { - trace_create_file("eval_map", 0444, d_tracer, + trace_create_file("eval_map", TRACE_MODE_READ, d_tracer, NULL, &tracing_eval_map_fops); }

@@ -8211,27 +8212,27 @@ tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) }

/* per cpu trace_pipe */ - trace_create_cpu_file("trace_pipe", 0444, d_cpu, + trace_create_cpu_file("trace_pipe", TRACE_MODE_READ, d_cpu, tr, cpu, &tracing_pipe_fops);

/* per cpu trace */ - trace_create_cpu_file("trace", 0644, d_cpu, + trace_create_cpu_file("trace", TRACE_MODE_WRITE, d_cpu, tr, cpu, &tracing_fops);

- trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu, + trace_create_cpu_file("trace_pipe_raw", TRACE_MODE_READ, d_cpu, tr, cpu, &tracing_buffers_fops);

- trace_create_cpu_file("stats", 0444, d_cpu, + trace_create_cpu_file("stats", TRACE_MODE_READ, d_cpu, tr, cpu, &tracing_stats_fops);

- trace_create_cpu_file("buffer_size_kb", 0444, d_cpu, + trace_create_cpu_file("buffer_size_kb", TRACE_MODE_READ, d_cpu, tr, cpu, &tracing_entries_fops);

#ifdef CONFIG_TRACER_SNAPSHOT - trace_create_cpu_file("snapshot", 0644, d_cpu, + trace_create_cpu_file("snapshot", TRACE_MODE_WRITE, d_cpu, tr, cpu, &snapshot_fops);

- trace_create_cpu_file("snapshot_raw", 0444, d_cpu, + trace_create_cpu_file("snapshot_raw", TRACE_MODE_READ, d_cpu, tr, cpu, &snapshot_raw_fops); #endif } @@ -8437,8 +8438,8 @@ create_trace_option_file(struct trace_array *tr, topt->opt = opt; topt->tr = tr;

- topt->entry = trace_create_file(opt->name, 0644, t_options, topt, - &trace_options_fops); + topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE, + t_options, topt, &trace_options_fops);

}

@@ -8513,7 +8514,7 @@ create_trace_option_core_file(struct trace_array *tr, if (!t_options) return NULL;

- return trace_create_file(option, 0644, t_options, + return trace_create_file(option, TRACE_MODE_WRITE, t_options, (void *)&tr->trace_flags_index[index], &trace_options_core_fops); } @@ -9037,28 +9038,28 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) struct trace_event_file *file; int cpu;

- trace_create_file("available_tracers", 0444, d_tracer, + trace_create_file("available_tracers", TRACE_MODE_READ, d_tracer, tr, &show_traces_fops);

- trace_create_file("current_tracer", 0644, d_tracer, + trace_create_file("current_tracer", TRACE_MODE_WRITE, d_tracer, tr, &set_tracer_fops);

- trace_create_file("tracing_cpumask", 0644, d_tracer, + trace_create_file("tracing_cpumask", TRACE_MODE_WRITE, d_tracer, tr, &tracing_cpumask_fops);

- trace_create_file("trace_options", 0644, d_tracer, + trace_create_file("trace_options", TRACE_MODE_WRITE, d_tracer, tr, &tracing_iter_fops);

- trace_create_file("trace", 0644, d_tracer, + trace_create_file("trace", TRACE_MODE_WRITE, d_tracer, tr, &tracing_fops);

- trace_create_file("trace_pipe", 0444, d_tracer, + trace_create_file("trace_pipe", TRACE_MODE_READ, d_tracer, tr, &tracing_pipe_fops);

- trace_create_file("buffer_size_kb", 0644, d_tracer, + trace_create_file("buffer_size_kb", TRACE_MODE_WRITE, d_tracer, tr, &tracing_entries_fops);

- trace_create_file("buffer_total_size_kb", 0444, d_tracer, + trace_create_file("buffer_total_size_kb", TRACE_MODE_READ, d_tracer, tr, &tracing_total_entries_fops);

trace_create_file("free_buffer", 0200, d_tracer, @@ -9069,25 +9070,25 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)

file = __find_event_file(tr, "ftrace", "print"); if (file && file->dir) - trace_create_file("trigger", 0644, file->dir, file, - &event_trigger_fops); + trace_create_file("trigger", TRACE_MODE_WRITE, file->dir, + file, &event_trigger_fops); tr->trace_marker_file = file;

trace_create_file("trace_marker_raw", 0220, d_tracer, tr, &tracing_mark_raw_fops);

- trace_create_file("trace_clock", 0644, d_tracer, tr, + trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr, &trace_clock_fops);

- trace_create_file("tracing_on", 0644, d_tracer, + trace_create_file("tracing_on", TRACE_MODE_WRITE, d_tracer, tr, &rb_simple_fops);

- trace_create_file("timestamp_mode", 0444, d_tracer, tr, + trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr, &trace_time_stamp_mode_fops);

tr->buffer_percent = 50;

- trace_create_file("buffer_percent", 0444, d_tracer, + trace_create_file("buffer_percent", TRACE_MODE_READ, d_tracer, tr, &buffer_percent_fops);

create_trace_options_dir(tr); @@ -9098,11 +9099,11 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) MEM_FAIL(1, "Could not allocate function filter files");

#ifdef CONFIG_TRACER_SNAPSHOT - trace_create_file("snapshot", 0644, d_tracer, + trace_create_file("snapshot", TRACE_MODE_WRITE, d_tracer, tr, &snapshot_fops); #endif

- trace_create_file("error_log", 0644, d_tracer, + trace_create_file("error_log", TRACE_MODE_WRITE, d_tracer, tr, &tracing_err_log_fops);

for_each_tracing_cpu(cpu) @@ -9265,19 +9266,19 @@ static __init int tracer_init_tracefs(void) init_tracer_tracefs(&global_trace, NULL); ftrace_init_tracefs_toplevel(&global_trace, NULL);

- trace_create_file("tracing_thresh", 0644, NULL, + trace_create_file("tracing_thresh", TRACE_MODE_WRITE, NULL, &global_trace, &tracing_thresh_fops);

- trace_create_file("README", 0444, NULL, + trace_create_file("README", TRACE_MODE_READ, NULL, NULL, &tracing_readme_fops);

- trace_create_file("saved_cmdlines", 0444, NULL, + trace_create_file("saved_cmdlines", TRACE_MODE_READ, NULL, NULL, &tracing_saved_cmdlines_fops);

- trace_create_file("saved_cmdlines_size", 0644, NULL, + trace_create_file("saved_cmdlines_size", TRACE_MODE_WRITE, NULL, NULL, &tracing_saved_cmdlines_size_fops);

- trace_create_file("saved_tgids", 0444, NULL, + trace_create_file("saved_tgids", TRACE_MODE_READ, NULL, NULL, &tracing_saved_tgids_fops);

trace_eval_init(); @@ -9289,7 +9290,7 @@ static __init int tracer_init_tracefs(void) #endif

#ifdef CONFIG_DYNAMIC_FTRACE - trace_create_file("dyn_ftrace_total_info", 0444, NULL, + trace_create_file("dyn_ftrace_total_info", TRACE_MODE_READ, NULL, NULL, &tracing_dyn_info_fops); #endif

diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 46c8a6ac9e3c..aaae8c24c82e 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -26,6 +26,9 @@ #include <asm/syscall.h> /* some archs define it here */ #endif

+#define TRACE_MODE_WRITE 0640 +#define TRACE_MODE_READ 0440 + enum trace_type { __TRACE_FIRST_TYPE = 0,

diff --git a/kernel/trace/trace_dynevent.c b/kernel/trace/trace_dynevent.c index 5fa49cfd2bb6..eb776cfb9a8f 100644 --- a/kernel/trace/trace_dynevent.c +++ b/kernel/trace/trace_dynevent.c @@ -213,7 +213,7 @@ static __init int init_dynamic_event(void) if (ret) return 0;

- entry = tracefs_create_file("dynamic_events", 0644, NULL, + entry = tracefs_create_file("dynamic_events", TRACE_MODE_WRITE, NULL, NULL, &dynamic_events_ops);

/* Event list interface */ diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 546a535f1490..5af763a73948 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -2187,12 +2187,12 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file) }

if (call->class->reg && !(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) - trace_create_file("enable", 0644, file->dir, file, + trace_create_file("enable", TRACE_MODE_WRITE, file->dir, file, &ftrace_enable_fops);

#ifdef CONFIG_PERF_EVENTS if (call->event.type && call->class->reg) - trace_create_file("id", 0444, file->dir, + trace_create_file("id", TRACE_MODE_READ, file->dir, (void *)(long)call->event.type, &ftrace_event_id_fops); #endif @@ -2208,22 +2208,22 @@ event_create_dir(struct dentry *parent, struct trace_event_file *file) * triggers or filters. */ if (!(call->flags & TRACE_EVENT_FL_IGNORE_ENABLE)) { - trace_create_file("filter", 0644, file->dir, file, - &ftrace_event_filter_fops); + trace_create_file("filter", TRACE_MODE_WRITE, file->dir, + file, &ftrace_event_filter_fops);

- trace_create_file("trigger", 0644, file->dir, file, - &event_trigger_fops); + trace_create_file("trigger", TRACE_MODE_WRITE, file->dir, + file, &event_trigger_fops); }

#ifdef CONFIG_HIST_TRIGGERS - trace_create_file("hist", 0444, file->dir, file, + trace_create_file("hist", TRACE_MODE_READ, file->dir, file, &event_hist_fops); #endif #ifdef CONFIG_HIST_TRIGGERS_DEBUG - trace_create_file("hist_debug", 0444, file->dir, file, + trace_create_file("hist_debug", TRACE_MODE_READ, file->dir, file, &event_hist_debug_fops); #endif - trace_create_file("format", 0444, file->dir, call, + trace_create_file("format", TRACE_MODE_READ, file->dir, call, &ftrace_event_format_fops);

#ifdef CONFIG_TRACE_EVENT_INJECT @@ -3212,7 +3212,7 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr) struct dentry *d_events; struct dentry *entry;

- entry = tracefs_create_file("set_event", 0644, parent, + entry = tracefs_create_file("set_event", TRACE_MODE_WRITE, parent, tr, &ftrace_set_event_fops); if (!entry) { pr_warn("Could not create tracefs 'set_event' entry\n"); @@ -3225,7 +3225,7 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr) return -ENOMEM; }

- entry = trace_create_file("enable", 0644, d_events, + entry = trace_create_file("enable", TRACE_MODE_WRITE, d_events, tr, &ftrace_tr_enable_fops); if (!entry) { pr_warn("Could not create tracefs 'enable' entry\n"); @@ -3234,24 +3234,25 @@ create_event_toplevel_files(struct dentry *parent, struct trace_array *tr)

/* There are not as crucial, just warn if they are not created */

- entry = tracefs_create_file("set_event_pid", 0644, parent, + entry = tracefs_create_file("set_event_pid", TRACE_MODE_WRITE, parent, tr, &ftrace_set_event_pid_fops); if (!entry) pr_warn("Could not create tracefs 'set_event_pid' entry\n");

- entry = tracefs_create_file("set_event_notrace_pid", 0644, parent, - tr, &ftrace_set_event_notrace_pid_fops); + entry = tracefs_create_file("set_event_notrace_pid", + TRACE_MODE_WRITE, parent, tr, + &ftrace_set_event_notrace_pid_fops); if (!entry) pr_warn("Could not create tracefs 'set_event_notrace_pid' entry\n");

/* ring buffer internal formats */ - entry = trace_create_file("header_page", 0444, d_events, + entry = trace_create_file("header_page", TRACE_MODE_READ, d_events, ring_buffer_print_page_header, &ftrace_show_header_fops); if (!entry) pr_warn("Could not create tracefs 'header_page' entry\n");

- entry = trace_create_file("header_event", 0444, d_events, + entry = trace_create_file("header_event", TRACE_MODE_READ, d_events, ring_buffer_print_entry_header, &ftrace_show_header_fops); if (!entry) @@ -3468,8 +3469,8 @@ __init int event_trace_init(void) if (!tr) return -ENODEV;

- entry = tracefs_create_file("available_events", 0444, NULL, - tr, &ftrace_avail_fops); + entry = tracefs_create_file("available_events", TRACE_MODE_READ, + NULL, tr, &ftrace_avail_fops); if (!entry) pr_warn("Could not create tracefs 'available_events' entry\n");

diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index 881df991742a..88109dc2dfa5 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -2086,8 +2086,8 @@ static __init int trace_events_synth_init(void) if (err) goto err;

- entry = tracefs_create_file("synthetic_events", 0644, NULL, - NULL, &synth_events_fops); + entry = tracefs_create_file("synthetic_events", TRACE_MODE_WRITE, + NULL, NULL, &synth_events_fops); if (!entry) { err = -ENODEV; goto err; diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index b086ba8bb3d6..677538c3b9ec 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -1340,7 +1340,7 @@ static __init int init_graph_tracefs(void) if (ret) return 0;

- trace_create_file("max_graph_depth", 0644, NULL, + trace_create_file("max_graph_depth", TRACE_MODE_WRITE, NULL, NULL, &graph_depth_fops);

return 0; diff --git a/kernel/trace/trace_hwlat.c b/kernel/trace/trace_hwlat.c index 4c01c5d8b9a7..fd5726c24d7f 100644 --- a/kernel/trace/trace_hwlat.c +++ b/kernel/trace/trace_hwlat.c @@ -544,14 +544,14 @@ static int init_tracefs(void) if (!top_dir) return -ENOMEM;

- hwlat_sample_window = tracefs_create_file("window", 0640, + hwlat_sample_window = tracefs_create_file("window", TRACE_MODE_WRITE, top_dir, &hwlat_data.sample_window, &window_fops); if (!hwlat_sample_window) goto err;

- hwlat_sample_width = tracefs_create_file("width", 0644, + hwlat_sample_width = tracefs_create_file("width", TRACE_MODE_WRITE, top_dir, &hwlat_data.sample_width, &width_fops); diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index 11242ff53663..d2780151f78e 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -1928,16 +1928,16 @@ static __init int init_kprobe_trace(void) if (ret) return 0;

- entry = tracefs_create_file("kprobe_events", 0644, NULL, - NULL, &kprobe_events_ops); + entry = tracefs_create_file("kprobe_events", TRACE_MODE_WRITE, + NULL, NULL, &kprobe_events_ops);

/* Event list interface */ if (!entry) pr_warn("Could not create tracefs 'kprobe_events' entry\n");

/* Profile interface */ - entry = tracefs_create_file("kprobe_profile", 0444, NULL, - NULL, &kprobe_profile_ops); + entry = tracefs_create_file("kprobe_profile", TRACE_MODE_READ, + NULL, NULL, &kprobe_profile_ops);

if (!entry) pr_warn("Could not create tracefs 'kprobe_profile' entry\n"); diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 9980047834b5..3cdbe9ae6fe7 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1943,27 +1943,27 @@ static int init_tracefs(void) if (!top_dir) return 0;

- tmp = tracefs_create_file("period_us", 0640, top_dir, + tmp = tracefs_create_file("period_us", TRACE_MODE_WRITE, top_dir, &osnoise_period, &trace_min_max_fops); if (!tmp) goto err;

- tmp = tracefs_create_file("runtime_us", 0644, top_dir, + tmp = tracefs_create_file("runtime_us", TRACE_MODE_WRITE, top_dir, &osnoise_runtime, &trace_min_max_fops); if (!tmp) goto err;

- tmp = tracefs_create_file("stop_tracing_us", 0640, top_dir, + tmp = tracefs_create_file("stop_tracing_us", TRACE_MODE_WRITE, top_dir, &osnoise_stop_tracing_in, &trace_min_max_fops); if (!tmp) goto err;

- tmp = tracefs_create_file("stop_tracing_total_us", 0640, top_dir, + tmp = tracefs_create_file("stop_tracing_total_us", TRACE_MODE_WRITE, top_dir, &osnoise_stop_tracing_total, &trace_min_max_fops); if (!tmp) goto err;

- tmp = trace_create_file("cpus", 0644, top_dir, NULL, &cpus_fops); + tmp = trace_create_file("cpus", TRACE_MODE_WRITE, top_dir, NULL, &cpus_fops); if (!tmp) goto err; #ifdef CONFIG_TIMERLAT_TRACER diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c index ff32476df072..697aeb3569fd 100644 --- a/kernel/trace/trace_printk.c +++ b/kernel/trace/trace_printk.c @@ -373,7 +373,7 @@ static __init int init_trace_printk_function_export(void) if (ret) return 0;

- trace_create_file("printk_formats", 0444, NULL, + trace_create_file("printk_formats", TRACE_MODE_READ, NULL, NULL, &ftrace_formats_fops);

return 0; diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index c408423e5d65..464fe0a7a032 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c @@ -560,14 +560,14 @@ static __init int stack_trace_init(void) if (ret) return 0;

- trace_create_file("stack_max_size", 0644, NULL, + trace_create_file("stack_max_size", TRACE_MODE_WRITE, NULL, &stack_trace_max_size, &stack_max_size_fops);

- trace_create_file("stack_trace", 0444, NULL, + trace_create_file("stack_trace", TRACE_MODE_READ, NULL, NULL, &stack_trace_fops);

#ifdef CONFIG_DYNAMIC_FTRACE - trace_create_file("stack_trace_filter", 0644, NULL, + trace_create_file("stack_trace_filter", TRACE_MODE_WRITE, NULL, &trace_ops, &stack_trace_filter_fops); #endif

diff --git a/kernel/trace/trace_stat.c b/kernel/trace/trace_stat.c index 8d141c3825a9..bb247beec447 100644 --- a/kernel/trace/trace_stat.c +++ b/kernel/trace/trace_stat.c @@ -297,9 +297,9 @@ static int init_stat_file(struct stat_session *session) if (!stat_dir && (ret = tracing_stat_init())) return ret;

- session->file = tracefs_create_file(session->ts->name, 0644, - stat_dir, - session, &tracing_stat_fops); + session->file = tracefs_create_file(session->ts->name, TRACE_MODE_WRITE, + stat_dir, session, + &tracing_stat_fops); if (!session->file) return -ENOMEM; return 0; diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c index a1bc68de1b29..a7e25b40ed34 100644 --- a/kernel/trace/trace_uprobe.c +++ b/kernel/trace/trace_uprobe.c @@ -1652,10 +1652,10 @@ static __init int init_uprobe_trace(void) if (ret) return 0;

- trace_create_file("uprobe_events", 0644, NULL, + trace_create_file("uprobe_events", TRACE_MODE_WRITE, NULL, NULL, &uprobe_events_ops); /* Profile interface */ - trace_create_file("uprobe_profile", 0444, NULL, + trace_create_file("uprobe_profile", TRACE_MODE_READ, NULL, NULL, &uprobe_profile_ops); return 0; }

From: Daniel Bristot de Oliveira bristot@kernel.org

mainline inclusion from mainline-v5.16-rc1 commit 2fac8d6486d5c34e2ec7028580142b8209da3f92 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

Currently, the user can start only one instance of timerlat/osnoise tracers and the tracers cannot run in parallel.

As starting point to add more flexibility, let's allow the same tracer to run on different trace instances. The workload will start when the first trace_array (instance) is registered and stop when the last instance is unregistered.

So, while this patch allows the same tracer to run in multiple instances (e.g., two instances running osnoise), it still does not allow instances of timerlat and osnoise in parallel (e.g., one timerlat and osnoise). That is because the osnoise: events have different behavior depending on which tracer is enabled (osnoise or timerlat). Enabling the parallel usage of these two tracers is in my TODO list.

Link: https://lkml.kernel.org/r/38c8f14b613492a4f3f938d9d3bf0b063b72f0f0.163570289...

Cc: Ingo Molnar mingo@redhat.com Cc: Tom Zanussi zanussi@kernel.org Cc: Masami Hiramatsu mhiramat@kernel.org Cc: Juri Lelli juri.lelli@redhat.com Cc: Clark Williams williams@redhat.com Cc: John Kacur jkacur@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Thomas Gleixner tglx@linutronix.de Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Daniel Bristot de Oliveira bristot@kernel.org Cc: linux-rt-users@vger.kernel.org Cc: linux-trace-devel@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Daniel Bristot de Oliveira bristot@kernel.org Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 101 +++++++++++++++++++++++++++-------- 1 file changed, 78 insertions(+), 23 deletions(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 4881861a66d7..b949631bc843 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -64,6 +64,24 @@ static bool osnoise_has_registered_instances(void) list); }

+/* + * osnoise_instance_registered - check if a tr is already registered + */ +static int osnoise_instance_registered(struct trace_array *tr) +{ + struct osnoise_instance *inst; + int found = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(inst, &osnoise_instances, list) { + if (inst->tr == tr) + found = 1; + } + rcu_read_unlock(); + + return found; +} + /* * osnoise_register_instance - register a new trace instance * @@ -2079,6 +2097,16 @@ static int osnoise_workload_start(void) { int retval;

+ /* + * Instances need to be registered after calling workload + * start. Hence, if there is already an instance, the + * workload was already registered. Otherwise, this + * code is on the way to register the first instance, + * and the workload will start. + */ + if (osnoise_has_registered_instances()) + return 0; + osn_var_reset_all();

retval = osnoise_hook_events(); @@ -2106,6 +2134,13 @@ static int osnoise_workload_start(void) */ static void osnoise_workload_stop(void) { + /* + * Instances need to be unregistered before calling + * stop. Hence, if there is a registered instance, more + * than one instance is running, and the workload will not + * yet stop. Otherwise, this code is on the way to disable + * the last instance, and the workload can stop. + */ if (osnoise_has_registered_instances()) return;

@@ -2127,7 +2162,11 @@ static void osnoise_tracer_start(struct trace_array *tr) { int retval;

- if (osnoise_has_registered_instances()) + /* + * If the instance is already registered, there is no need to + * register it again. + */ + if (osnoise_instance_registered(tr)) return;

retval = osnoise_workload_start(); @@ -2139,18 +2178,17 @@ static void osnoise_tracer_start(struct trace_array *tr)

static void osnoise_tracer_stop(struct trace_array *tr) { - if (!osnoise_has_registered_instances()) - return; - osnoise_unregister_instance(tr); osnoise_workload_stop(); }

static int osnoise_tracer_init(struct trace_array *tr) { - - /* Only allow one instance to enable this */ - if (osnoise_has_registered_instances()) + /* + * Only allow osnoise tracer if timerlat tracer is not running + * already. + */ + if (timerlat_enabled()) return -EBUSY;

tr->max_latency = 0; @@ -2179,45 +2217,55 @@ static void timerlat_tracer_start(struct trace_array *tr) { int retval;

- if (osnoise_has_registered_instances()) + /* + * If the instance is already registered, there is no need to + * register it again. + */ + if (osnoise_instance_registered(tr)) return;

- osnoise_data.timerlat_tracer = 1; - retval = osnoise_workload_start(); if (retval) - goto out_err; + pr_err(BANNER "Error starting timerlat tracer\n");

osnoise_register_instance(tr);

return; -out_err: - pr_err(BANNER "Error starting timerlat tracer\n"); }

static void timerlat_tracer_stop(struct trace_array *tr) { int cpu;

- if (!osnoise_has_registered_instances()) - return; - - for_each_online_cpu(cpu) - per_cpu(per_cpu_osnoise_var, cpu).sampling = 0; + osnoise_unregister_instance(tr);

- osnoise_tracer_stop(tr); + /* + * Instruct the threads to stop only if this is the last instance. + */ + if (!osnoise_has_registered_instances()) { + for_each_online_cpu(cpu) + per_cpu(per_cpu_osnoise_var, cpu).sampling = 0; + }

- osnoise_data.timerlat_tracer = 0; + osnoise_workload_stop(); }

static int timerlat_tracer_init(struct trace_array *tr) { - /* Only allow one instance to enable this */ - if (osnoise_has_registered_instances()) + /* + * Only allow timerlat tracer if osnoise tracer is not running already. + */ + if (osnoise_has_registered_instances() && !osnoise_data.timerlat_tracer) return -EBUSY;

- tr->max_latency = 0; + /* + * If this is the first instance, set timerlat_tracer to block + * osnoise tracer start. + */ + if (!osnoise_has_registered_instances()) + osnoise_data.timerlat_tracer = 1;

+ tr->max_latency = 0; timerlat_tracer_start(tr);

return 0; @@ -2226,6 +2274,13 @@ static int timerlat_tracer_init(struct trace_array *tr) static void timerlat_tracer_reset(struct trace_array *tr) { timerlat_tracer_stop(tr); + + /* + * If this is the last instance, reset timerlat_tracer allowing + * osnoise to be started. + */ + if (!osnoise_has_registered_instances()) + osnoise_data.timerlat_tracer = 0; }

static struct tracer timerlat_tracer __read_mostly = {

From: Daniel Bristot de Oliveira bristot@kernel.org

mainline inclusion from mainline-v5.16-rc1 commit 01e181c776fddf3a9e7a2ef229cc6e7ddf126fe7 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

Remove CONFIG_PREEMPT_RT from inside functions, avoiding compilation problems in the future.

Link: https://lkml.kernel.org/r/37ee0881b033cdc513efc84ebea26cf77880c8c2.163570289...

Cc: Ingo Molnar mingo@redhat.com Cc: Tom Zanussi zanussi@kernel.org Cc: Masami Hiramatsu mhiramat@kernel.org Cc: Juri Lelli juri.lelli@redhat.com Cc: Clark Williams williams@redhat.com Cc: John Kacur jkacur@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Thomas Gleixner tglx@linutronix.de Cc: Sebastian Andrzej Siewior bigeasy@linutronix.de Cc: Daniel Bristot de Oliveira bristot@kernel.org Cc: linux-rt-users@vger.kernel.org Cc: linux-trace-devel@vger.kernel.org Cc: linux-kernel@vger.kernel.org Suggested-by: Steven Rostedt rostedt@goodmis.org Signed-off-by: Daniel Bristot de Oliveira bristot@kernel.org Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index e77128f86e48..e9074dbcf5fb 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1495,9 +1495,11 @@ static enum hrtimer_restart timerlat_irq(struct hrtimer *timer) * running, the thread needs to receive the softirq delta_start. The * reason being is that the softirq will be the last to be unfolded, * resseting the thread delay to zero. + * + * The PREEMPT_RT is a special case, though. As softirqs run as threads + * on RT, moving the thread is enough. */ -#ifndef CONFIG_PREEMPT_RT - if (osn_var->softirq.delta_start) { + if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) { copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->softirq.delta_start);

@@ -1507,13 +1509,6 @@ static enum hrtimer_restart timerlat_irq(struct hrtimer *timer) copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->irq.delta_start); } -#else /* CONFIG_PREEMPT_RT */ - /* - * The sofirqs run as threads on RT, so there is not need - * to keep track of it. - */ - copy_int_safe_time(osn_var, &osn_var->thread.delta_start, &osn_var->irq.delta_start); -#endif /* CONFIG_PREEMPT_RT */

/* * Compute the current time with the expected time.

From: "Steven Rostedt (VMware)" rostedt@goodmis.org

mainline inclusion from mainline-v5.14-rc1 commit b96285e10aad234acfa0628f7e8336990f778c03 category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

ftracetest triggered:

INFO: rcu_tasks detected stalls on tasks: 00000000b92b832d: .. nvcsw: 1/1 holdout: 1 idle_cpu: -1/7 task:osnoise/7 state:R running task stack: 0 pid: 2133 ppid: 2 flags:0x00004000 Call Trace: ? asm_sysvec_apic_timer_interrupt+0x12/0x20 ? asm_sysvec_apic_timer_interrupt+0x12/0x20 ? trace_hardirqs_on+0x2b/0xe0 ? asm_sysvec_apic_timer_interrupt+0x12/0x20 ? trace_clock_local+0xc/0x20 ? osnoise_main+0x10e/0x450 ? trace_softirq_entry_callback+0x50/0x50 ? kthread+0x153/0x170 ? __kthread_bind_mask+0x60/0x60 ? ret_from_fork+0x22/0x30

While running osnoise tracer with other tracers that rely on synchronize_rcu_tasks(), where that just hung.

The reason is that osnoise_main() never schedules out if the interval is less than 1, and this will cause synchronize_rcu_tasks() to never return.

Link: https://lkml.kernel.org/r/20210628114953.6dc06a91@oasis.local.home

Fixes: bce29ac9ce0bb ("trace: Add osnoise tracer") Acked-by: Paul E. McKenney paulmck@kernel.org Reviewed-by: Daniel Bristot de Oliveira bristot@redhat.com Signed-off-by: Steven Rostedt (VMware) rostedt@goodmis.org Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index a32951f892cc..545c3d6489bd 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1441,8 +1441,11 @@ static int osnoise_main(void *data) * differently from hwlat_detector, the osnoise tracer can run * without a pause because preemption is on. */ - if (interval < 1) + if (interval < 1) { + /* Let synchronize_rcu_tasks() make progress */ + cond_resched_tasks_rcu_qs(); continue; + }

if (msleep_interruptible(interval)) break;

From: Wang ShaoBo bobo.shaobowang@huawei.com

hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4G64B CVE: NA

-------------------------------------------------

This patch fixes following compile warning:

---------------------- [WARNING] checkbuild_arm64 ----------------------

build warning: arm64, allmodconfig

kernel/trace/trace_osnoise.c: In function ‘start_kthread’: kernel/trace/trace_osnoise.c:1674:8: warning: ‘main’ is usually a function [-Wmain] void *main = osnoise_main; ^~~~ ---------------------- [WARNING] checkbuild_x86_64 ----------------------

build warning: x86_64, allmodconfig

kernel/trace/trace_osnoise.c: In function ‘start_kthread’: kernel/trace/trace_osnoise.c:1674:8: warning: ‘main’ is usually a function [-Wmain] void *main = osnoise_main; ^~~~

Signed-off-by: Wang ShaoBo bobo.shaobowang@huawei.com Acked-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/trace/trace_osnoise.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 545c3d6489bd..cfb80feb291e 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -1671,17 +1671,17 @@ static void stop_per_cpu_kthreads(void) static int start_kthread(unsigned int cpu) { struct task_struct *kthread; - void *main = osnoise_main; + void *func_main = osnoise_main; char comm[24];

if (timerlat_enabled()) { snprintf(comm, 24, "timerlat/%d", cpu); - main = timerlat_main; + func_main = timerlat_main; } else { snprintf(comm, 24, "osnoise/%d", cpu); }

- kthread = kthread_create_on_cpu(main, NULL, cpu, comm); + kthread = kthread_create_on_cpu(func_main, NULL, cpu, comm);

if (IS_ERR(kthread)) { pr_err(BANNER "could not start sampling thread\n");

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

Introduce function hugetlb_alloc_hugepage that alloc hugepages from static hugepages first. When the static hugepage is used up, it attempts to apply for hugepages from buddy system. Two additional modes are supported: static hugepages only and buddy hugepages only.

When the driver gets huge pages by alloc_huge_page_node, it attempts to apply for migrate hugepages after the reserved memory hugepages are used up. We expect that the migrated hugepages that are applied for can be charged in memcg to limit the memory usage. So we enable charge migrate hugepages, and default enable it.

Add hugetlb_insert_hugepage_pte[_by_pa] to insert hugepages into page table. The by_pa version performs like remap_pfn_range() that make the pte special and can be used for reserved physical memory.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Signed-off-by: Zhang Jian zhangjian210@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/Kconfig | 12 ++++ include/linux/hugetlb.h | 62 ++++++++++++++++++ mm/hugetlb.c | 139 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 212 insertions(+), 1 deletion(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 3afabc81551c..85610dd3ac0c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2022,6 +2022,18 @@ config ASCEND_OOM 0: disable oom killer 1: enable oom killer (default,compatible with mainline)

+config ASCEND_CHARGE_MIGRATE_HUGEPAGES + bool "Enable support for migrate hugepages" + depends on HUGETLBFS + default y + help + When reserved hugepages are used up, we attempts to apply for migrate + hugepages. We expect that the migrated hugepages that are applied for + can be charged in memcg to limit the memory usage. + + This option enable the feature to charge migrate hugepages to memory + cgroup. + endif

endmenu diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index fd9635a6a92f..397e6bfa8268 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -607,6 +607,45 @@ struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma, int huge_add_to_page_cache(struct page *page, struct address_space *mapping, pgoff_t idx);

+#ifdef CONFIG_ASCEND_FEATURES +#define HUGETLB_ALLOC_NONE 0x00 +#define HUGETLB_ALLOC_NORMAL 0x01 /* normal hugepage */ +#define HUGETLB_ALLOC_BUDDY 0x02 /* buddy hugepage */ +#define HUGETLB_ALLOC_MASK (HUGETLB_ALLOC_NONE | \ + HUGETLB_ALLOC_NORMAL | \ + HUGETLB_ALLOC_BUDDY) + +const struct hstate *hugetlb_get_hstate(void); +struct page *hugetlb_alloc_hugepage(int nid, int flag); +int hugetlb_insert_hugepage_pte(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, struct page *hpage); +int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr); +#else +static inline const struct hstate *hugetlb_get_hstate(void) +{ + return NULL; +} + +static inline struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + return NULL; +} + +static inline int hugetlb_insert_hugepage_pte(struct mm_struct *mm, + unsigned long addr, pgprot_t prot, struct page *hpage) +{ + return -EPERM; +} +static inline int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr) +{ + return -EPERM; +} +#endif + /* arch callback */ int __init __alloc_bootmem_huge_page(struct hstate *h); int __init alloc_bootmem_huge_page(struct hstate *h); @@ -1028,6 +1067,29 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr pte_t *ptep, pte_t pte, unsigned long sz) { } + +static inline const struct hstate *hugetlb_get_hstate(void) +{ + return NULL; +} + +static inline struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + return NULL; +} + +static inline int hugetlb_insert_hugepage_pte(struct mm_struct *mm, + unsigned long addr, pgprot_t prot, struct page *hpage) +{ + return -EPERM; +} + +static inline int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, + unsigned long vir_addr, + pgprot_t prot, unsigned long phy_addr) +{ + return -EPERM; +} #endif /* CONFIG_HUGETLB_PAGE */

#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP diff --git a/mm/hugetlb.c b/mm/hugetlb.c index d0672e482879..eaddb18d58e1 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -67,7 +67,6 @@ static struct hstate * __initdata parsed_hstate; static unsigned long __initdata default_hstate_max_huge_pages; static bool __initdata parsed_valid_hugepagesz = true; static bool __initdata parsed_default_hugepagesz; - /* * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages, * free_huge_pages, and surplus_huge_pages. @@ -5907,3 +5906,141 @@ void __init hugetlb_cma_check(void) }

#endif /* CONFIG_CMA */ + +#ifdef CONFIG_ASCEND_FEATURES +static int enable_charge_mighp __read_mostly; + +const struct hstate *hugetlb_get_hstate(void) +{ + return &default_hstate; +} +EXPORT_SYMBOL_GPL(hugetlb_get_hstate); + +static struct page *hugetlb_alloc_hugepage_normal(struct hstate *h, + gfp_t gfp_mask, int nid) +{ + struct page *page = NULL; + + spin_lock(&hugetlb_lock); + if (h->free_huge_pages - h->resv_huge_pages > 0) + page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL, NULL); + spin_unlock(&hugetlb_lock); + + return page; +} + +/* + * Allocate hugepage without reserve + */ +struct page *hugetlb_alloc_hugepage(int nid, int flag) +{ + struct hstate *h = &default_hstate; + gfp_t gfp_mask = htlb_alloc_mask(h); + struct page *page = NULL; + + if (nid == NUMA_NO_NODE) + nid = numa_mem_id(); + + if (nid < 0 || nid >= MAX_NUMNODES) + return NULL; + + if (flag & ~HUGETLB_ALLOC_MASK) + return NULL; + + gfp_mask |= __GFP_THISNODE; + if (enable_charge_mighp) + gfp_mask |= __GFP_ACCOUNT; + + if (flag & HUGETLB_ALLOC_NORMAL) + page = hugetlb_alloc_hugepage_normal(h, gfp_mask, nid); + else if (flag & HUGETLB_ALLOC_BUDDY) + page = alloc_migrate_huge_page(h, gfp_mask, nid, NULL); + else + page = alloc_huge_page_nodemask(h, nid, NULL, gfp_mask); + + return page; +} +EXPORT_SYMBOL_GPL(hugetlb_alloc_hugepage); + +static int __hugetlb_insert_hugepage(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, unsigned long pfn, bool special) +{ + int ret = 0; + pte_t *ptep, entry; + struct hstate *h; + struct vm_area_struct *vma; + struct address_space *mapping; + spinlock_t *ptl; + + h = size_to_hstate(PMD_SIZE); + if (!h) + return -EINVAL; + + if (!IS_ALIGNED(addr, PMD_SIZE)) + return -EINVAL; + + vma = find_vma(mm, addr); + if (!vma || !range_in_vma(vma, addr, addr + PMD_SIZE)) + return -EINVAL; + + mapping = vma->vm_file->f_mapping; + i_mmap_lock_read(mapping); + ptep = huge_pte_alloc(mm, addr, huge_page_size(h)); + if (!ptep) { + ret = -ENXIO; + goto out_unlock; + } + + if (WARN_ON(ptep && !pte_none(*ptep) && !pmd_huge(*(pmd_t *)ptep))) { + ret = -ENXIO; + goto out_unlock; + } + + entry = pfn_pte(pfn, prot); + entry = huge_pte_mkdirty(entry); + if (!(pgprot_val(prot) & PTE_RDONLY)) + entry = huge_pte_mkwrite(entry); + entry = pte_mkyoung(entry); + entry = pte_mkhuge(entry); + if (special) + entry = pte_mkspecial(entry); + + ptl = huge_pte_lockptr(h, mm, ptep); + spin_lock(ptl); + set_huge_pte_at(mm, addr, ptep, entry); + spin_unlock(ptl); + +out_unlock: + i_mmap_unlock_read(mapping); + + return ret; +} + +int hugetlb_insert_hugepage_pte(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, struct page *hpage) +{ + return __hugetlb_insert_hugepage(mm, addr, prot, page_to_pfn(hpage), false); +} +EXPORT_SYMBOL_GPL(hugetlb_insert_hugepage_pte); + +int hugetlb_insert_hugepage_pte_by_pa(struct mm_struct *mm, unsigned long addr, + pgprot_t prot, unsigned long phy_addr) +{ + return __hugetlb_insert_hugepage(mm, addr, prot, phy_addr >> PAGE_SHIFT, true); +} +EXPORT_SYMBOL_GPL(hugetlb_insert_hugepage_pte_by_pa); + +#ifdef CONFIG_ASCEND_CHARGE_MIGRATE_HUGEPAGES + +static int __init ascend_enable_charge_migrate_hugepages(char *s) +{ + enable_charge_mighp = 1; + + pr_info("Ascend enable charge migrate hugepage\n"); + + return 1; +} +__setup("enable_charge_mighp", ascend_enable_charge_migrate_hugepages); + +#endif +#endif

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

The do_mmap/mmap_region/__mm_populate could only be used to handle the current process, now the share pool need to handle the other process and create memory mmaping, so need to export new function to distinguish different process and handle it, it would not break the current logic and only valid for share pool.

Signed-off-by: Tang Yizhou tangyizhou@huawei.com Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/mm.h | 11 +++++++++++ mm/gup.c | 24 +++++++++++++++++------- mm/mmap.c | 41 ++++++++++++++++++++++++++++++----------- 3 files changed, 58 insertions(+), 18 deletions(-)

diff --git a/include/linux/mm.h b/include/linux/mm.h index f8ae3e41d5db..a4996e11cda6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2611,6 +2611,10 @@ extern int do_munmap(struct mm_struct *, unsigned long, size_t, struct list_head *uf); extern int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int behavior);

+extern unsigned long __do_mmap_mm(struct mm_struct *mm, struct file *file, + unsigned long addr, unsigned long len, unsigned long prot, + unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff, + unsigned long *populate, struct list_head *uf); #ifdef CONFIG_MMU extern int __mm_populate(unsigned long addr, unsigned long len, int ignore_errors); @@ -2619,8 +2623,15 @@ static inline void mm_populate(unsigned long addr, unsigned long len) /* Ignore errors */ (void) __mm_populate(addr, len, 1); } +extern int do_mm_populate(struct mm_struct *mm, unsigned long addr, unsigned long len, + int ignore_errors); #else static inline void mm_populate(unsigned long addr, unsigned long len) {} +static inline int do_mm_populate(struct mm_struct *mm, unsigned long addr, unsigned long len, + int ignore_errors) +{ + return -EPERM; +} #endif

/* These take the mm semaphore themselves */ diff --git a/mm/gup.c b/mm/gup.c index ab544708191e..ee9c2c39c299 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1507,15 +1507,12 @@ long populate_vma_page_range(struct vm_area_struct *vma, }

/* - * __mm_populate - populate and/or mlock pages within a range of address space. - * - * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap - * flags. VMAs must be already marked with the desired vm_flags, and - * mmap_lock must not be held. + * do_mm_populate - populate and/or mlock pages within a range of + * address space for the specified mm_struct. */ -int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) +int do_mm_populate(struct mm_struct *mm, unsigned long start, unsigned long len, + int ignore_errors) { - struct mm_struct *mm = current->mm; unsigned long end, nstart, nend; struct vm_area_struct *vma = NULL; int locked = 0; @@ -1565,6 +1562,19 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) mmap_read_unlock(mm); return ret; /* 0 or negative error code */ } + +/* + * __mm_populate - populate and/or mlock pages within a range of address space. + * + * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap + * flags. VMAs must be already marked with the desired vm_flags, and + * mmap_lock must not be held. + */ +int __mm_populate(unsigned long start, unsigned long len, int ignore_errors) +{ + return do_mm_populate(current->mm, start, len, ignore_errors); +} + #else /* CONFIG_MMU */ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, struct page **pages, diff --git a/mm/mmap.c b/mm/mmap.c index 3991634121d7..0af1300734a2 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1399,12 +1399,17 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, return true; }

-static inline unsigned long -__do_mmap(struct file *file, unsigned long addr, unsigned long len, - unsigned long prot, unsigned long flags, vm_flags_t vm_flags, - unsigned long pgoff, unsigned long *populate, struct list_head *uf) +static unsigned long __mmap_region(struct mm_struct *mm, + struct file *file, unsigned long addr, + unsigned long len, vm_flags_t vm_flags, + unsigned long pgoff, struct list_head *uf); + +unsigned long __do_mmap_mm(struct mm_struct *mm, struct file *file, + unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + vm_flags_t vm_flags, unsigned long pgoff, + unsigned long *populate, struct list_head *uf) { - struct mm_struct *mm = current->mm; int pkey = 0;

*populate = 0; @@ -1587,14 +1592,22 @@ __do_mmap(struct file *file, unsigned long addr, unsigned long len, if (flags & MAP_CHECKNODE) set_vm_checknode(&vm_flags, flags);

- addr = mmap_region(file, addr, len, vm_flags, pgoff, uf); + addr = __mmap_region(mm, file, addr, len, vm_flags, pgoff, uf); if (!IS_ERR_VALUE(addr) && ((vm_flags & VM_LOCKED) || (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE)) *populate = len; return addr; } +EXPORT_SYMBOL(__do_mmap_mm);

+static inline unsigned long +__do_mmap(struct file *file, unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, vm_flags_t vm_flags, + unsigned long pgoff, unsigned long *populate, struct list_head *uf) +{ + return __do_mmap_mm(current->mm, file, addr, len, prot, flags, vm_flags, pgoff, populate, uf); +} #ifdef CONFIG_USERSWAP /* * Check if pages between 'addr ~ addr+len' can be user swapped. If so, get @@ -1955,11 +1968,11 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags) return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE; }

-unsigned long mmap_region(struct file *file, unsigned long addr, - unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, - struct list_head *uf) +static unsigned long __mmap_region(struct mm_struct *mm, struct file *file, + unsigned long addr, unsigned long len, + vm_flags_t vm_flags, unsigned long pgoff, + struct list_head *uf) { - struct mm_struct *mm = current->mm; struct vm_area_struct *vma, *prev, *merge; int error; struct rb_node **rb_link, *rb_parent; @@ -2148,6 +2161,13 @@ unsigned long mmap_region(struct file *file, unsigned long addr, return error; }

+unsigned long mmap_region(struct file *file, unsigned long addr, + unsigned long len, vm_flags_t vm_flags, unsigned long pgoff, + struct list_head *uf) +{ + return __mmap_region(current->mm, file, addr, len, vm_flags, pgoff, uf); +} + static unsigned long unmapped_area(struct vm_unmapped_area_info *info) { /* @@ -3209,7 +3229,6 @@ SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) return __vm_munmap(addr, len, true); }

- /* * Emulation of deprecated remap_file_pages() syscall. */

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

The share pool features is a big feature, it is mainly used to share user virtual memory for different processes in the same group. It could be used by this steps: 1. Process A create a new group which is owned by process A. 2. Process A add process B to the group. 3. Process A add process C to the same group. 4. Process B alloc a new memory VA, and write something in it. 5. The VA was send to the process C by IPC, then process C got it. 6. The process C access the VA and got the data directly. 7. The process A could add more processes in the group to share the memory. 8. Fix the memory by use the free function or exit the group.

The new features is enabled both by CONFIG_ASCEND_SHARE_POOL and the enable_ascend_share_pool bootarg, it would not affect anything if disabled.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Tang Yizhou tangyizhou@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- arch/arm64/Kconfig | 9 ++ include/linux/mm_types.h | 4 + include/linux/share_pool.h | 226 +++++++++++++++++++++++++++++++++++++ kernel/fork.c | 4 + mm/Makefile | 1 + mm/share_pool.c | 45 ++++++++ 6 files changed, 289 insertions(+) create mode 100644 include/linux/share_pool.h create mode 100644 mm/share_pool.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 85610dd3ac0c..405e5ce460ce 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2034,6 +2034,15 @@ config ASCEND_CHARGE_MIGRATE_HUGEPAGES This option enable the feature to charge migrate hugepages to memory cgroup.

+config ASCEND_SHARE_POOL + bool "Enable support for the Share Pool Memory" + default n + depends on HAVE_ARCH_HUGE_VMALLOC + select ARCH_USES_HIGH_VMA_FLAGS + help + This feature allows multiple processes to share virtual memory both + in kernel and user level, which is only enabled for ascend platform. + endif

endmenu diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 2729eb58aca4..30b36a3adb87 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -585,6 +585,10 @@ struct mm_struct { struct kvm *kvm; #endif

+#if IS_ENABLED(CONFIG_ASCEND_SHARE_POOL) + struct sp_group_master *sp_group_master; +#endif + /* * The mm_cpumask needs to be at the end of mm_struct, because it * is dynamically sized based on nr_cpu_ids. diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h new file mode 100644 index 000000000000..84d9c539f12b --- /dev/null +++ b/include/linux/share_pool.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_SHARE_POOL_H +#define LINUX_SHARE_POOL_H + +#include <linux/mman.h> +#include <linux/mm_types.h> +#include <linux/notifier.h> +#include <linux/vmalloc.h> +#include <linux/printk.h> +#include <linux/hashtable.h> +#include <linux/numa.h> +#include <linux/jump_label.h> + +#define SP_HUGEPAGE (1 << 0) +#define SP_HUGEPAGE_ONLY (1 << 1) +#define SP_DVPP (1 << 2) +#define SP_SPEC_NODE_ID (1 << 3) + +#define DEVICE_ID_BITS 4UL +#define DEVICE_ID_MASK ((1UL << DEVICE_ID_BITS) - 1UL) +#define DEVICE_ID_SHIFT 32UL +#define NODE_ID_BITS NODES_SHIFT +#define NODE_ID_MASK ((1UL << NODE_ID_BITS) - 1UL) +#define NODE_ID_SHIFT (DEVICE_ID_SHIFT + DEVICE_ID_BITS) + +#define SP_FLAG_MASK (SP_HUGEPAGE | SP_HUGEPAGE_ONLY | SP_DVPP | \ + SP_SPEC_NODE_ID | \ + (DEVICE_ID_MASK << DEVICE_ID_SHIFT) | \ + (NODE_ID_MASK << NODE_ID_SHIFT)) + +#define sp_flags_device_id(flags) (((flags) >> DEVICE_ID_SHIFT) & DEVICE_ID_MASK) +#define sp_flags_node_id(flags) (((flags) >> NODE_ID_SHIFT) & NODE_ID_MASK) + +#define SPG_ID_NONE (-1) /* not associated with sp_group, only for specified thread */ +#define SPG_ID_DEFAULT 0 /* use the spg id of current thread */ +#define SPG_ID_MIN 1 /* valid id should be >= 1 */ +#define SPG_ID_MAX 99999 +#define SPG_ID_AUTO_MIN 100000 +#define SPG_ID_AUTO_MAX 199999 +#define SPG_ID_AUTO 200000 /* generate group id automatically */ + +#define MAX_DEVID 8 /* the max num of Da-vinci devices */ + +extern int sysctl_share_pool_hugepage_enable; + +extern int sysctl_ac_mode; + +extern int sysctl_sp_debug_mode; + +extern struct static_key_false share_pool_enabled_key; + +extern int sysctl_share_pool_map_lock_enable; + +extern int sysctl_sp_compact_enable; +extern unsigned long sysctl_sp_compact_interval; +extern unsigned long sysctl_sp_compact_interval_max; +extern int sysctl_sp_perf_alloc; + +extern int sysctl_sp_perf_k2u; + +/* we estimate an sp-group ususally contains at most 64 sp-group */ +#define SP_SPG_HASH_BITS 6 + +struct sp_spg_stat { + int spg_id; + /* record the number of hugepage allocation failures */ + atomic_t hugepage_failures; + /* number of sp_area */ + atomic_t spa_num; + /* total size of all sp_area from sp_alloc and k2u */ + atomic64_t size; + /* total size of all sp_area from sp_alloc 0-order page */ + atomic64_t alloc_nsize; + /* total size of all sp_area from sp_alloc hugepage */ + atomic64_t alloc_hsize; + /* total size of all sp_area from ap_alloc */ + atomic64_t alloc_size; + /* total size of all sp_area from sp_k2u */ + atomic64_t k2u_size; + struct mutex lock; /* protect hashtable */ + DECLARE_HASHTABLE(hash, SP_SPG_HASH_BITS); +}; + +/* we estimate a process ususally belongs to at most 16 sp-group */ +#define SP_PROC_HASH_BITS 4 + +/* per process memory usage statistics indexed by tgid */ +struct sp_proc_stat { + atomic_t use_count; + int tgid; + struct mm_struct *mm; + struct mutex lock; /* protect hashtable */ + DECLARE_HASHTABLE(hash, SP_PROC_HASH_BITS); + char comm[TASK_COMM_LEN]; + /* + * alloc amount minus free amount, may be negative when freed by + * another task in the same sp group. + */ + atomic64_t alloc_size; + atomic64_t k2u_size; +}; + +/* Processes in the same sp_group can share memory. + * Memory layout for share pool: + * + * |-------------------- 8T -------------------|---|------ 8T ------------| + * | Device 0 | Device 1 |...| | + * |----------------------------------------------------------------------| + * |------------- 16G -------------| 16G | | | + * | DVPP GROUP0 | DVPP GROUP1 | ... | ... |...| sp normal memory | + * | sp | sp | | | | | + * |----------------------------------------------------------------------| + * + * The host SVM feature reserves 8T virtual memory by mmap, and due to the + * restriction of DVPP, while SVM and share pool will both allocate memory + * for DVPP, the memory have to be in the same 32G range. + * + * Share pool reserves 16T memory, with 8T for normal uses and 8T for DVPP. + * Within this 8T DVPP memory, SVM will call sp_config_dvpp_range() to + * tell us which 16G memory range is reserved for share pool . + * + * In some scenarios where there is no host SVM feature, share pool uses + * the default 8G memory setting for DVPP. + */ +struct sp_group { + int id; + struct file *file; + struct file *file_hugetlb; + /* number of process in this group */ + int proc_num; + /* list head of processes (sp_group_node, each represents a process) */ + struct list_head procs; + /* list head of sp_area. it is protected by spin_lock sp_area_lock */ + struct list_head spa_list; + /* group statistics */ + struct sp_spg_stat *stat; + /* we define the creator process of a sp_group as owner */ + struct task_struct *owner; + /* is_alive == false means it's being destroyed */ + bool is_alive; + atomic_t use_count; + /* protect the group internal elements, except spa_list */ + struct rw_semaphore rw_lock; +}; + +/* a per-process(per mm) struct which manages a sp_group_node list */ +struct sp_group_master { + /* + * number of sp groups the process belongs to, + * a.k.a the number of sp_node in node_list + */ + unsigned int count; + /* list head of sp_node */ + struct list_head node_list; + struct mm_struct *mm; + struct sp_proc_stat *stat; +}; + +/* + * each instance represents an sp group the process belongs to + * sp_group_master : sp_group_node = 1 : N + * sp_group_node->spg : sp_group = 1 : 1 + * sp_group_node : sp_group->procs = N : 1 + */ +struct sp_group_node { + /* list node in sp_group->procs */ + struct list_head proc_node; + /* list node in sp_group_maseter->node_list */ + struct list_head group_node; + struct sp_group_master *master; + struct sp_group *spg; + unsigned long prot; +}; + +struct sp_walk_data { + struct page **pages; + unsigned int page_count; + unsigned long uva_aligned; + unsigned long page_size; + bool is_hugepage; + pmd_t *pmd; +}; + +#define MAP_SHARE_POOL 0x200000 + +#define MMAP_TOP_4G_SIZE 0x100000000UL + +/* 8T size */ +#define MMAP_SHARE_POOL_NORMAL_SIZE 0x80000000000UL +/* 8T size*/ +#define MMAP_SHARE_POOL_DVPP_SIZE 0x80000000000UL +/* 16G size */ +#define MMAP_SHARE_POOL_16G_SIZE 0x400000000UL +#define MMAP_SHARE_POOL_SIZE (MMAP_SHARE_POOL_NORMAL_SIZE + MMAP_SHARE_POOL_DVPP_SIZE) +/* align to 2M hugepage size, and MMAP_SHARE_POOL_TOP_16G_START should be align to 16G */ +#define MMAP_SHARE_POOL_END ((TASK_SIZE - MMAP_SHARE_POOL_DVPP_SIZE) & ~((1 << 21) - 1)) +#define MMAP_SHARE_POOL_START (MMAP_SHARE_POOL_END - MMAP_SHARE_POOL_SIZE) +#define MMAP_SHARE_POOL_16G_START (MMAP_SHARE_POOL_END - MMAP_SHARE_POOL_DVPP_SIZE) + +#ifdef CONFIG_ASCEND_SHARE_POOL + +static inline void sp_init_mm(struct mm_struct *mm) +{ + mm->sp_group_master = NULL; +} + +extern int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id); +extern int sp_group_add_task(int pid, int spg_id); + +static inline bool sp_is_enabled(void) +{ + return static_branch_likely(&share_pool_enabled_key); +} + +#else /* CONFIG_ASCEND_SHARE_POOL */ + +static inline void sp_init_mm(struct mm_struct *mm) { } + +static inline bool sp_is_enabled(void) +{ + return false; +} + +#endif /* !CONFIG_ASCEND_SHARE_POOL */ + +#endif /* LINUX_SHARE_POOL_H */ diff --git a/kernel/fork.c b/kernel/fork.c index d730b57c9c22..454b42af1de8 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -96,6 +96,7 @@ #include <linux/kasan.h> #include <linux/scs.h> #include <linux/io_uring.h> +#include <linux/share_pool.h>

#include <asm/pgalloc.h> #include <linux/uaccess.h> @@ -1055,6 +1056,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, goto fail_nocontext;

mm->user_ns = get_user_ns(user_ns); + + sp_init_mm(mm); + return mm;

fail_nocontext: diff --git a/mm/Makefile b/mm/Makefile index c14522bd17ed..ec3d0ab14a6a 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -126,3 +126,4 @@ obj-$(CONFIG_PAGE_REPORTING) += page_reporting.o obj-$(CONFIG_HAVE_BOOTMEM_INFO_NODE) += bootmem_info.o obj-$(CONFIG_PIN_MEMORY) += pin_mem.o obj-$(CONFIG_SHRINK_PAGECACHE) += page_cache_limit.o +obj-$(CONFIG_ASCEND_SHARE_POOL) += share_pool.o diff --git a/mm/share_pool.c b/mm/share_pool.c new file mode 100644 index 000000000000..32d473bada3a --- /dev/null +++ b/mm/share_pool.c @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Huawei Ascend Share Pool Memory + * + * Copyright (C) 2020 Huawei Limited + * Author: Tang Yizhou tangyizhou@huawei.com + * Zefan Li lizefan@huawei.com + * Wu Peng wupeng58@huawei.com + * Ding Tianhong dingtgianhong@huawei.com + * Zhou Guanghui zhouguanghui1@huawei.com + * Li Ming limingming.li@huawei.com + * + * This code is based on the hisilicon ascend platform. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#define pr_fmt(fmt) "share pool: " fmt + +#include <linux/share_pool.h> + +int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id) +{ + return 0; +} +EXPORT_SYMBOL_GPL(mg_sp_group_add_task); + +int sp_group_add_task(int pid, int spg_id) +{ + return 0; +} +EXPORT_SYMBOL_GPL(sp_group_add_task); + +DEFINE_STATIC_KEY_FALSE(share_pool_enabled_key); + +static int __init enable_share_pool(char *s) +{ + static_branch_enable(&share_pool_enabled_key); + pr_info("Ascend enable share pool features via bootargs\n"); + + return 1; +} +__setup("enable_ascend_share_pool", enable_share_pool);

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

Change the mmap_base in mm_struct and check the limit in get_unmapped_area.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Peng Wu wupeng58@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/share_pool.h | 12 +++++++++++- mm/mmap.c | 7 +++++++ mm/util.c | 4 ++++ 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 37a26487a7d8..3a56238c8a4d 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -262,6 +262,16 @@ static inline bool sp_is_enabled(void) return static_branch_likely(&share_pool_enabled_key); }

+static inline void sp_area_work_around(struct vm_unmapped_area_info *info, + unsigned long flags) +{ + /* the MAP_DVPP couldn't work with MAP_SHARE_POOL. In addition, the + * address ranges corresponding to the two flags must not overlap. + */ + if (sp_is_enabled() && !(flags & MAP_DVPP)) + info->high_limit = min(info->high_limit, MMAP_SHARE_POOL_START); +} + #else /* CONFIG_ASCEND_SHARE_POOL */

static inline int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id) @@ -442,7 +452,7 @@ static inline bool sp_is_enabled(void) return false; }

-static inline void sp_area_work_around(struct vm_unmapped_area_info *info) +static inline void sp_area_work_around(struct vm_unmapped_area_info *info, unsigned long flags) { }

diff --git a/mm/mmap.c b/mm/mmap.c index 0af1300734a2..d5a97a56dca7 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -48,6 +48,7 @@ #include <linux/oom.h> #include <linux/sched/mm.h> #include <linux/swapops.h> +#include <linux/share_pool.h>

#include <linux/uaccess.h> #include <asm/cacheflush.h> @@ -2450,6 +2451,8 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, if (enable_mmap_dvpp) dvpp_mmap_get_area(&info, flags);

+ sp_area_work_around(&info, flags); + return vm_unmapped_area(&info); } #endif @@ -2500,6 +2503,8 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, if (enable_mmap_dvpp) dvpp_mmap_get_area(&info, flags);

+ sp_area_work_around(&info, flags); + addr = vm_unmapped_area(&info);

/* @@ -2517,6 +2522,8 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, if (enable_mmap_dvpp) dvpp_mmap_get_area(&info, flags);

+ sp_area_work_around(&info, flags); + addr = vm_unmapped_area(&info); }

diff --git a/mm/util.c b/mm/util.c index 2350c064abc6..d31820abadb4 100644 --- a/mm/util.c +++ b/mm/util.c @@ -23,6 +23,7 @@ #include <linux/processor.h> #include <linux/sizes.h> #include <linux/compat.h> +#include <linux/share_pool.h>

#include <linux/uaccess.h> #include <linux/oom.h> @@ -392,6 +393,9 @@ static unsigned long mmap_base(unsigned long rnd, struct rlimit *rlim_stack) else if (gap > MAX_GAP) gap = MAX_GAP;

+ if (sp_is_enabled()) + return ALIGN_DOWN(MMAP_SHARE_POOL_START - rnd, PAGE_SIZE); + return PAGE_ALIGN(STACK_TOP - gap - rnd); }

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

This flag specify that the vm_area_struct is related to share_pool (in detail, has a spa corresponding to it).

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Tang Yizhou tangyizhou@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/mm.h | 6 ++++++ include/linux/share_pool.h | 8 ++++++++ 2 files changed, 14 insertions(+)

diff --git a/include/linux/mm.h b/include/linux/mm.h index a4996e11cda6..e2eb3ea63d6a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -378,6 +378,12 @@ extern unsigned int kobjsize(const void *objp); # define VM_MTE_ALLOWED VM_NONE #endif

+#if defined(CONFIG_ASCEND_SHARE_POOL) +# define VM_SHARE_POOL VM_HIGH_ARCH_4 +#else +# define VM_SHARE_POOL VM_NONE +#endif + #ifndef VM_GROWSUP # define VM_GROWSUP VM_NONE #endif diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 3a56238c8a4d..c39780ab5a87 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -272,6 +272,14 @@ static inline void sp_area_work_around(struct vm_unmapped_area_info *info, info->high_limit = min(info->high_limit, MMAP_SHARE_POOL_START); }

+static inline bool sp_check_vm_share_pool(unsigned long vm_flags) +{ + if (sp_is_enabled() && (vm_flags & VM_SHARE_POOL)) + return true; + + return false; +} + #else /* CONFIG_ASCEND_SHARE_POOL */

static inline int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id)

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

Sp-area specify a memory region used for share_pool. Add alloc/get/drop operations for sp-area.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Tang Yizhou tangyizhou@huawei.com Signed-off-by: Peng Wu wupeng58@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/share_pool.h | 8 + kernel/sysctl.c | 12 + mm/share_pool.c | 463 +++++++++++++++++++++++++++++++++++-- 3 files changed, 463 insertions(+), 20 deletions(-)

diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index c39780ab5a87..7e7ced34be57 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -257,6 +257,8 @@ extern bool mg_is_sharepool_addr(unsigned long addr); extern int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id); extern int sp_group_add_task(int pid, int spg_id);

+extern void sp_area_drop(struct vm_area_struct *vma); + static inline bool sp_is_enabled(void) { return static_branch_likely(&share_pool_enabled_key); @@ -280,6 +282,12 @@ static inline bool sp_check_vm_share_pool(unsigned long vm_flags) return false; }

+static inline void sp_dump_stack(void) +{ + if (sysctl_sp_debug_mode) + dump_stack(); +} + #else /* CONFIG_ASCEND_SHARE_POOL */

static inline int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id) diff --git a/kernel/sysctl.c b/kernel/sysctl.c index ce200213ccbd..8f417c7b12e8 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -71,6 +71,7 @@ #include <linux/coredump.h> #include <linux/latencytop.h> #include <linux/pid.h> +#include <linux/share_pool.h>

#include "../lib/kstrtox.h"

@@ -3257,6 +3258,17 @@ static struct ctl_table vm_table[] = { .proc_handler = proc_shrink_node_caches, .extra1 = SYSCTL_ZERO, }, +#endif +#ifdef CONFIG_ASCEND_SHARE_POOL + { + .procname = "sharepool_debug_mode", + .data = &sysctl_sp_debug_mode, + .maxlen = sizeof(sysctl_sp_debug_mode), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, #endif { } }; diff --git a/mm/share_pool.c b/mm/share_pool.c index 28bf0de8813b..0ea113b904cf 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -84,6 +84,9 @@ static const int mdc_default_group_id = 1; /* share the uva to the whole group */ static int __read_mostly enable_share_k2u_spg = 1;

+/* debug mode */ +int sysctl_sp_debug_mode; + static int share_pool_group_mode = SINGLE_GROUP_MODE;

static unsigned int sp_device_number; @@ -495,32 +498,136 @@ static struct file *spa_file(struct sp_area *spa) return spa->spg->file; }

-static inline void check_interrupt_context(void) -{ - if (unlikely(in_interrupt())) - panic("function can't be used in interrupt context\n"); +/* the caller should hold sp_area_lock */ +static void spa_inc_usage(struct sp_area *spa) +{ + enum spa_type type = spa->type; + unsigned long size = spa->real_size; + bool is_dvpp = spa->flags & SP_DVPP; + bool is_huge = spa->is_hugepage; + + switch (type) { + case SPA_TYPE_ALLOC: + spa_stat.alloc_num += 1; + spa_stat.alloc_size += size; + update_spg_stat_alloc(size, true, is_huge, spa->spg->stat); + break; + case SPA_TYPE_K2TASK: + spa_stat.k2u_task_num += 1; + spa_stat.k2u_task_size += size; + update_spg_stat_k2u(size, true, spg_none->stat); + break; + case SPA_TYPE_K2SPG: + spa_stat.k2u_spg_num += 1; + spa_stat.k2u_spg_size += size; + update_spg_stat_k2u(size, true, spa->spg->stat); + break; + default: + WARN(1, "invalid spa type"); + } + + if (is_dvpp) { + spa_stat.dvpp_size += size; + spa_stat.dvpp_va_size += ALIGN(size, PMD_SIZE); + } + + /* + * all the calculations won't overflow due to system limitation and + * parameter checking in sp_alloc_area() + */ + spa_stat.total_num += 1; + spa_stat.total_size += size; + + if (spa->spg != spg_none) { + atomic_inc(&sp_overall_stat.spa_total_num); + atomic64_add(size, &sp_overall_stat.spa_total_size); + } }

-static struct sp_group *create_spg(int spg_id) -{ - return NULL; +/* the caller should hold sp_area_lock */ +static void spa_dec_usage(struct sp_area *spa) +{ + enum spa_type type = spa->type; + unsigned long size = spa->real_size; + bool is_dvpp = spa->flags & SP_DVPP; + bool is_huge = spa->is_hugepage; + + switch (type) { + case SPA_TYPE_ALLOC: + spa_stat.alloc_num -= 1; + spa_stat.alloc_size -= size; + update_spg_stat_alloc(size, false, is_huge, spa->spg->stat); + break; + case SPA_TYPE_K2TASK: + spa_stat.k2u_task_num -= 1; + spa_stat.k2u_task_size -= size; + update_spg_stat_k2u(size, false, spg_none->stat); + break; + case SPA_TYPE_K2SPG: + spa_stat.k2u_spg_num -= 1; + spa_stat.k2u_spg_size -= size; + update_spg_stat_k2u(size, false, spa->spg->stat); + break; + default: + WARN(1, "invalid spa type"); + } + + if (is_dvpp) { + spa_stat.dvpp_size -= size; + spa_stat.dvpp_va_size -= ALIGN(size, PMD_SIZE); + } + + spa_stat.total_num -= 1; + spa_stat.total_size -= size; + + if (spa->spg != spg_none) { + atomic_dec(&sp_overall_stat.spa_total_num); + atomic64_sub(spa->real_size, &sp_overall_stat.spa_total_size); + } }

-static bool is_online_node_id(int node_id) +static void update_spg_proc_stat(unsigned long size, bool inc, + struct spg_proc_stat *stat, enum spa_type type) { - return node_id >= 0 && node_id < MAX_NUMNODES && node_online(node_id); + if (unlikely(!stat)) { + sp_dump_stack(); + WARN(1, "null process stat\n"); + return; + } + + switch (type) { + case SPA_TYPE_ALLOC: + update_spg_proc_stat_alloc(size, inc, stat); + break; + case SPA_TYPE_K2TASK: + case SPA_TYPE_K2SPG: + update_spg_proc_stat_k2u(size, inc, stat); + break; + default: + WARN(1, "invalid stat type\n"); + } }

-static bool is_device_addr(unsigned long addr) +static void sp_update_process_stat(struct task_struct *tsk, bool inc, + struct sp_area *spa) { - int i; + struct spg_proc_stat *stat; + unsigned long size = spa->real_size; + enum spa_type type = spa->type;

- for (i = 0; i < sp_device_number; i++) { - if (addr >= sp_dev_va_start[i] && - addr < sp_dev_va_start[i] + sp_dev_va_size[i]) - return true; - } - return false; + down_write(&sp_group_sem); + stat = sp_init_process_stat(tsk, tsk->mm, spa->spg); + up_write(&sp_group_sem); + if (unlikely(IS_ERR(stat))) + return; + + update_spg_proc_stat(size, inc, stat, type); +} + +static inline void check_interrupt_context(void) +{ + if (unlikely(in_interrupt())) + panic("function can't be used in interrupt context\n"); }

/** @@ -555,6 +662,28 @@ int mg_sp_group_id_by_pid(int pid, int *spg_ids, int *num) } EXPORT_SYMBOL_GPL(mg_sp_group_id_by_pid);

+static bool is_online_node_id(int node_id) +{ + return node_id >= 0 && node_id < MAX_NUMNODES && node_online(node_id); +} + +static bool is_device_addr(unsigned long addr) +{ + int i; + + for (i = 0; i < sp_device_number; i++) { + if (addr >= sp_dev_va_start[i] && + addr < sp_dev_va_start[i] + sp_dev_va_size[i]) + return true; + } + return false; +} + +static struct sp_group *create_spg(int spg_id) +{ + return NULL; +} + int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id) { return 0; @@ -567,9 +696,7 @@ int sp_group_add_task(int pid, int spg_id) } EXPORT_SYMBOL_GPL(sp_group_add_task);

-static void __sp_area_drop_locked(struct sp_area *spa) -{ -} +static void __sp_area_drop_locked(struct sp_area *spa);

/** * mg_sp_group_del_task() - delete a process from a sp group. @@ -595,6 +722,302 @@ int sp_group_del_task(int pid, int spg_id) } EXPORT_SYMBOL_GPL(sp_group_del_task);

+/* the caller must hold sp_area_lock */ +static void __insert_sp_area(struct sp_area *spa) +{ + struct rb_node **p = &sp_area_root.rb_node; + struct rb_node *parent = NULL; + + while (*p) { + struct sp_area *tmp; + + parent = *p; + tmp = rb_entry(parent, struct sp_area, rb_node); + if (spa->va_start < tmp->va_end) + p = &(*p)->rb_left; + else if (spa->va_end > tmp->va_start) + p = &(*p)->rb_right; + else + BUG(); + } + + rb_link_node(&spa->rb_node, parent, p); + rb_insert_color(&spa->rb_node, &sp_area_root); +} + +/* The sp_area cache globals are protected by sp_area_lock */ +static struct rb_node *free_sp_area_cache; +static unsigned long cached_hole_size; +static unsigned long cached_vstart; /* affected by SP_DVPP and sp_config_dvpp_range() */ + +/** + * sp_alloc_area() - Allocate a region of VA from the share pool. + * @size: the size of VA to allocate. + * @flags: how to allocate the memory. + * @spg: the share group that the memory is allocated to. + * @type: the type of the region. + * @applier: the pid of the task which allocates the region. + * + * Return: a valid pointer for success, NULL on failure. + */ +static struct sp_area *sp_alloc_area(unsigned long size, unsigned long flags, + struct sp_group *spg, enum spa_type type, + pid_t applier) +{ + struct sp_area *spa, *first, *err; + struct rb_node *n; + unsigned long vstart = MMAP_SHARE_POOL_START; + unsigned long vend = MMAP_SHARE_POOL_16G_START; + unsigned long addr; + unsigned long size_align = ALIGN(size, PMD_SIZE); /* va aligned to 2M */ + int device_id, node_id; + + device_id = sp_flags_device_id(flags); + node_id = flags & SP_SPEC_NODE_ID ? sp_flags_node_id(flags) : device_id; + + if (!is_online_node_id(node_id)) { + pr_err_ratelimited("invalid numa node id %d\n", node_id); + return ERR_PTR(-EINVAL); + } + + if ((flags & SP_DVPP)) { + if (!is_sp_dev_addr_enabled(device_id)) { + vstart = MMAP_SHARE_POOL_16G_START + + device_id * MMAP_SHARE_POOL_16G_SIZE; + vend = vstart + MMAP_SHARE_POOL_16G_SIZE; + } else { + vstart = sp_dev_va_start[device_id]; + vend = vstart + sp_dev_va_size[device_id]; + } + } + + spa = __kmalloc_node(sizeof(struct sp_area), GFP_KERNEL, node_id); + if (unlikely(!spa)) + return ERR_PTR(-ENOMEM); + + spin_lock(&sp_area_lock); + + /* + * Invalidate cache if we have more permissive parameters. + * cached_hole_size notes the largest hole noticed _below_ + * the sp_area cached in free_sp_area_cache: if size fits + * into that hole, we want to scan from vstart to reuse + * the hole instead of allocating above free_sp_area_cache. + * Note that sp_free_area may update free_sp_area_cache + * without updating cached_hole_size. + */ + if (!free_sp_area_cache || size_align < cached_hole_size || + vstart != cached_vstart) { + cached_hole_size = 0; + free_sp_area_cache = NULL; + } + + /* record if we encounter less permissive parameters */ + cached_vstart = vstart; + + /* find starting point for our search */ + if (free_sp_area_cache) { + first = rb_entry(free_sp_area_cache, struct sp_area, rb_node); + addr = first->va_end; + if (addr + size_align < addr) { + err = ERR_PTR(-EOVERFLOW); + goto error; + } + } else { + addr = vstart; + if (addr + size_align < addr) { + err = ERR_PTR(-EOVERFLOW); + goto error; + } + + n = sp_area_root.rb_node; + first = NULL; + + while (n) { + struct sp_area *tmp; + + tmp = rb_entry(n, struct sp_area, rb_node); + if (tmp->va_end >= addr) { + first = tmp; + if (tmp->va_start <= addr) + break; + n = n->rb_left; + } else + n = n->rb_right; + } + + if (!first) + goto found; + } + + /* from the starting point, traverse areas until a suitable hole is found */ + while (addr + size_align > first->va_start && addr + size_align <= vend) { + if (addr + cached_hole_size < first->va_start) + cached_hole_size = first->va_start - addr; + addr = first->va_end; + if (addr + size_align < addr) { + err = ERR_PTR(-EOVERFLOW); + goto error; + } + + n = rb_next(&first->rb_node); + if (n) + first = rb_entry(n, struct sp_area, rb_node); + else + goto found; + } + +found: + if (addr + size_align > vend) { + err = ERR_PTR(-EOVERFLOW); + goto error; + } + + spa->va_start = addr; + spa->va_end = addr + size_align; + spa->real_size = size; + spa->region_vstart = vstart; + spa->flags = flags; + spa->is_hugepage = (flags & SP_HUGEPAGE); + spa->is_dead = false; + spa->spg = spg; + atomic_set(&spa->use_count, 1); + spa->type = type; + spa->mm = NULL; + spa->kva = 0; /* NULL pointer */ + spa->applier = applier; + spa->node_id = node_id; + spa->device_id = device_id; + + spa_inc_usage(spa); + __insert_sp_area(spa); + free_sp_area_cache = &spa->rb_node; + if (spa->spg != spg_none) + list_add_tail(&spa->link, &spg->spa_list); + + spin_unlock(&sp_area_lock); + + return spa; + +error: + spin_unlock(&sp_area_lock); + kfree(spa); + return err; +} + +/* the caller should hold sp_area_lock */ +static struct sp_area *__find_sp_area_locked(unsigned long addr) +{ + struct rb_node *n = sp_area_root.rb_node; + + while (n) { + struct sp_area *spa; + + spa = rb_entry(n, struct sp_area, rb_node); + if (addr < spa->va_start) { + n = n->rb_left; + } else if (addr > spa->va_start) { + n = n->rb_right; + } else { + return spa; + } + } + + return NULL; +} + +static struct sp_area *__find_sp_area(unsigned long addr) +{ + struct sp_area *n; + + spin_lock(&sp_area_lock); + n = __find_sp_area_locked(addr); + if (n) + atomic_inc(&n->use_count); + spin_unlock(&sp_area_lock); + return n; +} + +/* + * Free the VA region starting from addr to the share pool + */ +static void sp_free_area(struct sp_area *spa) +{ + lockdep_assert_held(&sp_area_lock); + + if (free_sp_area_cache) { + struct sp_area *cache; + + cache = rb_entry(free_sp_area_cache, struct sp_area, rb_node); + if (spa->va_start <= cache->va_start) { + free_sp_area_cache = rb_prev(&spa->rb_node); + /* + * the new cache node may be changed to another region, + * i.e. from DVPP region to normal region + */ + if (free_sp_area_cache) { + cache = rb_entry(free_sp_area_cache, + struct sp_area, rb_node); + cached_vstart = cache->region_vstart; + } + /* + * We don't try to update cached_hole_size, + * but it won't go very wrong. + */ + } + } + + spa_dec_usage(spa); + if (spa->spg != spg_none) + list_del(&spa->link); + + rb_erase(&spa->rb_node, &sp_area_root); + RB_CLEAR_NODE(&spa->rb_node); + kfree(spa); +} + +static void __sp_area_drop_locked(struct sp_area *spa) +{ + /* + * Considering a situation where task A and B are in the same spg. + * A is exiting and calling remove_vma(). Before A calls this func, + * B calls sp_free() to free the same spa. So spa maybe NULL when A + * calls this func later. + */ + if (!spa) + return; + + if (atomic_dec_and_test(&spa->use_count)) + sp_free_area(spa); +} + +static void __sp_area_drop(struct sp_area *spa) +{ + spin_lock(&sp_area_lock); + __sp_area_drop_locked(spa); + spin_unlock(&sp_area_lock); +} + +void sp_area_drop(struct vm_area_struct *vma) +{ + struct sp_area *spa; + + if (!(vma->vm_flags & VM_SHARE_POOL)) + return; + + /* + * Considering a situation where task A and B are in the same spg. + * A is exiting and calling remove_vma() -> ... -> sp_area_drop(). + * Concurrently, B is calling sp_free() to free the same spa. + * __find_sp_area_locked() and __sp_area_drop_locked() should be + * an atomic operation. + */ + spin_lock(&sp_area_lock); + spa = __find_sp_area_locked(vma->vm_start); + __sp_area_drop_locked(spa); + spin_unlock(&sp_area_lock); +} + /** * sp_free() - Free the memory allocated by sp_alloc(). * @addr: the starting VA of the memory.

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

This function could map a user memory to kernel vmalloc space.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Tang Yizhou tangyizhou@huawei.com Signed-off-by: Peng Wu wupeng58@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/share_pool.c | 56 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-)

diff --git a/mm/share_pool.c b/mm/share_pool.c index 1cd1a64f2a8c..4d45a2519b10 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -1358,7 +1358,61 @@ static void __sp_walk_page_free(struct sp_walk_data *data) */ void *sp_make_share_u2k(unsigned long uva, unsigned long size, int pid) { - return NULL; + int ret = 0; + struct mm_struct *mm = current->mm; + void *p = ERR_PTR(-ESRCH); + struct sp_walk_data sp_walk_data = { + .page_count = 0, + }; + struct vm_struct *area; + + check_interrupt_context(); + + if (mm == NULL) { + pr_err("u2k: kthread is not allowed\n"); + return ERR_PTR(-EPERM); + } + + down_write(&mm->mmap_lock); + if (unlikely(mm->core_state)) { + up_write(&mm->mmap_lock); + pr_err("u2k: encountered coredump, abort\n"); + return p; + } + + ret = __sp_walk_page_range(uva, size, mm, &sp_walk_data); + if (ret) { + pr_err_ratelimited("walk page range failed %d\n", ret); + up_write(&mm->mmap_lock); + return ERR_PTR(ret); + } + + if (sp_walk_data.is_hugepage) + p = vmap_hugepage(sp_walk_data.pages, sp_walk_data.page_count, + VM_MAP, PAGE_KERNEL); + else + p = vmap(sp_walk_data.pages, sp_walk_data.page_count, VM_MAP, + PAGE_KERNEL); + up_write(&mm->mmap_lock); + + if (!p) { + pr_err("vmap(huge) in u2k failed\n"); + __sp_walk_page_free(&sp_walk_data); + return ERR_PTR(-ENOMEM); + } + + p = p + (uva - sp_walk_data.uva_aligned); + + /* + * kva p may be used later in k2u. Since p comes from uva originally, + * it's reasonable to add flag VM_USERMAP so that p can be remapped + * into userspace again. + */ + area = find_vm_area(p); + area->flags |= VM_USERMAP; + + kvfree(sp_walk_data.pages); + return p; } EXPORT_SYMBOL_GPL(sp_make_share_u2k);

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

This is used to find the group for a task. Also add spg management code.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Tang Yizhou tangyizhou@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/share_pool.c | 258 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 255 insertions(+), 3 deletions(-)

diff --git a/mm/share_pool.c b/mm/share_pool.c index 8e357056110e..10f1aa496116 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -89,6 +89,8 @@ int sysctl_sp_debug_mode;

static int share_pool_group_mode = SINGLE_GROUP_MODE;

+static int system_group_count; + static unsigned int sp_device_number; static unsigned long sp_dev_va_start[MAX_DEVID]; static unsigned long sp_dev_va_size[MAX_DEVID]; @@ -630,6 +632,136 @@ static inline void check_interrupt_context(void) panic("function can't be used in interrupt context\n"); }

+static void free_sp_group_id(int spg_id) +{ + /* ida operation is protected by an internal spin_lock */ + if (spg_id >= SPG_ID_AUTO_MIN && spg_id <= SPG_ID_AUTO_MAX) + ida_free(&sp_group_id_ida, spg_id); +} + +static void free_sp_group(struct sp_group *spg) +{ + fput(spg->file); + fput(spg->file_hugetlb); + free_spg_stat(spg->id); + down_write(&sp_group_sem); + idr_remove(&sp_group_idr, spg->id); + up_write(&sp_group_sem); + free_sp_group_id((unsigned int)spg->id); + kfree(spg); + system_group_count--; + WARN(system_group_count < 0, "unexpected group count\n"); +} + +static void sp_group_drop(struct sp_group *spg) +{ + if (atomic_dec_and_test(&spg->use_count)) + free_sp_group(spg); +} + +/* use with put_task_struct(task) */ +static int get_task(int pid, struct task_struct **task) +{ + struct task_struct *tsk; + + rcu_read_lock(); + tsk = find_task_by_vpid(pid); + if (!tsk || (tsk->flags & PF_EXITING)) { + rcu_read_unlock(); + return -ESRCH; + } + get_task_struct(tsk); + rcu_read_unlock(); + + *task = tsk; + return 0; +} + +static struct sp_group *get_first_group(struct mm_struct *mm) +{ + struct sp_group *spg = NULL; + struct sp_group_master *master = mm->sp_group_master; + + if (master && master->count >= 1) { + struct sp_group_node *spg_node = NULL; + + spg_node = list_first_entry(&master->node_list, + struct sp_group_node, group_node); + spg = spg_node->spg; + + /* don't revive a dead group */ + if (!spg || !atomic_inc_not_zero(&spg->use_count)) + spg = NULL; + } + + return spg; +} + +/* + * the caller must: + * 1. hold spg->rw_lock + * 2. ensure no concurrency problem for mm_struct + */ +static struct sp_group_node *is_process_in_group(struct sp_group *spg, + struct mm_struct *mm) +{ + struct sp_group_node *spg_node; + + list_for_each_entry(spg_node, &spg->procs, proc_node) + if (spg_node->master->mm == mm) + return spg_node; + + return NULL; +} + +/* user must call sp_group_drop() after use */ +static struct sp_group *__sp_find_spg_locked(int pid, int spg_id) +{ + struct sp_group *spg = NULL; + struct task_struct *tsk = NULL; + int ret = 0; + + ret = get_task(pid, &tsk); + if (ret) + return NULL; + + if (spg_id == SPG_ID_DEFAULT) { + /* + * Once we encounter a concurrency problem here. + * To fix it, we believe get_task_mm() and mmput() is too + * heavy because we just get the pointer of sp_group. + */ + task_lock(tsk); + if (tsk->mm == NULL) + spg = NULL; + else + spg = get_first_group(tsk->mm); + task_unlock(tsk); + } else { + spg = idr_find(&sp_group_idr, spg_id); + /* don't revive a dead group */ + if (!spg || !atomic_inc_not_zero(&spg->use_count)) + goto fail; + } + + put_task_struct(tsk); + return spg; + +fail: + put_task_struct(tsk); + return NULL; +} + +static struct sp_group *__sp_find_spg(int pid, int spg_id) +{ + struct sp_group *spg; + + down_read(&sp_group_sem); + spg = __sp_find_spg_locked(pid, spg_id); + up_read(&sp_group_sem); + return spg; +} + /** * sp_group_id_by_pid() - Get the sp_group ID of a process. * @pid: pid of target process. @@ -640,7 +772,22 @@ static inline void check_interrupt_context(void) */ int sp_group_id_by_pid(int pid) { - return 0; + struct sp_group *spg; + int spg_id = -ENODEV; + + check_interrupt_context(); + + spg = __sp_find_spg(pid, SPG_ID_DEFAULT); + if (!spg) + return -ENODEV; + + down_read(&spg->rw_lock); + if (spg_valid(spg)) + spg_id = spg->id; + up_read(&spg->rw_lock); + + sp_group_drop(spg); + return spg_id; } EXPORT_SYMBOL_GPL(sp_group_id_by_pid);

@@ -658,7 +805,48 @@ EXPORT_SYMBOL_GPL(sp_group_id_by_pid); */ int mg_sp_group_id_by_pid(int pid, int *spg_ids, int *num) { - return 0; + int ret = 0; + struct sp_group_node *node; + struct sp_group_master *master = NULL; + struct task_struct *tsk; + + check_interrupt_context(); + + if (!spg_ids || num <= 0) + return -EINVAL; + + ret = get_task(pid, &tsk); + if (ret) + return ret; + + down_read(&sp_group_sem); + task_lock(tsk); + if (tsk->mm) + master = tsk->mm->sp_group_master; + task_unlock(tsk); + + if (!master) { + ret = -ENODEV; + goto out_up_read; + } + + if (!master->count) { + ret = -ENODEV; + goto out_up_read; + } + if ((unsigned int)*num < master->count) { + ret = -E2BIG; + goto out_up_read; + } + *num = master->count; + + list_for_each_entry(node, &master->node_list, group_node) + *(spg_ids++) = node->spg->id; + +out_up_read: + up_read(&sp_group_sem); + put_task_struct(tsk); + return ret; } EXPORT_SYMBOL_GPL(mg_sp_group_id_by_pid);

@@ -681,7 +869,71 @@ static bool is_device_addr(unsigned long addr)

static struct sp_group *create_spg(int spg_id) { - return NULL; + int ret; + struct sp_group *spg; + char name[20]; + struct user_struct *user = NULL; + int hsize_log = MAP_HUGE_2MB >> MAP_HUGE_SHIFT; + + if (unlikely(system_group_count + 1 == MAX_GROUP_FOR_SYSTEM)) { + pr_err_ratelimited("reach system max group num\n"); + return ERR_PTR(-ENOSPC); + } + + spg = kzalloc(sizeof(*spg), GFP_KERNEL); + if (spg == NULL) + return ERR_PTR(-ENOMEM); + + ret = idr_alloc(&sp_group_idr, spg, spg_id, spg_id + 1, GFP_KERNEL); + if (ret < 0) { + pr_err_ratelimited("group %d idr alloc failed %d\n", + spg_id, ret); + goto out_kfree; + } + + spg->id = spg_id; + spg->is_alive = true; + spg->proc_num = 0; + spg->owner = current->group_leader; + atomic_set(&spg->use_count, 1); + INIT_LIST_HEAD(&spg->procs); + INIT_LIST_HEAD(&spg->spa_list); + init_rwsem(&spg->rw_lock); + + sprintf(name, "sp_group_%d", spg_id); + spg->file = shmem_kernel_file_setup(name, MAX_LFS_FILESIZE, + VM_NORESERVE); + if (IS_ERR(spg->file)) { + pr_err("spg file setup failed %ld\n", PTR_ERR(spg->file)); + ret = PTR_ERR(spg->file); + goto out_idr; + } + + spg->file_hugetlb = hugetlb_file_setup(name, MAX_LFS_FILESIZE, + VM_NORESERVE, &user, HUGETLB_ANONHUGE_INODE, hsize_log); + if (IS_ERR(spg->file_hugetlb)) { + pr_err("spg file_hugetlb setup failed %ld\n", + PTR_ERR(spg->file_hugetlb)); + ret = PTR_ERR(spg->file_hugetlb); + goto out_fput; + } + + ret = sp_init_spg_stat(spg); + if (ret < 0) + goto out_fput_all; + + system_group_count++; + return spg; + +out_fput_all: + fput(spg->file_hugetlb); +out_fput: + fput(spg->file); +out_idr: + idr_remove(&sp_group_idr, spg_id); +out_kfree: + kfree(spg); + return ERR_PTR(ret); }

int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id)

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

This share a kernel memory range to userspace. Introduce vm_struct flag VM_SHAREPOOL to indicate that a vm_struct shared to userspace and we cannot vfree such a vm_area.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Tang Yizhou tangyizhou@huawei.com Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/share_pool.h | 8 + include/linux/vmalloc.h | 5 + kernel/sysctl.c | 19 ++ mm/hugetlb.c | 8 + mm/share_pool.c | 404 ++++++++++++++++++++++++++++++++++++- mm/vmalloc.c | 7 + 6 files changed, 450 insertions(+), 1 deletion(-)

diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index 7e7ced34be57..6ec844708f83 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -288,6 +288,14 @@ static inline void sp_dump_stack(void) dump_stack(); }

+static inline bool is_vmalloc_sharepool(unsigned long vm_flags) +{ + if (sp_is_enabled() && (vm_flags & VM_SHAREPOOL)) + return true; + + return false; +} + #else /* CONFIG_ASCEND_SHARE_POOL */

static inline int mg_sp_group_add_task(int pid, unsigned long prot, int spg_id) diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 8f9e13944cc7..49c94afce25b 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -28,6 +28,11 @@ struct notifier_block; /* in notifier.h */ #define VM_MAP_PUT_PAGES 0x00000200 /* put pages and free array in vfree */ #define VM_NO_HUGE_VMAP 0x00000400 /* force PAGE_SIZE pte mapping */ #define VM_HUGE_PAGES 0x00001000 /* used for vmalloc hugepages */ +#ifdef CONFIG_ASCEND_SHARE_POOL +#define VM_SHAREPOOL 0x00002000 /* remapped to sharepool */ +#else +#define VM_SHAREPOOL 0 +#endif

/* * VM_KASAN is used slighly differently depending on CONFIG_KASAN_VMALLOC. diff --git a/kernel/sysctl.c b/kernel/sysctl.c index c7073b652b0c..9d242841ca3f 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3287,6 +3287,25 @@ static struct ctl_table vm_table[] = { .extra1 = &zero_ul, .extra2 = &sysctl_sp_compact_interval_max, }, + { + /* 0: map_unlock, 1: map_lock */ + .procname = "share_pool_map_lock_enable", + .data = &sysctl_share_pool_map_lock_enable, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "sharepool_perf_k2u", + .data = &sysctl_sp_perf_k2u, + .maxlen = sizeof(sysctl_sp_perf_k2u), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &ten_thousand, + }, #endif { } }; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index eaddb18d58e1..74d23542f9f2 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -31,6 +31,7 @@ #include <linux/llist.h> #include <linux/cma.h> #include <linux/mman.h> +#include <linux/share_pool.h>

#include <asm/page.h> #include <asm/pgalloc.h> @@ -4110,6 +4111,13 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,

pte = huge_ptep_get_and_clear(mm, address, ptep); tlb_remove_huge_tlb_entry(h, tlb, ptep, address); + + /* sharepool k2u mapped pages are marked special */ + if (sp_check_vm_share_pool(vma->vm_flags) && pte_special(pte)) { + spin_unlock(ptl); + continue; + } + if (huge_pte_dirty(pte)) set_page_dirty(page);

diff --git a/mm/share_pool.c b/mm/share_pool.c index bb134dd7ffc4..3c935d256b08 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -87,6 +87,10 @@ static int __read_mostly enable_share_k2u_spg = 1; /* debug mode */ int sysctl_sp_debug_mode;

+int sysctl_share_pool_map_lock_enable; + +int sysctl_sp_perf_k2u; + static int share_pool_group_mode = SINGLE_GROUP_MODE;

static int system_group_count; @@ -632,6 +636,13 @@ static inline void check_interrupt_context(void) panic("function can't be used in interrupt context\n"); }

+static unsigned long sp_mmap(struct mm_struct *mm, struct file *file, + struct sp_area *spa, unsigned long *populate, + unsigned long prot); +static void sp_munmap(struct mm_struct *mm, unsigned long addr, unsigned long size); +static unsigned long sp_remap_kva_to_vma(unsigned long kva, struct sp_area *spa, + struct mm_struct *mm, unsigned long prot); + static void free_sp_group_id(int spg_id) { /* ida operation is protected by an internal spin_lock */ @@ -1206,6 +1217,19 @@ static struct sp_area *__find_sp_area(unsigned long addr) return n; }

+static bool vmalloc_area_clr_flag(unsigned long kva, unsigned long flags) +{ + struct vm_struct *area; + + area = find_vm_area((void *)kva); + if (area) { + area->flags &= ~flags; + return true; + } + + return false; +} + /* * Free the VA region starting from addr to the share pool */ @@ -1235,6 +1259,9 @@ static void sp_free_area(struct sp_area *spa) } }

+ if (spa->kva && !vmalloc_area_clr_flag(spa->kva, VM_SHAREPOOL)) + pr_debug("clear spa->kva %ld is not valid\n", spa->kva); + spa_dec_usage(spa); if (spa->spg != spg_none) list_del(&spa->link); @@ -1542,6 +1569,37 @@ int mg_sp_free(unsigned long addr) } EXPORT_SYMBOL_GPL(mg_sp_free);

+/* wrapper of __do_mmap() and the caller must hold down_write(&mm->mmap_lock). */ +static unsigned long sp_mmap(struct mm_struct *mm, struct file *file, + struct sp_area *spa, unsigned long *populate, + unsigned long prot) +{ + unsigned long addr = spa->va_start; + unsigned long size = spa_size(spa); + unsigned long flags = MAP_FIXED | MAP_SHARED | MAP_POPULATE | + MAP_SHARE_POOL; + unsigned long vm_flags = VM_NORESERVE | VM_SHARE_POOL | VM_DONTCOPY; + unsigned long pgoff = addr_offset(spa) >> PAGE_SHIFT; + + /* Mark the mapped region to be locked. After the MAP_LOCKED is enable, + * multiple tasks will preempt resources, causing performance loss. + */ + if (sysctl_share_pool_map_lock_enable) + flags |= MAP_LOCKED; + + atomic_inc(&spa->use_count); + addr = __do_mmap_mm(mm, file, addr, size, prot, flags, vm_flags, pgoff, + populate, NULL); + if (IS_ERR_VALUE(addr)) { + atomic_dec(&spa->use_count); + pr_err("do_mmap fails %ld\n", addr); + } else { + BUG_ON(addr != spa->va_start); + } + + return addr; +} + /** * sp_alloc() - Allocate shared memory for all the processes in a sp_group. * @size: the size of memory to allocate. @@ -1596,6 +1654,314 @@ static int is_vmap_hugepage(unsigned long addr) return 0; }

+static unsigned long __sp_remap_get_pfn(unsigned long kva) +{ + unsigned long pfn; + + if (is_vmalloc_addr((void *)kva)) + pfn = vmalloc_to_pfn((void *)kva); + else + pfn = virt_to_pfn(kva); + + return pfn; +} + +/* when called by k2u to group, always make sure rw_lock of spg is down */ +static unsigned long sp_remap_kva_to_vma(unsigned long kva, struct sp_area *spa, + struct mm_struct *mm, unsigned long prot) +{ + struct vm_area_struct *vma; + unsigned long ret_addr; + unsigned long populate = 0; + int ret = 0; + unsigned long addr, buf, offset; + + down_write(&mm->mmap_lock); + if (unlikely(mm->core_state)) { + pr_err("k2u mmap: encountered coredump, abort\n"); + ret_addr = -EBUSY; + goto put_mm; + } + + ret_addr = sp_mmap(mm, spa_file(spa), spa, &populate, prot); + if (IS_ERR_VALUE(ret_addr)) { + pr_debug("k2u mmap failed %lx\n", ret_addr); + goto put_mm; + } + BUG_ON(ret_addr != spa->va_start); + + vma = find_vma(mm, ret_addr); + BUG_ON(vma == NULL); + if (prot & PROT_WRITE) + vma->vm_page_prot = __pgprot(((~PTE_RDONLY) & vma->vm_page_prot.pgprot) | PTE_DIRTY); + + if (is_vm_hugetlb_page(vma)) { + ret = remap_vmalloc_hugepage_range(vma, (void *)kva, 0); + if (ret) { + do_munmap(mm, ret_addr, spa_size(spa), NULL); + pr_debug("remap vmalloc hugepage failed, ret %d, kva is %lx\n", + ret, (unsigned long)kva); + ret_addr = ret; + goto put_mm; + } + vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; + } else { + buf = ret_addr; + addr = kva; + offset = 0; + do { + ret = remap_pfn_range(vma, buf, __sp_remap_get_pfn(addr), PAGE_SIZE, + __pgprot(vma->vm_page_prot.pgprot)); + if (ret) { + do_munmap(mm, ret_addr, spa_size(spa), NULL); + pr_err("remap_pfn_range failed %d\n", ret); + ret_addr = ret; + goto put_mm; + } + offset += PAGE_SIZE; + buf += PAGE_SIZE; + addr += PAGE_SIZE; + } while (offset < spa_size(spa)); + } + +put_mm: + up_write(&mm->mmap_lock); + + return ret_addr; +} + +/** + * sp_make_share_kva_to_task() - Share kernel memory to current task. + * @kva: the VA of shared kernel memory + * @size: the size of area to share, should be aligned properly + * @sp_flags: the flags for the opreation + * + * Return: + * * if succeed, return the shared user address to start at. + * * if fail, return the pointer of -errno. + */ +static void *sp_make_share_kva_to_task(unsigned long kva, unsigned long size, unsigned long sp_flags) +{ + void *uva; + struct sp_area *spa; + struct spg_proc_stat *stat; + unsigned long prot = PROT_READ | PROT_WRITE; + + down_write(&sp_group_sem); + stat = sp_init_process_stat(current, current->mm, spg_none); + up_write(&sp_group_sem); + if (IS_ERR(stat)) { + pr_err_ratelimited("k2u_task init process stat failed %lx\n", + PTR_ERR(stat)); + return stat; + } + + spa = sp_alloc_area(size, sp_flags, spg_none, SPA_TYPE_K2TASK, current->tgid); + if (IS_ERR(spa)) { + pr_err_ratelimited("alloc spa failed in k2u_task (potential no enough virtual memory when -75): %ld\n", + PTR_ERR(spa)); + return spa; + } + + spa->kva = kva; + + uva = (void *)sp_remap_kva_to_vma(kva, spa, current->mm, prot); + __sp_area_drop(spa); + if (IS_ERR(uva)) + pr_err("remap k2u to task failed %ld\n", PTR_ERR(uva)); + else { + update_spg_proc_stat(size, true, stat, SPA_TYPE_K2TASK); + spa->mm = current->mm; + } + + return uva; +} + +/** + * Share kernel memory to a spg, the current process must be in that group + * @kva: the VA of shared kernel memory + * @size: the size of area to share, should be aligned properly + * @sp_flags: the flags for the opreation + * @spg: the sp group to be shared with + * + * Return: the shared user address to start at + */ +static void *sp_make_share_kva_to_spg(unsigned long kva, unsigned long size, + unsigned long sp_flags, struct sp_group *spg) +{ + struct sp_area *spa; + struct mm_struct *mm; + struct sp_group_node *spg_node; + void *uva = ERR_PTR(-ENODEV); + + down_read(&spg->rw_lock); + spa = sp_alloc_area(size, sp_flags, spg, SPA_TYPE_K2SPG, current->tgid); + if (IS_ERR(spa)) { + up_read(&spg->rw_lock); + pr_err_ratelimited("alloc spa failed in k2u_spg (potential no enough virtual memory when -75): %ld\n", + PTR_ERR(spa)); + return spa; + } + + spa->kva = kva; + + list_for_each_entry(spg_node, &spg->procs, proc_node) { + mm = spg_node->master->mm; + uva = (void *)sp_remap_kva_to_vma(kva, spa, mm, spg_node->prot); + if (IS_ERR(uva)) { + pr_err("remap k2u to spg failed %ld\n", PTR_ERR(uva)); + __sp_free(spg, spa->va_start, spa_size(spa), mm); + goto out; + } + } + +out: + up_read(&spg->rw_lock); + __sp_area_drop(spa); + if (!IS_ERR(uva)) + sp_update_process_stat(current, true, spa); + + return uva; +} + +static bool vmalloc_area_set_flag(unsigned long kva, unsigned long flags) +{ + struct vm_struct *area; + + area = find_vm_area((void *)kva); + if (area) { + area->flags |= flags; + return true; + } + + return false; +} + +struct sp_k2u_context { + unsigned long kva; + unsigned long kva_aligned; + unsigned long size; + unsigned long size_aligned; + unsigned long sp_flags; + int spg_id; + bool to_task; + struct timespec64 start; + struct timespec64 end; +}; + +static void trace_sp_k2u_begin(struct sp_k2u_context *kc) +{ + if (!sysctl_sp_perf_k2u) + return; + + ktime_get_ts64(&kc->start); +} + +static void trace_sp_k2u_finish(struct sp_k2u_context *kc, void *uva) +{ + unsigned long cost; + + if (!sysctl_sp_perf_k2u) + return; + + ktime_get_ts64(&kc->end); + + cost = SEC2US(kc->end.tv_sec - kc->start.tv_sec) + + NS2US(kc->end.tv_nsec - kc->start.tv_nsec); + if (cost >= (unsigned long)sysctl_sp_perf_k2u) { + pr_err("Task %s(%d/%d) sp_k2u returns 0x%lx consumes %luus, size is %luKB, size_aligned is %luKB, sp_flags is %lx, to_task is %d\n", + current->comm, current->tgid, current->pid, + (unsigned long)uva, cost, byte2kb(kc->size), byte2kb(kc->size_aligned), + kc->sp_flags, kc->to_task); + } +} + +static int sp_k2u_prepare(unsigned long kva, unsigned long size, + unsigned long sp_flags, int spg_id, struct sp_k2u_context *kc) +{ + int is_hugepage; + unsigned int page_size = PAGE_SIZE; + unsigned long kva_aligned, size_aligned; + + trace_sp_k2u_begin(kc); + + if (sp_flags & ~SP_DVPP) { + pr_err_ratelimited("k2u sp_flags %lx error\n", sp_flags); + return -EINVAL; + } + + if (!current->mm) { + pr_err_ratelimited("k2u: kthread is not allowed\n"); + return -EPERM; + } + + is_hugepage = is_vmap_hugepage(kva); + if (is_hugepage > 0) { + sp_flags |= SP_HUGEPAGE; + page_size = PMD_SIZE; + } else if (is_hugepage == 0) { + /* do nothing */ + } else { + pr_err_ratelimited("k2u kva is not vmalloc address\n"); + return is_hugepage; + } + + /* aligned down kva is convenient for caller to start with any valid kva */ + kva_aligned = ALIGN_DOWN(kva, page_size); + size_aligned = ALIGN(kva + size, page_size) - kva_aligned; + + if (!vmalloc_area_set_flag(kva_aligned, VM_SHAREPOOL)) { + pr_debug("k2u_task kva %lx is not valid\n", kva_aligned); + return -EINVAL; + } + + kc->kva = kva; + kc->kva_aligned = kva_aligned; + kc->size = size; + kc->size_aligned = size_aligned; + kc->sp_flags = sp_flags; + kc->spg_id = spg_id; + kc->to_task = false; + return 0; +} + +static int sp_check_k2task(struct sp_k2u_context *kc) +{ + int ret = 0; + int spg_id = kc->spg_id; + + if (share_pool_group_mode == SINGLE_GROUP_MODE) { + struct sp_group *spg = get_first_group(current->mm); + + if (!spg) { + if (spg_id != SPG_ID_NONE && spg_id != SPG_ID_DEFAULT) + ret = -EINVAL; + else + kc->to_task = true; + } else { + if (spg_id != SPG_ID_DEFAULT && spg_id != spg->id) + ret = -EINVAL; + sp_group_drop(spg); + } + } else { + if (spg_id == SPG_ID_DEFAULT || spg_id == SPG_ID_NONE) + kc->to_task = true; + } + return ret; +} + +static void *sp_k2u_finish(void *uva, struct sp_k2u_context *kc) +{ + if (IS_ERR(uva)) + vmalloc_area_clr_flag(kc->kva_aligned, VM_SHAREPOOL); + else + uva = uva + (kc->kva - kc->kva_aligned); + + trace_sp_k2u_finish(kc, uva); + sp_dump_stack(); + return uva; +} + /** * sp_make_share_k2u() - Share kernel memory to current process or an sp_group. * @kva: the VA of shared kernel memory. @@ -1616,7 +1982,43 @@ static int is_vmap_hugepage(unsigned long addr) void *sp_make_share_k2u(unsigned long kva, unsigned long size, unsigned long sp_flags, int pid, int spg_id) { - return NULL; + void *uva; + int ret; + struct sp_k2u_context kc; + + check_interrupt_context(); + + ret = sp_k2u_prepare(kva, size, sp_flags, spg_id, &kc); + if (ret) + return ERR_PTR(ret); + + ret = sp_check_k2task(&kc); + if (ret) { + uva = ERR_PTR(ret); + goto out; + } + + if (kc.to_task) + uva = sp_make_share_kva_to_task(kc.kva_aligned, kc.size_aligned, kc.sp_flags); + else { + struct sp_group *spg; + + spg = __sp_find_spg(current->pid, kc.spg_id); + if (spg) { + ret = sp_check_caller_permission(spg, current->mm); + if (ret < 0) { + sp_group_drop(spg); + uva = ERR_PTR(ret); + goto out; + } + uva = sp_make_share_kva_to_spg(kc.kva_aligned, kc.size_aligned, kc.sp_flags, spg); + sp_group_drop(spg); + } else + uva = ERR_PTR(-ENODEV); + } + +out: + return sp_k2u_finish(uva, &kc); } EXPORT_SYMBOL_GPL(sp_make_share_k2u);

diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 755be0c19c81..dadbea29241d 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -37,6 +37,7 @@ #include <linux/pgtable.h> #include <linux/uaccess.h> #include <linux/hugetlb.h> +#include <linux/share_pool.h> #include <asm/io.h> #include <asm/tlbflush.h> #include <asm/shmparam.h> @@ -2622,6 +2623,12 @@ static void __vunmap(const void *addr, int deallocate_pages) return; }

+ /* unmap a sharepool vm area will cause meamleak! */ + if (is_vmalloc_sharepool(area->flags)) { + WARN(1, "Memory leak due to vfree() sharepool vm area (%p) !\n", addr); + return; + } + debug_check_no_locks_freed(area->addr, get_vm_area_size(area)); debug_check_no_obj_freed(area->addr, get_vm_area_size(area));

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

Alloc share memory for tasks in a share pool group. The tasks in the same pool could access the memory with the same virtual address.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Tang Yizhou tangyizhou@huawei.com Signed-off-by: Peng Wu wupeng58@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- kernel/sysctl.c | 9 ++ mm/share_pool.c | 397 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 405 insertions(+), 1 deletion(-)

diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 9d242841ca3f..97019c861d73 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -3306,6 +3306,15 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ZERO, .extra2 = &ten_thousand, }, + { + .procname = "sharepool_perf_alloc", + .data = &sysctl_sp_perf_alloc, + .maxlen = sizeof(sysctl_sp_perf_alloc), + .mode = 0600, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &ten_thousand, + }, #endif { } }; diff --git a/mm/share_pool.c b/mm/share_pool.c index 355c429b3843..d9f015e22abe 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -90,6 +90,7 @@ int sysctl_sp_debug_mode; int sysctl_share_pool_map_lock_enable;

int sysctl_sp_perf_k2u; +int sysctl_sp_perf_alloc;

static int share_pool_group_mode = SINGLE_GROUP_MODE;

@@ -1600,6 +1601,373 @@ static unsigned long sp_mmap(struct mm_struct *mm, struct file *file, return addr; }

+#define ALLOC_NORMAL 1 +#define ALLOC_RETRY 2 +#define ALLOC_NOMEM 3 + +struct sp_alloc_context { + struct sp_group *spg; + struct file *file; + unsigned long size; + unsigned long size_aligned; + unsigned long sp_flags; + unsigned long populate; + int state; + bool need_fallocate; + struct timespec64 start; + struct timespec64 end; +}; + +static void trace_sp_alloc_begin(struct sp_alloc_context *ac) +{ + if (!sysctl_sp_perf_alloc) + return; + + ktime_get_ts64(&ac->start); +} + +static void trace_sp_alloc_finish(struct sp_alloc_context *ac, unsigned long va) +{ + unsigned long cost; + bool is_pass_through = ac->spg == spg_none ? true : false; + + if (!sysctl_sp_perf_alloc) + return; + + ktime_get_ts64(&ac->end); + + cost = SEC2US(ac->end.tv_sec - ac->start.tv_sec) + + NS2US(ac->end.tv_nsec - ac->start.tv_nsec); + if (cost >= (unsigned long)sysctl_sp_perf_alloc) { + pr_err("Task %s(%d/%d) sp_alloc returns 0x%lx consumes %luus, size is %luKB, size_aligned is %luKB, sp_flags is %lx, pass through is %d\n", + current->comm, current->tgid, current->pid, + va, cost, byte2kb(ac->size), byte2kb(ac->size_aligned), ac->sp_flags, is_pass_through); + } +} + +static int sp_alloc_prepare(unsigned long size, unsigned long sp_flags, + int spg_id, struct sp_alloc_context *ac) +{ + struct sp_group *spg; + + check_interrupt_context(); + + trace_sp_alloc_begin(ac); + + /* mdc scene hack */ + if (enable_mdc_default_group) + spg_id = mdc_default_group_id; + + if (unlikely(!size || (size >> PAGE_SHIFT) > totalram_pages())) { + pr_err_ratelimited("allocation failed, invalid size %lu\n", size); + return -EINVAL; + } + + if (spg_id != SPG_ID_DEFAULT && spg_id < SPG_ID_MIN) { + pr_err_ratelimited("allocation failed, invalid group id %d\n", spg_id); + return -EINVAL; + } + + if (sp_flags & (~SP_FLAG_MASK)) { + pr_err_ratelimited("allocation failed, invalid flag %lx\n", sp_flags); + return -EINVAL; + } + + if (sp_flags & SP_HUGEPAGE_ONLY) + sp_flags |= SP_HUGEPAGE; + + if (share_pool_group_mode == SINGLE_GROUP_MODE) { + spg = __sp_find_spg(current->pid, SPG_ID_DEFAULT); + if (spg) { + if (spg_id != SPG_ID_DEFAULT && spg->id != spg_id) { + sp_group_drop(spg); + return -ENODEV; + } + + /* up_read will be at the end of sp_alloc */ + down_read(&spg->rw_lock); + if (!spg_valid(spg)) { + up_read(&spg->rw_lock); + sp_group_drop(spg); + pr_err_ratelimited("allocation failed, spg is dead\n"); + return -ENODEV; + } + } else { /* alocation pass through scene */ + if (enable_mdc_default_group) { + int ret = 0; + + ret = sp_group_add_task(current->tgid, spg_id); + if (ret < 0) { + pr_err_ratelimited("add group failed in pass through\n"); + return ret; + } + + spg = __sp_find_spg(current->pid, SPG_ID_DEFAULT); + + /* up_read will be at the end of sp_alloc */ + down_read(&spg->rw_lock); + if (!spg_valid(spg)) { + up_read(&spg->rw_lock); + sp_group_drop(spg); + pr_err_ratelimited("pass through allocation failed, spg is dead\n"); + return -ENODEV; + } + } else { + spg = spg_none; + } + } + } else { + if (spg_id != SPG_ID_DEFAULT) { + spg = __sp_find_spg(current->pid, spg_id); + if (!spg) { + pr_err_ratelimited("allocation failed, can't find group\n"); + return -ENODEV; + } + + /* up_read will be at the end of sp_alloc */ + down_read(&spg->rw_lock); + if (!spg_valid(spg)) { + up_read(&spg->rw_lock); + sp_group_drop(spg); + pr_err_ratelimited("allocation failed, spg is dead\n"); + return -ENODEV; + } + + if (!is_process_in_group(spg, current->mm)) { + up_read(&spg->rw_lock); + sp_group_drop(spg); + pr_err_ratelimited("allocation failed, task not in group\n"); + return -ENODEV; + } + } else { /* alocation pass through scene */ + spg = spg_none; + } + } + + if (sp_flags & SP_HUGEPAGE) { + ac->file = spg->file_hugetlb; + ac->size_aligned = ALIGN(size, PMD_SIZE); + } else { + ac->file = spg->file; + ac->size_aligned = ALIGN(size, PAGE_SIZE); + } + + ac->spg = spg; + ac->size = size; + ac->sp_flags = sp_flags; + ac->state = ALLOC_NORMAL; + ac->need_fallocate = false; + return 0; +} + +static void sp_alloc_unmap(struct mm_struct *mm, struct sp_area *spa, + struct sp_group_node *spg_node) +{ + if (spa->spg != spg_none) + __sp_free(spa->spg, spa->va_start, spa->real_size, mm); +} + +static int sp_alloc_mmap(struct mm_struct *mm, struct sp_area *spa, + struct sp_group_node *spg_node, struct sp_alloc_context *ac) +{ + int ret = 0; + unsigned long mmap_addr; + /* pass through default permission */ + unsigned long prot = PROT_READ | PROT_WRITE; + unsigned long sp_addr = spa->va_start; + unsigned long populate = 0; + struct vm_area_struct *vma; + + down_write(&mm->mmap_lock); + if (unlikely(mm->core_state)) { + up_write(&mm->mmap_lock); + sp_alloc_unmap(mm, spa, spg_node); + ac->state = ALLOC_NOMEM; + pr_info("allocation encountered coredump\n"); + return -EFAULT; + } + + if (spg_node) + prot = spg_node->prot; + + /* when success, mmap_addr == spa->va_start */ + mmap_addr = sp_mmap(mm, spa_file(spa), spa, &populate, prot); + if (IS_ERR_VALUE(mmap_addr)) { + up_write(&mm->mmap_lock); + sp_alloc_unmap(mm, spa, spg_node); + pr_err("sp mmap in allocation failed %ld\n", mmap_addr); + return PTR_ERR((void *)mmap_addr); + } + + if (unlikely(populate == 0)) { + up_write(&mm->mmap_lock); + pr_err("allocation sp mmap populate failed\n"); + ret = -EFAULT; + goto unmap; + } + ac->populate = populate; + + vma = find_vma(mm, sp_addr); + if (unlikely(!vma)) { + up_write(&mm->mmap_lock); + WARN(1, "allocation failed, can't find %lx vma\n", sp_addr); + ret = -EINVAL; + goto unmap; + } + /* clean PTE_RDONLY flags or trigger SMMU event */ + if (prot & PROT_WRITE) + vma->vm_page_prot = __pgprot(((~PTE_RDONLY) & vma->vm_page_prot.pgprot) | PTE_DIRTY); + up_write(&mm->mmap_lock); + + return ret; + +unmap: + if (spa->spg != spg_none) + sp_alloc_unmap(list_next_entry(spg_node, proc_node)->master->mm, spa, spg_node); + else + sp_munmap(mm, spa->va_start, spa->real_size); + return ret; +} + +static void sp_alloc_fallback(struct sp_area *spa, struct sp_alloc_context *ac) +{ + struct sp_spg_stat *stat = ac->spg->stat; + + if (ac->file == ac->spg->file) { + ac->state = ALLOC_NOMEM; + return; + } + + atomic_inc(&stat->hugepage_failures); + if (!(ac->sp_flags & SP_HUGEPAGE_ONLY)) { + ac->file = ac->spg->file; + ac->size_aligned = ALIGN(ac->size, PAGE_SIZE); + ac->sp_flags &= ~SP_HUGEPAGE; + ac->state = ALLOC_RETRY; + __sp_area_drop(spa); + return; + } + ac->state = ALLOC_NOMEM; +} + +static int sp_alloc_populate(struct mm_struct *mm, struct sp_area *spa, + struct sp_group_node *spg_node, struct sp_alloc_context *ac) +{ + int ret = 0; + unsigned long sp_addr = spa->va_start; + unsigned int noreclaim_flag = 0; + + /* + * The direct reclaim and compact may take a long + * time. As a result, sp mutex will be hold for too + * long time to casue the hung task problem. In this + * case, set the PF_MEMALLOC flag to prevent the + * direct reclaim and compact from being executed. + * Since direct reclaim and compact are not performed + * when the fragmentation is severe or the memory is + * insufficient, 2MB continuous physical pages fail + * to be allocated. This situation is allowed. + */ + if (spa->is_hugepage) + noreclaim_flag = memalloc_noreclaim_save(); + + /* + * We are not ignoring errors, so if we fail to allocate + * physical memory we just return failure, so we won't encounter + * page fault later on, and more importantly sp_make_share_u2k() + * depends on this feature (and MAP_LOCKED) to work correctly. + */ + ret = do_mm_populate(mm, sp_addr, ac->populate, 0); + if (spa->is_hugepage) { + memalloc_noreclaim_restore(noreclaim_flag); + if (ret) + sp_add_work_compact(); + } + if (ret) { + if (spa->spg != spg_none) + sp_alloc_unmap(list_next_entry(spg_node, proc_node)->master->mm, spa, spg_node); + else + sp_munmap(mm, spa->va_start, spa->real_size); + + if (unlikely(fatal_signal_pending(current))) + pr_warn_ratelimited("allocation failed, current thread is killed\n"); + else + pr_warn_ratelimited("allocation failed due to mm populate failed(potential no enough memory when -12): %d\n", + ret); + sp_fallocate(spa); /* need this, otherwise memleak */ + sp_alloc_fallback(spa, ac); + } else { + ac->need_fallocate = true; + } + return ret; +} + +static int __sp_alloc_mmap_populate(struct mm_struct *mm, struct sp_area *spa, + struct sp_group_node *spg_node, struct sp_alloc_context *ac) +{ + int ret; + + ret = sp_alloc_mmap(mm, spa, spg_node, ac); + if (ret < 0) { + if (ac->need_fallocate) { + /* e.g. second sp_mmap fail */ + sp_fallocate(spa); + ac->need_fallocate = false; + } + return ret; + } + + ret = sp_alloc_populate(mm, spa, spg_node, ac); + return ret; +} + +static int sp_alloc_mmap_populate(struct sp_area *spa, + struct sp_alloc_context *ac) +{ + int ret; + struct mm_struct *mm; + struct sp_group_node *spg_node; + + if (spa->spg == spg_none) { + ret = __sp_alloc_mmap_populate(current->mm, spa, NULL, ac); + } else { + /* create mapping for each process in the group */ + list_for_each_entry(spg_node, &spa->spg->procs, proc_node) { + mm = spg_node->master->mm; + ret = __sp_alloc_mmap_populate(mm, spa, spg_node, ac); + if (ret) + return ret; + } + } + return ret; +} + +/* spa maybe an error pointer, so introduce variable spg */ +static void sp_alloc_finish(int result, struct sp_area *spa, + struct sp_alloc_context *ac) +{ + struct sp_group *spg = ac->spg; + bool is_pass_through = spg == spg_none ? true : false; + + /* match sp_alloc_check_prepare */ + if (!is_pass_through) + up_read(&spg->rw_lock); + + if (!result) + sp_update_process_stat(current, true, spa); + + /* this will free spa if mmap failed */ + if (spa && !IS_ERR(spa)) + __sp_area_drop(spa); + + if (!is_pass_through) + sp_group_drop(spg); + + trace_sp_alloc_finish(ac, spa->va_start); + sp_dump_stack(); + sp_try_to_compact(); +} + /** * sp_alloc() - Allocate shared memory for all the processes in a sp_group. * @size: the size of memory to allocate. @@ -1614,7 +1982,34 @@ static unsigned long sp_mmap(struct mm_struct *mm, struct file *file, */ void *sp_alloc(unsigned long size, unsigned long sp_flags, int spg_id) { - return NULL; + struct sp_area *spa = NULL; + int ret = 0; + struct sp_alloc_context ac; + + ret = sp_alloc_prepare(size, sp_flags, spg_id, &ac); + if (ret) + return ERR_PTR(ret); + +try_again: + spa = sp_alloc_area(ac.size_aligned, ac.sp_flags, ac.spg, + SPA_TYPE_ALLOC, current->tgid); + if (IS_ERR(spa)) { + pr_err_ratelimited("alloc spa failed in allocation(potential no enough virtual memory when -75): %ld\n", + PTR_ERR(spa)); + ret = PTR_ERR(spa); + goto out; + } + + ret = sp_alloc_mmap_populate(spa, &ac); + if (ret && ac.state == ALLOC_RETRY) + goto try_again; + +out: + sp_alloc_finish(ret, spa, &ac); + if (ret) + return ERR_PTR(ret); + else + return (void *)(spa->va_start); } EXPORT_SYMBOL_GPL(sp_alloc);

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

This interface is added to support the function of exiting a process from a sharing group.

Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Signed-off-by: Guo Mengqi guomengqi3@huawei.com Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/share_pool.c | 121 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 2 deletions(-)

diff --git a/mm/share_pool.c b/mm/share_pool.c index 6d64086e0cd9..8dc64232f0db 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -156,6 +156,21 @@ static struct sp_group_master *sp_init_group_master_locked( return master; }

+static struct sp_proc_stat *sp_get_proc_stat(struct mm_struct *mm) +{ + struct sp_proc_stat *stat; + + if (!mm->sp_group_master) + return NULL; + + down_read(&sp_proc_stat_sem); + stat = mm->sp_group_master->stat; + up_read(&sp_proc_stat_sem); + + /* maybe NULL or not, we always return it */ + return stat; +} + static struct sp_proc_stat *create_proc_stat(struct mm_struct *mm, struct task_struct *tsk) { @@ -1383,7 +1398,26 @@ int sp_group_add_task(int pid, int spg_id) } EXPORT_SYMBOL_GPL(sp_group_add_task);

-static void __sp_area_drop_locked(struct sp_area *spa); +static void free_spg_proc_stat(struct mm_struct *mm, int spg_id) +{ + int i; + struct sp_proc_stat *proc_stat = sp_get_proc_stat(mm); + struct spg_proc_stat *stat; + struct sp_spg_stat *spg_stat; + struct hlist_node *tmp; + + hash_for_each_safe(proc_stat->hash, i, tmp, stat, pnode) { + if (stat->spg_id == spg_id) { + spg_stat = stat->spg_stat; + mutex_lock(&spg_stat->lock); + hash_del(&stat->gnode); + mutex_unlock(&spg_stat->lock); + hash_del(&stat->pnode); + kfree(stat); + break; + } + } +}

/** * mg_sp_group_del_task() - delete a process from a sp group. @@ -1399,7 +1433,90 @@ static void __sp_area_drop_locked(struct sp_area *spa); */ int mg_sp_group_del_task(int pid, int spg_id) { - return 0; + int ret = 0; + struct sp_group *spg; + struct sp_group_node *spg_node; + struct task_struct *tsk = NULL; + struct mm_struct *mm = NULL; + bool is_alive = true; + + if (spg_id < SPG_ID_MIN || spg_id > SPG_ID_AUTO) { + pr_err_ratelimited("del from group failed, invalid group id %d\n", spg_id); + return -EINVAL; + } + + spg = __sp_find_spg(pid, spg_id); + if (!spg) { + pr_err_ratelimited("spg not found or get task failed."); + return -EINVAL; + } + down_write(&sp_group_sem); + + if (!spg_valid(spg)) { + up_write(&sp_group_sem); + pr_err_ratelimited("spg dead."); + ret = -EINVAL; + goto out; + } + + if (!list_empty(&spg->spa_list)) { + up_write(&sp_group_sem); + pr_err_ratelimited("spa is not empty"); + ret = -EINVAL; + goto out; + } + + ret = get_task(pid, &tsk); + if (ret) { + up_write(&sp_group_sem); + pr_err_ratelimited("task is not found"); + goto out; + } + mm = get_task_mm(tsk->group_leader); + if (!mm) { + up_write(&sp_group_sem); + pr_err_ratelimited("mm is not found"); + ret = -ESRCH; + goto out_put_task; + } + + spg_node = is_process_in_group(spg, mm); + if (!spg_node) { + up_write(&sp_group_sem); + pr_err_ratelimited("process not in group"); + ret = -ESRCH; + goto out_put_mm; + } + + down_write(&spg->rw_lock); + if (list_is_singular(&spg->procs)) + is_alive = spg->is_alive = false; + spg->proc_num--; + list_del(&spg_node->proc_node); + sp_group_drop(spg); + up_write(&spg->rw_lock); + if (!is_alive) + blocking_notifier_call_chain(&sp_notifier_chain, 0, spg); + + list_del(&spg_node->group_node); + mm->sp_group_master->count--; + kfree(spg_node); + if (atomic_sub_and_test(1, &mm->mm_users)) { + up_write(&sp_group_sem); + WARN(1, "Invalid user counting\n"); + return -EINVAL; + } + + free_spg_proc_stat(mm, spg_id); + up_write(&sp_group_sem); + +out_put_mm: + mmput(mm); +out_put_task: + put_task_struct(tsk); +out: + sp_group_drop(spg); /* if spg dead, freed here */ + return ret; } EXPORT_SYMBOL_GPL(mg_sp_group_del_task);

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

The situation below is not allowed:

int *result = mmap(ADDR, sizeof(int), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);

As share pool uses an independent UVA allocation algorithm, it may produce an address that is conflicted with user-specified address.

Signed-off-by: Tang Yizhou tangyizhou@huawei.com Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/share_pool.h | 2 ++ mm/mmap.c | 9 +++++++++ mm/mremap.c | 4 ++++ mm/share_pool.c | 20 ++++++++++++++++++++ 4 files changed, 35 insertions(+)

diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index ac637359e158..0254ea95f034 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -260,6 +260,8 @@ extern int sp_group_add_task(int pid, int spg_id); extern void sp_area_drop(struct vm_area_struct *vma); extern int sp_group_exit(struct mm_struct *mm); extern void sp_group_post_exit(struct mm_struct *mm); +extern bool sp_check_addr(unsigned long addr); +extern bool sp_check_mmap_addr(unsigned long addr, unsigned long flags);

static inline bool sp_is_enabled(void) { diff --git a/mm/mmap.c b/mm/mmap.c index c616e99e7672..36ef4c2b93a9 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2430,6 +2430,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, if (len > mmap_end - mmap_min_addr) return -ENOMEM;

+ if (sp_check_mmap_addr(addr, flags)) + return -EINVAL; + if (flags & MAP_FIXED) return addr;

@@ -2481,6 +2484,9 @@ arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, if (len > mmap_end - mmap_min_addr) return -ENOMEM;

+ if (sp_check_mmap_addr(addr, flags)) + return -EINVAL; + if (flags & MAP_FIXED) return addr;

@@ -3209,6 +3215,9 @@ static int __vm_munmap(unsigned long start, size_t len, bool downgrade) struct mm_struct *mm = current->mm; LIST_HEAD(uf);

+ if (sp_check_addr(start)) + return -EINVAL; + if (mmap_write_lock_killable(mm)) return -EINTR;

diff --git a/mm/mremap.c b/mm/mremap.c index d4d66d1b6ec7..ecfca97b97ae 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -24,6 +24,7 @@ #include <linux/uaccess.h> #include <linux/mm-arch-hooks.h> #include <linux/userfaultfd_k.h> +#include <linux/share_pool.h>

#include <asm/cacheflush.h> #include <asm/tlb.h> @@ -929,6 +930,9 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, if (offset_in_page(addr)) return ret;

+ if (sp_check_addr(addr) || sp_check_addr(new_addr)) + return ret; + old_len = PAGE_ALIGN(old_len); new_len = PAGE_ALIGN(new_len);

diff --git a/mm/share_pool.c b/mm/share_pool.c index 96fc899617a5..7853054ed005 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -4139,6 +4139,26 @@ static void __init proc_sharepool_init(void)

/*** End of tatistical and maintenance functions ***/

+bool sp_check_addr(unsigned long addr) +{ + if (sp_is_enabled() && is_sharepool_addr(addr) && + !check_aoscore_process(current)) { + sp_dump_stack(); + return true; + } else + return false; +} + +bool sp_check_mmap_addr(unsigned long addr, unsigned long flags) +{ + if (sp_is_enabled() && is_sharepool_addr(addr) && + !check_aoscore_process(current) && !(flags & MAP_SHARE_POOL)) { + sp_dump_stack(); + return true; + } else + return false; +} + #define MM_WOULD_FREE 1

/*

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

In the share pool scenario, when the shared memory is applied for, the do_mm_populate function is performed at the same time, that is, the corresponding pages are allocated. In the current share pool implementation, the memory is charged to the memcg of the first task added to this share pool group.

This is unreasonable and may cause memcg of first task oom. So, we should charge the pages to the memcg of current task.

Signed-off-by: Zhou Guanghui zhouguanghui1@huawei.com Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Reviewed-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/shmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mm/shmem.c b/mm/shmem.c index 60644e9b1a7b..d2e86b35c7ff 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -1819,7 +1819,7 @@ static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, }

sbinfo = SHMEM_SB(inode->i_sb); - charge_mm = vma ? vma->vm_mm : current->mm; + charge_mm = vma && !sp_check_vm_share_pool(vma->vm_flags) ? vma->vm_mm : current->mm;

page = pagecache_get_page(mapping, index, FGP_ENTRY | FGP_HEAD | FGP_LOCK, 0);

From: Wang Wensheng wangwensheng4@huawei.com

ascend inclusion category: Feature bugzilla: https://gitee.com/openeuler/kernel/issues/I4NDAW CVE: NA

-------------------

Sharepool applies for a dedicated interface for large pages, which optimizes the efficiency of memory application

Signed-off-by: Weilong Chen chenweilong@huawei.com Signed-off-by: Wang Wensheng wangwensheng4@huawei.com Reviewed-by: Kefeng Wangwangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/hugetlb.h | 4 ++ include/linux/share_pool.h | 12 +++++ mm/hugetlb.c | 10 +++- mm/share_pool.c | 97 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 1 deletion(-)

diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 397e6bfa8268..1b586a324b8e 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -1120,4 +1120,8 @@ static inline __init void hugetlb_cma_check(void) } #endif

+#ifdef CONFIG_ASCEND_SHARE_POOL +pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, int writable); +#endif + #endif /* _LINUX_HUGETLB_H */ diff --git a/include/linux/share_pool.h b/include/linux/share_pool.h index ca9fcde90211..6f294911c6af 100644 --- a/include/linux/share_pool.h +++ b/include/linux/share_pool.h @@ -260,6 +260,10 @@ extern int sp_group_add_task(int pid, int spg_id); extern void sp_area_drop(struct vm_area_struct *vma); extern int sp_group_exit(struct mm_struct *mm); extern void sp_group_post_exit(struct mm_struct *mm); +vm_fault_t sharepool_no_page(struct mm_struct *mm, + struct vm_area_struct *vma, + struct address_space *mapping, pgoff_t idx, + unsigned long address, pte_t *ptep, unsigned int flags); extern bool sp_check_addr(unsigned long addr); extern bool sp_check_mmap_addr(unsigned long addr, unsigned long flags); extern int sp_node_id(struct vm_area_struct *vma); @@ -515,6 +519,14 @@ static inline bool sp_check_mmap_addr(unsigned long addr, unsigned long flags) return false; }

+static inline vm_fault_t sharepool_no_page(struct mm_struct *mm, + struct vm_area_struct *vma, + struct address_space *mapping, pgoff_t idx, + unsigned long address, pte_t *ptep, unsigned int flags) +{ + return VM_FAULT_SIGBUS; +} + #endif /* !CONFIG_ASCEND_SHARE_POOL */

#endif /* LINUX_SHARE_POOL_H */ diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 74d23542f9f2..44691ce8a9b8 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -3861,8 +3861,13 @@ const struct vm_operations_struct hugetlb_vm_ops = { .pagesize = hugetlb_vm_op_pagesize, };

+#ifdef CONFIG_ASCEND_SHARE_POOL +pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, + int writable) +#else static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page, int writable) +#endif { pte_t entry;

@@ -4727,7 +4732,10 @@ vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,

entry = huge_ptep_get(ptep); if (huge_pte_none(entry)) { - ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); + if (sp_check_vm_share_pool(vma->vm_flags)) + ret = sharepool_no_page(mm, vma, mapping, idx, address, ptep, flags); + else + ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags); goto out_mutex; }

diff --git a/mm/share_pool.c b/mm/share_pool.c index 05218c6dd250..494a829d6f3a 100644 --- a/mm/share_pool.c +++ b/mm/share_pool.c @@ -4177,6 +4177,103 @@ bool sp_check_mmap_addr(unsigned long addr, unsigned long flags) return false; }

+vm_fault_t sharepool_no_page(struct mm_struct *mm, + struct vm_area_struct *vma, + struct address_space *mapping, pgoff_t idx, + unsigned long address, pte_t *ptep, unsigned int flags) +{ + struct hstate *h = hstate_vma(vma); + vm_fault_t ret = VM_FAULT_SIGBUS; + unsigned long size; + struct page *page; + pte_t new_pte; + spinlock_t *ptl; + unsigned long haddr = address & huge_page_mask(h); + bool new_page = false; + int err; + int node_id; + struct sp_area *spa; + + spa = __find_sp_area(vma->vm_start); + if (!spa) { + pr_err("share pool: vma is invalid, not from sp mmap\n"); + return ret; + } + node_id = spa->node_id; + __sp_area_drop(spa); + +retry: + page = find_lock_page(mapping, idx); + if (!page) { + size = i_size_read(mapping->host) >> huge_page_shift(h); + if (idx >= size) + goto out; + + page = alloc_huge_page(vma, haddr, 0); + if (IS_ERR(page)) { + page = alloc_huge_page_nodemask(hstate_file(vma->vm_file), + node_id, NULL, GFP_KERNEL); + if (!page) + page = ERR_PTR(-ENOMEM); + } + if (IS_ERR(page)) { + ptl = huge_pte_lock(h, mm, ptep); + if (!huge_pte_none(huge_ptep_get(ptep))) { + ret = 0; + spin_unlock(ptl); + goto out; + } + spin_unlock(ptl); + ret = vmf_error(PTR_ERR(page)); + goto out; + } + __SetPageUptodate(page); + new_page = true; + + /* sharepool pages are all shared */ + err = huge_add_to_page_cache(page, mapping, idx); + if (err) { + put_page(page); + if (err == -EEXIST) + goto retry; + goto out; + } + } + + + ptl = huge_pte_lock(h, mm, ptep); + size = i_size_read(mapping->host) >> huge_page_shift(h); + if (idx >= size) + goto backout; + + ret = 0; + if (!huge_pte_none(huge_ptep_get(ptep))) + goto backout; + + page_dup_rmap(page, true); + new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE) + && (vma->vm_flags & VM_SHARED))); + set_huge_pte_at(mm, haddr, ptep, new_pte); + + hugetlb_count_add(pages_per_huge_page(h), mm); + + spin_unlock(ptl); + + if (new_page) { + SetPagePrivate(&page[1]); + } + + unlock_page(page); +out: + return ret; + +backout: + spin_unlock(ptl); + unlock_page(page); + put_page(page); + goto out; +} + #define MM_WOULD_FREE 1

/*