Add the monitoring part of the module and its runtime loop. Writing 1 to the dynthreaded module parameter starts a repeating hrtimer which schedules a work item every dyn_threaded_internal ms; writing 0 (or unloading the module) stops it again. The work item takes a sample of the scheduling statistics of all threads whose name matches the appname module parameter: their total runtime, the total number of context switches and their voluntary/involuntary split, as well as the union of the CPU masks allowed to those tasks. Each sample is compared with the previous one and judged as: DATA_LOSS counters moved backwards, the sample is dropped and the previous status is kept NO_TASK no thread with the target name exists NOT_RUNNING avg runtime below dyn_threaded_not_running_limit, the tasks hardly get the CPU LONG_RUNNING_WITH_INTERRUPTS avg runtime above dyn_threaded_avg_runtime_limit and a low voluntary switch ratio, i.e. the tasks are disturbed by softirq activity FRAGMENTATION short avg runtime per context switch, processing in softirq context is acceptable For the sample right after entering LONG_RUNNING_WITH_INTERRUPTS, the avg runtime and switch ratio of the last sample before the change are carried over, so that the status decision is not based on the transient sample. The subsequent patches act on this status. Co-developed-by: Yue Haibing <yuehaibing@huawei.com> Signed-off-by: Yue Haibing <yuehaibing@huawei.com> Assisted-by: opencode:glm-5.3 Signed-off-by: Cai Xinchen <caixinchen1@huawei.com> --- net/dynthread/net_dynthread.c | 347 +++++++++++++++++++++++++++++++++- 1 file changed, 344 insertions(+), 3 deletions(-) diff --git a/net/dynthread/net_dynthread.c b/net/dynthread/net_dynthread.c index 1aa7b35cfd94..c600428220f2 100644 --- a/net/dynthread/net_dynthread.c +++ b/net/dynthread/net_dynthread.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* Dynamic threaded NAPI scheduler * - * Record the RX queue IRQ information of the net devices configured - * via the ifname module parameter, so that the IRQ placement can be - * reworked at runtime. + * Monitor the scheduling behaviour of a target application and switch + * the NAPI processing of the configured net devices between softirq + * context and kernel threads, whose CPU affinity is adapted to the + * measured load. The RX queue IRQ placement of the devices is + * adjusted as well. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -12,6 +14,13 @@ #include <linux/module.h> #include <linux/kernel.h> #include <linux/string.h> +#include <linux/rcupdate.h> +#include <linux/cpumask.h> +#include <linux/ktime.h> +#include <linux/hrtimer.h> +#include <linux/workqueue.h> +#include <linux/sched.h> +#include <linux/sched/signal.h> #include <linux/irq.h> #include <linux/netdevice.h> #include <linux/topology.h> @@ -19,8 +28,10 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Dynamic threaded NAPI scheduler"); +#define ONE_HUNDRED 100 #define SCHED_MAX_NETDEV_NUM 8 #define MAX_DEV_RXQ_NUM 256 +#define DYN_THREAD_INTERVAL (dyn_threaded_internal * NSEC_PER_MSEC) static char ifname[64]; module_param_string(ifname, ifname, sizeof(ifname), 0644); @@ -30,6 +41,66 @@ static char irqname[64] = "comp"; module_param_string(irqname, irqname, sizeof(irqname), 0644); MODULE_PARM_DESC(irqname, "substring of the NIC RX queue IRQ names, default comp"); +static char appname[64] = "mysqld"; +module_param_string(appname, appname, sizeof(appname), 0644); +MODULE_PARM_DESC(appname, "name of the target application threads, default mysqld"); + +static uint dyn_threaded_nvcsw_per_limit = 95; +module_param(dyn_threaded_nvcsw_per_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_nvcsw_per_limit, + "max nvcsw percent for long running status, default 95"); + +static uint dyn_threaded_avg_runtime_limit = 350000; +module_param(dyn_threaded_avg_runtime_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_avg_runtime_limit, + "min avg runtime (ns) for long running status, default 350000"); + +static uint dyn_threaded_not_running_limit = 30000; +module_param(dyn_threaded_not_running_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_not_running_limit, + "max avg runtime (ns) for not running status, default 30000"); + +static uint dyn_threaded_internal = 1000; + +static int dyn_threaded_internal_set(const char *val, + const struct kernel_param *kp) +{ + uint interval; + int ret; + + ret = kstrtouint(val, 0, &interval); + if (ret) + return ret; + + /* a zero interval would divide by zero and spin the timer */ + if (!interval) + return -EINVAL; + + *(uint *)kp->arg = interval; + + return 0; +} + +static int dyn_threaded_internal_get(char *buffer, + const struct kernel_param *kp) +{ + return scnprintf(buffer, PAGE_SIZE, "%u\n", *(uint *)kp->arg); +} + +static const struct kernel_param_ops dyn_threaded_internal_ops = { + .set = dyn_threaded_internal_set, + .get = dyn_threaded_internal_get, +}; + +module_param_cb(dyn_threaded_internal, &dyn_threaded_internal_ops, + &dyn_threaded_internal, 0644); +MODULE_PARM_DESC(dyn_threaded_internal, "sample interval in ms, default 1000"); + +static uint dyn_threaded_avg_runtime_keep_per = 90; +module_param(dyn_threaded_avg_runtime_keep_per, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_avg_runtime_keep_per, + "min percent of avg runtime kept in long running status, default 90"); + /** * struct sched_netdev_queue_info - IRQ information of one RX queue * @irq: Linux IRQ number of the queue @@ -56,6 +127,51 @@ struct sched_netdev { struct sched_netdev_queue_info rxq[MAX_DEV_RXQ_NUM]; }; +/** + * enum task_status - scheduling state of the target tasks + * @DATA_LOSS: the counters moved backwards, the sample is not usable + * @NO_TASK: no task with the target name exists + * @NOT_RUNNING: the tasks hardly get the CPU + * @FRAGMENTATION: the tasks are scheduled out quickly, softirq mode is fine + * @LONG_RUNNING_WITH_INTERRUPTS: the tasks run long between context + * switches and should not be disturbed by softirq activity + */ +enum task_status { + DATA_LOSS, + NO_TASK, + NOT_RUNNING, + FRAGMENTATION, + LONG_RUNNING_WITH_INTERRUPTS, +}; + +/** + * struct sched_task_runtime_avg_info - scheduling statistics of the target tasks + * @total_switches: total context switches since boot + * @total_runtime_ns: total runtime in ns since boot + * @total_nvcsw: total voluntary context switches since boot + * @total_nivcsw: total involuntary context switches since boot + * @avg_runtime: avg runtime of the sample in ns, per context switch + * @nvcsw_per: percent of voluntary context switches of the sample + * @cpuload: total load of the sample in percent, over all allowed CPUs + * @thread_cnt: number of matching threads + * @status: task status judged for this sample + * @last_status: task status of the previous sample + * @comm: name of the target tasks + */ +struct sched_task_runtime_avg_info { + u64 total_switches; + u64 total_runtime_ns; + u64 total_nvcsw; + u64 total_nivcsw; + u64 avg_runtime; + u64 nvcsw_per; + u64 cpuload; + u32 thread_cnt; + enum task_status status; + enum task_status last_status; + char comm[TASK_COMM_LEN]; +}; + static int sched_netdev_num; static struct sched_netdev sched_netdev_table[SCHED_MAX_NETDEV_NUM]; @@ -251,6 +367,228 @@ static int init_netdevinfo(const char *netdev_str) return err; } +/* status of the last sample of the target tasks */ +static struct sched_task_runtime_avg_info snapshot; + +/* union of the CPU masks allowed to the target tasks */ +static struct cpumask target_cpumask; +static u32 target_cpucnt; + +/* Take a sample of the scheduling statistics of all threads whose name + * matches @target_comm, and compute the union of their allowed CPU + * masks. The info must be zeroed by the caller. + */ +static int sched_task_runtime_avg_snapshot(const char *target_comm, + struct sched_task_runtime_avg_info *infos) +{ + struct task_struct *g, *p; + char comm[TASK_COMM_LEN]; + u64 nvcsw, nivcsw, switches; + struct cpumask masks; + + if (!target_comm || !infos) + return -EINVAL; + + cpumask_clear(&masks); + strscpy(infos->comm, target_comm, sizeof(infos->comm)); + + rcu_read_lock(); + for_each_process(g) { + get_task_comm(comm, g); + if (strcmp(comm, infos->comm)) + continue; + + cpumask_or(&masks, &masks, g->cpus_ptr); + + for_each_thread(g, p) { + nvcsw = (u64)READ_ONCE(p->nvcsw); + nivcsw = READ_ONCE(p->nivcsw); + switches = nvcsw + nivcsw; + + infos->total_runtime_ns += READ_ONCE(p->se.sum_exec_runtime); + infos->total_switches += switches; + infos->total_nvcsw += nvcsw; + infos->total_nivcsw += nivcsw; + infos->thread_cnt++; + } + } + rcu_read_unlock(); + + cpumask_copy(&target_cpumask, &masks); + WRITE_ONCE(target_cpucnt, cpumask_weight(&target_cpumask)); + + + return 0; +} + +/* load of the target tasks over the CPUs allowed to them, in percent */ +static inline u64 cal_target_task_cpuload(struct sched_task_runtime_avg_info *infos) +{ + return 100 * (infos->total_runtime_ns - snapshot.total_runtime_ns) / + DYN_THREAD_INTERVAL / READ_ONCE(target_cpucnt); +} + +/* Compare the current sample with the previous one and judge the + * status of the target tasks. A sample whose counters moved backwards + * yields DATA_LOSS and is evaluated by the caller only. + */ +static void judge_target_task_status(struct sched_task_runtime_avg_info *info) +{ + u64 switches; + + if (info->total_switches < snapshot.total_switches || + info->total_runtime_ns < snapshot.total_runtime_ns || + info->total_nvcsw < snapshot.total_nvcsw || + info->total_nivcsw < snapshot.total_nivcsw || + strcmp(info->comm, snapshot.comm)) { + info->status = DATA_LOSS; + return; + } + + if (info->thread_cnt == 0) { + info->status = NO_TASK; + return; + } + + /* a sample may contain no context switch, in which case the + * runtime delta is the runtime of the whole sample; clamp the + * switch count to avoid a division by zero + */ + switches = info->total_switches - snapshot.total_switches; + switches = max_t(u64, switches, 1); + + info->avg_runtime = (info->total_runtime_ns - snapshot.total_runtime_ns) / + switches; + info->nvcsw_per = (info->total_nvcsw - snapshot.total_nvcsw) * 100 / + switches; + info->cpuload = cal_target_task_cpuload(info); + + if (info->avg_runtime < dyn_threaded_not_running_limit) { + info->status = NOT_RUNNING; + } else if (info->avg_runtime >= + snapshot.avg_runtime * dyn_threaded_avg_runtime_keep_per / ONE_HUNDRED && + snapshot.status == LONG_RUNNING_WITH_INTERRUPTS && + snapshot.last_status != LONG_RUNNING_WITH_INTERRUPTS && + snapshot.nvcsw_per < dyn_threaded_nvcsw_per_limit) { + /* the status was reached in the previous sample, carry + * over the baseline statistics of the last change + */ + info->status = LONG_RUNNING_WITH_INTERRUPTS; + info->nvcsw_per = snapshot.nvcsw_per; + info->avg_runtime = snapshot.avg_runtime; + } else if (info->avg_runtime > dyn_threaded_avg_runtime_limit && + info->nvcsw_per <= dyn_threaded_nvcsw_per_limit) { + info->status = LONG_RUNNING_WITH_INTERRUPTS; + } else { + info->status = FRAGMENTATION; + } +} + +static int dynthreaded; + +static struct work_struct dyn_threaded_work; +static struct hrtimer dyn_threaded_timer; + +/* periodic work, judge the load of the target tasks */ +static void dyn_threaded_handler(struct work_struct *work) +{ + struct sched_task_runtime_avg_info info = { }; + + /* take a sample and judge the current status */ + sched_task_runtime_avg_snapshot(appname, &info); + judge_target_task_status(&info); + + + switch (info.status) { + case DATA_LOSS: + /* invalid sample, reuse the previous status and statistics */ + info.status = snapshot.status; + info.nvcsw_per = snapshot.nvcsw_per; + info.avg_runtime = snapshot.avg_runtime; + break; + default: + break; + } + + /* remember the status preceding the current one */ + if (info.status == snapshot.status) + info.last_status = snapshot.last_status; + else + info.last_status = snapshot.status; + + + snapshot = info; +} + +static enum hrtimer_restart dyn_threaded_hrtimer_callback(struct hrtimer *timer) +{ + if (!dynthreaded) + return HRTIMER_NORESTART; + + schedule_work(&dyn_threaded_work); + hrtimer_forward_now(timer, ms_to_ktime(dyn_threaded_internal)); + + return HRTIMER_RESTART; +} + +static void start_worker(void) +{ + ktime_t interval = ms_to_ktime(dyn_threaded_internal); + + /* take the first sample as the baseline */ + sched_task_runtime_avg_snapshot(appname, &snapshot); + + INIT_WORK(&dyn_threaded_work, dyn_threaded_handler); + + hrtimer_init(&dyn_threaded_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + dyn_threaded_timer.function = dyn_threaded_hrtimer_callback; + hrtimer_start(&dyn_threaded_timer, interval, HRTIMER_MODE_REL); + +} + +static void stop_worker(void) +{ + hrtimer_cancel(&dyn_threaded_timer); + cancel_work_sync(&dyn_threaded_work); + +} + +/* enable/disable the scheduler through the dynthreaded module parameter */ +static int dynthreaded_set(const char *val, const struct kernel_param *kp) +{ + int new_val, ret; + + ret = kstrtoint(val, 10, &new_val); + if (ret < 0) + return ret; + + new_val = !!new_val; + if (new_val == dynthreaded) + return 0; + + dynthreaded = new_val; + + if (new_val) + start_worker(); + else + stop_worker(); + + return 0; +} + +static int dynthreaded_get(char *buffer, const struct kernel_param *kp) +{ + return scnprintf(buffer, PAGE_SIZE, "%d\n", dynthreaded); +} + +static const struct kernel_param_ops dynthreaded_ops = { + .set = dynthreaded_set, + .get = dynthreaded_get, +}; + +module_param_cb(dynthreaded, &dynthreaded_ops, NULL, 0644); +MODULE_PARM_DESC(dynthreaded, "enable the scheduler, default 0"); + static int __init dynthread_init(void) { int ret; @@ -265,6 +603,9 @@ static int __init dynthread_init(void) static void __exit dynthread_exit(void) { + /* make sure the worker is stopped before releasing the devices */ + dynthreaded_set("0", NULL); + deinit_netdevinfo(); } -- 2.18.0.huawei.25