From: Amritha Nambiar <amritha.nambiar@intel.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10109 ---------------------------------------- Add support to associate the interrupt vector number for a NAPI instance. Signed-off-by: Amritha Nambiar <amritha.nambiar@intel.com> Reviewed-by: Sridhar Samudrala <sridhar.samudrala@intel.com> Link: https://lore.kernel.org/r/170147334728.5260.13221803396905901904.stgit@anamb... Signed-off-by: Jakub Kicinski <kuba@kernel.org> Signed-off-by: Cai Xinchen <caixinchen1@huawei.com> --- include/linux/netdevice.h | 7 ++++++- net/core/dev.c | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 43cb840f0f2b..03372828a685 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -394,7 +394,7 @@ struct napi_struct { struct list_head dev_list; struct hlist_node napi_hash_node; - KABI_RESERVE(1) + KABI_USE(1, int irq) KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) @@ -2686,6 +2686,11 @@ void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index, enum netdev_queue_type type, struct napi_struct *napi); +static inline void netif_napi_set_irq(struct napi_struct *napi, int irq) +{ + napi->irq = irq; +} + /* Default NAPI poll() weight * Device drivers are strongly advised to not use bigger value */ diff --git a/net/core/dev.c b/net/core/dev.c index a8b10001905c..9dd9761928d9 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -6763,6 +6763,7 @@ void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi, */ if (dev->threaded && napi_kthread_create(napi)) dev->threaded = 0; + netif_napi_set_irq(napi, -1); } EXPORT_SYMBOL(netif_napi_add_weight); -- 2.18.0.huawei.25

From: Carolina Jubran <cjubran@nvidia.com> hulk inclusion category: bugfix bugzilla: https://atomgit.com/openeuler/kernel/issues/10109 ---------------------------------------- netif_queue_set_napi asserts whether RTNL lock is held if the netdev is initialized. Acquire the RTNL lock before activating or deactivating RQs/SQs if the lock has not been held before in the flow. Fixes: f25e7b82635f ("net/mlx5e: link NAPI instances to queues and IRQs") Cc: Joe Damato <jdamato@fastly.com> Signed-off-by: Carolina Jubran <cjubran@nvidia.com> Reviewed-by: Rahul Rameshbabu <rrameshbabu@nvidia.com> Signed-off-by: Tariq Toukan <tariqt@nvidia.com> Link: https://lore.kernel.org/r/20240411115444.374475-6-tariqt@nvidia.com Signed-off-by: Jakub Kicinski <kuba@kernel.org> Signed-off-by: Cai Xinchen <caixinchen1@huawei.com> --- drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 1319e9ee20fc..9ad9e375315a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -113,7 +113,10 @@ static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) mlx5e_reset_txqsq_cc_pc(sq); sq->stats->recover++; clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); + rtnl_lock(); mlx5e_activate_txqsq(sq); + rtnl_unlock(); + if (sq->channel) mlx5e_trigger_napi_icosq(sq->channel); else @@ -186,12 +189,16 @@ static int mlx5e_tx_reporter_ptpsq_unhealthy_recover(void *ctx) carrier_ok = netif_carrier_ok(netdev); netif_carrier_off(netdev); + rtnl_lock(); mlx5e_deactivate_priv_channels(priv); + rtnl_unlock(); mlx5e_ptp_close(chs->ptp); err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp); + rtnl_lock(); mlx5e_activate_priv_channels(priv); + rtnl_unlock(); /* return carrier back if needed */ if (carrier_ok) -- 2.18.0.huawei.25

Add the monitoring part of the module and its runtime loop. Writing 1 to the dynthreaded module parameter starts a repeating hrtimer which schedules a work item every dyn_threaded_internal ms; writing 0 (or unloading the module) stops it again. The work item takes a sample of the scheduling statistics of all threads whose name matches the appname module parameter: their total runtime, the total number of context switches and their voluntary/involuntary split, as well as the union of the CPU masks allowed to those tasks. Each sample is compared with the previous one and judged as: DATA_LOSS counters moved backwards, the sample is dropped and the previous status is kept NO_TASK no thread with the target name exists NOT_RUNNING avg runtime below dyn_threaded_not_running_limit, the tasks hardly get the CPU LONG_RUNNING_WITH_INTERRUPTS avg runtime above dyn_threaded_avg_runtime_limit and a low voluntary switch ratio, i.e. the tasks are disturbed by softirq activity FRAGMENTATION short avg runtime per context switch, processing in softirq context is acceptable For the sample right after entering LONG_RUNNING_WITH_INTERRUPTS, the avg runtime and switch ratio of the last sample before the change are carried over, so that the status decision is not based on the transient sample. The subsequent patches act on this status. Co-developed-by: Yue Haibing <yuehaibing@huawei.com> Signed-off-by: Yue Haibing <yuehaibing@huawei.com> Assisted-by: opencode:glm-5.3 Signed-off-by: Cai Xinchen <caixinchen1@huawei.com> --- net/dynthread/net_dynthread.c | 347 +++++++++++++++++++++++++++++++++- 1 file changed, 344 insertions(+), 3 deletions(-) diff --git a/net/dynthread/net_dynthread.c b/net/dynthread/net_dynthread.c index 1aa7b35cfd94..c600428220f2 100644 --- a/net/dynthread/net_dynthread.c +++ b/net/dynthread/net_dynthread.c @@ -1,9 +1,11 @@ // SPDX-License-Identifier: GPL-2.0-only /* Dynamic threaded NAPI scheduler * - * Record the RX queue IRQ information of the net devices configured - * via the ifname module parameter, so that the IRQ placement can be - * reworked at runtime. + * Monitor the scheduling behaviour of a target application and switch + * the NAPI processing of the configured net devices between softirq + * context and kernel threads, whose CPU affinity is adapted to the + * measured load. The RX queue IRQ placement of the devices is + * adjusted as well. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -12,6 +14,13 @@ #include <linux/module.h> #include <linux/kernel.h> #include <linux/string.h> +#include <linux/rcupdate.h> +#include <linux/cpumask.h> +#include <linux/ktime.h> +#include <linux/hrtimer.h> +#include <linux/workqueue.h> +#include <linux/sched.h> +#include <linux/sched/signal.h> #include <linux/irq.h> #include <linux/netdevice.h> #include <linux/topology.h> @@ -19,8 +28,10 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Dynamic threaded NAPI scheduler"); +#define ONE_HUNDRED 100 #define SCHED_MAX_NETDEV_NUM 8 #define MAX_DEV_RXQ_NUM 256 +#define DYN_THREAD_INTERVAL (dyn_threaded_internal * NSEC_PER_MSEC) static char ifname[64]; module_param_string(ifname, ifname, sizeof(ifname), 0644); @@ -30,6 +41,66 @@ static char irqname[64] = "comp"; module_param_string(irqname, irqname, sizeof(irqname), 0644); MODULE_PARM_DESC(irqname, "substring of the NIC RX queue IRQ names, default comp"); +static char appname[64] = "mysqld"; +module_param_string(appname, appname, sizeof(appname), 0644); +MODULE_PARM_DESC(appname, "name of the target application threads, default mysqld"); + +static uint dyn_threaded_nvcsw_per_limit = 95; +module_param(dyn_threaded_nvcsw_per_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_nvcsw_per_limit, + "max nvcsw percent for long running status, default 95"); + +static uint dyn_threaded_avg_runtime_limit = 350000; +module_param(dyn_threaded_avg_runtime_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_avg_runtime_limit, + "min avg runtime (ns) for long running status, default 350000"); + +static uint dyn_threaded_not_running_limit = 30000; +module_param(dyn_threaded_not_running_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_not_running_limit, + "max avg runtime (ns) for not running status, default 30000"); + +static uint dyn_threaded_internal = 1000; + +static int dyn_threaded_internal_set(const char *val, + const struct kernel_param *kp) +{ + uint interval; + int ret; + + ret = kstrtouint(val, 0, &interval); + if (ret) + return ret; + + /* a zero interval would divide by zero and spin the timer */ + if (!interval) + return -EINVAL; + + *(uint *)kp->arg = interval; + + return 0; +} + +static int dyn_threaded_internal_get(char *buffer, + const struct kernel_param *kp) +{ + return scnprintf(buffer, PAGE_SIZE, "%u\n", *(uint *)kp->arg); +} + +static const struct kernel_param_ops dyn_threaded_internal_ops = { + .set = dyn_threaded_internal_set, + .get = dyn_threaded_internal_get, +}; + +module_param_cb(dyn_threaded_internal, &dyn_threaded_internal_ops, + &dyn_threaded_internal, 0644); +MODULE_PARM_DESC(dyn_threaded_internal, "sample interval in ms, default 1000"); + +static uint dyn_threaded_avg_runtime_keep_per = 90; +module_param(dyn_threaded_avg_runtime_keep_per, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_avg_runtime_keep_per, + "min percent of avg runtime kept in long running status, default 90"); + /** * struct sched_netdev_queue_info - IRQ information of one RX queue * @irq: Linux IRQ number of the queue @@ -56,6 +127,51 @@ struct sched_netdev { struct sched_netdev_queue_info rxq[MAX_DEV_RXQ_NUM]; }; +/** + * enum task_status - scheduling state of the target tasks + * @DATA_LOSS: the counters moved backwards, the sample is not usable + * @NO_TASK: no task with the target name exists + * @NOT_RUNNING: the tasks hardly get the CPU + * @FRAGMENTATION: the tasks are scheduled out quickly, softirq mode is fine + * @LONG_RUNNING_WITH_INTERRUPTS: the tasks run long between context + * switches and should not be disturbed by softirq activity + */ +enum task_status { + DATA_LOSS, + NO_TASK, + NOT_RUNNING, + FRAGMENTATION, + LONG_RUNNING_WITH_INTERRUPTS, +}; + +/** + * struct sched_task_runtime_avg_info - scheduling statistics of the target tasks + * @total_switches: total context switches since boot + * @total_runtime_ns: total runtime in ns since boot + * @total_nvcsw: total voluntary context switches since boot + * @total_nivcsw: total involuntary context switches since boot + * @avg_runtime: avg runtime of the sample in ns, per context switch + * @nvcsw_per: percent of voluntary context switches of the sample + * @cpuload: total load of the sample in percent, over all allowed CPUs + * @thread_cnt: number of matching threads + * @status: task status judged for this sample + * @last_status: task status of the previous sample + * @comm: name of the target tasks + */ +struct sched_task_runtime_avg_info { + u64 total_switches; + u64 total_runtime_ns; + u64 total_nvcsw; + u64 total_nivcsw; + u64 avg_runtime; + u64 nvcsw_per; + u64 cpuload; + u32 thread_cnt; + enum task_status status; + enum task_status last_status; + char comm[TASK_COMM_LEN]; +}; + static int sched_netdev_num; static struct sched_netdev sched_netdev_table[SCHED_MAX_NETDEV_NUM]; @@ -251,6 +367,228 @@ static int init_netdevinfo(const char *netdev_str) return err; } +/* status of the last sample of the target tasks */ +static struct sched_task_runtime_avg_info snapshot; + +/* union of the CPU masks allowed to the target tasks */ +static struct cpumask target_cpumask; +static u32 target_cpucnt; + +/* Take a sample of the scheduling statistics of all threads whose name + * matches @target_comm, and compute the union of their allowed CPU + * masks. The info must be zeroed by the caller. + */ +static int sched_task_runtime_avg_snapshot(const char *target_comm, + struct sched_task_runtime_avg_info *infos) +{ + struct task_struct *g, *p; + char comm[TASK_COMM_LEN]; + u64 nvcsw, nivcsw, switches; + struct cpumask masks; + + if (!target_comm || !infos) + return -EINVAL; + + cpumask_clear(&masks); + strscpy(infos->comm, target_comm, sizeof(infos->comm)); + + rcu_read_lock(); + for_each_process(g) { + get_task_comm(comm, g); + if (strcmp(comm, infos->comm)) + continue; + + cpumask_or(&masks, &masks, g->cpus_ptr); + + for_each_thread(g, p) { + nvcsw = (u64)READ_ONCE(p->nvcsw); + nivcsw = READ_ONCE(p->nivcsw); + switches = nvcsw + nivcsw; + + infos->total_runtime_ns += READ_ONCE(p->se.sum_exec_runtime); + infos->total_switches += switches; + infos->total_nvcsw += nvcsw; + infos->total_nivcsw += nivcsw; + infos->thread_cnt++; + } + } + rcu_read_unlock(); + + cpumask_copy(&target_cpumask, &masks); + WRITE_ONCE(target_cpucnt, cpumask_weight(&target_cpumask)); + + + return 0; +} + +/* load of the target tasks over the CPUs allowed to them, in percent */ +static inline u64 cal_target_task_cpuload(struct sched_task_runtime_avg_info *infos) +{ + return 100 * (infos->total_runtime_ns - snapshot.total_runtime_ns) / + DYN_THREAD_INTERVAL / READ_ONCE(target_cpucnt); +} + +/* Compare the current sample with the previous one and judge the + * status of the target tasks. A sample whose counters moved backwards + * yields DATA_LOSS and is evaluated by the caller only. + */ +static void judge_target_task_status(struct sched_task_runtime_avg_info *info) +{ + u64 switches; + + if (info->total_switches < snapshot.total_switches || + info->total_runtime_ns < snapshot.total_runtime_ns || + info->total_nvcsw < snapshot.total_nvcsw || + info->total_nivcsw < snapshot.total_nivcsw || + strcmp(info->comm, snapshot.comm)) { + info->status = DATA_LOSS; + return; + } + + if (info->thread_cnt == 0) { + info->status = NO_TASK; + return; + } + + /* a sample may contain no context switch, in which case the + * runtime delta is the runtime of the whole sample; clamp the + * switch count to avoid a division by zero + */ + switches = info->total_switches - snapshot.total_switches; + switches = max_t(u64, switches, 1); + + info->avg_runtime = (info->total_runtime_ns - snapshot.total_runtime_ns) / + switches; + info->nvcsw_per = (info->total_nvcsw - snapshot.total_nvcsw) * 100 / + switches; + info->cpuload = cal_target_task_cpuload(info); + + if (info->avg_runtime < dyn_threaded_not_running_limit) { + info->status = NOT_RUNNING; + } else if (info->avg_runtime >= + snapshot.avg_runtime * dyn_threaded_avg_runtime_keep_per / ONE_HUNDRED && + snapshot.status == LONG_RUNNING_WITH_INTERRUPTS && + snapshot.last_status != LONG_RUNNING_WITH_INTERRUPTS && + snapshot.nvcsw_per < dyn_threaded_nvcsw_per_limit) { + /* the status was reached in the previous sample, carry + * over the baseline statistics of the last change + */ + info->status = LONG_RUNNING_WITH_INTERRUPTS; + info->nvcsw_per = snapshot.nvcsw_per; + info->avg_runtime = snapshot.avg_runtime; + } else if (info->avg_runtime > dyn_threaded_avg_runtime_limit && + info->nvcsw_per <= dyn_threaded_nvcsw_per_limit) { + info->status = LONG_RUNNING_WITH_INTERRUPTS; + } else { + info->status = FRAGMENTATION; + } +} + +static int dynthreaded; + +static struct work_struct dyn_threaded_work; +static struct hrtimer dyn_threaded_timer; + +/* periodic work, judge the load of the target tasks */ +static void dyn_threaded_handler(struct work_struct *work) +{ + struct sched_task_runtime_avg_info info = { }; + + /* take a sample and judge the current status */ + sched_task_runtime_avg_snapshot(appname, &info); + judge_target_task_status(&info); + + + switch (info.status) { + case DATA_LOSS: + /* invalid sample, reuse the previous status and statistics */ + info.status = snapshot.status; + info.nvcsw_per = snapshot.nvcsw_per; + info.avg_runtime = snapshot.avg_runtime; + break; + default: + break; + } + + /* remember the status preceding the current one */ + if (info.status == snapshot.status) + info.last_status = snapshot.last_status; + else + info.last_status = snapshot.status; + + + snapshot = info; +} + +static enum hrtimer_restart dyn_threaded_hrtimer_callback(struct hrtimer *timer) +{ + if (!dynthreaded) + return HRTIMER_NORESTART; + + schedule_work(&dyn_threaded_work); + hrtimer_forward_now(timer, ms_to_ktime(dyn_threaded_internal)); + + return HRTIMER_RESTART; +} + +static void start_worker(void) +{ + ktime_t interval = ms_to_ktime(dyn_threaded_internal); + + /* take the first sample as the baseline */ + sched_task_runtime_avg_snapshot(appname, &snapshot); + + INIT_WORK(&dyn_threaded_work, dyn_threaded_handler); + + hrtimer_init(&dyn_threaded_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + dyn_threaded_timer.function = dyn_threaded_hrtimer_callback; + hrtimer_start(&dyn_threaded_timer, interval, HRTIMER_MODE_REL); + +} + +static void stop_worker(void) +{ + hrtimer_cancel(&dyn_threaded_timer); + cancel_work_sync(&dyn_threaded_work); + +} + +/* enable/disable the scheduler through the dynthreaded module parameter */ +static int dynthreaded_set(const char *val, const struct kernel_param *kp) +{ + int new_val, ret; + + ret = kstrtoint(val, 10, &new_val); + if (ret < 0) + return ret; + + new_val = !!new_val; + if (new_val == dynthreaded) + return 0; + + dynthreaded = new_val; + + if (new_val) + start_worker(); + else + stop_worker(); + + return 0; +} + +static int dynthreaded_get(char *buffer, const struct kernel_param *kp) +{ + return scnprintf(buffer, PAGE_SIZE, "%d\n", dynthreaded); +} + +static const struct kernel_param_ops dynthreaded_ops = { + .set = dynthreaded_set, + .get = dynthreaded_get, +}; + +module_param_cb(dynthreaded, &dynthreaded_ops, NULL, 0644); +MODULE_PARM_DESC(dynthreaded, "enable the scheduler, default 0"); + static int __init dynthread_init(void) { int ret; @@ -265,6 +603,9 @@ static int __init dynthread_init(void) static void __exit dynthread_exit(void) { + /* make sure the worker is stopped before releasing the devices */ + dynthreaded_set("0", NULL); + deinit_netdevinfo(); } -- 2.18.0.huawei.25

hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10109 ---------------------------------------- Add the actual NAPI part of the module. In LONG_RUNNING_WITH_INTERRUPTS status the handler enables threaded NAPI (dev_set_threaded) on every monitored net device and binds the NAPI threads with bind_napi_threaded(), deriving the CPU mask of each thread from the NUMA node of its queue IRQ: - none of the target task CPUs is in the node: the whole node mask is used - the node has CPUs the target tasks are not allowed to use: those CPUs are used - the target tasks may use every CPU of the node: the first dyn_threaded_corenum CPUs of the node are used, so that the network load is capped instead of creeping into the CPUs of the target application; a thread already bound to such a subset keeps its current mask With dyn_threaded_bindirq set, the queue IRQ is bound along with its NAPI thread, round-robin over the same mask. The CPUs used by the NAPI threads are recorded in napi_cpumask, to evaluate their load in the next patch. In NOT_RUNNING and FRAGMENTATION status the handler switches NAPI back to softirq mode, as the target tasks currently do not suffer from softirq activity. Co-developed-by: Yue Haibing <yuehaibing@huawei.com> Signed-off-by: Yue Haibing <yuehaibing@huawei.com> Assisted-by: opencode:glm-5.3 Signed-off-by: Cai Xinchen <caixinchen1@huawei.com> --- net/dynthread/net_dynthread.c | 192 +++++++++++++++++++++++++++++++++- 1 file changed, 188 insertions(+), 4 deletions(-) diff --git a/net/dynthread/net_dynthread.c b/net/dynthread/net_dynthread.c index 719ca0d563ed..04292fba5b6e 100644 --- a/net/dynthread/net_dynthread.c +++ b/net/dynthread/net_dynthread.c @@ -25,6 +25,7 @@ #include <linux/topology.h> #include <linux/irq.h> #include <linux/netdevice.h> +#include <linux/rtnetlink.h> MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Dynamic threaded NAPI scheduler"); @@ -116,6 +117,54 @@ module_param(dyn_threaded_irq_cpucore_per, uint, 0644); MODULE_PARM_DESC(dyn_threaded_irq_cpucore_per, "percent of rx queues to keep per NUMA node, default 50"); +/* default number of cpus used by the napi threads, computed at init */ +static u32 dyn_threaded_corenum_default; + +static u32 dyn_threaded_corenum; + +static int dyn_threaded_corenum_set(const char *val, + const struct kernel_param *kp) +{ + u32 num; + int ret; + + ret = kstrtou32(val, 0, &num); + if (ret) + return ret; + /* a zero count would leave the napi threads without cpu */ + if (!num) + return -EINVAL; + + *(u32 *)kp->arg = num; + return 0; +} + +static int dyn_threaded_corenum_get(char *buffer, + const struct kernel_param *kp) +{ + return scnprintf(buffer, PAGE_SIZE, "%u\n", *(u32 *)kp->arg); +} + +static const struct kernel_param_ops dyn_threaded_corenum_ops = { + .set = dyn_threaded_corenum_set, + .get = dyn_threaded_corenum_get, +}; + +module_param_cb(dyn_threaded_corenum, &dyn_threaded_corenum_ops, + &dyn_threaded_corenum, 0644); +MODULE_PARM_DESC(dyn_threaded_corenum, + "number of cpus used by the napi threads, computed by default"); + +static u32 dyn_threaded_corenum_lower_limit = 4; +module_param(dyn_threaded_corenum_lower_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_corenum_lower_limit, + "lower limit of napi thread cpus, default 4"); + +static u32 dyn_threaded_corenum_upper_limit = 32; +module_param(dyn_threaded_corenum_upper_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_corenum_upper_limit, + "upper limit of napi thread cpus, default 32"); + /** * struct sched_netdev_queue_info - IRQ information of one RX queue * @irq: Linux IRQ number of the queue @@ -664,6 +713,103 @@ static void recover_irq_affinity(void) affinity_to_one_node = false; } +/* CPUs the napi threads of the monitored devices may run on */ +static struct cpumask napi_cpumask; + +/* bind_napi_threaded - bind the threaded NAPIs of @dev to CPUs + * + * The mask of each NAPI thread is derived from the NUMA node of its + * queue IRQ: + * + * - if none of the target task CPUs is in that node, the whole node + * mask is used + * - otherwise, if the node has CPUs the target tasks are not allowed + * to use, those CPUs are used + * - otherwise the first dyn_threaded_corenum CPUs of the node are + * used; if the thread is already bound to such a subset, its + * current mask is kept + * + * In the latter case, with dyn_threaded_bindirq set, the queue IRQ is + * bound along with its NAPI thread, round-robin over the same mask. + * The CPUs used by the threads are recorded in napi_cpumask for the + * load evaluation. + */ +static void bind_napi_threaded(struct net_device *dev) +{ + struct napi_struct *napi; + struct cpumask bind_mask, check_mask, napi_allowed_mask; + const struct cpumask *irq_mask, *node_mask; + u32 idx = 0; + + list_for_each_entry(napi, &dev->napi_list, dev_list) { + if (!napi->thread) + continue; + + cpumask_clear(&bind_mask); + + /* the node of the irq, cpu_online_mask if unknown */ + irq_mask = irq_get_effective_affinity_mask(napi->irq); + if (unlikely(!irq_mask) || cpumask_empty(irq_mask)) + node_mask = cpu_online_mask; + else + node_mask = cpu_cpu_mask(cpumask_first(irq_mask)); + + /* no target task cpu in this node: use the whole node */ + cpumask_and(&check_mask, node_mask, &target_cpumask); + if (cpumask_empty(&check_mask)) { + set_cpus_allowed_ptr(napi->thread, node_mask); + continue; + } + + /* preferred: the node's cpus without the target tasks' ones */ + cpumask_andnot(&napi_allowed_mask, node_mask, &target_cpumask); + if (!cpumask_empty(&napi_allowed_mask)) { + set_cpus_allowed_ptr(napi->thread, &napi_allowed_mask); + continue; + } + + /* no spare cpu in the node: fall back to the whole node, */ + cpumask_copy(&napi_allowed_mask, node_mask); + + /* already bound to a subset of the right size? keep it */ + if (cpumask_weight(napi->thread->cpus_ptr) == dyn_threaded_corenum && + cpumask_subset(napi->thread->cpus_ptr, &napi_allowed_mask)) { + cpumask_or(&napi_cpumask, &napi_cpumask, napi->thread->cpus_ptr); + continue; + } + + select_napi_cpus(dyn_threaded_corenum, &napi_allowed_mask, &bind_mask); + if (cpumask_empty(&bind_mask)) + continue; + + cpumask_or(&napi_cpumask, &napi_cpumask, &bind_mask); + set_cpus_allowed_ptr(napi->thread, &bind_mask); + if (dyn_threaded_bindirq) + bind_irq_in_cpumask(napi->irq, idx++, &bind_mask); + } +} + +static void set_all_napi_threaded(bool threaded) +{ + struct sched_netdev *sched_dev; + int id; + + /* napi_list, dev_set_threaded() and set_cpus_allowed_ptr() + * are all handled under the sleepable rtnl lock + */ + rtnl_lock(); + for (id = 0; id < sched_netdev_num; id++) { + sched_dev = get_sched_netdev(id); + if (!sched_dev) + continue; + + dev_set_threaded(sched_dev->netdev, threaded); + if (threaded) + bind_napi_threaded(sched_dev->netdev); + } + rtnl_unlock(); +} + static int dynthreaded; static struct work_struct dyn_threaded_work; @@ -696,15 +842,23 @@ static void dyn_threaded_handler(struct work_struct *work) info.avg_runtime = snapshot.avg_runtime; break; case NOT_RUNNING: - /* target tasks barely running: restore the original irqs */ + /* target tasks barely running: softirq mode is fine */ + set_all_napi_threaded(false); + /* and the original irq affinity can be restored */ recover_irq_affinity(); break; case FRAGMENTATION: - /* irqs may consume cpu time again, spread them over - * their NUMA node - */ + /* softirq mode is fine, irqs spread over their NUMA node */ + set_all_napi_threaded(false); bind_irq(); break; + case LONG_RUNNING_WITH_INTERRUPTS: + /* switch NAPI processing to threads bound away from + * the target task cpus + */ + cpumask_clear(&napi_cpumask); + set_all_napi_threaded(true); + break; default: break; } @@ -788,6 +942,33 @@ static const struct kernel_param_ops dynthreaded_ops = { module_param_cb(dynthreaded, &dynthreaded_ops, NULL, 0644); MODULE_PARM_DESC(dynthreaded, "enable the scheduler, default 0"); +/* set_default_dyn_threaded_corenum - compute the default napi thread cpu count + * + * The default is the number of RX queues of the busiest monitored device + * divided by the number of NUMA nodes, clamped between the core number + * lower and upper limits. + */ +static void set_default_dyn_threaded_corenum(void) +{ + struct sched_netdev *sched_dev; + unsigned int max_rxq = 0; + u32 num; + int id; + + for (id = 0; id < sched_netdev_num; id++) { + sched_dev = get_sched_netdev(id); + if (sched_dev) + max_rxq = max(max_rxq, sched_dev->netdev->real_num_rx_queues); + } + + num = max_rxq / num_online_nodes(); + if (num < dyn_threaded_corenum_lower_limit) + num = dyn_threaded_corenum_lower_limit; + if (num > dyn_threaded_corenum_upper_limit) + num = dyn_threaded_corenum_upper_limit; + dyn_threaded_corenum_default = num; +} + static int __init dynthread_init(void) { int ret; @@ -796,6 +977,9 @@ static int __init dynthread_init(void) if (ret) return ret; + set_default_dyn_threaded_corenum(); + if (!dyn_threaded_corenum) + dyn_threaded_corenum = dyn_threaded_corenum_default; return 0; } -- 2.18.0.huawei.25

hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10109 ---------------------------------------- Add a dyn_threaded_debug module parameter that enables debug instrumentation of the scheduler. The messages are recorded with trace_printk(), so that the periodic sampling, the status transitions and the decisions on the IRQ affinity, the NAPI thread masks and the napi thread CPU count can be followed from tracefs without flooding the console; without tracing support the messages fall back to dynamic debug. The switch is off by default. This replaces the previous pr_debug() instrumentation and adds the per-branch decisions of bind_napi_threaded(), the IRQ binding results, the napi thread CPU resize and the status transitions, matching the instrumentation that was used during the development of the module. Assisted-by: opencode:deepseek-v4.1-flash Signed-off-by: Cai Xinchen <caixinchen1@huawei.com> --- net/dynthread/net_dynthread.c | 88 +++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/net/dynthread/net_dynthread.c b/net/dynthread/net_dynthread.c index b50f09fcc281..43eda16818b1 100644 --- a/net/dynthread/net_dynthread.c +++ b/net/dynthread/net_dynthread.c @@ -182,6 +182,30 @@ module_param(dyn_threaded_cpuload_upper_limit, uint, 0644); MODULE_PARM_DESC(dyn_threaded_cpuload_upper_limit, "napi cpu load percent to grow for, default 90"); +static bool dyn_threaded_debug; +module_param(dyn_threaded_debug, bool, 0644); +MODULE_PARM_DESC(dyn_threaded_debug, + "enable the ftrace based debug instrumentation, default 0"); + +/* Debug instrumentation of the scheduler. It is controlled by the + * dyn_threaded_debug module parameter and recorded in the ftrace buffer, + * so that it can be read from tracefs without flooding the console. + * Without tracing support it falls back to dynamic debug. + */ +#ifdef CONFIG_TRACING +#define dyn_threaded_dbg(fmt, ...) \ + do { \ + if (dyn_threaded_debug) \ + trace_printk(fmt, ##__VA_ARGS__); \ + } while (0) +#else +#define dyn_threaded_dbg(fmt, ...) \ + do { \ + if (dyn_threaded_debug) \ + pr_debug(fmt, ##__VA_ARGS__); \ + } while (0) +#endif + /** * struct sched_netdev_queue_info - IRQ information of one RX queue * @irq: Linux IRQ number of the queue @@ -338,6 +362,9 @@ static void get_netdev_queue_info(struct sched_netdev *sched_dev) rxq_info = &sched_dev->rxq[sched_dev->rxq_num++]; rxq_info->irq = irq; rxq_info->affinity_cpu = cpu; + dyn_threaded_dbg("dev %s: irq=%d [%s] rxq_id=%d affinity_cpu=%d\n", + sched_dev->dev_name, irq, name, + sched_dev->rxq_num - 1, cpu); } } @@ -381,22 +408,27 @@ static int init_dev(const char *if_name, size_t length) strscpy(dev_name, if_name, min_t(size_t, length + 1, IFNAMSIZ)); netdev = dev_get_by_name(&init_net, dev_name); - if (!netdev) + if (!netdev) { + dyn_threaded_dbg("dev [%s] does not exist\n", dev_name); return -ENODEV; + } if (!(netdev->flags & IFF_UP)) { ret = -ENETDOWN; + dyn_threaded_dbg("dev [%s] is not up, flags=%u\n", dev_name, netdev->flags); goto err_dev_put; } if (netdev->flags & IFF_LOOPBACK) { ret = -EOPNOTSUPP; + dyn_threaded_dbg("loopback devices are not supported\n"); goto err_dev_put; } sched_dev = alloc_sched_netdev(); if (!sched_dev) { ret = -ENOMEM; + dyn_threaded_dbg("too many devices\n"); goto err_dev_put; } @@ -498,6 +530,10 @@ static int sched_task_runtime_avg_snapshot(const char *target_comm, cpumask_copy(&target_cpumask, &masks); WRITE_ONCE(target_cpucnt, cpumask_weight(&target_cpumask)); + dyn_threaded_dbg("target tasks '%s': cpumask %*pb (%u cpus), runtime %llu ns, switches %llu, threads %u\n", + infos->comm, cpumask_pr_args(&masks), + READ_ONCE(target_cpucnt), infos->total_runtime_ns, + infos->total_switches, infos->thread_cnt); return 0; } @@ -594,13 +630,19 @@ static void bind_irq_in_cpumask(int irq, int index, const struct cpumask *bind_m unsigned int len = cpumask_weight(bind_mask); int cpu; - if (!len) + if (!len) { + dyn_threaded_dbg("empty bind mask for irq %d\n", irq); return; + } cpu = cpumask_nth(index % len, bind_mask); - if (cpu >= nr_cpu_ids) + if (cpu >= nr_cpu_ids) { + dyn_threaded_dbg("no cpu for index %u of irq %d\n", index % len, irq); return; + } + dyn_threaded_dbg("bind irq %d to cpu %d of %*pb\n", + irq, cpu, cpumask_pr_args(bind_mask)); irq_set_affinity(irq, cpumask_of(cpu)); } @@ -688,17 +730,22 @@ static void set_irq_node_affinity(const struct cpumask *tasks_cpumask) if (cpumask_weight(&irq_prefer_mask) < dyn_threaded_irq_cpucore_limit) return; + dyn_threaded_dbg("irq affinity mask: %*pb\n", cpumask_pr_args(&irq_prefer_mask)); cpu = cpumask_first(&irq_prefer_mask); for (dev_id = 0; dev_id < sched_netdev_num; dev_id++) { sched_dev = get_sched_netdev(dev_id); for (rxq_id = 0; rxq_id < sched_dev->rxq_num; rxq_id++) { + dyn_threaded_dbg("set irq %d affinity to cpu %d\n", + sched_dev->rxq[rxq_id].irq, cpu); if (!irq_set_affinity(sched_dev->rxq[rxq_id].irq, cpumask_of(cpu))) { cpu = cpumask_next(cpu, &irq_prefer_mask); if (cpu >= nr_cpu_ids) cpu = cpumask_first(&irq_prefer_mask); } else { + dyn_threaded_dbg("failed to set irq %d affinity to cpu %d\n", + sched_dev->rxq[rxq_id].irq, cpu); } } } @@ -775,6 +822,8 @@ static void bind_napi_threaded(struct net_device *dev) cpumask_and(&check_mask, node_mask, &target_cpumask); if (cpumask_empty(&check_mask)) { set_cpus_allowed_ptr(napi->thread, node_mask); + dyn_threaded_dbg("%s: napi %d, no target cpu in node, use %*pb\n", + dev->name, napi->napi_id, cpumask_pr_args(node_mask)); continue; } @@ -782,6 +831,9 @@ static void bind_napi_threaded(struct net_device *dev) cpumask_andnot(&napi_allowed_mask, node_mask, &target_cpumask); if (!cpumask_empty(&napi_allowed_mask)) { set_cpus_allowed_ptr(napi->thread, &napi_allowed_mask); + dyn_threaded_dbg("%s: napi %d, use target-free cpus %*pb\n", + dev->name, napi->napi_id, + cpumask_pr_args(&napi_allowed_mask)); continue; } @@ -792,6 +844,9 @@ static void bind_napi_threaded(struct net_device *dev) if (cpumask_weight(napi->thread->cpus_ptr) == dyn_threaded_corenum && cpumask_subset(napi->thread->cpus_ptr, &napi_allowed_mask)) { cpumask_or(&napi_cpumask, &napi_cpumask, napi->thread->cpus_ptr); + dyn_threaded_dbg("%s: napi %d, keep mask %*pb\n", + dev->name, napi->napi_id, + cpumask_pr_args(napi->thread->cpus_ptr)); continue; } @@ -801,6 +856,9 @@ static void bind_napi_threaded(struct net_device *dev) cpumask_or(&napi_cpumask, &napi_cpumask, &bind_mask); set_cpus_allowed_ptr(napi->thread, &bind_mask); + dyn_threaded_dbg("%s: napi %d, use first %u cpus %*pb\n", + dev->name, napi->napi_id, dyn_threaded_corenum, + cpumask_pr_args(&bind_mask)); if (dyn_threaded_bindirq) bind_irq_in_cpumask(napi->irq, idx++, &bind_mask); } @@ -821,6 +879,8 @@ static void set_all_napi_threaded(bool threaded) continue; dev_set_threaded(sched_dev->netdev, threaded); + dyn_threaded_dbg("%s: set threaded napi %d\n", + sched_dev->dev_name, threaded); if (threaded) bind_napi_threaded(sched_dev->netdev); } @@ -935,6 +995,10 @@ static void change_dyn_threaded_corenum(void) /* percentual load of the napi cpus */ cal_cpustat_per(&napi_cpustat, &snapshot_napi_cpustat, &napi_per); + dyn_threaded_dbg("napi cpu user %llu system %llu softirq %llu irq %llu idle %llu iowait %llu\n", + napi_per.cpustat[CPUTIME_USER], napi_per.cpustat[CPUTIME_SYSTEM], + napi_per.cpustat[CPUTIME_SOFTIRQ], napi_per.cpustat[CPUTIME_IRQ], + napi_per.cpustat[CPUTIME_IDLE], napi_per.cpustat[CPUTIME_IOWAIT]); if (napi_per.cpustat[CPUTIME_SOFTIRQ] + napi_per.cpustat[CPUTIME_IRQ] >= IRQ_LIMIT && dyn_threaded_corenum < dyn_threaded_corenum_upper_limit) { @@ -964,11 +1028,15 @@ static void change_dyn_threaded_corenum(void) snapshot_napi_per = napi_per; last_threaded_corenum = dyn_threaded_corenum; dyn_threaded_corenum += action; + dyn_threaded_dbg("resize napi thread cpus %u -> %u\n", + last_threaded_corenum, dyn_threaded_corenum); } else { dynamic_corenum_keep = true; } out: + dyn_threaded_dbg("napi threads use %u cpus of %*pb\n", + dyn_threaded_corenum, cpumask_pr_args(&napi_cpumask)); snapshot_napi_cpustat = napi_cpustat; } @@ -1000,6 +1068,7 @@ static void dyn_threaded_handler(struct work_struct *work) dyn_threaded_bindirq) set_irq_node_affinity(&target_cpumask); + dyn_threaded_dbg("target task status %d\n", info.status); switch (info.status) { case NO_TASK: @@ -1038,6 +1107,7 @@ static void dyn_threaded_handler(struct work_struct *work) set_all_napi_threaded(true); break; default: + dyn_threaded_dbg("unexpected status %d\n", info.status); break; } @@ -1047,6 +1117,13 @@ static void dyn_threaded_handler(struct work_struct *work) else info.last_status = snapshot.status; + if (info.status != snapshot.status) + dyn_threaded_dbg("status %d -> %d\n", + snapshot.status, info.status); + + dyn_threaded_dbg("%s: avg_runtime %llu nvcsw_per %llu cpuload %llu status %d last %u irq_node %d\n", + __func__, info.avg_runtime, info.nvcsw_per, info.cpuload, + info.status, info.last_status, affinity_to_one_node); snapshot = info; } @@ -1075,6 +1152,7 @@ static void start_worker(void) dyn_threaded_timer.function = dyn_threaded_hrtimer_callback; hrtimer_start(&dyn_threaded_timer, interval, HRTIMER_MODE_REL); + dyn_threaded_dbg("worker started, interval %u ms\n", dyn_threaded_internal); } static void stop_worker(void) @@ -1082,6 +1160,7 @@ static void stop_worker(void) hrtimer_cancel(&dyn_threaded_timer); cancel_work_sync(&dyn_threaded_work); + dyn_threaded_dbg("worker stopped\n"); } /* enable/disable the scheduler through the dynthreaded module parameter */ @@ -1160,6 +1239,8 @@ static int __init dynthread_init(void) dyn_threaded_corenum = dyn_threaded_corenum_default; last_threaded_corenum = dyn_threaded_corenum; + dyn_threaded_dbg("module loaded\n"); + return 0; } @@ -1170,6 +1251,7 @@ static void __exit dynthread_exit(void) deinit_netdevinfo(); + dyn_threaded_dbg("module unloaded\n"); } module_init(dynthread_init); -- 2.18.0.huawei.25