While the target tasks stay in LONG_RUNNING_WITH_INTERRUPTS status, evaluate the load of the CPUs the NAPI threads were bound to (napi_cpumask) and adapt dyn_threaded_corenum over time: - the irq and softirq share of those CPUs reaches IRQ_LIMIT percent: grow the count by one - their user share reaches dyn_threaded_user_per_limit percent or their total load drops below dyn_threaded_cpuload_lower_limit percent: shrink the count by one - their total load reaches dyn_threaded_cpuload_upper_limit percent: grow the count by one The count stays within dyn_threaded_corenum_lower_limit and dyn_threaded_corenum_upper_limit, and it is not shrunk right over a count grown out of the previous one (and vice versa), so that both directions do not fight at the boundary. Fluctuations are damped additionally: an action right after a no-action period is deferred for one period, and after a change the evaluation is skipped for one period. The load is computed from the per-CPU kernel_cpustat of the napi cpus, summed by avg_cpustat() and normalized by cal_cpustat_per(). When the target tasks leave the threaded status again, the scaling state (baseline cpustat and last count) is reset by clear_corenum_napi(). Co-developed-by: Yue Haibing <yuehaibing@huawei.com> Signed-off-by: Yue Haibing <yuehaibing@huawei.com> Assisted-by: opencode:glm-5.3 Signed-off-by: Cai Xinchen <caixinchen1@huawei.com> --- net/dynthread/net_dynthread.c | 179 ++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/net/dynthread/net_dynthread.c b/net/dynthread/net_dynthread.c index 04292fba5b6e..b50f09fcc281 100644 --- a/net/dynthread/net_dynthread.c +++ b/net/dynthread/net_dynthread.c @@ -26,11 +26,13 @@ #include <linux/irq.h> #include <linux/netdevice.h> #include <linux/rtnetlink.h> +#include <linux/kernel_stat.h> MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Dynamic threaded NAPI scheduler"); #define ONE_HUNDRED 100 +#define IRQ_LIMIT 15 #define SCHED_MAX_NETDEV_NUM 8 #define MAX_DEV_RXQ_NUM 256 #define DYN_THREAD_INTERVAL (dyn_threaded_internal * NSEC_PER_MSEC) @@ -165,6 +167,21 @@ module_param(dyn_threaded_corenum_upper_limit, uint, 0644); MODULE_PARM_DESC(dyn_threaded_corenum_upper_limit, "upper limit of napi thread cpus, default 32"); +static u32 dyn_threaded_user_per_limit = 20; +module_param(dyn_threaded_user_per_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_user_per_limit, + "user percent of the napi cpus to shrink for, default 20"); + +static u32 dyn_threaded_cpuload_lower_limit = 70; +module_param(dyn_threaded_cpuload_lower_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_cpuload_lower_limit, + "napi cpu load percent to shrink for, default 70"); + +static u32 dyn_threaded_cpuload_upper_limit = 90; +module_param(dyn_threaded_cpuload_upper_limit, uint, 0644); +MODULE_PARM_DESC(dyn_threaded_cpuload_upper_limit, + "napi cpu load percent to grow for, default 90"); + /** * struct sched_netdev_queue_info - IRQ information of one RX queue * @irq: Linux IRQ number of the queue @@ -810,6 +827,160 @@ static void set_all_napi_threaded(bool threaded) rtnl_unlock(); } +/* percentual load of the napi cpus before the last change */ +static struct kernel_cpustat snapshot_napi_per; + +/* raw load of the napi cpus of the previous period */ +static struct kernel_cpustat snapshot_napi_cpustat; + +/* dyn_threaded_corenum of the last change */ +static u32 last_threaded_corenum; + +/* damping state of change_dyn_threaded_corenum: true right after + * the count changed, true if the previous period took no action + */ +static bool dynamic_corenum_changed = true; +static bool dynamic_corenum_keep; + +/* avg_cpustat - compute the per-CPU average of kernel_cpustat over + * the online CPUs of @mask + * @mask: desired CPUs + * @avg: result, zeroed if @mask contains no online CPU + */ +static void avg_cpustat(const struct cpumask *mask, struct kernel_cpustat *avg) +{ + struct kernel_cpustat sum = { { 0 } }; + int cpu, count = 0; + int i; + + for_each_cpu_and(cpu, mask, cpu_online_mask) { + struct kernel_cpustat *stat = &per_cpu(kernel_cpustat, cpu); + + for (i = 0; i < NR_STATS; i++) + sum.cpustat[i] += stat->cpustat[i]; + count++; + } + + if (!count) { + memset(avg, 0, sizeof(*avg)); + return; + } + + for (i = 0; i < NR_STATS; i++) + avg->cpustat[i] = sum.cpustat[i] / count; +} + +/* cal_cpustat_per - the increase of each kernel_cpustat field between + * @start and @end, in percent of the total increase + */ +static void cal_cpustat_per(const struct kernel_cpustat *end, + const struct kernel_cpustat *start, + struct kernel_cpustat *per) +{ + struct kernel_cpustat delta; + u64 total; + int i; + + for (i = 0; i < NR_STATS; i++) { + if (end->cpustat[i] > start->cpustat[i]) + delta.cpustat[i] = end->cpustat[i] - start->cpustat[i]; + else + delta.cpustat[i] = 0; + } + + total = delta.cpustat[CPUTIME_USER] + delta.cpustat[CPUTIME_NICE] + + delta.cpustat[CPUTIME_SYSTEM] + delta.cpustat[CPUTIME_IDLE] + + delta.cpustat[CPUTIME_IOWAIT] + delta.cpustat[CPUTIME_IRQ] + + delta.cpustat[CPUTIME_SOFTIRQ] + delta.cpustat[CPUTIME_STEAL]; + + /* no accounting delta, e.g. no online cpu in the mask: everything + * is zero, avoid the division by zero + */ + if (!total) { + memset(per, 0, sizeof(*per)); + return; + } + + for (i = 0; i < NR_STATS; i++) + per->cpustat[i] = 100 * delta.cpustat[i] / total; +} + +/* change_dyn_threaded_corenum - resize dyn_threaded_corenum + * + * The load of the napi cpus is evaluated from the deltas of their + * kernel_cpustat. The core count grows when the irq and softirq share + * of the napi cpus reaches IRQ_LIMIT percent or when their total load + * reaches dyn_threaded_cpuload_upper_limit percent; it shrinks when + * their user share reaches dyn_threaded_user_per_limit percent or + * their total load drops below dyn_threaded_cpuload_lower_limit + * percent, but not right over a change grown out of the target count. + * + * Fluctuations are damped: an action right after a no-action period + * is deferred for one period, and after a change the evaluation is + * skipped once. + */ +static void change_dyn_threaded_corenum(void) +{ + struct kernel_cpustat napi_per; + struct kernel_cpustat napi_cpustat; + int action = 0; + + avg_cpustat(&napi_cpumask, &napi_cpustat); + + if (dynamic_corenum_changed) { + dynamic_corenum_changed = false; + goto out; + } + + /* percentual load of the napi cpus */ + cal_cpustat_per(&napi_cpustat, &snapshot_napi_cpustat, &napi_per); + + + if (napi_per.cpustat[CPUTIME_SOFTIRQ] + napi_per.cpustat[CPUTIME_IRQ] >= IRQ_LIMIT && + dyn_threaded_corenum < dyn_threaded_corenum_upper_limit) { + action = 1; + } else if (napi_per.cpustat[CPUTIME_USER] >= dyn_threaded_user_per_limit && + dyn_threaded_corenum > dyn_threaded_corenum_lower_limit) { + action = -1; + } else if (100 - napi_per.cpustat[CPUTIME_IDLE] < dyn_threaded_cpuload_lower_limit && + dyn_threaded_corenum > dyn_threaded_corenum_lower_limit && + (snapshot_napi_per.cpustat[CPUTIME_SOFTIRQ] + + snapshot_napi_per.cpustat[CPUTIME_IRQ] < IRQ_LIMIT || + last_threaded_corenum != dyn_threaded_corenum - 1)) { + action = -1; + } else if (100 - napi_per.cpustat[CPUTIME_IDLE] >= dyn_threaded_cpuload_upper_limit && + dyn_threaded_corenum < dyn_threaded_corenum_upper_limit && + (snapshot_napi_per.cpustat[CPUTIME_USER] < dyn_threaded_user_per_limit || + last_threaded_corenum != dyn_threaded_corenum + 1)) { + action = 1; + } + + if (action) { + if (dynamic_corenum_keep) { + dynamic_corenum_keep = false; + goto out; + } + dynamic_corenum_changed = true; + snapshot_napi_per = napi_per; + last_threaded_corenum = dyn_threaded_corenum; + dyn_threaded_corenum += action; + } else { + dynamic_corenum_keep = true; + } + +out: + snapshot_napi_cpustat = napi_cpustat; +} + +/* reset the state collected while NAPI was threaded */ +static void clear_corenum_napi(void) +{ + memset(&snapshot_napi_per, 0, sizeof(snapshot_napi_per)); + last_threaded_corenum = dyn_threaded_corenum_default; + dynamic_corenum_changed = true; + dynamic_corenum_keep = false; +} + static int dynthreaded; static struct work_struct dyn_threaded_work; @@ -834,6 +1005,7 @@ static void dyn_threaded_handler(struct work_struct *work) case NO_TASK: /* no target tasks: restore the original irq affinity */ recover_irq_affinity(); + clear_corenum_napi(); break; case DATA_LOSS: /* invalid sample, reuse the previous status and statistics */ @@ -846,13 +1018,19 @@ static void dyn_threaded_handler(struct work_struct *work) set_all_napi_threaded(false); /* and the original irq affinity can be restored */ recover_irq_affinity(); + /* the napi thread cpu count goes back to the default */ + dyn_threaded_corenum = dyn_threaded_corenum_default; + clear_corenum_napi(); break; case FRAGMENTATION: /* softirq mode is fine, irqs spread over their NUMA node */ set_all_napi_threaded(false); bind_irq(); + clear_corenum_napi(); break; case LONG_RUNNING_WITH_INTERRUPTS: + /* adapt the number of thread cpus to the measured load */ + change_dyn_threaded_corenum(); /* switch NAPI processing to threads bound away from * the target task cpus */ @@ -980,6 +1158,7 @@ static int __init dynthread_init(void) set_default_dyn_threaded_corenum(); if (!dyn_threaded_corenum) dyn_threaded_corenum = dyn_threaded_corenum_default; + last_threaded_corenum = dyn_threaded_corenum; return 0; } -- 2.18.0.huawei.25