hulk inclusion category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/ICBFCS -------------------------------- Add numa affinity RPS to balance softirq, this fix high load performance regression. Fixes: d60758b0ffcd ("net/oenetcls: introduce oenetcls for network optimization") Signed-off-by: Yue Haibing <yuehaibing@huawei.com> --- include/linux/oenetcls.h | 32 ++++++++++++---- net/core/dev.c | 19 ++++++++- net/oenetcls/oenetcls_flow.c | 70 +++++++++++++++++++++++++++------- net/oenetcls/oenetcls_main.c | 11 ++++-- net/oenetcls/oenetcls_ntuple.c | 2 +- 5 files changed, 108 insertions(+), 26 deletions(-) diff --git a/include/linux/oenetcls.h b/include/linux/oenetcls.h index 29c0db40971f..09f89131f32b 100644 --- a/include/linux/oenetcls.h +++ b/include/linux/oenetcls.h @@ -5,12 +5,14 @@ struct oecls_hook_ops { void (*oecls_cfg_rxcls)(struct sock *sk, int is_del); void (*oecls_flow_update)(struct sock *sk); - void (*oecls_set_cpu)(struct sk_buff *skb); + void (*oecls_set_cpu)(struct sk_buff *skb, int *cpu, int *last_qtail); bool (*oecls_timeout)(struct net_device *dev, u16 rxq_index, u32 flow_id, u16 filter_id); }; +typedef int (*enqueue_f)(struct sk_buff *skb, int cpu, unsigned int *qtail); extern const struct oecls_hook_ops __rcu *oecls_ops; +extern struct static_key_false oecls_rps_needed; static inline void oenetcls_cfg_rxcls(struct sock *sk, int is_del) { @@ -34,27 +36,43 @@ static inline void oenetcls_flow_update(struct sock *sk) rcu_read_unlock(); } -static inline void oenetcls_skb_set_cpu(struct sk_buff *skb) +static inline bool +oenetcls_skb_set_cpu(struct sk_buff *skb, enqueue_f enq_func, int *ret) { const struct oecls_hook_ops *ops; + int cpu, last_qtail; + bool result = false; rcu_read_lock(); ops = rcu_dereference(oecls_ops); - if (ops && ops->oecls_set_cpu) - ops->oecls_set_cpu(skb); + if (ops && ops->oecls_set_cpu) { + ops->oecls_set_cpu(skb, &cpu, &last_qtail); + if (cpu >= 0) { + *ret = enq_func(skb, cpu, &last_qtail); + result = true; + } + } rcu_read_unlock(); + return result; } -static inline void oenetcls_skblist_set_cpu(struct list_head *head) +static inline void +oenetcls_skblist_set_cpu(struct list_head *head, enqueue_f enq_func) { const struct oecls_hook_ops *ops; struct sk_buff *skb, *next; + int cpu, last_qtail; rcu_read_lock(); ops = rcu_dereference(oecls_ops); if (ops && ops->oecls_set_cpu) { - list_for_each_entry_safe(skb, next, head, list) - ops->oecls_set_cpu(skb); + list_for_each_entry_safe(skb, next, head, list) { + ops->oecls_set_cpu(skb, &cpu, &last_qtail); + if (cpu >= 0) { + skb_list_del_init(skb); + enq_func(skb, cpu, &last_qtail); + } + } } rcu_read_unlock(); } diff --git a/net/core/dev.c b/net/core/dev.c index 05477c431afd..df6c0ac02a5e 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -154,6 +154,8 @@ #include <linux/oenetcls.h> const struct oecls_hook_ops __rcu *oecls_ops __read_mostly; EXPORT_SYMBOL_GPL(oecls_ops); +struct static_key_false oecls_rps_needed __read_mostly; +EXPORT_SYMBOL(oecls_rps_needed); #endif #define MAX_GRO_SKBS 8 @@ -5889,6 +5891,10 @@ static int netif_receive_skb_internal(struct sk_buff *skb) rcu_read_lock(); #ifdef CONFIG_RPS +#if IS_ENABLED(CONFIG_OENETCLS) + if (static_branch_unlikely(&oecls_rps_needed)) + goto oecls_rps; +#endif if (static_branch_unlikely(&rps_needed)) { struct rps_dev_flow voidflow, *rflow = &voidflow; int cpu = get_rps_cpu(skb->dev, skb, &rflow); @@ -5902,7 +5908,11 @@ static int netif_receive_skb_internal(struct sk_buff *skb) #endif #if IS_ENABLED(CONFIG_OENETCLS) - oenetcls_skb_set_cpu(skb); +oecls_rps: + if (oenetcls_skb_set_cpu(skb, enqueue_to_backlog, &ret)) { + rcu_read_unlock(); + return ret; + } #endif ret = __netif_receive_skb(skb); @@ -5926,6 +5936,10 @@ static void netif_receive_skb_list_internal(struct list_head *head) rcu_read_lock(); #ifdef CONFIG_RPS +#if IS_ENABLED(CONFIG_OENETCLS) + if (static_branch_unlikely(&oecls_rps_needed)) + goto oecls_rps_list; +#endif if (static_branch_unlikely(&rps_needed)) { list_for_each_entry_safe(skb, next, head, list) { struct rps_dev_flow voidflow, *rflow = &voidflow; @@ -5941,7 +5955,8 @@ static void netif_receive_skb_list_internal(struct list_head *head) #endif #if IS_ENABLED(CONFIG_OENETCLS) - oenetcls_skblist_set_cpu(head); +oecls_rps_list: + oenetcls_skblist_set_cpu(head, enqueue_to_backlog); #endif __netif_receive_skb_list(head); diff --git a/net/oenetcls/oenetcls_flow.c b/net/oenetcls/oenetcls_flow.c index 8d9fdb343d28..62bad2408966 100644 --- a/net/oenetcls/oenetcls_flow.c +++ b/net/oenetcls/oenetcls_flow.c @@ -133,8 +133,7 @@ static void set_oecls_cpu(struct net_device *dev, struct sk_buff *skb, struct oecls_dev_flow_table *dtb; struct oecls_dev_flow *rflow; u32 flow_id, hash; - u16 rxq_index; - int rc; + int rxq_index, rc; if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || !(dev->features & NETIF_F_NTUPLE)) @@ -152,7 +151,8 @@ static void set_oecls_cpu(struct net_device *dev, struct sk_buff *skb, hash = skb_get_hash(skb); flow_id = hash & dtb->mask; rflow = &dtb->flows[flow_id]; - if (rflow->isvalid && rflow->cpu == next_cpu) { + //Return if someone has configured this. + if (rflow->isvalid && cpu_to_node(rflow->cpu) == cpu_to_node(next_cpu)) { rflow->timeout = jiffies; return; } @@ -171,15 +171,41 @@ static void set_oecls_cpu(struct net_device *dev, struct sk_buff *skb, rflow->cpu = next_cpu; } +static int get_cpu_in_mask(int tcpu, u32 hash) +{ + const struct cpumask *mask; + int nr_cpus, cpu, index; + + mask = cpumask_of_node(cpu_to_node(tcpu)); + + nr_cpus = cpumask_weight(mask); + if (nr_cpus == 0) + return -1; + + index = reciprocal_scale(hash, nr_cpus); + if (index < 0) + return -1; + + cpu = cpumask_first(mask); + while (--nr_cpus > 0) { + if (index == 0) + break; + cpu = cpumask_next(cpu, mask); + index--; + } + + return cpu; +} + static void __oecls_set_cpu(struct sk_buff *skb, struct net_device *ndev, struct oecls_sock_flow_table *tb, struct oecls_dev_flow_table *dtb, - int old_rxq_id) + int old_rxq_id, int *rcpu, int *last_qtail) { + u32 last_recv_cpu, hash, val, cpu, tcpu; struct oecls_dev_flow *rflow; - u32 last_recv_cpu, hash, val; - u32 tcpu = 0; - u32 cpu = raw_smp_processor_id(); + int newcpu; + cpu = raw_smp_processor_id(); skb_reset_network_header(skb); hash = skb_get_hash(skb); if (!hash) @@ -193,14 +219,20 @@ static void __oecls_set_cpu(struct sk_buff *skb, struct net_device *ndev, if ((val ^ hash) & ~oecls_cpu_mask) return; - if (cpu_to_node(cpu) == cpu_to_node(last_recv_cpu)) + newcpu = get_cpu_in_mask(last_recv_cpu, hash); + if (newcpu >= 0) + *rcpu = newcpu; + else + newcpu = last_recv_cpu; + + if (cpu_to_node(cpu) == cpu_to_node(newcpu)) return; if (tcpu >= nr_cpu_ids) - set_oecls_cpu(ndev, skb, rflow, old_rxq_id, last_recv_cpu); + set_oecls_cpu(ndev, skb, rflow, old_rxq_id, newcpu); } -static void _oecls_set_cpu(struct sk_buff *skb) +static void _oecls_set_cpu(struct sk_buff *skb, int *cpu, int *last_qtail) { struct net_device *ndev = skb->dev; struct oecls_sock_flow_table *stb; @@ -208,6 +240,8 @@ static void _oecls_set_cpu(struct sk_buff *skb) struct netdev_rx_queue *rxqueue; int rxq_id = -1; + *cpu = -1; + last_qtail = 0;//unused if (!ndev) return; @@ -233,7 +267,7 @@ static void _oecls_set_cpu(struct sk_buff *skb) stb = rcu_dereference(oecls_sock_flow_table); dtb = rcu_dereference(rxqueue->oecls_ftb); if (stb && dtb) - __oecls_set_cpu(skb, ndev, stb, dtb, rxq_id); + __oecls_set_cpu(skb, ndev, stb, dtb, rxq_id, cpu, last_qtail); rcu_read_unlock(); } @@ -245,13 +279,13 @@ static void oecls_dev_flow_table_free(struct rcu_head *rcu) vfree(table); } -static void oecls_dev_flow_table_cleanup(struct net_device *netdev, int qid) +static void oecls_dev_flow_table_cleanup(struct net_device *netdev, int queues) { struct oecls_dev_flow_table *dtb; struct netdev_rx_queue *queue; int i; - for (i = 0; i < qid; i++) { + for (i = 0; i < queues; i++) { queue = netdev->_rx + i; spin_lock(&oecls_dev_flow_lock); dtb = rcu_dereference_protected(queue->oecls_ftb, @@ -407,11 +441,21 @@ int oecls_flow_res_init(void) RCU_INIT_POINTER(oecls_ops, &oecls_flow_ops); synchronize_rcu(); + +#ifdef CONFIG_RPS + static_branch_inc(&oecls_rps_needed); + oecls_debug("oecls_rps_needed true\n"); +#endif + return 0; } void oecls_flow_res_clean(void) { +#ifdef CONFIG_RPS + static_branch_dec(&oecls_rps_needed); + oecls_debug("oecls_rps_needed false\n"); +#endif rcu_assign_pointer(oecls_ops, NULL); synchronize_rcu(); diff --git a/net/oenetcls/oenetcls_main.c b/net/oenetcls/oenetcls_main.c index e5920351aada..ec37e38d7f48 100644 --- a/net/oenetcls/oenetcls_main.c +++ b/net/oenetcls/oenetcls_main.c @@ -23,11 +23,11 @@ static int mode; module_param(mode, int, 0444); MODULE_PARM_DESC(mode, "mode, default 0"); -static char ifname[64] = { 0 }; +static char ifname[128] = { 0 }; module_param_string(ifname, ifname, sizeof(ifname), 0444); MODULE_PARM_DESC(ifname, "ifname"); -static char appname[64] = "redis-server"; +static char appname[256] = "redis-server"; module_param_string(appname, appname, sizeof(appname), 0644); MODULE_PARM_DESC(appname, "appname, default redis-server"); @@ -43,6 +43,10 @@ static int check_cap = 1; module_param(check_cap, int, 0444); MODULE_PARM_DESC(check_cap, "check_cap, default 1"); +static char irqname[64] = "comp"; +module_param_string(irqname, irqname, sizeof(irqname), 0644); +MODULE_PARM_DESC(irqname, "nic irq name string, default comp"); + static bool check_params(void) { if (mode != 0 && mode != 1) @@ -371,7 +375,8 @@ static struct oecls_netdev_info *alloc_oecls_netdev_info(void) static bool check_irq_name(const char *irq_name, struct oecls_netdev_info *oecls_dev) { - if (!strstr(irq_name, "TxRx") && !strstr(irq_name, "comp") && !strstr(irq_name, "rx")) + if (!strstr(irq_name, "TxRx") && !strstr(irq_name, "comp") && !strstr(irq_name, "rx") && + strlen(irqname) > 0 && !strstr(irq_name, irqname)) return false; if (strstr(irq_name, oecls_dev->dev_name)) diff --git a/net/oenetcls/oenetcls_ntuple.c b/net/oenetcls/oenetcls_ntuple.c index 5bb1b1916a4c..888dc5b5005b 100644 --- a/net/oenetcls/oenetcls_ntuple.c +++ b/net/oenetcls/oenetcls_ntuple.c @@ -59,7 +59,7 @@ static void add_sk_rule(int devid, u32 dip4, u16 dport, void *sk, int action, in hlist_add_head(&entry->node, sk_hlist); return; out: - oecls_debug("alloc failed rule:%p entry:%p\n", rule, entry); + oecls_debug("alloc rule failed\n"); free_to_l0(entry); free_to_l0(rule); } -- 2.34.1