From: Philo Lu <lulie@linux.alibaba.com> mainline inclusion from mainline-v6.13-rc1 commit accdd51dc74ff65b7b7be1961b11723d228fbbbd category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/ICIAC2 CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... ------------------------------------------------- Preparing for udp 4-tuple hash (uhash4 for short). To implement uhash4 without cache line missing when lookup, hslot2 is used to record the number of hashed sockets in hslot4. Thus adding a new struct udp_hslot_main with field hash4_cnt, which is used by hash2. The new struct is used to avoid doubling the size of udp_hslot. Before uhash4 lookup, firstly checking hash4_cnt to see if there are hashed sks in hslot4. Because hslot2 is always used in lookup, there is no cache line miss. Related helpers are updated, and use the helpers as possible. uhash4 is implemented in following patches. Signed-off-by: Philo Lu <lulie@linux.alibaba.com> Acked-by: Willem de Bruijn <willemb@google.com> Acked-by: Paolo Abeni <pabeni@redhat.com> Signed-off-by: David S. Miller <davem@davemloft.net> Conflicts: net/ipv4/udp.c net/ipv6/udp.c include/net/udp.h [Mainly caused by OLK did not backport 9804985bf27f, 5788b3a07fc5, 4cdeeee9252a.] Signed-off-by: Liu Jian <liujian56@huawei.com> --- include/net/udp.h | 40 ++++++++++++++++++++++++++++++++++++---- net/ipv4/udp.c | 31 +++++++++++++++---------------- net/ipv6/udp.c | 17 +++++++---------- 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/include/net/udp.h b/include/net/udp.h index bd27e9dd6b84..5a55d280f517 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -52,7 +52,7 @@ struct udp_skb_cb { #define UDP_SKB_CB(__skb) ((struct udp_skb_cb *)((__skb)->cb)) /** - * struct udp_hslot - UDP hash slot + * struct udp_hslot - UDP hash slot used by udp_table.hash * * @head: head of list of sockets * @count: number of sockets in 'head' list @@ -62,7 +62,22 @@ struct udp_hslot { struct hlist_head head; int count; spinlock_t lock; -} __attribute__((aligned(2 * sizeof(long)))); +} __aligned(2 * sizeof(long)); + +/** + * struct udp_hslot_main - UDP hash slot used by udp_table.hash2 + * + * @hslot: basic hash slot + * @hash4_cnt: number of sockets in hslot4 of the same + * (local port, local address) + */ +struct udp_hslot_main { + struct udp_hslot hslot; /* must be the first member */ +#if !IS_ENABLED(CONFIG_BASE_SMALL) + u32 hash4_cnt; +#endif +} __aligned(2 * sizeof(long)); +#define UDP_HSLOT_MAIN(__hslot) ((struct udp_hslot_main *)(__hslot)) /** * struct udp_table - UDP table @@ -74,7 +89,7 @@ struct udp_hslot { */ struct udp_table { struct udp_hslot *hash; - struct udp_hslot *hash2; + struct udp_hslot_main *hash2; unsigned int mask; unsigned int log; }; @@ -85,6 +100,7 @@ static inline struct udp_hslot *udp_hashslot(struct udp_table *table, { return &table->hash[udp_hashfn(net, num, table->mask)]; } + /* * For secondary hash, net_hash_mix() is performed before calling * udp_hashslot2(), this explains difference with udp_hashslot() @@ -92,8 +108,24 @@ static inline struct udp_hslot *udp_hashslot(struct udp_table *table, static inline struct udp_hslot *udp_hashslot2(struct udp_table *table, unsigned int hash) { - return &table->hash2[hash & table->mask]; + return &table->hash2[hash & table->mask].hslot; +} + +#if IS_ENABLED(CONFIG_BASE_SMALL) +static inline void udp_table_hash4_init(struct udp_table *table) +{ +} +#else /* !CONFIG_BASE_SMALL */ + +/* Must be called with table->hash2 initialized */ +static inline void udp_table_hash4_init(struct udp_table *table) +{ + int i; + + for (i = 0; i <= table->mask; i++) + table->hash2[i].hash4_cnt = 0; } +#endif /* CONFIG_BASE_SMALL */ extern struct proto udp_prot; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 61af6d3388b6..4784ecd9d336 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -468,7 +468,7 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, { struct sock *sk, *result; unsigned short hnum = ntohs(dport); - unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); + unsigned int hash2, slot = udp_hashfn(net, hnum, udptable->mask); struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; bool exact_dif = udp_lib_exact_dif_match(net, skb); int score, badness; @@ -476,8 +476,7 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, if (hslot->count > 10) { hash2 = ipv4_portaddr_hash(net, daddr, hnum); - slot2 = hash2 & udptable->mask; - hslot2 = &udptable->hash2[slot2]; + hslot2 = udp_hashslot2(udptable, hash2); if (hslot->count < hslot2->count) goto begin; @@ -485,14 +484,13 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, daddr, hnum, dif, sdif, exact_dif, hslot2, skb); if (!result) { - unsigned int old_slot2 = slot2; + void *old_hslot2 = hslot2; hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum); - slot2 = hash2 & udptable->mask; + hslot2 = udp_hashslot2(udptable, hash2); /* avoid searching the same slot again. */ - if (unlikely(slot2 == old_slot2)) + if (unlikely(hslot2 == old_hslot2)) return result; - hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; @@ -2083,7 +2081,7 @@ static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, udptable->mask; hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask; start_lookup: - hslot = &udptable->hash2[hash2]; + hslot = &udptable->hash2[hash2].hslot; offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); } @@ -2338,8 +2336,7 @@ static struct sock *__udp4_lib_demux_lookup(struct net *net, { unsigned short hnum = ntohs(loc_port); unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum); - unsigned int slot2 = hash2 & udp_table.mask; - struct udp_hslot *hslot2 = &udp_table.hash2[slot2]; + struct udp_hslot *hslot2 = udp_hashslot2(&udp_table, hash2); INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr); const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum); struct sock *sk; @@ -2918,10 +2915,11 @@ __setup("uhash_entries=", set_uhash_entries); void __init udp_table_init(struct udp_table *table, const char *name) { - unsigned int i; + unsigned int i, slot_size; + slot_size = sizeof(struct udp_hslot) + sizeof(struct udp_hslot_main); table->hash = alloc_large_system_hash(name, - 2 * sizeof(struct udp_hslot), + slot_size, uhash_entries, 21, /* one slot per 2 MB */ 0, @@ -2930,17 +2928,18 @@ void __init udp_table_init(struct udp_table *table, const char *name) UDP_HTABLE_SIZE_MIN, 64 * 1024); - table->hash2 = table->hash + (table->mask + 1); + table->hash2 = (void *)(table->hash + (table->mask + 1)); for (i = 0; i <= table->mask; i++) { INIT_HLIST_HEAD(&table->hash[i].head); table->hash[i].count = 0; spin_lock_init(&table->hash[i].lock); } for (i = 0; i <= table->mask; i++) { - INIT_HLIST_HEAD(&table->hash2[i].head); - table->hash2[i].count = 0; - spin_lock_init(&table->hash2[i].lock); + INIT_HLIST_HEAD(&table->hash2[i].hslot.head); + table->hash2[i].hslot.count = 0; + spin_lock_init(&table->hash2[i].hslot.lock); } + udp_table_hash4_init(table); } u32 udp_flow_hashrnd(void) diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index bc10b241da84..48b48be8b167 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -215,7 +215,7 @@ struct sock *__udp6_lib_lookup(struct net *net, { struct sock *sk, *result; unsigned short hnum = ntohs(dport); - unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); + unsigned int hash2, slot = udp_hashfn(net, hnum, udptable->mask); struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; bool exact_dif = udp6_lib_exact_dif_match(net, skb); int score, badness; @@ -223,8 +223,7 @@ struct sock *__udp6_lib_lookup(struct net *net, if (hslot->count > 10) { hash2 = ipv6_portaddr_hash(net, daddr, hnum); - slot2 = hash2 & udptable->mask; - hslot2 = &udptable->hash2[slot2]; + hslot2 = udp_hashslot2(udptable, hash2); if (hslot->count < hslot2->count) goto begin; @@ -232,14 +231,13 @@ struct sock *__udp6_lib_lookup(struct net *net, daddr, hnum, dif, sdif, exact_dif, hslot2, skb); if (!result) { - unsigned int old_slot2 = slot2; + void *old_hslot2 = hslot2; hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum); - slot2 = hash2 & udptable->mask; + hslot2 = udp_hashslot2(udptable, hash2); /* avoid searching the same slot again. */ - if (unlikely(slot2 == old_slot2)) + if (unlikely(hslot2 == old_hslot2)) return result; - hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; @@ -713,7 +711,7 @@ static int __udp6_lib_mcast_deliver(struct net *net, struct sk_buff *skb, udptable->mask; hash2 = ipv6_portaddr_hash(net, daddr, hnum) & udptable->mask; start_lookup: - hslot = &udptable->hash2[hash2]; + hslot = &udptable->hash2[hash2].hslot; offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); } @@ -908,8 +906,7 @@ static struct sock *__udp6_lib_demux_lookup(struct net *net, { unsigned short hnum = ntohs(loc_port); unsigned int hash2 = ipv6_portaddr_hash(net, loc_addr, hnum); - unsigned int slot2 = hash2 & udp_table.mask; - struct udp_hslot *hslot2 = &udp_table.hash2[slot2]; + struct udp_hslot *hslot2 = udp_hashslot2(&udp_table, hash2); const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum); struct sock *sk; -- 2.34.1