From: Eric Dumazet edumazet@google.com
mainline inclusion from mainline-v5.16-rc1 commit c4777efa751d293e369aec464ce6875e957be255 category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I4RYXI CVE: NA
--------------------------------
While commit 097b9146c0e2 ("net: fix up truesize of cloned skb in skb_prepare_for_shift()") fixed immediate issues found when KFENCE was enabled/tested, there are still similar issues, when tcp_trim_head() hits KFENCE while the master skb is cloned.
This happens under heavy networking TX workloads, when the TX completion might be delayed after incoming ACK.
This patch fixes the WARNING in sk_stream_kill_queues when sk->sk_mem_queued/sk->sk_forward_alloc are not zero.
Fixes: d3fb45f370d9 ("mm, kfence: insert KFENCE hooks for SLAB") Signed-off-by: Eric Dumazet edumazet@google.com Acked-by: Marco Elver elver@google.com Link: https://lore.kernel.org/r/20211102004555.1359210-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski kuba@kernel.org Signed-off-by: YueHaibing yuehaibing@huawei.com Reviewed-by: Wei Yongjun weiyongjun1@huawei.com Reviewed-by: Liu Jian liujian56@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- include/linux/skbuff.h | 16 ++++++++++++++++ net/core/skbuff.c | 14 +------------- net/ipv4/tcp_output.c | 6 +++--- 3 files changed, 20 insertions(+), 16 deletions(-)
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index d485f17ff33a..e5f61bdd42a8 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1652,6 +1652,22 @@ static inline int skb_unclone(struct sk_buff *skb, gfp_t pri) return 0; }
+/* This variant of skb_unclone() makes sure skb->truesize is not changed */ +static inline int skb_unclone_keeptruesize(struct sk_buff *skb, gfp_t pri) +{ + might_sleep_if(gfpflags_allow_blocking(pri)); + + if (skb_cloned(skb)) { + unsigned int save = skb->truesize; + int res; + + res = pskb_expand_head(skb, 0, 0, pri); + skb->truesize = save; + return res; + } + return 0; +} + /** * skb_header_cloned - is the header a clone * @skb: buffer to check diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 8d73503c8b32..16c74a81b7bf 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3311,19 +3311,7 @@ EXPORT_SYMBOL(skb_split); */ static int skb_prepare_for_shift(struct sk_buff *skb) { - int ret = 0; - - if (skb_cloned(skb)) { - /* Save and restore truesize: pskb_expand_head() may reallocate - * memory where ksize(kmalloc(S)) != ksize(kmalloc(S)), but we - * cannot change truesize at this point. - */ - unsigned int save_truesize = skb->truesize; - - ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); - skb->truesize = save_truesize; - } - return ret; + return skb_unclone_keeptruesize(skb, GFP_ATOMIC); }
/** diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index c8420b48de36..da8bcdd90715 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -1613,7 +1613,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, return -ENOMEM; }
- if (skb_unclone(skb, gfp)) + if (skb_unclone_keeptruesize(skb, gfp)) return -ENOMEM;
/* Get a new skb... force flag on. */ @@ -1722,7 +1722,7 @@ int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len) { u32 delta_truesize;
- if (skb_unclone(skb, GFP_ATOMIC)) + if (skb_unclone_keeptruesize(skb, GFP_ATOMIC)) return -ENOMEM;
delta_truesize = __pskb_trim_head(skb, len); @@ -3236,7 +3236,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) cur_mss, GFP_ATOMIC)) return -ENOMEM; /* We'll try again later. */ } else { - if (skb_unclone(skb, GFP_ATOMIC)) + if (skb_unclone_keeptruesize(skb, GFP_ATOMIC)) return -ENOMEM;
diff = tcp_skb_pcount(skb);