From: Arjun Roy arjunroy@google.com
stable inclusion from stable-v5.10.210 commit 14690e419bb37c81dd7a4bf23daa1097773ecf01 category: bugfix bugzilla: https://gitee.com/src-openeuler/kernel/issues/I99JO6 CVE: CVE-2024-26640
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=...
--------------------------------
[ Upstream commit 98917cf0d6eda01e8c3c34d35398d46b247b6fd3 ]
Refactor frag-is-remappable test for tcp receive zerocopy. This is part of a patch set that introduces short-circuited hybrid copies for small receive operations, which results in roughly 33% fewer syscalls for small RPC scenarios.
Signed-off-by: Arjun Roy arjunroy@google.com Signed-off-by: Eric Dumazet edumazet@google.com Signed-off-by: Soheil Hassas Yeganeh soheil@google.com Signed-off-by: Jakub Kicinski kuba@kernel.org Stable-dep-of: 577e4432f3ac ("tcp: add sanity checks to rx zerocopy") Signed-off-by: Sasha Levin sashal@kernel.org Signed-off-by: Ziyang Xuan william.xuanziyang@huawei.com --- net/ipv4/tcp.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-)
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index dcbc5ec00da87..ac3d3bbe2a242 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1792,6 +1792,26 @@ static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb, return frag; }
+static bool can_map_frag(const skb_frag_t *frag) +{ + return skb_frag_size(frag) == PAGE_SIZE && !skb_frag_off(frag); +} + +static int find_next_mappable_frag(const skb_frag_t *frag, + int remaining_in_skb) +{ + int offset = 0; + + if (likely(can_map_frag(frag))) + return 0; + + while (offset < remaining_in_skb && !can_map_frag(frag)) { + offset += skb_frag_size(frag); + ++frag; + } + return offset; +} + static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc, struct sk_buff *skb, u32 copylen, u32 *offset, u32 *seq) @@ -1917,6 +1937,8 @@ static int tcp_zerocopy_receive(struct sock *sk, ret = 0; curr_addr = address; while (length + PAGE_SIZE <= zc->length) { + int mappable_offset; + if (zc->recv_skip_hint < PAGE_SIZE) { u32 offset_frag;
@@ -1944,15 +1966,11 @@ static int tcp_zerocopy_receive(struct sock *sk, if (!frags || offset_frag) break; } - if (skb_frag_size(frags) != PAGE_SIZE || skb_frag_off(frags)) { - int remaining = zc->recv_skip_hint;
- while (remaining && (skb_frag_size(frags) != PAGE_SIZE || - skb_frag_off(frags))) { - remaining -= skb_frag_size(frags); - frags++; - } - zc->recv_skip_hint -= remaining; + mappable_offset = find_next_mappable_frag(frags, + zc->recv_skip_hint); + if (mappable_offset) { + zc->recv_skip_hint = mappable_offset; break; } pages[pg_idx] = skb_frag_page(frags);