From: Eric Dumazet edumazet@google.com
mainline inclusion from mainline-v5.6 commit c54a274 category: bugfix bugzilla: 93967 CVE: NA
-----------------------------------------------
We would like to use hlist_unhashed() from timer_pending(), which runs without protection of a lock.
Note that other callers might also want to use this variant.
Instead of forcing a READ_ONCE() for all hlist_unhashed() callers, add a new helper with an explicit _lockless suffix in the name to better document what is going on.
Also add various WRITE_ONCE() in __hlist_del(), hlist_add_head() and hlist_add_before()/hlist_add_behind() to pair with the READ_ONCE().
Signed-off-by: Eric Dumazet edumazet@google.com Cc: Thomas Gleixner tglx@linutronix.de [ paulmck: Also add WRITE_ONCE() to rculist.h. ] Signed-off-by: Paul E. McKenney paulmck@kernel.org
Conflicts: include/linux/list.h [wangxiongfeng: include patch commit ae325dcd1 ("list: Don't use WRITE_ONCE() in hlist_add_behind()")] Signed-off-by: Xiongfeng Wang wangxiongfeng2@huawei.com Reviewed-by: Xie XiuQi xiexiuqi@huawei.com Signed-off-by: Yang Yingliang yangyingliang@huawei.com --- include/linux/list.h | 30 ++++++++++++++++++++---------- include/linux/rculist.h | 24 ++++++++++++------------ 2 files changed, 32 insertions(+), 22 deletions(-)
diff --git a/include/linux/list.h b/include/linux/list.h index 0e540581d52c4..fc0e87f94d286 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -676,6 +676,16 @@ static inline int hlist_unhashed(const struct hlist_node *h) return !h->pprev; }
+/* This variant of hlist_unhashed() must be used in lockless contexts + * to avoid potential load-tearing. + * The READ_ONCE() is paired with the various WRITE_ONCE() in hlist + * helpers that are defined below. + */ +static inline int hlist_unhashed_lockless(const struct hlist_node *h) +{ + return !READ_ONCE(h->pprev); +} + static inline int hlist_empty(const struct hlist_head *h) { return !READ_ONCE(h->first); @@ -688,7 +698,7 @@ static inline void __hlist_del(struct hlist_node *n)
WRITE_ONCE(*pprev, next); if (next) - next->pprev = pprev; + WRITE_ONCE(next->pprev, pprev); }
static inline void hlist_del(struct hlist_node *n) @@ -709,32 +719,32 @@ static inline void hlist_del_init(struct hlist_node *n) static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) { struct hlist_node *first = h->first; - n->next = first; + WRITE_ONCE(n->next, first); if (first) - first->pprev = &n->next; + WRITE_ONCE(first->pprev, &n->next); WRITE_ONCE(h->first, n); - n->pprev = &h->first; + WRITE_ONCE(n->pprev, &h->first); }
/* next must be != NULL */ static inline void hlist_add_before(struct hlist_node *n, struct hlist_node *next) { - n->pprev = next->pprev; - n->next = next; - next->pprev = &n->next; + WRITE_ONCE(n->pprev, next->pprev); + WRITE_ONCE(n->next, next); + WRITE_ONCE(next->pprev, &n->next); WRITE_ONCE(*(n->pprev), n); }
static inline void hlist_add_behind(struct hlist_node *n, struct hlist_node *prev) { - n->next = prev->next; + WRITE_ONCE(n->next, prev->next); WRITE_ONCE(prev->next, n); - n->pprev = &prev->next; + WRITE_ONCE(n->pprev, &prev->next);
if (n->next) - n->next->pprev = &n->next; + WRITE_ONCE(n->next->pprev, &n->next); }
/* after that we'll appear to be on some hlist and hlist_del will work */ diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 4786c2235b981..74ba25c858429 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h @@ -155,7 +155,7 @@ static inline void hlist_del_init_rcu(struct hlist_node *n) { if (!hlist_unhashed(n)) { __hlist_del(n); - n->pprev = NULL; + WRITE_ONCE(n->pprev, NULL); } }
@@ -455,7 +455,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list, static inline void hlist_del_rcu(struct hlist_node *n) { __hlist_del(n); - n->pprev = LIST_POISON2; + WRITE_ONCE(n->pprev, LIST_POISON2); }
/** @@ -471,11 +471,11 @@ static inline void hlist_replace_rcu(struct hlist_node *old, struct hlist_node *next = old->next;
new->next = next; - new->pprev = old->pprev; + WRITE_ONCE(new->pprev, old->pprev); rcu_assign_pointer(*(struct hlist_node __rcu **)new->pprev, new); if (next) - new->next->pprev = &new->next; - old->pprev = LIST_POISON2; + WRITE_ONCE(new->next->pprev, &new->next); + WRITE_ONCE(old->pprev, LIST_POISON2); }
/* @@ -510,10 +510,10 @@ static inline void hlist_add_head_rcu(struct hlist_node *n, struct hlist_node *first = h->first;
n->next = first; - n->pprev = &h->first; + WRITE_ONCE(n->pprev, &h->first); rcu_assign_pointer(hlist_first_rcu(h), n); if (first) - first->pprev = &n->next; + WRITE_ONCE(first->pprev, &n->next); }
/** @@ -546,7 +546,7 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n,
if (last) { n->next = last->next; - n->pprev = &last->next; + WRITE_ONCE(n->pprev, &last->next); rcu_assign_pointer(hlist_next_rcu(last), n); } else { hlist_add_head_rcu(n, h); @@ -574,10 +574,10 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n, static inline void hlist_add_before_rcu(struct hlist_node *n, struct hlist_node *next) { - n->pprev = next->pprev; + WRITE_ONCE(n->pprev, next->pprev); n->next = next; rcu_assign_pointer(hlist_pprev_rcu(n), n); - next->pprev = &n->next; + WRITE_ONCE(next->pprev, &n->next); }
/** @@ -602,10 +602,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, struct hlist_node *prev) { n->next = prev->next; - n->pprev = &prev->next; + WRITE_ONCE(n->pprev, &prev->next); rcu_assign_pointer(hlist_next_rcu(prev), n); if (n->next) - n->next->pprev = &n->next; + WRITE_ONCE(n->next->pprev, &n->next); }
#define __hlist_for_each_rcu(pos, head) \