hulk inclusion category: bugfix bugzilla: NA -------------------------------- Introduce ep_lock_* wrapper macros that select spinlock_t under CONFIG_PREEMPT_RT and rwlock_t otherwise. The rwlock read-side cannot be priority-boosted on RT (rwbase_rt readers are not rt_mutex owners), causing unbounded priority inversion; spinlock fixes this. On non-RT kernels the rwlock is preserved for producer concurrency. The lockless helpers list_add_tail_lockless() and chain_epi_lockless() are compiled out under CONFIG_PREEMPT_RT since the spinlock serializes all producers. Fixes: a218cc491420 ("epoll: use rwlock in order to reduce ep_poll_callback() contention") Signed-off-by: Ran Hongyun <ranhongyun1@huawei.com> --- fs/eventpoll.c | 95 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 24 deletions(-) diff --git a/fs/eventpoll.c b/fs/eventpoll.c index ba719222a2da..523e69de1e32 100644 --- a/fs/eventpoll.c +++ b/fs/eventpoll.c @@ -39,13 +39,45 @@ #include <linux/rculist.h> #include <net/busy_poll.h> +/* + * Under CONFIG_PREEMPT_RT the ep->lock is a spinlock because the rwlock + * read-side cannot be priority-boosted (rwbase_rt readers are not + * rt_mutex owners), which causes unbounded priority inversion when a + * high-priority consumer (writer) waits on low-priority producers + * (readers). On non-RT kernels the rwlock is preserved so that multiple + * producers can run concurrently, which scales better with large event + * sources. + */ + +#ifdef CONFIG_PREEMPT_RT + +#define ep_lock_t spinlock_t +#define ep_lock_init(lock) spin_lock_init(lock) +#define ep_lock_irq(lock) spin_lock_irq(lock) +#define ep_unlock_irq(lock) spin_unlock_irq(lock) +#define ep_lock_irqsave(lock, flags) spin_lock_irqsave(lock, flags) +#define ep_unlock_irqrestore(lock, flags) \ + spin_unlock_irqrestore(lock, flags) + +#else /* !CONFIG_PREEMPT_RT */ + +#define ep_lock_t rwlock_t +#define ep_lock_init(lock) rwlock_init(lock) +#define ep_lock_irq(lock) write_lock_irq(lock) +#define ep_unlock_irq(lock) write_unlock_irq(lock) +#define ep_lock_irqsave(lock, flags) read_lock_irqsave(lock, flags) +#define ep_unlock_irqrestore(lock, flags) \ + read_unlock_irqrestore(lock, flags) + +#endif /* CONFIG_PREEMPT_RT */ + /* * LOCKING: * There are three level of locking required by epoll : * * 1) epnested_mutex (mutex) * 2) ep->mtx (mutex) - * 3) ep->lock (rwlock) + * 3) ep->lock (rwlock or spinlock, see above) * * The acquire order is the one listed above, from 1 to 3. * We need a rwlock (ep->lock) because we manipulate objects @@ -194,7 +226,7 @@ struct eventpoll { struct list_head rdllist; /* Lock which protects rdllist and ovflist */ - rwlock_t lock; + ep_lock_t lock; /* RB tree root used to store monitored fd structs */ struct rb_root_cached rbr; @@ -628,10 +660,10 @@ static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist) * in a lockless way. */ lockdep_assert_irqs_enabled(); - write_lock_irq(&ep->lock); + ep_lock_irq(&ep->lock); list_splice_init(&ep->rdllist, txlist); WRITE_ONCE(ep->ovflist, NULL); - write_unlock_irq(&ep->lock); + ep_unlock_irq(&ep->lock); } static void ep_done_scan(struct eventpoll *ep, @@ -639,7 +671,7 @@ static void ep_done_scan(struct eventpoll *ep, { struct epitem *epi, *nepi; - write_lock_irq(&ep->lock); + ep_lock_irq(&ep->lock); /* * During the time we spent inside the "sproc" callback, some * other events might have been queued by the poll callback. @@ -680,7 +712,7 @@ static void ep_done_scan(struct eventpoll *ep, wake_up(&ep->wq); } - write_unlock_irq(&ep->lock); + ep_unlock_irq(&ep->lock); } static void epi_rcu_free(struct rcu_head *head) @@ -777,10 +809,10 @@ static bool ep_remove_epi(struct eventpoll *ep, struct epitem *epi) rb_erase_cached(&epi->rbn, &ep->rbr); - write_lock_irq(&ep->lock); + ep_lock_irq(&ep->lock); if (ep_is_linked(epi)) list_del_init(&epi->rdllink); - write_unlock_irq(&ep->lock); + ep_unlock_irq(&ep->lock); wakeup_source_unregister(ep_wakeup_source(epi)); /* @@ -1040,7 +1072,7 @@ static int ep_alloc(struct eventpoll **pep) return -ENOMEM; mutex_init(&ep->mtx); - rwlock_init(&ep->lock); + ep_lock_init(&ep->lock); init_waitqueue_head(&ep->wq); init_waitqueue_head(&ep->poll_wait); INIT_LIST_HEAD(&ep->rdllist); @@ -1127,6 +1159,7 @@ struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, } #endif /* CONFIG_KCMP */ +#ifndef CONFIG_PREEMPT_RT /* * Adds a new entry to the tail of the list in a lockless way, i.e. * multiple CPUs are allowed to call this function concurrently. @@ -1204,16 +1237,17 @@ static inline bool chain_epi_lockless(struct epitem *epi) return true; } +#endif /* !CONFIG_PREEMPT_RT */ /* * This is the callback that is passed to the wait queue wakeup * mechanism. It is called by the stored file descriptors when they * have events to report. * - * This callback takes a read lock in order not to contend with concurrent - * events from another file descriptor, thus all modifications to ->rdllist - * or ->ovflist are lockless. Read lock is paired with the write lock from - * ep_start/done_scan(), which stops all list modifications and guarantees - * that lists state is seen correctly. + * Under !CONFIG_PREEMPT_RT this callback takes a read lock in order not + * to contend with concurrent events from another file descriptor, thus + * all modifications to ->rdllist or ->ovflist are lockless. Read lock is + * paired with the write lock from ep_start/done_scan(), which stops all + * list modifications and guarantees that lists state is seen correctly. * * Another thing worth to mention is that ep_poll_callback() can be called * concurrently for the same @epi from different CPUs if poll table was inited @@ -1231,7 +1265,7 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v unsigned long flags; int ewake = 0; - read_lock_irqsave(&ep->lock, flags); + ep_lock_irqsave(&ep->lock, flags); ep_set_busy_poll_napi_id(epi); @@ -1260,12 +1294,25 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v * chained in ep->ovflist and requeued later on. */ if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) { +#ifdef CONFIG_PREEMPT_RT + if (epi->next == EP_UNACTIVE_PTR) { + epi->next = READ_ONCE(ep->ovflist); + WRITE_ONCE(ep->ovflist, epi); + ep_pm_stay_awake_rcu(epi); + } +#else if (chain_epi_lockless(epi)) ep_pm_stay_awake_rcu(epi); +#endif } else if (!ep_is_linked(epi)) { /* In the usual case, add event to ready list. */ +#ifdef CONFIG_PREEMPT_RT + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake_rcu(epi); +#else if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist)) ep_pm_stay_awake_rcu(epi); +#endif } /* @@ -1298,7 +1345,7 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v pwake++; out_unlock: - read_unlock_irqrestore(&ep->lock, flags); + ep_unlock_irqrestore(&ep->lock, flags); /* We have to call this outside the lock */ if (pwake) @@ -1633,7 +1680,7 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, } /* We have to drop the new item inside our item list to keep track of it */ - write_lock_irq(&ep->lock); + ep_lock_irq(&ep->lock); /* record NAPI ID of new item if present */ ep_set_busy_poll_napi_id(epi); @@ -1650,7 +1697,7 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event, pwake++; } - write_unlock_irq(&ep->lock); + ep_unlock_irq(&ep->lock); /* We have to call this outside the lock */ if (pwake) @@ -1714,7 +1761,7 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi, * list, push it inside. */ if (ep_item_poll(epi, &pt, 1)) { - write_lock_irq(&ep->lock); + ep_lock_irq(&ep->lock); if (!ep_is_linked(epi)) { list_add_tail(&epi->rdllink, &ep->rdllist); ep_pm_stay_awake(epi); @@ -1725,7 +1772,7 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi, if (waitqueue_active(&ep->poll_wait)) pwake++; } - write_unlock_irq(&ep->lock); + ep_unlock_irq(&ep->lock); } /* We have to call this outside the lock */ @@ -1958,7 +2005,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, init_wait(&wait); wait.func = ep_autoremove_wake_function; - write_lock_irq(&ep->lock); + ep_lock_irq(&ep->lock); /* * Barrierless variant, waitqueue_active() is called under * the same lock on wakeup ep_poll_callback() side, so it @@ -1977,7 +2024,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, if (!eavail) __add_wait_queue_exclusive(&ep->wq, &wait); - write_unlock_irq(&ep->lock); + ep_unlock_irq(&ep->lock); if (!eavail) timed_out = !schedule_hrtimeout_range(to, slack, @@ -1992,7 +2039,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, eavail = 1; if (!list_empty_careful(&wait.entry)) { - write_lock_irq(&ep->lock); + ep_lock_irq(&ep->lock); /* * If the thread timed out and is not on the wait queue, * it means that the thread was woken up after its @@ -2003,7 +2050,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, if (timed_out) eavail = list_empty(&wait.entry); __remove_wait_queue(&ep->wq, &wait); - write_unlock_irq(&ep->lock); + ep_unlock_irq(&ep->lock); } } } -- 2.52.0