From: Roman Gushchin guro@fb.com
[ Upstream commit 63b02ef7dc4ec239df45c018ac0adbd02ba30a0c ]
Currently the memcg_params.dying flag and the corresponding workqueue used for the asynchronous deactivation of kmem_caches is synchronized using the slab_mutex.
It makes impossible to check this flag from the irq context, which will be required in order to implement asynchronous release of kmem_caches.
So let's switch over to the irq-save flavor of the spinlock-based synchronization.
Link: http://lkml.kernel.org/r/20190611231813.3148843-8-guro@fb.com Signed-off-by: Roman Gushchin guro@fb.com Acked-by: Vladimir Davydov vdavydov.dev@gmail.com Reviewed-by: Shakeel Butt shakeelb@google.com Cc: Christoph Lameter cl@linux.com Cc: Johannes Weiner hannes@cmpxchg.org Cc: Michal Hocko mhocko@suse.com Cc: Waiman Long longman@redhat.com Cc: David Rientjes rientjes@google.com Cc: Joonsoo Kim iamjoonsoo.kim@lge.com Cc: Pekka Enberg penberg@kernel.org Cc: Andrei Vagin avagin@gmail.com Cc: Qian Cai cai@lca.pw Signed-off-by: Andrew Morton akpm@linux-foundation.org Signed-off-by: Linus Torvalds torvalds@linux-foundation.org Signed-off-by: Sasha Levin sashal@kernel.org --- mm/slab_common.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/mm/slab_common.c b/mm/slab_common.c index b5776b1301f0..3dbf693527dd 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -130,6 +130,7 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, #ifdef CONFIG_MEMCG_KMEM
LIST_HEAD(slab_root_caches); +static DEFINE_SPINLOCK(memcg_kmem_wq_lock);
void slab_init_memcg_params(struct kmem_cache *s) { @@ -717,14 +718,22 @@ void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s, WARN_ON_ONCE(s->memcg_params.deact_fn)) return;
+ /* + * memcg_kmem_wq_lock is used to synchronize memcg_params.dying + * flag and make sure that no new kmem_cache deactivation tasks + * are queued (see flush_memcg_workqueue() ). + */ + spin_lock_irq(&memcg_kmem_wq_lock); if (s->memcg_params.root_cache->memcg_params.dying) - return; + goto unlock;
/* pin memcg so that @s doesn't get destroyed in the middle */ css_get(&s->memcg_params.memcg->css);
s->memcg_params.deact_fn = deact_fn; call_rcu_sched(&s->memcg_params.deact_rcu_head, kmemcg_deactivate_rcufn); +unlock: + spin_unlock_irq(&memcg_kmem_wq_lock); }
void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg) @@ -834,9 +843,9 @@ static int shutdown_memcg_caches(struct kmem_cache *s)
static void flush_memcg_workqueue(struct kmem_cache *s) { - mutex_lock(&slab_mutex); + spin_lock_irq(&memcg_kmem_wq_lock); s->memcg_params.dying = true; - mutex_unlock(&slab_mutex); + spin_unlock_irq(&memcg_kmem_wq_lock);
/* * SLUB deactivates the kmem_caches through call_rcu_sched. Make