From: Mike Kravetz mike.kravetz@oracle.com
mainline inclusion from mainline-v5.13-rc1 commit db71ef79b59bb2e78dc4df83d0e4bf6beaa5c82d category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I9SZXR CVE: NA
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...
-------------------------------------------
Commit c77c0a8ac4c5 ("mm/hugetlb: defer freeing of huge pages if in non-task context") was added to address the issue of free_huge_page being called from irq context. That commit hands off free_huge_page processing to a workqueue if !in_task. However, this doesn't cover all the cases as pointed out by 0day bot lockdep report [1].
: Possible interrupt unsafe locking scenario: : : CPU0 CPU1 : ---- ---- : lock(hugetlb_lock); : local_irq_disable(); : lock(slock-AF_INET); : lock(hugetlb_lock); : <Interrupt> : lock(slock-AF_INET);
Shakeel has later explained that this is very likely TCP TX zerocopy from hugetlb pages scenario when the networking code drops a last reference to hugetlb page while having IRQ disabled. Hugetlb freeing path doesn't disable IRQ while holding hugetlb_lock so a lock dependency chain can lead to a deadlock.
This commit addresses the issue by doing the following: - Make hugetlb_lock irq safe. This is mostly a simple process of changing spin_*lock calls to spin_*lock_irq* calls. - Make subpool lock irq safe in a similar manner. - Revert the !in_task check and workqueue handoff.
[1] https://lore.kernel.org/linux-mm/000000000000f1c03b05bc43aadc@google.com/
Link: https://lkml.kernel.org/r/20210409205254.242291-8-mike.kravetz@oracle.com Signed-off-by: Mike Kravetz mike.kravetz@oracle.com Acked-by: Michal Hocko mhocko@suse.com Reviewed-by: Muchun Song songmuchun@bytedance.com Reviewed-by: Oscar Salvador osalvador@suse.de Cc: "Aneesh Kumar K . V" aneesh.kumar@linux.ibm.com Cc: Barry Song song.bao.hua@hisilicon.com Cc: David Hildenbrand david@redhat.com Cc: David Rientjes rientjes@google.com Cc: Hillf Danton hdanton@sina.com Cc: HORIGUCHI NAOYA naoya.horiguchi@nec.com Cc: Joonsoo Kim iamjoonsoo.kim@lge.com Cc: Matthew Wilcox willy@infradead.org Cc: Miaohe Lin linmiaohe@huawei.com Cc: Mina Almasry almasrymina@google.com Cc: Peter Xu peterx@redhat.com Cc: Peter Zijlstra peterz@infradead.org Cc: Roman Gushchin guro@fb.com Cc: Shakeel Butt shakeelb@google.com Cc: Waiman Long longman@redhat.com Cc: Will Deacon will@kernel.org Signed-off-by: Andrew Morton akpm@linux-foundation.org Signed-off-by: Linus Torvalds torvalds@linux-foundation.org
Conflicts: mm/hugetlb.c mm/hugetlb_cgroup.c fs/hugetlb/inode.c [Context conflicts. Dynamic Hugetlb feature and hugetlb_checknode() also use hugetlb_lock, convert these use too.] Signed-off-by: Jinjiang Tu tujinjiang@huawei.com --- fs/hugetlbfs/inode.c | 4 +- mm/hugetlb.c | 191 +++++++++++++++++-------------------------- mm/hugetlb_cgroup.c | 8 +- 3 files changed, 80 insertions(+), 123 deletions(-)
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 014ee6533e2e..92077383f320 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -131,7 +131,7 @@ static int hugetlb_checknode(struct vm_area_struct *vma, long nr) int ret = 0; struct hstate *h = &default_hstate;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock);
nid = vma->vm_flags >> CHECKNODE_BITS;
@@ -155,7 +155,7 @@ static int hugetlb_checknode(struct vm_area_struct *vma, long nr) }
err: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); return ret; }
diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 0d8d9ab65d5a..2844458b5fc5 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -105,11 +105,12 @@ static inline void ClearPageHugeFreed(struct page *head) static int hugetlb_acct_memory(struct hstate *h, long delta, struct dhugetlb_pool *hpool);
-static inline void unlock_or_release_subpool(struct hugepage_subpool *spool) +static inline void unlock_or_release_subpool(struct hugepage_subpool *spool, + unsigned long irq_flags) { bool free = (spool->count == 0) && (spool->used_hpages == 0);
- spin_unlock(&spool->lock); + spin_unlock_irqrestore(&spool->lock, irq_flags);
/* If no pages are used, and no other handles to the subpool * remain, give up any reservations mased on minimum size and @@ -148,10 +149,12 @@ struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
void hugepage_put_subpool(struct hugepage_subpool *spool) { - spin_lock(&spool->lock); + unsigned long flags; + + spin_lock_irqsave(&spool->lock, flags); BUG_ON(!spool->count); spool->count--; - unlock_or_release_subpool(spool); + unlock_or_release_subpool(spool, flags); }
/* @@ -174,7 +177,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, if (dhugetlb_enabled && hpool) return ret;
- spin_lock(&spool->lock); + spin_lock_irq(&spool->lock);
if (spool->max_hpages != -1) { /* maximum size accounting */ if ((spool->used_hpages + delta) <= spool->max_hpages) @@ -201,7 +204,7 @@ static long hugepage_subpool_get_pages(struct hugepage_subpool *spool, }
unlock_ret: - spin_unlock(&spool->lock); + spin_unlock_irq(&spool->lock); return ret; }
@@ -215,6 +218,7 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, long delta, struct dhugetlb_pool *hpool) { long ret = delta; + unsigned long flags;
if (!spool) return delta; @@ -223,7 +227,7 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, if (dhugetlb_enabled && hpool) return ret;
- spin_lock(&spool->lock); + spin_lock_irqsave(&spool->lock, flags);
if (spool->max_hpages != -1) /* maximum size accounting */ spool->used_hpages -= delta; @@ -244,7 +248,7 @@ static long hugepage_subpool_put_pages(struct hugepage_subpool *spool, * If hugetlbfs_put_super couldn't free spool due to an outstanding * quota reference, free it now. */ - unlock_or_release_subpool(spool); + unlock_or_release_subpool(spool, flags);
return ret; } @@ -1429,7 +1433,7 @@ void free_huge_page_to_dhugetlb_pool(struct page *page, bool restore_reserve) } #endif
-static void __free_huge_page(struct page *page) +void free_huge_page(struct page *page) { /* * Can't pass hstate in here because it is called from the @@ -1440,6 +1444,7 @@ static void __free_huge_page(struct page *page) struct hugepage_subpool *spool = (struct hugepage_subpool *)page_private(page); bool restore_reserve; + unsigned long flags;
sp_kmemcg_uncharge_hpage(page); set_page_private(page, 0); @@ -1450,12 +1455,12 @@ static void __free_huge_page(struct page *page) ClearPagePrivate(page);
if (dhugetlb_enabled && PagePool(page)) { - spin_lock(&hugetlb_lock); + spin_lock_irqsave(&hugetlb_lock, flags); clear_page_huge_active(page); list_del(&page->lru); hugetlb_cgroup_uncharge_page(hstate_index(h), pages_per_huge_page(h), page); - spin_unlock(&hugetlb_lock); + spin_unlock_irqrestore(&hugetlb_lock, flags); free_huge_page_to_dhugetlb_pool(page, restore_reserve); return; } @@ -1479,7 +1484,7 @@ static void __free_huge_page(struct page *page) restore_reserve = true; }
- spin_lock(&hugetlb_lock); + spin_lock_irqsave(&hugetlb_lock, flags); clear_page_huge_active(page); hugetlb_cgroup_uncharge_page(hstate_index(h), pages_per_huge_page(h), page); @@ -1490,78 +1495,30 @@ static void __free_huge_page(struct page *page) sp_memcg_uncharge_hpage(page); ClearPageHugeTemporary(page); remove_hugetlb_page(h, page, false); - spin_unlock(&hugetlb_lock); + spin_unlock_irqrestore(&hugetlb_lock, flags); update_and_free_page(h, page); } else if (h->surplus_huge_pages_node[nid]) { /* remove the page from active list */ remove_hugetlb_page(h, page, true); - spin_unlock(&hugetlb_lock); + spin_unlock_irqrestore(&hugetlb_lock, flags); update_and_free_page(h, page); } else { arch_clear_hugepage_flags(page); enqueue_huge_page(h, page); - spin_unlock(&hugetlb_lock); + spin_unlock_irqrestore(&hugetlb_lock, flags); } }
-/* - * As free_huge_page() can be called from a non-task context, we have - * to defer the actual freeing in a workqueue to prevent potential - * hugetlb_lock deadlock. - * - * free_hpage_workfn() locklessly retrieves the linked list of pages to - * be freed and frees them one-by-one. As the page->mapping pointer is - * going to be cleared in __free_huge_page() anyway, it is reused as the - * llist_node structure of a lockless linked list of huge pages to be freed. - */ -static LLIST_HEAD(hpage_freelist); - -static void free_hpage_workfn(struct work_struct *work) -{ - struct llist_node *node; - struct page *page; - - node = llist_del_all(&hpage_freelist); - - while (node) { - page = container_of((struct address_space **)node, - struct page, mapping); - node = node->next; - __free_huge_page(page); - } -} -static DECLARE_WORK(free_hpage_work, free_hpage_workfn); - -void free_huge_page(struct page *page) -{ - /* - * Defer freeing if in non-task context to avoid hugetlb_lock deadlock. - */ - if (!in_task()) { - /* - * Only call schedule_work() if hpage_freelist is previously - * empty. Otherwise, schedule_work() had been called but the - * workfn hasn't retrieved the list yet. - */ - if (llist_add((struct llist_node *)&page->mapping, - &hpage_freelist)) - schedule_work(&free_hpage_work); - return; - } - - __free_huge_page(page); -} - static void prep_new_huge_page(struct hstate *h, struct page *page, int nid) { INIT_LIST_HEAD(&page->lru); set_compound_page_dtor(page, HUGETLB_PAGE_DTOR); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); set_hugetlb_cgroup(page, NULL); h->nr_huge_pages++; h->nr_huge_pages_node[nid]++; ClearPageHugeFreed(page); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); }
static void prep_compound_gigantic_page(struct page *page, unsigned int order) @@ -1856,7 +1813,7 @@ int dissolve_free_huge_page(struct page *page) if (page_belong_to_dynamic_hugetlb(page)) return -EBUSY;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (!PageHuge(page)) { rc = 0; goto out; @@ -1873,7 +1830,7 @@ int dissolve_free_huge_page(struct page *page) * when it is dissolved. */ if (unlikely(!PageHugeFreed(head))) { - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); cond_resched();
/* @@ -1897,12 +1854,12 @@ int dissolve_free_huge_page(struct page *page) } remove_hugetlb_page(h, page, false); h->max_huge_pages--; - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); update_and_free_page(h, head); return 0; } out: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); return rc; }
@@ -1944,16 +1901,16 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, if (hstate_is_gigantic(h)) return NULL;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) goto out_unlock; - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL); if (!page) return NULL;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); /* * We could have raced with the pool size change. * Double check that and simply deallocate the new page @@ -1963,7 +1920,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, */ if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) { SetPageHugeTemporary(page); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); put_page(page); return NULL; } else { @@ -1972,7 +1929,7 @@ static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask, }
out_unlock: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
return page; } @@ -2027,10 +1984,10 @@ struct page *alloc_huge_page_node(struct hstate *h, int nid) if (nid != NUMA_NO_NODE) gfp_mask |= __GFP_THISNODE;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (h->free_huge_pages - h->resv_huge_pages > 0) page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL, NULL); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
if (!page) { if (enable_charge_mighp) @@ -2048,18 +2005,18 @@ struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid, { gfp_t gfp_mask = htlb_alloc_mask(h);
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (h->free_huge_pages - h->resv_huge_pages > 0) { struct page *page;
page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask, NULL); if (page) { - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); return page; } } - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask); } @@ -2106,7 +2063,7 @@ static int gather_surplus_pages(struct hstate *h, long delta)
ret = -ENOMEM; retry: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); for (i = 0; i < needed; i++) { page = alloc_surplus_huge_page(h, htlb_alloc_mask(h), NUMA_NO_NODE, NULL); @@ -2123,7 +2080,7 @@ static int gather_surplus_pages(struct hstate *h, long delta) * After retaking hugetlb_lock, we need to recalculate 'needed' * because either resv_huge_pages or free_huge_pages may have changed. */ - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); needed = (h->resv_huge_pages + delta) - (h->free_huge_pages + allocated); if (needed > 0) { @@ -2161,12 +2118,12 @@ static int gather_surplus_pages(struct hstate *h, long delta) enqueue_huge_page(h, page); } free: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
/* Free unnecessary surplus pages to the buddy allocator */ list_for_each_entry_safe(page, tmp, &surplus_list, lru) put_page(page); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock);
return ret; } @@ -2216,9 +2173,9 @@ static void return_unused_surplus_pages(struct hstate *h, }
out: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); update_and_free_pages_bulk(h, &page_list); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); }
@@ -2518,18 +2475,18 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, * Use hugetlb_lock to manage the account of * hugetlb cgroup. */ - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); list_add(&page->lru, &h->hugepage_activelist); hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(hstate_vma(vma)), h_cg, page); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); goto out; } goto out_uncharge_cgroup; }
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); /* * glb_chg is passed to indicate whether or not a page must be taken * from the global free pool (global change). gbl_chg == 0 indicates @@ -2537,11 +2494,11 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, */ page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg); if (!page) { - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); page = alloc_buddy_huge_page_with_mpol(h, vma, addr); if (!page) goto out_uncharge_cgroup; - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock);; if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) { SetPagePrivate(page); h->resv_huge_pages--; @@ -2550,7 +2507,7 @@ struct page *alloc_huge_page(struct vm_area_struct *vma, /* Fall through */ } hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); out: set_page_private(page, (unsigned long)spool);
@@ -2801,9 +2758,9 @@ static void try_to_free_low(struct hstate *h, unsigned long count, }
out: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); update_and_free_pages_bulk(h, &page_list); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); } #else static inline void try_to_free_low(struct hstate *h, unsigned long count, @@ -2868,7 +2825,7 @@ static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, return h->max_huge_pages; }
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock);
/* * Check for a node specific request. @@ -2912,14 +2869,14 @@ static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, * page, free_huge_page will handle it by freeing the page * and reducing the surplus. */ - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
/* yield cpu to avoid soft lockup */ cond_resched();
ret = alloc_pool_huge_page(h, nodes_allowed, node_alloc_noretry); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (!ret) goto out;
@@ -2958,9 +2915,9 @@ static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, list_add(&page->lru, &page_list); } /* free the pages after dropping lock */ - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); update_and_free_pages_bulk(h, &page_list); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock);
while (count < persistent_huge_pages(h)) { if (!adjust_pool_surplus(h, nodes_allowed, 1)) @@ -2968,7 +2925,7 @@ static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count, } out: ret = persistent_huge_pages(h); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
NODEMASK_FREE(node_alloc_noretry);
@@ -3135,9 +3092,9 @@ static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj, if (err) return err;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); h->nr_overcommit_huge_pages = input; - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
return count; } @@ -3448,7 +3405,7 @@ int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool, return -ENOMEM;
spin_lock(&hpool->lock); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (h->free_huge_pages_node[nid] < size) { ret = -ENOMEM; goto out_unlock; @@ -3470,7 +3427,7 @@ int alloc_hugepage_from_hugetlb(struct dhugetlb_pool *hpool, } ret = 0; out_unlock: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); spin_unlock(&hpool->lock); return ret; } @@ -3835,7 +3792,7 @@ static void free_back_hugetlb(struct dhugetlb_pool *hpool) if (!h) return;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); list_for_each_entry_safe(page, page_next, &hpool->dhugetlb_1G_freelists, lru) { nr_pages = 1 << huge_page_order(h); @@ -3862,7 +3819,7 @@ static void free_back_hugetlb(struct dhugetlb_pool *hpool) hpool->free_reserved_1G = 0; hpool->total_reserved_1G = 0; INIT_LIST_HEAD(&hpool->dhugetlb_1G_freelists); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); }
bool free_dhugetlb_pool(struct dhugetlb_pool *hpool) @@ -4632,9 +4589,9 @@ int hugetlb_overcommit_handler(struct ctl_table *table, int write, goto out;
if (write) { - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); h->nr_overcommit_huge_pages = tmp; - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); } out: return ret; @@ -4731,7 +4688,7 @@ static int hugetlb_acct_memory(struct hstate *h, long delta, if (dhugetlb_enabled && hpool) return dhugetlb_acct_memory(h, delta, hpool);
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); /* * When cpuset is configured, it breaks the strict hugetlb page * reservation as the accounting is done on a global variable. Such @@ -4764,7 +4721,7 @@ static int hugetlb_acct_memory(struct hstate *h, long delta, return_unused_surplus_pages(h, (unsigned long) -delta);
out: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); return ret; }
@@ -6671,7 +6628,7 @@ bool isolate_huge_page(struct page *page, struct list_head *list) { bool ret = true;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (!PageHeadHuge(page) || !page_huge_active(page) || !get_page_unless_zero(page)) { ret = false; @@ -6680,17 +6637,17 @@ bool isolate_huge_page(struct page *page, struct list_head *list) clear_page_huge_active(page); list_move_tail(&page->lru, list); unlock: - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); return ret; }
void putback_active_hugepage(struct page *page) { VM_BUG_ON_PAGE(!PageHead(page), page); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); set_page_huge_active(page); list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); put_page(page); }
@@ -6718,12 +6675,12 @@ void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason) SetPageHugeTemporary(oldpage); ClearPageHugeTemporary(newpage);
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (h->surplus_huge_pages_node[old_nid]) { h->surplus_huge_pages_node[old_nid]--; h->surplus_huge_pages_node[new_nid]++; } - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); } }
@@ -6739,10 +6696,10 @@ static struct page *hugetlb_alloc_hugepage_normal(struct hstate *h, { struct page *page = NULL;
- spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); if (h->free_huge_pages - h->resv_huge_pages > 0) page = dequeue_huge_page_nodemask(h, gfp_mask, nid, NULL, NULL); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock);
return page; } diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index 7a93e1e439dd..13110d8ca3ea 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -167,11 +167,11 @@ static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css)
do { for_each_hstate(h) { - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); list_for_each_entry(page, &h->hugepage_activelist, lru) hugetlb_cgroup_move_parent(idx, h_cg, page);
- spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); idx++; } cond_resched(); @@ -422,14 +422,14 @@ void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) return;
VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage); - spin_lock(&hugetlb_lock); + spin_lock_irq(&hugetlb_lock); h_cg = hugetlb_cgroup_from_page(oldhpage); set_hugetlb_cgroup(oldhpage, NULL);
/* move the h_cg details to new cgroup */ set_hugetlb_cgroup(newhpage, h_cg); list_move(&newhpage->lru, &h->hugepage_activelist); - spin_unlock(&hugetlb_lock); + spin_unlock_irq(&hugetlb_lock); return; }