hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 ---------------------------------------- For each KSM candidate page, kmsd needs a struct rmap_item to store the state of the candidate page. struct rmap_item is both used when the page is trying to be merged and when the page has been merged. If the system has too many KSM candidate pages that fail to be merged, the memory consumption of metadata is even larger than the memory saving brought by merging. This patch adds a new mode called ksm_only_merge_zero_pages. When this mode is enabled, ksmd only tries to merge empty pages. This ensures ksmd could reduce memory usage without worrying that the benefits of mering will be offset by metadata overhead. Besides, this mode could increase the scanning speed. When ksm_use_zero_pages mode is enabled, the empty pages are merged with the system zero pages. System zero pages don't need reserve mapping, so the memory consumption of rmap_item metdata are not needed too. If ksm_use_zero_pages mode is enabled too when ksm_only_merge_zero_pages mode is enabled, the profit will be improved more. During ksmd walks the page table to find candidate pages, We add a trivial check to filter out nozero pages. walk_page_range_vma() needs to walk servel levels of page table to find the target pte. Too many calls of walk_page_range_vma() are too slow. So we add a trivial check to filter out nozero pages in one walk_page_range_vma() call, to speed up scanning. Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- mm/ksm.c | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/mm/ksm.c b/mm/ksm.c index e73cdde7b537..c7be71802847 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -281,6 +281,8 @@ static unsigned int zero_checksum __read_mostly; /* Whether to merge empty (zeroed) pages with actual zero pages */ static bool ksm_use_zero_pages __read_mostly; +static bool ksm_only_merge_zero_pages __read_mostly; + /* The number of zero pages which is placed by KSM */ atomic_long_t ksm_zero_pages = ATOMIC_LONG_INIT(0); @@ -2324,8 +2326,22 @@ struct ksm_next_page_arg { struct folio *folio; struct page *page; unsigned long addr; + unsigned int nr_skipped; }; +static inline bool nozero_page_inprecise_check(struct page *page) +{ + unsigned long *p; + bool ret = true; + unsigned int last_pos = PAGE_SIZE / sizeof(*p) - 1; + + p = kmap_atomic(page); + if (p[0] || p[last_pos]) + ret = false; + kunmap_atomic(p); + return ret; +} + static int ksm_next_page_pmd_entry(pmd_t *pmdp, unsigned long addr, unsigned long end, struct mm_walk *walk) { @@ -2385,6 +2401,12 @@ static int ksm_next_page_pmd_entry(pmd_t *pmdp, unsigned long addr, unsigned lon if (folio_is_zone_device(folio) || !folio_test_anon(folio)) continue; + + if (ksm_only_merge_zero_pages && private->nr_skipped--) { + if (!nozero_page_inprecise_check(page)) + continue; + } + goto found_unlock; } @@ -2409,6 +2431,40 @@ static struct mm_walk_ops ksm_next_page_ops = { .walk_lock = PGWALK_RDLOCK, }; +static bool should_skip_page(struct ksm_rmap_item **rmap_list, + unsigned long addr, struct page *page) +{ + struct ksm_rmap_item *rmap_item; + unsigned long *p; + int i; + + if (!ksm_only_merge_zero_pages) + return false; + + if (PageKsm(page)) + return false; + + p = kmap_atomic(page); + for (i = 0; i < PAGE_SIZE / sizeof(unsigned long); ++i) { + if (p[i] != 0) { + kunmap_atomic(p); + while (*rmap_list) { + rmap_item = *rmap_list; + if (rmap_item->address > addr) + break; + *rmap_list = rmap_item->rmap_list; + remove_rmap_item_from_tree(rmap_item); + free_rmap_item(rmap_item); + } + return true; + } + } + kunmap_atomic(p); + + return false; +} + + static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) { struct mm_struct *mm; @@ -2504,6 +2560,7 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) int found; + ksm_next_page_arg.nr_skipped = 1024; found = walk_page_range_vma(vma, ksm_scan.address, vma->vm_end, &ksm_next_page_ops, @@ -2520,6 +2577,14 @@ static struct ksm_rmap_item *scan_get_next_rmap_item(struct page **page) if (tmp_page) { flush_anon_page(vma, tmp_page, ksm_scan.address); flush_dcache_page(tmp_page); + + if (should_skip_page(ksm_scan.rmap_list, ksm_scan.address, tmp_page)) { + folio_put(folio); + ksm_scan.address += PAGE_SIZE; + mmap_read_unlock(mm); + return ERR_PTR(-EAGAIN); + } + rmap_item = get_next_rmap_item(mm_slot, ksm_scan.rmap_list, ksm_scan.address); if (rmap_item) { @@ -2613,6 +2678,10 @@ static void ksm_do_scan(unsigned int scan_npages) rmap_item = scan_get_next_rmap_item(&page); if (!rmap_item) return; + if (rmap_item == ERR_PTR(-EAGAIN)) { + ksm_pages_scanned++; + continue; + } cmp_and_merge_page(page, rmap_item); put_page(page); ksm_pages_scanned++; @@ -3428,6 +3497,28 @@ static ssize_t use_zero_pages_store(struct kobject *kobj, } KSM_ATTR(use_zero_pages); +static ssize_t only_merge_zero_pages_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%u\n", ksm_only_merge_zero_pages); +} +static ssize_t only_merge_zero_pages_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int err; + bool value; + + err = kstrtobool(buf, &value); + if (err) + return -EINVAL; + + ksm_only_merge_zero_pages = value; + + return count; +} +KSM_ATTR(only_merge_zero_pages); + static ssize_t max_page_sharing_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -3599,6 +3690,7 @@ static struct attribute *ksm_attrs[] = { &stable_node_dups_attr.attr, &stable_node_chains_prune_millisecs_attr.attr, &use_zero_pages_attr.attr, + &only_merge_zero_pages_attr.attr, &general_profit_attr.attr, NULL, }; -- 2.43.0