From: xu xin <xu.xin16@zte.com.cn> mainline inclusion from mainline-v5.19-rc1 commit 7609385337a4feb6236e42dcd0df2185683ce839 category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- Some applications or containers want to use KSM by calling madvise() to advise areas of address space to be MERGEABLE. But they may not know which applications are more likely to cause real merges in the deployment. If this patch is applied, it helps them know their corresponding number of merged pages, and then optimize their app code. As current KSM only counts the number of KSM merging pages(e.g. ksm_pages_sharing and ksm_pages_shared) of the whole system, we cannot see the more fine-grained KSM merging, for the upper application optimization, the merging area cannot be set easily according to the KSM page merging probability of each process. Therefore, it is necessary to add extra statistical means so that the upper level users can know the detailed KSM merging information of each process. We add a new proc file named as ksm_merging_pages under /proc/<pid>/ to indicate the involved ksm merging pages of this process. [akpm@linux-foundation.org: fix comment typo, remove BUG_ON()s] Link: https://lkml.kernel.org/r/20220325082318.2352853-1-xu.xin16@zte.com.cn Signed-off-by: xu xin <xu.xin16@zte.com.cn> Reported-by: kernel test robot <lkp@intel.com> Reviewed-by: Yang Yang <yang.yang29@zte.com.cn> Reviewed-by: Ran Xiaokai <ran.xiaokai@zte.com.cn> Reported-by: Zeal Robot <zealci@zte.com.cn> Cc: Kees Cook <keescook@chromium.org> Cc: Alexey Dobriyan <adobriyan@gmail.com> Cc: Stephen Rothwell <sfr@canb.auug.org.au> Cc: Ohhoon Kwon <ohoono.kwon@samsung.com> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Cc: Stephen Brennan <stephen.s.brennan@oracle.com> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Feng Tang <feng.tang@intel.com> Cc: Yang Yang <yang.yang29@zte.com.cn> Cc: Ran Xiaokai <ran.xiaokai@zte.com.cn> Cc: Zeal Robot <zealci@zte.com.cn> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: fs/proc/base.c include/linux/mm_types.h [Context conflicts.] Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- fs/proc/base.c | 22 ++++++++++++++++++++++ include/linux/mm_types.h | 8 ++++++++ mm/ksm.c | 8 ++++++++ 3 files changed, 38 insertions(+) diff --git a/fs/proc/base.c b/fs/proc/base.c index dc9841826264..1807474532d1 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3042,6 +3042,22 @@ static int proc_pid_patch_state(struct seq_file *m, struct pid_namespace *ns, } #endif /* CONFIG_LIVEPATCH */ +#ifdef CONFIG_KSM +static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct mm_struct *mm; + + mm = get_task_mm(task); + if (mm) { + seq_printf(m, "%lu\n", mm->ksm_merging_pages); + mmput(mm); + } + + return 0; +} +#endif /* CONFIG_KSM */ + /* * Thread groups */ @@ -3151,6 +3167,9 @@ static const struct pid_entry tgid_base_stuff[] = { #ifdef CONFIG_ASCEND_SHARE_POOL ONE("sp_group", S_IRUGO, proc_sp_group_state), #endif +#ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), +#endif }; static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx) @@ -3537,6 +3556,9 @@ static const struct pid_entry tid_base_stuff[] = { #ifdef CONFIG_ASCEND_SHARE_POOL ONE("sp_group", S_IRUGO, proc_sp_group_state), #endif +#ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), +#endif }; static int proc_tid_base_readdir(struct file *file, struct dir_context *ctx) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 0a46332b79d9..5bba4140936c 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -564,7 +564,15 @@ struct mm_struct { KABI_RESERVE(3) #endif +#if defined(CONFIG_KSM) && !defined(__GENKSYMS__) + /* + * Represent how many pages of this process are involved in KSM + * merging. + */ + unsigned long ksm_merging_pages; +#else KABI_RESERVE(4) +#endif KABI_RESERVE(5) KABI_RESERVE(6) KABI_RESERVE(7) diff --git a/mm/ksm.c b/mm/ksm.c index fc6b6526099f..f45d0178bd05 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -641,6 +641,9 @@ static void remove_node_from_stable_tree(struct stable_node *stable_node) ksm_pages_sharing--; else ksm_pages_shared--; + + rmap_item->mm->ksm_merging_pages--; + VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node->rmap_hlist_len--; put_anon_vma(rmap_item->anon_vma); @@ -789,6 +792,9 @@ static void remove_rmap_item_from_tree(struct rmap_item *rmap_item) ksm_pages_sharing--; else ksm_pages_shared--; + + rmap_item->mm->ksm_merging_pages--; + VM_BUG_ON(stable_node->rmap_hlist_len <= 0); stable_node->rmap_hlist_len--; @@ -2043,6 +2049,8 @@ static void stable_tree_append(struct rmap_item *rmap_item, ksm_pages_sharing++; else ksm_pages_shared++; + + rmap_item->mm->ksm_merging_pages++; } /* -- 2.43.0

From: Stefan Roesch <shr@devkernel.io> mainline inclusion from mainline-v6.4-rc1 commit d7597f59d1d33e9efbffa7060deb9ee5bd119e62 category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- Patch series "mm: process/cgroup ksm support", v9. So far KSM can only be enabled by calling madvise for memory regions. To be able to use KSM for more workloads, KSM needs to have the ability to be enabled / disabled at the process / cgroup level. Use case 1: The madvise call is not available in the programming language. An example for this are programs with forked workloads using a garbage collected language without pointers. In such a language madvise cannot be made available. In addition the addresses of objects get moved around as they are garbage collected. KSM sharing needs to be enabled "from the outside" for these type of workloads. Use case 2: The same interpreter can also be used for workloads where KSM brings no benefit or even has overhead. We'd like to be able to enable KSM on a workload by workload basis. Use case 3: With the madvise call sharing opportunities are only enabled for the current process: it is a workload-local decision. A considerable number of sharing opportunities may exist across multiple workloads or jobs (if they are part of the same security domain). Only a higler level entity like a job scheduler or container can know for certain if its running one or more instances of a job. That job scheduler however doesn't have the necessary internal workload knowledge to make targeted madvise calls. Security concerns: In previous discussions security concerns have been brought up. The problem is that an individual workload does not have the knowledge about what else is running on a machine. Therefore it has to be very conservative in what memory areas can be shared or not. However, if the system is dedicated to running multiple jobs within the same security domain, its the job scheduler that has the knowledge that sharing can be safely enabled and is even desirable. Performance: Experiments with using UKSM have shown a capacity increase of around 20%. Here are the metrics from an instagram workload (taken from a machine with 64GB main memory): full_scans: 445 general_profit: 20158298048 max_page_sharing: 256 merge_across_nodes: 1 pages_shared: 129547 pages_sharing: 5119146 pages_to_scan: 4000 pages_unshared: 1760924 pages_volatile: 10761341 run: 1 sleep_millisecs: 20 stable_node_chains: 167 stable_node_chains_prune_millisecs: 2000 stable_node_dups: 2751 use_zero_pages: 0 zero_pages_sharing: 0 After the service is running for 30 minutes to an hour, 4 to 5 million shared pages are common for this workload when using KSM. Detailed changes: 1. New options for prctl system command This patch series adds two new options to the prctl system call. The first one allows to enable KSM at the process level and the second one to query the setting. The setting will be inherited by child processes. With the above setting, KSM can be enabled for the seed process of a cgroup and all processes in the cgroup will inherit the setting. 2. Changes to KSM processing When KSM is enabled at the process level, the KSM code will iterate over all the VMA's and enable KSM for the eligible VMA's. When forking a process that has KSM enabled, the setting will be inherited by the new child process. 3. Add general_profit metric The general_profit metric of KSM is specified in the documentation, but not calculated. This adds the general profit metric to /sys/kernel/debug/mm/ksm. 4. Add more metrics to ksm_stat This adds the process profit metric to /proc/<pid>/ksm_stat. 5. Add more tests to ksm_tests and ksm_functional_tests This adds an option to specify the merge type to the ksm_tests. This allows to test madvise and prctl KSM. It also adds a two new tests to ksm_functional_tests: one to test the new prctl options and the other one is a fork test to verify that the KSM process setting is inherited by client processes. This patch (of 3): So far KSM can only be enabled by calling madvise for memory regions. To be able to use KSM for more workloads, KSM needs to have the ability to be enabled / disabled at the process / cgroup level. 1. New options for prctl system command This patch series adds two new options to the prctl system call. The first one allows to enable KSM at the process level and the second one to query the setting. The setting will be inherited by child processes. With the above setting, KSM can be enabled for the seed process of a cgroup and all processes in the cgroup will inherit the setting. 2. Changes to KSM processing When KSM is enabled at the process level, the KSM code will iterate over all the VMA's and enable KSM for the eligible VMA's. When forking a process that has KSM enabled, the setting will be inherited by the new child process. 1) Introduce new MMF_VM_MERGE_ANY flag This introduces the new flag MMF_VM_MERGE_ANY flag. When this flag is set, kernel samepage merging (ksm) gets enabled for all vma's of a process. 2) Setting VM_MERGEABLE on VMA creation When a VMA is created, if the MMF_VM_MERGE_ANY flag is set, the VM_MERGEABLE flag will be set for this VMA. 3) support disabling of ksm for a process This adds the ability to disable ksm for a process if ksm has been enabled for the process with prctl. 4) add new prctl option to get and set ksm for a process This adds two new options to the prctl system call - enable ksm for all vmas of a process (if the vmas support it). - query if ksm has been enabled for a process. 3. Disabling MMF_VM_MERGE_ANY for storage keys in s390 In the s390 architecture when storage keys are used, the MMF_VM_MERGE_ANY will be disabled. Link: https://lkml.kernel.org/r/20230418051342.1919757-1-shr@devkernel.io Link: https://lkml.kernel.org/r/20230418051342.1919757-2-shr@devkernel.io Signed-off-by: Stefan Roesch <shr@devkernel.io> Acked-by: David Hildenbrand <david@redhat.com> Cc: David Hildenbrand <david@redhat.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Michal Hocko <mhocko@suse.com> Cc: Rik van Riel <riel@surriel.com> Cc: Bagas Sanjaya <bagasdotme@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: arch/s390/mm/gmap.c include/linux/ksm.h include/linux/sched/coredump.h include/uapi/linux/prctl.h kernel/sys.c mm/ksm.c mm/mmap.c Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- include/linux/ksm.h | 20 +++++- include/linux/sched/coredump.h | 1 + include/uapi/linux/prctl.h | 2 + kernel/sys.c | 27 ++++++++ mm/ksm.c | 110 +++++++++++++++++++++++++++------ mm/mmap.c | 3 + 6 files changed, 141 insertions(+), 22 deletions(-) diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 161e8164abcf..a7cfcab99b0d 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -21,13 +21,26 @@ struct mem_cgroup; #ifdef CONFIG_KSM int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags); + +void ksm_add_vma(struct vm_area_struct *vma); +int ksm_enable_merge_any(struct mm_struct *mm); + int __ksm_enter(struct mm_struct *mm); void __ksm_exit(struct mm_struct *mm); static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { - if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) - return __ksm_enter(mm); + int ret; + + if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) { + ret = __ksm_enter(mm); + if (ret) + return ret; + } + + if (test_bit(MMF_VM_MERGE_ANY, &oldmm->flags)) + set_bit(MMF_VM_MERGE_ANY, &mm->flags); + return 0; } @@ -55,6 +68,9 @@ void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc); void ksm_migrate_page(struct page *newpage, struct page *oldpage); #else /* !CONFIG_KSM */ +static inline void ksm_add_vma(struct vm_area_struct *vma) +{ +} static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h index dfd82eab2902..6a4d85c7a5f3 100644 --- a/include/linux/sched/coredump.h +++ b/include/linux/sched/coredump.h @@ -78,4 +78,5 @@ static inline int get_dumpable(struct mm_struct *mm) #define MMF_INIT_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ MMF_DISABLE_THP_MASK) +#define MMF_VM_MERGE_ANY 29 #endif /* _LINUX_SCHED_COREDUMP_H */ diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h index b17201edfa09..511003727252 100644 --- a/include/uapi/linux/prctl.h +++ b/include/uapi/linux/prctl.h @@ -220,4 +220,6 @@ struct prctl_mm_map { # define PR_SPEC_DISABLE (1UL << 2) # define PR_SPEC_FORCE_DISABLE (1UL << 3) +#define PR_SET_MEMORY_MERGE 67 +#define PR_GET_MEMORY_MERGE 68 #endif /* _LINUX_PRCTL_H */ diff --git a/kernel/sys.c b/kernel/sys.c index b088b71732b7..6b1958150a49 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -14,6 +14,7 @@ #include <linux/highuid.h> #include <linux/fs.h> #include <linux/kmod.h> +#include <linux/ksm.h> #include <linux/perf_event.h> #include <linux/resource.h> #include <linux/kernel.h> @@ -2480,6 +2481,32 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3, return -EINVAL; error = arch_prctl_spec_ctrl_set(me, arg2, arg3); break; +#ifdef CONFIG_KSM + case PR_SET_MEMORY_MERGE: + if (arg3 || arg4 || arg5) + return -EINVAL; + if (down_write_killable(&me->mm->mmap_sem)) + return -EINTR; + + if (arg2) { + error = ksm_enable_merge_any(me->mm); + } else { + /* + * TODO: we might want disable KSM on all VMAs and + * trigger unsharing to completely disable KSM. + */ + clear_bit(MMF_VM_MERGE_ANY, &me->mm->flags); + error = 0; + } + up_write(&me->mm->mmap_sem); + break; + case PR_GET_MEMORY_MERGE: + if (arg2 || arg3 || arg4 || arg5) + return -EINVAL; + + error = !!test_bit(MMF_VM_MERGE_ANY, &me->mm->flags); + break; +#endif default: error = -EINVAL; break; diff --git a/mm/ksm.c b/mm/ksm.c index 82eafea0c1b7..d0624f645ca3 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -518,6 +518,31 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) return (ret & VM_FAULT_OOM) ? -ENOMEM : 0; } +static bool vma_ksm_compatible(struct vm_area_struct *vma) +{ + if (vma->vm_flags & (VM_SHARED | VM_MAYSHARE | VM_PFNMAP | + VM_IO | VM_DONTEXPAND | VM_HUGETLB | + VM_MIXEDMAP)) + return false; /* just ignore the advice */ + + if (vma_is_dax(vma)) + return false; +#ifdef CONFIG_COHERENT_DEVICE + if (is_cdm_vma(vma)) + return false; +#endif +#ifdef VM_SAO + if (*vm_flags & VM_SAO) + return false; +#endif +#ifdef VM_SPARC_ADI + if (*vm_flags & VM_SPARC_ADI) + return false; +#endif + + return true; +} + static struct vm_area_struct *find_mergeable_vma(struct mm_struct *mm, unsigned long addr) { @@ -1009,6 +1034,7 @@ static int unmerge_and_remove_all_rmap_items(void) free_mm_slot(mm_slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); mmdrop(mm); } else spin_unlock(&ksm_mmlist_lock); @@ -2389,6 +2415,7 @@ static struct rmap_item *scan_get_next_rmap_item(struct page **page) free_mm_slot(slot); clear_bit(MMF_VM_MERGEABLE, &mm->flags); + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); up_read(&mm->mmap_sem); mmdrop(mm); } else { @@ -2461,6 +2488,65 @@ static int ksm_scan_thread(void *nothing) return 0; } +static void __ksm_add_vma(struct vm_area_struct *vma) +{ + unsigned long vm_flags = vma->vm_flags; + + if (vm_flags & VM_MERGEABLE) + return; + + if (vma_ksm_compatible(vma)) + vma->vm_flags |= VM_MERGEABLE; +} + +/** + * ksm_add_vma - Mark vma as mergeable if compatible + * + * @vma: Pointer to vma + */ +void ksm_add_vma(struct vm_area_struct *vma) +{ + struct mm_struct *mm = vma->vm_mm; + + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + __ksm_add_vma(vma); +} + +static void ksm_add_vmas(struct mm_struct *mm) +{ + struct vm_area_struct *vma; + + for (vma = mm->mmap; vma; vma = vma->vm_next) + __ksm_add_vma(vma); +} + +/** + * ksm_enable_merge_any - Add mm to mm ksm list and enable merging on all + * compatible VMA's + * + * @mm: Pointer to mm + * + * Returns 0 on success, otherwise error code + */ +int ksm_enable_merge_any(struct mm_struct *mm) +{ + int err; + + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + return 0; + + if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { + err = __ksm_enter(mm); + if (err) + return err; + } + + set_bit(MMF_VM_MERGE_ANY, &mm->flags); + ksm_add_vmas(mm); + + return 0; +} + int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, unsigned long *vm_flags) { @@ -2469,28 +2555,11 @@ int ksm_madvise(struct vm_area_struct *vma, unsigned long start, switch (advice) { case MADV_MERGEABLE: - /* - * Be somewhat over-protective for now! - */ - if (*vm_flags & (VM_MERGEABLE | VM_SHARED | VM_MAYSHARE | - VM_PFNMAP | VM_IO | VM_DONTEXPAND | - VM_HUGETLB | VM_MIXEDMAP)) - return 0; /* just ignore the advice */ - - if (vma_is_dax(vma)) - return 0; -#ifdef CONFIG_COHERENT_DEVICE - if (is_cdm_vma(vma)) + if (vma->vm_flags & VM_MERGEABLE) return 0; -#endif -#ifdef VM_SAO - if (*vm_flags & VM_SAO) - return 0; -#endif -#ifdef VM_SPARC_ADI - if (*vm_flags & VM_SPARC_ADI) + + if (!vma_ksm_compatible(vma)) return 0; -#endif if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) { err = __ksm_enter(mm); @@ -2587,6 +2656,7 @@ void __ksm_exit(struct mm_struct *mm) if (easy_to_free) { free_mm_slot(mm_slot); + clear_bit(MMF_VM_MERGE_ANY, &mm->flags); clear_bit(MMF_VM_MERGEABLE, &mm->flags); mmdrop(mm); } else if (mm_slot) { diff --git a/mm/mmap.c b/mm/mmap.c index ad643f9dd3a4..3c4401602233 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -48,6 +48,7 @@ #include <linux/sched/mm.h> #include <linux/swapops.h> #include <linux/share_pool.h> +#include <linux/ksm.h> #include <linux/uaccess.h> #include <asm/cacheflush.h> @@ -2045,6 +2046,7 @@ static unsigned long __mmap_region(struct mm_struct *mm, struct file *file, allow_write_access(file); } file = vma->vm_file; + ksm_add_vma(vma); out: perf_event_mmap(vma); @@ -3401,6 +3403,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long fla vma->vm_flags = flags; vma->vm_page_prot = vm_get_page_prot(flags); vma_link(mm, vma, prev, rb_link, rb_parent); + ksm_add_vma(vma); out: perf_event_mmap(vma); mm->total_vm += len >> PAGE_SHIFT; -- 2.43.0

From: Stefan Roesch <shr@devkernel.io> mainline inclusion from mainline-v6.4-rc1 commit d21077fbc2fc987c2e593c34dc3b4d84e546dc9f category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- This adds the general_profit KSM sysfs knob and the process profit metric knobs to ksm_stat. 1) expose general_profit metric The documentation mentions a general profit metric, however this metric is not calculated. In addition the formula depends on the size of internal structures, which makes it more difficult for an administrator to make the calculation. Adding the metric for a better user experience. 2) document general_profit sysfs knob 3) calculate ksm process profit metric The ksm documentation mentions the process profit metric and how to calculate it. This adds the calculation of the metric. 4) mm: expose ksm process profit metric in ksm_stat This exposes the ksm process profit metric in /proc/<pid>/ksm_stat. The documentation mentions the formula for the ksm process profit metric, however it does not calculate it. In addition the formula depends on the size of internal structures. So it makes sense to expose it. 5) document new procfs ksm knobs Link: https://lkml.kernel.org/r/20230418051342.1919757-3-shr@devkernel.io Signed-off-by: Stefan Roesch <shr@devkernel.io> Reviewed-by: Bagas Sanjaya <bagasdotme@gmail.com> Acked-by: David Hildenbrand <david@redhat.com> Cc: David Hildenbrand <david@redhat.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Michal Hocko <mhocko@suse.com> Cc: Rik van Riel <riel@surriel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: Documentation/admin-guide/mm/ksm.rst fs/proc/base.c include/linux/ksm.h Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- Documentation/ABI/testing/sysfs-kernel-mm-ksm | 8 +++++++ Documentation/admin-guide/mm/ksm.rst | 2 ++ fs/proc/base.c | 3 +++ include/linux/ksm.h | 4 ++++ mm/ksm.c | 21 +++++++++++++++++++ 5 files changed, 38 insertions(+) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-ksm b/Documentation/ABI/testing/sysfs-kernel-mm-ksm index dfc13244cda3..e1352bdd7688 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-ksm +++ b/Documentation/ABI/testing/sysfs-kernel-mm-ksm @@ -50,3 +50,11 @@ Description: Control merging pages across different NUMA nodes. When it is set to 0 only pages from the same node are merged, otherwise pages from all nodes can be merged together (default). + +What: /sys/kernel/mm/ksm/general_profit +Date: April 2023 +KernelVersion: 6.4 +Contact: Linux memory management mailing list <linux-mm@kvack.org> +Description: Measure how effective KSM is. + general_profit: how effective is KSM. The formula for the + calculation is in Documentation/admin-guide/mm/ksm.rst. diff --git a/Documentation/admin-guide/mm/ksm.rst b/Documentation/admin-guide/mm/ksm.rst index 9303786632d1..153c04dc0003 100644 --- a/Documentation/admin-guide/mm/ksm.rst +++ b/Documentation/admin-guide/mm/ksm.rst @@ -159,6 +159,8 @@ stable_node_chains_prune_millisecs The effectiveness of KSM and MADV_MERGEABLE is shown in ``/sys/kernel/mm/ksm/``: +general_profit + how effective is KSM. The calculation is explained below. pages_shared how many shared pages are being used pages_sharing diff --git a/fs/proc/base.c b/fs/proc/base.c index 6b1622c2085e..64d471fa99b3 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -95,6 +95,7 @@ #include <linux/flex_array.h> #include <linux/posix-timers.h> #include <linux/share_pool.h> +#include <linux/ksm.h> #include <trace/events/oom.h> #include "internal.h" #include "fd.h" @@ -3064,6 +3065,8 @@ static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, mm = get_task_mm(task); if (mm) { seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); + seq_printf(m, "ksm_merging_pages %lu\n", mm->ksm_merging_pages); + seq_printf(m, "ksm_process_profit %ld\n", ksm_process_profit(mm)); mmput(mm); } diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 99a4484ac37c..1e99aff30cdb 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -68,6 +68,10 @@ struct page *ksm_might_need_to_copy(struct page *page, void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc); void ksm_migrate_page(struct page *newpage, struct page *oldpage); +#ifdef CONFIG_PROC_FS +long ksm_process_profit(struct mm_struct *); +#endif /* CONFIG_PROC_FS */ + #else /* !CONFIG_KSM */ static inline void ksm_add_vma(struct vm_area_struct *vma) { diff --git a/mm/ksm.c b/mm/ksm.c index ac1ecc5d85ed..d77fa56ec369 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -2974,6 +2974,14 @@ static void wait_while_offlining(void) } #endif /* CONFIG_MEMORY_HOTREMOVE */ +#ifdef CONFIG_PROC_FS +long ksm_process_profit(struct mm_struct *mm) +{ + return mm->ksm_merging_pages * PAGE_SIZE - + mm->ksm_rmap_items * sizeof(struct rmap_item); +} +#endif /* CONFIG_PROC_FS */ + #ifdef CONFIG_SYSFS /* * This all compiles without CONFIG_SYSFS, but is a waste of space. @@ -3238,6 +3246,18 @@ static ssize_t pages_volatile_show(struct kobject *kobj, } KSM_ATTR_RO(pages_volatile); +static ssize_t general_profit_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + long general_profit; + + general_profit = ksm_pages_sharing * PAGE_SIZE - + ksm_rmap_items * sizeof(struct rmap_item); + + return sysfs_emit(buf, "%ld\n", general_profit); +} +KSM_ATTR_RO(general_profit); + static ssize_t stable_node_dups_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { @@ -3302,6 +3322,7 @@ static struct attribute *ksm_attrs[] = { &stable_node_dups_attr.attr, &stable_node_chains_prune_millisecs_attr.attr, &use_zero_pages_attr.attr, + &general_profit_attr.attr, NULL, }; -- 2.43.0

mainline inclusion from mainline-v6.10-rc1 commit 3a9e567ca45fb5280065283d10d9a11f0db61d2b category: bugfix bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- Patch series "mm/ksm: fix ksm exec support for prctl", v4. commit 3c6f33b7273a ("mm/ksm: support fork/exec for prctl") inherits MMF_VM_MERGE_ANY flag when a task calls execve(). However, it doesn't create the mm_slot, so ksmd will not try to scan this task. The first patch fixes the issue. The second patch refactors to prepare for the third patch. The third patch extends the selftests of ksm to verfity the deduplication really happens after fork/exec inherits ths KSM setting. This patch (of 3): commit 3c6f33b7273a ("mm/ksm: support fork/exec for prctl") inherits MMF_VM_MERGE_ANY flag when a task calls execve(). Howerver, it doesn't create the mm_slot, so ksmd will not try to scan this task. To fix it, allocate and add the mm_slot to ksm_mm_head in __bprm_mm_init() when the mm has MMF_VM_MERGE_ANY flag. Link: https://lkml.kernel.org/r/20240328111010.1502191-1-tujinjiang@huawei.com Link: https://lkml.kernel.org/r/20240328111010.1502191-2-tujinjiang@huawei.com Fixes: 3c6f33b7273a ("mm/ksm: support fork/exec for prctl") Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> Reviewed-by: David Hildenbrand <david@redhat.com> Cc: Johannes Weiner <hannes@cmpxchg.org> Cc: Kefeng Wang <wangkefeng.wang@huawei.com> Cc: Nanyong Sun <sunnanyong@huawei.com> Cc: Rik van Riel <riel@surriel.com> Cc: Stefan Roesch <shr@devkernel.io> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: fs/exec.c Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- fs/exec.c | 13 +++++++++++++ include/linux/ksm.h | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/fs/exec.c b/fs/exec.c index 1d66dd0c759e..13ad06761780 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -62,6 +62,9 @@ #include <linux/oom.h> #include <linux/compat.h> #include <linux/vmalloc.h> +#ifndef __GENKSYMS__ +#include <linux/ksm.h> +#endif #include <linux/uaccess.h> #include <asm/mmu_context.h> @@ -300,6 +303,14 @@ static int __bprm_mm_init(struct linux_binprm *bprm) goto err_free; } + /* + * Need to be called with mmap write lock + * held, to avoid race with ksmd. + */ + err = ksm_execve(mm); + if (err) + goto err_ksm; + /* * Place the stack at the largest stack address the architecture * supports. Later, we'll move this to an appropriate place. We don't @@ -322,6 +333,8 @@ static int __bprm_mm_init(struct linux_binprm *bprm) bprm->p = vma->vm_end - sizeof(void *); return 0; err: + ksm_exit(mm); +err_ksm: up_write(&mm->mmap_sem); err_free: bprm->vma = NULL; diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 1e99aff30cdb..c5de58e80984 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -45,6 +45,14 @@ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) return 0; } +static inline int ksm_execve(struct mm_struct *mm) +{ + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + return __ksm_enter(mm); + + return 0; +} + static inline void ksm_exit(struct mm_struct *mm) { if (test_bit(MMF_VM_MERGEABLE, &mm->flags)) @@ -82,6 +90,11 @@ static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) return 0; } +static inline int ksm_execve(struct mm_struct *mm) +{ + return 0; +} + static inline void ksm_exit(struct mm_struct *mm) { } -- 2.43.0

From: xu xin <xu.xin16@zte.com.cn> mainline inclusion from mainline-v6.6-rc1 commit 79271476b3362a9e69adae949a520647f8af3559 category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- Patch series "ksm: support tracking KSM-placed zero-pages", v10. The core idea of this patch set is to enable users to perceive the number of any pages merged by KSM, regardless of whether use_zero_page switch has been turned on, so that users can know how much free memory increase is really due to their madvise(MERGEABLE) actions. But the problem is, when enabling use_zero_pages, all empty pages will be merged with kernel zero pages instead of with each other as use_zero_pages is disabled, and then these zero-pages are no longer monitored by KSM. The motivations to do this is seen at: https://lore.kernel.org/lkml/202302100915227721315@zte.com.cn/ In one word, we hope to implement the support for KSM-placed zero pages tracking without affecting the feature of use_zero_pages, so that app developer can also benefit from knowing the actual KSM profit by getting KSM-placed zero pages to optimize applications eventually when /sys/kernel/mm/ksm/use_zero_pages is enabled. This patch (of 5): When use_zero_pages of ksm is enabled, madvise(addr, len, MADV_UNMERGEABLE) and other ways (like write 2 to /sys/kernel/mm/ksm/run) to trigger unsharing will *not* actually unshare the shared zeropage as placed by KSM (which is against the MADV_UNMERGEABLE documentation). As these KSM-placed zero pages are out of the control of KSM, the related counts of ksm pages don't expose how many zero pages are placed by KSM (these special zero pages are different from those initially mapped zero pages, because the zero pages mapped to MADV_UNMERGEABLE areas are expected to be a complete and unshared page). To not blindly unshare all shared zero_pages in applicable VMAs, the patch use pte_mkdirty (related with architecture) to mark KSM-placed zero pages. Thus, MADV_UNMERGEABLE will only unshare those KSM-placed zero pages. In addition, we'll reuse this mechanism to reliably identify KSM-placed ZeroPages to properly account for them (e.g., calculating the KSM profit that includes zeropages) in the latter patches. The patch will not degrade the performance of use_zero_pages as it doesn't change the way of merging empty pages in use_zero_pages's feature. Link: https://lkml.kernel.org/r/202306131104554703428@zte.com.cn Link: https://lkml.kernel.org/r/20230613030928.185882-1-yang.yang29@zte.com.cn Signed-off-by: xu xin <xu.xin16@zte.com.cn> Acked-by: David Hildenbrand <david@redhat.com> Cc: Claudio Imbrenda <imbrenda@linux.ibm.com> Cc: Xuexin Jiang <jiang.xuexin@zte.com.cn> Reviewed-by: Xiaokai Ran <ran.xiaokai@zte.com.cn> Reviewed-by: Yang Yang <yang.yang29@zte.com.cn> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: mm/ksm.c Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- include/linux/ksm.h | 6 ++++++ mm/ksm.c | 31 ++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 286944d61a4a..63cd98cd75bf 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -28,6 +28,12 @@ int ksm_disable_merge_any(struct mm_struct *mm); int __ksm_enter(struct mm_struct *mm); void __ksm_exit(struct mm_struct *mm); +/* + * To identify zeropages that were mapped by KSM, we reuse the dirty bit + * in the PTE. If the PTE is dirty, the zeropage was mapped by KSM when + * deduplicating memory. + */ +#define is_ksm_zero_pte(pte) (is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte)) static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { diff --git a/mm/ksm.c b/mm/ksm.c index d77fa56ec369..63b3b0b23e78 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -454,6 +454,27 @@ static inline bool ksm_test_exit(struct mm_struct *mm) return atomic_read(&mm->mm_users) == 0; } +static inline bool is_ksm_zero_page(struct mm_struct *mm, struct page *page, + unsigned long addr) +{ + pmd_t *pmd; + pte_t *ptep; + spinlock_t *ptl; + bool is_dirty; + + if (!is_zero_pfn(page_to_pfn(page))) + return false; + + pmd = mm_find_pmd(mm, addr); + if (!pmd) + return false; + + ptep = pte_offset_map_lock(mm, pmd, addr, &ptl); + is_dirty = !!pte_dirty(*ptep); + pte_unmap_unlock(ptep, ptl); + return is_dirty; +} + /* * We use break_ksm to break COW on a ksm page: it's a stripped down * @@ -480,7 +501,7 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr) FOLL_GET | FOLL_MIGRATION | FOLL_REMOTE); if (IS_ERR_OR_NULL(page)) break; - if (PageKsm(page)) + if (PageKsm(page) || is_ksm_zero_page(vma->vm_mm, page, addr)) ret = handle_mm_fault(vma, addr, FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE); else @@ -1208,8 +1229,12 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, page_add_anon_rmap(kpage, vma, addr, false); newpte = mk_pte(kpage, vma->vm_page_prot); } else { - newpte = pte_mkspecial(pfn_pte(page_to_pfn(kpage), - vma->vm_page_prot)); + /* + * Use pte_mkdirty to mark the zero page mapped by KSM, and then + * we can easily track all KSM-placed zero pages by checking if + * the dirty bit in zero page's PTE is set. + */ + newpte = pte_mkdirty(pte_mkspecial(pfn_pte(page_to_pfn(kpage), vma->vm_page_prot))); /* * We're replacing an anonymous page with a zero page, which is * not anonymous. We need to do proper accounting otherwise we -- 2.43.0

From: xu xin <xu.xin16@zte.com.cn> mainline inclusion from mainline-v6.6-rc1 commit 6080d19f07043ade61094d0f58b14c05e1694a39 category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- As the number of ksm zero pages is not included in ksm_merging_pages per process when enabling use_zero_pages, it's unclear of how many actual pages are merged by KSM. To let users accurately estimate their memory demands when unsharing KSM zero-pages, it's necessary to show KSM zero- pages per process. In addition, it help users to know the actual KSM profit because KSM-placed zero pages are also benefit from KSM. since unsharing zero pages placed by KSM accurately is achieved, then tracking empty pages merging and unmerging is not a difficult thing any longer. Since we already have /proc/<pid>/ksm_stat, just add the information of 'ksm_zero_pages' in it. Link: https://lkml.kernel.org/r/20230613030938.185993-1-yang.yang29@zte.com.cn Signed-off-by: xu xin <xu.xin16@zte.com.cn> Acked-by: David Hildenbrand <david@redhat.com> Reviewed-by: Xiaokai Ran <ran.xiaokai@zte.com.cn> Reviewed-by: Yang Yang <yang.yang29@zte.com.cn> Cc: Claudio Imbrenda <imbrenda@linux.ibm.com> Cc: Xuexin Jiang <jiang.xuexin@zte.com.cn> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: include/linux/mm_types.h mm/khugepaged.c mm/memory.c Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- fs/proc/base.c | 1 + include/linux/ksm.h | 8 +++++--- include/linux/mm_types.h | 11 ++++++++++- mm/khugepaged.c | 2 +- mm/ksm.c | 1 + mm/memory.c | 4 ++-- 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index 64d471fa99b3..80ceb036bae2 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3065,6 +3065,7 @@ static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, mm = get_task_mm(task); if (mm) { seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); + seq_printf(m, "ksm_zero_pages %lu\n", mm->ksm_zero_pages); seq_printf(m, "ksm_merging_pages %lu\n", mm->ksm_merging_pages); seq_printf(m, "ksm_process_profit %ld\n", ksm_process_profit(mm)); mmput(mm); diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 9a01f2a406a6..f2a83a2676cb 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -37,10 +37,12 @@ void __ksm_exit(struct mm_struct *mm); extern unsigned long ksm_zero_pages; -static inline void ksm_might_unmap_zero_page(pte_t pte) +static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte) { - if (is_ksm_zero_pte(pte)) + if (is_ksm_zero_pte(pte)) { ksm_zero_pages--; + mm->ksm_zero_pages--; + } } static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) @@ -105,7 +107,7 @@ static inline void ksm_exit(struct mm_struct *mm) { } -static inline void ksm_might_unmap_zero_page(pte_t pte) +static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte) { } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 2db486c0ee5b..5d7de361db2a 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -567,7 +567,7 @@ struct mm_struct { #if defined(CONFIG_KSM) && !defined(__GENKSYMS__) /* * Represent how many pages of this process are involved in KSM - * merging. + * merging (not including ksm_zero_pages). */ unsigned long ksm_merging_pages; #else @@ -583,7 +583,16 @@ struct mm_struct { #else KABI_RESERVE(5) #endif + +#if defined(CONFIG_KSM) && !defined(__GENKSYMS__) + /* + * Represent how many empty pages are merged with kernel zero + * pages when enabling KSM use_zero_pages. + */ + unsigned long ksm_zero_pages; +#else KABI_RESERVE(6) +#endif KABI_RESERVE(7) KABI_RESERVE(8) diff --git a/mm/khugepaged.c b/mm/khugepaged.c index e98b755dc0d1..37c5768b6ac8 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -674,7 +674,7 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, */ pte_clear(vma->vm_mm, address, _pte); spin_unlock(ptl); - ksm_might_unmap_zero_page(pteval); + ksm_might_unmap_zero_page(vma->vm_mm, pteval); } } else { src_page = pte_page(pteval); diff --git a/mm/ksm.c b/mm/ksm.c index 0cddf1b8cd05..5938de557780 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1239,6 +1239,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, */ newpte = pte_mkdirty(pte_mkspecial(pfn_pte(page_to_pfn(kpage), vma->vm_page_prot))); ksm_zero_pages++; + mm->ksm_zero_pages++; /* * We're replacing an anonymous page with a zero page, which is * not anonymous. We need to do proper accounting otherwise we diff --git a/mm/memory.c b/mm/memory.c index acdf8907becb..209439e02647 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -1106,7 +1106,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, tlb->fullmm); tlb_remove_tlb_entry(tlb, pte, addr); if (unlikely(!page)) { - ksm_might_unmap_zero_page(ptent); + ksm_might_unmap_zero_page(mm, ptent); continue; } @@ -2518,7 +2518,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf) reliable_page_counter(new_page, mm, 1); } } else { - ksm_might_unmap_zero_page(vmf->orig_pte); + ksm_might_unmap_zero_page(mm, vmf->orig_pte); inc_mm_counter_fast(mm, MM_ANONPAGES); reliable_page_counter(new_page, mm, 1); } -- 2.43.0

From: Chengming Zhou <chengming.zhou@linux.dev> mainline inclusion from mainline-v6.10-rc3 commit c2dc78b86e0821ecf9a9d0c35dba2618279a5bb6 category: bugfix bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- We normally ksm_zero_pages++ in ksmd when page is merged with zero page, but ksm_zero_pages-- is done from page tables side, where there is no any accessing protection of ksm_zero_pages. So we can read very exceptional value of ksm_zero_pages in rare cases, such as -1, which is very confusing to users. Fix it by changing to use atomic_long_t, and the same case with the mm->ksm_zero_pages. Link: https://lkml.kernel.org/r/20240528-b4-ksm-counters-v3-2-34bb358fdc13@linux.d... Fixes: e2942062e01d ("ksm: count all zero pages placed by KSM") Fixes: 6080d19f0704 ("ksm: add ksm zero pages for each process") Signed-off-by: Chengming Zhou <chengming.zhou@linux.dev> Acked-by: David Hildenbrand <david@redhat.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: Ran Xiaokai <ran.xiaokai@zte.com.cn> Cc: Stefan Roesch <shr@devkernel.io> Cc: xu xin <xu.xin16@zte.com.cn> Cc: Yang Yang <yang.yang29@zte.com.cn> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Conflicts: include/linux/mm_types.h mm/ksm.c Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- fs/proc/base.c | 2 +- include/linux/ksm.h | 17 ++++++++++++++--- include/linux/mm_types.h | 2 +- mm/ksm.c | 11 +++++------ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/fs/proc/base.c b/fs/proc/base.c index 80ceb036bae2..caffb7e70b8c 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3065,7 +3065,7 @@ static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, mm = get_task_mm(task); if (mm) { seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); - seq_printf(m, "ksm_zero_pages %lu\n", mm->ksm_zero_pages); + seq_printf(m, "ksm_zero_pages %ld\n", mm_ksm_zero_pages(mm)); seq_printf(m, "ksm_merging_pages %lu\n", mm->ksm_merging_pages); seq_printf(m, "ksm_process_profit %ld\n", ksm_process_profit(mm)); mmput(mm); diff --git a/include/linux/ksm.h b/include/linux/ksm.h index f2a83a2676cb..8e5352ddfe4b 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -35,16 +35,27 @@ void __ksm_exit(struct mm_struct *mm); */ #define is_ksm_zero_pte(pte) (is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte)) -extern unsigned long ksm_zero_pages; +extern atomic_long_t ksm_zero_pages; + +static inline void ksm_map_zero_page(struct mm_struct *mm) +{ + atomic_long_inc(&ksm_zero_pages); + atomic_long_inc(&mm->ksm_zero_pages); +} static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte) { if (is_ksm_zero_pte(pte)) { - ksm_zero_pages--; - mm->ksm_zero_pages--; + atomic_long_dec(&ksm_zero_pages); + atomic_long_dec(&mm->ksm_zero_pages); } } +static inline long mm_ksm_zero_pages(struct mm_struct *mm) +{ + return atomic_long_read(&mm->ksm_zero_pages); +} + static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm) { if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags)) diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 5d7de361db2a..682f75c03d68 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -589,7 +589,7 @@ struct mm_struct { * Represent how many empty pages are merged with kernel zero * pages when enabling KSM use_zero_pages. */ - unsigned long ksm_zero_pages; + atomic_long_t ksm_zero_pages; #else KABI_RESERVE(6) #endif diff --git a/mm/ksm.c b/mm/ksm.c index f35d9a490d00..86f750c603a7 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -281,7 +281,7 @@ static unsigned int zero_checksum __read_mostly; static bool ksm_use_zero_pages __read_mostly; /* The number of zero pages which is placed by KSM */ -unsigned long ksm_zero_pages; +atomic_long_t ksm_zero_pages = ATOMIC_LONG_INIT(0); #ifdef CONFIG_NUMA /* Zeroed when merging across nodes is not allowed */ @@ -1238,8 +1238,7 @@ static int replace_page(struct vm_area_struct *vma, struct page *page, * the dirty bit in zero page's PTE is set. */ newpte = pte_mkdirty(pte_mkspecial(pfn_pte(page_to_pfn(kpage), vma->vm_page_prot))); - ksm_zero_pages++; - mm->ksm_zero_pages++; + ksm_map_zero_page(mm); /* * We're replacing an anonymous page with a zero page, which is * not anonymous. We need to do proper accounting otherwise we @@ -3007,7 +3006,7 @@ static void wait_while_offlining(void) #ifdef CONFIG_PROC_FS long ksm_process_profit(struct mm_struct *mm) { - return (long)(mm->ksm_merging_pages + mm->ksm_zero_pages) * PAGE_SIZE - + return (long)(mm->ksm_merging_pages + mm_ksm_zero_pages(mm)) * PAGE_SIZE - mm->ksm_rmap_items * sizeof(struct rmap_item); } #endif /* CONFIG_PROC_FS */ @@ -3279,7 +3278,7 @@ KSM_ATTR_RO(pages_volatile); static ssize_t ksm_zero_pages_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - return sysfs_emit(buf, "%ld\n", ksm_zero_pages); + return sysfs_emit(buf, "%ld\n", atomic_long_read(&ksm_zero_pages)); } KSM_ATTR_RO(ksm_zero_pages); @@ -3288,7 +3287,7 @@ static ssize_t general_profit_show(struct kobject *kobj, { long general_profit; - general_profit = (ksm_pages_sharing + ksm_zero_pages) * PAGE_SIZE - + general_profit = (ksm_pages_sharing + atomic_long_read(&ksm_zero_pages)) * PAGE_SIZE - ksm_rmap_items * sizeof(struct rmap_item); return sysfs_emit(buf, "%ld\n", general_profit); -- 2.43.0

From: Nanyong Sun <sunnanyong@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 ---------------------------------------- Add control file "memory.ksm" to enable ksm per cgroup. Echo to 1 will set all tasks currently in the cgroup to ksm merge any mode, which means ksm gets enabled for all vma's of a process. Meanwhile echo to 0 will disable ksm for them and unmerge the merged pages. Cat the file will show the above state and ksm related profits of this cgroup. Signed-off-by: Nanyong Sun <sunnanyong@huawei.com> Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- mm/memcontrol.c | 107 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index b9d083303cf6..043ed63c554c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -75,6 +75,7 @@ #include <linux/uaccess.h> #include <trace/events/vmscan.h> +#include <linux/ksm.h> struct cgroup_subsys memory_cgrp_subsys __read_mostly; EXPORT_SYMBOL(memory_cgrp_subsys); @@ -5003,6 +5004,104 @@ bool dhugetlb_pool_is_free(struct cgroup_subsys_state *css) } #endif /* CONFIG_DYNAMIC_HUGETLB */ +static inline bool __task_is_dying(struct task_struct *task) +{ + return tsk_is_oom_victim(task) || fatal_signal_pending(task) || + (task->flags & PF_EXITING); +} + +#ifdef CONFIG_KSM +static int memcg_set_ksm_for_tasks(struct mem_cgroup *memcg, bool enable) +{ + struct task_struct *task; + struct mm_struct *mm; + struct css_task_iter it; + int ret = 0; + + css_task_iter_start(&memcg->css, CSS_TASK_ITER_PROCS, &it); + while (!ret && (task = css_task_iter_next(&it))) { + if (__task_is_dying(task)) + continue; + + mm = get_task_mm(task); + if (!mm) + continue; + + if (down_write_killable(&mm->mmap_sem)) { + mmput(mm); + continue; + } + + if (enable) + ret = ksm_enable_merge_any(mm); + else + ret = ksm_disable_merge_any(mm); + + up_write(&mm->mmap_sem); + mmput(mm); + } + css_task_iter_end(&it); + + return ret; +} + +static int memory_ksm_show(struct seq_file *m, void *v) +{ + unsigned long ksm_merging_pages = 0; + unsigned long ksm_rmap_items = 0; + long ksm_process_profits = 0; + unsigned int tasks = 0; + struct task_struct *task; + struct mm_struct *mm; + struct css_task_iter it; + struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); + + css_task_iter_start(&memcg->css, CSS_TASK_ITER_PROCS, &it); + while ((task = css_task_iter_next(&it))) { + mm = get_task_mm(task); + if (!mm) + continue; + + if (test_bit(MMF_VM_MERGE_ANY, &mm->flags)) + tasks++; + + ksm_rmap_items += mm->ksm_rmap_items; + ksm_merging_pages += mm->ksm_merging_pages; + ksm_process_profits += ksm_process_profit(mm); + mmput(mm); + } + css_task_iter_end(&it); + + seq_printf(m, "merge any tasks: %u\n", tasks); + seq_printf(m, "ksm_rmap_items %lu\n", ksm_rmap_items); + seq_printf(m, "ksm_merging_pages %lu\n", ksm_merging_pages); + seq_printf(m, "ksm_process_profits %ld\n", ksm_process_profits); + return 0; +} + +static ssize_t memory_ksm_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + bool enable; + int err; + struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of)); + + buf = strstrip(buf); + if (!buf) + return -EINVAL; + + err = kstrtobool(buf, &enable); + if (err) + return err; + + err = memcg_set_ksm_for_tasks(memcg, enable); + if (err) + return err; + + return nbytes; +} +#endif /* CONFIG_KSM */ + static struct cftype mem_cgroup_legacy_files[] = { { .name = "usage_in_bytes", @@ -5165,6 +5264,14 @@ static struct cftype mem_cgroup_legacy_files[] = { .write = mem_cgroup_reset, .read_u64 = mem_cgroup_read_u64, }, +#ifdef CONFIG_KSM + { + .name = "ksm", + .flags = CFTYPE_NOT_ON_ROOT, + .write = memory_ksm_write, + .seq_show = memory_ksm_show, + }, +#endif { }, /* terminate */ }; -- 2.43.0

hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 ---------------------------------------- When a task moves to a new memcg, enable KSM for the task if the auto_ksm_enabled of the memcg is 1. Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- mm/memcontrol.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a234f07d5fe0..b4d2af749d4a 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -5125,6 +5125,39 @@ static ssize_t memory_ksm_write(struct kernfs_open_file *of, char *buf, return nbytes; } + +static void memcg_attach_ksm(struct cgroup_taskset *tset) +{ + struct cgroup_subsys_state *css; + struct mem_cgroup *memcg; + struct task_struct *task; + + cgroup_taskset_first(tset, &css); + memcg = mem_cgroup_from_css(css); + if (!READ_ONCE(memcg->auto_ksm_enabled)) + return; + + cgroup_taskset_for_each(task, css, tset) { + struct mm_struct *mm = get_task_mm(task); + + if (!mm) + continue; + + if (down_write_killable(&mm->mmap_sem)) { + mmput(mm); + continue; + } + + ksm_enable_merge_any(mm); + + up_write(&mm->mmap_sem); + mmput(mm); + } +} +#else +static inline void memcg_attach_ksm(struct cgroup_taskset *tset) +{ +} #endif /* CONFIG_KSM */ static struct cftype mem_cgroup_legacy_files[] = { @@ -6415,6 +6448,12 @@ static void mem_cgroup_move_charge(void) atomic_dec(&mc.from->moving_account); } +static void mem_cgroup_attach(struct cgroup_taskset *tset) +{ + if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) + memcg_attach_ksm(tset); +} + static void mem_cgroup_move_task(void) { if (mc.to) { @@ -6430,6 +6469,9 @@ static int mem_cgroup_can_attach(struct cgroup_taskset *tset) static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset) { } +static void mem_cgroup_attach(struct cgroup_taskset *tset) +{ +} static void mem_cgroup_move_task(void) { } @@ -6819,6 +6861,7 @@ struct cgroup_subsys memory_cgrp_subsys = { .css_reset = mem_cgroup_css_reset, .can_attach = mem_cgroup_can_attach, .cancel_attach = mem_cgroup_cancel_attach, + .attach = mem_cgroup_attach, .post_attach = mem_cgroup_move_task, .bind = mem_cgroup_bind, .dfl_cftypes = memory_files, -- 2.43.0

From: Liangcai Fan <liangcaifan19@gmail.com> mainline inclusion from mainline-v5.16-rc1 commit bd3400ea173fb611cdf2030d03620185ff6c0b0e category: bugfix bugzilla: https://atomgit.com/openeuler/kernel/issues/10169 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- When initializing transparent huge pages, min_free_kbytes would be calculated according to what khugepaged expected. So when transparent huge pages get disabled, min_free_kbytes should be recalculated instead of the higher value set by khugepaged. Link: https://lkml.kernel.org/r/1633937809-16558-1-git-send-email-liangcaifan19@gm... Signed-off-by: Liangcai Fan <liangcaifan19@gmail.com> Signed-off-by: Chunyan Zhang <zhang.lyra@gmail.com> Cc: Mike Kravetz <mike.kravetz@oracle.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Jinjiang Tu <tujinjiang@huawei.com> --- include/linux/mm.h | 1 + mm/khugepaged.c | 10 ++++++++-- mm/page_alloc.c | 7 ++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 67e299374ac8..1e41f215abef 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2302,6 +2302,7 @@ extern void set_dma_reserve(unsigned long new_dma_reserve); extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long, enum meminit_context, struct vmem_altmap *); extern void setup_per_zone_wmarks(void); +extern void calculate_min_free_kbytes(void); extern int __meminit init_per_zone_wmark_min(void); extern void mem_init(void); extern void __init mmap_init(void); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 37c5768b6ac8..4aa61e4f151b 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -2004,6 +2004,11 @@ static void set_recommended_min_free_kbytes(void) int nr_zones = 0; unsigned long recommended_min; + if (!khugepaged_enabled()) { + calculate_min_free_kbytes(); + goto update_wmarks; + } + for_each_populated_zone(zone) { /* * We don't need to worry about fragmentation of @@ -2039,6 +2044,8 @@ static void set_recommended_min_free_kbytes(void) min_free_kbytes = recommended_min; } + +update_wmarks: setup_per_zone_wmarks(); } @@ -2060,12 +2067,11 @@ int start_stop_khugepaged(void) if (!list_empty(&khugepaged_scan.mm_head)) wake_up_interruptible(&khugepaged_wait); - - set_recommended_min_free_kbytes(); } else if (khugepaged_thread) { kthread_stop(khugepaged_thread); khugepaged_thread = NULL; } + set_recommended_min_free_kbytes(); fail: mutex_unlock(&khugepaged_mutex); return err; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index c2577d3048d9..c38a7ebe25bd 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -7946,7 +7946,7 @@ void setup_per_zone_wmarks(void) * 8192MB: 11584k * 16384MB: 16384k */ -int __meminit init_per_zone_wmark_min(void) +void calculate_min_free_kbytes(void) { unsigned long lowmem_kbytes; int new_min_free_kbytes; @@ -7964,6 +7964,11 @@ int __meminit init_per_zone_wmark_min(void) pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", new_min_free_kbytes, user_min_free_kbytes); } +} + +int __meminit init_per_zone_wmark_min(void) +{ + calculate_min_free_kbytes(); setup_per_zone_wmarks(); refresh_zone_stat_thresholds(); setup_per_zone_lowmem_reserve(); -- 2.43.0