hulk inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/IC8KS8 CVE: NA -------------------------------- Improve DAMON's region access tracking by integrating hardware-sampled memory access information when available via the mem_sampling framework. Currently, DAMON estimates region hotness based on software-based page access checks across monitoring intervals. This approach is limited by: - Random region selection - Long monitoring convergence time - Sensitivity to parameter tuning With hardware sampling (e.g. via ARM SPE), DAMON can: - Observe real memory access patterns with higher precision - Detect hot/cold regions more quickly - Reduce overhead and improve responsiveness This patch adds optional support for consuming mem_sampling data within DAMON. When hardware sampling is available and enabled, DAMON will use access records provided by the sampling infrastructure instead of relying on traditional page access bits. The integration preserves backward compatibility with existing DAMON logic, falling back to software-based tracking when hardware sampling is not available or disabled. This hybrid approach enhances DAMON's adaptability across platforms while improving accuracy and reducing latency in memory access monitoring. Signed-off-by: Ze Zuo <zuoze1@huawei.com> Signed-off-by: Tong Tiangen <tongtiangen@huawei.com> Signed-off-by: Shuang Yan <yanshuang7@huawei.com> --- include/linux/damon.h | 4 ++ include/linux/mem_sampling.h | 18 ++++++++ include/linux/mm_types.h | 4 ++ kernel/fork.c | 3 ++ mm/damon/Kconfig | 14 ++++++ mm/damon/core.c | 34 ++++++++++++++ mm/damon/vaddr.c | 87 ++++++++++++++++++++++++++++++++++++ mm/mem_sampling.c | 70 +++++++++++++++++++++++++++++ 8 files changed, 234 insertions(+) diff --git a/include/linux/damon.h b/include/linux/damon.h index 343132a146cf..424431d5a100 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -13,6 +13,7 @@ #include <linux/time64.h> #include <linux/types.h> #include <linux/random.h> +#include <linux/mem_sampling.h> /* Minimal region size. Every damon_region is aligned by this. */ #define DAMON_MIN_REGION PAGE_SIZE @@ -73,6 +74,9 @@ struct damon_region { */ struct damon_target { struct pid *pid; +#ifdef CONFIG_DAMON_MEM_SAMPLING + struct damon_mem_sampling_fifo damon_fifo; +#endif unsigned int nr_regions; struct list_head regions_list; struct list_head list; diff --git a/include/linux/mem_sampling.h b/include/linux/mem_sampling.h index d93324ce8269..00997b761b81 100644 --- a/include/linux/mem_sampling.h +++ b/include/linux/mem_sampling.h @@ -12,6 +12,8 @@ #ifndef __MEM_SAMPLING_H #define __MEM_SAMPLING_H +#include <linux/kfifo.h> + enum mem_sampling_sample_type { MEM_SAMPLING_L1D_ACCESS = 1 << 0, MEM_SAMPLING_L1D_MISS = 1 << 1, @@ -112,4 +114,20 @@ void mem_sampling_sched_in(struct task_struct *prev, struct task_struct *curr); static inline void set_mem_sampling_state(bool enabled) { } static inline void mem_sampling_sched_in(struct task_struct *prev, struct task_struct *curr) { } #endif /* CONFIG_MEM_SAMPLING */ + +#ifdef CONFIG_DAMON_MEM_SAMPLING +#define DAMOS_FIFO_MAX_RECORD (1024) +struct damon_mem_sampling_record { + u64 vaddr; +}; + +struct damon_mem_sampling_fifo { + struct kfifo rx_kfifo; + spinlock_t rx_kfifo_lock; /* protect SPE Rx data kfifo */ +}; + +bool damon_use_mem_sampling(void); +#else +static inline bool damon_use_mem_sampling(void) { return false; } +#endif /* CONFIG_DAMON_MEM_SAMPLING */ #endif /* __MEM_SAMPLING_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index b4442fbbf17b..64c38b09e18d 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1011,7 +1011,11 @@ struct mm_struct { #endif } __randomize_layout; +#ifdef CONFIG_DAMON_MEM_SAMPLING + KABI_USE(1, struct damon_mem_sampling_fifo *damon_fifo) +#else KABI_RESERVE(1) +#endif KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) diff --git a/kernel/fork.c b/kernel/fork.c index 698d7829f2e4..4b37cb915f7b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1362,6 +1362,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, init_tlb_flush_pending(mm); #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS mm->pmd_huge_pte = NULL; +#endif +#if defined(CONFIG_DAMON_MEM_SAMPLING) + mm->damon_fifo = NULL; #endif mm_init_uprobes_state(mm); hugetlb_count_init(mm); diff --git a/mm/damon/Kconfig b/mm/damon/Kconfig index 436c6b4cb5ec..d6ed1ef6ad4a 100644 --- a/mm/damon/Kconfig +++ b/mm/damon/Kconfig @@ -32,6 +32,20 @@ config DAMON_VADDR This builds the default data access monitoring operations for DAMON that work for virtual address spaces. +config DAMON_MEM_SAMPLING + bool "Set DAMON to use records from hardware sample" + depends on MEM_SAMPLING && DAMON_VADDR + help + This enables DAMON to utilize hardware sampling-based memory access + monitoring data (e.g., ARM SPE, Intel PEBS, AMD IBS) instead of + software-based sampling. When enabled, DAMON will: + + - Use CPU performance monitoring unit (PMU) samples as data source + - Correlate hardware samples with process virtual address spaces + - Provide lower overhead monitoring compared to pure software approaches + + If unsure, say N. + config DAMON_PADDR bool "Data access monitoring operations for the physical address space" depends on DAMON && MMU diff --git a/mm/damon/core.c b/mm/damon/core.c index 1daa8793c44b..c8a4427d1d63 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -112,6 +112,32 @@ int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id) return err; } +#if IS_ENABLED(CONFIG_DAMON_MEM_SAMPLING) +int damon_target_init_kfifo(struct damon_target *t) +{ + struct damon_mem_sampling_fifo *damon_fifo; + int ret = 0; + unsigned int fifo_size = sizeof(struct damon_mem_sampling_record) * DAMOS_FIFO_MAX_RECORD; + + damon_fifo = &t->damon_fifo; + + ret = kfifo_alloc(&damon_fifo->rx_kfifo, fifo_size, GFP_KERNEL); + if (ret) + return -ENOMEM; + + spin_lock_init(&damon_fifo->rx_kfifo_lock); + return 0; +} + +void damon_target_deinit_kfifo(struct damon_target *t) +{ + kfifo_free(&t->damon_fifo.rx_kfifo); +} +#else +static inline int damon_target_init_kfifo(struct damon_target *t) {return 0; } +static inline void damon_target_deinit_kfifo(struct damon_target *t) { } +#endif /* CONFIG_DAMON_MEM_SAMPLING */ + /* * Construct a damon_region struct * @@ -388,11 +414,18 @@ void damon_destroy_scheme(struct damos *s) struct damon_target *damon_new_target(void) { struct damon_target *t; + int ret; t = kmalloc(sizeof(*t), GFP_KERNEL); if (!t) return NULL; + ret = damon_target_init_kfifo(t); + if (ret) { + kfree(t); + return NULL; + } + t->pid = NULL; t->nr_regions = 0; INIT_LIST_HEAD(&t->regions_list); @@ -422,6 +455,7 @@ void damon_free_target(struct damon_target *t) damon_for_each_region_safe(r, next, t) damon_free_region(r); + damon_target_deinit_kfifo(t); kfree(t); } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 5764b9885e7d..167ca23d77ae 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -402,6 +402,83 @@ static void damon_va_mkold(struct mm_struct *mm, unsigned long addr) mmap_read_unlock(mm); } +#if IS_ENABLED(CONFIG_DAMON_MEM_SAMPLING) +/* + * Functions for the access checking of the regions with mem sampling + */ +static void __hw_damon_va_prepare_access_check(struct damon_region *r) +{ + r->sampling_addr = 0; +} + +static void hw_damon_va_prepare_access_checks(struct damon_ctx *ctx) +{ + struct damon_target *t; + struct mm_struct *mm; + struct damon_region *r; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + mm->damon_fifo = &t->damon_fifo; + damon_for_each_region(r, t) + __hw_damon_va_prepare_access_check(r); + mmput(mm); + } +} + +static void find_damon_region(struct damon_mem_sampling_record *damon_record, + struct damon_target *t, unsigned int *max_nr_accesses) +{ + struct damon_region *r; + unsigned long addr = damon_record->vaddr; + + damon_for_each_region(r, t) { + if (r->sampling_addr != 0) + return; + if (addr > r->ar.start && addr < r->ar.end) { + r->nr_accesses++; + r->sampling_addr = addr; + *max_nr_accesses = max(r->nr_accesses, *max_nr_accesses); + return; + } + } +} + +static unsigned int hw_damon_va_check_accesses(struct damon_ctx *ctx) +{ + unsigned int outs; + struct damon_target *t; + struct mm_struct *mm; + unsigned int max_nr_accesses = 0; + struct damon_mem_sampling_record damon_record; + + damon_for_each_target(t, ctx) { + mm = damon_get_mm(t); + if (!mm) + continue; + mm->damon_fifo = NULL; + mmput(mm); + while (!kfifo_is_empty(&t->damon_fifo.rx_kfifo)) { + outs = kfifo_out(&t->damon_fifo.rx_kfifo, &damon_record, + sizeof(struct damon_mem_sampling_record)); + if (outs != sizeof(struct damon_mem_sampling_record)) { + pr_debug("damon hw spe record corrupted header. Flush.\n"); + continue; + } + find_damon_region(&damon_record, t, &max_nr_accesses); + } + kfifo_reset_out(&t->damon_fifo.rx_kfifo); + } + + return max_nr_accesses; +} +#else +static inline void hw_damon_va_prepare_access_checks(struct damon_ctx *ctx) { } +static inline unsigned int hw_damon_va_check_accesses(struct damon_ctx *ctx) {return 0; } +#endif + /* * Functions for the access checking of the regions */ @@ -420,6 +497,11 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; + if (damon_use_mem_sampling()) { + hw_damon_va_prepare_access_checks(ctx); + return; + } + damon_for_each_target(t, ctx) { mm = damon_get_mm(t); if (!mm) @@ -589,6 +671,11 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) unsigned int max_nr_accesses = 0; bool same_target; + if (damon_use_mem_sampling()) { + max_nr_accesses = hw_damon_va_check_accesses(ctx); + return max_nr_accesses; + } + damon_for_each_target(t, ctx) { mm = damon_get_mm(t); if (!mm) diff --git a/mm/mem_sampling.c b/mm/mem_sampling.c index 3550c71b3f3d..65f047ae8b84 100644 --- a/mm/mem_sampling.c +++ b/mm/mem_sampling.c @@ -290,6 +290,75 @@ static void set_numabalancing_mem_sampling_state(bool enabled) static inline void set_numabalancing_mem_sampling_state(bool enabled) { } #endif /* CONFIG_NUMABALANCING_MEM_SAMPLING */ +DEFINE_STATIC_KEY_FALSE(mm_damon_mem_sampling); +#ifdef CONFIG_DAMON_MEM_SAMPLING +static void damon_mem_sampling_record_cb(struct mem_sampling_record *record) +{ + struct damon_mem_sampling_fifo *damon_fifo; + struct damon_mem_sampling_record domon_record; + struct task_struct *task = NULL; + struct mm_struct *mm; + + /* Discard kernel address accesses */ + if (record->virt_addr & (1UL << 63)) + return; + + task = find_get_task_by_vpid((pid_t)record->context_id); + if (!task) + return; + + mm = get_task_mm(task); + put_task_struct(task); + if (!mm) + return; + + damon_fifo = mm->damon_fifo; + mmput(mm); + + domon_record.vaddr = record->virt_addr; + + /* only the proc under monitor now has damon_fifo */ + if (damon_fifo) { + if (kfifo_is_full(&damon_fifo->rx_kfifo)) + return; + + kfifo_in_locked(&damon_fifo->rx_kfifo, &domon_record, + sizeof(struct damon_mem_sampling_record), + &damon_fifo->rx_kfifo_lock); + return; + } +} + +static void damon_mem_sampling_record_cb_register(void) +{ + mem_sampling_record_cb_register(damon_mem_sampling_record_cb); +} + +static void damon_mem_sampling_record_cb_unregister(void) +{ + mem_sampling_record_cb_unregister(damon_mem_sampling_record_cb); +} + +static void set_damon_mem_sampling_state(bool enabled) +{ + if (enabled) { + damon_mem_sampling_record_cb_register(); + static_branch_enable(&mm_damon_mem_sampling); + } else { + damon_mem_sampling_record_cb_unregister(); + static_branch_disable(&mm_damon_mem_sampling); + } +} + +bool damon_use_mem_sampling(void) +{ + return static_branch_unlikely(&mem_sampling_access_hints) && + static_branch_unlikely(&mm_damon_mem_sampling); +} +#else +static inline void set_damon_mem_sampling_state(bool enabled) { } +#endif + void mem_sampling_process(void) { int i, nr_records; @@ -336,6 +405,7 @@ static void __set_mem_sampling_state(bool enabled) else { static_branch_disable(&mem_sampling_access_hints); set_numabalancing_mem_sampling_state(enabled); + set_damon_mem_sampling_state(enabled); } } -- 2.25.1