[PATCH openEuler-26.09 1/3] mm: Cache_ext framework

From: Nikita Panov <panov.nikita@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10164 ---------------------------------------- Cache_ext framework project integration. Heavily inspired by "cache_ext: Customizing the Page Cache with eBPF" by Tal Zussman, Ioannis Zarkadas and others. A lot of cleanups and reimplementation's of the original project were performed, fixed a lot of major design issues, including but not limited to ref-counting, memory leaks, data corruption bugs, use-after-free, and many more. Significantly improved scalability and decreased overhead compare to the original implementation. Supported policy loading per-memory cgroup. On policy load, child memory cgroups inherit policy from the parent. For now, we recommend loading a single policy into the root memory cgroup. Migration, drop_caches and standard reclaim - all these features are supported. Page_ext is used for internal structures, be aware about extra memory consumption. Append "cache_ext=on" to the kernel cmdline if you want to enable this feature. Co-developed-by: Alexander Grubnikov <alexander.grubnikov@huawei.com> Signed-off-by: Alexander Grubnikov <alexander.grubnikov@huawei.com> Co-developed-by: Anatoly Stepanov <stepanov.anatoly@huawei.com> Signed-off-by: Anatoly Stepanov <stepanov.anatoly@huawei.com> Co-developed-by: Nikita Panov <panov.nikita@huawei.com> Signed-off-by: Nikita Panov <panov.nikita@huawei.com> --- .../admin-guide/kernel-parameters.txt | 11 + Documentation/admin-guide/sysctl/vm.rst | 11 + arch/arm64/configs/openeuler_defconfig | 1 + include/linux/cache_ext.h | 249 ++++ include/linux/memcontrol.h | 26 + include/linux/mm_types.h | 33 + include/linux/mmzone.h | 5 + include/linux/page_ext.h | 4 + include/linux/sched.h | 6 + include/linux/swap.h | 12 + kernel/bpf/verifier.c | 129 +- kernel/sysctl.c | 10 + mm/Kconfig | 11 + mm/Makefile | 1 + mm/cache_ext.c | 952 +++++++++++++++ mm/cache_ext_ds.c | 1050 +++++++++++++++++ mm/filemap.c | 10 + mm/memcontrol.c | 43 + mm/migrate.c | 16 +- mm/page_ext.c | 7 + mm/swap.c | 1 + mm/truncate.c | 12 +- mm/vmscan.c | 262 +++- mm/vmstat.c | 6 +- mm/workingset.c | 68 +- 25 files changed, 2924 insertions(+), 12 deletions(-) create mode 100644 include/linux/cache_ext.h create mode 100644 mm/cache_ext.c create mode 100644 mm/cache_ext_ds.c diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 17160ad5aefe..ab5ecf034b22 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -538,6 +538,17 @@ possible to determine what the correct size should be. This option provides an override for these situations. + cache_ext= [KNL] + Format: [on|off] + Default: off + This feature enables support of the cache_ext framework. + Cache_ext allows implementing custom policies for page + cache reclaim and loading them per memory cgroup. + Page_ext is used in this framework for internal data + structures. If policy is not loaded, or policy is unable + to reclaim the requested number of pages, fallback to + vanilla reclaim is performed. + carrier_timeout= [NET] Specifies amount of time (in seconds) that the kernel should wait for a network carrier. By default diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index 2ad3bf320c55..cdbcf806b7fe 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -80,6 +80,7 @@ Currently, these files are in /proc/sys/vm: - cache_reclaim_weight - cache_reclaim_enable - cache_limit_mbytes +- allow_direct_cache_ext admin_reserve_kbytes @@ -1099,3 +1100,13 @@ cache_limit_mbytes This is used to set the upper limit of page cache in megabytes. Page cache will be reclaimed periodically if page cache is over limit. + + +allow_direct_cache_ext +=================== + +When CONFIG_CACHE_EXT is enabled, bpf policies for page cache reclaim might be +implemented and loaded per memory cgroup. This file controls whether direct +reclaim is allowed to be performed from the custom policy or not. If not, +only kswapd will reclaim from the cache_ext policy. Noop if cache_ext=on is +not specified in kernel cmdline. Default value is true. diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 476efcfb46fe..fc503fff4c29 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -1266,6 +1266,7 @@ CONFIG_RECLAIM_NOTIFY=y # CONFIG_BPF_READAHEAD is not set CONFIG_MEM_SAMPLING=y CONFIG_NUMABALANCING_MEM_SAMPLING=y +CONFIG_CACHE_EXT=y # # Data Access Monitoring diff --git a/include/linux/cache_ext.h b/include/linux/cache_ext.h new file mode 100644 index 000000000000..40e3c4230942 --- /dev/null +++ b/include/linux/cache_ext.h @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_CACHE_EXT_H +#define _LINUX_CACHE_EXT_H + +#include <linux/list.h> +#include <linux/hashtable.h> +#include <linux/spinlock.h> +#include <linux/slab.h> +#include <linux/mm.h> +#include <linux/bitops.h> +#include <linux/page_ext.h> + + +#ifdef CONFIG_CACHE_EXT + +DECLARE_STATIC_KEY_TRUE(cache_ext_disabled); + +static inline bool folio_test_cache_ext(struct folio *folio) +{ + bool result; + struct page_ext *page_ext; + + if (static_branch_unlikely(&cache_ext_disabled)) + return false; + + page_ext = page_ext_get(folio_page(folio, 0)); + + if (unlikely(!page_ext)) + return false; + + result = test_bit(PAGE_EXT_CACHE_EXT, &page_ext->flags); + + page_ext_put(page_ext); + return result; +} + +static inline void folio_clear_cache_ext(struct folio *folio) +{ + struct page_ext *page_ext; + + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + page_ext = page_ext_get(folio_page(folio, 0)); + + if (unlikely(!page_ext)) + return; + + clear_bit(PAGE_EXT_CACHE_EXT, &page_ext->flags); + + page_ext_put(page_ext); +} + +static inline void folio_set_cache_ext(struct folio *folio) +{ + struct page_ext *page_ext; + + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + page_ext = page_ext_get(folio_page(folio, 0)); + + if (unlikely(!page_ext)) + return; + + set_bit(PAGE_EXT_CACHE_EXT, &page_ext->flags); + + page_ext_put(page_ext); +} + +static inline bool folio_test_cache_ext_special_list(struct folio *folio) +{ + bool result; + struct page_ext *page_ext; + + if (static_branch_unlikely(&cache_ext_disabled)) + return false; + + page_ext = page_ext_get(folio_page(folio, 0)); + + if (unlikely(!page_ext)) + return false; + + result = test_bit(PAGE_EXT_CACHE_EXT, &page_ext->flags); + + page_ext_put(page_ext); + return result; +} + +static inline void folio_clear_cache_ext_special_list(struct folio *folio) +{ + struct page_ext *page_ext; + + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + page_ext = page_ext_get(folio_page(folio, 0)); + + if (unlikely(!page_ext)) + return; + + clear_bit(PAGE_EXT_CACHE_EXT_SPECIAL_LIST, &page_ext->flags); + + page_ext_put(page_ext); +} + +static inline void folio_set_cache_ext_special_list(struct folio *folio) +{ + struct page_ext *page_ext; + + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + page_ext = page_ext_get(folio_page(folio, 0)); + + if (unlikely(!page_ext)) + return; + + set_bit(PAGE_EXT_CACHE_EXT_SPECIAL_LIST, &page_ext->flags); + + page_ext_put(page_ext); +} + +typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); + +bool cache_ext_is_callback_calling_kfunc_iterate(u32 btf_id); +bool cache_ext_is_callback_calling_kfunc_sample(u32 btf_id); + +struct cache_ext_list { + struct list_head head; +}; + +/* + * Cookie might be set only + * for non-cache_ext folios. + */ +struct cache_ext_list_node { + union { + struct folio *folio; + s64 cookie; + }; + struct list_head node; +}; + +struct cache_ext_iterate_opts { + u64 continue_list; + u64 continue_mode; + + u64 nr_folios_continue; + u64 nr_folios_isolated; +}; + +struct sampling_options { + __u32 sample_size; + __u32 select_size; +}; + +int bpf_cache_ext_list_splice(u64 list_id, u64 head_id, struct mem_cgroup *memcg, int nid); +int bpf_cache_ext_list_splice_tail(u64 list_id, u64 head_id, struct mem_cgroup *memcg, int nid); + +int bpf_cache_ext_list_add(u64 list, struct folio *folio); +int bpf_cache_ext_list_add_tail(u64 list, struct folio *folio); +int bpf_cache_ext_list_move(u64 list, struct folio *folio, bool tail); +int bpf_cache_ext_list_del(struct folio *folio); +int bpf_cache_ext_list_iterate(struct mem_cgroup *memcg, u64 list_id, + int (*iter_fn)(int idx, + struct cache_ext_list_node *node), + struct cache_ext_isolation_ctx *ctx); +int bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, + s32 (*score_fn)(struct cache_ext_list_node *a), + struct sampling_options *opts, + struct cache_ext_isolation_ctx *ctx); +int bpf_cache_ext_list_iterate_extended(struct mem_cgroup *memcg, u64 list, + int (*iter_fn)(int idx, struct cache_ext_list_node *node), + struct cache_ext_iterate_opts *opts, + struct cache_ext_isolation_ctx *ctx); +u64 bpf_cache_ext_ds_registry_new_list(struct mem_cgroup *memcg); +bool bpf_cache_ext_filename_strstr(const char *str, u32 str__sz, struct folio *folio); +bool bpf_cache_ext_list_empty(u64 list_id, struct mem_cgroup *memcg, int nid); +int bpf_folio_nid(struct folio *folio); + +#define CACHE_EXT_REGISTRY_MAX_ENTRIES 8 + +struct cache_ext_ds_registry { + /* + * rwlock serializing the following: + * - item add/del + * - per-list add/del/move + */ + rwlock_t lock; + struct cache_ext_list items[CACHE_EXT_REGISTRY_MAX_ENTRIES]; + int nr_entries; +}; + +struct cache_ext_policy { + struct cache_ext_ops *cache_ext_ops; + rwlock_t lock; + struct mem_cgroup *memcg; +}; + +int cache_ext_policy_init(struct mem_cgroup *memcg); +void cache_ext_policy_free(struct mem_cgroup *memcg); + +void cache_ext_folio_added(struct folio *folio, bool increase_ref, s64 cookie); +void cache_ext_folio_accessed(struct folio *folio); +s64 cache_ext_folio_evicted(struct folio *folio); +int cache_ext_isolate_folios(struct cache_ext_isolation_ctx *ctx, struct mem_cgroup *memcg); +int cache_ext_policy_stat_show(struct seq_file *m, void *v); + +int cache_ext_folio_meta_init(struct folio *folio, bool increase_ref); +int cache_ext_folio_meta_fini(struct folio *folio); + +struct cache_ext_list_node *get_cache_ext_list_node(struct folio *folio); +void put_cache_ext_list_node(struct cache_ext_list_node *node); + +void set_cookie_to_folio_page_ext(struct folio *folio, s64 cookie); +s64 get_and_clear_cookie_from_folio_page_ext(struct folio *folio); + +bool cache_ext_kfunc(u32 btf_id); + +extern bool allow_direct_cache_ext; + +#else + +static inline bool folio_test_cache_ext(struct folio *folio) { return false; } +static inline void folio_clear_cache_ext(struct folio *folio) { } +static inline void folio_set_cache_ext(struct folio *folio) { } +static inline bool folio_test_cache_ext_special_list(struct folio *folio) { return false; } +static inline void folio_clear_cache_ext_special_list(struct folio *folio) { } +static inline void folio_set_cache_ext_special_list(struct folio *folio) { } + +static inline bool cache_ext_is_callback_calling_kfunc_iterate(u32 btf_id) { return false; } +static inline bool cache_ext_is_callback_calling_kfunc_sample(u32 btf_id) { return false; } + +static inline int cache_ext_policy_init(struct mem_cgroup *memcg) { return 0; } +static inline void cache_ext_policy_free(struct mem_cgroup *memcg) { } + +static inline void cache_ext_folio_added(struct folio *folio, bool increase_ref, s64 cookie) { } +static inline void cache_ext_folio_accessed(struct folio *folio) { } +static inline int cache_ext_folio_evicted(struct folio *folio) { return -EOPNOTSUPP; } + +static inline void set_cookie_to_folio_page_ext(struct folio *folio, s64 cookie) { } +static inline s64 get_and_clear_cookie_from_folio_page_ext(struct folio *folio) { return 0; } + +static inline bool cache_ext_kfunc(u32 btf_id) { return false; } + +#endif + +#endif diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 89ef30f20275..c2221d106b00 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -21,6 +21,7 @@ #include <linux/vmstat.h> #include <linux/writeback.h> #include <linux/page-flags.h> +#include <linux/cache_ext.h> #include <linux/kabi.h> #ifndef __GENKSYMS__ @@ -157,6 +158,10 @@ struct mem_cgroup_per_node { struct mem_cgroup *memcg; /* Back pointer, we cannot */ /* use container_of */ +#ifdef CONFIG_CACHE_EXT + struct cache_ext_ds_registry *cache_ext_ds_registry; +#endif + KABI_RESERVE(1) KABI_RESERVE(2) KABI_RESERVE(3) @@ -412,6 +417,11 @@ struct mem_cgroup { struct dynamic_pool *dpool; #endif +#ifdef CONFIG_CACHE_EXT + struct cache_ext_policy *cache_ext_policy; + struct cache_ext_policy *cache_ext_ptr; +#endif + KABI_RESERVE(1) KABI_RESERVE(2) KABI_RESERVE(3) @@ -2107,4 +2117,20 @@ static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, } #endif +#ifdef CONFIG_CACHE_EXT +static inline struct cache_ext_ops *get_cache_ext_ops_memcg(struct mem_cgroup *memcg) +{ + if (memcg && memcg->cache_ext_ptr) + return memcg->cache_ext_ptr->cache_ext_ops; + return NULL; +} + +static inline struct cache_ext_ops *get_cache_ext_ops_lruvec(struct lruvec *lruvec) +{ + if (!lruvec) + return NULL; + return get_cache_ext_ops_memcg(lruvec_memcg(lruvec)); +} +#endif + #endif /* _LINUX_MEMCONTROL_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 20c62bc70558..e2050defc043 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1500,4 +1500,37 @@ enum { /* See also internal only FOLL flags in mm/internal.h */ }; +#define CACHE_EXT_NR_ISOLATED (32) +struct cache_ext_isolation_ctx { + /* Requested nr */ + unsigned long nr_folios_to_isolate; + int numa_node_id; + /* Actual nr */ + unsigned long nr_folios_isolated; + unsigned long nr_base_pages_isolated; + struct folio *folios[CACHE_EXT_NR_ISOLATED]; + s64 scores[CACHE_EXT_NR_ISOLATED]; + s64 cookies[CACHE_EXT_NR_ISOLATED]; +}; + +#define MAX_STATS_COUNT (4) +#define MAX_STAT_LEN (32) +struct cache_ext_stats { + u64 n; + char names[MAX_STATS_COUNT][MAX_STAT_LEN]; + u64 values[MAX_STATS_COUNT]; +}; + +struct lruvec; +struct cache_ext_ops { + /* Implement bpf_verifier_ops */ + s32 (*init)(struct mem_cgroup *memcg); + int (*isolate_folios)(struct cache_ext_isolation_ctx *ctx, struct mem_cgroup *memcg); + void (*folio_added)(struct folio *folio, s64 cookie); + void (*folio_accessed)(struct folio *folio); + s64 (*folio_evicted)(struct folio *folio); + void (*get_statistics)(struct lruvec *lruvec, struct cache_ext_stats *stats); + struct mem_cgroup *memcg; +}; + #endif /* _LINUX_MM_TYPES_H */ diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index b831463c2e6e..3fbc6b5fbba9 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -219,6 +219,11 @@ enum node_stat_item { PGDEMOTE_KSWAPD, PGDEMOTE_DIRECT, PGDEMOTE_KHUGEPAGED, +#ifdef CONFIG_CACHE_EXT + NR_FILE_CACHE_EXT, + NR_FILE_CACHE_EXT_KSWAPD_RECLAIMED, + NR_FILE_CACHE_EXT_REFAULT, +#endif NR_VM_NODE_STAT_ITEMS }; diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index be98564191e6..d8c9fb64d5ee 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -40,6 +40,10 @@ enum page_ext_flags { PAGE_EXT_YOUNG, PAGE_EXT_IDLE, #endif +#ifdef CONFIG_CACHE_EXT + PAGE_EXT_CACHE_EXT, + PAGE_EXT_CACHE_EXT_SPECIAL_LIST, +#endif }; /* diff --git a/include/linux/sched.h b/include/linux/sched.h index 3014da820ba6..d7f1c1095cbb 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1862,7 +1862,13 @@ extern struct pid *cad_pid; #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ #define PF__HOLE__00800000 0x00800000 + +#ifdef CONFIG_CACHE_EXT +#define PF_CACHE_EXT_RECLAIM 0x01000000 /* I reclaim from cache_ext right now */ +#else #define PF__HOLE__01000000 0x01000000 +#endif + #define PF__HOLE__02000000 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ diff --git a/include/linux/swap.h b/include/linux/swap.h index f11b96cec851..9f1f77f4cf59 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -39,6 +39,18 @@ static inline int current_is_kswapd(void) return current->flags & PF_KSWAPD; } +#ifdef CONFIG_CACHE_EXT +static inline int current_is_cache_ext_reclaim(void) +{ + return current->flags & PF_CACHE_EXT_RECLAIM; +} +#else +static inline int current_is_cache_ext_reclaim(void) +{ + return 0; +} +#endif + /* * MAX_SWAPFILES defines the maximum number of swaptypes: things which can * be swapped to. The swap type and the offset into that swap type are diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 5c24f719437f..1dff96c87ead 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -26,6 +26,7 @@ #include <linux/poison.h> #include <linux/module.h> #include <linux/cpumask.h> +#include <linux/cache_ext.h> #include <net/xdp.h> #ifdef CONFIG_BPF_SCHED @@ -9758,6 +9759,73 @@ static int set_rbtree_add_callback_state(struct bpf_verifier_env *env, return 0; } +#ifdef CONFIG_CACHE_EXT + +BTF_ID_LIST_SINGLE(bpf_cache_ext_list_node_btf_id, struct, cache_ext_list_node) + +static int set_cache_ext_list_iterate_callback_state( + struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + /* int bpf_cache_ext_list_iterate( + * struct mem_cgroup *memcg, + * u64 list, + * int (iter_fn)(int idx, struct cache_ext_list_node *node), + * struct cache_ext_eviction_ctx *ctx) + */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); + + /* Set scalar register */ + __mark_reg_unknown(env, &callee->regs[BPF_REG_1]); + + /* Set node pointer */ + callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); + callee->regs[BPF_REG_2].btf = btf_vmlinux; + callee->regs[BPF_REG_2].btf_id = bpf_cache_ext_list_node_btf_id[0]; + + /* unused */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + + callee->in_callback_fn = true; + callee->callback_ret_range = retval_range(0, 2); + return 0; +} + +static int set_cache_ext_list_sample_callback_state( + struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + /* int bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, + * s64(score_fn)(struct cache_ext_list_node *a), + * struct sampling_options *opts, + * struct cache_ext_eviction_ctx *ctx) + */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_0]); + + /* Set node pointer */ + callee->regs[BPF_REG_1].type = PTR_TO_BTF_ID; + __mark_reg_known_zero(&callee->regs[BPF_REG_1]); + callee->regs[BPF_REG_1].btf = btf_vmlinux; + callee->regs[BPF_REG_1].btf_id = bpf_cache_ext_list_node_btf_id[0]; + + /* unused */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_3]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + + callee->in_callback_fn = true; + callee->callback_ret_range = retval_range(S32_MIN, S32_MAX); + return 0; +} +#endif + static bool is_rbtree_lock_required_kfunc(u32 btf_id); /* Are we currently verifying the callback for a rbtree helper that must @@ -11238,7 +11306,9 @@ static bool is_bpf_graph_api_kfunc(u32 btf_id) static bool is_sync_callback_calling_kfunc(u32 btf_id) { - return btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl]; + return (btf_id == special_kfunc_list[KF_bpf_rbtree_add_impl] || + cache_ext_is_callback_calling_kfunc_iterate(btf_id) || + cache_ext_is_callback_calling_kfunc_sample(btf_id)); } static bool is_rbtree_lock_required_kfunc(u32 btf_id) @@ -11931,6 +12001,36 @@ static int fetch_kfunc_meta(struct bpf_verifier_env *env, return 0; } +#ifdef CONFIG_CACHE_EXT +static bool in_cache_ext_list_cb(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + struct bpf_insn *insn = env->prog->insnsi; + struct bpf_func_state *callee; + int kfunc_btf_id; + u32 frame; + + if (static_branch_unlikely(&cache_ext_disabled)) + return false; + + for (frame = state->curframe; frame; frame--) { + callee = state->frame[frame]; + + if (!callee->in_callback_fn) + continue; + + kfunc_btf_id = insn[callee->callsite].imm; + if (cache_ext_is_callback_calling_kfunc_iterate(kfunc_btf_id) || + cache_ext_is_callback_calling_kfunc_sample(kfunc_btf_id)) + return true; + } + + return false; +} +#else +static bool in_cache_ext_list_cb(struct bpf_verifier_env *env) { return false; } +#endif + static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx_p) { @@ -11971,6 +12071,12 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return -EACCES; } + if (in_cache_ext_list_cb(env) && cache_ext_kfunc(meta.func_id)) { + verbose(env, "kfunc %s#%d cannot be called from cache_ext list callback\n", + func_name, meta.func_id); + return -EACCES; + } + /* Check the arguments */ err = check_kfunc_args(env, &meta, insn_idx); if (err < 0) @@ -11986,6 +12092,27 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } } +#ifdef CONFIG_CACHE_EXT + if (cache_ext_is_callback_calling_kfunc_iterate(meta.func_id)) { + err = push_callback_call(env, insn, insn_idx, meta.subprogno, + set_cache_ext_list_iterate_callback_state); + if (err) { + verbose(env, "kfunc %s#%d failed callback verification\n", + func_name, meta.func_id); + return err; + } + } + if (cache_ext_is_callback_calling_kfunc_sample(meta.func_id)) { + err = push_callback_call(env, insn, insn_idx, meta.subprogno, + set_cache_ext_list_sample_callback_state); + if (err) { + verbose(env, "kfunc %s#%d failed callback verification\n", + func_name, meta.func_id); + return err; + } + } +#endif + rcu_lock = is_kfunc_bpf_rcu_read_lock(&meta); rcu_unlock = is_kfunc_bpf_rcu_read_unlock(&meta); diff --git a/kernel/sysctl.c b/kernel/sysctl.c index e84df0818517..88d8ca3c0fd3 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -63,6 +63,7 @@ #include <linux/mount.h> #include <linux/userfaultfd_k.h> #include <linux/pid.h> +#include <linux/cache_ext.h> #include "../lib/kstrtox.h" @@ -2118,6 +2119,15 @@ static struct ctl_table vm_table[] = { .extra1 = SYSCTL_ONE, .extra2 = SYSCTL_FOUR, }, +#ifdef CONFIG_CACHE_EXT + { + .procname = "allow_direct_cache_ext", + .data = &allow_direct_cache_ext, + .maxlen = sizeof(bool), + .mode = 0644, + .proc_handler = proc_dobool + }, +#endif { .procname = "page_lock_unfairness", .data = &sysctl_page_lock_unfairness, diff --git a/mm/Kconfig b/mm/Kconfig index b99e90efcc2c..62646204f67e 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1528,6 +1528,17 @@ config GMEM say Y here to enable gmem subsystem +config CACHE_EXT + bool "Cache_ext framework" + depends on BPF_SYSCALL && MEMCG + select PAGE_EXTENSION + default n + help + This feature provides infrastructure for implementing + custom reclaim policies for page cache reclaim in BPF + + say Y here to enable + source "mm/damon/Kconfig" config THP_CONTROL diff --git a/mm/Makefile b/mm/Makefile index e45cdeda47b7..7a026d41588c 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -150,3 +150,4 @@ obj-$(CONFIG_RECLAIM_NOTIFY) += reclaim_notify.o obj-$(CONFIG_DYNAMIC_POOL) += dynamic_pool.o obj-$(CONFIG_MEM_SAMPLING) += mem_sampling.o obj-$(CONFIG_THP_CONTROL) += thp_ctl.o +obj-$(CONFIG_CACHE_EXT) += cache_ext.o cache_ext_ds.o diff --git a/mm/cache_ext.c b/mm/cache_ext.c new file mode 100644 index 000000000000..2b349030209f --- /dev/null +++ b/mm/cache_ext.c @@ -0,0 +1,952 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/bpf_verifier.h> +#include <linux/bpf.h> +#include <linux/btf.h> +#include <linux/kernel.h> +#include <linux/memcontrol.h> +#include <linux/mm_types.h> +#include <linux/types.h> +#include <linux/cgroup.h> +#include <linux/pagevec.h> +#include <linux/mm.h> +#include <linux/mm_inline.h> +#include <linux/page_ext.h> + +#include "internal.h" + +DEFINE_STATIC_KEY_TRUE(cache_ext_disabled); + +static int __init setup_cache_ext(char *str) +{ + int ret = -EINVAL; + + if (!str) + goto out; + if (!strcmp(str, "on")) { + static_branch_disable(&cache_ext_disabled); + pr_info("cache_ext enabled via cmdline\n"); + ret = 0; + } else if (!strcmp(str, "off")) { + static_branch_enable(&cache_ext_disabled); + pr_info("cache_ext disabled via cmdline\n"); + ret = 0; + } +out: + if (ret) + pr_warn("cache_ext= cannot parse, ignored\n"); + return ret; +} +early_param("cache_ext", setup_cache_ext); + +static const struct btf_type *cache_ext_isolation_ctx_type; +static const struct btf_type *cache_ext_stats_ctx_type; + +static int bpf_cache_ext_init(struct btf *btf) +{ + u32 isolation_type_id, stats_type_id; + + isolation_type_id = btf_find_by_name_kind(btf, "cache_ext_isolation_ctx", + BTF_KIND_STRUCT); + if (isolation_type_id < 0) { + pr_err("cache_ext: failed to find struct cache_ext_isolation_ctx\n"); + return -EINVAL; + } + + stats_type_id = btf_find_by_name_kind(btf, "cache_ext_stats", + BTF_KIND_STRUCT); + if (stats_type_id < 0) { + pr_err("cache_ext: failed to find struct cache_ext_stats\n"); + return -EINVAL; + } + + cache_ext_isolation_ctx_type = btf_type_by_id(btf, isolation_type_id); + cache_ext_stats_ctx_type = btf_type_by_id(btf, stats_type_id); + return 0; +} + +static const struct bpf_func_proto * +bpf_cache_ext_get_func_proto(enum bpf_func_id func_id, + const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_get_current_pid_tgid: + return &bpf_get_current_pid_tgid_proto; + default: + return bpf_base_func_proto(func_id); + } +} + +static bool bpf_cache_ext_is_valid_access(int off, int size, + enum bpf_access_type type, + const struct bpf_prog *prog, + struct bpf_insn_access_aux *info) +{ + if (off % size != 0) + return false; + + return btf_ctx_access(off, size, type, prog, info); +} + +static int bpf_cache_ext_btf_struct_access(struct bpf_verifier_log *log, + const struct bpf_reg_state *reg, + int off, int size) +{ + /* + * For each of the allowed types, allow read access. + */ + const struct btf_type *t; + + t = btf_type_by_id(reg->btf, reg->btf_id); + if (t == cache_ext_isolation_ctx_type) { + unsigned long start, end; + + if (off + size > sizeof(struct cache_ext_isolation_ctx)) { + bpf_log(log, "out of bounds access at off %d with size %d\n", off, size); + return -EACCES; + } + start = offsetof(struct cache_ext_isolation_ctx, cookies); + end = start + (CACHE_EXT_NR_ISOLATED * sizeof(s64)); + if (off >= start && off + size <= end) + return SCALAR_VALUE; + + bpf_log(log, "Invalid write in isolation ctx at off %d with size %d\n", off, size); + return -EACCES; + } else if (t == cache_ext_stats_ctx_type) { + if (off + size > sizeof(struct cache_ext_stats)) { + bpf_log(log, "out of bounds access at off %d with size %d\n", off, size); + return -EACCES; + } + return SCALAR_VALUE; + } + + return -EACCES; +} + +static int bpf_cache_ext_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + return 0; +} + +static int bpf_cache_ext_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + const char *member_name; + + member_name = btf_name_by_offset(bpf_get_btf_vmlinux(), member->name_off); + + if (prog->aux->sleepable) { + + if (member_name && strcmp(member_name, "init") == 0) + return 0; + + pr_warn_ratelimited("cache_ext: sleepable BPF program is not allowed for atomic hook '%s'\n", + member_name ? member_name : "unknown"); + return -EINVAL; + } + return 0; +} + +static int bpf_cache_ext_validate(void *kdata) +{ + return 0; +} + +#define MAX_CACHE_EXT_BATCH (256) + +/* + * Inspired by isolate lru folios / move folio + */ +static bool isolate_folio_list(struct list_head *src, struct list_head *dst) +{ + int remaining = MAX_CACHE_EXT_BATCH; + bool result = true; + + LIST_HEAD(return_folios); + + while (!list_empty(src)) { + struct list_head *move_to = &return_folios; + struct folio *folio; + + folio = lru_to_folio(src); + + if (!folio_test_lru(folio)) + goto move; + + if (unlikely(!folio_try_get(folio))) + goto move; + + if (!folio_test_clear_lru(folio)) { + /* + * Another thread is already isolating this folio + */ + folio_put(folio); + goto move; + } + move_to = dst; +move: + list_move(&folio->lru, move_to); + remaining--; + + if (!remaining) { + result = false; + break; + } + } + list_splice_tail(&return_folios, src); + + return result; +} + +static bool return_folios_to_lru(struct list_head *dst, struct list_head *src, + struct list_head *putback_list, struct folio_batch *free_folios) +{ + int remaining = MAX_CACHE_EXT_BATCH; + bool result = true; + + while (!list_empty(src)) { + struct folio *folio = lru_to_folio(src); + + VM_BUG_ON_FOLIO(folio_test_lru(folio), folio); + + list_del(&folio->lru); + if (unlikely(!folio_evictable(folio))) { + list_add(&folio->lru, putback_list); + continue; + } + + folio_set_lru(folio); + + if (unlikely(folio_put_testzero(folio))) { + __folio_clear_lru_flags(folio); + + folio_unqueue_deferred_split(folio); + if (folio_batch_add(free_folios, folio) == 0) { + result = false; + break; + } + + continue; + } + + + list_add(&folio->lru, dst); + remaining--; + + if (!remaining) { + result = false; + break; + } + } + + return result; +} + +static bool add_folio_list_to_policy(struct list_head *src, struct list_head *return_list, + struct lruvec *lruvec, bool activate) +{ + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + struct cache_ext_ops *ops = memcg->cache_ext_ptr ? + memcg->cache_ext_ptr->cache_ext_ops : NULL; + int remaining = MAX_CACHE_EXT_BATCH; + bool result = true; + + if (!ops) + return result; + + while (!list_empty(src)) { + struct folio *folio = lru_to_folio(src); + int nr = folio_nr_pages(folio); + + /* + * Since we relase policy and lru locks during isolation, + * some of the isolated folios might be already added to policy. + * Check flag and skip if they are already in policy + */ + if (folio_test_cache_ext(folio)) + goto skip; + + if (cache_ext_folio_meta_init(folio, true)) + goto skip; + + if (ops->folio_added) + ops->folio_added(folio, 0); + + mod_lruvec_state(lruvec, NR_FILE_CACHE_EXT, nr); + + if (activate && ops->folio_accessed) + ops->folio_accessed(folio); + +skip: + list_move(&folio->lru, return_list); + + remaining--; + + if (!remaining) { + result = false; + break; + } + } + + return result; +} + +static void load_lru_list_to_policy(struct lruvec *lruvec, struct list_head *lru_list, + rwlock_t *policy_lock, unsigned long *policy_lock_flags, + bool activate) +{ + unsigned long flags; + struct folio_batch free_folios; + LIST_HEAD(isolated); + LIST_HEAD(return_list); + LIST_HEAD(putback_list); + + folio_batch_init(&free_folios); + + spin_lock_irqsave(&lruvec->lru_lock, flags); + + while (!isolate_folio_list(lru_list, &isolated)) { + spin_unlock_irqrestore(&lruvec->lru_lock, flags); + write_unlock_irqrestore(policy_lock, *policy_lock_flags); + + cond_resched(); + + write_lock_irqsave(policy_lock, *policy_lock_flags); + spin_lock_irqsave(&lruvec->lru_lock, flags); + } + + spin_unlock_irqrestore(&lruvec->lru_lock, flags); + + while (!add_folio_list_to_policy(&isolated, &return_list, lruvec, activate)) { + write_unlock_irqrestore(policy_lock, *policy_lock_flags); + + cond_resched(); + + write_lock_irqsave(policy_lock, *policy_lock_flags); + } + + spin_lock_irqsave(&lruvec->lru_lock, flags); + + while (!return_folios_to_lru(lru_list, &return_list, + &putback_list, &free_folios)) { + spin_unlock_irqrestore(&lruvec->lru_lock, flags); + write_unlock_irqrestore(policy_lock, *policy_lock_flags); + + while (!list_empty(&putback_list)) { + struct folio *folio = lru_to_folio(&putback_list); + + list_del(&folio->lru); + folio_putback_lru(folio); + } + + mem_cgroup_uncharge_folios(&free_folios); + free_unref_folios(&free_folios); + + cond_resched(); + + write_lock_irqsave(policy_lock, *policy_lock_flags); + spin_lock_irqsave(&lruvec->lru_lock, flags); + } + + spin_unlock_irqrestore(&lruvec->lru_lock, flags); + + WARN_ON(!list_empty(&isolated)); + WARN_ON(!list_empty(&return_list)); + WARN_ON(!list_empty(&putback_list)); +} + +#ifdef CONFIG_LRU_GEN +static void load_mglru_lists_to_policy(struct lruvec *lruvec, rwlock_t *policy_lock, + unsigned long *policy_lock_flags) +{ + int gen, zone, type = LRU_GEN_FILE; + + for (gen = 0; gen < MAX_NR_GENS; gen++) { + bool active = lru_gen_is_active(lruvec, gen); + + for (zone = 0; zone < MAX_NR_ZONES; zone++) { + struct list_head *head = &lruvec->lrugen.folios[gen][type][zone]; + + load_lru_list_to_policy(lruvec, head, + policy_lock, policy_lock_flags, active); + } + } +} +#else +static void load_mglru_lists_to_policy(struct lruvec *lruvec, rwlock_t *policy_lock, + unsigned long *policy_lock_flags) +{ } +#endif + +static void load_lru_lists_to_policy(struct mem_cgroup *memcg, rwlock_t *policy_lock, + unsigned long *policy_lock_flags) +{ + int nid; + + if (!memcg->cache_ext_ptr) + return; + + for_each_node_state(nid, N_MEMORY) { + struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; + + /* + * cgroup_lock protects from concurrent enabling/disabling mglru + */ + if (lru_gen_enabled()) { + load_mglru_lists_to_policy(lruvec, policy_lock, policy_lock_flags); + } else { + load_lru_list_to_policy(lruvec, &lruvec->lists[LRU_INACTIVE_FILE], + policy_lock, policy_lock_flags, false); + load_lru_list_to_policy(lruvec, &lruvec->lists[LRU_ACTIVE_FILE], + policy_lock, policy_lock_flags, true); + } + + } +} + +static bool unload_folios_from_policy(struct mem_cgroup *memcg) +{ + int nid; + struct cache_ext_ops *ops; + int remaining = MAX_CACHE_EXT_BATCH; + + if (!memcg->cache_ext_ptr) + return true; + + ops = memcg->cache_ext_ptr->cache_ext_ops; + + if (!ops) + return true; + + for_each_node_state(nid, N_MEMORY) { + struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec; + struct cache_ext_ds_registry *registry = + memcg->nodeinfo[nid]->cache_ext_ds_registry; + int i; + + for (i = 0; i < CACHE_EXT_REGISTRY_MAX_ENTRIES; i++) { + struct cache_ext_list *head = ®istry->items[i]; + + while (!list_empty(&head->head)) { + struct list_head *prev = head->head.prev; + struct cache_ext_list_node *node = + container_of(prev, struct cache_ext_list_node, node); + struct folio *folio = node->folio; + int nr = folio_nr_pages(folio); + + if (ops->folio_evicted) + ops->folio_evicted(folio); + + cache_ext_folio_meta_fini(folio); + + mod_lruvec_state(lruvec, NR_FILE_CACHE_EXT, -nr); + + remaining--; + + if (!remaining) + return false; + } + + } + registry->nr_entries = 0; + } + + return true; +} + +static void unload_folios_from_prev_policy(struct mem_cgroup *memcg) +{ + struct mem_cgroup *iter = NULL; + unsigned long flags; + struct cache_ext_policy *policy = memcg->cache_ext_ptr; + + cgroup_lock(); + + while ((iter = mem_cgroup_iter(memcg, iter, NULL))) { + write_lock_irqsave(&iter->cache_ext_policy->lock, flags); + if (iter->cache_ext_ptr != policy) { + write_unlock_irqrestore(&iter->cache_ext_policy->lock, flags); + /* + * This memcg is loaded into something else. Skip + */ + continue; + } + + while (!unload_folios_from_policy(iter)) { + write_unlock_irqrestore(&iter->cache_ext_policy->lock, flags); + cond_resched(); + write_lock_irqsave(&iter->cache_ext_policy->lock, flags); + } + + iter->cache_ext_ptr = NULL; + + write_unlock_irqrestore(&iter->cache_ext_policy->lock, flags); + cond_resched(); + } + + cgroup_unlock(); +} + +static void load_folios_to_new_policy(struct mem_cgroup *memcg, struct cache_ext_policy *new_policy) +{ + struct mem_cgroup *iter = NULL; + unsigned long flags; + + cgroup_lock(); + + while ((iter = mem_cgroup_iter(memcg, iter, NULL))) { + cond_resched(); + write_lock_irqsave(&iter->cache_ext_policy->lock, flags); + if (iter->cache_ext_ptr) { + write_unlock_irqrestore(&iter->cache_ext_policy->lock, flags); + /* + * Somethig is already loaded into this memcg. Skip + */ + continue; + } + iter->cache_ext_ptr = new_policy; + + load_lru_lists_to_policy(iter, &iter->cache_ext_policy->lock, &flags); + + write_unlock_irqrestore(&iter->cache_ext_policy->lock, flags); + } + + cgroup_unlock(); +} + + +DEFINE_MUTEX(cache_ext_reg_mutex); + +static int bpf_cache_ext_reg(void *kdata) +{ + struct cache_ext_ops *ops = kdata; + struct mem_cgroup *memcg = get_mem_cgroup_from_current(); + int ret = 0; + + if (memcg == NULL) { + pr_crit("cache_ext: failed to get memcg for registration!\n"); + return -EINVAL; + } + + ops->memcg = memcg; + + mutex_lock(&cache_ext_reg_mutex); + + if (memcg->cache_ext_policy->cache_ext_ops) { + pr_info("cache_ext: something already loaded\n"); + mutex_unlock(&cache_ext_reg_mutex); + mem_cgroup_put(memcg); + return -EINVAL; + } + + lru_cache_disable(); + + unload_folios_from_prev_policy(memcg); + + pr_info("cache_ext: Calling init\n"); + if (ops->init) { + ret = ops->init(memcg); + if (ret) { + struct mem_cgroup *parent_memcg = parent_mem_cgroup(memcg); + struct cache_ext_policy *parent_policy = NULL; + + pr_err("cache_ext: init failed with error code: %d\n", ret); + + if (parent_memcg) + parent_policy = parent_memcg->cache_ext_ptr; + /* + * Return folios to the previous policy + */ + load_folios_to_new_policy(memcg, parent_policy); + + lru_cache_enable(); + mutex_unlock(&cache_ext_reg_mutex); + mem_cgroup_put(memcg); + return ret; + } + } + + memcg->cache_ext_policy->cache_ext_ops = ops; + + load_folios_to_new_policy(memcg, memcg->cache_ext_policy); + + lru_cache_enable(); + mutex_unlock(&cache_ext_reg_mutex); + pr_info("cache_ext: Policy %016lx loaded into memcg %016lx\n", + (unsigned long)ops, (unsigned long)memcg); + return 0; +} + +static void bpf_cache_ext_unreg(void *kdata) +{ + struct cache_ext_ops *prev_ops = kdata; + struct mem_cgroup *memcg = prev_ops->memcg; + struct mem_cgroup *parent_memcg = parent_mem_cgroup(memcg); + struct cache_ext_policy *parent_policy = NULL; + + pr_info("cache_ext: Unregistering struct ops\n"); + + if (memcg == NULL) { + pr_crit("cache_ext: failed to get memcg for release!\n"); + return; + } + + mutex_lock(&cache_ext_reg_mutex); + lru_cache_disable(); + + if (parent_memcg) + parent_policy = parent_memcg->cache_ext_ptr; + + unload_folios_from_prev_policy(memcg); + + memcg->cache_ext_policy->cache_ext_ops = NULL; + + load_folios_to_new_policy(memcg, parent_policy); + + lru_cache_enable(); + mutex_unlock(&cache_ext_reg_mutex); + + pr_info("cache_ext: Policy %016lx unloaded from memcg %016lx\n", + (unsigned long)prev_ops, (unsigned long)memcg); + + + mem_cgroup_put(memcg); +} + +static const struct bpf_verifier_ops bpf_cache_ext_verifier_ops = { + .get_func_proto = bpf_cache_ext_get_func_proto, + .is_valid_access = bpf_cache_ext_is_valid_access, + .btf_struct_access = bpf_cache_ext_btf_struct_access, +}; + +/* + * cache_ext stubs + */ +static s32 cache_ext_init_stub(struct mem_cgroup *memcg) { return -EOPNOTSUPP; } +static int isolate_folios_stub(struct cache_ext_isolation_ctx *ctx, + struct mem_cgroup *memcg) +{ + return -EOPNOTSUPP; +} +static void folio_added_stub(struct folio *folio, s64 cookie) {} +static void folio_accessed_stub(struct folio *folio) {} +static s64 folio_evicted_stub(struct folio *folio) { return -EOPNOTSUPP; } +static void get_statistics_stub(struct lruvec *lruvec, struct cache_ext_stats *stats) {} + +static struct cache_ext_ops __bpf_cache_ext_ops = { + .init = cache_ext_init_stub, + .isolate_folios = isolate_folios_stub, + .folio_added = folio_added_stub, + .folio_accessed = folio_accessed_stub, + .folio_evicted = folio_evicted_stub, + .get_statistics = get_statistics_stub, +}; + +/* + * Callbacks to define the struct_ops map + */ +struct bpf_struct_ops bpf_cache_ext_ops = { + .verifier_ops = &bpf_cache_ext_verifier_ops, + .init = bpf_cache_ext_init, + .reg = bpf_cache_ext_reg, + .unreg = bpf_cache_ext_unreg, + .init_member = bpf_cache_ext_init_member, + .check_member = bpf_cache_ext_check_member, + .validate = bpf_cache_ext_validate, + .cfi_stubs = &__bpf_cache_ext_ops, + .name = "cache_ext_ops", + .owner = THIS_MODULE, +}; + +static int __init cache_ext_ops_init(void) +{ + int ret; + + if (static_branch_unlikely(&cache_ext_disabled)) + return 0; + + ret = register_bpf_struct_ops(&bpf_cache_ext_ops, cache_ext_ops); + + if (ret) { + pr_err("cache_ext: Failed to register struct_ops (%d)\n", ret); + return ret; + } + + return 0; +} + +late_initcall(cache_ext_ops_init); + + +void cache_ext_folio_added(struct folio *folio, bool increase_ref, s64 cookie) +{ + struct mem_cgroup *memcg = folio_memcg(folio); + struct cache_ext_ops *pcext_ops = NULL; + unsigned long flags; + struct lruvec *lruvec = folio_lruvec(folio); + int ret; + + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + if (cookie < 0) + return; + + if (!memcg) + return; + + if (!memcg->cache_ext_ptr) + return; + + if (!folio_is_file_lru(folio)) + return; + + read_lock_irqsave(&memcg->cache_ext_policy->lock, flags); + + pcext_ops = get_cache_ext_ops_memcg(memcg); + + if (pcext_ops && folio_test_cache_ext(folio)) { + read_unlock_irqrestore(&memcg->cache_ext_policy->lock, flags); + return; + } + + if (pcext_ops) { + ret = cache_ext_folio_meta_init(folio, increase_ref); + if (ret) + goto out; + } + + if (pcext_ops && pcext_ops->folio_added) + pcext_ops->folio_added(folio, cookie); + + if (pcext_ops) + mod_lruvec_state(lruvec, NR_FILE_CACHE_EXT, folio_nr_pages(folio)); + +out: + + read_unlock_irqrestore(&memcg->cache_ext_policy->lock, flags); +} + +void cache_ext_folio_accessed(struct folio *folio) +{ + struct mem_cgroup *memcg = folio_memcg(folio); + struct cache_ext_ops *pcext_ops = NULL; + unsigned long flags; + + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + if (!memcg) + return; + + if (!memcg->cache_ext_ptr) + return; + + if (!folio_test_cache_ext(folio)) + return; + + WARN_ON(!folio_is_file_lru(folio)); + + read_lock_irqsave(&memcg->cache_ext_policy->lock, flags); + + pcext_ops = get_cache_ext_ops_memcg(memcg); + if (pcext_ops && pcext_ops->folio_accessed) + pcext_ops->folio_accessed(folio); + + read_unlock_irqrestore(&memcg->cache_ext_policy->lock, flags); +} + +s64 cache_ext_folio_evicted(struct folio *folio) +{ + s64 ret = -EINVAL; + struct mem_cgroup *memcg = folio_memcg(folio); + struct cache_ext_ops *pcext_ops = NULL; + unsigned long flags; + + if (static_branch_unlikely(&cache_ext_disabled)) + return ret; + + if (!memcg) + return ret; + + if (!folio_test_cache_ext(folio)) + return ret; + + if (!memcg->cache_ext_ptr) + return ret; + + WARN_ON(!folio_is_file_lru(folio)); + + read_lock_irqsave(&memcg->cache_ext_policy->lock, flags); + pcext_ops = get_cache_ext_ops_memcg(memcg); + + if (pcext_ops && pcext_ops->folio_evicted) + ret = pcext_ops->folio_evicted(folio); + + if (pcext_ops) { + int cache_ext_ret = cache_ext_folio_meta_fini(folio); + /* + * Folio was part of the policy, adjust the cookie + */ + if (!cache_ext_ret) { + struct lruvec *lruvec = folio_lruvec(folio); + + ret = max(0, ret); + mod_lruvec_state(lruvec, NR_FILE_CACHE_EXT, -folio_nr_pages(folio)); + } else + ret = cache_ext_ret; + } + + read_unlock_irqrestore(&memcg->cache_ext_policy->lock, flags); + return ret; +} + +/* + * Isolate folios from registered cache_ext lists. + * + * The folios are guaranteed to be removed from any registered cache_ext list. + * If a user keeps extra folio information in eBPF-maps, for example, + * then it's his responsibility to remove the information from corresponding structures. + * + * Expected return value: "rc < 0" - on error, "rc = 0" - otherwise. + */ +int cache_ext_isolate_folios(struct cache_ext_isolation_ctx *ctx, struct mem_cgroup *memcg) +{ + int ret = -EINVAL; + struct cache_ext_ops *pcext_ops = NULL; + unsigned long flags; + + if (static_branch_unlikely(&cache_ext_disabled)) + return ret; + + if (!memcg) + return ret; + + if (!memcg->cache_ext_ptr) + return ret; + + read_lock_irqsave(&memcg->cache_ext_policy->lock, flags); + + pcext_ops = get_cache_ext_ops_memcg(memcg); + + if (pcext_ops && pcext_ops->isolate_folios) + ret = pcext_ops->isolate_folios(ctx, memcg); + + read_unlock_irqrestore(&memcg->cache_ext_policy->lock, flags); + return ret; +} + +static bool need_page_ext_cache_ext(void) +{ + if (static_branch_unlikely(&cache_ext_disabled)) + return false; + + return true; +} + +struct page_ext_operations page_ext_cache_ext_ops = { + .size = sizeof(struct cache_ext_list_node), + .need = need_page_ext_cache_ext, + .need_shared_flags = true, +}; + +struct cache_ext_list_node *get_cache_ext_list_node(struct folio *folio) +{ + struct page_ext *page_ext = page_ext_get(folio_page(folio, 0)); + + if (unlikely(!page_ext)) + return NULL; + + return page_ext_data(page_ext, &page_ext_cache_ext_ops); +} + +void put_cache_ext_list_node(struct cache_ext_list_node *node) +{ + struct page_ext *page_ext = + (struct page_ext *)((void *)node - page_ext_cache_ext_ops.offset); + + page_ext_put(page_ext); +} + +void set_cookie_to_folio_page_ext(struct folio *folio, s64 cookie) +{ + struct cache_ext_list_node *node; + + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + node = get_cache_ext_list_node(folio); + + if (unlikely(!node)) + return; + + VM_BUG_ON(folio_test_cache_ext(folio)); + node->cookie = cookie; + put_cache_ext_list_node(node); +} + +s64 get_and_clear_cookie_from_folio_page_ext(struct folio *folio) +{ + struct cache_ext_list_node *node; + s64 cookie; + + if (static_branch_unlikely(&cache_ext_disabled)) + return 0; + + node = get_cache_ext_list_node(folio); + + if (unlikely(!node)) + return 0; + + cookie = node->cookie; + + + VM_BUG_ON(folio_test_cache_ext(folio)); + node->cookie = 0; + put_cache_ext_list_node(node); + return cookie; +} + +int cache_ext_policy_stat_show(struct seq_file *m, void *v) +{ + struct mem_cgroup *memcg = mem_cgroup_from_seq(m); + int nid; + + if (static_branch_unlikely(&cache_ext_disabled)) + return 0; + + if (!memcg) + return 0; + + if (!memcg->cache_ext_ptr) + return 0; + + for_each_node_state(nid, N_MEMORY) { + struct cache_ext_stats stats = {}; + struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid)); + struct cache_ext_ops *pcext_ops = NULL; + unsigned long flags; + int i; + + read_lock_irqsave(&memcg->cache_ext_policy->lock, flags); + pcext_ops = get_cache_ext_ops_memcg(memcg); + if (!pcext_ops || !pcext_ops->get_statistics) { + read_unlock_irqrestore(&memcg->cache_ext_policy->lock, flags); + break; + } + + pcext_ops->get_statistics(lruvec, &stats); + read_unlock_irqrestore(&memcg->cache_ext_policy->lock, flags); + + seq_printf(m, "Node # %d\n", nid); + for (i = 0; i < stats.n && i < MAX_STATS_COUNT; i++) + seq_printf(m, "%.*s : %llu\n", MAX_STAT_LEN, + stats.names[i], stats.values[i]); + } + + return 0; +} + + diff --git a/mm/cache_ext_ds.c b/mm/cache_ext_ds.c new file mode 100644 index 000000000000..dca000894684 --- /dev/null +++ b/mm/cache_ext_ds.c @@ -0,0 +1,1050 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/slab.h> +#include <linux/memcontrol.h> +#include <linux/atomic.h> +#include <linux/bpf.h> +#include <linux/cache_ext.h> +#include <linux/btf.h> +#include <linux/sort.h> +#include <linux/vmalloc.h> +#include <linux/mm_inline.h> + +static int cache_ext_ds_registry_init(struct mem_cgroup *memcg) +{ + int nid; + int i; + + for_each_node(nid) { + struct cache_ext_ds_registry *registry = + kzalloc_node(sizeof(struct cache_ext_ds_registry), GFP_KERNEL, nid); + + if (!registry) + return -ENOMEM; + + if (memcg->nodeinfo[nid]->cache_ext_ds_registry) { + kfree(registry); + WARN_ON(true); + return -EINVAL; + } + + rwlock_init(®istry->lock); + + for (i = 0; i < CACHE_EXT_REGISTRY_MAX_ENTRIES; i++) + INIT_LIST_HEAD(®istry->items[i].head); + + memcg->nodeinfo[nid]->cache_ext_ds_registry = registry; + } + + return 0; +} + + +static void cache_ext_folio_meta_fini_unlocked(struct cache_ext_ds_registry *registry, + struct cache_ext_list_node *node); + +static void cache_ext_ds_registry_free(struct mem_cgroup *memcg) +{ + int nid; + + for_each_node(nid) { + struct cache_ext_ds_registry *registry = + memcg->nodeinfo[nid]->cache_ext_ds_registry; + + kfree(registry); + memcg->nodeinfo[nid]->cache_ext_ds_registry = NULL; + } +} + +int cache_ext_policy_init(struct mem_cgroup *memcg) +{ + struct cache_ext_policy *policy; + + if (static_branch_unlikely(&cache_ext_disabled)) + return 0; + + policy = kmalloc(sizeof(struct cache_ext_policy), GFP_KERNEL | __GFP_ZERO); + + if (!policy) + return -ENOMEM; + + policy->memcg = memcg; + rwlock_init(&policy->lock); + memcg->cache_ext_policy = policy; + + if (cache_ext_ds_registry_init(memcg)) + return -ENOMEM; + + return 0; +} + +void cache_ext_policy_free(struct mem_cgroup *memcg) +{ + if (static_branch_unlikely(&cache_ext_disabled)) + return; + + if (!memcg->cache_ext_policy) + return; + cache_ext_ds_registry_free(memcg); + + kfree(memcg->cache_ext_policy); +} + +static struct cache_ext_list * +cache_ext_ds_registry_get(struct cache_ext_ds_registry *registry, u64 list_id) +{ + struct cache_ext_list *list; + unsigned long flags; + + if (list_id <= 0 || list_id > CACHE_EXT_REGISTRY_MAX_ENTRIES) + return NULL; + + read_lock_irqsave(®istry->lock, flags); + list = ®istry->items[list_id]; + read_unlock_irqrestore(®istry->lock, flags); + return list; +} + +static struct cache_ext_list * +cache_ext_ds_registry_get_unlocked(struct cache_ext_ds_registry *registry, u64 list_id) +{ + struct cache_ext_list *list; + + if (list_id <= 0 || list_id > CACHE_EXT_REGISTRY_MAX_ENTRIES) + return NULL; + + list = ®istry->items[list_id]; + return list; +} + +static struct cache_ext_list * +cache_ext_ds_registry_get_special_list_unlocked(struct cache_ext_ds_registry *registry) +{ + struct cache_ext_list *list; + + list = ®istry->items[0]; + + return list; +} + +static inline struct cache_ext_ds_registry * +cache_ext_ds_registry_from_memcg(struct mem_cgroup *memcg, int nid) +{ + return memcg->nodeinfo[nid]->cache_ext_ds_registry; +} + +static struct cache_ext_ds_registry * +cache_ext_ds_registry_from_folio(struct folio *folio) +{ + /* + * These kfuncs should be called only on page cache + * folios, which are charged and should not return NULL + */ + VM_BUG_ON_FOLIO(!folio_memcg(folio), folio); + + return cache_ext_ds_registry_from_memcg(folio_memcg(folio), folio_nid(folio)); +} + +/* + * Register list for each NUMA node + * Return rergistered list id or 0 in case of error. + * List id ranging from 1 to CACHE_EXT_REGISTRY_MAX_ENTRIES. + * Since this functions is sleepable, can be called only on policy init + */ +static u64 cache_ext_ds_registry_new_list(struct mem_cgroup *memcg) +{ + int nid; + u64 lock_id = 0; + unsigned long flags; + int fail_node; + + for_each_node(nid) { + struct cache_ext_ds_registry *registry = + cache_ext_ds_registry_from_memcg(memcg, nid); + + write_lock_irqsave(®istry->lock, flags); + if (registry->nr_entries >= CACHE_EXT_REGISTRY_MAX_ENTRIES) { + fail_node = nid; + write_unlock_irqrestore(®istry->lock, flags); + goto cleanup; + } + + registry->nr_entries++; + + if (!lock_id) + lock_id = registry->nr_entries; + write_unlock_irqrestore(®istry->lock, flags); + } + + return lock_id; + +cleanup: + for_each_node(nid) { + struct cache_ext_ds_registry *registry = + cache_ext_ds_registry_from_memcg(memcg, nid); + + if (nid == fail_node) + break; + + write_lock_irqsave(®istry->lock, flags); + registry->nr_entries--; + write_unlock_irqrestore(®istry->lock, flags); + } + return 0; +} + +__bpf_kfunc u64 bpf_cache_ext_ds_registry_new_list(struct mem_cgroup *memcg) +{ + return (u64)cache_ext_ds_registry_new_list(memcg); +} + +static int cache_ext_list_add(struct cache_ext_ds_registry *registry, + struct cache_ext_list *list, struct folio *folio, bool tail) +{ + struct cache_ext_list_node *node = get_cache_ext_list_node(folio); + int ret = 0; + unsigned long flags; + + if (unlikely(!node)) + return -EINVAL; + + if (!folio_test_cache_ext(folio)) { + ret = -EINVAL; + goto ret; + } + + write_lock_irqsave(®istry->lock, flags); + + if (!folio_test_cache_ext(folio)) { + ret = -EINVAL; + goto unlock_ret; + } + + if (!folio_test_cache_ext_special_list(folio)) { + ret = -EINVAL; + goto unlock_ret; + } + + folio_clear_cache_ext_special_list(folio); + + if (tail) + list_move_tail(&node->node, &list->head); + else + list_move(&node->node, &list->head); + +unlock_ret: + write_unlock_irqrestore(®istry->lock, flags); +ret: + put_cache_ext_list_node(node); + return ret; +} + +static int cache_ext_list_splice(struct cache_ext_ds_registry *registry, + struct cache_ext_list *list, struct cache_ext_list *head, bool tail) +{ + unsigned long flags; + + write_lock_irqsave(®istry->lock, flags); + + if (tail) + list_splice_tail_init(&list->head, &head->head); + else + list_splice_init(&list->head, &head->head); + + write_unlock_irqrestore(®istry->lock, flags); + + return 0; +} + + +static int cache_ext_list_move(struct cache_ext_ds_registry *registry, + struct cache_ext_list *list, struct folio *folio, + bool tail) +{ + struct cache_ext_list_node *node = get_cache_ext_list_node(folio); + int ret = 0; + unsigned long flags; + + if (unlikely(!node)) + return -EINVAL; + + if (!folio_test_cache_ext(folio)) { + ret = -EINVAL; + goto ret; + } + + write_lock_irqsave(®istry->lock, flags); + if (!folio_test_cache_ext(folio)) { + ret = -EINVAL; + goto unlock_ret; + } + + if (folio_test_cache_ext_special_list(folio)) { + ret = -EEXIST; + goto unlock_ret; + } + + if (tail) + list_move_tail(&node->node, &list->head); + else + list_move(&node->node, &list->head); + + +unlock_ret: + write_unlock_irqrestore(®istry->lock, flags); +ret: + put_cache_ext_list_node(node); + return ret; +} + +static int cache_ext_list_del(struct cache_ext_ds_registry *registry, + struct folio *folio) +{ + struct cache_ext_list_node *node = get_cache_ext_list_node(folio); + struct cache_ext_list *special_list = + cache_ext_ds_registry_get_special_list_unlocked(registry); + + int ret = 0; + unsigned long flags; + + if (unlikely(!node)) + return -EINVAL; + + if (!folio_test_cache_ext(folio)) { + ret = -EINVAL; + goto ret; + } + + write_lock_irqsave(®istry->lock, flags); + if (!folio_test_cache_ext(folio)) { + ret = -EINVAL; + goto unlock_ret; + } + + if (folio_test_cache_ext_special_list(folio)) { + ret = -EINVAL; + goto unlock_ret; + } + + folio_set_cache_ext_special_list(folio); + list_move(&node->node, &special_list->head); + +unlock_ret: + write_unlock_irqrestore(®istry->lock, flags); +ret: + put_cache_ext_list_node(node); + return ret; +} + +enum cache_ext_iter_callback_ret { + CACHE_EXT_CONTINUE_ITER = 0, + CACHE_EXT_STOP_ITER = 1, + CACHE_EXT_ISOLATE_NODE = 2, +}; + +enum cache_ext_iter_ret { + CACHE_EXT_DONE_ITER = 0, + CACHE_EXT_MAX_ITER_REACHED = 8, + CACHE_EXT_EVICT_ARRAY_FILLED = 9, +}; + +static int cache_ext_list_iterate(struct cache_ext_ds_registry *registry, + u64 list_id, void *iter_fn, + struct cache_ext_isolation_ctx *ctx) +{ + uint64_t ret = CACHE_EXT_DONE_ITER, cb_ret, iter = 0; + struct cache_ext_list_node *node, *node2; + struct cache_ext_list *list; + bpf_callback_t bpf_iter_fn = (bpf_callback_t)iter_fn; + unsigned long flags; + + if (ctx->nr_folios_isolated >= ARRAY_SIZE(ctx->folios)) + return CACHE_EXT_EVICT_ARRAY_FILLED; + + write_lock_irqsave(®istry->lock, flags); + list = cache_ext_ds_registry_get_unlocked(registry, list_id); + + if (!list) + goto exit; + + list_for_each_entry_safe(node, node2, &list->head, node) { + struct folio *folio = node->folio; + + if (!folio_test_lru(folio)) + continue; + + /* + * TODO: Ensure that we don't let the callback use any of the list + * helpers, or we will have a deadlock. + */ + cb_ret = bpf_iter_fn((u64)iter, (u64)node, (u64)0, (u64)0, (u64)0); + iter++; + + if (cb_ret == CACHE_EXT_CONTINUE_ITER) { + continue; + } else if (cb_ret == CACHE_EXT_STOP_ITER) { + ret = CACHE_EXT_DONE_ITER; + break; + } else if (cb_ret == CACHE_EXT_ISOLATE_NODE) { + /* + * Isolate should clear cache_ext flag, but keep single + * folio reference. + * Just keep cache_ext folio ref, so don't need extra + * folio_get() for isolate reference. + */ + cache_ext_folio_meta_fini_unlocked(registry, node); + ctx->folios[ctx->nr_folios_isolated] = folio; + ctx->nr_folios_isolated++; + ctx->nr_base_pages_isolated += folio_nr_pages(folio); + + mod_lruvec_state(folio_lruvec(folio), + NR_FILE_CACHE_EXT, -folio_nr_pages(folio)); + + if (ctx->nr_folios_isolated == ARRAY_SIZE(ctx->folios)) { + ret = CACHE_EXT_EVICT_ARRAY_FILLED; + break; + } + } else { + break; + } + } +exit: + write_unlock_irqrestore(®istry->lock, flags); + return ret; +} + +enum cache_ext_iterate_mode { + CACHE_EXT_ITERATE_SKIP = 0, + CACHE_EXT_ITERATE_HEAD, + CACHE_EXT_ITERATE_TAIL, + CACHE_EXT_ITERATE_MAX, +}; + +enum cache_ext_iterate_list { + CACHE_EXT_ITERATE_SELF = 0, +}; + +static bool cache_ext_validate_iterate_opts(struct cache_ext_iterate_opts *opts) +{ + if (opts->continue_mode >= CACHE_EXT_ITERATE_MAX) + return false; + + if (opts->continue_list != CACHE_EXT_ITERATE_SELF && + opts->continue_mode == CACHE_EXT_ITERATE_SKIP) + return false; + return true; +} + +static int cache_ext_list_iterate_extended(struct cache_ext_ds_registry *registry, + u64 list_id, void *iter_fn, + struct cache_ext_iterate_opts *opts, + struct cache_ext_isolation_ctx *ctx) +{ + uint64_t ret = CACHE_EXT_DONE_ITER, cb_ret, iter = 0; + struct cache_ext_list_node *node, *node2; + bpf_callback_t bpf_iter_fn = (bpf_callback_t)iter_fn; + unsigned long flags; + struct cache_ext_list *list = NULL; + struct cache_ext_list *continue_list = NULL; + + if (!cache_ext_validate_iterate_opts(opts)) + return -EINVAL; + + if (ctx->nr_folios_isolated >= ARRAY_SIZE(ctx->folios)) + return CACHE_EXT_EVICT_ARRAY_FILLED; + + if (opts->continue_list != CACHE_EXT_ITERATE_SELF) { + continue_list = cache_ext_ds_registry_get(registry, opts->continue_list); + if (!continue_list) + return -EINVAL; + } + + write_lock_irqsave(®istry->lock, flags); + + list = cache_ext_ds_registry_get_unlocked(registry, list_id); + + if (!list) { + ret = -EINVAL; + goto out_unlock; + } + + if (!continue_list) + continue_list = list; + + list_for_each_entry_safe(node, node2, &list->head, node) { + struct folio *folio = node->folio; + + if (!folio_test_lru(folio)) + cb_ret = CACHE_EXT_CONTINUE_ITER; + else + cb_ret = bpf_iter_fn((u64)iter, (u64)node, (u64)0, (u64)0, (u64)0); + iter++; + + if (cb_ret == CACHE_EXT_CONTINUE_ITER) { + if (opts->continue_mode == CACHE_EXT_ITERATE_HEAD) + list_move(&node->node, &continue_list->head); + else if (opts->continue_mode == CACHE_EXT_ITERATE_TAIL) + list_move_tail(&node->node, &continue_list->head); + + opts->nr_folios_continue++; + + continue; + } else if (cb_ret == CACHE_EXT_STOP_ITER) { + ret = CACHE_EXT_DONE_ITER; + break; + } else if (cb_ret == CACHE_EXT_ISOLATE_NODE) { + /* + * Isolate should clear cache_ext flag, but keep single + * folio reference. + * Just keep cache_ext folio ref, so don't need extra + * folio_get() for isolate reference. + */ + cache_ext_folio_meta_fini_unlocked(registry, node); + ctx->folios[ctx->nr_folios_isolated] = folio; + ctx->nr_folios_isolated++; + ctx->nr_base_pages_isolated += folio_nr_pages(folio); + opts->nr_folios_isolated++; + + mod_lruvec_state(folio_lruvec(folio), + NR_FILE_CACHE_EXT, -folio_nr_pages(folio)); + + if (ctx->nr_folios_isolated == ARRAY_SIZE(ctx->folios)) { + ret = CACHE_EXT_EVICT_ARRAY_FILLED; + break; + } + } else + break; + } +out_unlock: + write_unlock_irqrestore(®istry->lock, flags); + return ret; +} + +int cache_ext_folio_meta_init(struct folio *folio, bool increase_ref) +{ + struct cache_ext_list_node *node = get_cache_ext_list_node(folio); + struct cache_ext_ds_registry *registry = cache_ext_ds_registry_from_folio(folio); + struct cache_ext_list *special_list; + + unsigned long flags; + + if (unlikely(!node)) + return -EINVAL; + + if (increase_ref) + folio_get(folio); + + INIT_LIST_HEAD(&node->node); + node->folio = folio; + + /* Paired with smp_rmb() after folio_test_cache_ext() + * Required to correctly order node initilaiztion with flag setting. + */ + + write_lock_irqsave(®istry->lock, flags); + + special_list = cache_ext_ds_registry_get_special_list_unlocked(registry); + + folio_set_cache_ext(folio); + folio_set_cache_ext_special_list(folio); + + list_add(&node->node, &special_list->head); + + write_unlock_irqrestore(®istry->lock, flags); + + + put_cache_ext_list_node(node); + + return 0; +} + +static void cache_ext_folio_meta_fini_unlocked(struct cache_ext_ds_registry *registry, + struct cache_ext_list_node *node) +{ + folio_clear_cache_ext(node->folio); + folio_clear_cache_ext_special_list(node->folio); + + list_del(&node->node); + node->folio = NULL; +} + +/* Returns 0 if folio was removed from cache_ext */ +int cache_ext_folio_meta_fini(struct folio *folio) +{ + struct cache_ext_ds_registry *registry = cache_ext_ds_registry_from_folio(folio); + struct cache_ext_list_node *node = get_cache_ext_list_node(folio); + unsigned long flags; + int ret = -EINVAL; + + if (unlikely(!node)) + return -EINVAL; + + write_lock_irqsave(®istry->lock, flags); + if (node->folio) { + cache_ext_folio_meta_fini_unlocked(registry, node); + folio_put(folio); + ret = 0; + } + write_unlock_irqrestore(®istry->lock, flags); + + put_cache_ext_list_node(node); + return ret; +} + +__bpf_kfunc int +bpf_cache_ext_list_splice(u64 list_id, u64 head_id, struct mem_cgroup *memcg, int nid) +{ + struct cache_ext_ds_registry *registry; + struct cache_ext_list *list; + struct cache_ext_list *head; + + if (nid < 0 || nid >= MAX_NUMNODES) + return -EINVAL; + + if (!node_online(nid) || !node_state(nid, N_MEMORY)) + return -EINVAL; + + if (list_id == head_id) + return -EINVAL; + + registry = cache_ext_ds_registry_from_memcg(memcg, nid); + list = cache_ext_ds_registry_get(registry, list_id); + head = cache_ext_ds_registry_get(registry, head_id); + + if (!list || !head) + return -EINVAL; + + return cache_ext_list_splice(registry, list, head, false); +}; + + +__bpf_kfunc int +bpf_cache_ext_list_splice_tail(u64 list_id, u64 head_id, struct mem_cgroup *memcg, int nid) +{ + struct cache_ext_ds_registry *registry; + struct cache_ext_list *list; + struct cache_ext_list *head; + + + if (nid < 0 || nid >= MAX_NUMNODES) + return -EINVAL; + + if (!node_online(nid) || !node_state(nid, N_MEMORY)) + return -EINVAL; + + if (list_id == head_id) + return -EINVAL; + + registry = cache_ext_ds_registry_from_memcg(memcg, nid); + list = cache_ext_ds_registry_get(registry, list_id); + head = cache_ext_ds_registry_get(registry, head_id); + + if (!list || !head) + return -EINVAL; + + return cache_ext_list_splice(registry, list, head, true); +}; + +__bpf_kfunc int bpf_cache_ext_list_add(u64 list_id, struct folio *folio) +{ + struct cache_ext_ds_registry *registry = cache_ext_ds_registry_from_folio(folio); + struct cache_ext_list *list = cache_ext_ds_registry_get(registry, list_id); + + if (!list) + return -EINVAL; + + return cache_ext_list_add(registry, list, folio, false); +}; + +__bpf_kfunc int bpf_cache_ext_list_add_tail(u64 list_id, struct folio *folio) +{ + struct cache_ext_ds_registry *registry = cache_ext_ds_registry_from_folio(folio); + struct cache_ext_list *list = cache_ext_ds_registry_get(registry, list_id); + + if (!list) + return -EINVAL; + + return cache_ext_list_add(registry, list, folio, true); +}; + +__bpf_kfunc int bpf_cache_ext_list_move(u64 list_id, struct folio *folio, bool tail) +{ + struct cache_ext_ds_registry *registry = cache_ext_ds_registry_from_folio(folio); + struct cache_ext_list *list = cache_ext_ds_registry_get(registry, list_id); + + if (!list) + return -EINVAL; + + return cache_ext_list_move(registry, list, folio, tail); +}; + +__bpf_kfunc int bpf_cache_ext_list_del(struct folio *folio) +{ + struct cache_ext_ds_registry *registry = cache_ext_ds_registry_from_folio(folio); + + return cache_ext_list_del(registry, folio); +}; + +__bpf_kfunc int bpf_cache_ext_list_iterate( + struct mem_cgroup *memcg, u64 list_id, + int (*iter_fn)(int idx, struct cache_ext_list_node *node), + struct cache_ext_isolation_ctx *ctx) +{ + struct cache_ext_ds_registry *registry = + cache_ext_ds_registry_from_memcg(memcg, ctx->numa_node_id); + + if (ctx->nr_folios_to_isolate > ARRAY_SIZE(ctx->folios)) + return -EINVAL; + + if (!registry) + return -EINVAL; + + return cache_ext_list_iterate(registry, list_id, (void *)iter_fn, ctx); +}; + +__bpf_kfunc int bpf_cache_ext_list_iterate_extended( + struct mem_cgroup *memcg, u64 list_id, + int (*iter_fn)(int idx, struct cache_ext_list_node *node), + struct cache_ext_iterate_opts *opts, + struct cache_ext_isolation_ctx *ctx) +{ + struct cache_ext_ds_registry *registry; + + if (!opts) + return -EINVAL; + + if (ctx->nr_folios_to_isolate > ARRAY_SIZE(ctx->folios)) + return -EINVAL; + + registry = cache_ext_ds_registry_from_memcg(memcg, ctx->numa_node_id); + if (!registry) + return -EINVAL; + + return cache_ext_list_iterate_extended(registry, list_id, (void *)iter_fn, opts, ctx); +}; + +#define MAX_SAMPLE_FOLIOS 2048 +DEFINE_PER_CPU(struct cache_ext_list_node *, sample_folios[MAX_SAMPLE_FOLIOS]); + +static void __putback_list_nodes(struct cache_ext_list *list, + struct cache_ext_list_node **sample_folios_arr, int size) +{ + for (int i = 0; i < size; i++) { + if (sample_folios_arr[i]->folio != NULL) + list_add_tail(&sample_folios_arr[i]->node, &list->head); + } +} + +static int __bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list_id, + s32 (*score_fn)(struct cache_ext_list_node *a), + struct sampling_options *opts, + struct cache_ext_isolation_ctx *ctx) +{ + unsigned int sample_size = opts->sample_size; + int i = 0; + unsigned long num_folios_to_sample = ctx->nr_folios_to_isolate * sample_size; + unsigned long flags; + struct cache_ext_list_node **sample_folios_arr; + struct cache_ext_ds_registry *registry; + struct cache_ext_list *list_ptr; + int sample_folios_size = 0; + int sample_folios_idx = 0; + + if (num_folios_to_sample > MAX_SAMPLE_FOLIOS) + return -EINVAL; + + sample_folios_arr = this_cpu_ptr(sample_folios); + + registry = cache_ext_ds_registry_from_memcg(memcg, ctx->numa_node_id); + write_lock_irqsave(®istry->lock, flags); + list_ptr = cache_ext_ds_registry_get_unlocked(registry, list_id); + + if (!list_ptr) { + write_unlock_irqrestore(®istry->lock, flags); + return -EINVAL; + } + + while (i < num_folios_to_sample) { + struct cache_ext_list_node *node; + + if (list_empty(&list_ptr->head)) { + if (i == 0) { + __putback_list_nodes(list_ptr, sample_folios_arr, + sample_folios_size); + + write_unlock_irqrestore(®istry->lock, flags); + return -EINVAL; + } + + if (sample_folios_size < ctx->nr_folios_to_isolate) { + ctx->nr_folios_to_isolate = i; + sample_size = 1; + break; + } + + sample_size = sample_folios_size / ctx->nr_folios_to_isolate; + break; + } + node = list_first_entry( + &list_ptr->head, struct cache_ext_list_node, node); + + if (!folio_test_cache_ext(node->folio)) { + list_del_init(&node->node); + continue; + } + + sample_folios_arr[i] = node; + sample_folios_size++; + i++; + list_del_init(&node->node); + } + + ctx->nr_folios_isolated = 0; + for (int i = 0; i < ctx->nr_folios_to_isolate; i++) { + struct cache_ext_list_node *min_node = sample_folios_arr[sample_folios_idx]; + struct folio *folio = NULL; + s64 min_score = score_fn(min_node); + + sample_folios_idx++; + + for (int j = 1; j < sample_size; j++) { + struct cache_ext_list_node *curr_node = + sample_folios_arr[sample_folios_idx]; + s64 curr_score = score_fn(curr_node); + + sample_folios_idx++; + if (curr_score < min_score) { + min_score = curr_score; + min_node = curr_node; + } + } + + folio = min_node->folio; + + cache_ext_folio_meta_fini_unlocked(registry, min_node); + ctx->folios[ctx->nr_folios_isolated] = folio; + ctx->scores[ctx->nr_folios_isolated] = min_score; + ctx->nr_folios_isolated++; + ctx->nr_base_pages_isolated += folio_nr_pages(folio); + + mod_lruvec_state(folio_lruvec(folio), + NR_FILE_CACHE_EXT, -folio_nr_pages(folio)); + } + + __putback_list_nodes(list_ptr, sample_folios_arr, sample_folios_size); + write_unlock_irqrestore(®istry->lock, flags); + + return 0; +} + +__bpf_kfunc int +bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, + s32(score_fn)(struct cache_ext_list_node *a), + struct sampling_options *opts, + struct cache_ext_isolation_ctx *ctx) +{ + if (opts->sample_size == 0) + return -EINVAL; + + if (ctx->nr_folios_to_isolate > ARRAY_SIZE(ctx->folios)) + return -EINVAL; + + scoped_guard(preempt) { + return __bpf_cache_ext_list_sample(memcg, list, score_fn, opts, ctx); + } +} + +#define MAX_STRSTR_LEN (128) + +__bpf_kfunc bool +bpf_cache_ext_filename_strstr(const char *str, u32 str__sz, struct folio *folio) +{ + struct address_space *mapping; + struct inode *inode; + struct dentry *dentry; + int ret = 0; + + if (!str) + return ret; + + if (str__sz == 0) + return true; + + if (str__sz > MAX_STRSTR_LEN || str[str__sz - 1] != '\0') + return ret; + + if (!folio) + return ret; + + if (!folio_is_file_lru(folio)) + return ret; + + rcu_read_lock(); + + mapping = folio_mapping(folio); + + if (!mapping) + goto out; + + inode = mapping->host; + + if (!inode) + goto out; + + dentry = d_find_alias_rcu(inode); + if (dentry) { + const char *name; + + spin_lock(&dentry->d_lock); + name = dentry->d_name.name; + + if (name && (strnstr(name, str, MAX_STRSTR_LEN))) + ret = 1; + + spin_unlock(&dentry->d_lock); + } +out: + rcu_read_unlock(); + return ret; +} + + +__bpf_kfunc bool bpf_cache_ext_list_empty(u64 list_id, struct mem_cgroup *memcg, int nid) +{ + bool empty = true; + unsigned long flags; + struct cache_ext_ds_registry *registry; + struct cache_ext_list *list; + + if (!node_online(nid) || !node_state(nid, N_MEMORY)) + return empty; + + registry = cache_ext_ds_registry_from_memcg(memcg, nid); + list = cache_ext_ds_registry_get(registry, list_id); + + if (!list) + return empty; + + read_lock_irqsave(®istry->lock, flags); + empty = list_empty(&list->head); + read_unlock_irqrestore(®istry->lock, flags); + + return empty; +} + +__bpf_kfunc int bpf_folio_nid(struct folio *folio) +{ + return folio_nid(folio); +} + +enum cache_ext_list_ops_type { + KF_bpf_cache_ext_list_add = 0, + KF_bpf_cache_ext_list_add_tail, + KF_bpf_cache_ext_list_del, + KF_bpf_cache_ext_list_iterate, + KF_bpf_cache_ext_list_sample, + KF_bpf_cache_ext_list_move, + KF_bpf_cache_ext_list_iterate_extended, + KF_bpf_cache_ext_filename_strstr, + KF_bpf_cache_ext_list_empty, + KF_bpf_folio_nid, + KF_bpf_cache_ext_list_splice, + KF_bpf_cache_ext_list_splice_tail, + __CACHE_EXT_KF_NR +}; + +BTF_KFUNCS_START(cache_ext_list_ops) +BTF_ID_FLAGS(func, bpf_cache_ext_list_add) +BTF_ID_FLAGS(func, bpf_cache_ext_list_add_tail) +BTF_ID_FLAGS(func, bpf_cache_ext_list_del) +BTF_ID_FLAGS(func, bpf_cache_ext_list_iterate) +BTF_ID_FLAGS(func, bpf_cache_ext_list_sample) +BTF_ID_FLAGS(func, bpf_cache_ext_list_move) +BTF_ID_FLAGS(func, bpf_cache_ext_list_iterate_extended) +BTF_ID_FLAGS(func, bpf_cache_ext_filename_strstr) +BTF_ID_FLAGS(func, bpf_cache_ext_list_empty) +BTF_ID_FLAGS(func, bpf_folio_nid) +BTF_ID_FLAGS(func, bpf_cache_ext_list_splice) +BTF_ID_FLAGS(func, bpf_cache_ext_list_splice_tail) +BTF_KFUNCS_END(cache_ext_list_ops) + +BTF_ID_LIST(cache_ext_list_ops_list) +BTF_ID(func, bpf_cache_ext_list_add) +BTF_ID(func, bpf_cache_ext_list_add_tail) +BTF_ID(func, bpf_cache_ext_list_del) +BTF_ID(func, bpf_cache_ext_list_iterate) +BTF_ID(func, bpf_cache_ext_list_sample) +BTF_ID(func, bpf_cache_ext_list_move) +BTF_ID(func, bpf_cache_ext_list_iterate_extended) +BTF_ID(func, bpf_cache_ext_filename_strstr) +BTF_ID(func, bpf_cache_ext_list_empty) +BTF_ID(func, bpf_folio_nid) +BTF_ID(func, bpf_cache_ext_list_splice) +BTF_ID(func, bpf_cache_ext_list_splice_tail) + +bool cache_ext_kfunc(u32 btf_id) +{ + int i; + + if (static_branch_unlikely(&cache_ext_disabled)) + return false; + + for (i = 0; i < __CACHE_EXT_KF_NR; i++) + if (btf_id == cache_ext_list_ops_list[i]) + return true; + + return false; +} + +bool cache_ext_is_callback_calling_kfunc_iterate(u32 btf_id) +{ + if (static_branch_unlikely(&cache_ext_disabled)) + return false; + + return (btf_id == cache_ext_list_ops_list[KF_bpf_cache_ext_list_iterate] || + btf_id == cache_ext_list_ops_list[KF_bpf_cache_ext_list_iterate_extended]); +} + +bool cache_ext_is_callback_calling_kfunc_sample(u32 btf_id) +{ + if (static_branch_unlikely(&cache_ext_disabled)) + return false; + + return (btf_id == cache_ext_list_ops_list[KF_bpf_cache_ext_list_sample]); +} + +static const struct btf_kfunc_id_set cache_ext_kfunc_set_list_ops = { + .owner = THIS_MODULE, + .set = &cache_ext_list_ops, +}; + +BTF_KFUNCS_START(cache_ext_registry_ops) +BTF_ID_FLAGS(func, bpf_cache_ext_ds_registry_new_list, KF_SLEEPABLE) +BTF_KFUNCS_END(cache_ext_registry_ops) + +static const struct btf_kfunc_id_set cache_ext_kfunc_set_registry_ops = { + .owner = THIS_MODULE, + .set = &cache_ext_registry_ops, +}; + +static int __init register_cache_ext_kfuncs(void) +{ + int ret = 0; + + if (static_branch_unlikely(&cache_ext_disabled)) + return 0; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &cache_ext_kfunc_set_list_ops) || + register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &cache_ext_kfunc_set_registry_ops); + + if (ret) { + pr_err("cache_ext: failed to register kfunc sets (%d)\n", ret); + return ret; + } + + return 0; +} +late_initcall(register_cache_ext_kfuncs); diff --git a/mm/filemap.c b/mm/filemap.c index a94d7dd3f0ff..58dd2fde49a6 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -255,6 +255,9 @@ void filemap_remove_folio(struct folio *folio) struct address_space *mapping = folio->mapping; BUG_ON(!folio_test_locked(folio)); + + cache_ext_folio_evicted(folio); + spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); __filemap_remove_folio(folio, NULL); @@ -328,6 +331,12 @@ void delete_from_page_cache_batch(struct address_space *mapping, if (!folio_batch_count(fbatch)) return; + for (i = 0; i < folio_batch_count(fbatch); i++) { + struct folio *folio = fbatch->folios[i]; + + cache_ext_folio_evicted(folio); + } + spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); for (i = 0; i < folio_batch_count(fbatch); i++) { @@ -975,6 +984,7 @@ int filemap_add_folio(struct address_space *mapping, struct folio *folio, if (!(gfp & __GFP_WRITE) && shadow) workingset_refault(folio, shadow); folio_add_lru(folio); + cache_ext_folio_added(folio, true, 0); } return ret; } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index a67d363accad..f6be624801e1 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -68,6 +68,7 @@ #include <linux/sched/isolation.h> #include <linux/parser.h> #include <linux/dynamic_pool.h> +#include <linux/cache_ext.h> #ifdef CONFIG_MEMCG_SWAP_QOS #include <linux/blkdev.h> @@ -1625,6 +1626,11 @@ static int memcg_page_state_unit(int item) case WORKINGSET_RESTORE_ANON: case WORKINGSET_RESTORE_FILE: case WORKINGSET_NODERECLAIM: +#ifdef CONFIG_CACHE_EXT + case NR_FILE_CACHE_EXT: + case NR_FILE_CACHE_EXT_KSWAPD_RECLAIMED: + case NR_FILE_CACHE_EXT_REFAULT: +#endif return 1; case NR_KERNEL_STACK_KB: return SZ_1K; @@ -4972,6 +4978,11 @@ static const unsigned int memcg1_stats[] = { MEMCG_SWAP, NR_SWAPCACHE, #endif +#ifdef CONFIG_CACHE_EXT + NR_FILE_CACHE_EXT, + NR_FILE_CACHE_EXT_KSWAPD_RECLAIMED, + NR_FILE_CACHE_EXT_REFAULT, +#endif }; static const char *const memcg1_stat_names[] = { @@ -4990,6 +5001,12 @@ static const char *const memcg1_stat_names[] = { "swap", "swapcached", #endif +#ifdef CONFIG_CACHE_EXT + "file_cache_ext", + "file_cache_ext_reclaimed_kswapd", + "file_cache_ext_refault", + +#endif }; /* Universal VM events cgroup1 shows, original sort order */ @@ -6273,6 +6290,12 @@ static struct cftype mem_cgroup_legacy_files[] = { .name = "stat", .seq_show = memory_stat_show, }, +#ifdef CONFIG_CACHE_EXT + { + .name = "cache_ext_policy_stat", + .seq_show = cache_ext_policy_stat_show, + }, +#endif { .name = "force_empty", .write = mem_cgroup_force_empty_write, @@ -6628,6 +6651,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg) if (memcg->orig_objcg) obj_cgroup_put(memcg->orig_objcg); + cache_ext_policy_free(memcg); for_each_node(node) free_mem_cgroup_per_node_info(memcg, node); kfree(memcg->vmstats); @@ -6676,6 +6700,9 @@ static struct mem_cgroup *mem_cgroup_alloc(void) if (alloc_mem_cgroup_per_node_info(memcg, node)) goto fail; + if (cache_ext_policy_init(memcg)) + goto fail; + if (memcg_wb_domain_init(memcg, GFP_KERNEL)) goto fail; @@ -6784,6 +6811,16 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css) FLUSH_TIME); lru_gen_online_memcg(memcg); +#ifdef CONFIG_CACHE_EXT + if (parent_mem_cgroup(memcg)) + /* + * cache_ext_ptr setting is protected via cgroup_lock + * on policy registration. We hold the same lock here, + * so there is no need in additinal synchronisation + */ + memcg->cache_ext_ptr = parent_mem_cgroup(memcg)->cache_ext_ptr; +#endif + #ifdef CONFIG_MEMCG_OOM_PRIORITY memcg_oom_prio_init(memcg); #endif @@ -8308,6 +8345,12 @@ static struct cftype memory_files[] = { .name = "stat", .seq_show = memory_stat_show, }, +#ifdef CONFIG_CACHE_EXT + { + .name = "cache_ext_policy_stat", + .seq_show = cache_ext_policy_stat_show, + }, +#endif #ifdef CONFIG_NUMA { .name = "numa_stat", diff --git a/mm/migrate.c b/mm/migrate.c index 4edd29d9a041..44cc91e0587e 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -433,6 +433,7 @@ static int __folio_migrate_mapping(struct address_space *mapping, int dirty; long nr = folio_nr_pages(folio); long entries, i; + s64 cache_ext_cookie = 0; if (!mapping) { /* Take off deferred split queue while frozen and memcg set */ @@ -458,12 +459,19 @@ static int __folio_migrate_mapping(struct address_space *mapping, oldzone = folio_zone(folio); newzone = folio_zone(newfolio); + if (folio_is_file_lru(folio)) + cache_ext_cookie = cache_ext_folio_evicted(folio); + xas_lock_irq(&xas); if (!folio_ref_freeze(folio, expected_count)) { xas_unlock_irq(&xas); + if (folio_is_file_lru(folio) && cache_ext_cookie >= 0) + cache_ext_folio_added(folio, true, cache_ext_cookie); return -EAGAIN; } + set_cookie_to_folio_page_ext(newfolio, cache_ext_cookie); + /* Take off deferred split queue while frozen and memcg set */ folio_unqueue_deferred_split(folio); @@ -566,7 +574,7 @@ int folio_migrate_mapping(struct address_space *mapping, { int expected_count = folio_expected_refs(mapping, folio) + extra_count; - if (folio_ref_count(folio) != expected_count) + if (folio_ref_count(folio) != expected_count + folio_test_cache_ext(folio)) return -EAGAIN; return __folio_migrate_mapping(mapping, newfolio, folio, expected_count); @@ -577,7 +585,7 @@ static int folio_migrate_mc_copy(struct folio *dst, struct folio *src, enum migrate_mode mode, int expected_count) { /* Check whether src does not have extra refs before we do more work */ - if (folio_ref_count(src) != expected_count) + if (folio_ref_count(src) != expected_count + folio_test_cache_ext(src)) return -EAGAIN; if (mode == MIGRATE_SYNC_NO_COPY) @@ -813,7 +821,7 @@ static int __buffer_migrate_folio(struct address_space *mapping, /* Check whether page does not have extra refs before we do more work */ expected_count = folio_expected_refs(mapping, src); - if (folio_ref_count(src) != expected_count) + if (folio_ref_count(src) != expected_count + folio_test_cache_ext(src)) return -EAGAIN; if (!buffer_migrate_lock_buffers(head, mode)) @@ -1347,6 +1355,8 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private, * isolated from the unevictable LRU: but this case is the easiest. */ folio_add_lru(dst); + if (folio_is_file_lru(dst)) + cache_ext_folio_added(dst, true, get_and_clear_cookie_from_folio_page_ext(dst)); if (old_page_state & PAGE_WAS_MLOCKED) lru_add_drain(); diff --git a/mm/page_ext.c b/mm/page_ext.c index 4548fcc66d74..c29c115232cf 100644 --- a/mm/page_ext.c +++ b/mm/page_ext.c @@ -75,6 +75,10 @@ static struct page_ext_operations page_idle_ops __initdata = { }; #endif +#ifdef CONFIG_CACHE_EXT +extern struct page_ext_operations page_ext_cache_ext_ops; +#endif + static struct page_ext_operations *page_ext_ops[] __initdata = { #ifdef CONFIG_PAGE_OWNER &page_owner_ops, @@ -85,6 +89,9 @@ static struct page_ext_operations *page_ext_ops[] __initdata = { #ifdef CONFIG_PAGE_TABLE_CHECK &page_table_check_ops, #endif +#ifdef CONFIG_CACHE_EXT + &page_ext_cache_ext_ops, +#endif }; unsigned long page_ext_size; diff --git a/mm/swap.c b/mm/swap.c index 3c0e14640951..607c8a521351 100644 --- a/mm/swap.c +++ b/mm/swap.c @@ -449,6 +449,7 @@ static void folio_inc_refs(struct folio *folio) */ void folio_mark_accessed(struct folio *folio) { + cache_ext_folio_accessed(folio); if (lru_gen_enabled()) { folio_inc_refs(folio); return; diff --git a/mm/truncate.c b/mm/truncate.c index 1557a0503f8e..c1488c290f8c 100644 --- a/mm/truncate.c +++ b/mm/truncate.c @@ -21,6 +21,8 @@ #include <linux/task_io_accounting_ops.h> #include <linux/shmem_fs.h> #include <linux/rmap.h> +#include <linux/cache_ext.h> + #include "internal.h" /* @@ -286,7 +288,8 @@ long mapping_evict_folio(struct address_space *mapping, struct folio *folio) return 0; /* The refcount will be elevated if any page in the folio is mapped */ if (folio_ref_count(folio) > - folio_nr_pages(folio) + folio_has_private(folio) + 1) + folio_nr_pages(folio) + folio_has_private(folio) + 1 + + !!folio_test_cache_ext(folio)) return 0; if (!filemap_release_folio(folio, 0)) return 0; @@ -561,12 +564,15 @@ EXPORT_SYMBOL(invalidate_mapping_pages); static int invalidate_complete_folio2(struct address_space *mapping, struct folio *folio) { + s64 cookie; if (folio->mapping != mapping) return 0; if (!filemap_release_folio(folio, GFP_KERNEL)) return 0; + cookie = cache_ext_folio_evicted(folio); + spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); if (folio_test_dirty(folio)) @@ -584,6 +590,10 @@ static int invalidate_complete_folio2(struct address_space *mapping, failed: xa_unlock_irq(&mapping->i_pages); spin_unlock(&mapping->host->i_lock); + + if (cookie >= 0) + cache_ext_folio_added(folio, true, cookie); + return 0; } diff --git a/mm/vmscan.c b/mm/vmscan.c index 9e4aedbfd85e..f521d2d82a69 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -444,7 +444,7 @@ static inline int is_page_cache_freeable(struct folio *folio) * private data at folio->private. */ return folio_ref_count(folio) - folio_test_private(folio) == - 1 + folio_nr_pages(folio); + 1 + folio_nr_pages(folio) + folio_test_cache_ext(folio); } /* @@ -699,10 +699,18 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, { int refcount; void *shadow = NULL; + s64 cookie = -EINVAL; BUG_ON(!folio_test_locked(folio)); BUG_ON(mapping != folio_mapping(folio)); + if (folio_is_file_lru(folio)) { + if (folio_test_cache_ext(folio) && + folio_ref_count(folio) > 2 + folio_nr_pages(folio)) + goto out; + cookie = cache_ext_folio_evicted(folio); + } + if (!folio_test_swapcache(folio)) spin_lock(&mapping->host->i_lock); xa_lock_irq(&mapping->i_pages); @@ -788,6 +796,12 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, xa_unlock_irq(&mapping->i_pages); if (!folio_test_swapcache(folio)) spin_unlock(&mapping->host->i_lock); + /* + * If the folio was successfully evicted above - re-add back + */ + if (folio_is_file_lru(folio) && (cookie >= 0)) + cache_ext_folio_added(folio, true, cookie); +out: return 0; } @@ -2581,6 +2595,187 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, return can_demote(pgdat->node_id, sc); } +#ifdef CONFIG_CACHE_EXT + +bool allow_direct_cache_ext = true; + +static unsigned int reclaim_cache_ext_folio_list(struct list_head *folio_list, + struct pglist_data *pgdat, + bool ignore_references) +{ + struct reclaim_stat dummy_stat; + unsigned int nr_reclaimed; + struct folio *folio; + struct scan_control sc = { + .gfp_mask = GFP_KERNEL, + .may_writepage = 0, + .may_unmap = 1, + .may_swap = 0, + .no_demotion = 0, + }; + + nr_reclaimed = shrink_folio_list(folio_list, pgdat, &sc, &dummy_stat, ignore_references); + while (!list_empty(folio_list)) { + folio = lru_to_folio(folio_list); + list_del(&folio->lru); + + if (folio_is_file_lru(folio)) + cache_ext_folio_added(folio, true, + get_and_clear_cookie_from_folio_page_ext(folio)); + + folio_putback_lru(folio); + } + + return nr_reclaimed; +} + +static bool isolate_cache_ext_folio(struct folio *folio) +{ + struct lruvec *lruvec; + + if (WARN_ON(folio_test_anon(folio))) + return false; + + if (folio_test_unevictable(folio)) + return false; + + if (!folio_test_clear_lru(folio)) + return false; + + folio_clear_referenced(folio); + folio_test_clear_young(folio); + folio_clear_reclaim(folio); + + lruvec = folio_lruvec(folio); + + lruvec_del_folio(lruvec, folio); + folio_clear_active(folio); + + return true; +} + +static void return_isolated_folios_to_policy(struct cache_ext_isolation_ctx *ctx) +{ + for (int i = 0; i < ctx->nr_folios_isolated; i++) { + struct folio *folio = ctx->folios[i]; + + if (!folio) + continue; + + VM_BUG_ON(!pfn_valid(folio_pfn(folio))); + + cache_ext_folio_added(folio, false, ctx->cookies[i]); + } +} + +static unsigned long cache_ext_isolate_and_reclaim(struct lruvec *lruvec, int nr_to_reclaim) +{ + int ret = 0; + LIST_HEAD(free_folios); + unsigned long nr_reclaimed = 0; + struct cache_ext_isolation_ctx ctx; + int nr_to_reclaim_batch = min(CACHE_EXT_NR_ISOLATED, nr_to_reclaim); + int nid = lruvec_pgdat(lruvec)->node_id; + + while (nr_to_reclaim > 0) { + unsigned long nr_reclaimed_batch = 0; + + memset(&ctx, 0, sizeof(ctx)); + ctx.nr_folios_to_isolate = nr_to_reclaim_batch; + ctx.numa_node_id = nid; + + ret = cache_ext_isolate_folios(&ctx, lruvec_memcg(lruvec)); + + /* + * If for some reason, policy return error from isolation, + * but isolated something, we need to return such folios to the policy. + * Only kfuncs can populate ctx.folios, so it is guranteed that they are valid + */ + if (ret < 0) { + return_isolated_folios_to_policy(&ctx); + goto out; + } + + /* + * ctx.folios is populated only via kfuncs, + * policy can not perform direct writes into it. + * So, we have full control over its conten and + * VM_BUG_ON conditions are bugs + */ + spin_lock_irq(&lruvec->lru_lock); + for (int i = 0; i < ctx.nr_folios_isolated; i++) { + struct folio *untrusted_folio = ctx.folios[i]; + + VM_BUG_ON(!untrusted_folio); + VM_BUG_ON(!pfn_valid(folio_pfn(untrusted_folio))); + + set_cookie_to_folio_page_ext(untrusted_folio, ctx.cookies[i]); + + VM_BUG_ON_FOLIO(lruvec != folio_lruvec(untrusted_folio), untrusted_folio); + + if (!isolate_cache_ext_folio(untrusted_folio)) + continue; + + list_add(&untrusted_folio->lru, &free_folios); + + /* + * Keep in this array only folios which must be re-added to cache_ext + * due to isolation failure or lruvec mismatch + */ + ctx.folios[i] = NULL; + } + spin_unlock_irq(&lruvec->lru_lock); + + return_isolated_folios_to_policy(&ctx); + + current->flags |= PF_CACHE_EXT_RECLAIM; + nr_reclaimed_batch = + reclaim_cache_ext_folio_list(&free_folios, NODE_DATA(nid), true); + current->flags &= ~PF_CACHE_EXT_RECLAIM; + + nr_reclaimed += nr_reclaimed_batch; + nr_to_reclaim -= max(CACHE_EXT_NR_ISOLATED, nr_reclaimed_batch); + + cond_resched(); + } +out: + return nr_reclaimed; +} + +static unsigned long +kswapd_reclaim_from_cache_ext_memcg(struct lruvec *lruvec, unsigned long nr_to_reclaim) +{ + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + unsigned long reclaimed; + + if (!memcg) + return 0; + + if (!memcg->cache_ext_ptr) + return 0; + + if (!nr_to_reclaim) + return 0; + + reclaimed = cache_ext_isolate_and_reclaim(lruvec, nr_to_reclaim); + + mod_lruvec_state(lruvec, NR_FILE_CACHE_EXT_KSWAPD_RECLAIMED, reclaimed); + + return reclaimed; +} + +#else + +#define allow_direct_cache_ext (false) + +static unsigned long +kswapd_reclaim_from_cache_ext_memcg(struct lruvec *lruvec, unsigned long nr_to_reclaim) +{ + return 0; +} + +#endif + #ifdef CONFIG_LRU_GEN #ifdef CONFIG_LRU_GEN_ENABLED @@ -4554,6 +4749,44 @@ static int isolate_folios(struct lruvec *lruvec, struct scan_control *sc, int sw return scanned; } +#ifdef CONFIG_CACHE_EXT + +static bool should_lru_gen_reclaim_file(struct lruvec *lruvec, int swappiness) +{ + int type; + int tier = -1; + DEFINE_MIN_SEQ(lruvec); + + /* + * Try to make the obvious choice first. When anon and file are both + * available from the same generation, interpret swappiness 1 as file + * first and 200 as anon first. + */ + if (!swappiness) + type = LRU_GEN_FILE; + else if (min_seq[LRU_GEN_ANON] < min_seq[LRU_GEN_FILE]) + type = LRU_GEN_ANON; + else if (swappiness == 1) + type = LRU_GEN_FILE; + else if (swappiness == 200) + type = LRU_GEN_ANON; + else + type = get_type_to_scan(lruvec, swappiness, &tier); + + return type == LRU_GEN_FILE; +} + +#else + +#define allow_direct_cache_ext (false) + +static bool should_lru_gen_reclaim_file(struct lruvec *lruvec, int swappiness) +{ + return false; +} + +#endif + static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swappiness) { int type; @@ -4570,6 +4803,19 @@ static int evict_folios(struct lruvec *lruvec, struct scan_control *sc, int swap struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); + if (should_lru_gen_reclaim_file(lruvec, swappiness) && + (allow_direct_cache_ext || current_is_kswapd())) { + unsigned long nr_to_reclaim = min(sc->nr_to_reclaim, MIN_LRU_BATCH); + unsigned long reclaimed = + kswapd_reclaim_from_cache_ext_memcg(lruvec, nr_to_reclaim); + + sc->nr_reclaimed += reclaimed; + + if (reclaimed >= nr_to_reclaim) + return reclaimed; + } + + spin_lock_irq(&lruvec->lru_lock); scanned = isolate_folios(lruvec, sc, swappiness, &type, &list); @@ -5720,6 +5966,20 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX); nr[lru] -= nr_to_scan; + if (lru == LRU_INACTIVE_FILE && + (allow_direct_cache_ext || current_is_kswapd())) { + unsigned long cache_ext_reclaimed = + kswapd_reclaim_from_cache_ext_memcg(lruvec, + nr_to_scan); + + nr_reclaimed += cache_ext_reclaimed; + + if (cache_ext_reclaimed >= nr_to_scan) + continue; + + nr_to_scan -= cache_ext_reclaimed; + } + nr_reclaimed += shrink_list(lru, nr_to_scan, lruvec, sc); } diff --git a/mm/vmstat.c b/mm/vmstat.c index c183908a6623..b6d4a0f33de4 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1250,7 +1250,11 @@ const char * const vmstat_text[] = { "pgdemote_kswapd", "pgdemote_direct", "pgdemote_khugepaged", - +#ifdef CONFIG_CACHE_EXT + "nr_file_cache_ext", + "nr_cache_ext_reclaimed_kswapd", + "nr_cache_ext_refaulted", +#endif /* enum writeback_stat_item counters */ "nr_dirty_threshold", "nr_dirty_background_threshold", diff --git a/mm/workingset.c b/mm/workingset.c index 80588168ba41..613705f8a4ec 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -16,6 +16,8 @@ #include <linux/dax.h> #include <linux/fs.h> #include <linux/mm.h> +#include <linux/cache_ext.h> + #include "internal.h" /* @@ -181,9 +183,14 @@ */ #define WORKINGSET_SHIFT 1 +#ifdef CONFIG_CACHE_EXT +#define CACHE_EXT_RECLAIM_SHIFT 1 +#else +#define CACHE_EXT_RECLAIM_SHIFT 0 +#endif #define EVICTION_SHIFT ((BITS_PER_LONG - BITS_PER_XA_VALUE) + \ - WORKINGSET_SHIFT + NODES_SHIFT + \ - MEM_CGROUP_ID_SHIFT) + CACHE_EXT_RECLAIM_SHIFT + WORKINGSET_SHIFT + \ + NODES_SHIFT + MEM_CGROUP_ID_SHIFT) #define EVICTION_MASK (~0UL >> EVICTION_SHIFT) /* @@ -199,20 +206,30 @@ static unsigned int bucket_order __read_mostly; static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, bool workingset) { + bool cache_ext = false; + + if (current_is_cache_ext_reclaim()) + cache_ext = true; + eviction &= EVICTION_MASK; eviction = (eviction << MEM_CGROUP_ID_SHIFT) | memcgid; eviction = (eviction << NODES_SHIFT) | pgdat->node_id; eviction = (eviction << WORKINGSET_SHIFT) | workingset; + eviction = (eviction << CACHE_EXT_RECLAIM_SHIFT) | cache_ext; return xa_mk_value(eviction); } static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, - unsigned long *evictionp, bool *workingsetp) + unsigned long *evictionp, bool *workingsetp, bool *cache_extp) { unsigned long entry = xa_to_value(shadow); int memcgid, nid; bool workingset; + bool cache_ext; + + cache_ext = entry & ((1UL << CACHE_EXT_RECLAIM_SHIFT) - 1); + entry >>= CACHE_EXT_RECLAIM_SHIFT; workingset = entry & ((1UL << WORKINGSET_SHIFT) - 1); entry >>= WORKINGSET_SHIFT; @@ -225,7 +242,41 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, *pgdat = NODE_DATA(nid); *evictionp = entry; *workingsetp = workingset; + + if (cache_extp) + *cache_extp = cache_ext; +} + +#ifdef CONFIG_CACHE_EXT +static void account_cache_ext_refault(void *shadow, long nr) +{ + struct mem_cgroup *eviction_memcg; + struct lruvec *lruvec; + int memcgid; + struct pglist_data *pgdat; + unsigned long eviction; + bool workingset; + bool cache_ext; + + unpack_shadow(shadow, &memcgid, &pgdat, &eviction, &workingset, &cache_ext); + + if (!cache_ext) + return; + + eviction_memcg = mem_cgroup_from_id(memcgid); + /* We need to account this somewhere */ + if (!mem_cgroup_disabled() && !eviction_memcg) + eviction_memcg = root_mem_cgroup; + if (!eviction_memcg->cache_ext_ptr) + return; + + lruvec = &eviction_memcg->nodeinfo[pgdat->node_id]->lruvec; + + mod_lruvec_state(lruvec, NR_FILE_CACHE_EXT_REFAULT, nr); } +#else +static inline void account_cache_ext_refault(void *shadow, long nr) { } +#endif #ifdef CONFIG_LRU_GEN @@ -268,7 +319,7 @@ static bool lru_gen_test_recent(void *shadow, bool file, struct lruvec **lruvec, struct mem_cgroup *memcg; struct pglist_data *pgdat; - unpack_shadow(shadow, &memcg_id, &pgdat, token, workingset); + unpack_shadow(shadow, &memcg_id, &pgdat, token, workingset, NULL); memcg = mem_cgroup_from_id(memcg_id); *lruvec = mem_cgroup_lruvec(memcg, pgdat); @@ -291,6 +342,10 @@ static void lru_gen_refault(struct folio *folio, void *shadow) rcu_read_lock(); recent = lru_gen_test_recent(shadow, type, &lruvec, &token, &workingset); + + if (type) + account_cache_ext_refault(shadow, delta); + if (lruvec != folio_lruvec(folio)) goto unlock; @@ -429,7 +484,7 @@ bool workingset_test_recent(void *shadow, bool file, bool *workingset) if (lru_gen_enabled()) return lru_gen_test_recent(shadow, file, &eviction_lruvec, &eviction, workingset); - unpack_shadow(shadow, &memcgid, &pgdat, &eviction, workingset); + unpack_shadow(shadow, &memcgid, &pgdat, &eviction, workingset, NULL); eviction <<= bucket_order; /* @@ -540,6 +595,9 @@ void workingset_refault(struct folio *folio, void *shadow) mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr); + if (file) + account_cache_ext_refault(shadow, nr); + if (!workingset_test_recent(shadow, file, &workingset)) goto out; -- 2.34.1