From: Nikita Panov <panov.nikita@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10164 ---------------------------------------- This mechanism is inteneded to refill page cache pages which were evicted through cache_ext mechanism. Per super block thread is spawned on mount, only for rotational devices. This mechanism is DISABLED by default. /proc/sys/vm/page_cache_refill_threshold (default=0) /proc/sys/vm/refill_hardware_mq_usage_threshold (default=500(5%)) /proc/sys/vm/refill_ms_sleep_duration (default=500(500ms)) Files above are used to control its behaviour, Second and third values are already set to something meaningful for our HDD setup, regarding page_cache_refill_threshold - we recommend to test different values, our test shows 6000 (60%) most promising. Other values we keep as a default Append "cache_ext_refill=on" to the kernel cmdline if you want to enable this feature. Co-developed-by: Alexander Grubnikov <alexander.grubnikov@huawei.com> Signed-off-by: Alexander Grubnikov <alexander.grubnikov@huawei.com> Co-developed-by: Anatoly Stepanov <stepanov.anatoly@huawei.com> Signed-off-by: Anatoly Stepanov <stepanov.anatoly@huawei.com> Co-developed-by: Nikita Panov <panov.nikita@huawei.com> Signed-off-by: Nikita Panov <panov.nikita@huawei.com> --- .../admin-guide/kernel-parameters.txt | 10 + Documentation/admin-guide/sysctl/vm.rst | 30 ++ fs/drop_caches.c | 6 + fs/inode.c | 13 + fs/super.c | 20 +- include/linux/cache_ext.h | 10 + include/linux/fs.h | 4 + include/linux/mempolicy.h | 6 + include/linux/mmzone.h | 2 + include/linux/pagemap.h | 3 + include/linux/sched.h | 3 +- include/linux/swap.h | 8 + include/linux/sysctl.h | 8 +- kernel/sysctl.c | 28 +- mm/Kconfig | 2 +- mm/cache_ext.c | 383 ++++++++++++++++++ mm/filemap.c | 2 +- mm/memcontrol.c | 7 +- mm/mempolicy.c | 5 + mm/vmscan.c | 5 + mm/vmstat.c | 2 + mm/workingset.c | 56 ++- 22 files changed, 589 insertions(+), 24 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ab5ecf034b22..7cb0af3a33f1 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -549,6 +549,16 @@ to reclaim the requested number of pages, fallback to vanilla reclaim is performed. + cache_ext_refill= + [KNL] + Format: [on|off] + Default: off + Might be enabled only if cache_ext=on. + This feature proactively reads file pages that were + previously reclaimed using the cache_ext policy. + For precise configuration, check + /proc/sys/vm/documentation. + carrier_timeout= [NET] Specifies amount of time (in seconds) that the kernel should wait for a network carrier. By default diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst index cdbcf806b7fe..d3b8e4390de3 100644 --- a/Documentation/admin-guide/sysctl/vm.rst +++ b/Documentation/admin-guide/sysctl/vm.rst @@ -81,6 +81,9 @@ Currently, these files are in /proc/sys/vm: - cache_reclaim_enable - cache_limit_mbytes - allow_direct_cache_ext +- page_cache_refill_threshold +- refill_hardware_mq_usage_threshold +- refill_ms_sleep_duration admin_reserve_kbytes @@ -1110,3 +1113,30 @@ implemented and loaded per memory cgroup. This file controls whether direct reclaim is allowed to be performed from the custom policy or not. If not, only kswapd will reclaim from the cache_ext policy. Noop if cache_ext=on is not specified in kernel cmdline. Default value is true. + + +page_cache_refill_threshold +=================== + +When CONFIG_CACHE_EXT is enabled, it sets the allowed memory threshold for the +refill feature. The refill mechanism will proactively read file pages that +were evicted using the cache_ext policy. Refill will work only if the overall +system memory usage is below this threshold. 10000 for 100% (always try to +refill), 5000 for 50%, and 0 for 0% (default) - refill is disabled. Noop if +cache_ext_refill=on is not specified in kernel cmdline. + + +refill_hardware_mq_usage_threshold +=================== + +When CONFIG_CACHE_EXT is enabled and page_cache_refill_threshold is not 0, +this entry sets a limit on block device mq usage. If usage is above the set +value, a refill will not be performed. Noop if cache_ext_refill=on is not +specified in the kernel cmdline. Default value is 500 (5%). + + +refill_ms_sleep_duration +=================== + +Sleep interval for refill threads between refill attempts in ms. 100 ms is the +minimal allowed value to prevent busy loops. diff --git a/fs/drop_caches.c b/fs/drop_caches.c index b9575957a7c2..b7a518f2bd9b 100644 --- a/fs/drop_caches.c +++ b/fs/drop_caches.c @@ -34,6 +34,12 @@ static void drop_pagecache_sb(struct super_block *sb, void *unused) continue; } __iget(inode); + +#ifdef CONFIG_CACHE_EXT + if (test_bit(AS_CACHE_EXT, &inode->i_mapping->flags)) + clear_bit(AS_CACHE_EXT, &inode->i_mapping->flags); +#endif + spin_unlock(&inode->i_lock); spin_unlock(&sb->s_inode_list_lock); diff --git a/fs/inode.c b/fs/inode.c index 878a9c671c91..3fe99641a9d7 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -20,6 +20,7 @@ #include <linux/ratelimit.h> #include <linux/list_lru.h> #include <linux/iversion.h> +#include <linux/cache_ext.h> #include <trace/events/writeback.h> #include "internal.h" @@ -880,6 +881,18 @@ static enum lru_status inode_lru_isolate(struct list_head *item, if (!spin_trylock(&inode->i_lock)) return LRU_SKIP; +#ifdef CONFIG_CACHE_EXT + /* + * Skip cache_ext inodes in direct or kswapd reclaim, because they are candidates + * for refill. + */ + if (!static_branch_unlikely(&cache_ext_refill_disabled) && + (current->flags & PF_MEMALLOC) && + test_bit(AS_CACHE_EXT, &inode->i_mapping->flags)) { + spin_unlock(&inode->i_lock); + return LRU_SKIP; + } +#endif /* * Inodes can get referenced, redirtied, or repopulated while * they're already on the LRU, and this can make them diff --git a/fs/super.c b/fs/super.c index 6f5e2881fd10..a35e16f711bf 100644 --- a/fs/super.c +++ b/fs/super.c @@ -36,13 +36,14 @@ #include <linux/lockdep.h> #include <linux/user_namespace.h> #include <linux/fs_context.h> +#include <linux/cache_ext.h> #include <uapi/linux/mount.h> #include "internal.h" static int thaw_super_locked(struct super_block *sb, enum freeze_holder who); static LIST_HEAD(super_blocks); -static DEFINE_SPINLOCK(sb_lock); +DEFINE_SPINLOCK(sb_lock); static char *sb_writers_name[SB_FREEZE_LEVELS] = { "sb_writers", @@ -76,7 +77,7 @@ static inline void super_unlock_excl(struct super_block *sb) super_unlock(sb, true); } -static inline void super_unlock_shared(struct super_block *sb) +void super_unlock_shared(struct super_block *sb) { super_unlock(sb, false); } @@ -141,7 +142,7 @@ static __must_check bool super_lock(struct super_block *sb, bool excl) } /* wait and acquire read-side of @sb->s_umount */ -static inline bool super_lock_shared(struct super_block *sb) +bool super_lock_shared(struct super_block *sb) { return super_lock(sb, false); } @@ -285,6 +286,14 @@ static void destroy_super_work(struct work_struct *work) for (i = 0; i < SB_FREEZE_LEVELS; i++) percpu_free_rwsem(&s->s_writers.rw_sem[i]); + +#ifdef CONFIG_CACHE_EXT + if (s->krefilld) { + kthread_stop(s->krefilld); + s->krefilld = NULL; + } +#endif + kfree(s); } @@ -1537,6 +1546,11 @@ int setup_bdev_super(struct super_block *sb, int sb_flags, bdev_release(bdev_handle); return -EBUSY; } + + if (!bdev_nonrot(bdev)) + if (krefilld_run(sb, bdev->bd_disk->disk_name)) + pr_warn("Failed to start krefilld for %s\n", bdev->bd_disk->disk_name); + spin_lock(&sb_lock); sb->s_bdev_handle = bdev_handle; sb->s_bdev = bdev; diff --git a/include/linux/cache_ext.h b/include/linux/cache_ext.h index 40e3c4230942..0e51dcec01f1 100644 --- a/include/linux/cache_ext.h +++ b/include/linux/cache_ext.h @@ -11,9 +11,12 @@ #include <linux/page_ext.h> +struct super_block; + #ifdef CONFIG_CACHE_EXT DECLARE_STATIC_KEY_TRUE(cache_ext_disabled); +DECLARE_STATIC_KEY_TRUE(cache_ext_refill_disabled); static inline bool folio_test_cache_ext(struct folio *folio) { @@ -219,6 +222,11 @@ s64 get_and_clear_cookie_from_folio_page_ext(struct folio *folio); bool cache_ext_kfunc(u32 btf_id); extern bool allow_direct_cache_ext; +extern unsigned long refill_threshold; +extern unsigned long hardware_mq_usage_threshold; +extern unsigned long refill_ms_sleep_duration; + +int krefilld_run(struct super_block *sb, const char *name); #else @@ -244,6 +252,8 @@ static inline s64 get_and_clear_cookie_from_folio_page_ext(struct folio *folio) static inline bool cache_ext_kfunc(u32 btf_id) { return false; } +static inline int krefilld_run(struct super_block *sb, const char *name) { return 0; } + #endif #endif diff --git a/include/linux/fs.h b/include/linux/fs.h index 37ff2e2a36fa..69f1bb960528 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1494,6 +1494,10 @@ struct super_block { spinlock_t s_inode_wblist_lock; struct list_head s_inodes_wb; /* writeback inodes */ +#ifdef CONFIG_CACHE_EXT + struct task_struct *krefilld; +#endif + KABI_USE(1, struct shrinker_v2 *s_shrink) KABI_RESERVE(2) diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 5cb841943283..43c609506dd8 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -139,6 +139,7 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, bool vma_policy_mof(struct vm_area_struct *vma); extern void numa_default_policy(void); +extern long numa_set_interleave_policy(nodemask_t *nodes); extern void numa_policy_init(void); extern void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new); extern void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new); @@ -253,6 +254,11 @@ static inline void numa_default_policy(void) { } +static inline long numa_set_interleave_policy(nodemask_t *nodes) +{ + return 0; +} + static inline void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) { diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 3fbc6b5fbba9..596a5a2e6aac 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -223,6 +223,8 @@ enum node_stat_item { NR_FILE_CACHE_EXT, NR_FILE_CACHE_EXT_KSWAPD_RECLAIMED, NR_FILE_CACHE_EXT_REFAULT, + NR_FILE_CACHE_EXT_REFILLED, + REFILL_REFAULT_FILE, #endif NR_VM_NODE_STAT_ITEMS }; diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 624712273947..6ed3ba3ecfef 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -211,6 +211,9 @@ enum mapping_flags { AS_FOLIO_ORDER_BITS = 5, AS_FOLIO_ORDER_MIN = 16, AS_FOLIO_ORDER_MAX = AS_FOLIO_ORDER_MIN + AS_FOLIO_ORDER_BITS, +#ifdef CONFIG_CACHE_EXT + AS_CACHE_EXT = AS_FOLIO_ORDER_MAX + AS_FOLIO_ORDER_BITS, +#endif }; #define AS_FOLIO_ORDER_BITS_MASK ((1u << AS_FOLIO_ORDER_BITS) - 1) diff --git a/include/linux/sched.h b/include/linux/sched.h index d7f1c1095cbb..61055ce6bb40 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1865,11 +1865,12 @@ extern struct pid *cad_pid; #ifdef CONFIG_CACHE_EXT #define PF_CACHE_EXT_RECLAIM 0x01000000 /* I reclaim from cache_ext right now */ +#define PF_REFILL_THREAD 0x02000000 /* I am refill thread */ #else #define PF__HOLE__01000000 0x01000000 +#define PF__HOLE__02000000 0x02000000 #endif -#define PF__HOLE__02000000 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ #define PF_MEMALLOC_PIN 0x10000000 /* Allocation context constrained to zones which allow long term pinning. */ diff --git a/include/linux/swap.h b/include/linux/swap.h index 9f1f77f4cf59..05cdf1047e9c 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -44,11 +44,19 @@ static inline int current_is_cache_ext_reclaim(void) { return current->flags & PF_CACHE_EXT_RECLAIM; } +static inline int current_is_krefilld(void) +{ + return current->flags & PF_REFILL_THREAD; +} #else static inline int current_is_cache_ext_reclaim(void) { return 0; } +static inline int current_is_krefilld(void) +{ + return 0; +} #endif /* diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 61b40ea81f4d..02b512fc43db 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -55,9 +55,11 @@ struct ctl_dir; extern const int sysctl_vals[]; -#define SYSCTL_LONG_ZERO ((void *)&sysctl_long_vals[0]) -#define SYSCTL_LONG_ONE ((void *)&sysctl_long_vals[1]) -#define SYSCTL_LONG_MAX ((void *)&sysctl_long_vals[2]) +#define SYSCTL_LONG_ZERO ((void *)&sysctl_long_vals[0]) +#define SYSCTL_LONG_ONE ((void *)&sysctl_long_vals[1]) +#define SYSCTL_LONG_MAX ((void *)&sysctl_long_vals[2]) +#define SYSCTL_LONG_ONE_HUNDRED ((void *)&sysctl_long_vals[3]) +#define SYSCTL_LONG_TEN_THOUSAND ((void *)&sysctl_long_vals[4]) extern const unsigned long sysctl_long_vals[]; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index 88d8ca3c0fd3..9c11d522631c 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -86,7 +86,7 @@ const int sysctl_vals[] = { 0, 1, 2, 3, 4, 100, 200, 1000, 3000, INT_MAX, 65535, -1 }; EXPORT_SYMBOL(sysctl_vals); -const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX }; +const unsigned long sysctl_long_vals[] = { 0, 1, LONG_MAX, 100, 10000 }; EXPORT_SYMBOL_GPL(sysctl_long_vals); #if defined(CONFIG_SYSCTL) @@ -2127,6 +2127,32 @@ static struct ctl_table vm_table[] = { .mode = 0644, .proc_handler = proc_dobool }, + { + .procname = "page_cache_refill_threshold", + .data = &refill_threshold, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = SYSCTL_LONG_ZERO, + .extra2 = SYSCTL_LONG_TEN_THOUSAND, + }, + { + .procname = "refill_hardware_mq_usage_threshold", + .data = &hardware_mq_usage_threshold, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = SYSCTL_LONG_ZERO, + .extra2 = SYSCTL_LONG_TEN_THOUSAND, + }, + { + .procname = "refill_ms_sleep_duration", + .data = &refill_ms_sleep_duration, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = SYSCTL_LONG_ONE_HUNDRED + }, #endif { .procname = "page_lock_unfairness", diff --git a/mm/Kconfig b/mm/Kconfig index 9198e1f037ab..e98dbdb770a7 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1530,7 +1530,7 @@ config GMEM config CACHE_EXT bool "Cache_ext framework" - depends on BPF_SYSCALL && MEMCG && 64BIT + depends on BPF_SYSCALL && MEMCG && 64BIT && BLOCK select PAGE_EXTENSION default n help diff --git a/mm/cache_ext.c b/mm/cache_ext.c index 2b349030209f..9b9f61139bb7 100644 --- a/mm/cache_ext.c +++ b/mm/cache_ext.c @@ -11,10 +11,17 @@ #include <linux/mm.h> #include <linux/mm_inline.h> #include <linux/page_ext.h> +#include <linux/blk-mq.h> +#include <linux/mempolicy.h> +#include <linux/fs.h> +#include <linux/backing-dev.h> +#include <linux/freezer.h> +#include <linux/delay.h> #include "internal.h" DEFINE_STATIC_KEY_TRUE(cache_ext_disabled); +DEFINE_STATIC_KEY_TRUE(cache_ext_refill_disabled); static int __init setup_cache_ext(char *str) { @@ -38,6 +45,35 @@ static int __init setup_cache_ext(char *str) } early_param("cache_ext", setup_cache_ext); +static int __init setup_cache_ext_refill(char *str) +{ + int ret = 0; + + if (!str) + goto out; + + if (!strcmp(str, "on")) { + + if (static_branch_unlikely(&cache_ext_disabled)) { + pr_warn("cache_ext_refill= cannot enable without cache_ext\n"); + return 0; + } + + static_branch_disable(&cache_ext_refill_disabled); + pr_info("cache_ext_refill enabled via cmdline\n"); + ret = 1; + } else if (!strcmp(str, "off")) { + static_branch_enable(&cache_ext_refill_disabled); + pr_info("cache_ext_refill disabled via cmdline\n"); + ret = 1; + } +out: + if (!ret) + pr_warn("cache_ext_refill= cannot parse, ignored\n"); + return ret; +} +__setup("cache_ext_refill=", setup_cache_ext_refill); + static const struct btf_type *cache_ext_isolation_ctx_type; static const struct btf_type *cache_ext_stats_ctx_type; @@ -949,4 +985,351 @@ int cache_ext_policy_stat_show(struct seq_file *m, void *v) return 0; } +unsigned long refill_threshold; /* 0%, disabled by default */ +unsigned long hardware_mq_usage_threshold = 500; +unsigned long refill_ms_sleep_duration = 500; + +static void krefilld_file_sleep(void) +{ + msleep_interruptible(READ_ONCE(refill_ms_sleep_duration)); +} + +void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, + unsigned long *evictionp, bool *workingsetp, bool *cache_ext_p); + +static bool below_refill_threshold(void) +{ + unsigned long total_pages = 0, free_pages = 0; + unsigned long refill_threshold_local = READ_ONCE(refill_threshold); + struct zone *zone; + + free_pages = global_zone_page_state(NR_FREE_PAGES); + + for_each_populated_zone(zone) { + total_pages += zone_managed_pages(zone); + } + + if (!total_pages) + return false; + + return (((total_pages - free_pages) * 10000) / total_pages) < refill_threshold_local; +} + +static bool is_candidate_for_refill(struct folio *folio) +{ + int memcg_id; + struct pglist_data *pgdat; + unsigned long token; + bool workingset; + bool cache_ext; + + if (WARN_ON(!xa_is_value(folio))) + return false; + + unpack_shadow(folio, &memcg_id, &pgdat, &token, &workingset, &cache_ext); + + return cache_ext; +} + +struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, xa_mark_t mark); + +static unsigned int find_exceptional_entry(struct address_space *mapping, pgoff_t *start, + pgoff_t end, pgoff_t *index) +{ + XA_STATE(xas, &mapping->i_pages, *start); + struct folio *folio; + unsigned long res = 0; + + rcu_read_lock(); + while ((folio = find_get_entry(&xas, end, XA_PRESENT))) { + unsigned long base; + unsigned long nr; + + if (!xa_is_value(folio)) { + nr = folio_nr_pages(folio); + base = folio->index; + folio_put(folio); + /* Omit large folio which begins before the start */ + if (base < *start) + break; + /* Omit large folio which extends beyond the end */ + if (base + nr - 1 > end) + break; + + *start = base + nr; + } else { + nr = 1 << xas_get_order(&xas); + base = xas.xa_index & ~(nr - 1); + /* Omit order>0 value which begins before the start */ + if (base < *start) + continue; + /* Omit order>0 value which extends beyond the end */ + if (base + nr - 1 > end) + break; + + *start = base + nr; + + if (!is_candidate_for_refill(folio)) + continue; + + *index = xas.xa_index; + res = 1; + break; + } + } + rcu_read_unlock(); + + return res; +} + +static unsigned long refill_page_cache_mapping_pages(struct address_space *mapping, + pgoff_t start, pgoff_t end, bool *has_refill_candidates) +{ + pgoff_t read_index; + pgoff_t index = start; + unsigned long count = 0; + *has_refill_candidates = false; + + while (find_exceptional_entry(mapping, &index, end, &read_index)) { + unsigned long read_size = inode_to_bdi(mapping->host)->ra_pages; + struct file_ra_state ra = { + .start = -1, + .size = read_size, + .prev_pos = read_index << PAGE_SHIFT, + .ra_pages = read_size, + }; + DEFINE_READAHEAD(ractl, NULL, &ra, mapping, read_index); + + *has_refill_candidates = true; + + page_cache_sync_ra(&ractl, read_size); + + count += ra.ra_pages; + + cond_resched(); + } + + return count; +} + +static bool below_hardware_queue_threshold(struct super_block *sb) +{ + struct block_device *bdev; + struct request_queue *q; + struct blk_mq_hw_ctx *hctx; + unsigned long ret; + unsigned int active_hw_requests = 0; + unsigned long hw_capacity = 0; + unsigned long i; + unsigned long hardware_usage_local = READ_ONCE(hardware_mq_usage_threshold); + bool result = true; + + if (!sb->s_bdev) + return false; + + bdev = sb->s_bdev; + + if (!bdev->bd_disk || !bdev->bd_disk->queue) + return false; + + q = bdev->bd_disk->queue; + + ret = blk_queue_enter(q, 0); + if (ret) + return false; + + queue_for_each_hw_ctx(q, hctx, i) { + if (!hctx || !hctx->tags) + continue; + + hw_capacity = hctx->tags->nr_tags; + active_hw_requests = sbitmap_weight(&hctx->tags->bitmap_tags.sb); + + if (!hw_capacity) { + result = false; + break; + } + + if (((active_hw_requests * 10000) / hw_capacity) > hardware_usage_local) { + result = false; + break; + } + } + + blk_queue_exit(q); + + return result; +} + +static void refill_pagecache_sb(struct super_block *sb) +{ + struct inode *inode, *toput_inode = NULL; + unsigned long total_page_cache_refilled = 0; + + if (!below_refill_threshold()) + return; + + spin_lock(&sb->s_inode_list_lock); + + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + bool has_refill_candidates = false; + + if (!spin_trylock(&inode->i_lock)) + continue; + /* + * We must skip inodes in unusual state. + */ + if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) { + spin_unlock(&inode->i_lock); + continue; + } + + if (!test_bit(AS_CACHE_EXT, &inode->i_mapping->flags)) { + spin_unlock(&inode->i_lock); + continue; + } + + if (!inode->i_mapping->a_ops->readahead) { + clear_bit(AS_CACHE_EXT, &inode->i_mapping->flags); + spin_unlock(&inode->i_lock); + continue; + } + + __iget(inode); + spin_unlock(&inode->i_lock); + spin_unlock(&sb->s_inode_list_lock); + + total_page_cache_refilled += + refill_page_cache_mapping_pages(inode->i_mapping, 0, -1, + &has_refill_candidates); + + if (!has_refill_candidates) + clear_bit(AS_CACHE_EXT, &inode->i_mapping->flags); + + mod_memcg_state(root_mem_cgroup, NR_FILE_CACHE_EXT_REFILLED, + total_page_cache_refilled); + + total_page_cache_refilled = 0; + + iput(toput_inode); + toput_inode = inode; + + cond_resched(); + + if (!below_refill_threshold() || !below_hardware_queue_threshold(sb)) + goto out; + + spin_lock(&sb->s_inode_list_lock); + + + } + spin_unlock(&sb->s_inode_list_lock); +out: + iput(toput_inode); + +} + +extern bool super_lock_shared(struct super_block *sb); +extern void super_unlock_shared(struct super_block *sb); +extern spinlock_t sb_lock; +extern void put_super(struct super_block *sb); + +static int krefilld(void *p) +{ + struct super_block *sb = (struct super_block *)p; + struct task_struct *tsk = current; + nodemask_t target_nodes; + unsigned long ret = 0; + + nodes_clear(target_nodes); + nodes_or(target_nodes, target_nodes, node_states[N_MEMORY]); + + if (WARN_ON(numa_set_interleave_policy(&target_nodes))) + goto out; + + if (WARN_ON(set_task_ioprio(current, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0)))) + goto out; + + tsk->flags |= PF_REFILL_THREAD; + + set_freezable(); + for ( ; ; ) { + bool born; + + krefilld_file_sleep(); + + ret = try_to_freeze(); + + if (kthread_should_stop()) + break; + + if (ret) + continue; + + if (!below_refill_threshold()) + continue; + + spin_lock(&sb_lock); + if (sb->s_count == 0) { + spin_unlock(&sb_lock); + continue; + } + + if (kthread_should_stop()) { + spin_unlock(&sb_lock); + break; + } + + sb->s_count++; + spin_unlock(&sb_lock); + + + born = super_lock_shared(sb); + + if (!below_hardware_queue_threshold(sb)) + goto skip; + + if (born && sb->s_root) + refill_pagecache_sb(sb); + +skip: + super_unlock_shared(sb); + + put_super(sb); + } + + tsk->flags &= ~(PF_REFILL_THREAD); +out: + /* + * Why? + * To prevent kthread_stop on already stopped kthreads and avoid UAF + */ + for ( ; ; ) { + set_current_state(TASK_INTERRUPTIBLE); + + if (kthread_should_stop()) + break; + schedule(); + } + + set_current_state(TASK_RUNNING); + return 0; + +} + +int krefilld_run(struct super_block *sb, const char *name) +{ + struct task_struct *tsk; + if (static_branch_unlikely(&cache_ext_refill_disabled)) + return 0; + + tsk = kthread_create(krefilld, sb, "krefilld_%s", name); + + if (IS_ERR(tsk)) + return PTR_ERR(tsk); + + sb->krefilld = tsk; + wake_up_process(tsk); + return 0; +} diff --git a/mm/filemap.c b/mm/filemap.c index 58dd2fde49a6..c4155d24a218 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1995,7 +1995,7 @@ struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index, } EXPORT_SYMBOL(__filemap_get_folio); -static inline struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, +struct folio *find_get_entry(struct xa_state *xas, pgoff_t max, xa_mark_t mark) { struct folio *folio; diff --git a/mm/memcontrol.c b/mm/memcontrol.c index f6be624801e1..d88ac91860a1 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1630,6 +1630,8 @@ static int memcg_page_state_unit(int item) case NR_FILE_CACHE_EXT: case NR_FILE_CACHE_EXT_KSWAPD_RECLAIMED: case NR_FILE_CACHE_EXT_REFAULT: + case NR_FILE_CACHE_EXT_REFILLED: + case REFILL_REFAULT_FILE: #endif return 1; case NR_KERNEL_STACK_KB: @@ -4982,6 +4984,8 @@ static const unsigned int memcg1_stats[] = { NR_FILE_CACHE_EXT, NR_FILE_CACHE_EXT_KSWAPD_RECLAIMED, NR_FILE_CACHE_EXT_REFAULT, + NR_FILE_CACHE_EXT_REFILLED, + REFILL_REFAULT_FILE, #endif }; @@ -5005,7 +5009,8 @@ static const char *const memcg1_stat_names[] = { "file_cache_ext", "file_cache_ext_reclaimed_kswapd", "file_cache_ext_refault", - + "file_cache_ext_refilled", + "refill_refault_file", #endif }; diff --git a/mm/mempolicy.c b/mm/mempolicy.c index b960310a70f9..7b6b7b011960 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -3317,6 +3317,11 @@ void numa_default_policy(void) do_set_mempolicy(MPOL_DEFAULT, 0, NULL); } +long numa_set_interleave_policy(nodemask_t *nodes) +{ + return do_set_mempolicy(MPOL_INTERLEAVE, 0, nodes); +} + /* * Parse and format mempolicy from/to strings */ diff --git a/mm/vmscan.c b/mm/vmscan.c index a3c871965507..edfaf3385688 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -782,6 +782,11 @@ static int __remove_mapping(struct address_space *mapping, struct folio *folio, shadow = workingset_eviction(folio, target_memcg); __filemap_remove_folio(folio, shadow); xa_unlock_irq(&mapping->i_pages); +#ifdef CONFIG_CACHE_EXT + if (current_is_cache_ext_reclaim()) + /* Mark inode for future refill*/ + set_bit(AS_CACHE_EXT, &mapping->flags); +#endif if (mapping_shrinkable(mapping)) inode_add_lru(mapping->host); spin_unlock(&mapping->host->i_lock); diff --git a/mm/vmstat.c b/mm/vmstat.c index b6d4a0f33de4..bde8f3f69e51 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1254,6 +1254,8 @@ const char * const vmstat_text[] = { "nr_file_cache_ext", "nr_cache_ext_reclaimed_kswapd", "nr_cache_ext_refaulted", + "nr_cache_ext_refilled", + "refill_refault_file", #endif /* enum writeback_stat_item counters */ "nr_dirty_threshold", diff --git a/mm/workingset.c b/mm/workingset.c index 613705f8a4ec..2fc06a4cbcfe 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -220,7 +220,7 @@ static void *pack_shadow(int memcgid, pg_data_t *pgdat, unsigned long eviction, return xa_mk_value(eviction); } -static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, +void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, unsigned long *evictionp, bool *workingsetp, bool *cache_extp) { unsigned long entry = xa_to_value(shadow); @@ -274,8 +274,24 @@ static void account_cache_ext_refault(void *shadow, long nr) mod_lruvec_state(lruvec, NR_FILE_CACHE_EXT_REFAULT, nr); } + +static void account_refault(struct lruvec *lruvec, int type, int nr, void *shadow) +{ + if (current_is_krefilld()) { + mod_lruvec_state(lruvec, REFILL_REFAULT_FILE, nr); + } else { + mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, nr); + if (type) + account_cache_ext_refault(shadow, nr); + } + +} + #else -static inline void account_cache_ext_refault(void *shadow, long nr) { } +static void account_refault(struct lruvec *lruvec, int type, int nr, void *shadow) +{ + mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, nr); +} #endif #ifdef CONFIG_LRU_GEN @@ -343,13 +359,10 @@ static void lru_gen_refault(struct folio *folio, void *shadow) recent = lru_gen_test_recent(shadow, type, &lruvec, &token, &workingset); - if (type) - account_cache_ext_refault(shadow, delta); - if (lruvec != folio_lruvec(folio)) goto unlock; - mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta); + account_refault(lruvec, type, delta, shadow); if (!recent) goto unlock; @@ -593,10 +606,7 @@ void workingset_refault(struct folio *folio, void *shadow) pgdat = folio_pgdat(folio); lruvec = mem_cgroup_lruvec(memcg, pgdat); - mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr); - - if (file) - account_cache_ext_refault(shadow, nr); + account_refault(lruvec, file, nr, shadow); if (!workingset_test_recent(shadow, file, &workingset)) goto out; @@ -660,10 +670,10 @@ struct list_lru shadow_nodes; void workingset_update_node(struct xa_node *node) { struct address_space *mapping; - + bool cache_ext_node = false; /* * Track non-empty nodes that contain only shadow entries; - * unlink those that contain pages or are being freed. + * unlink those that contain pages or are being freed or might be refilled. * * Avoid acquiring the list_lru lock when the nodes are * already where they should be. The list_empty() test is safe @@ -672,11 +682,31 @@ void workingset_update_node(struct xa_node *node) mapping = container_of(node->array, struct address_space, i_pages); lockdep_assert_held(&mapping->i_pages.xa_lock); +#ifdef CONFIG_CACHE_EXT + if (static_branch_unlikely(&cache_ext_refill_disabled)) + cache_ext_node = false; + else + cache_ext_node = test_bit(AS_CACHE_EXT, &mapping->flags); +#endif + if (node->count && node->count == node->nr_values) { - if (list_empty(&node->private_list)) { + + /* + * Do not add cache_ext nodes to shadow list + */ + if (list_empty(&node->private_list) && !cache_ext_node) { list_lru_add(&shadow_nodes, &node->private_list); __inc_lruvec_kmem_state(node, WORKINGSET_NODES); } + + /* + * If cache_ext node in shadow list - remove it + */ + if (!list_empty(&node->private_list) && cache_ext_node) { + list_lru_del(&shadow_nodes, &node->private_list); + __dec_lruvec_kmem_state(node, WORKINGSET_NODES); + } + } else { if (!list_empty(&node->private_list)) { list_lru_del(&shadow_nodes, &node->private_list); -- 2.34.1