From: Nikita Panov <panov.nikita@huawei.com> hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/10164 ---------------------------------------- Co-developed-by: Alexander Grubnikov <alexander.grubnikov@huawei.com> Signed-off-by: Alexander Grubnikov <alexander.grubnikov@huawei.com> Co-developed-by: Anatoly Stepanov <stepanov.anatoly@huawei.com> Signed-off-by: Anatoly Stepanov <stepanov.anatoly@huawei.com> Co-developed-by: Nikita Panov <panov.nikita@huawei.com> Signed-off-by: Nikita Panov <panov.nikita@huawei.com> --- tools/cache_ext/.gitignore | 2 + tools/cache_ext/Makefile | 229 ++++++++++++++++++ tools/cache_ext/cache_ext_simple_fifo.bpf.c | 109 +++++++++ tools/cache_ext/cache_ext_simple_fifo.c | 52 ++++ .../cache_ext/include/cache_ext_common.bpf.h | 178 ++++++++++++++ 5 files changed, 570 insertions(+) create mode 100644 tools/cache_ext/.gitignore create mode 100644 tools/cache_ext/Makefile create mode 100644 tools/cache_ext/cache_ext_simple_fifo.bpf.c create mode 100644 tools/cache_ext/cache_ext_simple_fifo.c create mode 100644 tools/cache_ext/include/cache_ext_common.bpf.h diff --git a/tools/cache_ext/.gitignore b/tools/cache_ext/.gitignore new file mode 100644 index 000000000000..d6264fe1c8cd --- /dev/null +++ b/tools/cache_ext/.gitignore @@ -0,0 +1,2 @@ +tools/ +build/ diff --git a/tools/cache_ext/Makefile b/tools/cache_ext/Makefile new file mode 100644 index 000000000000..29db925673c0 --- /dev/null +++ b/tools/cache_ext/Makefile @@ -0,0 +1,229 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copied from tools/sched_ext/Makefile +include ../build/Build.include +include ../scripts/Makefile.arch +include ../scripts/Makefile.include + +all: all_targets + +ifneq ($(LLVM),) +ifneq ($(filter %/,$(LLVM)),) +LLVM_PREFIX := $(LLVM) +else ifneq ($(filter -%,$(LLVM)),) +LLVM_SUFFIX := $(LLVM) +endif + +CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi +CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu +CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl +CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu +CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu +CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu +CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu +CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu +CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu +CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH)) + +ifeq ($(CROSS_COMPILE),) +ifeq ($(CLANG_TARGET_FLAGS),) +$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk) +else +CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS) +endif # CLANG_TARGET_FLAGS +else +CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) +endif # CROSS_COMPILE + +CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as +else +CC := $(CROSS_COMPILE)gcc +endif # LLVM + +CURDIR := $(abspath .) +TOOLSDIR := $(abspath ..) +LIBDIR := $(TOOLSDIR)/lib +BPFDIR := $(LIBDIR)/bpf +TOOLSINCDIR := $(TOOLSDIR)/include +BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool +APIDIR := $(TOOLSINCDIR)/uapi +GENDIR := $(abspath ../../include/generated) +GENHDR := $(GENDIR)/autoconf.h + +ifeq ($(O),) +OUTPUT_DIR := $(CURDIR)/build +else +OUTPUT_DIR := $(O)/build +endif # O +OBJ_DIR := $(OUTPUT_DIR)/obj +INCLUDE_DIR := $(OUTPUT_DIR)/include +BPFOBJ_DIR := $(OBJ_DIR)/libbpf +CACHE_EXT_OBJ_DIR := $(OBJ_DIR)/cache_ext +BINDIR := $(OUTPUT_DIR)/bin +BPFOBJ := $(BPFOBJ_DIR)/libbpf.a +ifneq ($(CROSS_COMPILE),) +HOST_BUILD_DIR := $(OBJ_DIR)/host +HOST_OUTPUT_DIR := host-tools +HOST_INCLUDE_DIR := $(HOST_OUTPUT_DIR)/include +else +HOST_BUILD_DIR := $(OBJ_DIR) +HOST_OUTPUT_DIR := $(OUTPUT_DIR) +HOST_INCLUDE_DIR := $(INCLUDE_DIR) +endif +HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a +RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids +DEFAULT_BPFTOOL := $(HOST_OUTPUT_DIR)/sbin/bpftool + +VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \ + $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \ + ../../vmlinux \ + /sys/kernel/btf/vmlinux \ + /boot/vmlinux-$(shell uname -r) +VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS)))) +ifeq ($(VMLINUX_BTF),) +$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)") +endif + +BPFTOOL ?= $(DEFAULT_BPFTOOL) + +ifneq ($(wildcard $(GENHDR)),) + GENFLAGS := -DHAVE_GENHDR +endif + +CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \ + -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ + -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include + +# Silence some warnings when compiled with clang +ifneq ($(LLVM),) +CFLAGS += -Wno-unused-command-line-argument +endif + +LDFLAGS = -lelf -lz -lpthread -static + +IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \ + grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__') + +# Get Clang's default includes on this system, as opposed to those seen by +# '-target bpf'. This fixes "missing" files on some architectures/distros, +# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc. +# +# Use '-idirafter': Don't interfere with include mechanics except where the +# build would have failed anyways. +define get_sys_includes +$(shell $(1) -v -E - </dev/null 2>&1 \ + | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \ +$(shell $(1) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}') +endef + +BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \ + $(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) \ + -I$(CURDIR)/include -I$(CURDIR)/include/bpf-compat \ + -I$(INCLUDE_DIR) -I$(APIDIR) \ + -I../../include \ + $(call get_sys_includes,$(CLANG)) \ + -Wall -Wno-compare-distinct-pointer-types \ + -O2 -mcpu=v3 + +# sort removes libbpf duplicates when not cross-building +MAKE_DIRS := $(sort $(OBJ_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \ + $(HOST_BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/resolve_btfids \ + $(INCLUDE_DIR) $(CACHE_EXT_OBJ_DIR) $(BINDIR)) + +$(MAKE_DIRS): + $(call msg,MKDIR,,$@) + $(Q)mkdir -p $@ + +$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ + $(APIDIR)/linux/bpf.h \ + | $(OBJ_DIR)/libbpf + $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(OBJ_DIR)/libbpf/ \ + EXTRA_CFLAGS='-g -O0 -fPIC' \ + DESTDIR=$(OUTPUT_DIR) prefix= all install_headers + +$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ + $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool + $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \ + ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) \ + EXTRA_CFLAGS='-g -O0' \ + OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \ + LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \ + LIBBPF_DESTDIR=$(HOST_OUTPUT_DIR)/ \ + prefix= DESTDIR=$(HOST_OUTPUT_DIR)/ install-bin + +$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR) +ifeq ($(VMLINUX_H),) + $(call msg,GEN,,$@) + $(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@ +else + $(call msg,CP,,$@) + $(Q)cp "$(VMLINUX_H)" $@ +endif + +$(CACHE_EXT_OBJ_DIR)/%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h include/*.h \ + | $(BPFOBJ) $(CACHE_EXT_OBJ_DIR) + $(call msg,CLNG-BPF,,$(notdir $@)) + $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@ + +$(INCLUDE_DIR)/%.bpf.skel.h: $(CACHE_EXT_OBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BPFTOOL) + $(eval cache_ext_policy=$(notdir $@)) + $(call msg,GEN-SKEL,,$(cache_ext_policy)) + $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $< + $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o) + $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o) + $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o) + $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(cache_ext_policy)) > $@ + $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(cache_ext_policy)) > $(@:.skel.h=.subskel.h) + +CACHE_EXT_COMMON_DEPS := include/cache_ext_common.bpf.h | $(BINDIR) + +c-cache-ext-policy-targets = cache_ext_simple_fifo + +$(addprefix $(BINDIR)/,$(c-cache-ext-policy-targets)): \ + $(BINDIR)/%: \ + $(filter-out %.bpf.c,%.c) \ + $(INCLUDE_DIR)/%.bpf.skel.h \ + $(CACHE_EXT_COMMON_DEPS) + $(eval cache_ext_policy=$(notdir $@)) + $(CC) $(CFLAGS) -c $(cache_ext_policy).c -o $(CACHE_EXT_OBJ_DIR)/$(cache_ext_policy).o + $(CC) -o $@ $(CACHE_EXT_OBJ_DIR)/$(cache_ext_policy).o $(HOST_BPFOBJ) $(LDFLAGS) + +$(c-cache-ext-policy-targets): %: $(BINDIR)/% + +install: all + $(Q)mkdir -p $(DESTDIR)/usr/local/bin/ + $(Q)cp $(BINDIR)/* $(DESTDIR)/usr/local/bin/ + +clean: + rm -rf $(OUTPUT_DIR) $(HOST_OUTPUT_DIR) + rm -f *.o *.bpf.o *.bpf.skel.h *.bpf.subskel.h + rm -f $(c-cache-ext-policy-targets) + +help: + @echo 'Building targets' + @echo '================' + @echo '' + @echo ' all - Compile all policies' + @echo '' + @echo 'Alternatively, you may compile individual policies:' + @echo '' + @printf ' %s\n' $(c-cache-ext-policy-targets) + @echo '' + @echo 'For any policy build target, you may specify an alternative' + @echo 'build output path with the O= environment variable. For example:' + @echo '' + @echo ' O=/tmp/cache_ext make all' + @echo '' + @echo 'will compile all policies, and emit the build artifacts to' + @echo '/tmp/cache_ext/build.' + @echo '' + + +all_targets: $(c-cache-ext-policy-targets) + +.PHONY: all all_targets $(c-cache-ext-policy-targets) clean help + +# delete failed targets +.DELETE_ON_ERROR: + +# keep intermediate (.bpf.skel.h, .bpf.o, etc) targets +.SECONDARY: diff --git a/tools/cache_ext/cache_ext_simple_fifo.bpf.c b/tools/cache_ext/cache_ext_simple_fifo.bpf.c new file mode 100644 index 000000000000..5aa7786a4bee --- /dev/null +++ b/tools/cache_ext/cache_ext_simple_fifo.bpf.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +#include "cache_ext_common.bpf.h" + +char _license[] SEC("license") = "GPL"; + +u64 simple_fifo_list; + +unsigned long isolated_stats[4] = {}; +const char stat_str[16] = "isolated: "; + +#define MAX_SCAN_SIZE (1024) + +SEC("struct_ops.s/simple_fifo_init") +s32 BPF_PROG(simple_fifo_init, struct mem_cgroup *memcg) +{ + simple_fifo_list = bpf_cache_ext_ds_registry_new_list(memcg); + + if (simple_fifo_list == 0) + return -EINVAL; + + return 0; +} + +static int bpf_simple_fifo_evict_cb(int idx, struct cache_ext_list_node *a) +{ + bool can_be_reclaimed = folio_test_lru(a->folio) && !folio_test_dirty(a->folio) && + !folio_test_writeback(a->folio); + + if (!can_be_reclaimed) { + if (idx < MAX_SCAN_SIZE) + return CACHE_EXT_CONTINUE_ITER; + else + return CACHE_EXT_STOP_ITER; + } + + return CACHE_EXT_ISOLATE_NODE; +} + +static void update_stats(int node, unsigned long nr) +{ + if (node >= 0 && node <= 3) + __sync_fetch_and_add(&isolated_stats[node], nr); +} + +SEC("struct_ops/simple_fifo_isolate_folios") +int BPF_PROG(simple_fifo_isolate_folios, struct cache_ext_isolation_ctx *eviction_ctx, + struct mem_cgroup *memcg) +{ + unsigned long isolated; + + + bpf_cache_ext_list_iterate(memcg, simple_fifo_list, bpf_simple_fifo_evict_cb, eviction_ctx); + + isolated = eviction_ctx->nr_base_pages_isolated; + + update_stats(eviction_ctx->numa_node_id, isolated); + + return 0; +} + +SEC("struct_ops/simple_fifo_folio_accessed") +void BPF_PROG(simple_fifo_folio_accessed, struct folio *folio) +{ + bpf_cache_ext_list_move(simple_fifo_list, folio, true); +} + +SEC("struct_ops/simple_fifo_folio_evicted") +s64 BPF_PROG(simple_fifo_folio_evicted, struct folio *folio) +{ + return bpf_cache_ext_list_del(folio); +} + +SEC("struct_ops/simple_fifo_folio_added") +void BPF_PROG(simple_fifo_folio_added, struct folio *folio, s64 cookie) +{ + bpf_cache_ext_list_add_tail(simple_fifo_list, folio); +} + +SEC("struct_ops/simple_fifo_get_statistics") +void BPF_PROG(simple_fifo_get_statistics, struct lruvec *lruvec, + struct cache_ext_stats *stats) +{ + int node = BPF_CORE_READ(lruvec, pgdat, node_id); + char *stat = (char *)stats + offsetof(struct cache_ext_stats, names); + + if (node < 0 || node >= 4) + return; + + stats->n = 1; + + __builtin_memcpy(stat, stat_str, sizeof(stat_str)); + stats->values[0] = READ_ONCE(isolated_stats[node]); + +} + +SEC(".struct_ops.link") +struct cache_ext_ops simple_fifo_ops = { + .init = simple_fifo_init, + .isolate_folios = simple_fifo_isolate_folios, + .folio_accessed = simple_fifo_folio_accessed, + .folio_evicted = simple_fifo_folio_evicted, + .folio_added = simple_fifo_folio_added, + .get_statistics = simple_fifo_get_statistics, +}; diff --git a/tools/cache_ext/cache_ext_simple_fifo.c b/tools/cache_ext/cache_ext_simple_fifo.c new file mode 100644 index 000000000000..85a2e2fa463b --- /dev/null +++ b/tools/cache_ext/cache_ext_simple_fifo.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <argp.h> +#include <bpf/bpf.h> +#include <fcntl.h> +#include <limits.h> +#include <signal.h> +#include <stdio.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <stdint.h> +#include <inttypes.h> + +typedef uint64_t u64; +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint8_t u8; + +#include "cache_ext_simple_fifo.bpf.skel.h" + +int main(int argc, char **argv) +{ + struct cache_ext_simple_fifo *skel = NULL; + struct bpf_link *link = NULL; + int ret = 1; + + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + + + skel = cache_ext_simple_fifo__open(); + if (!skel) { + perror("Failed to open BPF skeleton"); + goto cleanup; + } + + if (cache_ext_simple_fifo__load(skel)) { + perror("Failed to load BPF skeleton"); + goto cleanup; + } + + link = bpf_map__attach_struct_ops(skel->maps.simple_fifo_ops); + if (link == NULL) { + perror("Failed to attach cache_ext_ops to cgroup"); + goto cleanup; + } + + system("sleep infinity"); + ret = 0; +cleanup: + bpf_link__destroy(link); + return ret; +} diff --git a/tools/cache_ext/include/cache_ext_common.bpf.h b/tools/cache_ext/include/cache_ext_common.bpf.h new file mode 100644 index 000000000000..ab2ff78c3162 --- /dev/null +++ b/tools/cache_ext/include/cache_ext_common.bpf.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _CACHE_EXT_COMMON_BPF_H +#define _CACHE_EXT_COMMON_BPF_H + +#define U32_MAX ((u32)~0U) +#define U64_MAX ((u64)~0ULL) +#define S64_MAX ((s64)(U64_MAX >> 1)) + +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define max(a, b) ((a) > (b) ? (a) : (b)) +#define min(a, b) ((a) < (b) ? (a) : (b)) + +#define barrier() \ + do { \ + asm volatile("" ::: "memory"); \ + } while (0) + +typedef __u8 __attribute__((__may_alias__)) __u8_alias_t; +typedef __u16 __attribute__((__may_alias__)) __u16_alias_t; +typedef __u32 __attribute__((__may_alias__)) __u32_alias_t; +typedef __u64 __attribute__((__may_alias__)) __u64_alias_t; + +#define EINVAL 22 + + +static __always_inline void __read_once_size(const volatile void *p, void *res, int size) +{ + switch (size) { + case 1: + *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; + break; + case 2: + *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; + break; + case 4: + *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; + break; + case 8: + *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; + break; + default: + barrier(); + __builtin_memcpy((void *)res, (const void *)p, size); + barrier(); + } +} + +#define READ_ONCE(x) \ +({ \ + union { typeof(x) __val; char __c[1]; } __u = \ + { .__c = { 0 } }; \ + __read_once_size(&(x), __u.__c, sizeof(x)); \ + __u.__val; \ +}) + + +static __always_inline void __write_once_size(volatile void *p, void *res, int size) +{ + switch (size) { + case 1: + *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; + break; + case 2: + *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; + break; + case 4: + *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; + break; + case 8: + *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; + break; + default: + barrier(); + __builtin_memcpy((void *)p, (const void *)res, size); + barrier(); + } +} + +#define WRITE_ONCE(x, val) \ +({ \ + union { typeof(x) __val; char __c[1]; } __u = \ + { .__val = (val) }; \ + __write_once_size(&(x), __u.__c, sizeof(x)); \ + __u.__val; \ +}) + +int bpf_cache_ext_list_add(u64 list, struct folio *folio) __ksym; +int bpf_cache_ext_list_add_tail(u64 list, struct folio *folio) __ksym; +int bpf_cache_ext_list_del(struct folio *folio) __ksym; +int bpf_cache_ext_list_move(u64 list, struct folio *folio, bool tail) __ksym; +int bpf_cache_ext_list_iterate(struct mem_cgroup *memcg, u64 list, + int(iter_fn)(int idx, struct cache_ext_list_node *node), + struct cache_ext_isolation_ctx *ctx) __ksym; +int bpf_cache_ext_list_iterate_extended(struct mem_cgroup *memcg, u64 list, + int(iter_fn)(int idx, struct cache_ext_list_node *node), + struct cache_ext_iterate_opts *opts, + struct cache_ext_isolation_ctx *ctx) __ksym; +int bpf_cache_ext_list_sample(struct mem_cgroup *memcg, u64 list, + s32(score_fn)(struct cache_ext_list_node *a), + struct sampling_options *opts, + struct cache_ext_isolation_ctx *ctx) __ksym; +u64 bpf_cache_ext_ds_registry_new_list(struct mem_cgroup *memcg) __ksym; +bool bpf_cache_ext_filename_strstr(const char *str, struct folio *folio) __ksym; +bool bpf_cache_ext_list_empty(u64 list_id, struct mem_cgroup *memcg, int nid) __ksym; +int bpf_folio_nid(struct folio *folio) __ksym; +int bpf_cache_ext_list_splice(u64 list_id, u64 head_id, + struct mem_cgroup *memcg, int nid) __ksym; +int bpf_cache_ext_list_splice_tail(u64 list_id, u64 head_id, + struct mem_cgroup *memcg, int nid) __ksym; + +#define BITS_PER_LONG 64 +#define BIT_WORD(nr) ((nr) / BITS_PER_LONG) + +static __always_inline bool +generic_test_bit(unsigned long nr, const volatile unsigned long *addr) +{ + /* + * Unlike the bitops with the '__' prefix above, this one *is* atomic, + * so `volatile` must always stay here with no cast-aways. See + * `Documentation/atomic_bitops.txt` for the details. + */ + return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); +} + +static inline unsigned long *folio_flags(struct folio *folio, unsigned int n) +{ + struct page *page = &folio->page; + + return &page[n].flags; +} + + +static inline bool folio_test_uptodate(struct folio *folio) +{ + bool ret = generic_test_bit(PG_uptodate, folio_flags(folio, 0)); + + return ret; +} + +static inline bool folio_test_lru(struct folio *folio) +{ + bool ret = generic_test_bit(PG_lru, folio_flags(folio, 0)); + + return ret; +} + +static inline bool folio_test_dirty(struct folio *folio) +{ + bool ret = generic_test_bit(PG_dirty, folio_flags(folio, 0)); + + return ret; +} + +static inline bool folio_test_reclaim(struct folio *folio) +{ + bool ret = generic_test_bit(PG_reclaim, folio_flags(folio, 0)); + + return ret; +} + +static inline bool folio_test_writeback(struct folio *folio) +{ + bool ret = generic_test_bit(PG_writeback, folio_flags(folio, 0)); + + return ret; +} + +static inline bool folio_test_locked(struct folio *folio) +{ + bool ret = generic_test_bit(PG_locked, folio_flags(folio, 0)); + + return ret; +} + +#endif /* _CACHE_EXT_COMMON_BPF_H */ -- 2.34.1