--- test/zdtm.py | 66 ++++++++-- test/zdtm/customization/Makefile | 3 +- test/zdtm/customization/notifier00.c | 68 ++++++++++ test/zdtm/customization/notifier00.desc | 1 + test/zdtm/mod/.gitignore | 163 ++++++++++++++++++++++++ test/zdtm/mod/Makefile | 28 ++++ test/zdtm/mod/notifier.c | 145 +++++++++++++++++++++ 7 files changed, 464 insertions(+), 10 deletions(-) create mode 100644 test/zdtm/customization/notifier00.c create mode 100644 test/zdtm/customization/notifier00.desc create mode 100644 test/zdtm/mod/.gitignore create mode 100644 test/zdtm/mod/Makefile create mode 100644 test/zdtm/mod/notifier.c
diff --git a/test/zdtm.py b/test/zdtm.py index 111f9f1..2a29400 100755 --- a/test/zdtm.py +++ b/test/zdtm.py @@ -25,6 +25,7 @@ import tempfile import time import pathlib from builtins import (input, int, open, range, str, zip) +import platform
import pycriu as crpc
@@ -1460,6 +1461,13 @@ class criu: return True return False
+ @staticmethod + def check_sysfs(pathes): + for path in pathes.split(): + if not pathlib.Path(path).exists(): + return True + return False + @staticmethod def available(): if not os.access(opts['criu_bin'], os.X_OK): @@ -1983,17 +1991,45 @@ class Launcher: testline = u"ok %d - %s # SKIP %s" % (self.__runtest, name, reason) print(testline, file=self.__file_report)
+ def check_module(self, mod): + found = False + with open("/proc/modules") as f: + for line in f.readlines(): + if "pin_memory" == line.split()[0]: + found = True + return found + def modprobe_pin_memory(self, load): + mod = "pin_memory" if not load: return - else: - found = False - with open("/proc/modules") as f: - for line in f.readlines(): - if "pin_memory" == line.split()[0]: - found = True - if not found: - subprocess.check_call(["modprobe", "pin_memory"]) + elif not self.check_module(mod): + subprocess.check_call(["modprobe", "pin_memory"]) + + def build_and_load_mod(self, target, kdir): + if platform.machine() != "aarch64" or not target: + return + + if not os.access("zdtm/mod", os.R_OK): + print("should be executed in the test subdir") + sys.exit(0) + + dirpath = f"MOD={os.getcwd()}/zdtm/mod" + build_mod = ["make", "-C", "zdtm/mod", dirpath, target] + if kdir: + build_mod.append(f"KDIR={kdir}") + subprocess.check_call(build_mod) + + # ensure the module has been unloaded + if self.check_module(target.rstrip(".ko")): + subprocess.run(["rmmod", target], check=False) + + modpath = f"zdtm/mod/{target}" + subprocess.check_call(["insmod", modpath]) + + def unload_mod(self, mod): + if mod: + subprocess.check_call(["rmmod", mod])
def run_test(self, name, desc, flavor):
@@ -2034,6 +2070,8 @@ class Launcher: # `--use-fork-pid`, so don't care `--pin-memory` option self.modprobe_pin_memory(no_pid_ns)
+ self.build_and_load_mod(desc.get("mod", ""), opts["kdir"]) + sub = subprocess.Popen(["./zdtm_ct", "zdtm.py"], env=dict(os.environ, CR_CT_TEST_INFO=arg, ZDTM_NO_PID_NS=zdtm_no_pid_ns), @@ -2048,9 +2086,11 @@ class Launcher: }
# pin memory function don't support concurrency - if test_flag(desc, 'excl') or test_value(desc, "opts", "--pin-memory"): + if test_flag(desc, 'excl') or test_value(desc, "opts", "--pin-memory") or desc.get("mod", ""): self.wait()
+ self.unload_mod(desc.get("mod", "")) + def __wait_one(self, flags): pid = -1 status = -1 @@ -2390,6 +2430,11 @@ def run_tests(opts): t, f"cmdline '{cmdline}' isn't support, or don't set") continue
+ sysfs = tdesc.get('sysfs', '') + if sysfs and criu.check_sysfs(sysfs): + launcher.skip(t, f"sysfs file {sysfs} don't exist") + continue + test_flavs = tdesc.get('flavor', 'h ns uns').split() opts_flavs = (opts['flavor'] or 'h,ns,uns').split(',') if opts_flavs != ['best']: @@ -2412,6 +2457,7 @@ def run_tests(opts): launcher.run_test(t, tdesc, run_flavs) else: launcher.skip(t, "no flavors") + finally: fail = launcher.finish() if opts['join_ns']: @@ -2701,6 +2747,8 @@ rp.add_argument("--pre-dump-mode", rp.add_argument("--kdat", help="Path to criu.kdat, default '/run/criu.kdat'", default="/run/criu.kdat") +rp.add_argument( + "--kdir", help="specific kernel devel path, the default value is `/lib/modules/$(uname -r)/build`")
lp = sp.add_parser("list", help="List tests") lp.set_defaults(action=list_tests) diff --git a/test/zdtm/customization/Makefile b/test/zdtm/customization/Makefile index 82348f2..93922c7 100644 --- a/test/zdtm/customization/Makefile +++ b/test/zdtm/customization/Makefile @@ -10,7 +10,8 @@ TST_NOFILE = \ maps04 \ maps05 \ maps007 \ - maps008 + maps008 \ + notifier00
TST_FILE = \ maps00 \ diff --git a/test/zdtm/customization/notifier00.c b/test/zdtm/customization/notifier00.c new file mode 100644 index 0000000..5fc3d54 --- /dev/null +++ b/test/zdtm/customization/notifier00.c @@ -0,0 +1,68 @@ +#include <stdio.h> +/* Historical reasons: in order to compatible with R10 */ +#define CONFIG_EULEROS_MODRESTORE_NOTIFY +#include <linux/modrestore.h> + +#include "zdtmtst.h" + +const char *test_doc = "Tests the basic function of the notifiers"; +static char *nvwa_notifiers[] = { + "PRE_FREEZE", + "FREEZE_TO_KILL", + "PRE_UPDATE_KERNEL", + "POST_UPDATE_KERNEL", + "UNFREEZE_TO_RUN", + "POST_RUN" +}; + +_Static_assert(sizeof(nvwa_notifiers)/sizeof(nvwa_notifiers[0]) == KUP_HOOK_MAX, "nvwa_notifiers number is wrong!"); + +int main(int argc, char *argv[]) +{ + int orig_values[KUP_HOOK_MAX] = {0}; + bool failure = false; + FILE *fp; + + test_init(argc, argv); + + fp = fopen("/sys/kernel/criu_notifier", "r"); + if (fp == NULL) { + pr_perror("fopen"); + return 1; + } + + for (int i = 0; i < KUP_HOOK_MAX; i++) + fscanf(fp, "%d ", orig_values+i); + + test_daemon(); + test_waitsig(); + + if (fseek(fp, 0, SEEK_SET) != 0) { + pr_perror("fseek"); + return 2; + } + + for (int i = 0; i < KUP_HOOK_MAX; i++) { + int val = 0; + int should = orig_values[i]+1; + + fscanf(fp, "%d ", &val); + + /* those are not called in criu */ + if (i == PRE_UPDATE_KERNEL || i == POST_UPDATE_KERNEL) + continue; + + if (val != should) { + pr_err("%s notifier is abnormal, it should be %d, but %d.\n", + nvwa_notifiers[i], should, val); + failure = true; + } + } + + if (failure) + fail("notifier is abnormal."); + else + pass(); + + return 0; +} diff --git a/test/zdtm/customization/notifier00.desc b/test/zdtm/customization/notifier00.desc new file mode 100644 index 0000000..1c6b512 --- /dev/null +++ b/test/zdtm/customization/notifier00.desc @@ -0,0 +1 @@ +{'arch': 'aarch64', 'opts': '--with-notifier', 'flavor': 'h', 'flags': 'suid', 'sysfs': '/sys/kernel/modrestore/nvwa_notifier', 'mod': 'notifier.ko'} diff --git a/test/zdtm/mod/.gitignore b/test/zdtm/mod/.gitignore new file mode 100644 index 0000000..7afd412 --- /dev/null +++ b/test/zdtm/mod/.gitignore @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# NOTE! Don't add files that are generated in specific +# subdirectories here. Add them in the ".gitignore" file +# in that subdirectory instead. +# +# NOTE! Please use 'git ls-files -i --exclude-standard' +# command after changing this file, to see if there are +# any tracked files which get ignored after the change. +# +# Normal rules (sorted alphabetically) +# +.* +*.a +*.asn1.[ch] +*.bin +*.bz2 +*.c.[012]*.* +*.dt.yaml +*.dtb +*.dtbo +*.dtb.S +*.dwo +*.elf +*.gcno +*.gz +*.i +*.ko +*.lex.c +*.ll +*.lst +*.lz4 +*.lzma +*.lzo +*.mod +*.mod.c +*.o +*.o.* +*.patch +*.s +*.so +*.so.dbg +*.su +*.symtypes +*.symversions +*.tab.[ch] +*.tar +*.xz +*.zst +Module.symvers +modules.order + +# +# Top-level generic files +# +/linux +/modules-only.symvers +/vmlinux +/vmlinux.32 +/vmlinux.map +/vmlinux.symvers +/vmlinux-gdb.py +/vmlinuz +/System.map +/Module.markers +/modules.builtin +/modules.builtin.modinfo +/modules.nsdeps + +# +# RPM spec file (make rpm-pkg) +# +/*.spec + +# +# Debian directory (make deb-pkg) +# +/debian/ + +# +# Snap directory (make snap-pkg) +# +/snap/ + +# +# tar directory (make tar*-pkg) +# +/tar-install/ + +# +# We don't want to ignore the following even if they are dot-files +# +!.clang-format +!.cocciconfig +!.get_maintainer.ignore +!.gitattributes +!.gitignore +!.mailmap + +# +# Generated include files +# +/include/config/ +/include/generated/ +/include/ksym/ +/arch/*/include/generated/ + +# stgit generated dirs +patches-* + +# quilt's files +patches +series + +# ctags files +tags +TAGS + +# cscope files +cscope.* +ncscope.* + +# gnu global files +GPATH +GRTAGS +GSYMS +GTAGS + +# id-utils files +ID + +*.orig +*~ +#*# + +# +# Leavings from module signing +# +extra_certificates +signing_key.pem +signing_key.priv +signing_key.x509 +x509.genkey + +# Kconfig presets +/all.config +/alldef.config +/allmod.config +/allno.config +/allrandom.config +/allyes.config + +# Kconfig savedefconfig output +/defconfig + +# Kdevelop4 +*.kdev4 + +# Clang's compilation database file +/compile_commands.json + +# Documentation toolchain +sphinx_*/ diff --git a/test/zdtm/mod/Makefile b/test/zdtm/mod/Makefile new file mode 100644 index 0000000..10c9c9a --- /dev/null +++ b/test/zdtm/mod/Makefile @@ -0,0 +1,28 @@ +# notice: +# `ARCH` var is used in both criu and kernel, but they have the different value +# for the same architecture(e.g. arm64). Therefore, this Makefile can't be +# included in the criu Makefile. +obj-m += notifier.o + +# specific the kernel devel path +# example (use `/home/me/kernel` as `KDIR`): +# $ export KDIR="/home/me/kernel" +ifeq ($(KDIR),) + KDIR := /lib/modules/$(shell uname -r)/build +endif + +# specific the mod src path +ifeq ($(MOD),) + MOD := $(PWD) +endif + +all: + $(MAKE) -C $(KDIR) M=$(MOD) modules + +clean: + $(MAKE) -C $(KDIR) M=$(MOD) clean + +.PHONY: all clean + +notifier.ko: + $(MAKE) -C $(KDIR) M=$(MOD) notifier.ko diff --git a/test/zdtm/mod/notifier.c b/test/zdtm/mod/notifier.c new file mode 100644 index 0000000..70a5b33 --- /dev/null +++ b/test/zdtm/mod/notifier.c @@ -0,0 +1,145 @@ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/modrestore.h> + +static int values[KUP_HOOK_MAX]; +static char *nvwa_actions[] = { + "PREPARE", + "ROLLBACK", +}; +static char *nvwa_notifiers[] = { + "PRE_FREEZE", + "FREEZE_TO_KILL", + "PRE_UPDATE_KERNEL", + "POST_UPDATE_KERNEL", + "UNFREEZE_TO_RUN", + "POST_RUN" +}; + +static int nvwa_notifier_func(struct notifier_block *nb, unsigned long val, void *data) +{ + struct nvwa_action *action = data; + + switch (action->cmd) { + case PREPARE: + values[val] += 1; + break; + case ROLLBACK: + values[val] -= 1; + break; + default: + pr_err("invalid cmd: %d", action->cmd); + return NOTIFY_BAD; + } + + pr_info("nvwa notifier action %s", nvwa_actions[action->cmd]); + + return NOTIFY_DONE; +} + +#define DEFINE_NVWA_NB(name) \ + static struct notifier_block nvwa_##name##_nb = { \ + .notifier_call = nvwa_notifier_func, \ + } + +DEFINE_NVWA_NB(pre_freeze); +DEFINE_NVWA_NB(freeze_to_kill); +DEFINE_NVWA_NB(pre_update_kernel); +DEFINE_NVWA_NB(post_update_kernel); +DEFINE_NVWA_NB(unfreeze_to_run); +DEFINE_NVWA_NB(post_run); + +static struct notifier_block *nvwa_nbs[] = { + &nvwa_pre_freeze_nb, + &nvwa_freeze_to_kill_nb, + &nvwa_pre_update_kernel_nb, + &nvwa_post_update_kernel_nb, + &nvwa_unfreeze_to_run_nb, + &nvwa_post_run_nb, +}; + +static int register_nvwa_notifiers(void) +{ + int i; + + BUILD_BUG_ON_MSG(ARRAY_SIZE(nvwa_nbs) != KUP_HOOK_MAX, + "wrong nvwa notifier block size!"); + + for (i = 0; i < ARRAY_SIZE(nvwa_nbs); i++) { + if (register_nvwa_notifier(i, nvwa_nbs[i]) != 0) { + pr_err("register nvwa %s notifier failed!", nvwa_notifiers[i]); + goto error; + } + } + + return 0; + +error: + + for (i -= 1; i >= 0; i -= 1) + unregister_nvwa_notifier(i, nvwa_nbs[i]); + + return -1; +} + +static void unregister_nvwa_notifiers(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(nvwa_nbs); i++) + unregister_nvwa_notifier(i, nvwa_nbs[i]); +} + +static ssize_t criu_notifier_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(values); i++) + values[i] = 0; + + return count; +} + +static ssize_t criu_notifier_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + int i; + ssize_t count = 0; + + for (i = 0; i < ARRAY_SIZE(values); i++) + count += sprintf(buf+count, "%d ", values[i]); + + buf[count-1] = '\n'; + + return count; +} + +static struct kobj_attribute notifier_file = __ATTR_RW(criu_notifier); + +static int __init notifier_init(void) +{ + if (register_nvwa_notifiers() != 0) + return -1; + + if (sysfs_create_file(kernel_kobj, ¬ifier_file.attr) != 0) { + unregister_nvwa_notifiers(); + return -1; + } + + return 0; +} + +static void __exit notifier_exit(void) +{ + sysfs_remove_file(kernel_kobj, ¬ifier_file.attr); + unregister_nvwa_notifiers(); +} + +module_init(notifier_init); +module_exit(notifier_exit); +MODULE_LICENSE("GPL");