From 38acd556ea35729de2a5b5c35d3b156b60b1c7fa Mon Sep 17 00:00:00 2001
From: "fu.lin" <fulin10@huawei.com>
Date: Tue, 23 Nov 2021 16:08:17 +0800
Subject: [PATCH 54/54] ptrace: trace specific syscall

criu use `ptrace(PTRACE_SYSCALL)` to watch whether the tracee steps in
correct status, it isn't necessory to stop tracee at every syscall.
Therefore, customizing `ptrace(PTRACE_SYSCALL)` to make tracee stop at
the specific syscall can save time (1000 threads consume about 140ms).

ptrace syntax:
    long ptrace(PTRACE_SYSCALL, pid_t pid, void *addr, void *data);

the argument `addr` is unused in original `ptrace(PTRACE_SYSCALL)`,
use `addr` parameter to give the specific sysno which is wanted to
trace.

Signed-off-by: fu.lin <fulin10@huawei.com>
---
 compel/Makefile              |   1 +
 compel/include/uapi/bisect.h |  30 +++++++
 compel/include/uapi/infect.h |  11 ++-
 compel/src/lib/bisect.c      |  92 +++++++++++++++++++
 compel/src/lib/infect.c      | 167 ++++++++++++++++++++++++++++++++---
 criu/config.c                |   1 +
 criu/cr-dump.c               |   2 +-
 criu/cr-restore.c            |  97 +++++++++++++++++++-
 criu/crtools.c               |   1 +
 criu/include/cr_options.h    |   1 +
 10 files changed, 385 insertions(+), 18 deletions(-)
 create mode 100644 compel/include/uapi/bisect.h
 create mode 100644 compel/src/lib/bisect.c

diff --git a/compel/Makefile b/compel/Makefile
index de9318c..eea93a7 100644
--- a/compel/Makefile
+++ b/compel/Makefile
@@ -27,6 +27,7 @@ lib-y			+= src/lib/infect-rpc.o
 lib-y			+= src/lib/infect-util.o
 lib-y			+= src/lib/infect.o
 lib-y			+= src/lib/ptrace.o
+lib-y			+= src/lib/bisect.o
 
 # handle_elf() has no support of ELF relocations on ARM (yet?)
 ifneq ($(filter arm aarch64,$(ARCH)),)
diff --git a/compel/include/uapi/bisect.h b/compel/include/uapi/bisect.h
new file mode 100644
index 0000000..9c00513
--- /dev/null
+++ b/compel/include/uapi/bisect.h
@@ -0,0 +1,30 @@
+#ifndef __COMPEL_BISECT_H__
+#define __COMPEL_BISECT_H__
+
+#include <sys/types.h>
+
+enum tf {
+	TRACE_INTERRUPT	= 0x17173,
+	TRACE_SYSCALL_ENTER,
+	TRACE_SYSCALL_EXIT,
+};
+
+struct trace_flag {
+	pid_t key;
+	enum tf flag; /* TODO: enum trace_flags flag */
+};
+
+struct bisect_meta {
+	int size;
+	int used;
+	void *data;	/* data pointer array */
+	void *__data;	/* data array */
+};
+
+struct trace_flag *tf_bisect(struct bisect_meta *meta, pid_t key);
+struct trace_flag *tf_insert(struct bisect_meta *meta, pid_t key);
+int tf_create(struct bisect_meta *meta, int len);
+void tf_destroy(struct bisect_meta *meta);
+void tf_clear(struct bisect_meta *meta);
+
+#endif /* __COMPEL_BISECT_H__ */
diff --git a/compel/include/uapi/infect.h b/compel/include/uapi/infect.h
index 257658a..99448a9 100644
--- a/compel/include/uapi/infect.h
+++ b/compel/include/uapi/infect.h
@@ -8,6 +8,7 @@
 #include <compel/ksigset.h>
 #include <compel/handle-elf.h>
 #include <compel/task-state.h>
+#include <compel/bisect.h>
 
 #include "common/compiler.h"
 
@@ -41,7 +42,7 @@ extern int __must_check compel_infect(struct parasite_ctl *ctl,
 extern struct parasite_thread_ctl __must_check *compel_prepare_thread(struct parasite_ctl *ctl, int pid);
 extern void compel_release_thread(struct parasite_thread_ctl *);
 
-extern int __must_check compel_stop_daemon(struct parasite_ctl *ctl);
+extern int __must_check compel_stop_daemon(struct parasite_ctl *ctl, bool customize);
 extern int __must_check compel_cure_remote(struct parasite_ctl *ctl);
 extern int __must_check compel_cure_local(struct parasite_ctl *ctl);
 extern int __must_check compel_cure(struct parasite_ctl *ctl);
@@ -90,6 +91,14 @@ extern int __must_check compel_stop_pie(pid_t pid, void *addr,
 
 extern int __must_check compel_unmap(struct parasite_ctl *ctl, unsigned long addr);
 
+extern int __must_check compel_stop_on_syscall_customize(int tasks,
+		const int sys_nr, const int exit_sys_nr, struct bisect_meta *meta);
+
+extern int __must_check compel_stop_pie_customize(pid_t pid,
+		const int sys_nr, struct trace_flag *tf);
+
+extern int __must_check compel_unmap_customize(struct parasite_ctl *ctl, unsigned long addr);
+
 extern int compel_mode_native(struct parasite_ctl *ctl);
 
 extern k_rtsigset_t *compel_task_sigmask(struct parasite_ctl *ctl);
diff --git a/compel/src/lib/bisect.c b/compel/src/lib/bisect.c
new file mode 100644
index 0000000..807a5a9
--- /dev/null
+++ b/compel/src/lib/bisect.c
@@ -0,0 +1,92 @@
+#include <stddef.h>
+
+#include "log.h"
+#include "common/xmalloc.h"
+#include "bisect.h"
+
+struct trace_flag *tf_bisect(struct bisect_meta *meta, pid_t key)
+{
+	struct trace_flag **tfs = meta->data;
+	int lo = 0, hi = meta->used, mid;
+
+	if (meta->used <= 0)
+		return NULL;
+
+	while (lo < hi) {
+		mid = (int)((lo + hi) / 2);
+		if (tfs[mid]->key == key) {
+			return tfs[mid];
+		} else if (tfs[mid]->key > key) {
+			hi = mid;
+		} else {
+			lo = mid + 1;
+		}
+	}
+
+	return NULL;
+}
+
+/* used in cr-restore */
+struct trace_flag *tf_insert(struct bisect_meta *meta, pid_t key)
+{
+	struct trace_flag **tfs = meta->data;
+	struct trace_flag *tf = &((struct trace_flag *)meta->__data)[meta->used];
+	int i = 0, j = 0;
+
+	if (meta->used == meta->size)
+		return NULL;
+
+	for (i = 0; i < meta->used; i++) {
+		if (tfs[i]->key >= key) /* impossible condition: `tfs[i]->key == key` */
+			break;
+	}
+
+	j = meta->used;
+	meta->used += 1;
+
+	while (j > i) {
+		tfs[j] = tfs[j-1];
+		j -= 1;
+	}
+
+	tfs[i] = tf;
+	tf->key = key;
+
+	return tf;
+}
+
+int tf_create(struct bisect_meta *meta, int len)
+{
+	struct trace_flag *tfs;
+	struct trace_flag **tfs_ptr;
+
+	tfs = xzalloc(sizeof(*tfs) * len);
+	if (tfs == NULL)
+		return -1;
+
+	tfs_ptr = xmalloc(sizeof(*tfs_ptr) * len);
+	if (tfs_ptr == NULL)
+		goto err;
+
+	meta->size = len;
+	meta->used = 0;
+	meta->__data = tfs;
+	meta->data = tfs_ptr;
+
+	return 0;
+err:
+	xfree(tfs);
+	return -1;
+}
+
+void tf_destroy(struct bisect_meta *meta)
+{
+	xfree(meta->__data);
+	xfree(meta->data);
+}
+
+void tf_clear(struct bisect_meta *meta)
+{
+	meta->used = 0;
+	__builtin_memset(meta->data, 0, sizeof(struct trace_flag **)*meta->size);
+}
diff --git a/compel/src/lib/infect.c b/compel/src/lib/infect.c
index 38846c2..2582cd2 100644
--- a/compel/src/lib/infect.c
+++ b/compel/src/lib/infect.c
@@ -442,7 +442,7 @@ static int restore_child_handler(struct parasite_ctl *ctl)
 }
 
 static int parasite_run(pid_t pid, int cmd, unsigned long ip, void *stack,
-		user_regs_struct_t *regs, struct thread_ctx *octx)
+		user_regs_struct_t *regs, struct thread_ctx *octx, void *addr)
 {
 	k_rtsigset_t block;
 
@@ -458,7 +458,7 @@ static int parasite_run(pid_t pid, int cmd, unsigned long ip, void *stack,
 		goto err_regs;
 	}
 
-	if (ptrace(cmd, pid, NULL, NULL)) {
+	if (ptrace(cmd, pid, addr, NULL)) {
 		pr_perror("Can't run parasite at %d", pid);
 		goto err_cont;
 	}
@@ -565,7 +565,7 @@ int compel_execute_syscall(struct parasite_ctl *ctl,
 		return -1;
 	}
 
-	err = parasite_run(pid, PTRACE_CONT, ctl->ictx.syscall_ip, 0, regs, &ctl->orig);
+	err = parasite_run(pid, PTRACE_CONT, ctl->ictx.syscall_ip, 0, regs, &ctl->orig, NULL);
 	if (!err)
 		err = parasite_trap(ctl, pid, regs, &ctl->orig);
 
@@ -583,7 +583,7 @@ int compel_run_at(struct parasite_ctl *ctl, unsigned long ip, user_regs_struct_t
 	user_regs_struct_t regs = ctl->orig.regs;
 	int ret;
 
-	ret = parasite_run(ctl->rpid, PTRACE_CONT, ip, 0, &regs, &ctl->orig);
+	ret = parasite_run(ctl->rpid, PTRACE_CONT, ip, 0, &regs, &ctl->orig, NULL);
 	if (!ret)
 		ret = parasite_trap(ctl, ctl->rpid, ret_regs ? ret_regs : &regs, &ctl->orig);
 	return ret;
@@ -632,7 +632,7 @@ static int parasite_init_daemon(struct parasite_ctl *ctl)
 		goto err;
 
 	regs = ctl->orig.regs;
-	if (parasite_run(pid, PTRACE_CONT, ctl->parasite_ip, ctl->rstack, &regs, &ctl->orig))
+	if (parasite_run(pid, PTRACE_CONT, ctl->parasite_ip, ctl->rstack, &regs, &ctl->orig, NULL))
 		goto err;
 
 	futex_wait_while_eq(&args->daemon_connected, 0);
@@ -1272,7 +1272,7 @@ static bool task_in_parasite(struct parasite_ctl *ctl, user_regs_struct_t *regs)
 		addr < ctl->remote_map + ctl->map_length;
 }
 
-static int parasite_fini_seized(struct parasite_ctl *ctl)
+static int parasite_fini_seized(struct parasite_ctl *ctl, bool customize)
 {
 	pid_t pid = ctl->rpid;
 	user_regs_struct_t regs;
@@ -1317,9 +1317,38 @@ static int parasite_fini_seized(struct parasite_ctl *ctl)
 	if (ret)
 		return -1;
 
+	/* use customize ptrace */
+	if (customize) {
+		struct trace_flag tf = { .key = pid, .flag = TRACE_SYSCALL_ENTER };
+		struct trace_flag *tf_ptr[] = { &tf };
+		struct bisect_meta meta = {
+			.size = 1,
+			.used = 1,
+			.__data = &tf,
+			.data = tf_ptr,
+		};
+
+		// TODO: compitable?
+		ret = compel_stop_pie_customize(pid, __NR(rt_sigreturn, 0), &tf);
+		if (ret < 0)
+			return ret;
+
+		/* The process is going to execute the required syscall, the
+		 * original syscall should be forgot(set `-1`) in
+		 * `syscall_trace_enter()` handler in kernel when no other
+		 * else operation in tracer.
+		 *
+		 * Note: -1 means NO_SYSCALL which is defined in
+		 *  `arch/arm64/include/asm/ptrace.h`.
+		 */
+		return compel_stop_on_syscall_customize(1,
+							__NR(rt_sigreturn, 0),
+							-1, &meta);
+	}
+
 	/* Go to sigreturn as closer as we can */
 	ret = compel_stop_pie(pid, ctl->sigreturn_addr, &flag,
-			ctl->ictx.flags & INFECT_NO_BREAKPOINTS);
+				ctl->ictx.flags & INFECT_NO_BREAKPOINTS);
 	if (ret < 0)
 		return ret;
 
@@ -1339,7 +1368,7 @@ static int parasite_fini_seized(struct parasite_ctl *ctl)
 	return 0;
 }
 
-int compel_stop_daemon(struct parasite_ctl *ctl)
+int compel_stop_daemon(struct parasite_ctl *ctl, bool customize)
 {
 	if (ctl->daemonized) {
 		/*
@@ -1349,7 +1378,7 @@ int compel_stop_daemon(struct parasite_ctl *ctl)
 		if (ctl->tsock < 0)
 			return -1;
 
-		if (parasite_fini_seized(ctl)) {
+		if (parasite_fini_seized(ctl, customize)) {
 			close_safe(&ctl->tsock);
 			return -1;
 		}
@@ -1365,7 +1394,7 @@ int compel_cure_remote(struct parasite_ctl *ctl)
 	long ret;
 	int err;
 
-	if (compel_stop_daemon(ctl))
+	if (compel_stop_daemon(ctl, false))
 		return -1;
 
 	if (!ctl->remote_map)
@@ -1434,7 +1463,7 @@ int compel_run_in_thread(struct parasite_thread_ctl *tctl, unsigned int cmd)
 
 	*ctl->cmd = cmd;
 
-	ret = parasite_run(pid, PTRACE_CONT, ctl->parasite_ip, stack, &regs, octx);
+	ret = parasite_run(pid, PTRACE_CONT, ctl->parasite_ip, stack, &regs, octx, NULL);
 	if (ret == 0)
 		ret = parasite_trap(ctl, pid, &regs, octx);
 	if (ret == 0)
@@ -1457,7 +1486,7 @@ int compel_unmap(struct parasite_ctl *ctl, unsigned long addr)
 	pid_t pid = ctl->rpid;
 	int ret = -1;
 
-	ret = parasite_run(pid, PTRACE_SYSCALL, addr, ctl->rstack, &regs, &ctl->orig);
+	ret = parasite_run(pid, PTRACE_SYSCALL, addr, ctl->rstack, &regs, &ctl->orig, NULL);
 	if (ret)
 		goto err;
 
@@ -1470,6 +1499,44 @@ err:
 	return ret;
 }
 
+int compel_unmap_customize(struct parasite_ctl *ctl, unsigned long addr)
+{
+	user_regs_struct_t regs = ctl->orig.regs;
+	pid_t pid = ctl->rpid;
+	int ret = -1;
+	struct trace_flag tf = { .key = pid, .flag = TRACE_SYSCALL_ENTER };
+	struct trace_flag *tf_ptr[] = { &tf };
+	struct bisect_meta meta = {
+		.size = 1,
+		.used = 1,
+		.__data = &tf,
+		.data = tf_ptr,
+	};
+
+	/*
+	 * Here it parasite code. Unlike trap code `compel_stop_pie()`, it
+	 * won't let tracee forget the original syscall. In such way,  tracer
+	 * just trace the syscall called by tracee. The log likes the following:
+	 *
+	 *   [  817.638332] set pid 1877 ptrace sysno 215
+	 *   [  817.638343] syscall_trace_enter: pid 1877 ptrace_sysno 0 current_sysno 215
+	 *   [  817.638363] (00.006280) Error (compel/src/lib/infect.c:1582): 1877 (native) is going to execute the syscall 215, required is 215
+	 *   [  817.638368] set pid 1877 ptrace sysno 0
+	 *   [  817.638402] syscall_trace_exit: pid 1877 ptrace_sysno 0 current_sysno 215
+	 */
+	ret = parasite_run(pid, PTRACE_SYSCALL, addr, ctl->rstack, &regs,
+				&ctl->orig, (void *)(long)__NR(munmap, 0));
+	if (ret)
+		goto err;
+
+	ret = compel_stop_on_syscall_customize(1, __NR(munmap, 0), 0, &meta);
+
+	if (restore_thread_ctx(pid, &ctl->orig))
+		ret = -1;
+err:
+	return ret;
+}
+
 int compel_stop_pie(pid_t pid, void *addr, enum trace_flags *tf, bool no_bp)
 {
 	int ret;
@@ -1505,6 +1572,17 @@ int compel_stop_pie(pid_t pid, void *addr, enum trace_flags *tf, bool no_bp)
 	return 0;
 }
 
+int compel_stop_pie_customize(pid_t pid, const int sys_nr, struct trace_flag *tf)
+{
+	if (ptrace(PTRACE_SYSCALL, pid, sys_nr, NULL)) {
+		pr_perror("Unable to restart the %d process", pid);
+		return -1;
+	}
+
+	tf->flag = TRACE_SYSCALL_ENTER;
+	return 0;
+}
+
 static bool task_is_trapped(int status, pid_t pid)
 {
 	if (WIFSTOPPED(status) && WSTOPSIG(status) == SIGTRAP)
@@ -1617,6 +1695,71 @@ goon:
 	return 0;
 }
 
+int compel_stop_on_syscall_customize(int tasks, const int sys_nr,
+	const int exit_sys_nr, struct bisect_meta *meta)
+{
+	struct trace_flag *tf;
+	user_regs_struct_t regs;
+	int status, ret;
+	pid_t pid;
+
+	while (tasks) {
+		pid = wait4(-1, &status, __WALL, NULL);
+		if (pid == -1) {
+			pr_perror("wait4 failed");
+			return -1;
+		}
+
+		if (!task_is_trapped(status, pid))
+			return -1;
+
+		tf = tf_bisect(meta, pid);
+		if (tf == NULL) {
+			pr_err("Can't find ptrace status for %d\n", pid);
+			return -1;
+		}
+
+		switch (tf->flag) {
+			case TRACE_SYSCALL_ENTER:
+				pr_debug("%d was trapped\n", pid);
+				pr_debug("`- Expecting exit\n");
+
+				ret = ptrace_get_regs(pid, &regs);
+				if (ret) {
+					pr_perror("ptrace");
+					return -1;
+				}
+
+				if (is_required_syscall(&regs, pid, sys_nr, sys_nr)) {
+					ret = ptrace(PTRACE_SYSCALL, pid, exit_sys_nr, NULL);
+					if (ret) {
+						pr_perror("ptrace");
+						return -1;
+					}
+					tf->flag = TRACE_SYSCALL_EXIT;
+				} else {
+					pr_warn("Impossible condition, check the system, try our best to restore...\n");
+					ret = ptrace(PTRACE_SYSCALL, pid, sys_nr, NULL);
+					if (ret) {
+						pr_perror("ptrace");
+						return -1;
+					}
+				}
+				break;
+			case TRACE_SYSCALL_EXIT:
+				pr_debug("%d was stopped\n", pid);
+				tasks--;
+				break;
+
+			default:
+				pr_err("pid %d invalid status: %d\n", pid, tf->flag);
+				return -1;
+		}
+	}
+
+	return 0;
+}
+
 int compel_mode_native(struct parasite_ctl *ctl)
 {
 	return user_regs_native(&ctl->orig.regs);
diff --git a/criu/config.c b/criu/config.c
index e09445a..b0d3639 100644
--- a/criu/config.c
+++ b/criu/config.c
@@ -555,6 +555,7 @@ int parse_options(int argc, char **argv, bool *usage_error,
 		{"reserve-ports", 		required_argument,	0, 'P'	},
 		BOOL_OPT("use-nft", &opts.use_nft),
 		BOOL_OPT("parallel", &opts.parallel),
+		BOOL_OPT("customize-ptrace", &opts.customize_ptrace),
 		{ },
 	};
 
diff --git a/criu/cr-dump.c b/criu/cr-dump.c
index 24b0ef5..21ae657 100644
--- a/criu/cr-dump.c
+++ b/criu/cr-dump.c
@@ -1428,7 +1428,7 @@ static int dump_one_task(struct pstree_item *item, InventoryEntry *parent_ie)
 		goto err_cure;
 	}
 
-	ret = compel_stop_daemon(parasite_ctl);
+	ret = compel_stop_daemon(parasite_ctl, opts.customize_ptrace);
 	if (ret) {
 		pr_err("Can't stop daemon in parasite (pid: %d)\n", pid);
 		goto err_cure;
diff --git a/criu/cr-restore.c b/criu/cr-restore.c
index 83bdc38..33f56a3 100644
--- a/criu/cr-restore.c
+++ b/criu/cr-restore.c
@@ -2171,6 +2171,64 @@ static int catch_tasks(bool root_seized, enum trace_flags *flag)
 	return 0;
 }
 
+static int cache_tasks_customize(bool root_seized, struct bisect_meta *meta)
+{
+	struct pstree_item *item;
+	struct trace_flag *tf;
+
+	for_each_pstree_item(item) {
+		int status, i, ret;
+		pid_t pid;
+
+		if (!task_alive(item))
+			continue;
+
+		if (item->nr_threads == 1) {
+			item->threads[0].real = item->pid->real;
+		} else {
+			if (parse_threads(item->pid->real, &item->threads, &item->nr_threads))
+				return -1;
+		}
+
+		for (i = 0; i < item->nr_threads; i++) {
+			pid = item->threads[i].real;
+
+			if (ptrace(PTRACE_INTERRUPT, pid, 0, 0)) {
+				pr_perror("Can't interrupt the %d task", pid);
+				return -1;
+			}
+
+			tf = tf_insert(meta, pid);
+			if (tf == NULL) {
+				pr_err("Can't find trace flag for %d, used %d\n",
+						pid, meta->used);
+				return -1;
+			}
+			tf->flag = TRACE_INTERRUPT;
+		}
+
+		for (i = 0; i < item->nr_threads; i++) {
+			pid = wait4(-1, &status, __WALL, NULL);
+			
+			tf = tf_bisect(meta, pid);
+			if (tf == NULL) {
+				pr_err("Can't find trace flag for %d, used %d\n",
+						pid, meta->used);
+				return -1;
+			}
+
+			ret = compel_stop_pie_customize(pid,
+							__NR(rt_sigreturn, 0),
+							tf);
+			if (ret < 0)
+				return -1;
+
+		}
+	}
+
+	return 0;
+}
+
 static int clear_breakpoints(void)
 {
 	struct pstree_item *item;
@@ -2197,6 +2255,7 @@ static void finalize_restore(void)
 		pid_t pid = item->pid->real;
 		struct parasite_ctl *ctl;
 		unsigned long restorer_addr;
+		int retval;
 
 		if (!task_alive(item))
 			continue;
@@ -2207,7 +2266,12 @@ static void finalize_restore(void)
 			continue;
 
 		restorer_addr = (unsigned long)rsti(item)->munmap_restorer;
-		if (compel_unmap(ctl, restorer_addr))
+		if (!opts.customize_ptrace)
+			retval = compel_unmap(ctl, restorer_addr);
+		else
+			retval = compel_unmap_customize(ctl, restorer_addr);
+
+		if (retval)
 			pr_err("Failed to unmap restorer from %d\n", pid);
 
 		xfree(ctl);
@@ -2312,11 +2376,18 @@ static int write_restored_pid(void)
 
 static int restore_root_task(struct pstree_item *init)
 {
+	struct bisect_meta tfs_meta;
 	enum trace_flags flag = TRACE_ALL;
 	int ret, fd, mnt_ns_fd = -1;
 	int root_seized = 0;
 	struct pstree_item *item;
 
+	if (opts.customize_ptrace
+			&& tf_create(&tfs_meta, task_entries->nr_threads) != 0) {
+		pr_err("Can't alloc memory, tf_create failed\n");
+		return -1;
+	}
+
 	ret = run_scripts(ACT_PRE_RESTORE);
 	if (ret != 0) {
 		pr_err("Aborting restore due to pre-restore script ret code %d\n", ret);
@@ -2521,7 +2592,12 @@ skip_ns_bouncing:
 
 	timing_stop(TIME_RESTORE);
 
-	if (catch_tasks(root_seized, &flag)) {
+	if (!opts.customize_ptrace)
+		ret = catch_tasks(root_seized, &flag);
+	else
+		ret = cache_tasks_customize(root_seized, &tfs_meta);
+
+	if (ret) {
 		pr_err("Can't catch all tasks\n");
 		goto out_kill_network_unlocked;
 	}
@@ -2531,8 +2607,14 @@ skip_ns_bouncing:
 
 	__restore_switch_stage(CR_STATE_COMPLETE);
 
-	ret = compel_stop_on_syscall(task_entries->nr_threads,
-		__NR(rt_sigreturn, 0), __NR(rt_sigreturn, 1), flag);
+	if (!opts.customize_ptrace) {
+		ret = compel_stop_on_syscall(task_entries->nr_threads,
+			__NR(rt_sigreturn, 0), __NR(rt_sigreturn, 1), flag);
+	} else {
+		ret = compel_stop_on_syscall_customize(task_entries->nr_threads,
+							__NR(rt_sigreturn, 0),
+							-1, &tfs_meta);
+	}
 	if (ret) {
 		pr_err("Can't stop all tasks on rt_sigreturn\n");
 		goto out_kill_network_unlocked;
@@ -2571,6 +2653,9 @@ skip_ns_bouncing:
 	if (!opts.restore_detach && !opts.exec_cmd)
 		wait(NULL);
 
+	if (opts.customize_ptrace)
+		tf_destroy(&tfs_meta);
+
 	return 0;
 
 out_kill_network_unlocked:
@@ -2605,6 +2690,10 @@ out:
 	stop_usernsd();
 	__restore_switch_stage(CR_STATE_FAIL);
 	pr_err("Restoring FAILED.\n");
+
+	if (opts.customize_ptrace)
+		tf_destroy(&tfs_meta);
+
 	return -1;
 }
 
diff --git a/criu/crtools.c b/criu/crtools.c
index 5127cde..0ef76b5 100644
--- a/criu/crtools.c
+++ b/criu/crtools.c
@@ -491,6 +491,7 @@ usage:
 "  --reserve-ports       Reserve src ports in kernel\n"
 "  --use-nft             Use nft API instead of iptables cmd in network locking\n"
 "  --parallel            Parallel to accellrate dumping speed\n"
+"  --customize-ptrace	 Use customize ptrace(PTRACE_SYSCALL)\n"
 "\n"
 "Check options:\n"
 "  Without options, \"criu check\" checks availability of absolutely required\n"
diff --git a/criu/include/cr_options.h b/criu/include/cr_options.h
index cc8d1ae..b7080ca 100644
--- a/criu/include/cr_options.h
+++ b/criu/include/cr_options.h
@@ -189,6 +189,7 @@ struct cr_options {
 	int			reserve_ports;
 	int			use_nft;
 	int			parallel;
+	int			customize_ptrace;
 };
 
 extern struct cr_options opts;
-- 
2.33.1

