From 636c36822e99ffda5b66a895a2c2d962b3ab08be Mon Sep 17 00:00:00 2001
From: "fu.lin" <fulin10@huawei.com>
Date: Mon, 29 Nov 2021 16:03:02 +0800
Subject: [PATCH 53/54] parallel: parallel nft delete set

The nft has two part: rules and set. criu delete nft rules to unlock network
during restoration. The set deletion action consumes about hundreds of ms when
there are too many elements in nft set. Dealying set deletion is helpful
to save restoration time.

Signed-off-by: fu.lin <fulin10@huawei.com>
---
 criu/cr-dump.c           |  1 +
 criu/cr-restore.c        |  3 ++-
 criu/include/taskqueue.h | 12 ++++++++++-
 criu/nftables.c          | 14 +++++++++++--
 criu/taskqueue.c         | 45 ++++++++++++++++++++++++++++++++++++++++
 5 files changed, 71 insertions(+), 4 deletions(-)

diff --git a/criu/cr-dump.c b/criu/cr-dump.c
index 30bbc5b..24b0ef5 100644
--- a/criu/cr-dump.c
+++ b/criu/cr-dump.c
@@ -1799,6 +1799,7 @@ static int cr_dump_finish(int ret)
 	 */
 	if (ret || post_dump_ret || opts.final_state == TASK_ALIVE) {
 		network_unlock(opts.tree_id);
+		parallel_nft_clean((long)opts.tree_id);
 		delete_link_remaps();
 		clean_cr_time_mounts();
 	}
diff --git a/criu/cr-restore.c b/criu/cr-restore.c
index e94d5a6..83bdc38 100644
--- a/criu/cr-restore.c
+++ b/criu/cr-restore.c
@@ -2566,7 +2566,7 @@ skip_ns_bouncing:
 	if (ret != 0)
 		pr_err("Post-resume script ret code %d\n", ret);
 
-	network_delete_set(vpid(init));
+	parallel_nft_clean((long)vpid(init));
 
 	if (!opts.restore_detach && !opts.exec_cmd)
 		wait(NULL);
@@ -2575,6 +2575,7 @@ skip_ns_bouncing:
 
 out_kill_network_unlocked:
 	pr_err("Killing processes because of failure on restore.\nThe Network was unlocked so some data or a connection may have been lost.\n");
+	parallel_nft_clean((long)vpid(init));
 out_kill:
 	/*
 	 * The processes can be killed only when all of them have been created,
diff --git a/criu/include/taskqueue.h b/criu/include/taskqueue.h
index 16f9e3d..906c784 100644
--- a/criu/include/taskqueue.h
+++ b/criu/include/taskqueue.h
@@ -6,7 +6,6 @@
 #include <semaphore.h>
 
 #include "vma.h"
-#include "pstree.h"
 
 #include "common/list.h"
 
@@ -47,4 +46,15 @@ struct mappings_info {
 int start_collect_mappings_thread(void);
 int end_collect_mappings_thread(struct pstree_item *item);
 
+#define STACK_SIZE	(1024 *1024)
+typedef void (*daemon_t)(void *);
+int parallel_task(daemon_t fn, void *_arg);
+
+struct daemon {
+	daemon_t fn;
+	void *arg;
+};
+
+void parallel_nft_clean(long tree_id);
+
 #endif /* __CR_TASKQUEUE_H__ */
diff --git a/criu/nftables.c b/criu/nftables.c
index 739aee4..0c529ed 100644
--- a/criu/nftables.c
+++ b/criu/nftables.c
@@ -16,6 +16,7 @@
 
 #include "sk-inet.h"
 #include "nftables.h"
+#include "taskqueue.h"
 
 #include "../soccr/soccr.h"
 
@@ -661,10 +662,9 @@ static int network_delete_rule_internal(struct mnl_params *params,
 	return nft_rule_common(params, tree_id, false);
 }
 
+/* here split the deletion of rule and set to accelete the restoration process */
 void network_delete_rule(pid_t tree_id)
 {
-	pr_info("unlock network\n");
-
 	mnl_common(network_delete_rule_internal, NULL, &tree_id);
 }
 
@@ -683,6 +683,16 @@ void network_delete_set(pid_t tree_id)
 	mnl_common(network_delete_set_internal, NULL, &tree_id);
 }
 
+void parallel_nft_clean_internal(void *arg)
+{
+	network_delete_set((long)arg);
+}
+
+void parallel_nft_clean(long tree_id)
+{
+	parallel_task(parallel_nft_clean_internal, (void *)tree_id);
+}
+
 static int add_set_elem_internal(struct nftnl_set *s, void *data, size_t len)
 {
 	struct nftnl_set_elem *e;
diff --git a/criu/taskqueue.c b/criu/taskqueue.c
index 1196a5e..7d500e9 100644
--- a/criu/taskqueue.c
+++ b/criu/taskqueue.c
@@ -122,3 +122,48 @@ int end_collect_mappings_thread(struct pstree_item *item)
 	 */
 	return retval;
 }
+
+static int daemonize(void *arg)
+{
+	struct daemon *d = arg;
+
+	if (daemon(0, 0) < 0)
+		pr_perror("daemonize failed");
+
+	d->fn(d->arg);
+
+	return 0;
+}
+
+int parallel_task(daemon_t fn, void *_arg)
+{
+	struct daemon arg = {
+		.fn = fn,
+		.arg = _arg,
+	};
+	char *stack;
+	char *stack_top;
+	pid_t pid;
+
+	stack = mmap(NULL, STACK_SIZE, PROT_READ | PROT_WRITE,
+			MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+	if (stack == MAP_FAILED) {
+		pr_perror("mmap failed");
+		return -1;
+	}
+
+	stack_top = stack + STACK_SIZE;
+
+	/* ignore SIGCHLD signal */
+	pid = clone(daemonize, stack_top, 0, &arg);
+	if (pid > 0)
+		return 0; /* parent */
+	else if (pid < 0) {
+		pr_perror("clone failed");
+		return -1;
+	}
+
+	/* unreachable */
+	__builtin_unreachable();
+	return 0;
+}
-- 
2.33.1

