From: Jingxian He hejingxian@huawei.com
In order to reduce the cost time of network unlock, make the nft table clearing operation run after resuming.
The nft has two part: rules and set. criu delete nft rules to unlock network during restoration. The set deletion action consumes about hundreds of ms when there are too many elements in nft set. Dealying set deletion is helpful to save restoration time.
Conflict:NA Reference:https://gitee.com/src-openeuler/criu/pulls/21
Signed-off-by: Jingxian He hejingxian@huawei.com Signed-off-by: fu.lin fulin10@huawei.com --- criu/config.c | 1 + criu/cr-dump.c | 7 ++++-- criu/cr-restore.c | 11 ++++++++++ criu/crtools.c | 1 + criu/include/cr_options.h | 1 + criu/include/net.h | 3 ++- criu/include/taskqueue.h | 12 ++++++++++- criu/net.c | 2 +- criu/nftables.c | 37 ++++++++++++++++++++++++-------- criu/taskqueue.c | 45 +++++++++++++++++++++++++++++++++++++++ 10 files changed, 106 insertions(+), 14 deletions(-)
diff --git a/criu/config.c b/criu/config.c index e09445a..5d195cb 100644 --- a/criu/config.c +++ b/criu/config.c @@ -555,6 +555,7 @@ int parse_options(int argc, char **argv, bool *usage_error, {"reserve-ports", required_argument, 0, 'P' }, BOOL_OPT("use-nft", &opts.use_nft), BOOL_OPT("parallel", &opts.parallel), + BOOL_OPT("async-clear-nft", &opts.async_clear_nft), { }, };
diff --git a/criu/cr-dump.c b/criu/cr-dump.c index 6caee06..0c212a8 100644 --- a/criu/cr-dump.c +++ b/criu/cr-dump.c @@ -1801,7 +1801,10 @@ static int cr_dump_finish(int ret) network_unlock(opts.tree_id); delete_link_remaps(); clean_cr_time_mounts(); - + if (opts.async_clear_nft) + parallel_nft_clean((long)opts.tree_id); + else + network_delete_set(opts.tree_id); }
if (!ret && opts.lazy_pages) @@ -1831,7 +1834,7 @@ static int cr_dump_finish(int ret) clear_pin_mem(0); }
- if (ret != 0 && opts.with_notifier_kup) { + if (ret) { pr_info("repair off netlink fd\n"); netlink_repair_off(); } diff --git a/criu/cr-restore.c b/criu/cr-restore.c index 6114dd4..3049e07 100644 --- a/criu/cr-restore.c +++ b/criu/cr-restore.c @@ -2566,6 +2566,12 @@ skip_ns_bouncing: if (ret != 0) pr_err("Post-resume script ret code %d\n", ret);
+ pr_info("start delete_set\n"); + if (opts.async_clear_nft) + parallel_nft_clean((long)vpid(init)); + else + network_delete_set(vpid(init)); + if (!opts.restore_detach && !opts.exec_cmd) wait(NULL);
@@ -2712,6 +2718,11 @@ err: } if ((network_status & NETWORK_UNLOCK) == 0) network_unlock(vpid(root_item)); + + if (opts.use_nft && opts.async_clear_nft) + parallel_nft_clean(vpid(root_item)); + else if (opts.use_nft) + network_delete_set(vpid(root_item)); }
return ret; diff --git a/criu/crtools.c b/criu/crtools.c index 18945e7..35a479f 100644 --- a/criu/crtools.c +++ b/criu/crtools.c @@ -490,6 +490,7 @@ usage: " --reserve-ports Reserve src ports in kernel\n" " --use-nft Use nft API instead of iptables cmd in network locking\n" " --parallel Parallel to accellrate dumping speed\n" +" --async-clear-nft Async to clear nft table set" "\n" "Check options:\n" " Without options, "criu check" checks availability of absolutely required\n" diff --git a/criu/include/cr_options.h b/criu/include/cr_options.h index cc8d1ae..aa519c8 100644 --- a/criu/include/cr_options.h +++ b/criu/include/cr_options.h @@ -189,6 +189,7 @@ struct cr_options { int reserve_ports; int use_nft; int parallel; + int async_clear_nft; };
extern struct cr_options opts; diff --git a/criu/include/net.h b/criu/include/net.h index 4e704cc..9daea8d 100644 --- a/criu/include/net.h +++ b/criu/include/net.h @@ -31,7 +31,8 @@ struct veth_pair { extern int collect_net_namespaces(bool for_dump);
extern int network_prepare(pid_t tree_id); -extern void network_unprepare(pid_t tree_id); +extern void network_delete_rule(pid_t tree_id); +extern void network_delete_set(pid_t tree_id); extern int network_lock(pid_t tree_id); extern void network_unlock(pid_t tree_id); extern int network_lock_internal(void); diff --git a/criu/include/taskqueue.h b/criu/include/taskqueue.h index 16f9e3d..906c784 100644 --- a/criu/include/taskqueue.h +++ b/criu/include/taskqueue.h @@ -6,7 +6,6 @@ #include <semaphore.h>
#include "vma.h" -#include "pstree.h"
#include "common/list.h"
@@ -47,4 +46,15 @@ struct mappings_info { int start_collect_mappings_thread(void); int end_collect_mappings_thread(struct pstree_item *item);
+#define STACK_SIZE (1024 *1024) +typedef void (*daemon_t)(void *); +int parallel_task(daemon_t fn, void *_arg); + +struct daemon { + daemon_t fn; + void *arg; +}; + +void parallel_nft_clean(long tree_id); + #endif /* __CR_TASKQUEUE_H__ */ diff --git a/criu/net.c b/criu/net.c index 30b1491..2bd6f77 100644 --- a/criu/net.c +++ b/criu/net.c @@ -2943,7 +2943,7 @@ void network_unlock(pid_t tree_id) }
if (opts.use_nft && opts.tcp_established_ok) - network_unprepare(tree_id); + network_delete_rule(tree_id);
cpt_unlock_tcp_connections(opts.use_nft);
diff --git a/criu/nftables.c b/criu/nftables.c index 817f157..500d485 100644 --- a/criu/nftables.c +++ b/criu/nftables.c @@ -16,6 +16,7 @@
#include "sk-inet.h" #include "nftables.h" +#include "taskqueue.h"
#include "../soccr/soccr.h"
@@ -653,25 +654,43 @@ int network_prepare(pid_t tree_id) return mnl_common(network_prepare_internal, NULL, &tree_id); }
-static int network_unprepare_internal(struct mnl_params *params, +static int network_delete_rule_internal(struct mnl_params *params, batch_func_t _, void *args) { pid_t tree_id = *(pid_t *)args;
- if (nft_rule_common(params, tree_id, false) < 0) - return -1; + return nft_rule_common(params, tree_id, false); +}
- if (nft_set_common(params, tree_id, false) < 0) - return -1; +/* here split the deletion of rule and set to accelete the restoration process */ +void network_delete_rule(pid_t tree_id) +{ + mnl_common(network_delete_rule_internal, NULL, &tree_id); +}
- return 0; +static int network_delete_set_internal(struct mnl_params *params, + batch_func_t _, void *args) +{ + pid_t tree_id = *(pid_t *)args; + + return nft_set_common(params, tree_id, false); }
-void network_unprepare(pid_t tree_id) +void network_delete_set(pid_t tree_id) { - pr_info("Unprepare network\n"); + pr_info("clear nft set\n"); + + mnl_common(network_delete_set_internal, NULL, &tree_id); +}
- mnl_common(network_unprepare_internal, NULL, &tree_id); +void parallel_nft_clean_internal(void *arg) +{ + network_delete_set((long)arg); +} + +void parallel_nft_clean(long tree_id) +{ + parallel_task(parallel_nft_clean_internal, (void *)tree_id); }
static int add_set_elem_internal(struct nftnl_set *s, void *data, size_t len) diff --git a/criu/taskqueue.c b/criu/taskqueue.c index 1196a5e..7d500e9 100644 --- a/criu/taskqueue.c +++ b/criu/taskqueue.c @@ -122,3 +122,48 @@ int end_collect_mappings_thread(struct pstree_item *item) */ return retval; } + +static int daemonize(void *arg) +{ + struct daemon *d = arg; + + if (daemon(0, 0) < 0) + pr_perror("daemonize failed"); + + d->fn(d->arg); + + return 0; +} + +int parallel_task(daemon_t fn, void *_arg) +{ + struct daemon arg = { + .fn = fn, + .arg = _arg, + }; + char *stack; + char *stack_top; + pid_t pid; + + stack = mmap(NULL, STACK_SIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) { + pr_perror("mmap failed"); + return -1; + } + + stack_top = stack + STACK_SIZE; + + /* ignore SIGCHLD signal */ + pid = clone(daemonize, stack_top, 0, &arg); + if (pid > 0) + return 0; /* parent */ + else if (pid < 0) { + pr_perror("clone failed"); + return -1; + } + + /* unreachable */ + __builtin_unreachable(); + return 0; +}