From 80f1111b17388477cd298d01a908a5e898c2575f Mon Sep 17 00:00:00 2001
From: Liu Chao <liuchao173@huawei.com>
Date: Mon, 19 Jul 2021 03:19:30 +0000
Subject: [PATCH 31/54] save src ports to ip_local_reserved_ports when dump
 tasks and retore it when restore tasks

---
 criu/config.c             |  8 +++-
 criu/cr-dump.c            |  3 ++
 criu/crtools.c            |  1 +
 criu/include/cr_options.h |  1 +
 criu/include/sk-inet.h    |  4 ++
 criu/include/util.h       |  2 +
 criu/net.c                |  6 ++-
 criu/sk-tcp.c             | 85 +++++++++++++++++++++++++++++++++++++++
 8 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/criu/config.c b/criu/config.c
index 6beed04..9268cd1 100644
--- a/criu/config.c
+++ b/criu/config.c
@@ -461,7 +461,7 @@ int parse_options(int argc, char **argv, bool *usage_error,
 		{OPT_NAME, no_argument, SAVE_TO, true},\
 		{"no-" OPT_NAME, no_argument, SAVE_TO, false}
 
-	static const char short_opts[] = "dSsRt:hD:o:v::x::Vr:jJ:lW:L:M:";
+	static const char short_opts[] = "dSsRt:hD:o:v::x::Vr:jJ:lW:L:M:P:";
 	static struct option long_opts[] = {
 		{ "tree",			required_argument,	0, 't'	},
 		{ "leave-stopped",		no_argument,		0, 's'	},
@@ -553,6 +553,7 @@ int parse_options(int argc, char **argv, bool *usage_error,
 		BOOL_OPT("weak-file-check", &opts.weak_file_check),
 		BOOL_OPT("ignore-special-dump", &opts.ignore_special_dump),
 		BOOL_OPT("file-locks-repair", &opts.file_locks_repair),
+		{"reserve-ports", 		required_argument,	0, 'P'	},
 		{ },
 	};
 
@@ -896,6 +897,11 @@ int parse_options(int argc, char **argv, bool *usage_error,
 		case 'h':
 			*usage_error = false;
 			return 2;
+		case 'P':
+			opts.reserve_ports = atoi(optarg);
+			if (opts.reserve_ports < 0)
+				goto bad_arg;
+			break;
 		default:
 			return 2;
 		}
diff --git a/criu/cr-dump.c b/criu/cr-dump.c
index 9e899d0..695c98a 100644
--- a/criu/cr-dump.c
+++ b/criu/cr-dump.c
@@ -1955,6 +1955,9 @@ int cr_dump_tasks(pid_t pid)
 			goto err;
 	}
 
+	if (opts.reserve_ports > 0)
+		set_reserved_ports();
+
 	if (parent_ie) {
 		inventory_entry__free_unpacked(parent_ie, NULL);
 		parent_ie = NULL;
diff --git a/criu/crtools.c b/criu/crtools.c
index d16961a..888714c 100644
--- a/criu/crtools.c
+++ b/criu/crtools.c
@@ -458,6 +458,7 @@ usage:
 "  --weak-file-check	Allow file size and mod larger than dumping value\n"
 "  --ignore-special-dump Ignore special task tid page dump\n"
 "  --file-locks-repair    Use repair mode to dump and restore file locks\n"
+"  --reserve-ports       Reserve src ports in kernel\n"
 "\n"
 "Check options:\n"
 "  Without options, \"criu check\" checks availability of absolutely required\n"
diff --git a/criu/include/cr_options.h b/criu/include/cr_options.h
index 361aebc..85d852f 100644
--- a/criu/include/cr_options.h
+++ b/criu/include/cr_options.h
@@ -187,6 +187,7 @@ struct cr_options {
 	int			file_locks_repair;
 	char			*share_dst_ports;
 	char			*share_src_ports;
+	int			reserve_ports;
 };
 
 extern struct cr_options opts;
diff --git a/criu/include/sk-inet.h b/criu/include/sk-inet.h
index 2e28444..4181fbe 100644
--- a/criu/include/sk-inet.h
+++ b/criu/include/sk-inet.h
@@ -83,6 +83,10 @@ extern void tcp_locked_conn_add(struct inet_sk_info *);
 extern void rst_unlock_tcp_connections(void);
 extern void cpt_unlock_tcp_connections(void);
 
+extern void read_reserved_ports(char *path);
+extern void write_reserved_ports(char *path);
+extern void set_reserved_ports(void);
+
 extern int dump_one_tcp(int sk, struct inet_sk_desc *sd, SkOptsEntry *soe);
 extern int restore_one_tcp(int sk, struct inet_sk_info *si);
 
diff --git a/criu/include/util.h b/criu/include/util.h
index 3a4b8f9..d1510fc 100644
--- a/criu/include/util.h
+++ b/criu/include/util.h
@@ -430,4 +430,6 @@ bool find_devname(const char *name);
 #define UNMASK_EXIT_NOTIFY_DIR "/sys/kernel/unmask_exit_notify"
 int mask_task_exit_notify(int pid, bool mask);
 
+#define RESERVED_PORTS_PATH "/proc/sys/net/ipv4/ip_local_reserved_ports"
+
 #endif /* __CR_UTIL_H__ */
diff --git a/criu/net.c b/criu/net.c
index 4f1f7d4..19329cf 100644
--- a/criu/net.c
+++ b/criu/net.c
@@ -2897,7 +2897,6 @@ static int network_unlock_internal(void)
 	if (switch_ns(root_item->pid->real, &net_ns_desc, &nsret))
 		return -1;
 
-
 	ret |= iptables_restore(false, conf, sizeof(conf) - 1);
 	if (kdat.ipv6)
 		ret |= iptables_restore(true, conf, sizeof(conf) - 1);
@@ -2926,6 +2925,11 @@ void network_unlock(void)
 {
 	pr_info("Unlock network\n");
 
+	if (opts.reserve_ports) {
+		read_reserved_ports("ip_local_reserved_ports");
+		write_reserved_ports(RESERVED_PORTS_PATH);
+	}
+
 	cpt_unlock_tcp_connections();
 	rst_unlock_tcp_connections();
 
diff --git a/criu/sk-tcp.c b/criu/sk-tcp.c
index 0409e22..67846c3 100644
--- a/criu/sk-tcp.c
+++ b/criu/sk-tcp.c
@@ -23,6 +23,7 @@
 #include "kerndat.h"
 #include "restorer.h"
 #include "rst-malloc.h"
+#include "xmalloc.h"
 
 #include "protobuf.h"
 #include "images/tcp-stream.pb-c.h"
@@ -33,6 +34,9 @@
 static LIST_HEAD(cpt_tcp_repair_sockets);
 static LIST_HEAD(rst_tcp_repair_sockets);
 
+static char* reserved_ports;
+static int reserved_ports_num;
+
 static int tcp_repair_established(int fd, struct inet_sk_desc *sk)
 {
 	int ret;
@@ -475,3 +479,84 @@ void rst_unlock_tcp_connections(void)
 	list_for_each_entry(ii, &rst_tcp_repair_sockets, rlist)
 		nf_unlock_connection_info(ii);
 }
+
+void read_reserved_ports(char *path)
+{
+	FILE *file = NULL;
+	char *ch = NULL;
+	size_t size = 0;
+
+	if (reserved_ports) {
+		free(reserved_ports);
+		reserved_ports = NULL;
+	}
+
+	file = fopen(path, "r");
+	if (!file) {
+		pr_err("Cannot fopen %s\n", path);
+		return;
+	}
+
+	if (getline(&reserved_ports, &size, file) <= 0)
+		pr_err("Cannot getline from %s\n", path);
+	fclose(file);
+
+	if (!reserved_ports)
+		return;
+
+	ch = strstr(reserved_ports, "\n");
+	if (ch)
+		*ch = '\0';
+}
+
+void write_reserved_ports(char *path)
+{
+	int fd = -1;
+	char buf[PATH_MAX];
+
+	fd = open(path, O_RDWR | O_CREAT, 0640);
+	if (fd < 0) {
+		pr_err("Cannot open %s ret %d cwd: %s\n", path, fd, buf);
+		return;
+	}
+
+	cr_system(-1, fd, -1, "/usr/bin/echo",
+			(char *[]) { "echo", reserved_ports, NULL}, 0);
+	close(fd);
+}
+
+static int add_reserved_ports(struct inet_sk_desc *sk)
+{
+	if (reserved_ports_num >= opts.reserve_ports)
+		return -1;
+
+	if (strlen(reserved_ports) == 0)
+		snprintf(reserved_ports, 6, "%u", sk->src_port);
+	else
+		snprintf(reserved_ports + strlen(reserved_ports), 7, ",%u", sk->src_port);
+	reserved_ports_num++;
+
+	return 0;
+}
+
+void set_reserved_ports(void)
+{
+	struct inet_sk_desc *sk = NULL;
+	size_t size = 0;
+
+	read_reserved_ports(RESERVED_PORTS_PATH);
+
+	write_reserved_ports("ip_local_reserved_ports");
+
+	size = strlen(reserved_ports) + 6 * opts.reserve_ports + 1;
+	if (xrealloc_safe(&reserved_ports, size))
+		exit(1);
+
+	list_for_each_entry(sk, &cpt_tcp_repair_sockets, rlist)
+		add_reserved_ports(sk);
+
+	write_reserved_ports(RESERVED_PORTS_PATH);
+
+	free(reserved_ports);
+	reserved_ports = NULL;
+}
-- 
2.33.1

