From 504f69180bbcc8d8dd36ec98c1a3ec9b33d0b4b7 Mon Sep 17 00:00:00 2001
From: "fu.lin" <fulin10@huawei.com>
Date: Mon, 8 Nov 2021 15:08:12 +0800
Subject: [PATCH 50/54] infiniband: fix the infiniband fd conflict

Phenomenon:
  Operating uverbs device will generate anonymous fd named
`anon_inode:[infinibandevent]`. When `anon_inode:[infinibandevent]` fd
is the last opened fd, and some kind of unix socket fd exist, which is
generated by syscalls like `socketpair()` at the same tim,
`anon_inode:[infinibandevent]` will restore fail probabilistically.

log as the following:

```
(00.254523)  63959: open file flags:1
(00.254526)  63959: unix: Opening standalone (stage 0 id 0x1ff ino 1019605 peer 0)
(00.254571)  63959: *******flags: 0
(00.254575)  63959: 		Create fd for 1408 # the fake fd
(00.254578)  63959: *******flags: 1
(00.254580)  63959: 		Create fd for 445 # the restoration fd
```

Reason:
  During the restoration of unix socket, `socketpair()` will generate
two fds, one is used to the current restoration, another is called fake
fd which fd nr is owned by `find_unused_fd()`. When
`anon_inode:[infinibandevent]` fd is the last one, criu don't dump the
fd information for `anon_inode:[infinibandevent]` in original
implementation, and criu think the fd nr which should belong to
`anon_inode:[infinibandevent]` isn't used. Therefore, it cause the
`anon_inode:[infinibandevent]` restoration fail.

This patch fix the above problem. Core: dump
`anon_inode:[infinibandevent]` fd information, make the criu is aware
that that fd nr is used.

Signed-off-by: fu.lin <fulin10@huawei.com>
---
 criu/Makefile.crtools        |  1 +
 criu/char.c                  | 68 ++++++++++++++++++++++++++++++++++++
 criu/files.c                 | 23 ++++++------
 criu/include/char.h          | 17 +++++++++
 criu/include/image-desc.h    |  1 +
 criu/include/protobuf-desc.h |  1 +
 images/chr.proto             |  3 ++
 images/fdinfo.proto          |  2 ++
 8 files changed, 103 insertions(+), 13 deletions(-)
 create mode 100644 criu/char.c
 create mode 100644 criu/include/char.h

diff --git a/criu/Makefile.crtools b/criu/Makefile.crtools
index 0bea576..70d1b73 100644
--- a/criu/Makefile.crtools
+++ b/criu/Makefile.crtools
@@ -93,6 +93,7 @@ obj-y			+= devname.o
 obj-y			+= mnl.o
 obj-y			+= nftables.o
 obj-y			+= kmsg.o
+obj-y			+= char.o
 obj-$(CONFIG_HAS_LIBBPF)	+= bpfmap.o
 obj-$(CONFIG_COMPAT)	+= pie-util-vdso-elf32.o
 CFLAGS_pie-util-vdso-elf32.o	+= -DCONFIG_VDSO_32
diff --git a/criu/char.c b/criu/char.c
new file mode 100644
index 0000000..153145f
--- /dev/null
+++ b/criu/char.c
@@ -0,0 +1,68 @@
+#include "imgset.h"
+#include "char.h"
+#include "log.h"
+
+#include "protobuf.h"
+
+static void pr_info_infiniband(char *action, InfinibandEntry *infiniband)
+{
+	pr_info("%sinfiniband: id %#08x\n", action, infiniband->id);
+}
+
+/* Checks if file descriptor @lfd is infinibandevent */
+int is_infiniband_link(char *link)
+{
+	return is_anon_link_type(link, "[infinibandevent]");
+}
+
+static int dump_one_infiniband(int lfd, u32 id, const struct fd_parms *p)
+{
+	FileEntry fe = FILE_ENTRY__INIT;
+	InfinibandEntry infiniband = INFINIBAND_ENTRY__INIT;
+
+	infiniband.id = id;
+
+	fe.type  = FD_TYPES__INFINIBAND;
+	fe.id = infiniband.id;
+	fe.infiniband = &infiniband;
+
+	pr_info_infiniband("Dumping ", &infiniband);
+
+	return pb_write_one(img_from_set(glob_imgset, CR_FD_FILES), &fe, PB_FILE);
+}
+
+const struct fdtype_ops infiniband_dump_ops = {
+	.type	= FD_TYPES__INFINIBAND,
+	.dump	= dump_one_infiniband,
+};
+
+static int infiniband_open(struct file_desc *d, int *new_fd) {
+	/*
+	 * `*new_fd == -1` at this time, it means this open operation shouldn't
+	 * be served out, which is why this function does nothing here.
+	 */
+	return 0;
+};
+
+static struct file_desc_ops infiniband_desc_ops = {
+	.type	= FD_TYPES__INFINIBAND,
+	.open	= infiniband_open,
+};
+
+static int collect_one_infiniband(void *o, ProtobufCMessage *base, struct cr_img *i)
+{
+	struct infiniband_file_info *info = o;
+
+	info->infiniband = pb_msg(base, InfinibandEntry);
+	pr_info_infiniband("Collected ", info->infiniband);
+
+	/* add the fd to `file_desc_hash` list to prevent from NULL pointer */
+	return file_desc_add(&info->d, info->infiniband->id, &infiniband_desc_ops);
+}
+
+struct collect_image_info infiniband_cinfo = {
+	.fd_type = CR_FD_INFINIBAND,
+	.pb_type = PB_INFINIBAND,
+	.priv_size = sizeof(struct infiniband_file_info),
+	.collect = collect_one_infiniband,
+};
diff --git a/criu/files.c b/criu/files.c
index cb50aca..34c4e18 100644
--- a/criu/files.c
+++ b/criu/files.c
@@ -49,6 +49,7 @@
 #include "kerndat.h"
 #include "fdstore.h"
 #include "bpfmap.h"
+#include "char.h"
 
 #include "protobuf.h"
 #include "util.h"
@@ -593,12 +594,6 @@ static int dump_chrdev(struct fd_parms *p, int lfd, FdinfoEntry *e)
 	return err;
 }
 
-/* Checks if file descriptor @lfd is infinibandevent */
-int is_infiniband_link(char *link)
-{
-	return is_anon_link_type(link, "[infinibandevent]");
-}
-
 static int dump_one_file(struct pid *pid, int fd, int lfd, struct fd_opts *opts,
 		struct parasite_ctl *ctl, FdinfoEntry *e,
 		struct parasite_drain_fd *dfds)
@@ -654,7 +649,7 @@ static int dump_one_file(struct pid *pid, int fd, int lfd, struct fd_opts *opts,
 		else if (is_timerfd_link(link))
 			ops = &timerfd_dump_ops;
 		else if (is_infiniband_link(link))
-			return 1;
+			ops = &infiniband_dump_ops;
 #ifdef CONFIG_HAS_LIBBPF
 		else if (is_bpfmap_link(link))
 			ops = &bpfmap_dump_ops;
@@ -765,11 +760,7 @@ int dump_task_files_seized(struct parasite_ctl *ctl, struct pstree_item *item,
 						lfds[i], opts + i, ctl, &e, dfds);
 			if (ret < 0)
 				break;
-			/* infiniband link file */
-			if (ret > 0) {
-				ret = 0;
-				continue;
-			}
+
 			e.flags |= need_reuse_flag;
 			pr_info("write fdinfoEntry fd=%d id=%d\n", (&e)->fd, (&e)->id);
 			ret = pb_write_one(img, &e, PB_FDINFO);
@@ -1864,8 +1855,11 @@ static int chrfile_open(struct file_desc *d, int *new_fd)
 	pr_info("charfile: Opening %s (repair %d index %d)\n",
 			ci->path, ci->cfe->repair, ci->cfe->index);
 
+	if (ci->cfe->repair)
+		ci->cfe->flags |= O_REPAIR;
+
 	mntns_root = open_pid_proc(getpid());
-	fd = openat(mntns_root, ci->path, ci->cfe->flags | O_REPAIR);
+	fd = openat(mntns_root, ci->path, ci->cfe->flags);
 	if (fd < 0){
 		pr_err("open chr file failed\n");
 		return -1;
@@ -1991,6 +1985,9 @@ static int collect_one_file(void *o, ProtobufCMessage *base, struct cr_img *i)
 	case FD_TYPES__CHR:
 		ret = collect_one_file_entry(fe, fe->chr->id, &fe->chr->base, &chrfile_cinfo);
 		break;
+	case FD_TYPES__INFINIBAND:
+		ret = collect_one_file_entry(fe, fe->infiniband->id, &fe->infiniband->base, &infiniband_cinfo);
+		break;
 	}
 
 	return ret;
diff --git a/criu/include/char.h b/criu/include/char.h
new file mode 100644
index 0000000..c63b8f1
--- /dev/null
+++ b/criu/include/char.h
@@ -0,0 +1,17 @@
+#ifndef __CR_CHAR_H__
+#define __CR_CHAR_H__
+
+#include "files.h"
+#include "images/chr.pb-c.h"
+
+struct infiniband_file_info {
+	InfinibandEntry	*infiniband;
+	struct file_desc d;
+};
+
+extern const struct fdtype_ops infiniband_dump_ops;
+extern struct collect_image_info infiniband_cinfo;
+
+int is_infiniband_link(char *link);
+
+#endif /* __CR_CHAR_H__ */
diff --git a/criu/include/image-desc.h b/criu/include/image-desc.h
index 22676ae..4231716 100644
--- a/criu/include/image-desc.h
+++ b/criu/include/image-desc.h
@@ -115,6 +115,7 @@ enum {
 
 	CR_FD_AUTOFS,
 	CR_FD_CHRFILE,
+	CR_FD_INFINIBAND,
 
 	CR_FD_MAX
 };
diff --git a/criu/include/protobuf-desc.h b/criu/include/protobuf-desc.h
index e7df57e..023bbfc 100644
--- a/criu/include/protobuf-desc.h
+++ b/criu/include/protobuf-desc.h
@@ -70,6 +70,7 @@ enum {
 	PB_BPFMAP_FILE,
 	PB_BPFMAP_DATA,
 	PB_CHRFILE,
+	PB_INFINIBAND,
 
 	/* PB_AUTOGEN_STOP */
 
diff --git a/images/chr.proto b/images/chr.proto
index 67929db..ed65005 100644
--- a/images/chr.proto
+++ b/images/chr.proto
@@ -10,3 +10,6 @@ message chrfile_entry {
 	required bool		repair	= 5;
 };
 
+message infiniband_entry {
+	required uint32		id	= 1;
+};
diff --git a/images/fdinfo.proto b/images/fdinfo.proto
index 8561da4..2fa34f8 100644
--- a/images/fdinfo.proto
+++ b/images/fdinfo.proto
@@ -42,6 +42,7 @@ enum fd_types {
 	MEMFD		= 18;
 	BPFMAP		= 19;
 	CHR		= 21;
+	INFINIBAND	= 22;
 
 	/* Any number above the real used. Not stored to image */
 	CTL_TTY		= 65534;
@@ -79,4 +80,5 @@ message file_entry {
 	optional memfd_file_entry	memfd	= 20;
 	optional bpfmap_file_entry	bpf	= 21;
 	optional chrfile_entry          chr     = 23;
+	optional infiniband_entry	infiniband	= 25;
 }
-- 
2.33.1

