Phenomenon: Operating uverbs device will generate anonymous fd named `anon_inode:[infinibandevent]`. When `anon_inode:[infinibandevent]` fd is the last opened fd, and some kind of unix socket fd exist, which is generated by syscalls like `socketpair()` at the same tim, `anon_inode:[infinibandevent]` will restore fail probabilistically.
log as the following:
``` (00.254523) 63959: open file flags:1 (00.254526) 63959: unix: Opening standalone (stage 0 id 0x1ff ino 1019605 peer 0) (00.254571) 63959: *******flags: 0 (00.254575) 63959: Create fd for 1408 # the fake fd (00.254578) 63959: *******flags: 1 (00.254580) 63959: Create fd for 445 # the restoration fd ```
Reason: During the restoration of unix socket, `socketpair()` will generate two fds, one is used to the current restoration, another is called fake fd which fd nr is owned by `find_unused_fd()`. When `anon_inode:[infinibandevent]` fd is the last one, criu don't dump the fd information for `anon_inode:[infinibandevent]` in original implementation, and criu think the fd nr which should belong to `anon_inode:[infinibandevent]` isn't used. Therefore, it cause the `anon_inode:[infinibandevent]` restoration fail.
This patch fix the above problem. Core: dump `anon_inode:[infinibandevent]` fd information, make the criu is aware that that fd nr is used.
Conflict:NA Reference:https://gitee.com/src-openeuler/criu/pulls/21 Signed-off-by: fu.lin fulin10@huawei.com --- criu/Makefile.crtools | 1 + criu/char.c | 68 ++++++++++++++++++++++++++++++++++++ criu/files.c | 23 ++++++------ criu/include/char.h | 17 +++++++++ criu/include/image-desc.h | 1 + criu/include/protobuf-desc.h | 1 + images/chr.proto | 3 ++ images/fdinfo.proto | 2 ++ 8 files changed, 103 insertions(+), 13 deletions(-) create mode 100644 criu/char.c create mode 100644 criu/include/char.h
diff --git a/criu/Makefile.crtools b/criu/Makefile.crtools index 0bea576..70d1b73 100644 --- a/criu/Makefile.crtools +++ b/criu/Makefile.crtools @@ -93,6 +93,7 @@ obj-y += devname.o obj-y += mnl.o obj-y += nftables.o obj-y += kmsg.o +obj-y += char.o obj-$(CONFIG_HAS_LIBBPF) += bpfmap.o obj-$(CONFIG_COMPAT) += pie-util-vdso-elf32.o CFLAGS_pie-util-vdso-elf32.o += -DCONFIG_VDSO_32 diff --git a/criu/char.c b/criu/char.c new file mode 100644 index 0000000..153145f --- /dev/null +++ b/criu/char.c @@ -0,0 +1,68 @@ +#include "imgset.h" +#include "char.h" +#include "log.h" + +#include "protobuf.h" + +static void pr_info_infiniband(char *action, InfinibandEntry *infiniband) +{ + pr_info("%sinfiniband: id %#08x\n", action, infiniband->id); +} + +/* Checks if file descriptor @lfd is infinibandevent */ +int is_infiniband_link(char *link) +{ + return is_anon_link_type(link, "[infinibandevent]"); +} + +static int dump_one_infiniband(int lfd, u32 id, const struct fd_parms *p) +{ + FileEntry fe = FILE_ENTRY__INIT; + InfinibandEntry infiniband = INFINIBAND_ENTRY__INIT; + + infiniband.id = id; + + fe.type = FD_TYPES__INFINIBAND; + fe.id = infiniband.id; + fe.infiniband = &infiniband; + + pr_info_infiniband("Dumping ", &infiniband); + + return pb_write_one(img_from_set(glob_imgset, CR_FD_FILES), &fe, PB_FILE); +} + +const struct fdtype_ops infiniband_dump_ops = { + .type = FD_TYPES__INFINIBAND, + .dump = dump_one_infiniband, +}; + +static int infiniband_open(struct file_desc *d, int *new_fd) { + /* + * `*new_fd == -1` at this time, it means this open operation shouldn't + * be served out, which is why this function does nothing here. + */ + return 0; +}; + +static struct file_desc_ops infiniband_desc_ops = { + .type = FD_TYPES__INFINIBAND, + .open = infiniband_open, +}; + +static int collect_one_infiniband(void *o, ProtobufCMessage *base, struct cr_img *i) +{ + struct infiniband_file_info *info = o; + + info->infiniband = pb_msg(base, InfinibandEntry); + pr_info_infiniband("Collected ", info->infiniband); + + /* add the fd to `file_desc_hash` list to prevent from NULL pointer */ + return file_desc_add(&info->d, info->infiniband->id, &infiniband_desc_ops); +} + +struct collect_image_info infiniband_cinfo = { + .fd_type = CR_FD_INFINIBAND, + .pb_type = PB_INFINIBAND, + .priv_size = sizeof(struct infiniband_file_info), + .collect = collect_one_infiniband, +}; diff --git a/criu/files.c b/criu/files.c index 3bb68c7..0e5be91 100644 --- a/criu/files.c +++ b/criu/files.c @@ -49,6 +49,7 @@ #include "kerndat.h" #include "fdstore.h" #include "bpfmap.h" +#include "char.h"
#include "protobuf.h" #include "util.h" @@ -593,12 +594,6 @@ static int dump_chrdev(struct fd_parms *p, int lfd, FdinfoEntry *e) return err; }
-/* Checks if file descriptor @lfd is infinibandevent */ -int is_infiniband_link(char *link) -{ - return is_anon_link_type(link, "[infinibandevent]"); -} - static int dump_one_file(struct pid *pid, int fd, int lfd, struct fd_opts *opts, struct parasite_ctl *ctl, FdinfoEntry *e, struct parasite_drain_fd *dfds) @@ -654,7 +649,7 @@ static int dump_one_file(struct pid *pid, int fd, int lfd, struct fd_opts *opts, else if (is_timerfd_link(link)) ops = &timerfd_dump_ops; else if (is_infiniband_link(link)) - return 1; + ops = &infiniband_dump_ops; #ifdef CONFIG_HAS_LIBBPF else if (is_bpfmap_link(link)) ops = &bpfmap_dump_ops; @@ -765,11 +760,7 @@ int dump_task_files_seized(struct parasite_ctl *ctl, struct pstree_item *item, lfds[i], opts + i, ctl, &e, dfds); if (ret < 0) break; - /* infiniband link file */ - if (ret > 0) { - ret = 0; - continue; - } + e.flags |= need_reuse_flag; pr_info("write fdinfoEntry fd=%d id=%d\n", (&e)->fd, (&e)->id); ret = pb_write_one(img, &e, PB_FDINFO); @@ -1864,8 +1855,11 @@ static int chrfile_open(struct file_desc *d, int *new_fd) pr_info("charfile: Opening %s (repair %d index %d)\n", ci->path, ci->cfe->repair, ci->cfe->index);
+ if (ci->cfe->repair) + ci->cfe->flags |= O_REPAIR; + mntns_root = open_pid_proc(getpid()); - fd = openat(mntns_root, ci->path, ci->cfe->flags | O_REPAIR); + fd = openat(mntns_root, ci->path, ci->cfe->flags); if (fd < 0){ pr_err("open chr file failed\n"); return -1; @@ -1991,6 +1985,9 @@ static int collect_one_file(void *o, ProtobufCMessage *base, struct cr_img *i) case FD_TYPES__CHR: ret = collect_one_file_entry(fe, fe->chr->id, &fe->chr->base, &chrfile_cinfo); break; + case FD_TYPES__INFINIBAND: + ret = collect_one_file_entry(fe, fe->infiniband->id, &fe->infiniband->base, &infiniband_cinfo); + break; }
return ret; diff --git a/criu/include/char.h b/criu/include/char.h new file mode 100644 index 0000000..c63b8f1 --- /dev/null +++ b/criu/include/char.h @@ -0,0 +1,17 @@ +#ifndef __CR_CHAR_H__ +#define __CR_CHAR_H__ + +#include "files.h" +#include "images/chr.pb-c.h" + +struct infiniband_file_info { + InfinibandEntry *infiniband; + struct file_desc d; +}; + +extern const struct fdtype_ops infiniband_dump_ops; +extern struct collect_image_info infiniband_cinfo; + +int is_infiniband_link(char *link); + +#endif /* __CR_CHAR_H__ */ diff --git a/criu/include/image-desc.h b/criu/include/image-desc.h index 22676ae..4231716 100644 --- a/criu/include/image-desc.h +++ b/criu/include/image-desc.h @@ -115,6 +115,7 @@ enum {
CR_FD_AUTOFS, CR_FD_CHRFILE, + CR_FD_INFINIBAND,
CR_FD_MAX }; diff --git a/criu/include/protobuf-desc.h b/criu/include/protobuf-desc.h index e7df57e..023bbfc 100644 --- a/criu/include/protobuf-desc.h +++ b/criu/include/protobuf-desc.h @@ -70,6 +70,7 @@ enum { PB_BPFMAP_FILE, PB_BPFMAP_DATA, PB_CHRFILE, + PB_INFINIBAND,
/* PB_AUTOGEN_STOP */
diff --git a/images/chr.proto b/images/chr.proto index 67929db..ed65005 100644 --- a/images/chr.proto +++ b/images/chr.proto @@ -10,3 +10,6 @@ message chrfile_entry { required bool repair = 5; };
+message infiniband_entry { + required uint32 id = 1; +}; diff --git a/images/fdinfo.proto b/images/fdinfo.proto index 8561da4..2fa34f8 100644 --- a/images/fdinfo.proto +++ b/images/fdinfo.proto @@ -42,6 +42,7 @@ enum fd_types { MEMFD = 18; BPFMAP = 19; CHR = 21; + INFINIBAND = 22;
/* Any number above the real used. Not stored to image */ CTL_TTY = 65534; @@ -79,4 +80,5 @@ message file_entry { optional memfd_file_entry memfd = 20; optional bpfmap_file_entry bpf = 21; optional chrfile_entry chr = 23; + optional infiniband_entry infiniband = 25; }