Kernel

kernel@openeuler.org

33 participants
24393 discussions

[PATCH OLK-6.6 v8] Add XMFS Support
by Yifan Qiao 11 Aug '26

11 Aug '26

hulk inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/9732 ---------------------------------------- Memory-semantic interconnects such as CXL 3.0 and Huawei United Bus make remote memory directly addressable. This raises a question for Linux: should cross-node memory become another storage tier that can be exposed through existing POSIX filesystem interfaces? To explore this question, we built XMFS, an in-kernel prototype that exports cross-node shared memory through a standard POSIX filesystem interface. Initial experiments with metadata-intensive workloads and container image sharing suggest the approach is practical. Signed-off-by: Yifan Qiao <qiaoyifan4(a)huawei.com> --- fs/xmfs/xmfs_i.h | 1098 ++++++++++++++++++++ fs/xmfs/dir.c | 160 +++ fs/xmfs/file.c | 2261 +++++++++++++++++++++++++++++++++++++++++ fs/xmfs/gc.c | 953 +++++++++++++++++ fs/xmfs/inode.c | 344 +++++++ fs/xmfs/namei.c | 1794 ++++++++++++++++++++++++++++++++ fs/xmfs/replay.c | 907 +++++++++++++++++ fs/xmfs/super.c | 1677 ++++++++++++++++++++++++++++++ fs/xmfs/symlink.c | 87 ++ fs/xmfs/urma.c | 2482 +++++++++++++++++++++++++++++++++++++++++++++ fs/xmfs/xattr.c | 591 +++++++++++ fs/Kconfig | 1 + fs/Makefile | 1 + fs/xmfs/Kconfig | 21 + fs/xmfs/Makefile | 16 + 15 files changed, 12393 insertions(+) create mode 100644 fs/xmfs/xmfs_i.h create mode 100644 fs/xmfs/dir.c create mode 100644 fs/xmfs/file.c create mode 100644 fs/xmfs/gc.c create mode 100644 fs/xmfs/inode.c create mode 100644 fs/xmfs/namei.c create mode 100644 fs/xmfs/replay.c create mode 100644 fs/xmfs/super.c create mode 100644 fs/xmfs/symlink.c create mode 100644 fs/xmfs/urma.c create mode 100644 fs/xmfs/xattr.c create mode 100644 fs/xmfs/Kconfig create mode 100644 fs/xmfs/Makefile diff --git a/fs/xmfs/xmfs_i.h b/fs/xmfs/xmfs_i.h new file mode 100644 index 000000000000..7f58b2f64f7b --- /dev/null +++ b/fs/xmfs/xmfs_i.h @@ -0,0 +1,1098 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ +#ifndef _XMFS_H +#define _XMFS_H + +#include <linux/atomic.h> +#include <linux/fs.h> +#include <linux/xarray.h> +#include <linux/spinlock.h> +#include <linux/mm.h> +#include <linux/ratelimit.h> +#include <linux/xattr.h> +#include <linux/vmalloc.h> +#include <linux/workqueue.h> + +#include <linux/pgtable.h> +#include <linux/atomic.h> +#include <linux/libnvdimm.h> + +#ifdef CONFIG_XMFS_FS_URMA +#include <ub/urma/ubcore_types.h> +#include <ub/urma/ubcore_uapi.h> +#include <ub/urma/ubcore_api.h> +#endif + +/* #define PER_FS 1 */ + +#define XMFS_MAGIC 0x53464D58 + +#define TOTAL_SIZE (8UL << 30) +#define DATA_ALIGN_KB 4UL + +#ifdef PER_FS +#define MAX_LOG_PER_TRUNK 18724 +#else +#define MAX_LOG_PER_TRUNK 6096 +#endif + +#define ALIGN_TO_MB(size) \ + (((size) + (2 * 1024 * 1024 - 1)) & ~(2 * 1024 * 1024 - 1)) + +#define ALIGN_TO_2MB_LOWER(size) ((size) & ~(2 * 1024 * 1024 - 1)) + +/* sb is 4MB, supports up to ~8TB */ +#define SB_OFFSET 0 +#define SB_SIZE (4UL << 20) + +/* mount msg is 2MB */ +#define MOUNT_MSG_OFFSET (SB_OFFSET + SB_SIZE) +#define MOUNT_MSG_SIZE (1UL << 21) + +/* msg ring is 2MB */ +#define MSG_RING_OFFSET (MOUNT_MSG_OFFSET + MOUNT_MSG_SIZE) +#define MSG_RING_SIZE (1 << 21) + +#define INODE_TABLE_OFFSET (MSG_RING_OFFSET + MSG_RING_SIZE) +#define INODE_TABLE_SIZE (2UL << 21) +#define XMFS_INODE_TABLE_MAGIC 0x494E4F44UL + +#define DATA_AREA_OFFSET (INODE_TABLE_OFFSET + INODE_TABLE_SIZE) + +/* XMFS trunk is 2MB */ +#define XMFS_DEFAULT_CHUNK_SIZE (1UL << 21) + +/* XMFS data align is 64KB */ +#define XMFS_DATA_SHIFT 16 + +/* XMFS log ops */ +#define XMFS_CREATE (1 << 0) +#define XMFS_DELETE (1 << 1) +#define XMFS_UPDATE (1 << 2) +#define XMFS_LINK (1 << 3) +#define XMFS_MKDIR (1 << 4) +#define XMFS_RENAME (1 << 5) +#define XMFS_SETATTR (1 << 6) +#define XMFS_SYMLINK (1 << 7) +#define XMFS_XATTR (1 << 8) +#define XMFS_ROOT (1 << 9) +#define XMFS_GC (1 << 10) + +#define RENAME_HOLD (1 << 4) + +#define XMFS_DATA_TRUNK_MAGIC 0x44415441UL +#define XMFS_GC_INTERVAL_DEFAULT 5 +#define XMFS_GC_BATCH 51200 +#define XMFS_BITMAP_CAPACITY (SB_SIZE - offsetof(struct xmfs_sb_index, bitmap)) + +enum xmfs_rw_msg_type { + MSG_RW_REQUEST = 1, + MSG_RW_GRANT, + MSG_RW_COMPLETE, + MSG_RW_ERROR, +}; + +struct xmfs_data_trunk_header { + unsigned long magic; + unsigned long live_page_count; + unsigned int total_page_count; + unsigned long last_modify_version; +}; + +struct xmfs_gc_ctl { + spinlock_t gc_lock; + wait_queue_head_t wait; + unsigned long gc_cursor; + unsigned long gc_batch; + bool gc_running; + bool need_replay; +}; + +struct xmfs_rw_msg { + enum xmfs_rw_msg_type type; + int magic; + int slave_id; + unsigned long request_id; + unsigned int ops; + unsigned long data_pages; + unsigned long log_entries; + umode_t mode; + unsigned long p_ino; + unsigned long i_ino; + char name[256]; + loff_t size; + unsigned long log_trunk_addr; + int log_slot_start; + int log_slot_count; + unsigned long data_addr; + unsigned long data_page_count; + unsigned long second_data_addr; + unsigned long second_data_pages; + unsigned long assigned_ino; + int status; + unsigned long version; +}; + +struct xmfs_notify_msg { + int slave_id; + unsigned long request_id; + int msg_type; + int _pad[5]; +}; + +extern const struct xattr_handler *xmfs_host_xattr_handlers[]; +extern const struct xattr_handler *xmfs_slave_xattr_handlers[]; + +enum xmfs_msg_type { + MSG_STOP_READ = 1, + MSG_READ, +}; + +enum xmfs_mode { + XMFS_SHARED_MEM_MODE, + XMFS_URMA_MODE, + XMFS_HYBRID_MODE, +}; + +enum xmfs_param { + Opt_host, + Opt_replay, + Opt_addr, + Opt_size, + Opt_freq, + Opt_log, + Opt_debug, + Opt_always_hot, + Opt_fallocate, + Opt_mode, +#ifdef CONFIG_XMFS_FS_URMA + Opt_urma_va, + Opt_urma_eid_subnet_prefix, + Opt_urma_eid_interface_id, + Opt_urma_jetty_id, + Opt_urma_token_id, + Opt_urma_dev, +#endif +}; + +/* 4KB */ +struct xmfs_msg_queue { + unsigned short len; + unsigned short type; + unsigned long addr; + char pad[4072]; +}; + +struct xmfs_fs_context { + bool host; + bool replay; + unsigned long addr; + unsigned long size; + unsigned long fallocate_size; + int freq; + int log_size; + int mode; + bool debug; + bool always_hot; +#ifdef CONFIG_XMFS_FS_URMA + uint64_t urma_va; + uint64_t urma_eid_subnet_prefix; + uint64_t urma_eid_interface_id; + uint32_t urma_jetty_id; + uint32_t urma_token_id; + char dev_name[100]; +#endif +}; + +#ifdef PER_FS +struct xmfs_sb_index { + unsigned long count; + unsigned long log_start; + unsigned long log_current; + unsigned long data_current; + unsigned long data_count; + unsigned long last_update; + char pad[4048]; +}; + +struct xmfs_log { + unsigned long index; + unsigned long end_index; + unsigned long data_offset; + loff_t size; + unsigned int ops; + umode_t mode; + char name[256]; + unsigned long p_ino; + unsigned long i_ino; + unsigned long offset; + unsigned long version; + unsigned long writer_id; +}; + +struct xmfs_log_trunk { + unsigned long used; + unsigned long next_trunk; + unsigned long version_base; + unsigned long version_max; + char pad[1192]; + struct xmfs_log log[MAX_LOG_PER_TRUNK]; +}; + +#else +struct xmfs_sb_index { + unsigned long magic; + /* High-water mark: next slot index to allocate. */ + unsigned long count; + /* Actual number of used trunks (bitmap != 'u'). */ + unsigned long used_trunk_count; + /* Log of start inode. */ + unsigned long log_start; + /* Log of last inode. */ + unsigned long log_current; + /* Current address of data. */ + unsigned long data_current; + /* Number of 4 KiB data blocks in the current trunk. */ + unsigned long data_count; + /* Timestamp. */ + unsigned long last_update; + /* Filesystem ID. */ + __kernel_fsid_t fsid; + int log_size; + unsigned long len; + int bitmap_size; + char bitmap[]; +}; + +struct xmfs_log { + unsigned long index; + unsigned long end_index; + unsigned long data_offset; + loff_t size; + unsigned int ops; + umode_t mode; + char name[256]; + unsigned long p_ino; + unsigned long i_ino; + unsigned long offset; + unsigned long version; + unsigned long writer_id; +}; /* 336 */ + +struct xmfs_xattr { + unsigned long size; + unsigned long name_size; + char *name; + char value[]; +}; + +#define MAX_LOG (4096 / (sizeof(struct xmfs_log))) + +struct xmfs_log_trunk { + unsigned long used; + unsigned long next_trunk; + unsigned long version_base; + unsigned long version_max; + struct xmfs_log log[]; +}; +#endif + +#define XMLT_INDEX_SIZE (offsetof(struct xmfs_log_trunk, log[0])) + +enum xmfs_urma_ops { + XMFS_URMA_READ, + XMFS_URMA_WRITE, + XMFS_URMA_CAS, + XMFS_URMA_SEND, +}; + +struct xmfs_urma_msg { + enum xmfs_urma_ops ops; + uint64_t offset; + uint64_t len; + void *local_addr; + void *write_buffer; + int wr_cnt; + int slot; + uint64_t cas_cmp_data; + uint64_t cas_swap_data; +}; + +struct xmfs_urma_pool { + void *base; + size_t total_size; + size_t slot_size; + u32 nr_slots; + u32 free_slots; + unsigned long *bitmap; + spinlock_t lock; + wait_queue_head_t waitq; +}; + +struct urma_info { + uint64_t urma_remote_va; + uint64_t urma_send_va; + uint64_t urma_recv_va; + uint64_t urma_eid_subnet_prefix; + uint64_t urma_eid_interface_id; + uint32_t urma_jetty_id; + uint32_t urma_token_id; + uint32_t urma_send_token_id; + uint32_t urma_recv_token_id; +}; + +struct xmfs_grant_wait { + struct completion done; + struct xmfs_rw_msg grant; +}; + +#ifdef CONFIG_XMFS_FS_URMA + +struct xmfs_urma_ctl { + struct ubcore_client ubc_client; + int ubc_index; + struct ubcore_device *ubc_dev; + struct ubcore_jetty *jetty[5]; + struct ubcore_jetty *send_jetty; + struct ubcore_jfc *jfc; + struct ubcore_jfc *send_jfc; + struct ubcore_jfc *recv_jfc; + struct ubcore_jfr *jfr; + struct ubcore_jfr *send_jfr; + struct ubcore_target_seg *local_seg; + struct ubcore_target_seg *send_seg; + struct ubcore_target_seg *recv_seg; + struct ubcore_tjetty *tjetty; + struct ubcore_tjetty *send_tjettys[64]; + struct ubcore_tjetty *tjettys[64]; + struct ubcore_target_seg *remote_seg; + struct xmfs_urma_pool pool; + struct xmfs_urma_pool send_pool; + struct xarray grant_waits; + struct urma_info own_info; + struct urma_info send_info; + void *urma_local_va; + void *recv_buf; + void *send_buf; + uint64_t urma_remote_va; + uint64_t urma_eid_subnet_prefix; + uint64_t urma_eid_interface_id; + uint32_t urma_jetty_id; + uint32_t urma_token_id; + uint64_t last_log_start; + uint64_t last_data_start; +}; + +#define XMFS_URMA_DEFAULT_WR_SIZE 104857600 /* 100 MiB */ +#else +struct xmfs_urma_ctl { + struct xmfs_urma_pool pool; + struct xmfs_urma_pool send_pool; + struct xarray grant_waits; + struct urma_info own_info; + struct urma_info send_info; + int ubc_index; + void *urma_local_va; + uint64_t urma_remote_va; + uint64_t urma_eid_subnet_prefix; + uint64_t urma_eid_interface_id; + uint32_t urma_jetty_id; + uint32_t urma_token_id; + uint64_t last_log_start; + uint64_t last_data_start; +}; +#endif + +struct xmfs_inode_entry { + unsigned long i_ino; + unsigned long p_ino; + umode_t mode; + unsigned int nlink; + loff_t size; + struct timespec64 ctime; + struct timespec64 mtime; + struct timespec64 atime; + char name[256]; +}; + +struct xmfs_inode_table { + unsigned long magic; + unsigned long count; + struct xmfs_inode_entry entries[]; +}; + +struct xmfs_sb_info { + int magic; + int id; + struct super_block *sb; + void *kaddr; + phys_addr_t start; + unsigned long len; + struct task_struct *thread; + struct file *dax_filp; + struct dax_device *dax_dev; + struct bdev_handle *handlep; + unsigned long used_ino; + unsigned long max_chunk; + dev_t dev; + int freq; + int log_size; + bool debug; + bool always_hot; + unsigned long trunk_size; + int mode; + struct xarray pinned_inodes; + char dev_name[100]; + + /* Ratelimit XMFS diagnostics. */ + struct ratelimit_state s_error_ratelimit_state; + struct ratelimit_state s_warning_ratelimit_state; + struct ratelimit_state s_msg_ratelimit_state; + atomic_t s_error_count; + atomic_t s_warning_count; + atomic_t s_msg_count; + + /* host uses */ + spinlock_t space_lock; + struct rw_semaphore chain_rwsem; + spinlock_t version_lock; + struct mutex replay_lock; + bool full; + unsigned long log_current; + unsigned long fallocate_size; + bool during_replay; + struct xmfs_gc_ctl gc_ctl; + struct task_struct *host_worker; + int max_slave_id; + struct task_struct *gc_thread; + unsigned long gc_interval; + int mount_slave; + + /* slave uses */ + int stop_cnt; + unsigned long addr; + unsigned long last_update; + unsigned long log_start; + unsigned long last_replayed_slot; + unsigned long skip_no_ops; + spinlock_t local_ring_lock; + unsigned long rw_request_id; + + /* urma related */ + struct xmfs_urma_ctl urma_cfg; + + struct workqueue_struct *prefetch_wq; + struct workqueue_struct *send_wq; +}; + +static inline struct xmfs_sb_info *XMFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +void xmfs_init_log_ratelimits(struct xmfs_sb_info *sbi); + +__printf(5, 6) void __xmfs_error(struct super_block *sb, const char *function, + unsigned int line, int error, const char *fmt, + ...); +__printf(5, 6) void __xmfs_error_inode(struct inode *inode, + const char *function, unsigned int line, + int error, const char *fmt, ...); +__printf(5, 6) void __xmfs_error_file(struct file *file, const char *function, + unsigned int line, int error, + const char *fmt, ...); +__printf(4, 5) void __xmfs_warning(struct super_block *sb, const char *function, + unsigned int line, const char *fmt, ...); +__printf(4, 5) void __xmfs_warning_inode(const struct inode *inode, + const char *function, + unsigned int line, const char *fmt, + ...); +__printf(2, 3) void __xmfs_msg(struct super_block *sb, const char *fmt, ...); + +#ifdef CONFIG_PRINTK +#define xmfs_error(sb, fmt, ...) \ + __xmfs_error((sb), __func__, __LINE__, 0, (fmt), ##__VA_ARGS__) +#define xmfs_error_err(sb, err, fmt, ...) \ + __xmfs_error((sb), __func__, __LINE__, (err), (fmt), ##__VA_ARGS__) +#define xmfs_error_inode(inode, fmt, ...) \ + __xmfs_error_inode((inode), __func__, __LINE__, 0, (fmt), ##__VA_ARGS__) +#define xmfs_error_inode_err(inode, err, fmt, ...) \ + __xmfs_error_inode((inode), __func__, __LINE__, (err), (fmt), \ + ##__VA_ARGS__) +#define xmfs_error_file(file, fmt, ...) \ + __xmfs_error_file((file), __func__, __LINE__, 0, (fmt), ##__VA_ARGS__) +#define xmfs_warning(sb, fmt, ...) \ + __xmfs_warning((sb), __func__, __LINE__, (fmt), ##__VA_ARGS__) +#define xmfs_warning_inode(inode, fmt, ...) \ + __xmfs_warning_inode((inode), __func__, __LINE__, (fmt), ##__VA_ARGS__) +#define xmfs_msg(sb, fmt, ...) \ + __xmfs_msg((sb), (fmt), ##__VA_ARGS__) +#else +#define xmfs_error(sb, fmt, ...) no_printk((fmt), ##__VA_ARGS__) +#define xmfs_error_err(sb, err, fmt, ...) no_printk((fmt), ##__VA_ARGS__) +#define xmfs_error_inode(inode, fmt, ...) no_printk((fmt), ##__VA_ARGS__) +#define xmfs_error_inode_err(inode, err, fmt, ...) \ + no_printk((fmt), ##__VA_ARGS__) +#define xmfs_error_file(file, fmt, ...) no_printk((fmt), ##__VA_ARGS__) +#define xmfs_warning(sb, fmt, ...) no_printk((fmt), ##__VA_ARGS__) +#define xmfs_warning_inode(inode, fmt, ...) no_printk((fmt), ##__VA_ARGS__) +#define xmfs_msg(sb, fmt, ...) no_printk((fmt), ##__VA_ARGS__) +#endif + +#define xmfs_info(sb, fmt, ...) xmfs_msg((sb), KERN_INFO fmt, ##__VA_ARGS__) + +struct mount_msg_info { + atomic_t id; + atomic_t host_mount; + /* 'D' for done, 'C' for unmount, 'T' for during mount. */ + char msg[256]; + /* Each slave writes its last_update here. */ + unsigned long slave_versions[64]; + struct urma_info urma_infos[64]; + struct urma_info send_infos[64]; +}; + +struct page_freq { + atomic_t read_count; + bool in_cache; + bool never_in_cache; + bool already_in_cache; +}; + +struct xmfs_cache_work { + struct work_struct work; + struct inode *inode; + pgoff_t index; + pgoff_t end_index; +}; + +struct xmfs_write_work { + struct work_struct work; + struct inode *inode; + pgoff_t index; + pgoff_t end_index; + size_t size; +}; + +struct xmfs_send_work { + struct work_struct work; + struct xmfs_rw_msg grant; + struct xmfs_sb_info *sbi; +}; + +struct xmfs_inode_info { + unsigned long i_ino; + struct inode vfs_inode; + /* Inode log start address. */ + unsigned long log_start; + /* Log trunk where the last replay ends. */ + unsigned long log_replay; + /* Symlink name address. */ + unsigned long symname; + struct xarray shared_pages; + struct simple_xattrs xattrs; + struct timespec64 btime; +#ifdef CONFIG_XMFS_FS_URMA + struct ubcore_jfs_wr *wr; + void *write_buffer; + int wr_cnt; + int *slots; +#endif + struct xarray page_freq; + unsigned long start_write; + unsigned long second_write; + unsigned long second_write_count; + unsigned long count; + unsigned long data_start; + unsigned long data_size; + bool leaf; + bool tmp; + bool die; + bool fallocate; + bool pinned; + unsigned long inode_table_slot; + struct workqueue_struct *write_wq; +}; + +struct xmfs_read_extent { + pgoff_t start_index; + unsigned long start_offset; + unsigned int nr_pages; +}; + +struct host_mount_args { + struct xmfs_sb_info *sbi; + int id; +}; + +void xmfs_init_file_inode(struct inode *inode); +void xmfs_init_file_mapping(struct inode *inode); +void xmfs_init_dir_inode(struct inode *inode, bool host); +void xmfs_init_dir_file(struct inode *inode); +void xmfs_init_dentry_operations(struct super_block *sb); +void xmfs_init_symlink_mapping(struct inode *inode); +void xmfs_init_symlink_operations(struct inode *inode); +void xmfs_init_special_inode_operations(struct inode *inode); +int xmfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int xmfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int xmfs_slave_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int xmfs_slave_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int flags); +int xmfs_update_time(struct inode *inode, int flags); +struct inode *xmfs_get_root(struct super_block *sb, int host_id); +int xmfs_send_req_wait_reply(struct xmfs_sb_info *sbi, int type); +void arch_invalidate_pmem(void *addr, size_t size); +void xmfs_mark_page_dead(struct xmfs_sb_info *sbi, unsigned long data_offset); +void xmfs_mark_page_live(struct xmfs_sb_info *sbi, unsigned long data_offset); + +struct xmfs_data_reserve_result { + unsigned long data_addr; + unsigned long data_page_count; + unsigned long second_data_addr; + unsigned long second_data_pages; + unsigned long trunk_addrs[4]; + int trunk_count; +}; + +unsigned long xmfs_add_log_v3(struct xmfs_log *xm_log, struct xmfs_sb_info *sbi, + size_t namelen, unsigned long ino, + bool update_version); +long xmfs_find_free_slot(struct xmfs_sb_index *xmsi, struct xmfs_sb_info *sbi); +long xmfs_alloc_and_mark_slot(struct xmfs_sb_index *xmsi, + struct xmfs_sb_info *sbi, char type); +int xmfs_gc_thread_fn(void *data); +int xmfs_gc_thread_slave_fn(void *data); +int xmfs_quiesce_wait(struct xmfs_sb_info *sbi); +int xmfs_host_worker_fn(void *data); +void xmfs_host_process_ring(struct xmfs_sb_info *sbi); +int xmfs_slave_request_write(struct xmfs_sb_info *sbi, struct xmfs_rw_msg *req); +int xmfs_slave_wait_grant(struct xmfs_sb_info *sbi, struct xmfs_rw_msg *grant, + unsigned long grant_slot, unsigned long request_id); +void xmfs_slave_send_complete(struct xmfs_sb_info *sbi, + unsigned long request_id, loff_t comp_size, + struct xmfs_rw_msg *orig_req, + struct xmfs_rw_msg *grant); +int xmfs_pmem_write(struct xmfs_sb_info *sbi, unsigned long offset, + const void *buf, unsigned long size); +int xmfs_pmem_write_multi_pages(struct xmfs_sb_info *sbi, unsigned long offset, + struct page **pages, unsigned long page_num); +int xmfs_pmem_read_multi_pages(struct xmfs_sb_info *sbi, unsigned long *offsets, + struct page **pages, unsigned long page_num); +int xmfs_pmem_read(struct xmfs_sb_info *sbi, unsigned long offset, void *buf, + unsigned long size); +int xmfs_pmem_cas(struct xmfs_sb_info *sbi, unsigned long offset, + uint64_t old_val, uint64_t new_val); +int xmfs_import_jetty(struct xmfs_sb_info *sbi, struct urma_info info, + int index, bool is_send); +int xmfs_send_request(struct xmfs_sb_info *sbi, struct xmfs_rw_msg *req, + struct xmfs_grant_wait *gw, int id); +#ifdef CONFIG_XMFS_FS_URMA +int xmfs_urma_send_write_pages(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr *wr); +#endif +void *urma_pool_acquire_nslots(struct xmfs_urma_pool *pool, int nr, u32 *slots); +void urma_pool_release_nslots(struct xmfs_urma_pool *pool, int nr, + const u32 *slots); +int xmfs_urma_read_ubuf(struct xmfs_sb_info *sbi, void __user *buf, + struct xmfs_read_extent *exts, int ext_cnt, + size_t size); +void xmfs_prefetch_worker(struct work_struct *work); + +static inline int xmfs_data_write(struct xmfs_sb_info *sbi, + unsigned long offset, const void *buf, + unsigned long size) +{ + if (sbi->mode != XMFS_URMA_MODE) { + memcpy(sbi->kaddr + offset, buf, size); + return 0; + } + return xmfs_pmem_write(sbi, offset, buf, size); +} + +static inline int xmfs_data_read(struct xmfs_sb_info *sbi, unsigned long offset, + void *buf, unsigned long size) +{ + if (sbi->mode != XMFS_URMA_MODE) { + memcpy(buf, sbi->kaddr + offset, size); + return 0; + } + return xmfs_pmem_read(sbi, offset, buf, size); +} + +int xmfs_gc_main(struct xmfs_sb_info *sbi); +int xmfs_inode_eq(struct inode *inode, void *_ino); +int xmfs_inode_set(struct inode *inode, void *_ino); +unsigned int xmfs_inode_table_sync_nlink(struct xmfs_inode_table *tbl, + unsigned long ino); +bool xmfs_inode_table_remove(struct xmfs_inode_table *tbl, unsigned long ino, + unsigned long p_ino, const char *name); +void xmfs_replay_plays_gc(struct xmfs_sb_info *sbi); +void xmfs_clear_inode_bits(struct xmfs_sb_info *sbi, unsigned long ino); +ssize_t xmfs_listxattr(struct dentry *dentry, char *buffer, size_t size); +int xmfs_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info); +int xmfs_replay_new_entries(struct xmfs_sb_info *sbi, struct dentry *dentry, + unsigned long last_update); +void xmfs_populate_from_inode_table(struct xmfs_sb_info *sbi); +ssize_t xmfs_simple_xattr_list(struct inode *inode, + struct simple_xattrs *xattrs, char *buffer, + size_t size); +int xmfs_xattr_list_one(char **buffer, ssize_t *remaining_size, + const char *name); +int xmfs_simple_xattr_get(struct simple_xattrs *xattrs, const char *name, + void *buffer, size_t size); +struct simple_xattr *xmfs_simple_xattr_set(struct simple_xattrs *xattrs, + const char *name, const void *value, + size_t size, int flags); +struct simple_xattr *xmfs_simple_xattr_alloc(const void *value, size_t size); +void xmfs_simple_xattrs_init(struct simple_xattrs *xattrs); +void xmfs_simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space); +size_t xmfs_simple_xattr_space(const char *name, size_t size); +void xmfs_simple_xattr_free(struct simple_xattr *xattr); +long xmfs_common_fallocate(struct inode *inode, int mode, loff_t offset, + loff_t len); +int xmfs_urma_register(struct xmfs_sb_info *sbi, struct xmfs_fs_context *ctx); +void xmfs_urma_unregister(struct xmfs_sb_info *sbi); +int xmfs_urma_import(struct xmfs_sb_info *sbi); +void xmfs_urma_unimport(struct xmfs_sb_info *sbi); +int xmfs_urma_send(struct xmfs_sb_info *sbi, struct xmfs_urma_msg *msg); +void xmfs_host_handle_urma_request(struct xmfs_sb_info *sbi, int slave_i, + struct xmfs_rw_msg *req, + struct xmfs_rw_msg *grant); +#ifdef CONFIG_XMFS_FS_URMA +int xmfs_urma_fill_write_page(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr *wr, + struct xmfs_urma_msg *msg, int slot); +#endif +int xmfs_urma_write_ubuf(struct xmfs_sb_info *sbi, void __user *buf, + size_t size, unsigned long offset); + +static inline struct xmfs_inode_info *XMFS_I(struct inode *inode) +{ + return container_of(inode, struct xmfs_inode_info, vfs_inode); +} + +static inline int local_update_log_and_data(struct xmfs_sb_info *sbi, + unsigned long offset, + unsigned long size) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + return 0; + case XMFS_URMA_MODE: + return xmfs_pmem_read(sbi, offset, sbi->urma_cfg.urma_local_va, + size); + default: + return -EINVAL; + } +} + +static inline int get_meta_and_copy(struct xmfs_sb_info *sbi, + unsigned long offset, void *addr, + unsigned long size, void *local_addr, + unsigned long local_offset) +{ + int err; + struct xmfs_urma_msg msg; + + if (addr == NULL) + return -EINVAL; + if (offset > sbi->len - 2097152) + return -EINVAL; + + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + return copy_mc_to_kernel(addr, sbi->kaddr + offset, size); + case XMFS_URMA_MODE: + if (!local_addr) { + msg.ops = XMFS_URMA_READ; + msg.offset = offset; + msg.len = size; + msg.local_addr = addr; + err = xmfs_urma_send(sbi, &msg); + return err; + } else { + return copy_mc_to_kernel(addr, + local_addr + offset - + local_offset, + size); + } + default: + return -EINVAL; + } +} + +static inline int get_data_and_copy(struct xmfs_sb_info *sbi, + unsigned long offset, void *addr, + unsigned long size, void *local_addr, + unsigned long local_offset) +{ + int err; + struct xmfs_urma_msg msg; + + if (addr == NULL) + return -EINVAL; + if (offset > sbi->len - 2097152) + return -EINVAL; + + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + return copy_mc_to_kernel(addr, sbi->kaddr + offset, size); + case XMFS_URMA_MODE: + case XMFS_HYBRID_MODE: + if (!local_addr) { + msg.ops = XMFS_URMA_READ; + msg.offset = offset; + msg.len = size; + msg.local_addr = addr; + err = xmfs_urma_send(sbi, &msg); + return err; + } else { + return copy_mc_to_kernel(addr, + local_addr + offset - + local_offset, + size); + } + default: + return -EINVAL; + } +} + +static inline void *get_meta(struct xmfs_sb_info *sbi, unsigned long offset, + unsigned long size) +{ + struct xmfs_urma_msg msg; + int err = 0; + void *local_addr; + + if (offset > sbi->len - 2097152) + return NULL; + + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + return sbi->kaddr + offset; + case XMFS_URMA_MODE: + local_addr = vmalloc(size); + if (!local_addr) + return local_addr; + msg.ops = XMFS_URMA_READ; + msg.offset = offset; + msg.len = size; + msg.local_addr = local_addr; + err = xmfs_urma_send(sbi, &msg); + if (err) { + xmfs_info(sbi->sb, "get meta err out %d", err); + vfree(local_addr); + local_addr = NULL; + } + return local_addr; + default: + return NULL; + } +} + +static inline void put_meta(struct xmfs_sb_info *sbi, void *local_addr) +{ + if (sbi->mode == XMFS_URMA_MODE) + vfree(local_addr); +} + +static inline void *get_data(struct xmfs_sb_info *sbi, unsigned long offset, + unsigned long size) +{ + struct xmfs_urma_msg msg; + int err = 0; + void *local_addr; + + if (offset > sbi->len - 2097152) + return NULL; + + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + return sbi->kaddr + offset; + case XMFS_URMA_MODE: + case XMFS_HYBRID_MODE: + local_addr = vmalloc(size); + if (!local_addr) + return local_addr; + msg.ops = XMFS_URMA_READ; + msg.offset = offset; + msg.len = size; + msg.local_addr = local_addr; + err = xmfs_urma_send(sbi, &msg); + if (err) { + vfree(local_addr); + local_addr = NULL; + } + return local_addr; + default: + return NULL; + } +} + +static inline void put_data(struct xmfs_sb_info *sbi, void *local_addr) +{ + if (sbi->mode == XMFS_URMA_MODE || sbi->mode == XMFS_HYBRID_MODE) + vfree(local_addr); +} + +static inline struct xmfs_sb_index *get_xmsi(struct xmfs_sb_info *sbi) +{ + return (struct xmfs_sb_index *)get_meta(sbi, SB_OFFSET, + sizeof(struct xmfs_sb_index)); +} + +static inline void put_xmsi(struct xmfs_sb_info *sbi, void *local_buf) +{ + put_meta(sbi, local_buf); +} + +static inline int get_xmsi_and_copy(struct xmfs_sb_info *sbi, + struct xmfs_sb_index *xmsi) +{ + return get_meta_and_copy(sbi, SB_OFFSET, xmsi, + sizeof(struct xmfs_sb_index), NULL, 0); +} + +static inline struct xmfs_log *get_log(struct xmfs_sb_info *sbi, + unsigned long offset) +{ + return (struct xmfs_log *)get_meta(sbi, offset, + sizeof(struct xmfs_log)); +} + +static inline void put_log(struct xmfs_sb_info *sbi, void *local_buf) +{ + put_meta(sbi, local_buf); +} + +static inline int get_log_and_copy(struct xmfs_sb_info *sbi, + unsigned long offset, struct xmfs_log *log) +{ + return get_meta_and_copy(sbi, offset, log, sizeof(struct xmfs_log), + NULL, 0); +} + +static inline struct xmfs_log_trunk *get_log_trunk(struct xmfs_sb_info *sbi, + unsigned long offset) +{ + return (struct xmfs_log_trunk *)get_meta(sbi, offset, 2097152); +} + +static inline struct xmfs_log_trunk * +get_log_trunk_meta(struct xmfs_sb_info *sbi, unsigned long offset) +{ + return (struct xmfs_log_trunk *)get_meta(sbi, offset, + sizeof(struct xmfs_log_trunk)); +} + +static inline struct xmfs_log_trunk * +get_log_trunk_data(struct xmfs_sb_info *sbi, unsigned long offset, + unsigned long idx) +{ + return (struct xmfs_log_trunk *)get_meta(sbi, offset, + sizeof(struct xmfs_log_trunk) + + idx * sizeof(struct xmfs_log)); +} + +static inline void put_log_trunk(struct xmfs_sb_info *sbi, void *local_buf) +{ + put_meta(sbi, local_buf); +} + +static inline int get_log_trunk_and_copy(struct xmfs_sb_info *sbi, + unsigned long offset, + struct xmfs_log_trunk *xmlt, + bool get_full_trunk) +{ + if (get_full_trunk) + return get_meta_and_copy(sbi, offset, xmlt, sbi->trunk_size, + NULL, 0); + return get_meta_and_copy(sbi, offset, xmlt, + sizeof(struct xmfs_log_trunk), NULL, 0); +} + +static inline struct mount_msg_info *get_mnt_msg(struct xmfs_sb_info *sbi) +{ + return (struct mount_msg_info *)get_meta(sbi, MOUNT_MSG_OFFSET, + sizeof(struct mount_msg_info)); +} + +static inline void put_mnt_msg(struct xmfs_sb_info *sbi, void *local_buf) +{ + put_meta(sbi, local_buf); +} + +static inline int get_mnt_msg_and_copy(struct xmfs_sb_info *sbi, + struct mount_msg_info *mm_info) +{ + return get_meta_and_copy(sbi, MOUNT_MSG_OFFSET, mm_info, + sizeof(struct mount_msg_info), NULL, 0); +} + +static inline int mnt_msg_write(struct xmfs_sb_info *sbi, int id, char c) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: { + struct mount_msg_info *mm_info = sbi->kaddr + MOUNT_MSG_OFFSET; + + return mm_info->msg[id] = c; + } + case XMFS_URMA_MODE: + return xmfs_pmem_write(sbi, + MOUNT_MSG_OFFSET + + offsetof(struct mount_msg_info, + msg) + + id, + &c, 1) ? + -1 : + c; + default: + return -1; + } +} + +static inline int xmfs_cas(struct xmfs_sb_info *sbi, int old, int new) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: { + struct mount_msg_info *mm_info = sbi->kaddr + MOUNT_MSG_OFFSET; + + return atomic_cmpxchg(&mm_info->id, old, new); + } + case XMFS_URMA_MODE: { + int ret = xmfs_pmem_cas(sbi, + MOUNT_MSG_OFFSET + + offsetof(struct mount_msg_info, + id), + (uint64_t)old, (uint64_t)new); + + return (ret == 0) ? old : 0; + } + default: + return 0; + } +} + +static inline int xmfs_atomic_read(struct xmfs_sb_info *sbi, + struct mount_msg_info *mm_info) +{ + return atomic_read(&mm_info->id); +} + +#endif diff --git a/fs/xmfs/dir.c b/fs/xmfs/dir.c new file mode 100644 index 000000000000..eb6f23d48e67 --- /dev/null +++ b/fs/xmfs/dir.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "xmfs_i.h" + +#include <linux/file.h> +#include <linux/kernel.h> +#include <linux/slab.h> + +static int xmfs_revalidate(struct dentry *dentry, unsigned int flags) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dentry->d_sb); + struct xmfs_sb_index *xmsi = kmalloc(sizeof(*xmsi), GFP_ATOMIC); + int err = get_xmsi_and_copy(sbi, xmsi); + + if (unlikely(err)) { + xmfs_error(sbi->sb, + "hardware memory error when reading superblock during revalidate"); + kfree(xmsi); + return err; + } + /* + * xmfs_info(sbi->sb, "qyf revalidate %ld %ld", + * xmsi->last_update, sbi->last_update); + */ + if (xmsi->last_update != sbi->last_update) { + kfree(xmsi); + return 0; + } + kfree(xmsi); + return 1; +} + +static const struct dentry_operations xmfs_dentry_operations = { + .d_delete = always_delete_dentry, + .d_revalidate = xmfs_revalidate, +}; + +static int xmfs_iterate(struct file *file, struct dir_context *ctx) +{ + struct xmfs_sb_info *sbi = XMFS_SB(file->f_inode->i_sb); + struct inode *dir = file->f_inode; + struct xmfs_inode_table *tbl; + unsigned long max_entries, i __maybe_unused; + + if (!dir_emit_dots(file, ctx)) + return 0; + + { + struct xmfs_sb_index *xmsi = kmalloc(sizeof(*xmsi), GFP_KERNEL); + + if (xmsi) { + int err = get_xmsi_and_copy(sbi, xmsi); + + if (!err && xmsi->last_update != sbi->last_update) + xmfs_replay_new_entries(sbi, NULL, + xmsi->last_update); + kfree(xmsi); + } + } + + if (sbi->mode == XMFS_SHARED_MEM_MODE || sbi->mode == XMFS_HYBRID_MODE) + tbl = (struct xmfs_inode_table *)(sbi->kaddr + + INODE_TABLE_OFFSET); + else { + tbl = (struct xmfs_inode_table *)get_meta(sbi, + INODE_TABLE_OFFSET, + INODE_TABLE_SIZE); + /* + * tbl = (struct xmfs_inode_table *)get_meta( + * sbi, INODE_TABLE_OFFSET, + * max(sizeof(struct xmfs_inode_table) + + * tbl->count * sizeof(struct xmfs_inode_entry), + * INODE_TABLE_SIZE)); + */ + } + + if (!tbl || tbl->magic != XMFS_INODE_TABLE_MAGIC) { + if (tbl && sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + return 0; + } + /* xmfs_info(sbi->sb, "qyf iterate %d %d", tbl->count, ctx->pos); */ + /* + * for (i = 0; i < tbl->count; i++) { + * struct xmfs_inode_entry *e = &tbl->entries[i]; + * + * xmfs_info(sbi->sb, + * "tbl id %d ino %d pino %d name %s size %d", + * i, e->i_ino, e->p_ino, e->name, e->size); + * } + */ + if (ctx->pos - 2 >= tbl->count) + return 0; + max_entries = + (INODE_TABLE_SIZE - 4096) / sizeof(struct xmfs_inode_entry); + for (; ctx->pos - 2 < tbl->count && ctx->pos - 2 < max_entries; + ctx->pos++) { + struct xmfs_inode_entry *e = &tbl->entries[ctx->pos - 2]; + unsigned char d_type; + /* + * xmfs_info(sbi->sb, + * "qyf iterate iter %d %d name: %s size %ld", + * e->i_ino, e->p_ino, e->name, e->size); + */ + if (e->i_ino == 0 || e->p_ino != dir->i_ino) + continue; + + if (S_ISREG(e->mode)) + d_type = DT_REG; + else if (S_ISDIR(e->mode)) + d_type = DT_DIR; + else if (S_ISLNK(e->mode)) + d_type = DT_LNK; + else if (S_ISCHR(e->mode)) + d_type = DT_CHR; + else if (S_ISBLK(e->mode)) + d_type = DT_BLK; + else if (S_ISFIFO(e->mode)) + d_type = DT_FIFO; + else if (S_ISSOCK(e->mode)) + d_type = DT_SOCK; + else + d_type = DT_UNKNOWN; + + if (!dir_emit(ctx, e->name, strlen(e->name), e->i_ino, d_type)) + break; + } + + if (sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + return 0; +} + +static const struct file_operations xmfs_dir_operations = { + .iterate_shared = xmfs_iterate, + .open = dcache_dir_open, + .release = dcache_dir_close, + .llseek = dcache_dir_lseek, + .read = generic_read_dir, + .fsync = noop_fsync, +}; + +void xmfs_init_dir_file(struct inode *inode) +{ + inode->i_fop = &xmfs_dir_operations; +} + +void xmfs_init_dentry_operations(struct super_block *sb) +{ + sb->s_d_op = &xmfs_dentry_operations; +} diff --git a/fs/xmfs/file.c b/fs/xmfs/file.c new file mode 100644 index 000000000000..ef45c2b31f47 --- /dev/null +++ b/fs/xmfs/file.c @@ -0,0 +1,2261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#include "xmfs_i.h" + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/swap.h> +#include <linux/uio.h> +#include <linux/file.h> +#include <linux/seq_file.h> +#include <linux/pagemap.h> +#include <linux/fadvise.h> +#include <linux/splice.h> +#include <linux/falloc.h> + +#ifdef CONFIG_XMFS_FS_URMA +static ssize_t xmfs_file_write_iter_slave(struct kiocb *iocb, + struct iov_iter *from); +static ssize_t xmfs_file_write_ubuf_slave(struct kiocb *iocb, + struct iov_iter *from); +#else +static ssize_t xmfs_file_write_iter_slave(struct kiocb *iocb, + struct iov_iter *from) +{ + return -EOPNOTSUPP; +} +#endif +static ssize_t xmfs_file_read_ubuf_slave(struct kiocb *iocb, + struct iov_iter *to); +static int xmfs_read_page(struct file *file, struct page *page) +{ + struct inode *inode = page->mapping->host; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + unsigned long mapped_offset = + (unsigned long)xa_load(&xmfs_inode->shared_pages, page->index); + void *kaddr = kmap_local_page(page); + + if (mapped_offset) { + memcpy(kaddr, get_data(sbi, mapped_offset, PAGE_SIZE), + PAGE_SIZE); + } + kunmap_local(kaddr); + + return 0; +} + +#ifdef CONFIG_XMFS_FS_URMA +static void xmfs_write_worker(struct work_struct *work) +{ + struct xmfs_write_work *ww = + container_of(work, struct xmfs_write_work, work); + struct inode *inode = ww->inode; + struct address_space *mapping = inode->i_mapping; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct page *page; + struct folio *folio; + struct xmfs_rw_msg grant = {}; + struct xmfs_rw_msg req = {}; + unsigned long end_index = ww->end_index; + unsigned long start_index = ww->index; + unsigned long page_num = end_index - start_index; + struct ubcore_jfs_wr *wrs; + int *slots; + void *write_buffer; + int wr_cnt = 0; + unsigned long mapped_offset; + struct xmfs_urma_msg msg = {}; + char *addr; + struct ubcore_jfs_wr *wr; + int err; + unsigned long log_base; + /* + * xmfs_info(sbi->sb, "qyf write work %d %d %d", start_index, + * end_index, ww->size); + */ + + req.ops = XMFS_UPDATE; + req.i_ino = inode->i_ino; + req.data_pages = page_num; + req.log_entries = 1; + req.size = ww->size; + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + if (xmfs_quiesce_wait(sbi)) + return; + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) { + iput(inode); + return; + } + grant = gw.grant; + log_base = + grant.log_trunk_addr + offsetof(struct xmfs_log_trunk, log[0]); + /* + * xmfs_info(sbi->sb, "grant %d %d %d %d %d", + * grant.log_slot_count, grant.data_addr, + * grant.data_page_count, grant.second_data_addr, + * grant.log_slot_start); + */ + wrs = vzalloc(page_num * sizeof(struct ubcore_jfs_wr)); + if (!wrs) { + iput(inode); + return; + } + slots = vzalloc(sizeof(int) * page_num); + if (!slots) { + vfree(wrs); + iput(inode); + return; + } + + write_buffer = + urma_pool_acquire_nslots(&sbi->urma_cfg.pool, page_num, slots); + if (!write_buffer) { + vfree(wrs); + vfree(slots); + iput(inode); + return; + } + + unsigned long start_write = grant.data_addr; + unsigned long count = grant.data_page_count; + unsigned long second_write = grant.second_data_addr; + + for (int i = start_index; i < end_index; i++) { + if (count > 0) { + mapped_offset = start_write; + count--; + start_write += PAGE_SIZE; + if (start_write % XMFS_DEFAULT_CHUNK_SIZE == 0) + start_write += PAGE_SIZE; + } else if (second_write != 0) { + mapped_offset = second_write; + second_write += PAGE_SIZE; + if (second_write % XMFS_DEFAULT_CHUNK_SIZE == 0) + second_write += PAGE_SIZE; + } + + folio = filemap_get_folio(mapping, i); + if (IS_ERR(folio)) { + xmfs_warning(sbi->sb, "write work cannot get folio %d", + i); + continue; + } + xa_store(&(XMFS_I(inode)->shared_pages), i, + (void *)mapped_offset, GFP_KERNEL); + page = folio_page(folio, 0); + addr = kmap_local_page(page); + msg.ops = XMFS_URMA_WRITE; + msg.offset = mapped_offset; + msg.len = PAGE_SIZE; + msg.write_buffer = write_buffer + wr_cnt * PAGE_SIZE; + msg.local_addr = addr; + wr = &wrs[wr_cnt]; + err = xmfs_urma_fill_write_page(sbi, wr, &msg, slots[wr_cnt]); + if (err) { + xmfs_info(sbi->sb, + "write work fill write page failed %d folio", + i); + kunmap_local(addr); + folio_put(folio); + continue; + } + if (wr_cnt > 0) + wrs[wr_cnt - 1].next = wr; + wr_cnt++; + kunmap_local(addr); + folio_put(folio); + } + + if (wr_cnt > 0) { + wrs[wr_cnt - 1].next = NULL; + xmfs_urma_send_write_pages(sbi, wrs); + } + + urma_pool_release_nslots(&sbi->urma_cfg.pool, page_num, slots); + vfree(slots); + + for (int s = 0; s < grant.log_slot_count; s++) { + struct xmfs_log xm_log = {}; + + xm_log.ops = XMFS_UPDATE; + xm_log.i_ino = inode->i_ino; + xm_log.version = grant.version + s; + xm_log.writer_id = sbi->id; + if (s == 0) { + xm_log.index = start_index; + xm_log.end_index = start_index + grant.data_page_count; + xm_log.data_offset = grant.data_addr; + xm_log.size = grant.size; + } else { + xm_log.index = start_index + grant.data_page_count; + xm_log.end_index = end_index; + xm_log.data_offset = grant.second_data_addr; + xm_log.size = grant.size; + } + + xmfs_data_write(sbi, + log_base + (grant.log_slot_start + s) * + sizeof(struct xmfs_log), + &xm_log, sizeof(xm_log)); + /* + * xmfs_info(sbi->sb, "write work log %ld %ld %ld", + * log_base + (grant.log_slot_start + s) * + * sizeof(struct xmfs_log), + * xm_log.index, xm_log.end_index); + */ + } + iput(inode); + kfree(ww); +} +#endif + +void xmfs_prefetch_worker(struct work_struct *work) +{ + struct xmfs_cache_work *cw = + container_of(work, struct xmfs_cache_work, work); + struct inode *inode = cw->inode; + struct address_space *mapping = inode->i_mapping; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct folio *folio; + unsigned long *offsets; + struct page **pages; + unsigned long mapped_offset; + int i; + int count = 0; + + offsets = + kzalloc((cw->end_index - cw->index + 1) * sizeof(unsigned long), + GFP_KERNEL); + if (!offsets) + return; + + pages = kzalloc((cw->end_index - cw->index + 1) * sizeof(struct page *), + GFP_KERNEL); + if (!pages) { + kfree(offsets); + return; + } + + for (i = cw->index; i < cw->end_index; i++) { + mapped_offset = + (unsigned long)xa_load(&XMFS_I(inode)->shared_pages, i); + if (!mapped_offset) + continue; + + folio = filemap_grab_folio(mapping, i); + + if (IS_ERR(folio)) + break; + + offsets[count] = mapped_offset; + pages[count] = &folio->page; + count++; + } + + if (count > 0) { + /* + * xmfs_info(sbi->sb, "qyf backgrount cp page %ld to %ld", + * cw->index, cw->end_index); + */ + xmfs_pmem_read_multi_pages(sbi, offsets, pages, count); + for (i = 0; i < count; i++) { + folio = page_folio(pages[i]); + + folio_mark_uptodate(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + } + } + kfree(offsets); + kfree(pages); + + kfree(cw); +} + +static int xmfs_write_begin(struct file *filp, struct address_space *mapping, + loff_t pos, unsigned int len, struct page **pagep, + void **fsdata) +{ + struct inode *inode = mapping->host; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + int err = simple_write_begin(filp, mapping, pos, len, pagep, fsdata); + + if (err || sbi->id != 0 || sbi->during_replay || PageUptodate(*pagep) || + len == PAGE_SIZE) + return err; + return xmfs_read_page(filp, *pagep); +} + +unsigned long xmfs_get_data_offset_nolock(struct inode *inode, + struct xmfs_sb_info *sbi, + struct page *page) +{ + char *kaddr; + + if (XMFS_I(inode)->count > 0) { + unsigned long ret_addr; + + if (sbi->id == 0) { + kaddr = kmap_local_page(page); + xmfs_data_write(sbi, XMFS_I(inode)->start_write, kaddr, + PAGE_SIZE); + kunmap_local(kaddr); + } + ret_addr = XMFS_I(inode)->start_write; + XMFS_I(inode)->count--; + XMFS_I(inode)->start_write += PAGE_SIZE; + if (XMFS_I(inode)->start_write % XMFS_DEFAULT_CHUNK_SIZE == 0) + XMFS_I(inode)->start_write += PAGE_SIZE; + return ret_addr; + } else if (XMFS_I(inode)->second_write != 0) { + unsigned long ret_addr; + + if (sbi->id == 0) { + kaddr = kmap_local_page(page); + xmfs_data_write(sbi, XMFS_I(inode)->second_write, kaddr, + PAGE_SIZE); + kunmap_local(kaddr); + } + ret_addr = XMFS_I(inode)->second_write; + XMFS_I(inode)->second_write += PAGE_SIZE; + if (XMFS_I(inode)->second_write % XMFS_DEFAULT_CHUNK_SIZE == 0) + XMFS_I(inode)->second_write += PAGE_SIZE; + return ret_addr; + } + /* + * xmfs_error(inode->i_sb, + * "no reserved space for inode %ld", inode->i_ino); + */ + return 0; +} + +unsigned long xmfs_get_data_offset_fallocate(struct inode *inode, + struct xmfs_sb_info *sbi, + struct page *page) +{ + unsigned long index = page->index; + unsigned long data_start = XMFS_I(inode)->data_start; + unsigned long len = XMFS_I(inode)->data_size; + + if (index * PAGE_SIZE >= len) { + xmfs_error(sbi->sb, + "Not support write more than fallocate size now"); + return -1; + } + + char *kaddr = kmap_local_page(page); + + xmfs_data_write(sbi, data_start + index * PAGE_SIZE, kaddr, PAGE_SIZE); + kunmap_local(kaddr); + + return data_start + index * PAGE_SIZE; +} + +static int xmfs_write_end(struct file *file, struct address_space *mapping, + loff_t pos, unsigned int len, unsigned int copied, + struct page *page, void *fsdata) +{ + struct folio *folio = page_folio(page); + struct inode *inode = folio->mapping->host; + loff_t last_pos = pos + copied; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (!folio_test_uptodate(folio)) { + /* zero the stale part of the folio if we did a short copy */ + if (copied < len) { + size_t from = offset_in_folio(folio, pos); + + folio_zero_range(folio, from + copied, len - copied); + } + folio_mark_uptodate(folio); + } + folio_mark_dirty(folio); + + if (i_size_read(inode) < last_pos) + i_size_write(inode, last_pos); + + unsigned long offset; + + if (XMFS_I(inode)->fallocate) + offset = xmfs_get_data_offset_fallocate(inode, sbi, page); + else + offset = xmfs_get_data_offset_nolock(inode, sbi, page); + if (offset == 0) { + folio_unlock(folio); + folio_put(folio); + return copied; + } else if (offset < 0) { + folio_unlock(folio); + folio_put(folio); + return offset; + } + /* + * xmfs_info(sbi->sb, "write end index %d offset %ld", + * page->index, offset); + */ + /* + * if (sbi->id != 0) { + * folio_mark_dirty(folio); + * struct xmfs_urma_msg msg = {}; + * char *addr = kmap_local_page(page); + * struct ubcore_jfs_wr *wr; + * + * msg.ops = XMFS_URMA_WRITE; + * msg.offset = offset; + * msg.len = PAGE_SIZE; + * msg.write_buffer = XMFS_I(inode)->write_buffer + + * XMFS_I(inode)->wr_cnt * PAGE_SIZE; + * msg.local_addr = addr; + * wr = &XMFS_I(inode)->wr[XMFS_I(inode)->wr_cnt]; + * xmfs_urma_fill_write_page( + * sbi, wr, &msg, + * XMFS_I(inode)->slots[XMFS_I(inode)->wr_cnt]); + * if (XMFS_I(inode)->wr_cnt > 0) + * XMFS_I(inode)->wr[XMFS_I(inode)->wr_cnt - 1].next = wr; + * XMFS_I(inode)->wr_cnt++; + * kunmap_local(addr); + * } + */ + + void *old_ret = xa_store(&(XMFS_I(inode)->shared_pages), folio->index, + (void *)offset, GFP_KERNEL); + if (xa_err(old_ret)) { + xmfs_error(sbi->sb, "cannot store shared pages for page %ld", + folio->index); + } + if (sbi->id == 0) { + xmfs_mark_page_dead(sbi, (unsigned long)old_ret); + xmfs_mark_page_live(sbi, offset); + } + + folio_unlock(folio); + folio_put(folio); + + return copied; +} + +static ssize_t xmfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t ret; + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (sbi->id != 0) { +#ifdef CONFIG_XMFS_FS_URMA + if (iter_is_ubuf(from)) + return xmfs_file_write_ubuf_slave(iocb, from); +#endif + return xmfs_file_write_iter_slave(iocb, from); + } + inode_lock(inode); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (XMFS_I(inode)->die) { + xmfs_info(sbi->sb, + "inode logs have been interrupted before write iter"); + inode_unlock(inode); + return -EINVAL; + } + + struct xmfs_log xm_log = {}; + + unsigned long start_index = iocb->ki_pos / PAGE_SIZE; + unsigned long end_index = + (iov_iter_count(from) + iocb->ki_pos + PAGE_SIZE - 1) / + PAGE_SIZE; + unsigned long start_offset = iocb->ki_pos; + unsigned long xmlt_addr; + struct xmfs_rw_msg req = {}, grant = {}; + + ret = generic_write_checks(iocb, from); + if (ret > 0) { + start_index = iocb->ki_pos / PAGE_SIZE; + end_index = + (iov_iter_count(from) + iocb->ki_pos + PAGE_SIZE - 1) / + PAGE_SIZE; + start_offset = iocb->ki_pos; + if (XMFS_I(inode)->fallocate && + end_index > XMFS_I(inode)->data_size / PAGE_SIZE) { + xmfs_error(sbi->sb, + "write iter do not support write more than fallocate size"); + inode_unlock(inode); + return -EOPNOTSUPP; + } + + ret = file_remove_privs(file); + if (ret) { + xmfs_error(sbi->sb, + "write iter remove privs failed %ld", ret); + inode_unlock(inode); + return ret; + } + + ret = file_update_time(file); + if (ret) { + xmfs_error(sbi->sb, "write iter update time failed %ld", + ret); + inode_unlock(inode); + return ret; + } + + if (XMFS_I(inode)->fallocate) + goto start_write; + + req.ops = XMFS_UPDATE; + req.i_ino = inode->i_ino; + req.data_pages = end_index - start_index; + req.log_entries = 1; + req.size = iocb->ki_pos + iov_iter_count(from); + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + xmfs_host_handle_urma_request(sbi, sbi->id, &req, &grant); + XMFS_I(inode)->start_write = grant.data_addr; + XMFS_I(inode)->count = grant.data_page_count; + XMFS_I(inode)->second_write = grant.second_data_addr; + XMFS_I(inode)->second_write_count = 0; + +start_write: + /* + * xmfs_info(sbi->sb, "write start %ld %ld %d %ld", + * XMFS_I(inode)->start_write, + * XMFS_I(inode)->second_write, XMFS_I(inode)->count, + * XMFS_I(inode)->second_write_count); + */ + ret = generic_perform_write(iocb, from); + } else { + inode_unlock(inode); + return ret; + } + /* + * xmfs_info(sbi->sb, + * "start offset %ld index %ld end index %ld pos %ld %ld", + * start_offset, start_index, end_index, iocb->ki_pos, + * (iocb->ki_pos + PAGE_SIZE - 1) / PAGE_SIZE); + */ + XMFS_I(inode)->start_write = 0; + XMFS_I(inode)->second_write = 0; + XMFS_I(inode)->count = 0; + XMFS_I(inode)->second_write_count = 0; + if (ret <= 0) { + inode_unlock(inode); + return ret; + } + + for (int s = 0; s < grant.log_slot_count; s++) { + xm_log.ops = XMFS_UPDATE; + xm_log.i_ino = inode->i_ino; + xm_log.version = grant.version + s; + xm_log.writer_id = sbi->id; + if (s == 0) { + xm_log.index = start_index; + xm_log.end_index = start_index + grant.data_page_count; + xm_log.data_offset = grant.data_addr; + xm_log.size = req.size; + } else { + xm_log.index = start_index + grant.data_page_count; + xm_log.end_index = end_index; + xm_log.data_offset = grant.second_data_addr; + xm_log.size = req.size; + } + xmlt_addr = + xmfs_add_log_v3(&xm_log, sbi, 0, inode->i_ino, true); + } + + inode_unlock(inode); + return ret; +} + +const struct pipe_buf_operations xmfs_pipe_buf_ops = { + .release = generic_pipe_buf_release, + .try_steal = generic_pipe_buf_try_steal, + .get = generic_pipe_buf_get, +}; + +static size_t splice_folio_into_pipe(struct pipe_inode_info *pipe, + struct folio *folio, loff_t fpos, + size_t size) +{ + struct page *page; + size_t spliced = 0, offset = offset_in_folio(folio, fpos); + + page = folio_page(folio, offset / PAGE_SIZE); + size = min(size, folio_size(folio) - offset); + offset %= PAGE_SIZE; + + while (spliced < size && + !pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { + struct pipe_buffer *buf = pipe_head_buf(pipe); + size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced); + + *buf = (struct pipe_buffer) { + .ops = &xmfs_pipe_buf_ops, + .page = page, + .offset = offset, + .len = part, + }; + folio_get(folio); + pipe->head++; + page++; + spliced += part; + offset = 0; + } + + return spliced; +} + +static bool zero_pipe_buf_get(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + return true; +} + +static void zero_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ +} + +static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + return false; +} + +static const struct pipe_buf_operations zero_pipe_buf_ops = { + .release = zero_pipe_buf_release, + .try_steal = zero_pipe_buf_try_steal, + .get = zero_pipe_buf_get, +}; + +static size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe, + loff_t fpos, size_t size) +{ + size_t offset = fpos & ~PAGE_MASK; + + size = min_t(size_t, size, PAGE_SIZE - offset); + + if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { + struct pipe_buffer *buf = pipe_head_buf(pipe); + + *buf = (struct pipe_buffer) { + .ops = &zero_pipe_buf_ops, + .page = ZERO_PAGE(0), + .offset = offset, + .len = size, + }; + pipe->head++; + } + + return size; +} + +static ssize_t xmfs_file_read_iter_slave(struct kiocb *iocb, + struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + pgoff_t index; + pgoff_t end_index; + unsigned long offset; + unsigned long *offsets; + int error = 0; + int count = 0; + ssize_t retval = 0; + loff_t *ppos = &iocb->ki_pos; + loff_t i_size = iov_iter_count(to); + struct page **pages; + /* xmfs_info(sbi->sb, "read iter slave"); */ + + /* + * struct xmfs_sb_index *xmsi = + * kmalloc(sizeof(struct xmfs_sb_index), GFP_KERNEL); + * int err = get_xmsi_and_copy(sbi, xmsi); + * + * if (unlikely(err)) { + * xmfs_error(sbi->sb, + * "hardware memory error when reading superblock during slave lookup"); + * kfree(xmsi); + * return err; + * } + * + * if (xmsi->last_update != sbi->last_update) { + * if (!sbi->debug) + * xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + * else + * xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + * } + * + * kfree(xmsi); + */ + + index = *ppos >> PAGE_SHIFT; + offset = *ppos & ~PAGE_MASK; + end_index = (i_size + *ppos) / PAGE_SIZE; + offsets = kzalloc((end_index - index + 1) * sizeof(unsigned long), + GFP_KERNEL); + if (!offsets) + return -ENOMEM; + + pages = kzalloc((end_index - index + 1) * sizeof(struct page *), + GFP_KERNEL); + if (!pages) { + kfree(offsets); + return -ENOMEM; + } + for (;;) { + struct folio *folio = NULL; + unsigned long nr, ret; + + if (index > end_index) + break; + if (index == end_index) { + nr = (i_size + *ppos) & ~PAGE_MASK; + if (nr <= offset) + break; + } else { + nr = PAGE_SIZE; + } + nr -= offset; + + folio = filemap_get_folio(inode->i_mapping, index); + if (IS_ERR(folio)) + folio = NULL; + + if (folio) { + folio_put(folio); + } else { + /* + * Copy to user tends to be so well optimized, but + * clear_user() not so much, that it is noticeably + * faster to copy the zero page instead of clearing. + */ + /* xmfs_error(sbi->sb, "qyf bad things happened"); */ + unsigned long mapped_offset = + (unsigned long)xa_load(&xmfs_inode->shared_pages, + index); + if (mapped_offset) { + folio = __filemap_get_folio(inode->i_mapping, index, + FGP_LOCK | FGP_WRITE | + FGP_CREAT, + inode->i_mapping->gfp_mask); + if (IS_ERR(folio)) { + xmfs_info(sbi->sb, + "splice cannot get the folio"); + error = PTR_ERR(folio); + folio = NULL; + break; + } + offsets[count] = mapped_offset; + pages[count] = &folio->page; + count++; + } else { + xmfs_info(sbi->sb, "no offset for %ld", index); + } + } + ret = nr; + offset += ret; + index += offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + } + + if (count > 0) { + int idx = 0; + + while (count > 0) { + int num = min(512, count); + + count -= num; + xmfs_pmem_read_multi_pages(sbi, offsets + idx, + pages + idx, num); + idx += num; + } + for (int i = 0; i < idx; i++) { + struct folio *folio = page_folio(pages[i]); + + folio_mark_uptodate(folio); + folio_unlock(folio); + folio_put(folio); + } + } + kfree(offsets); + kfree(pages); + index = *ppos >> PAGE_SHIFT; + offset = *ppos & ~PAGE_MASK; + i_size = i_size_read(inode); + end_index = i_size >> PAGE_SHIFT; + + for (;;) { + struct folio *folio = NULL; + struct page *page = NULL; + unsigned long nr, ret; + + if (index > end_index) + break; + if (index == end_index) { + nr = i_size & ~PAGE_MASK; + if (nr <= offset) + break; + } else { + nr = PAGE_SIZE; + } + nr -= offset; + + folio = filemap_get_folio(inode->i_mapping, index); + if (!IS_ERR(folio)) { + bool uptodate; + + folio_lock(folio); + uptodate = folio_test_uptodate(folio); + folio_unlock(folio); + if (!uptodate) { + folio_put(folio); + folio = NULL; + } + } else { + folio = NULL; + } + if (IS_ERR(folio)) + folio = NULL; + + if (folio) { + page = folio_page(folio, 0); + ret = copy_page_to_iter(page, offset, nr, to); + folio_put(folio); + } else { + ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to); + } + + retval += ret; + offset += ret; + index += offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + + if (!iov_iter_count(to)) + break; + if (ret < nr) { + error = -EFAULT; + break; + } + } + + *ppos = ((loff_t)index << PAGE_SHIFT) + offset; + file_accessed(file); + return retval ? retval : error; +} + +int xmfs_collect_read_extents(struct xmfs_inode_info *inode, pgoff_t start, + pgoff_t end, struct xmfs_read_extent *exts, + unsigned int max_exts) +{ + XA_STATE(xas, &inode->shared_pages, start); + + void *entry; + unsigned long offset; + pgoff_t index; + unsigned int nr_exts = 0; + bool in_extent = false; + + pgoff_t cur_index = 0; + unsigned long cur_offset = 0; + unsigned int cur_pages = 0; + + xas_lock(&xas); + + xas_for_each(&xas, entry, end - 1) { + offset = (unsigned long)entry; + + index = xas.xa_index; + + /* + * xa_load() 返回 NULL 时不会进入这里， + * xas_for_each() 已经跳过空洞。 + */ + + if (!in_extent) { + cur_index = index; + cur_offset = offset; + cur_pages = 1; + in_extent = true; + continue; + } + + /* + * 是否仍然连续。 + */ + if (index == cur_index + cur_pages && + offset == cur_offset + ((unsigned long)cur_pages + << PAGE_SHIFT)) { + cur_pages++; + continue; + } + + /* + * 保存上一段 extent。 + */ + if (nr_exts >= max_exts) { + xas_unlock(&xas); + return -ENOSPC; + } + exts[nr_exts].start_index = cur_index; + exts[nr_exts].start_offset = cur_offset; + exts[nr_exts].nr_pages = cur_pages; + nr_exts++; + /* + * 新 extent。 + */ + cur_index = index; + cur_offset = offset; + cur_pages = 1; + } + + xas_unlock(&xas); + + /* + * Flush 最后一个 extent。 + */ + if (in_extent) { + if (nr_exts >= max_exts) + return -ENOSPC; + exts[nr_exts].start_index = cur_index; + exts[nr_exts].start_offset = cur_offset; + exts[nr_exts].nr_pages = cur_pages; + nr_exts++; + } + + return nr_exts; +} + +static ssize_t xmfs_file_read_ubuf_slave(struct kiocb *iocb, + struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + pgoff_t index; + pgoff_t end_index; + unsigned long offset; + int err = 0; + int count = 0; + ssize_t retval = 0; + loff_t *ppos = &iocb->ki_pos; + loff_t i_size = iov_iter_count(to); + loff_t end_pos = min(i_size + iocb->ki_pos, inode->i_size); + struct xmfs_read_extent *exts; + + index = *ppos >> PAGE_SHIFT; + offset = *ppos & ~PAGE_MASK; + end_index = (end_pos + PAGE_SIZE - 1) / PAGE_SIZE; + + count = end_index - index + 1; + + exts = vzalloc(count * sizeof(struct xmfs_read_extent)); + /* + * xmfs_info(sbi->sb, "qyf before count is %d end is %ld %ld %ld", + * count, inode->i_size, *ppos, i_size); + */ + count = xmfs_collect_read_extents(xmfs_inode, index, end_index, exts, + count); + /* xmfs_info(sbi->sb, "qyf count is %d %ld", count, i_size); */ + for (int i = 0; i < count; i++) + retval += exts[i].nr_pages * PAGE_SIZE; + + retval = min(retval, end_pos - *ppos); + if (count != 0 && retval != 0) { + exts[0].start_offset += offset; + err = xmfs_urma_read_ubuf(sbi, to->ubuf, exts, count, retval); + } + vfree(exts); + iocb->ki_pos += retval; + file_accessed(file); + /* xmfs_info(sbi->sb, "qyf read return %ld %ld", err, retval); */ + return err ? err : retval; +} + +static ssize_t xmfs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + struct inode *inode = file_inode(in); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct address_space *mapping = inode->i_mapping; + struct folio *folio = NULL; + size_t total_spliced = 0, used, npages, n, part; + loff_t isize; + int error = 0; + + struct xmfs_sb_index *xmsi = + kmalloc(sizeof(struct xmfs_sb_index), GFP_KERNEL); + int err = get_xmsi_and_copy(sbi, xmsi); + + if (unlikely(err)) { + xmfs_error(sbi->sb, + "hardware memory error when reading superblock during splice read"); + kfree(xmsi); + return err; + } + + if (xmsi->last_update != sbi->last_update) { + if (!sbi->debug) + xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + else + xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + } + kfree(xmsi); + + used = pipe_occupancy(pipe->head, pipe->tail); + npages = max_t(ssize_t, pipe->max_usage - used, 0); + len = min_t(size_t, len, (npages * PAGE_SIZE)); + + do { + if (*ppos >= i_size_read(inode)) + break; + + folio = filemap_get_folio(inode->i_mapping, *ppos / PAGE_SIZE); + if (!IS_ERR(folio)) { + bool uptodate; + + folio_lock(folio); + uptodate = folio_test_uptodate(folio); + folio_unlock(folio); + if (!uptodate) { + folio_put(folio); + folio = NULL; + } + } else { + folio = NULL; + } + + /* + * i_size must be checked after we know the pages are Uptodate. + * + * Checking i_size after the check allows us to calculate + * the correct value for "nr", which means the zero-filled + * part of the page is not copied back to userspace (unless + * another truncate extends the file - this is desired though). + */ + isize = i_size_read(inode); + if (unlikely(*ppos >= isize)) + break; + part = min_t(loff_t, isize - *ppos, len); + + if (folio) { + /* + * If users can be writing to this page using arbitrary + * virtual addresses, take care about potential aliasing + * before reading the page on the kernel side. + */ + if (mapping_writably_mapped(mapping)) + flush_dcache_folio(folio); + folio_mark_accessed(folio); + /* + * Ok, we have the page, and it's up-to-date, so we can + * now splice it into the pipe. + */ + n = splice_folio_into_pipe(pipe, folio, *ppos, part); + folio_put(folio); + folio = NULL; + } else { + unsigned long mapped_offset = + (unsigned long)xa_load(&xmfs_inode->shared_pages, + *ppos / PAGE_SIZE); + if (!mapped_offset) { + n = splice_zeropage_into_pipe(pipe, *ppos, + part); + } else { + folio = __filemap_get_folio(inode->i_mapping, + *ppos / PAGE_SIZE, + FGP_LOCK | FGP_WRITE | + FGP_CREAT, + inode->i_mapping->gfp_mask); + if (IS_ERR(folio)) { + xmfs_info(sbi->sb, + "splice cannot get the folio"); + error = PTR_ERR(folio); + folio = NULL; + break; + } + void *kaddr = + kmap_local_page(folio_file_page(folio, + *ppos / PAGE_SIZE)); + error = get_data_and_copy(sbi, mapped_offset, + kaddr, PAGE_SIZE, + NULL, 0); + if (unlikely(error)) { + xmfs_error(sbi->sb, + "hardware memory error when reading data during splice read"); + folio_unlock(folio); + folio_put(folio); + folio = NULL; + break; + } + kunmap_local(kaddr); + folio_mark_uptodate(folio); + n = splice_folio_into_pipe(pipe, folio, *ppos, + part); + folio_unlock(folio); + folio_put(folio); + folio = NULL; + } + } + + if (!n) + break; + len -= n; + total_spliced += n; + *ppos += n; + in->f_ra.prev_pos = *ppos; + if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) + break; + + } while (len); + + if (folio) + folio_put(folio); + + file_accessed(in); + return total_spliced ? total_spliced : error; +} + +static ssize_t xmfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (sbi->id != 0) { + if (iter_is_ubuf(to)) + return xmfs_file_read_ubuf_slave(iocb, to); + return xmfs_file_read_iter_slave(iocb, to); + } + + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + pgoff_t index; + unsigned long offset; + int error = 0; + ssize_t retval = 0; + loff_t *ppos = &iocb->ki_pos; + + struct xmfs_sb_index *xmsi = + kmalloc(sizeof(struct xmfs_sb_index), GFP_KERNEL); + int err = get_xmsi_and_copy(sbi, xmsi); + + if (unlikely(err)) { + xmfs_error(sbi->sb, + "hardware memory error when reading superblock during read iter"); + kfree(xmsi); + return err; + } + + /* + * if (xmsi->last_update != sbi->last_update) { + * if (!sbi->debug) + * xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + * else + * xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + * } + */ + + kfree(xmsi); + + index = *ppos >> PAGE_SHIFT; + offset = *ppos & ~PAGE_MASK; + + for (;;) { + struct folio *folio = NULL; + struct page *page = NULL; + pgoff_t end_index; + unsigned long nr, ret; + loff_t i_size = i_size_read(inode); + + end_index = i_size >> PAGE_SHIFT; + if (index > end_index) + break; + if (index == end_index) { + nr = i_size & ~PAGE_MASK; + if (nr <= offset) + break; + } else { + nr = PAGE_SIZE; + } + nr -= offset; + + folio = filemap_get_folio(inode->i_mapping, index); + if (!IS_ERR(folio)) { + bool uptodate; + + folio_lock(folio); + uptodate = folio_test_uptodate(folio); + folio_unlock(folio); + if (!uptodate) { + folio_put(folio); + folio = NULL; + } + } else { + folio = NULL; + } + if (IS_ERR(folio)) + folio = NULL; + + if (folio) { + page = folio_page(folio, 0); + ret = copy_page_to_iter(page, offset, nr, to); + folio_put(folio); + } else { + /* + * Copy to user tends to be so well optimized, but + * clear_user() not so much, that it is noticeably + * faster to copy the zero page instead of clearing. + */ + /* xmfs_error(sbi->sb, "qyf bad things happened"); */ + unsigned long mapped_offset = + (unsigned long)xa_load(&xmfs_inode->shared_pages, + index); + if (!mapped_offset) { + ret = copy_page_to_iter(ZERO_PAGE(0), offset, + nr, to); + } else { + void *data; + + data = get_data(sbi, mapped_offset + offset, + nr); + if (IS_ERR(data)) { + error = PTR_ERR(data); + break; + } + ret = _copy_to_iter(data, nr, to); + put_data(sbi, data); + } + } + + retval += ret; + offset += ret; + index += offset >> PAGE_SHIFT; + offset &= ~PAGE_MASK; + + if (!iov_iter_count(to)) + break; + if (ret < nr) { + error = -EFAULT; + break; + } + } + + *ppos = ((loff_t)index << PAGE_SHIFT) + offset; + file_accessed(file); + return retval ? retval : error; +} + +#ifdef CONFIG_XMFS_FS_URMA +static ssize_t xmfs_file_write_iter_slave(struct kiocb *iocb, + struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + unsigned long start_index = iocb->ki_pos / PAGE_SIZE; + unsigned long end_index; + unsigned long page_num; + ssize_t ret; + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + return ret; + + end_index = (iov_iter_count(from) + iocb->ki_pos + PAGE_SIZE - 1) / + PAGE_SIZE; + page_num = end_index - start_index; + + /* + * req.ops = XMFS_UPDATE; + * req.i_ino = inode->i_ino; + * req.data_pages = page_num; + * req.log_entries = 1; + * req.size = iocb->ki_pos + iov_iter_count(from); + * req.type = MSG_RW_REQUEST; + * req.slave_id = sbi->id; + * spin_lock(&sbi->local_ring_lock); + * req.request_id = ++sbi->rw_request_id; + * spin_unlock(&sbi->local_ring_lock); + * + * struct xmfs_grant_wait gw; + * init_completion(&gw.done); + * xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, + * GFP_KERNEL); + * err = xmfs_send_request(sbi, &req, &gw, 0); + * if (err) + * return err; + * grant = gw.grant; + * log_base = grant.log_trunk_addr + + * offsetof(struct xmfs_log_trunk, log[0]); + */ + + inode_lock(inode); + /* + * xmfs_inode->start_write = grant.data_addr; + * xmfs_inode->count = grant.data_page_count; + * xmfs_inode->second_write = grant.second_data_addr; + * xmfs_inode->second_write_count = 0; + */ + + /* + * xmfs_inode->wr = vzalloc(page_num * sizeof(struct ubcore_jfs_wr)); + * xmfs_inode->slots = vzalloc(sizeof(int) * page_num); + * xmfs_inode->write_buffer = urma_pool_acquire_nslots( + * &sbi->urma_cfg.pool, page_num, xmfs_inode->slots); + * xmfs_inode->wr_cnt = 0; + */ + + ret = generic_perform_write(iocb, from); + + struct xmfs_write_work *ww; + + ww = kmalloc(sizeof(*ww), GFP_KERNEL); + if (!ww) { + inode_unlock(inode); + return ret; + } + + ww->inode = inode; + ww->index = start_index; + ww->end_index = end_index; + ww->size = inode->i_size; + ihold(inode); + INIT_WORK(&ww->work, xmfs_write_worker); + if (xmfs_inode->write_wq) + queue_work(xmfs_inode->write_wq, &ww->work); + else + queue_work(sbi->prefetch_wq, &ww->work); + + /* + * for (s = 0; s < grant.log_slot_count; s++) { + * struct xmfs_log xm_log = {}; + * + * xm_log.ops = XMFS_UPDATE; + * xm_log.i_ino = inode->i_ino; + * xm_log.version = grant.version + s; + * xm_log.writer_id = sbi->id; + * if (s == 0) { + * xm_log.index = start_index; + * xm_log.end_index = + * start_index + grant.data_page_count; + * xm_log.data_offset = grant.data_addr; + * xm_log.size = req.size; + * } else { + * xm_log.index = start_index + grant.data_page_count; + * xm_log.end_index = end_index; + * xm_log.data_offset = grant.second_data_addr; + * xm_log.size = req.size; + * } + * xmfs_data_write( + * sbi, log_base + (grant.log_slot_start + s) * + * sizeof(struct xmfs_log), + * &xm_log, sizeof(xm_log)); + * + * pages = kzalloc((xm_log.end_index - xm_log.index) * + * sizeof(struct page *), GFP_KERNEL); + * for (int i = xm_log.index; i < xm_log.end_index; i++) { + * struct folio *folio = + * filemap_get_folio(inode->i_mapping, i); + * + * if (IS_ERR(folio)) { + * xm_log.end_index = i; + * break; + * } + * pages[i - xm_log.index] = &folio->page; + * } + * xmfs_pmem_write_multi_pages( + * sbi, xm_log.data_offset, pages, + * xm_log.end_index - xm_log.index); + * for (int i = xm_log.index; i < xm_log.end_index; i++) { + * struct folio *folio = + * page_folio(pages[i - xm_log.index]); + * + * folio_unlock(folio); + * folio_put(folio); + * } + * kfree(pages); + * } + */ + /* + * xmfs_urma_send_write_pages(sbi, xmfs_inode->wr); + * + * urma_pool_release_nslots(&sbi->urma_cfg.pool, page_num, + * xmfs_inode->slots); + * vfree(xmfs_inode->slots); + * xmfs_inode->start_write = 0; + * xmfs_inode->second_write = 0; + * xmfs_inode->count = 0; + * xmfs_inode->second_write_count = 0; + */ + inode_unlock(inode); + + return ret; +} + +static ssize_t xmfs_file_write_ubuf_slave(struct kiocb *iocb, + struct iov_iter *from) +{ + /* xmfs_info(sbi->sb, "qyf start write ubuf"); */ + struct inode *inode = file_inode(iocb->ki_filp); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_rw_msg req = {}, grant = {}; + unsigned long start_index = iocb->ki_pos / PAGE_SIZE; + unsigned long end_index; + unsigned long page_num; + unsigned long log_base; + int err, s; + ssize_t ret; + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + return ret; + + end_index = (iov_iter_count(from) + iocb->ki_pos + PAGE_SIZE - 1) / + PAGE_SIZE; + page_num = end_index - start_index; + + req.ops = XMFS_UPDATE; + req.i_ino = inode->i_ino; + req.data_pages = page_num; + req.log_entries = 1; + req.size = iocb->ki_pos + iov_iter_count(from); + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + log_base = + grant.log_trunk_addr + offsetof(struct xmfs_log_trunk, log[0]); + + inode_lock(inode); + xmfs_inode->start_write = grant.data_addr; + xmfs_inode->count = grant.data_page_count; + xmfs_inode->second_write = grant.second_data_addr; + xmfs_inode->second_write_count = 0; + /* xmfs_info(sbi->sb, "qyf start write ubuf data"); */ + xmfs_urma_write_ubuf(sbi, from->ubuf, grant.data_page_count * PAGE_SIZE, + grant.data_addr); + if (grant.log_slot_count > 1 && grant.second_data_addr != 0 && + grant.second_data_pages != 0) { + xmfs_urma_write_ubuf(sbi, + from->ubuf + + grant.data_page_count * PAGE_SIZE, + iov_iter_count(from) - + grant.data_page_count * PAGE_SIZE, + grant.second_data_addr); + } + /* xmfs_info(sbi->sb, "qyf start end ubuf data"); */ + for (s = 0; s < grant.log_slot_count; s++) { + struct xmfs_log xm_log = {}; + + xm_log.ops = XMFS_UPDATE; + xm_log.i_ino = inode->i_ino; + xm_log.version = grant.version + s; + xm_log.writer_id = sbi->id; + if (s == 0) { + xm_log.index = start_index; + xm_log.end_index = start_index + grant.data_page_count; + xm_log.data_offset = grant.data_addr; + xm_log.size = req.size; + int chunk_header_cnt = 0; + + for (int i = 0; i < grant.data_page_count; i++) { + unsigned long offset = + grant.data_addr + i * PAGE_SIZE + + chunk_header_cnt * PAGE_SIZE; + if (offset % XMFS_DEFAULT_CHUNK_SIZE == 0) { + chunk_header_cnt++; + offset = grant.data_addr + + i * PAGE_SIZE + + chunk_header_cnt * PAGE_SIZE; + } + xa_store(&(XMFS_I(inode)->shared_pages), + i + start_index, (void *)offset, + GFP_KERNEL); + } + } else { + xm_log.index = start_index + grant.data_page_count; + xm_log.end_index = end_index; + xm_log.data_offset = grant.second_data_addr; + xm_log.size = req.size; + int chunk_header_cnt = 0; + + for (int i = 0; i < grant.second_data_pages; i++) { + unsigned long offset = + grant.second_data_addr + i * PAGE_SIZE + + chunk_header_cnt * PAGE_SIZE; + if (offset % XMFS_DEFAULT_CHUNK_SIZE == 0) { + chunk_header_cnt++; + offset = grant.second_data_addr + + i * PAGE_SIZE + + chunk_header_cnt * PAGE_SIZE; + } + xa_store(&(XMFS_I(inode)->shared_pages), + i + start_index + + grant.data_page_count, + (void *)offset, GFP_KERNEL); + } + } + xmfs_data_write(sbi, + log_base + (grant.log_slot_start + s) * + sizeof(struct xmfs_log), + &xm_log, sizeof(xm_log)); + } + /* xmfs_info(sbi->sb, "qyf start write ubuf log"); */ + iocb->ki_pos += iov_iter_count(from); + if (i_size_read(inode) < iocb->ki_pos) + i_size_write(inode, iocb->ki_pos); + inode_unlock(inode); + + /* + * struct xmfs_cache_work *cw; + * + * cw = kmalloc(sizeof(*cw), GFP_KERNEL); + * if (!cw) + * return ret; + * + * cw->inode = inode; + * cw->index = start_index; + * cw->end_index = end_index; + * INIT_WORK(&cw->work, xmfs_prefetch_worker); + * queue_work(sbi->prefetch_wq, &cw->work); + */ + /* xmfs_info(sbi->sb, "qyf end write ubuf"); */ + return ret; +} +#endif + +static loff_t xmfs_seek_block(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + loff_t maxbytes = inode->i_sb->s_maxbytes; + pgoff_t pgofs, preofs; + loff_t data_ofs = offset; + loff_t isize; + + inode_lock_shared(inode); + + isize = i_size_read(inode); + + if (offset >= isize) { + inode_unlock_shared(inode); + return -ENXIO; + } + + pgofs = (pgoff_t)(offset >> PAGE_SHIFT); + preofs = pgofs; + unsigned long i; + void *ptr; + + if (whence == SEEK_DATA) { + bool found = false; + + xa_for_each_start(&xmfs_inode->shared_pages, i, ptr, pgofs) { + if (i == pgofs) { + /* + * 场景 A：offset 刚好落在一个包含数据的页内， + * 直接返回原 offset。 + */ + data_ofs = offset; + } else { + /* + * 场景 B：offset 落在空洞内，下一个数据页在 i， + * 返回新数据页的起始地址。 + */ + data_ofs = (loff_t)i << PAGE_SHIFT; + } + found = true; + /* 找到第一个数据块即可，退出循环。 */ + break; + } + if (!found) { + /* 之后没有数据了。 */ + inode_unlock_shared(inode); + return -ENXIO; + } + } else if (whence == SEEK_HOLE) { + unsigned long expected = pgofs; + bool in_hole = true; + + xa_for_each_start(&xmfs_inode->shared_pages, i, ptr, pgofs) { + if (i > expected) { + /* + * 发现断层：本来期望页号是 expected，但拿到的 i + * 却比它大，说明 expected 处存在一个洞， + * 跳出循环。 + */ + break; + } + /* + * 如果走到这里，说明 i == expected，该页有数据。 + * 既然第一页有数据，说明 offset 不在洞里。 + */ + in_hole = false; + /* 期望下一页。 */ + expected++; + } + if (in_hole) { + /* + * 场景 A：offset 所在的起始页没有数据（或者没有后续 + * 数据），所以 offset 本身就在空洞里， + * 直接返回原 offset。 + */ + data_ofs = offset; + } else { + /* + * 场景 B：遇到连续的数据页， + * 在期望页 expected 发生断层， + * 返回断层起始地址。 + */ + data_ofs = (loff_t)expected << PAGE_SHIFT; + } + /* + * 场景 C：隐式文件尾空洞。如果算出的洞偏移超过文件大小， + * 则取文件大小。 + */ + if (data_ofs > isize) + data_ofs = isize; + } else { + inode_unlock_shared(inode); + return -EINVAL; + } + inode_unlock_shared(inode); + return vfs_setpos(file, data_ofs, maxbytes); +} + +static loff_t xmfs_file_llseek(struct file *file, loff_t offset, int whence) +{ + loff_t size = i_size_read(file->f_mapping->host); + + switch (whence) { + case SEEK_END: + case SEEK_CUR: + case SEEK_SET: + return generic_file_llseek(file, offset, whence); + case SEEK_DATA: + case SEEK_HOLE: + if (offset < 0 || offset >= size) + return -ENXIO; + return xmfs_seek_block(file, offset, whence); + } + + return -EINVAL; +} + +static vm_fault_t xmfs_fault(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + vm_fault_t ret = 0; + unsigned long offset = vmf->pgoff << PAGE_SHIFT; + pgoff_t index = vmf->pgoff; + unsigned long mapped_offset; + int err; + + if (offset >= i_size_read(inode)) + return VM_FAULT_SIGBUS; + + filemap_invalidate_lock_shared(inode->i_mapping); + + struct folio *folio = __filemap_get_folio(inode->i_mapping, index, + FGP_CREAT | FGP_FOR_MMAP, + vmf->gfp_mask); + if (IS_ERR(folio)) { + filemap_invalidate_unlock_shared(inode->i_mapping); + return vmf_error(PTR_ERR(folio)); + } + + folio_lock(folio); + + void *kaddr = kmap_local_page(folio_file_page(folio, index)); + + mapped_offset = + (unsigned long)xa_load(&xmfs_inode->shared_pages, index); + if (mapped_offset) { + err = get_data_and_copy(sbi, mapped_offset, kaddr, PAGE_SIZE, NULL, + 0); + if (unlikely(err)) { + kunmap_local(kaddr); + folio_unlock(folio); + folio_put(folio); + filemap_invalidate_unlock_shared(inode->i_mapping); + return VM_FAULT_SIGBUS; + } + } else { + memset(kaddr, 0, PAGE_SIZE); + } + kunmap_local(kaddr); + folio_mark_uptodate(folio); + + vmf->page = folio_file_page(folio, index); + filemap_invalidate_unlock_shared(inode->i_mapping); + ret |= VM_FAULT_LOCKED; + return ret; +} + +static vm_fault_t xmfs_slave_fault(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + vm_fault_t ret = 0; + unsigned long offset = vmf->pgoff << PAGE_SHIFT; + pgoff_t index = vmf->pgoff; + struct page_freq *page_freq; + + if (offset >= i_size_read(inode)) { + xmfs_info(sbi->sb, "offset is larger than size"); + return VM_FAULT_SIGBUS; + } + if (!sbi->always_hot) { + page_freq = (struct page_freq *)xa_load(&xmfs_inode->page_freq, + index); + if (!page_freq) { + page_freq = + kzalloc(sizeof(struct page_freq), GFP_KERNEL); + if (!page_freq) { + xmfs_info(sbi->sb, "fault page freq no"); + return VM_FAULT_SIGBUS; + } + void *store_ret = xa_store(&xmfs_inode->page_freq, + index, (void *)page_freq, + GFP_KERNEL); + if (xa_err(store_ret)) { + xmfs_error(sbi->sb, + "cannot store shared pages for page %lu", + index); + } + } + page_freq->in_cache = true; + page_freq->never_in_cache = false; + } + filemap_invalidate_lock_shared(inode->i_mapping); + struct folio *folio = __filemap_get_folio(inode->i_mapping, index, + FGP_CREAT | FGP_FOR_MMAP, + vmf->gfp_mask); + if (IS_ERR(folio)) { + filemap_invalidate_unlock_shared(inode->i_mapping); + xmfs_info(sbi->sb, "fault oom %ld %d", PTR_ERR(folio), + vmf_error(PTR_ERR(folio))); + return vmf_error(PTR_ERR(folio)); + } + folio_lock(folio); + + void *kaddr = kmap_local_page(folio_file_page(folio, index)); + unsigned long mapped_offset = + (unsigned long)xa_load(&xmfs_inode->shared_pages, index); + int err = get_data_and_copy(sbi, mapped_offset, kaddr, PAGE_SIZE, NULL, + 0); + if (unlikely(err)) + xmfs_error(sbi->sb, + "hardware memory error when reading data during slave page fault"); + kunmap_local(kaddr); + folio_mark_uptodate(folio); + + vmf->page = folio_file_page(folio, index); + filemap_invalidate_unlock_shared(inode->i_mapping); + if (!sbi->always_hot) + page_freq->already_in_cache = true; + + ret |= VM_FAULT_LOCKED; + return ret; +} + +static vm_fault_t xmfs_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, + pgoff_t end_pgoff) +{ + vm_fault_t ret = filemap_map_pages(vmf, start_pgoff, end_pgoff); + return ret; +} + +static vm_fault_t xmfs_mkwrite(struct vm_fault *vmf) +{ + vm_fault_t ret = filemap_page_mkwrite(vmf); + return ret; +} + +static const struct vm_operations_struct xmfs_file_vm_ops = { + .fault = xmfs_fault, + .map_pages = xmfs_map_pages, + .page_mkwrite = xmfs_mkwrite, +}; + +static const struct vm_operations_struct xmfs_slave_file_vm_ops = { + .fault = xmfs_slave_fault, + .map_pages = xmfs_map_pages, + .page_mkwrite = xmfs_mkwrite, +}; + +static int xmfs_file_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct xmfs_sb_info *sbi = XMFS_SB(file_inode(file)->i_sb); + + file_accessed(file); + if (sbi->id) + vma->vm_ops = &xmfs_slave_file_vm_ops; + else + vma->vm_ops = &xmfs_file_vm_ops; + return 0; +} + +static int xmfs_slave_writepage(struct page *page, + struct writeback_control *wbc) +{ + struct address_space *mapping = page->mapping; + struct inode *inode = mapping->host; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + unsigned long mapped_offset; + + if (!(inode->i_sb->s_flags & SB_ACTIVE)) { + unlock_page(page); + return 0; + } + + mapped_offset = + (unsigned long)xa_load(&xmfs_inode->shared_pages, page->index); + if (mapped_offset) { + unlock_page(page); + return 0; + } + { + struct xmfs_rw_msg req = {}, grant = {}; + unsigned long log_off; + int err; + + req.ops = XMFS_UPDATE; + req.i_ino = inode->i_ino; + req.data_pages = 1; + req.log_entries = 1; + req.size = i_size_read(inode); + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, + GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) { + if (!(inode->i_sb->s_flags & SB_ACTIVE)) { + unlock_page(page); + return 0; + } + redirty_page_for_writepage(wbc, page); + unlock_page(page); + return err; + } + grant = gw.grant; + if (grant.type == MSG_RW_ERROR) { + if (!(inode->i_sb->s_flags & SB_ACTIVE)) { + unlock_page(page); + return 0; + } + redirty_page_for_writepage(wbc, page); + unlock_page(page); + return -ENOSPC; + } + { + char *kaddr = kmap_local_page(page); + + xmfs_data_write(sbi, grant.data_addr, kaddr, PAGE_SIZE); + kunmap_local(kaddr); + } + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + { + struct xmfs_log xm_log = {}; + + xm_log.index = page->index; + xm_log.end_index = page->index + 1; + xm_log.ops = XMFS_UPDATE; + xm_log.size = i_size_read(inode); + xm_log.data_offset = grant.data_addr; + xm_log.version = grant.version; + xm_log.writer_id = sbi->id; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + } + /* xmfs_mark_page_live(sbi, grant.data_addr); */ + { + void *old_ret = + xa_store(&xmfs_inode->shared_pages, page->index, + (void *)grant.data_addr, GFP_KERNEL); + /* xmfs_mark_page_dead(sbi, (unsigned long)old_ret); */ + if (xa_err(old_ret)) + xmfs_error(sbi->sb, + "slave writepage cannot store shared pages for page %ld", + page->index); + } + unlock_page(page); + return 0; + } +} + +static int xmfs_writepage(struct page *page, struct writeback_control *wbc) +{ + struct address_space *mapping = page->mapping; + struct inode *inode = mapping->host; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (sbi->id != 0) + return xmfs_slave_writepage(page, wbc); + + if (sbi->mode == XMFS_URMA_MODE) { + redirty_page_for_writepage(wbc, page); + unlock_page(page); + return 0; + } + + if (inode_is_locked(inode)) { + redirty_page_for_writepage(wbc, page); + unlock_page(page); + return 0; + } + if (!inode || !XMFS_I(inode)) { + xmfs_info(sbi->sb, "no inode now for write page"); + unlock_page(page); + return -EINVAL; + } + struct xmfs_log xm_log; + + if (sbi == NULL || sbi->kaddr == NULL) { + unlock_page(page); + return 0; + } + unsigned long xmlt_addr; + + if (!spin_trylock(&sbi->space_lock)) { + redirty_page_for_writepage(wbc, page); + unlock_page(page); + return -EAGAIN; + } + if (XMFS_I(inode)->die) { + xmfs_info(sbi->sb, + "inode logs have been interrupted before writepage"); + spin_unlock(&sbi->space_lock); + unlock_page(page); + return -EINVAL; + } + + if (XMFS_I(inode)->fallocate && + page->index > XMFS_I(inode)->data_size / PAGE_SIZE) { + xmfs_error(sbi->sb, + "writepage do not support write more than fallocate size"); + spin_unlock(&sbi->space_lock); + unlock_page(page); + return -EOPNOTSUPP; + } + + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + + if (!xmsi) { + spin_unlock(&sbi->space_lock); + unlock_page(page); + return -EINVAL; + } + unsigned long data_current = xmsi->data_current; + unsigned long data_count = xmsi->data_count; + unsigned long data_offset; + + int logcount = 1; + int need_data_trunk = (data_count == 511 || data_current == 0) ? 1 : 0; + + if (logcount + xmsi->used_trunk_count + need_data_trunk > + sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + unlock_page(page); + return -ENOSPC; + } + if (data_current == 0 || data_count == 511) { + long slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'd'); + + if (slot < 0) { + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + unlock_page(page); + return -ENOSPC; + } + data_offset = + DATA_AREA_OFFSET + slot * sbi->trunk_size + PAGE_SIZE; + xmsi->data_current = data_offset; + xmsi->data_count = 0; + } else { + data_offset = data_current + PAGE_SIZE; + xmsi->data_current = data_offset; + xmsi->data_count = data_count + 1; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + + loff_t pos = (xm_log.index + 1) * PAGE_SIZE; + + xm_log.index = page->index; + xm_log.end_index = xm_log.index + 1; + xm_log.ops = XMFS_UPDATE; + xm_log.p_ino = 0; + xm_log.size = min(i_size_read(inode), pos); + xm_log.offset = xm_log.index * PAGE_SIZE; + xm_log.writer_id = sbi->id; + if (XMFS_I(inode)->fallocate) { + xm_log.data_offset = + xmfs_get_data_offset_fallocate(inode, sbi, page); + } else { + char *kaddr = kmap_local_page(page); + + xmfs_data_write(sbi, data_offset, kaddr, PAGE_SIZE); + arch_invalidate_pmem(sbi->kaddr + data_offset, PAGE_SIZE); + kunmap_local(kaddr); + xm_log.data_offset = data_offset; + } + + void *old_ret = xa_store(&(XMFS_I(inode)->shared_pages), page->index, + (void *)xm_log.data_offset, GFP_KERNEL); + if (xa_err(old_ret)) { + xmfs_error(sbi->sb, "wb cannot store shared pages for page %ld", + page->index); + } + xmfs_mark_page_dead(sbi, (unsigned long)old_ret); + xmfs_mark_page_live(sbi, xm_log.data_offset); + + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, 0, inode->i_ino, true); + if (xmlt_addr < 0) { + xmfs_info(sbi->sb, "writepage wrong"); + XMFS_I(inode)->die = true; + unlock_page(page); + return -EINVAL; + } + unlock_page(page); + + return 0; +} + +static int xmfs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + return write_inode_now(file->f_mapping->host, 1); +} + +static int xmfs_release(struct inode *inode, struct file *file) +{ + if (XMFS_I(inode)->tmp) { + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + struct xmfs_log xm_log = { 0 }; + unsigned long xmlt_addr; + + if (inode->__i_nlink > 1) + xm_log.index = -1; + else + xm_log.index = 0; + xm_log.end_index = -1; + xm_log.ops = XMFS_DELETE; + xm_log.data_offset = 0; + memcpy(xm_log.name, file->f_path.dentry->d_name.name, + min(strlen(file->f_path.dentry->d_name.name), + 255UL)); + xm_log.p_ino = parent_ino(file->f_path.dentry); + + struct xmfs_sb_index *xmsi; + + spin_lock(&sbi->space_lock); + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + if (XMFS_I(inode)->die) { + xmfs_info(sbi->sb, + "inode logs have been interrupted before close tmpfile"); + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -EINVAL; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, + min(strlen(file->f_path.dentry->d_name.name), + 255UL), + inode->i_ino, true); + if (xmlt_addr < 0) { + XMFS_I(inode)->die = true; + return -EINVAL; + } + } + return 0; +} + +long xmfs_common_fallocate(struct inode *inode, int mode, loff_t offset, + loff_t len) +{ + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + /* per-FS log: data_size/data_start now in inode_info */ + while (XMFS_I(inode)->fallocate && XMFS_I(inode)->data_size != 0) { + struct xmfs_log_trunk *xmlt_local = + kmalloc(sizeof(struct xmfs_log_trunk), GFP_KERNEL); + int err = get_log_trunk_and_copy(sbi, XMFS_I(inode)->log_start, + xmlt_local, false); + if (unlikely(err)) { + xmfs_error(sbi->sb, + "hardware memory error when reading xm log trunk during fallocate."); + kfree(xmlt_local); + break; + } + XMFS_I(inode)->fallocate = true; + /* data_start from inode_info */ + /* data_size from inode_info */ + kfree(xmlt_local); + return 0; + } + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + + if (!xmsi) + return 0; + long first_slot = -1, slot; + int ti; + unsigned long num_trunks = + (len + sbi->trunk_size - 1) / sbi->trunk_size; + if (xmsi->used_trunk_count + num_trunks > sbi->max_chunk) { + xmfs_info(sbi->sb, "fallocate more space %lu %lu than total", + (unsigned long)len, num_trunks); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + XMFS_I(inode)->fallocate = true; + for (ti = 0; ti < (int)num_trunks; ti++) { + slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'd'); + if (slot < 0) { + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + if (first_slot < 0) + first_slot = slot; + } + XMFS_I(inode)->data_start = + DATA_AREA_OFFSET + first_slot * sbi->trunk_size; + XMFS_I(inode)->data_size = len; + XMFS_I(inode)->data_start = XMFS_I(inode)->data_start; + XMFS_I(inode)->data_size = XMFS_I(inode)->data_size; + xmsi->data_current = DATA_AREA_OFFSET + first_slot * sbi->trunk_size + + num_trunks * sbi->trunk_size; + xmsi->data_count = 0; + + put_xmsi(sbi, xmsi); + + return 0; +} + +static long xmfs_file_fallocate(struct file *file, int mode, loff_t offset, + loff_t len) +{ + if ((mode & FALLOC_FL_KEEP_SIZE) == 0 || offset != 0) + return -EOPNOTSUPP; + + struct inode *inode = file_inode(file); + + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (!sbi->debug) + return -EOPNOTSUPP; + + spin_lock(&sbi->space_lock); + long err = xmfs_common_fallocate(inode, mode, offset, len); + + spin_unlock(&sbi->space_lock); + return err; +} + +static const struct address_space_operations xmfs_aops = { + .writepage = xmfs_writepage, + .write_begin = xmfs_write_begin, + .write_end = xmfs_write_end, + .dirty_folio = filemap_dirty_folio, + /* .free_folio = xmfs_free_folio, */ +}; + +static const struct file_operations xmfs_file_operations = { + .release = xmfs_release, + .read_iter = xmfs_file_read_iter, + .write_iter = xmfs_file_write_iter, + .splice_read = xmfs_file_splice_read, + .splice_write = iter_file_splice_write, + .mmap = xmfs_file_mmap, + .fsync = xmfs_fsync, + .llseek = xmfs_file_llseek, + .fallocate = xmfs_file_fallocate, +}; + +static const struct inode_operations xmfs_file_inode_operations = { + .getattr = xmfs_getattr, + .setattr = xmfs_setattr, + .listxattr = xmfs_listxattr, + .update_time = xmfs_update_time, +}; + +void xmfs_init_file_mapping(struct inode *inode) +{ + inode->i_mapping->a_ops = &xmfs_aops; +} + +void xmfs_init_file_inode(struct inode *inode) +{ + inode->i_op = &xmfs_file_inode_operations; + xmfs_init_file_mapping(inode); + inode->i_fop = &xmfs_file_operations; +} diff --git a/fs/xmfs/gc.c b/fs/xmfs/gc.c new file mode 100644 index 000000000000..7df7d8a1c6fe --- /dev/null +++ b/fs/xmfs/gc.c @@ -0,0 +1,953 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "xmfs_i.h" +#include <linux/dcache.h> +#include <linux/kthread.h> +#include <linux/delay.h> +#include <linux/ktime.h> +#include <linux/cacheflush.h> + +void xmfs_mark_page_dead(struct xmfs_sb_info *sbi, unsigned long data_offset) +{ + unsigned long trunk_offset; + + if (data_offset == 0 || data_offset < DATA_AREA_OFFSET) + return; + + trunk_offset = data_offset & ~(sbi->trunk_size - 1); + if (trunk_offset < DATA_AREA_OFFSET) + return; + + if (sbi->mode != XMFS_URMA_MODE) { + struct xmfs_data_trunk_header *header; + + header = (struct xmfs_data_trunk_header *)(sbi->kaddr + + trunk_offset); + if (header->magic != XMFS_DATA_TRUNK_MAGIC) + return; + if (header->live_page_count == 0) + return; + header->live_page_count--; + } else { + unsigned long magic_off = trunk_offset; + unsigned long magic_val; + unsigned long count_off = + trunk_offset + offsetof(struct xmfs_data_trunk_header, + live_page_count); + uint64_t old_count, new_count; + + xmfs_pmem_read(sbi, magic_off, &magic_val, sizeof(magic_val)); + if (magic_val != XMFS_DATA_TRUNK_MAGIC) + return; + + do { + xmfs_pmem_read(sbi, count_off, &old_count, + sizeof(old_count)); + new_count = old_count - 1; + } while (xmfs_pmem_cas(sbi, count_off, old_count, new_count) != + 0); + } +} +EXPORT_SYMBOL(xmfs_mark_page_dead); + +void xmfs_mark_page_live(struct xmfs_sb_info *sbi, unsigned long data_offset) +{ + unsigned long trunk_offset; + + if (data_offset == 0 || data_offset < DATA_AREA_OFFSET) + return; + + trunk_offset = data_offset & ~(sbi->trunk_size - 1); + if (trunk_offset < DATA_AREA_OFFSET) + return; + + if (sbi->mode != XMFS_URMA_MODE) { + struct xmfs_data_trunk_header *header; + + header = (void *)(sbi->kaddr + trunk_offset); + if (header->magic != XMFS_DATA_TRUNK_MAGIC) + return; + header->live_page_count++; + /* cmpxchg(&header->last_modify_version, (unsigned long)-1, 0); */ + } else { + unsigned long magic_off = trunk_offset; + unsigned long magic_val; + /* + * unsigned long lmv_off = trunk_offset + + * offsetof(struct xmfs_data_trunk_header, + * last_modify_version); + */ + unsigned long count_off = + trunk_offset + offsetof(struct xmfs_data_trunk_header, + live_page_count); + uint64_t old_count, new_count; + + xmfs_pmem_read(sbi, magic_off, &magic_val, sizeof(magic_val)); + if (magic_val != XMFS_DATA_TRUNK_MAGIC) + return; + + do { + xmfs_pmem_read(sbi, count_off, &old_count, + sizeof(old_count)); + new_count = old_count + 1; + } while (xmfs_pmem_cas(sbi, count_off, old_count, new_count) != + 0); + + /* xmfs_pmem_cas(sbi, lmv_off, (uint64_t)-1, 0); */ + } +} +EXPORT_SYMBOL(xmfs_mark_page_live); + +int xmfs_quiesce_wait(struct xmfs_sb_info *sbi) +{ + int ret; + + ret = wait_event_interruptible(sbi->gc_ctl.wait, + !READ_ONCE(sbi->gc_ctl.gc_running)); + + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL(xmfs_quiesce_wait); + +long xmfs_find_free_slot(struct xmfs_sb_index *xmsi, struct xmfs_sb_info *sbi) +{ + unsigned long i; + + if (xmsi->count < sbi->max_chunk) + return xmsi->count; + + for (i = 0; i < xmsi->count && i < XMFS_BITMAP_CAPACITY; i++) { + if (xmsi->bitmap[i] == 'u') + return i; + } + + return -1; +} + +long xmfs_alloc_and_mark_slot(struct xmfs_sb_index *xmsi, + struct xmfs_sb_info *sbi, char type) +{ + long slot = xmfs_find_free_slot(xmsi, sbi); + + if (slot < 0) + return -1; + if (slot < (long)xmsi->count) + memset(sbi->kaddr + DATA_AREA_OFFSET + slot * sbi->trunk_size, + 0, sbi->trunk_size); + xmsi->bitmap[slot] = type; + if (type == 'd') { + struct xmfs_data_trunk_header *hdr; + + hdr = (void *)(sbi->kaddr + DATA_AREA_OFFSET + + slot * sbi->trunk_size); + hdr->magic = XMFS_DATA_TRUNK_MAGIC; + hdr->last_modify_version = 0; + hdr->live_page_count = 0; + } + if (slot == (long)xmsi->count) + xmsi->count = slot + 1; + xmsi->used_trunk_count++; + return slot; +} + +static void write_log_entry_fields(struct xmfs_log_trunk *xmlt, + struct xmfs_log *xm_log, unsigned long ino, + unsigned long version, int slot, + size_t namelen) +{ + xmlt->log[slot].index = xm_log->index; + xmlt->log[slot].end_index = xm_log->end_index; + xmlt->log[slot].ops = xm_log->ops; + xmlt->log[slot].data_offset = xm_log->data_offset; + xmlt->log[slot].mode = xm_log->mode; + xmlt->log[slot].size = xm_log->size; + xmlt->log[slot].i_ino = ino; + xmlt->log[slot].offset = xm_log->offset; + xmlt->log[slot].version = version; + xmlt->log[slot].writer_id = xm_log->writer_id; + + if (xm_log->ops != XMFS_UPDATE) { + memcpy(xmlt->log[slot].name, xm_log->name, namelen); + xmlt->log[slot].name[namelen] = '\0'; + xmlt->log[slot].p_ino = xm_log->p_ino; + } +} + +unsigned long xmfs_add_log_v3(struct xmfs_log *xm_log, struct xmfs_sb_info *sbi, + size_t namelen, unsigned long ino, + bool update_version) +{ + unsigned long version; + struct xmfs_sb_index *xmsi; + struct xmfs_log_trunk *xmlt; + unsigned long addr; + long slot; + + down_read(&sbi->chain_rwsem); + + spin_lock(&sbi->version_lock); + sbi->last_update++; + version = sbi->last_update; + spin_unlock(&sbi->version_lock); + + spin_lock(&sbi->space_lock); + + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + up_read(&sbi->chain_rwsem); + return (unsigned long)-1; + } + + xm_log->writer_id = 0; + + if (xmsi->log_current == 0 || xmsi->count == 0) { + slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'l'); + if (slot < 0) { + put_xmsi(sbi, xmsi); + spin_unlock(&sbi->space_lock); + up_read(&sbi->chain_rwsem); + return (unsigned long)-1; + } + addr = DATA_AREA_OFFSET + slot * sbi->trunk_size; + xmlt = get_log_trunk(sbi, addr); + xmlt->used = 0; + xmlt->next_trunk = 0; + xmlt->version_base = version; + xmlt->version_max = 0; + + xmsi->log_start = addr; + xmsi->log_current = addr; + } else { + xmlt = get_log_trunk(sbi, xmsi->log_current); + } + + if (xmlt->used >= MAX_LOG_PER_TRUNK) { + xmlt->version_max = version - 1; + + slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'l'); + if (slot < 0) { + put_xmsi(sbi, xmsi); + spin_unlock(&sbi->space_lock); + up_read(&sbi->chain_rwsem); + return (unsigned long)-1; + } + addr = DATA_AREA_OFFSET + slot * sbi->trunk_size; + xmlt->next_trunk = addr; + put_log_trunk(sbi, xmlt); + xmlt = get_log_trunk(sbi, addr); + xmlt->used = 0; + xmlt->next_trunk = 0; + xmlt->version_base = version; + xmlt->version_max = 0; + xmsi->log_current = addr; + } + + write_log_entry_fields(xmlt, xm_log, ino, version, xmlt->used, namelen); + xmlt->used++; + + addr = xmsi->log_current; + + if (xm_log->ops == XMFS_UPDATE) { + unsigned long trunk_off = xm_log->data_offset & + ~(sbi->trunk_size - 1); + struct xmfs_data_trunk_header *hdr = + (void *)(sbi->kaddr + trunk_off); + if (hdr->magic == XMFS_DATA_TRUNK_MAGIC && + version > hdr->last_modify_version) + hdr->last_modify_version = version; + } else if (xm_log->ops == XMFS_XATTR && + xm_log->data_offset >= DATA_AREA_OFFSET) { + unsigned long trunk_off = xm_log->data_offset & + ~(sbi->trunk_size - 1); + struct xmfs_data_trunk_header *hdr = + (void *)(sbi->kaddr + trunk_off); + if (hdr->magic == XMFS_DATA_TRUNK_MAGIC && + version > hdr->last_modify_version) + hdr->last_modify_version = version; + } else if (xm_log->ops == XMFS_SYMLINK && + xm_log->end_index >= DATA_AREA_OFFSET) { + unsigned long trunk_off = xm_log->end_index & + ~(sbi->trunk_size - 1); + struct xmfs_data_trunk_header *hdr = + (void *)(sbi->kaddr + trunk_off); + if (hdr->magic == XMFS_DATA_TRUNK_MAGIC && + version > hdr->last_modify_version) + hdr->last_modify_version = version; + } + + if (update_version) { + spin_lock(&sbi->version_lock); + if (version > xmsi->last_update) + xmsi->last_update = version; + spin_unlock(&sbi->version_lock); + } + + put_log_trunk(sbi, xmlt); + put_xmsi(sbi, xmsi); + spin_unlock(&sbi->space_lock); + up_read(&sbi->chain_rwsem); + + return addr + offsetof(struct xmfs_log_trunk, log[0]) + + (xmlt->used - 1) * sizeof(struct xmfs_log); +} + +static void gc_tail_compact(struct xmfs_sb_index *xmsi) +{ + while (xmsi->count > 0 && xmsi->count <= XMFS_BITMAP_CAPACITY && + xmsi->bitmap[xmsi->count - 1] == 'u') + xmsi->count--; +} + +static unsigned long gc_reclaim_log_trunks(struct xmfs_sb_info *sbi, + unsigned long safe_version) +{ + struct xmfs_sb_index *xmsi; + struct xmfs_log_trunk *xmlt; + unsigned long trunk_offset; + unsigned long next_trunk_offset; + unsigned long max_ver_reclaimed = 0; + unsigned long reclaimed = 0; + unsigned long first_surviving_trunk = 0; + + xmsi = get_xmsi(sbi); + if (!xmsi) + return 0; + if (safe_version == 0) { + put_xmsi(sbi, xmsi); + return 0; + } + + trunk_offset = xmsi->log_start; + + while (trunk_offset != 0) { + xmlt = get_log_trunk(sbi, trunk_offset); + if (!xmlt) + break; + + if (xmlt->version_max == 0) { + if (first_surviving_trunk == 0) + first_surviving_trunk = trunk_offset; + put_log_trunk(sbi, xmlt); + break; + } + + if (xmlt->version_max > safe_version) { + if (first_surviving_trunk == 0) + first_surviving_trunk = trunk_offset; + put_log_trunk(sbi, xmlt); + break; + } + + if (xmlt->version_max > max_ver_reclaimed) + max_ver_reclaimed = xmlt->version_max; + + next_trunk_offset = xmlt->next_trunk; + put_log_trunk(sbi, xmlt); + + spin_lock(&sbi->space_lock); + { + unsigned long idx = (trunk_offset - DATA_AREA_OFFSET) / + sbi->trunk_size; + memset(sbi->kaddr + trunk_offset, 0, sbi->trunk_size); + if (idx < XMFS_BITMAP_CAPACITY) + xmsi->bitmap[idx] = 'u'; + if (xmsi->used_trunk_count > 0) + xmsi->used_trunk_count--; + sbi->full = false; + } + spin_unlock(&sbi->space_lock); + reclaimed++; + + trunk_offset = next_trunk_offset; + } + + if (first_surviving_trunk != 0) + xmsi->log_start = first_surviving_trunk; + xmfs_info(sbi->sb, "gc log ends, gc to %ld, release %ld", + first_surviving_trunk, reclaimed); + spin_lock(&sbi->space_lock); + while (xmsi->count > 0 && xmsi->count <= XMFS_BITMAP_CAPACITY && + xmsi->bitmap[xmsi->count - 1] == 'u') + xmsi->count--; + spin_unlock(&sbi->space_lock); + + put_xmsi(sbi, xmsi); + return max_ver_reclaimed; +} + +static void gc_scan_data_trunks(struct xmfs_sb_info *sbi, + unsigned long safe_version) +{ + struct xmfs_sb_index *xmsi; + struct xmfs_data_trunk_header *header; + unsigned long trunk_offset; + unsigned long trunk_index; + unsigned long batch_count = 0; + + xmsi = get_xmsi(sbi); + if (!xmsi) + return; + + for (trunk_index = sbi->gc_ctl.gc_cursor; + trunk_index < + sbi->max_chunk /*&& batch_count < sbi->gc_ctl.gc_batch*/; + trunk_index++) { + if (trunk_index < XMFS_BITMAP_CAPACITY && + xmsi->bitmap[trunk_index] != 'd') + continue; + + trunk_offset = DATA_AREA_OFFSET + trunk_index * sbi->trunk_size; + header = (struct xmfs_data_trunk_header *)(sbi->kaddr + + trunk_offset); + if (header->magic != XMFS_DATA_TRUNK_MAGIC) + continue; + + if (header->live_page_count > 0) + continue; + + if (header->last_modify_version > safe_version) + continue; + + xmfs_info(sbi->sb, + "gc: reclaiming data trunk[%lu] live_pages=%lu ver=%lu", + trunk_index, header->live_page_count, + header->last_modify_version); + spin_lock(&sbi->space_lock); + memset(sbi->kaddr + trunk_offset, 0, sbi->trunk_size); + if (trunk_index < XMFS_BITMAP_CAPACITY) + xmsi->bitmap[trunk_index] = 'u'; + if (xmsi->used_trunk_count > 0) + xmsi->used_trunk_count--; + sbi->full = false; + batch_count++; + spin_unlock(&sbi->space_lock); + } + + spin_lock(&sbi->space_lock); + sbi->gc_ctl.gc_cursor = trunk_index; + if (sbi->gc_ctl.gc_cursor >= sbi->max_chunk) + sbi->gc_ctl.gc_cursor = 0; + gc_tail_compact(xmsi); + spin_unlock(&sbi->space_lock); + + put_xmsi(sbi, xmsi); +} + +static unsigned long gc_compute_safe_version(struct xmfs_sb_info *sbi) +{ + struct mount_msg_info *mm_info; + unsigned long safe_version; + int i; + + safe_version = sbi->last_update; + + if (sbi->mode == XMFS_SHARED_MEM_MODE || + sbi->mode == XMFS_HYBRID_MODE) { + mm_info = sbi->kaddr + MOUNT_MSG_OFFSET; + } else { + mm_info = get_mnt_msg(sbi); + if (!mm_info) + return safe_version; + } + + for (i = 1; i <= sbi->max_slave_id && i < 64; i++) { + if (mm_info->msg[i] == 'D') { + unsigned long sv = mm_info->slave_versions[i]; + + if (sv < safe_version) + safe_version = sv; + } + } + + if (sbi->mode == XMFS_URMA_MODE) + put_mnt_msg(sbi, mm_info); + + xmfs_info(sbi->sb, + "gc: safe_version=%lu (last_update=%lu, max_slave=%d)", + safe_version, sbi->last_update, sbi->max_slave_id); + return safe_version; +} + +static void xmfs_gc_run(struct xmfs_sb_info *sbi) +{ + unsigned long safe_version; + struct mount_msg_info *mm_info; +#ifdef CONFIG_XMFS_FS_URMA + struct xmfs_rw_msg req; +#endif + bool all_caught_up; + int i; + + if (sbi->id != 0) + return; + + xmfs_replay_new_entries(sbi, NULL, -1); + + spin_lock(&sbi->gc_ctl.gc_lock); + if (sbi->gc_ctl.gc_running) { + spin_unlock(&sbi->gc_ctl.gc_lock); + return; + } + WRITE_ONCE(sbi->gc_ctl.gc_running, true); + spin_unlock(&sbi->gc_ctl.gc_lock); + + safe_version = gc_compute_safe_version(sbi); + + mm_info = get_mnt_msg(sbi); + + if (mm_info) { +#ifdef CONFIG_XMFS_FS_URMA + for (i = 1; i < 16; i++) { + if (sbi->urma_cfg.send_tjettys[i] == NULL) + continue; + req.ops = XMFS_GC; + req.status = 2; + xmfs_send_request(sbi, &req, NULL, i); + } +#endif + all_caught_up = true; + for (i = 1; i <= sbi->max_slave_id && i < 64; i++) { + if (mm_info->msg[i] == 'D' && + mm_info->slave_versions[i] < safe_version) { + all_caught_up = false; + break; + } + } + + if (!all_caught_up) { + xmfs_info(sbi->sb, + "gc: waiting for slaves to catch up to version %lu", + safe_version); + schedule_timeout_interruptible(10 * HZ); + mm_info = get_mnt_msg(sbi); + if (mm_info) { + all_caught_up = true; + for (i = 1; i <= sbi->max_slave_id && i < 64; + i++) { + if (mm_info->msg[i] == 'D' && + mm_info->slave_versions[i] < + safe_version) { + all_caught_up = false; + break; + } + } + put_mnt_msg(sbi, mm_info); + } + } + } + + down_write(&sbi->chain_rwsem); + if (all_caught_up) { +#ifdef CONFIG_XMFS_FS_URMA + for (i = 1; i < 16; i++) { + if (sbi->urma_cfg.send_tjettys[i] == NULL) + continue; + req.ops = XMFS_GC; + req.status = 1; + xmfs_send_request(sbi, &req, NULL, i); + } +#endif + gc_reclaim_log_trunks(sbi, safe_version); +#ifdef CONFIG_XMFS_FS_URMA + for (i = 1; i < 16; i++) { + if (sbi->urma_cfg.send_tjettys[i] == NULL) + continue; + req.ops = XMFS_GC; + req.status = 0; + xmfs_send_request(sbi, &req, NULL, i); + } +#endif + } + gc_scan_data_trunks(sbi, safe_version); + up_write(&sbi->chain_rwsem); + + spin_lock(&sbi->gc_ctl.gc_lock); + WRITE_ONCE(sbi->gc_ctl.gc_running, false); + wake_up_all(&sbi->gc_ctl.wait); + spin_unlock(&sbi->gc_ctl.gc_lock); +} + +int xmfs_gc_main(struct xmfs_sb_info *sbi) +{ + xmfs_gc_run(sbi); + return 0; +} + +int xmfs_gc_thread_fn(void *data) +{ + struct xmfs_sb_info *sbi = data; + struct xmfs_sb_index *xmsi; + unsigned long interval; + + interval = sbi->gc_interval ? sbi->gc_interval : + XMFS_GC_INTERVAL_DEFAULT; + + while (!kthread_should_stop()) { + xmsi = get_xmsi(sbi); + /* + * xmfs_info(sbi->sb, "gc %d %d %d", + * xmsi->used_trunk_count, + * sbi->max_chunk * 80 / 100, sbi->max_chunk); + */ + if (xmsi && + (xmsi->used_trunk_count >= sbi->max_chunk * 80 / 100 || + sbi->full)) + xmfs_gc_run(sbi); + if (xmsi) + put_xmsi(sbi, xmsi); + schedule_timeout_interruptible(interval * HZ); + } + + return 0; +} + +int xmfs_gc_thread_slave_fn(void *data) +{ + struct xmfs_sb_info *sbi = data; + unsigned long interval; + + interval = sbi->gc_interval ? sbi->gc_interval : + XMFS_GC_INTERVAL_DEFAULT; + + while (!kthread_should_stop()) { + if (sbi->gc_ctl.need_replay) { + xmfs_info(sbi->sb, "gc need to replay"); + xmfs_replay_new_entries(sbi, NULL, -1); + xmfs_info(sbi->sb, "gc need to replay done"); + sbi->gc_ctl.need_replay = false; + } + schedule_timeout_interruptible(interval * HZ); + } + + return 0; +} + +static int xmfs_host_reserve_data_space(struct xmfs_sb_info *sbi, + unsigned long page_count, + struct xmfs_data_reserve_result *result) +{ + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + unsigned long data_current, data_count; + long slot; + struct xmfs_data_trunk_header *hdr; + unsigned long remaining; + int trunk_idx; + bool first_partial = false; + + if (!xmsi) + return -ENOSPC; + + memset(result, 0, sizeof(*result)); + trunk_idx = 0; + + data_current = xmsi->data_current; + data_count = xmsi->data_count; + remaining = 511 - data_count; + if (remaining + 511 * (sbi->max_chunk - xmsi->count) < page_count) { + xmfs_info(sbi->sb, + "try to allocate more than usable space %lu > %lu", + page_count, + remaining + 511 * (sbi->max_chunk - xmsi->count)); + return -ENOSPC; + } + + if (data_current != 0 && remaining > 0) { + unsigned long use = min(page_count, remaining); + + hdr = (struct xmfs_data_trunk_header *)(sbi->kaddr + + (data_current & + ~(sbi->trunk_size - + 1))); + result->data_addr = data_current; + result->data_page_count = use; + result->trunk_addrs[trunk_idx] = data_current & + ~(sbi->trunk_size - 1); + trunk_idx++; + + xmsi->data_current = data_current + use * PAGE_SIZE; + xmsi->data_count = data_count + use; + hdr->total_page_count = data_count + use; + /* hdr->live_page_count += use; */ + + page_count -= use; + if (page_count == 0) { + put_xmsi(sbi, xmsi); + result->trunk_count = trunk_idx; + return 0; + } + first_partial = true; + } + + while (page_count > 0) { + unsigned long use; + + slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'd'); + if (slot < 0) { + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + data_current = DATA_AREA_OFFSET + slot * sbi->trunk_size; + hdr = (struct xmfs_data_trunk_header *)(sbi->kaddr + + data_current); + memset(hdr, 0, PAGE_SIZE); + hdr->magic = XMFS_DATA_TRUNK_MAGIC; + hdr->last_modify_version = 0; + + use = min_t(unsigned long, page_count, 511); + + if (!first_partial && trunk_idx == 0) { + result->data_addr = data_current + PAGE_SIZE; + result->data_page_count = use; + } else if (result->second_data_addr == 0) { + result->second_data_addr = data_current + PAGE_SIZE; + result->second_data_pages = use; + } else { + result->second_data_pages += use; + } + result->trunk_addrs[trunk_idx] = data_current; + trunk_idx++; + + hdr->total_page_count = use; + /* hdr->live_page_count = use; */ + + if (use < 511) { + xmsi->data_current = + data_current + PAGE_SIZE + use * PAGE_SIZE; + xmsi->data_count = use; + } else { + xmsi->data_current = 0; + xmsi->data_count = 511; + } + + page_count -= use; + } + + result->trunk_count = trunk_idx; + put_xmsi(sbi, xmsi); + return 0; +} + +static unsigned long xmfs_host_reserve_log_space(struct xmfs_sb_info *sbi, + unsigned long ino, + int entry_count) +{ + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + struct xmfs_log_trunk *xmlt; + unsigned long addr; + long slot; + + if (!xmsi) + return 0; + + xmlt = get_log_trunk(sbi, xmsi->log_current); + if (!xmlt || xmlt->used + entry_count > MAX_LOG_PER_TRUNK) { + if (xmlt) { + xmlt->version_max = sbi->last_update; + put_log_trunk(sbi, xmlt); + } + slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'l'); + if (slot < 0 || slot >= sbi->max_chunk) { + xmfs_info(sbi->sb, + "cannot get a correct slot %ld, maybe space is full", + slot); + put_xmsi(sbi, xmsi); + return 0; + } + addr = DATA_AREA_OFFSET + slot * sbi->trunk_size; + xmlt = get_log_trunk(sbi, addr); + xmlt->used = 0; + xmlt->next_trunk = 0; + xmlt->version_base = sbi->last_update + 1; + xmlt->version_max = 0; + put_log_trunk(sbi, xmlt); + { + struct xmfs_log_trunk *old = + get_log_trunk(sbi, xmsi->log_current); + if (old) { + old->next_trunk = addr; + put_log_trunk(sbi, old); + } + } + xmsi->log_current = addr; + sbi->log_current = + addr + offsetof(struct xmfs_log_trunk, log[0]); + xmlt = get_log_trunk(sbi, addr); + } + + addr = xmsi->log_current; + put_log_trunk(sbi, xmlt); + put_xmsi(sbi, xmsi); + return addr; +} + +void xmfs_host_handle_urma_request(struct xmfs_sb_info *sbi, int slave_i, + struct xmfs_rw_msg *req, + struct xmfs_rw_msg *grant) +{ + struct xmfs_log_trunk *xmlt; + struct xmfs_data_reserve_result dr; + unsigned long log_addr; + int log_needed; + int data_err; + + if (xmfs_quiesce_wait(sbi)) + return; + + if (req->ops == XMFS_SETATTR) { + spin_lock(&sbi->space_lock); + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long ci; + struct iattr *attr = (struct iattr *)req->name; + time64_t ts_sec = ktime_get_real_seconds(); + + for (ci = 0; ci < tbl->count; ci++) { + if (tbl->entries[ci].i_ino == req->i_ino) { + if (attr->ia_valid & ATTR_SIZE) + tbl->entries[ci].size = attr->ia_size; + tbl->entries[ci].ctime.tv_sec = ts_sec; + tbl->entries[ci].mtime.tv_sec = ts_sec; + tbl->entries[ci].atime.tv_sec = ts_sec; + break; + } + } + spin_unlock(&sbi->space_lock); + return; + } + + grant->type = MSG_RW_GRANT; + grant->request_id = req->request_id; + grant->slave_id = req->slave_id; + grant->ops = req->ops; + spin_lock(&sbi->space_lock); + if (req->data_pages > 0) { + data_err = + xmfs_host_reserve_data_space(sbi, req->data_pages, &dr); + if (data_err) { + spin_unlock(&sbi->space_lock); + grant->type = MSG_RW_ERROR; + return; + } + grant->data_addr = dr.data_addr; + grant->data_page_count = dr.data_page_count; + grant->second_data_addr = dr.second_data_addr; + grant->second_data_pages = dr.second_data_pages; + } else { + data_err = 0; + grant->data_addr = 0; + grant->data_page_count = 0; + } + + log_needed = req->log_entries; + if (req->ops == XMFS_UPDATE && grant->second_data_addr != 0) + log_needed = max(log_needed, 2); + if (req->ops == XMFS_RENAME) + log_needed = max(log_needed, 2); + if (req->ops == XMFS_SYMLINK) + log_needed = max(log_needed, 2); + + if (log_needed > 0 && slave_i != 0) { + log_addr = xmfs_host_reserve_log_space(sbi, req->i_ino, + log_needed); + if (log_addr == 0) { + spin_unlock(&sbi->space_lock); + grant->type = MSG_RW_ERROR; + return; + } + + xmlt = get_log_trunk(sbi, log_addr); + if (xmlt) { + grant->log_trunk_addr = log_addr; + grant->log_slot_start = xmlt->used; + grant->log_slot_count = log_needed; + xmlt->used += log_needed; + put_log_trunk(sbi, xmlt); + } + } else { + grant->log_trunk_addr = 0; + grant->log_slot_start = 0; + grant->log_slot_count = log_needed; + } + + if (req->i_ino == 0) + grant->assigned_ino = get_next_ino(); + else + grant->assigned_ino = req->i_ino; + + { + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + + if (xmsi) { + grant->version = xmsi->last_update + 1; + xmsi->last_update += max(log_needed, 1); + put_xmsi(sbi, xmsi); + } else { + grant->version = 1; + } + } + + if (req->ops == XMFS_CREATE || req->ops == XMFS_MKDIR || + req->ops == XMFS_SYMLINK || req->ops == XMFS_LINK) { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + struct timespec64 ts; + unsigned int nlink; + { + struct xmfs_inode_entry *e = &tbl->entries[tbl->count]; + + memset(e, 0, sizeof(*e)); + e->i_ino = grant->assigned_ino; + e->p_ino = req->p_ino; + e->mode = req->mode; + if (req->ops != XMFS_LINK) + e->nlink = 1; + memcpy(e->name, req->name, strnlen(req->name, 255)); + e->name[strnlen(req->name, 255)] = '\0'; + e->size = req->size; + ktime_get_real_ts64(&ts); + e->ctime = ts; + e->mtime = ts; + e->atime = ts; + } + tbl->count++; + if (req->ops == XMFS_LINK) { + nlink = xmfs_inode_table_sync_nlink(tbl, req->i_ino); + if (nlink == 1) + xmfs_warning(sbi->sb, + "LINK inode %lu not found in table", + req->i_ino); + } + } + + if (req->ops == XMFS_DELETE) { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + + xmfs_inode_table_remove(tbl, req->i_ino, req->p_ino, + req->name); + xmfs_inode_table_sync_nlink(tbl, req->i_ino); + } + + if (req->ops == XMFS_UPDATE) { + struct xmfs_inode_table *ctbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long ci; + /* xmfs_info(sbi->sb, "write update size %ld", req->size); */ + for (ci = 0; ci < ctbl->count; ci++) { + if (ctbl->entries[ci].i_ino == req->i_ino) { + /* + * xmfs_info(sbi->sb, + * "write find inode to update size %ld", + * req->size); + */ + ctbl->entries[ci].size = req->size; + break; + } + } + } + + spin_unlock(&sbi->space_lock); +} diff --git a/fs/xmfs/inode.c b/fs/xmfs/inode.c new file mode 100644 index 000000000000..9f066782545a --- /dev/null +++ b/fs/xmfs/inode.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "xmfs_i.h" + +#include <linux/kernel.h> +#include <linux/namei.h> +#include <linux/pagemap.h> +#include <linux/slab.h> + +int xmfs_inode_eq(struct inode *inode, void *_ino) +{ + u64 ino = *(u64 *)_ino; + + if (XMFS_I(inode)->i_ino == ino && XMFS_I(inode)->pinned && + !XMFS_I(inode)->die) + return 1; + else + return 0; +} + +int xmfs_inode_set(struct inode *inode, void *_ino) +{ + u64 ino = *(u64 *)_ino; + + XMFS_I(inode)->i_ino = ino; + inode->i_ino = ino; + XMFS_I(inode)->leaf = true; + XMFS_I(inode)->tmp = false; + XMFS_I(inode)->die = false; + XMFS_I(inode)->fallocate = false; + XMFS_I(inode)->pinned = false; + XMFS_I(inode)->start_write = 0; + XMFS_I(inode)->second_write = 0; + XMFS_I(inode)->count = 0; + xa_init(&XMFS_I(inode)->shared_pages); + xmfs_simple_xattrs_init(&XMFS_I(inode)->xattrs); + XMFS_I(inode)->write_wq = NULL; + + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (!sbi->always_hot) + xa_init(&XMFS_I(inode)->page_freq); + + sbi->used_ino++; + return 0; +} + +unsigned int xmfs_inode_table_sync_nlink(struct xmfs_inode_table *tbl, + unsigned long ino) +{ + unsigned long i; + unsigned int nlink = 0; + + for (i = 0; i < tbl->count; i++) { + if (tbl->entries[i].i_ino == ino) + nlink++; + } + for (i = 0; i < tbl->count; i++) { + if (tbl->entries[i].i_ino == ino) + tbl->entries[i].nlink = nlink; + } + + return nlink; +} + +bool xmfs_inode_table_remove(struct xmfs_inode_table *tbl, unsigned long ino, + unsigned long p_ino, const char *name) +{ + unsigned long i; + + for (i = 0; i < tbl->count; i++) { + struct xmfs_inode_entry *entry = &tbl->entries[i]; + + if (entry->i_ino != ino || entry->p_ino != p_ino || + strcmp(entry->name, name)) + continue; + + tbl->count--; + if (i < tbl->count) + tbl->entries[i] = tbl->entries[tbl->count]; + memset(&tbl->entries[tbl->count], 0, + sizeof(struct xmfs_inode_entry)); + return true; + } + + return false; +} + +int xmfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + void *ret; + struct inode *inode = dentry->d_inode; + int err; + + if (inode->i_ino == 0) + return simple_setattr(idmap, dentry, attr); + + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (sbi->id != 0) + return xmfs_slave_setattr(idmap, dentry, attr); + + err = setattr_prepare(idmap, dentry, attr); + if (err) { + xmfs_info(inode->i_sb, "set attr wrong %d %d %d", err, + attr->ia_mode, attr->ia_valid); + return err; + } + setattr_copy(idmap, d_inode(dentry), attr); + mark_inode_dirty(inode); + unsigned int ia_valid = attr->ia_valid; + + if (ia_valid & ATTR_SIZE) { + loff_t new_size = attr->ia_size; + + if (new_size > sbi->len) + return -EFBIG; + + if (i_size_read(inode) > new_size) { + for (unsigned long i = + (new_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + i <= (i_size_read(inode) >> PAGE_SHIFT); i++) { + ret = xa_erase(&xmfs_inode->shared_pages, i); + if (ret) + inode->i_blocks--; + + } + i_size_write(inode, new_size); + truncate_pagecache(inode, new_size); + + } else { + i_size_write(inode, new_size); + } + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long ci; + + for (ci = 0; ci < tbl->count; ci++) { + if (tbl->entries[ci].i_ino == inode->i_ino) { + tbl->entries[ci].size = + i_size_read(inode); + tbl->entries[ci].ctime = + inode_get_ctime(inode); + tbl->entries[ci].mtime = inode->i_mtime; + break; + } + } + } + } + + return err; +} + +int xmfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + + generic_fillattr(idmap, request_mask, inode, stat); + if (request_mask & STATX_BTIME) { + stat->result_mask |= STATX_BTIME; + stat->btime = XMFS_I(inode)->btime; + } + return 0; +} + +int xmfs_slave_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + + generic_fillattr(idmap, request_mask, inode, stat); + if (request_mask & STATX_BTIME) { + stat->result_mask |= STATX_BTIME; + stat->btime = XMFS_I(inode)->btime; + } + return 0; +} + +int xmfs_update_time(struct inode *inode, int flags) +{ + struct iattr attr = {}; + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_log xm_log; + + if (inode->i_ino == 0) + return 0; + + if (flags == S_ATIME) + return 0; + + if ((flags & (S_MTIME | S_CTIME | S_ATIME)) == 0) + return 0; + + generic_update_time(inode, flags); + return 0; + if (inode->i_ino == 0) + return 0; + + if (flags == S_ATIME) + return 0; + + int err; + + if ((flags & (S_MTIME | S_CTIME | S_ATIME)) == 0) + return err; + + if (flags & S_MTIME) { + attr.ia_mtime.tv_sec = inode->i_mtime.tv_sec; + attr.ia_mtime.tv_nsec = inode->i_mtime.tv_nsec; + attr.ia_valid |= ATTR_MTIME; + } + if (flags & S_CTIME) { + attr.ia_ctime.tv_sec = inode->__i_ctime.tv_sec; + attr.ia_ctime.tv_nsec = inode->__i_ctime.tv_nsec; + attr.ia_valid |= ATTR_CTIME; + } + if (flags & S_ATIME) { + attr.ia_atime.tv_sec = inode->i_atime.tv_sec; + attr.ia_atime.tv_nsec = inode->i_atime.tv_nsec; + attr.ia_valid |= ATTR_ATIME; + } + + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + if (xmfs_inode->die) { + xmfs_info(inode->i_sb, + "inode logs have been interrupted before update time"); + return -EINVAL; + } + + struct xmfs_sb_index *xmsi; + + spin_lock(&sbi->space_lock); + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + err = 0; + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + + xm_log.ops = XMFS_SETATTR; + memcpy(xm_log.name, &attr, sizeof(struct iattr)); + + unsigned long xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, + sizeof(struct iattr), + inode->i_ino, true); + if (xmlt_addr < 0) { + xmfs_inode->die = true; + return -EINVAL; + } + + return 0; +} + +static const struct inode_operations xmfs_special_inode_operations = { + .getattr = xmfs_getattr, + .setattr = xmfs_setattr, + .listxattr = xmfs_listxattr, + .update_time = xmfs_update_time, +}; + +void xmfs_init_special_inode_operations(struct inode *inode) +{ + inode->i_op = &xmfs_special_inode_operations; +} + +int xmfs_slave_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_rw_msg req = {}; + int err; + + err = setattr_prepare(idmap, dentry, attr); + if (err) + return err; + + req.ops = XMFS_SETATTR; + req.i_ino = inode->i_ino; + req.p_ino = 0; + req.data_pages = 0; + req.log_entries = 0; + memcpy(req.name, attr, sizeof(struct iattr)); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + err = xmfs_send_request(sbi, &req, NULL, 0); + if (err) + return err; + + setattr_copy(idmap, inode, attr); + + unsigned int ia_valid = attr->ia_valid; + + if (ia_valid & ATTR_SIZE) { + xmfs_info(inode->i_sb, "slave setattr size %lld", + (long long)attr->ia_size); + loff_t new_size = attr->ia_size; + + if (new_size > sbi->len) + return -EFBIG; + + if (i_size_read(inode) > new_size) { + for (unsigned long i = + (new_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + i <= (i_size_read(inode) >> PAGE_SHIFT); i++) { + xa_erase(&XMFS_I(inode)->shared_pages, i); + } + i_size_write(inode, new_size); + truncate_pagecache(inode, new_size); + } else { + i_size_write(inode, new_size); + } + } + return 0; +} diff --git a/fs/xmfs/namei.c b/fs/xmfs/namei.c new file mode 100644 index 000000000000..7c725bc9b654 --- /dev/null +++ b/fs/xmfs/namei.c @@ -0,0 +1,1794 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "xmfs_i.h" + +#include <linux/file.h> +#include <linux/kernel.h> +#include <linux/namei.h> +#include <linux/pagemap.h> +#include <linux/security.h> +#include <linux/slab.h> + +static const struct inode_operations xmfs_host_dir_inode_operations; +static const struct inode_operations xmfs_dir_inode_operations; + +static int xmfs_do_create_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, umode_t mode); +static int xmfs_do_mkdir_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, umode_t mode); +static int xmfs_do_rename_slave(struct xmfs_sb_info *sbi, struct inode *old_dir, + struct dentry *old_dentry, + struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags); +static int xmfs_do_mknod_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev); +static int xmfs_do_unlink_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry); +static int xmfs_do_symlink_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, const char *symname); +static int xmfs_do_link_slave(struct xmfs_sb_info *sbi, + struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry); + +static void xmfs_unpin_inode(struct xmfs_sb_info *sbi, struct inode *inode) +{ + if (!XMFS_I(inode)->pinned) + return; + + XMFS_I(inode)->pinned = false; + xa_erase(&sbi->pinned_inodes, inode->i_ino); + iput(inode); +} + +static unsigned long xmfs_symlink_write_symname(struct inode *inode, + const char *name, + struct xmfs_log *xm_log) +{ + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_sb_index *xmsi; + unsigned long data_current, data_count; + long slot; + int err = 0; + + spin_lock(&sbi->space_lock); + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + + data_current = xmsi->data_current; + data_count = xmsi->data_count; + + WARN_ON(xmsi->count == 0); + + if (data_current == 0 || data_count == 511) { + slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'd'); + if (slot < 0) { + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + xmsi->data_current = + DATA_AREA_OFFSET + slot * sbi->trunk_size + PAGE_SIZE; + xmsi->data_count = 0; + } else { + xmsi->data_current = data_current + PAGE_SIZE; + } + + xmfs_data_write(sbi, xmsi->data_current, name, + min(strlen(name) + 1, PATH_MAX)); + xmfs_mark_page_live(sbi, xmsi->data_current); + + xmsi->data_count = xmsi->data_count + 1; + xm_log->end_index = xmsi->data_current; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return err; +} + +static int xmfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (sbi->id != 0) + return xmfs_do_symlink_slave(sbi, dir, dentry, symname); + struct xmfs_inode_info *xmfs_inode; + unsigned int len = strlen(symname) + 1; + + if (len > PAGE_SIZE) + return -ENAMETOOLONG; + + const char *name = dentry->d_name.name; + struct inode *inode; + struct xmfs_log xm_log; + unsigned long xmlt_addr; + + spin_lock(&sbi->space_lock); + if (sbi->full) { + spin_unlock(&sbi->space_lock); + return -ENOSPC; + } + + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int err = 0; + + int logcount = 1; + int need_data_trunk = + (xmsi->data_count == 511 || xmsi->data_current == 0) ? 1 : 0; + if (len >= 256 && xmsi->used_trunk_count + logcount + need_data_trunk > + sbi->max_chunk) { + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } else if (len < 256 && + xmsi->used_trunk_count + logcount > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + + unsigned long ino = get_next_ino(); + + inode = iget5_locked(dir->i_sb, ino, xmfs_inode_eq, xmfs_inode_set, + &ino); + if (!inode) + return -ENOMEM; + + xmfs_inode = XMFS_I(inode); + inode->i_ino = ino; + + i_size_write(inode, 0); + inode_init_owner(idmap, inode, dir, S_IFLNK | 0777); + + xmfs_init_symlink_operations(inode); + err = security_inode_init_security(inode, dir, &dentry->d_name, + xmfs_initxattrs, NULL); + if (err) { + unlock_new_inode(inode); + iput(inode); + return err; + } + inode_nohighmem(inode); + xmfs_init_symlink_mapping(inode); + + xmfs_inode->btime = inode->i_atime = inode->i_mtime = + inode_set_ctime_current(inode); + dir->i_mtime = inode_set_ctime_current(dir); + unlock_new_inode(inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + xm_log.index = 0; + xm_log.end_index = 0; + xm_log.ops = XMFS_SYMLINK; + xm_log.data_offset = xmfs_inode->btime.tv_sec; + xm_log.mode = S_IFLNK | 0777; + memcpy(xm_log.name, name, min(strlen(name), 255)); + xm_log.p_ino = parent_ino(dentry); + + struct xmfs_log xm_log1; + + xm_log1.index = 1; + xm_log1.ops = XMFS_SYMLINK; + xm_log1.data_offset = xmfs_inode->btime.tv_nsec; + xm_log1.mode = S_IFLNK | 0777; + xm_log1.p_ino = parent_ino(dentry); + + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, min(strlen(name), 255), + inode->i_ino, false); + if (xmlt_addr < 0) { + xmfs_inode->die = true; + clear_nlink(inode); + iput(inode); + return -EINVAL; + } + + if (len <= 255) { + memcpy(xm_log1.name, symname, min(strlen(symname), 255)); + xm_log1.name[min(strlen(symname), 255)] = '\0'; + xm_log1.end_index = 0; + xmlt_addr = xmfs_add_log_v3(&xm_log1, sbi, + min(strlen(symname), 255), + inode->i_ino, true); + } else { + err = xmfs_symlink_write_symname(inode, symname, &xm_log1); + xm_log1.index = len; + if (err) { + xmfs_inode->die = true; + clear_nlink(inode); + iput(inode); + return err; + } + xmlt_addr = + xmfs_add_log_v3(&xm_log1, sbi, 0, inode->i_ino, true); + } + if (xmlt_addr < 0) { + xmfs_inode->die = true; + clear_nlink(inode); + iput(inode); + return -EINVAL; + } + xmfs_inode->symname = xmlt_addr; + + err = page_symlink(inode, symname, len); + if (err) { + clear_nlink(inode); + iput(inode); + return err; + } + + d_instantiate(dentry, inode); + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long slot = tbl->count; + + tbl->count++; + { + struct xmfs_inode_entry *e = &tbl->entries[slot]; + + memset(e, 0, sizeof(*e)); + e->i_ino = inode->i_ino; + e->p_ino = dir->i_ino; + e->mode = S_IFLNK | 0777; + e->nlink = 1; + e->size = 0; + e->ctime = inode_get_ctime(inode); + e->mtime = inode->i_mtime; + e->atime = inode->i_atime; + memcpy(e->name, dentry->d_name.name, + dentry->d_name.len); + if (dentry->d_name.len < 256) + e->name[dentry->d_name.len] = '\0'; + } + } + + return 0; +} + +static int xmfs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (sbi->id != 0) + return xmfs_do_link_slave(sbi, old_dentry, dir, dentry); + const char *name = dentry->d_name.name; + struct inode *inode = d_inode(old_dentry); + struct xmfs_log xm_log; + unsigned long xmlt_addr; + + if (XMFS_I(inode)->die) { + xmfs_info(sbi->sb, + "inode logs have been interrupted before link"); + return -EINVAL; + } + struct xmfs_sb_index *xmsi; + + spin_lock(&sbi->space_lock); + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + + dir->i_mtime = + inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode)); + inc_nlink(inode); + ihold(inode); + + d_instantiate(dentry, inode); + + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long slot; + + spin_lock(&sbi->space_lock); + slot = tbl->count; + tbl->count++; + { + struct xmfs_inode_entry *e = &tbl->entries[slot]; + + memset(e, 0, sizeof(*e)); + e->i_ino = inode->i_ino; + e->p_ino = dir->i_ino; + e->mode = inode->i_mode; + e->nlink = inode->i_nlink; + e->size = i_size_read(inode); + e->ctime = inode_get_ctime(inode); + e->mtime = inode->i_mtime; + e->atime = inode->i_atime; + memcpy(e->name, dentry->d_name.name, + dentry->d_name.len); + if (dentry->d_name.len < 256) + e->name[dentry->d_name.len] = '\0'; + } + xmfs_inode_table_sync_nlink(tbl, inode->i_ino); + spin_unlock(&sbi->space_lock); + } + + xm_log.index = 0; + xm_log.end_index = inode->i_ino; + xm_log.ops = XMFS_LINK; + xm_log.data_offset = dir->i_mtime.tv_sec; + xm_log.mode = inode->i_mode; + xm_log.offset = dir->i_mtime.tv_nsec; + memcpy(xm_log.name, name, min(strlen(name), 255UL)); + xm_log.p_ino = parent_ino(dentry); + + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, + min(strlen(name), 255UL), + inode->i_ino, true); + if (xmlt_addr < 0) { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + + spin_lock(&sbi->space_lock); + xmfs_inode_table_remove(tbl, inode->i_ino, dir->i_ino, name); + xmfs_inode_table_sync_nlink(tbl, inode->i_ino); + spin_unlock(&sbi->space_lock); + XMFS_I(inode)->die = true; + drop_nlink(inode); + iput(inode); + dput(dentry); + return -EINVAL; + } + + return 0; +} + +static int xmfs_do_create_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct xmfs_rw_msg req = {}, grant = {}; + struct inode *inode; + struct xmfs_inode_info *xmfs_inode; + int err; + struct xmfs_log xm_log; + unsigned long log_off; + + req.ops = XMFS_CREATE; + req.mode = mode | S_IFREG; + req.p_ino = dir->i_ino; + req.i_ino = 0; + req.data_pages = 0; + req.log_entries = 1; + memcpy(req.name, dentry->d_name.name, dentry->d_name.len + 1); + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + xm_log.index = 0; + xm_log.end_index = 0; + xm_log.ops = XMFS_CREATE; + xm_log.data_offset = ktime_get_real_seconds(); + xm_log.mode = mode | S_IFREG; + xm_log.offset = ktime_get_real_ns() % NSEC_PER_SEC; + xm_log.version = grant.version; + memcpy(xm_log.name, dentry->d_name.name, + min(dentry->d_name.len, 255UL)); + if (dentry->d_name.len < 256) + xm_log.name[dentry->d_name.len] = '\0'; + xm_log.p_ino = dir->i_ino; + xm_log.i_ino = grant.assigned_ino; + xm_log.writer_id = sbi->id; + xm_log.size = 0; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + inode = iget5_locked(dir->i_sb, grant.assigned_ino, xmfs_inode_eq, + xmfs_inode_set, &grant.assigned_ino); + if (!inode) + return -ENOMEM; + + if (!(inode->i_state & I_NEW)) + return 0; + + xmfs_inode = XMFS_I(inode); + i_size_write(inode, 0); + inode->i_ino = grant.assigned_ino; + inode_init_owner(&nop_mnt_idmap, inode, dir, mode | S_IFREG); + xmfs_init_file_inode(inode); + /* + * xmfs_inode->write_wq = alloc_workqueue("xmfs-prefetch-inode", + * WQ_UNBOUND | WQ_MEM_RECLAIM, + * 8); + */ + security_inode_init_security(inode, dir, &dentry->d_name, + xmfs_initxattrs, NULL); + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode_set_ctime_current(inode); + dir->i_mtime = inode_set_ctime_current(dir); + unlock_new_inode(inode); + d_instantiate(dentry, inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + return 0; +} + +static int xmfs_do_mkdir_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct xmfs_rw_msg req = {}, grant = {}; + struct inode *inode; + struct xmfs_inode_info *xmfs_inode; + int err; + struct xmfs_log xm_log; + unsigned long log_off; + + req.ops = XMFS_MKDIR; + req.mode = mode | S_IFDIR; + req.p_ino = dir->i_ino; + req.i_ino = 0; + req.data_pages = 0; + req.log_entries = 1; + req.size = 4096; + memcpy(req.name, dentry->d_name.name, dentry->d_name.len + 1); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + xm_log.index = 0; + xm_log.end_index = 0; + xm_log.ops = XMFS_MKDIR; + xm_log.data_offset = ktime_get_real_seconds(); + xm_log.mode = mode | S_IFDIR; + xm_log.offset = ktime_get_real_ns() % NSEC_PER_SEC; + xm_log.version = grant.version; + memcpy(xm_log.name, dentry->d_name.name, + min(dentry->d_name.len, 255UL)); + if (dentry->d_name.len < 256) + xm_log.name[dentry->d_name.len] = '\0'; + xm_log.p_ino = dir->i_ino; + xm_log.i_ino = grant.assigned_ino; + xm_log.writer_id = sbi->id; + xm_log.size = 4096; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + inode = iget5_locked(dir->i_sb, grant.assigned_ino, xmfs_inode_eq, + xmfs_inode_set, &grant.assigned_ino); + if (!inode) + return -ENOMEM; + + if (!(inode->i_state & I_NEW)) + return 0; + + xmfs_inode = XMFS_I(inode); + i_size_write(inode, 4096); + inode->i_ino = grant.assigned_ino; + inode_init_owner(&nop_mnt_idmap, inode, dir, S_IFDIR | mode); + xmfs_init_dir_inode(inode, false); + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode_set_ctime_current(inode); + dir->i_mtime = inode_set_ctime_current(dir); + inc_nlink(dir); + inc_nlink(inode); + unlock_new_inode(inode); + d_instantiate(dentry, inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + return 0; +} + +static int xmfs_do_unlink_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry) +{ + struct xmfs_rw_msg req = {}, grant = {}; + int err; + struct xmfs_log xm_log; + unsigned long log_off; + + req.ops = XMFS_DELETE; + req.p_ino = dir->i_ino; + req.i_ino = dentry->d_inode->i_ino; + req.data_pages = 0; + req.log_entries = 1; + memcpy(req.name, dentry->d_name.name, dentry->d_name.len + 1); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + xm_log.ops = XMFS_DELETE; + xm_log.i_ino = dentry->d_inode->i_ino; + xm_log.p_ino = dir->i_ino; + xm_log.version = grant.version; + memcpy(xm_log.name, dentry->d_name.name, + min(dentry->d_name.len, 255UL)); + if (dentry->d_name.len < 256) + xm_log.name[dentry->d_name.len] = '\0'; + xm_log.writer_id = sbi->id; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + if (d_inode(dentry)->i_nlink > 0) + drop_nlink(d_inode(dentry)); + if (!d_inode(dentry)->i_nlink) + xmfs_unpin_inode(sbi, d_inode(dentry)); + return 0; +} + +static int xmfs_do_symlink_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct xmfs_rw_msg req = {}; + struct inode *inode; + struct xmfs_inode_info *xmfs_inode; + unsigned int len = strlen(symname) + 1; + struct xmfs_log xm_log; + unsigned long log_off; + int err; + + if (len > PAGE_SIZE) + return -ENAMETOOLONG; + + req.ops = XMFS_SYMLINK; + req.mode = S_IFLNK | 0777; + req.p_ino = dir->i_ino; + req.i_ino = 0; + req.data_pages = (len > 255) ? 1 : 0; + req.log_entries = 2; + memcpy(req.name, dentry->d_name.name, dentry->d_name.len + 1); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + + inode = iget5_locked(dir->i_sb, gw.grant.assigned_ino, xmfs_inode_eq, + xmfs_inode_set, &gw.grant.assigned_ino); + if (!inode) + return -ENOMEM; + + if (!(inode->i_state & I_NEW)) + return 0; + + xmfs_inode = XMFS_I(inode); + inode->i_ino = gw.grant.assigned_ino; + i_size_write(inode, 0); + inode_init_owner(&nop_mnt_idmap, inode, dir, S_IFLNK | 0777); + xmfs_init_symlink_operations(inode); + inode_nohighmem(inode); + xmfs_init_symlink_mapping(inode); + xmfs_inode->btime = inode->i_atime = inode->i_mtime = + inode_set_ctime_current(inode); + dir->i_mtime = inode_set_ctime_current(dir); + unlock_new_inode(inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + log_off = gw.grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + gw.grant.log_slot_start * sizeof(struct xmfs_log); + xm_log.index = 0; + xm_log.end_index = 0; + xm_log.ops = XMFS_SYMLINK; + xm_log.data_offset = ktime_get_real_seconds(); + xm_log.mode = S_IFLNK | 0777; + xm_log.version = gw.grant.version; + memcpy(xm_log.name, dentry->d_name.name, + min(dentry->d_name.len, 255UL)); + if (dentry->d_name.len < 256) + xm_log.name[dentry->d_name.len] = '\0'; + xm_log.p_ino = dir->i_ino; + xm_log.i_ino = gw.grant.assigned_ino; + xm_log.writer_id = sbi->id; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + if (gw.grant.log_slot_count >= 2) { + struct xmfs_log xm_log1 = {}; + + xm_log1.index = 1; + xm_log1.ops = XMFS_SYMLINK; + xm_log1.version = gw.grant.version + 1; + xm_log1.i_ino = gw.grant.assigned_ino; + xm_log1.p_ino = dir->i_ino; + xm_log1.writer_id = sbi->id; + if (len <= 255) { + memcpy(xm_log1.name, symname, + min(len, 255UL)); + xm_log1.name[min(len - 1, 254UL)] = '\0'; + xm_log1.end_index = 0; + } else { + xm_log1.end_index = gw.grant.data_addr; + xm_log1.index = len; + XMFS_I(inode)->start_write = gw.grant.data_addr; + XMFS_I(inode)->count = gw.grant.data_page_count; + } + log_off = gw.grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + (gw.grant.log_slot_start + 1) * sizeof(struct xmfs_log); + xmfs_inode->symname = log_off; + xmfs_data_write(sbi, log_off, &xm_log1, sizeof(xm_log1)); + } + + if (len > 255 && gw.grant.data_addr != 0) { + xmfs_data_write(sbi, gw.grant.data_addr, symname, + min(len, (unsigned long)PAGE_SIZE)); + } + + err = page_symlink(inode, symname, len); + if (err) { + clear_nlink(inode); + iput(inode); + return err; + } + + d_instantiate(dentry, inode); + + return 0; +} + +static int xmfs_do_link_slave(struct xmfs_sb_info *sbi, + struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct xmfs_rw_msg req = {}, grant = {}; + struct inode *inode = d_inode(old_dentry); + int err; + struct xmfs_log xm_log; + unsigned long log_off; + + req.ops = XMFS_LINK; + req.p_ino = dir->i_ino; + req.i_ino = inode->i_ino; + req.mode = inode->i_mode; + req.data_pages = 0; + req.log_entries = 1; + memcpy(req.name, dentry->d_name.name, dentry->d_name.len + 1); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + xm_log.index = 0; + xm_log.end_index = inode->i_ino; + xm_log.ops = XMFS_LINK; + xm_log.data_offset = ktime_get_real_seconds(); + xm_log.mode = inode->i_mode; + xm_log.offset = ktime_get_real_ns() % NSEC_PER_SEC; + xm_log.version = grant.version; + memcpy(xm_log.name, dentry->d_name.name, + min(dentry->d_name.len, 255UL)); + if (dentry->d_name.len < 256) + xm_log.name[dentry->d_name.len] = '\0'; + xm_log.p_ino = dir->i_ino; + xm_log.i_ino = inode->i_ino; + xm_log.writer_id = sbi->id; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + inc_nlink(inode); + ihold(inode); + + d_instantiate(dentry, inode); + + return 0; +} + +static int xmfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, bool excl) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (sbi->id != 0) + return xmfs_do_create_slave(sbi, dir, dentry, mode); + struct xmfs_log xm_log; + unsigned long xmlt_addr; + struct xmfs_sb_index *xmsi; + int err = 0; + struct xmfs_inode_info *xmfs_inode; + const char *name = dentry->d_name.name; + struct inode *inode; + + spin_lock(&sbi->space_lock); + if (sbi->full) { + spin_unlock(&sbi->space_lock); + return -ENOSPC; + } + + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + unsigned long ino = get_next_ino(); + + inode = iget5_locked(dir->i_sb, ino, xmfs_inode_eq, xmfs_inode_set, + &ino); + if (!inode) + return -ENOMEM; + + xmfs_inode = XMFS_I(inode); + i_size_write(inode, 0); + + inode->i_ino = ino; + inode_init_owner(idmap, inode, dir, mode | S_IFREG); + + xmfs_init_file_inode(inode); + /* + * xmfs_inode->write_wq = alloc_workqueue("xmfs-prefetch-inode", + * WQ_UNBOUND | WQ_MEM_RECLAIM, + * 8); + */ + err = security_inode_init_security(inode, dir, &dentry->d_name, + xmfs_initxattrs, NULL); + if (err) { + unlock_new_inode(inode); + iput(inode); + return err; + } + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode_set_ctime_current(inode); + + dir->i_mtime = inode_set_ctime_current(dir); + unlock_new_inode(inode); + d_instantiate(dentry, inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long slot = tbl->count; + + tbl->count++; + { + struct xmfs_inode_entry *e = &tbl->entries[slot]; + + memset(e, 0, sizeof(*e)); + e->i_ino = inode->i_ino; + e->p_ino = dir->i_ino; + e->mode = mode | S_IFREG; + e->nlink = 1; + e->size = 0; + e->ctime = inode_get_ctime(inode); + e->mtime = inode->i_mtime; + e->atime = inode->i_atime; + memcpy(e->name, dentry->d_name.name, + dentry->d_name.len); + if (dentry->d_name.len < 256) + e->name[dentry->d_name.len] = '\0'; + } + } + + xm_log.index = 0; + xm_log.end_index = 0; + xm_log.ops = XMFS_CREATE; + xm_log.data_offset = inode->i_mtime.tv_sec; + xm_log.mode = mode | S_IFREG; + xm_log.offset = inode->i_mtime.tv_nsec; + memcpy(xm_log.name, name, min(strlen(name), 255UL)); + xm_log.p_ino = parent_ino(dentry); + xm_log.writer_id = sbi->id; + xm_log.size = 0; + + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, + min(strlen(name), 255UL), + inode->i_ino, true); + if (xmlt_addr < 0) { + xmfs_inode->die = true; + iput(inode); + dput(dentry); + return -EINVAL; + } + + if (sbi->fallocate_size) + xmfs_common_fallocate(inode, 0x01, 0, sbi->fallocate_size); + + return 0; +} + +static int xmfs_do_mknod_slave(struct xmfs_sb_info *sbi, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) +{ + struct xmfs_rw_msg req = {}, grant = {}; + struct inode *inode; + struct xmfs_inode_info *xmfs_inode; + int err; + struct xmfs_log xm_log; + unsigned long log_off; + + req.ops = XMFS_CREATE; + req.mode = mode; + req.p_ino = dir->i_ino; + req.i_ino = 0; + req.data_pages = 0; + req.log_entries = 1; + memcpy(req.name, dentry->d_name.name, dentry->d_name.len + 1); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + memset(&xm_log, 0, sizeof(xm_log)); + xm_log.ops = XMFS_CREATE; + xm_log.mode = mode; + xm_log.p_ino = dir->i_ino; + xm_log.i_ino = grant.assigned_ino; + xm_log.index = 1; + xm_log.end_index = dev; + xm_log.data_offset = ktime_get_real_seconds(); + xm_log.offset = ktime_get_real_ns() % NSEC_PER_SEC; + xm_log.version = grant.version; + memcpy(xm_log.name, dentry->d_name.name, + min(dentry->d_name.len, 255UL)); + if (dentry->d_name.len < 256) + xm_log.name[dentry->d_name.len] = '\0'; + xm_log.writer_id = sbi->id; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + inode = iget5_locked(dir->i_sb, grant.assigned_ino, xmfs_inode_eq, + xmfs_inode_set, &grant.assigned_ino); + if (!inode) + return -ENOMEM; + + if (!(inode->i_state & I_NEW)) + return 0; + + xmfs_inode = XMFS_I(inode); + i_size_write(inode, 0); + inode->i_ino = grant.assigned_ino; + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + init_special_inode(inode, mode, dev); + xmfs_init_special_inode_operations(inode); + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode_set_ctime_current(inode); + unlock_new_inode(inode); + d_instantiate(dentry, inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + return 0; +} + +static int xmfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (sbi->id != 0) + return xmfs_do_mknod_slave(sbi, dir, dentry, mode, dev); + struct xmfs_log xm_log; + unsigned long xmlt_addr; + struct xmfs_sb_index *xmsi; + int err = 0; + struct xmfs_inode_info *xmfs_inode; + const char *name = dentry->d_name.name; + struct inode *inode; + + spin_lock(&sbi->space_lock); + if (sbi->full) { + spin_unlock(&sbi->space_lock); + return -ENOSPC; + } + + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + unsigned long ino = get_next_ino(); + + inode = iget5_locked(dir->i_sb, ino, xmfs_inode_eq, xmfs_inode_set, + &ino); + if (!inode) + return -ENOMEM; + + xmfs_inode = XMFS_I(inode); + i_size_write(inode, 0); + + inode->i_ino = ino; + inode_init_owner(idmap, inode, dir, mode); + init_special_inode(inode, mode, dev); + xmfs_init_special_inode_operations(inode); + err = security_inode_init_security(inode, dir, &dentry->d_name, + xmfs_initxattrs, NULL); + if (err) { + unlock_new_inode(inode); + iput(inode); + return err; + } + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode_set_ctime_current(inode); + + dir->i_mtime = inode_set_ctime_current(dir); + unlock_new_inode(inode); + d_instantiate(dentry, inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long slot = tbl->count; + + tbl->count++; + { + struct xmfs_inode_entry *e = &tbl->entries[slot]; + + memset(e, 0, sizeof(*e)); + e->i_ino = inode->i_ino; + e->p_ino = dir->i_ino; + e->mode = mode; + e->nlink = 1; + e->size = 0; + e->ctime = inode_get_ctime(inode); + e->mtime = inode->i_mtime; + e->atime = inode->i_atime; + memcpy(e->name, dentry->d_name.name, + dentry->d_name.len); + if (dentry->d_name.len < 256) + e->name[dentry->d_name.len] = '\0'; + } + } + + xm_log.index = 1; + xm_log.end_index = dev; + xm_log.ops = XMFS_CREATE; + xm_log.data_offset = inode->i_mtime.tv_sec; + xm_log.mode = mode; + xm_log.offset = inode->i_mtime.tv_nsec; + memcpy(xm_log.name, name, min(strlen(name), 255UL)); + xm_log.p_ino = parent_ino(dentry); + + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, + min(strlen(name), 255UL), + inode->i_ino, true); + if (xmlt_addr < 0) { + xmfs_inode->die = true; + iput(inode); + dput(dentry); + return -EINVAL; + } + + return 0; +} + +static int xmfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (sbi->id != 0) + return xmfs_do_mkdir_slave(sbi, dir, dentry, mode); + struct xmfs_log xm_log; + struct xmfs_sb_index *xmsi; + int err = 0; + struct xmfs_inode_info *xmfs_inode; + const char *name = dentry->d_name.name; + struct inode *inode; + unsigned long xmlt_addr; + + spin_lock(&sbi->space_lock); + if (sbi->full) { + spin_unlock(&sbi->space_lock); + return -ENOSPC; + } + + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + + unsigned long ino = get_next_ino(); + + inode = iget5_locked(dir->i_sb, ino, xmfs_inode_eq, xmfs_inode_set, + &ino); + if (!inode) + return -ENOMEM; + + xmfs_inode = XMFS_I(inode); + i_size_write(inode, 4096); + + inode->i_ino = ino; + inode_init_owner(idmap, inode, dir, S_IFDIR | mode); + xmfs_init_dir_inode(inode, true); + + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode_set_ctime_current(inode); + + dir->i_mtime = inode_set_ctime_current(dir); + + err = security_inode_init_security(inode, dir, &dentry->d_name, + xmfs_initxattrs, NULL); + if (err) { + unlock_new_inode(inode); + iput(inode); + return err; + } + inc_nlink(dir); + inc_nlink(inode); + unlock_new_inode(inode); + d_instantiate(dentry, inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long slot = tbl->count; + + tbl->count++; + { + struct xmfs_inode_entry *e = &tbl->entries[slot]; + + memset(e, 0, sizeof(*e)); + e->i_ino = inode->i_ino; + e->p_ino = dir->i_ino; + e->mode = mode | S_IFDIR; + e->nlink = 1; + e->size = 4096; + e->ctime = inode_get_ctime(inode); + e->mtime = inode->i_mtime; + e->atime = inode->i_atime; + memcpy(e->name, dentry->d_name.name, + dentry->d_name.len); + if (dentry->d_name.len < 256) + e->name[dentry->d_name.len] = '\0'; + } + } + + xm_log.index = 0; + xm_log.end_index = 0; + xm_log.ops = XMFS_MKDIR; + xm_log.data_offset = dir->i_mtime.tv_sec; + xm_log.mode = mode | S_IFDIR; + xm_log.offset = dir->i_mtime.tv_nsec; + memcpy(xm_log.name, name, min(strlen(name), 255UL)); + xm_log.p_ino = parent_ino(dentry); + xm_log.size = 4096; + + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, + min(strlen(name), 255UL), + inode->i_ino, true); + if (xmlt_addr < 0) { + xmfs_inode->die = true; + drop_nlink(dir); + drop_nlink(inode); + iput(inode); + dput(dentry); + return 0; + } + + return 0; +} + +static int xmfs_do_rename_slave(struct xmfs_sb_info *sbi, struct inode *old_dir, + struct dentry *old_dentry, + struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct xmfs_rw_msg req = {}, grant = {}; + const char *old_name = old_dentry->d_name.name; + const char *new_name = new_dentry->d_name.name; + unsigned long old_len = old_dentry->d_name.len; + unsigned long new_len = new_dentry->d_name.len; + int err; + struct xmfs_log xm_log; + unsigned long log_off; + + if (old_len + 1 + new_len + 1 > 256) + return -ENAMETOOLONG; + if (flags & ~(RENAME_NOREPLACE)) + return -EINVAL; + + req.ops = XMFS_RENAME; + req.p_ino = old_dir->i_ino; + req.i_ino = old_dentry->d_inode->i_ino; + req.data_pages = 0; + req.log_entries = 2; + memcpy(req.name, old_name, old_len + 1); + memcpy(req.name + old_len + 1, new_name, new_len + 1); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + memset(&xm_log, 0, sizeof(xm_log)); + xm_log.ops = XMFS_RENAME; + xm_log.p_ino = old_dir->i_ino; + xm_log.i_ino = old_dentry->d_inode->i_ino; + xm_log.end_index = -1; + xm_log.version = grant.version; + memcpy(xm_log.name, old_name, min(old_len, 255UL)); + if (old_len < 256) + xm_log.name[old_len] = '\0'; + xm_log.writer_id = sbi->id; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + (grant.log_slot_start + 1) * sizeof(struct xmfs_log); + memset(&xm_log, 0, sizeof(xm_log)); + xm_log.ops = XMFS_RENAME; + xm_log.p_ino = new_dir->i_ino; + xm_log.i_ino = old_dentry->d_inode->i_ino; + xm_log.version = grant.version + 1; + memcpy(xm_log.name, new_name, min(new_len, 255UL)); + if (new_len < 256) + xm_log.name[new_len] = '\0'; + xm_log.writer_id = sbi->id; + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + err = simple_rename(&nop_mnt_idmap, old_dir, old_dentry, new_dir, + new_dentry, flags); + if (err) + return err; + + return 0; +} + +static int xmfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + /* RENAME_EXCHANGE is disabled because d_exchange() is not exported. */ + if (flags & ~(RENAME_NOREPLACE)) + return -EINVAL; + struct xmfs_sb_info *sbi = XMFS_SB(old_dir->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (sbi->id != 0) + return xmfs_do_rename_slave(sbi, old_dir, old_dentry, new_dir, + new_dentry, flags); + if (XMFS_I(old_dentry->d_inode)->die) { + xmfs_info(sbi->sb, + "inode logs have been interrupted before rename"); + return -EINVAL; + } + struct xmfs_sb_index *xmsi; + int err = 0; + + spin_lock(&sbi->space_lock); + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + + err = simple_rename(idmap, old_dir, old_dentry, new_dir, new_dentry, + flags); + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + unsigned long j; + + for (j = 0; j < tbl->count; j++) { + if (tbl->entries[j].i_ino == + old_dentry->d_inode->i_ino && + tbl->entries[j].p_ino == old_dir->i_ino && + strcmp(tbl->entries[j].name, + old_dentry->d_name.name) == 0) { + tbl->entries[j].p_ino = new_dir->i_ino; + memset(tbl->entries[j].name, 0, 256); + memcpy(tbl->entries[j].name, + new_dentry->d_name.name, + min(new_dentry->d_name.len, + 255UL)); + if (new_dentry->d_name.len < 256) + tbl->entries[j] + .name[new_dentry->d_name.len] = + '\0'; + break; + } + } + } + if (err < 0) + return err; + + struct xmfs_log xm_log_old; + + xm_log_old.index = flags; + xm_log_old.ops = XMFS_RENAME; + xm_log_old.data_offset = old_dir->i_mtime.tv_sec; + xm_log_old.end_index = -1; + + const char *name = old_dentry->d_name.name; + + memcpy(xm_log_old.name, name, min(strlen(name), 255UL)); + xm_log_old.p_ino = old_dir->i_ino; + + struct xmfs_log xm_log_new; + + xm_log_new.ops = XMFS_RENAME; + xm_log_new.data_offset = old_dir->i_mtime.tv_nsec; + xm_log_new.p_ino = new_dir->i_ino; + xm_log_new.end_index = old_dentry->d_inode->i_ino; + + const char *new_name = new_dentry->d_name.name; + + memcpy(xm_log_new.name, new_name, + min(strlen(new_name), 255UL)); + + unsigned long addr; + + addr = xmfs_add_log_v3(&xm_log_old, sbi, + min(strlen(name), 255UL), + old_dentry->d_inode->i_ino, false); + if (addr < 0) { + XMFS_I(old_dentry->d_inode)->die = true; + return 0; + } + addr = xmfs_add_log_v3(&xm_log_new, sbi, + min(strlen(new_name), 255UL), + old_dentry->d_inode->i_ino, true); + if (addr < 0) { + XMFS_I(old_dentry->d_inode)->die = true; + return 0; + } + + return 0; +} + +static int xmfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + if (sbi->id != 0) + return xmfs_do_unlink_slave(sbi, dir, dentry); + if (XMFS_I(dentry->d_inode)->die) { + xmfs_info(sbi->sb, + "inode logs have been interrupted before unlink"); + return -EINVAL; + } + struct xmfs_sb_index *xmsi; + + struct xmfs_log xm_log = { 0 }; + const char *name = dentry->d_name.name; + unsigned long ino = d_inode(dentry)->i_ino; + + spin_lock(&sbi->space_lock); + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + /* + * int logcount = 0; + * + * if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + * sbi->full = true; + * spin_unlock(&sbi->space_lock); + * put_xmsi(sbi, xmsi); + * return -ENOSPC; + * } + */ + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + + if (d_inode(dentry)->__i_nlink > 1) + xm_log.index = -1; + else + xm_log.index = 0; + xm_log.end_index = -1; + xm_log.ops = XMFS_DELETE; + xm_log.data_offset = dir->i_mtime.tv_sec; + xm_log.offset = dir->i_mtime.tv_nsec; + memcpy(xm_log.name, name, min(strlen(name), 255UL)); + xm_log.p_ino = parent_ino(dentry); + + unsigned long addr = + xmfs_add_log_v3(&xm_log, sbi, + min(strlen(name), 255UL), ino, + true); + if (addr < 0) { + XMFS_I(d_inode(dentry))->die = true; + return 0; + } + + spin_lock(&sbi->space_lock); + xmsi = get_xmsi(sbi); + if (xmsi) + put_xmsi(sbi, xmsi); + + spin_unlock(&sbi->space_lock); + + if (d_inode(dentry)->i_nlink > 0) + drop_nlink(d_inode(dentry)); + { + struct xmfs_inode_table *tbl = + (void *)(sbi->kaddr + INODE_TABLE_OFFSET); + bool removed; + + spin_lock(&sbi->space_lock); + removed = xmfs_inode_table_remove(tbl, ino, dir->i_ino, name); + xmfs_inode_table_sync_nlink(tbl, ino); + spin_unlock(&sbi->space_lock); + if (!removed) + xmfs_warning(sbi->sb, "inode %lu entry %s not found", ino, + name); + } + if (!d_inode(dentry)->i_nlink) + xmfs_unpin_inode(sbi, d_inode(dentry)); + + return 0; +} + +static int xmfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + if (!simple_empty(dentry)) + return -ENOTEMPTY; + + drop_nlink(d_inode(dentry)); + int err = xmfs_unlink(dir, dentry); + + if (!err) + drop_nlink(dir); + + return err; +} + +static struct dentry *xmfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + struct xmfs_sb_index *xmsi; + struct dentry *p_dentry, *res = NULL; + struct xmfs_inode_table *tbl; + unsigned long max_entries, i; + struct inode *inode; + + if (dentry->d_name.len > NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + + xmsi = kmalloc(sizeof(struct xmfs_sb_index), GFP_KERNEL); + if (get_xmsi_and_copy(sbi, xmsi)) { + kfree(xmsi); + return ERR_PTR(-EIO); + } + /* + * xmfs_info(sbi->sb, "qyf id %ld lookup %ld %ld", sbi->id, + * xmsi->last_update, sbi->last_update); + */ + sbi->stop_cnt++; + if (xmsi->last_update != sbi->last_update) + xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + kfree(xmsi); + + struct qstr qname = QSTR_INIT(dentry->d_name.name, dentry->d_name.len); + + hlist_for_each_entry(p_dentry, &dir->i_dentry, d_u.d_alias) { + res = d_hash_and_lookup(p_dentry, &qname); + if (res) { + sbi->stop_cnt--; + return res; + } + } + + if (sbi->mode == XMFS_SHARED_MEM_MODE || sbi->mode == XMFS_HYBRID_MODE) + tbl = (struct xmfs_inode_table *)(sbi->kaddr + + INODE_TABLE_OFFSET); + else { + tbl = (struct xmfs_inode_table *)get_meta(sbi, + INODE_TABLE_OFFSET, + INODE_TABLE_SIZE); + /* + * tbl = (struct xmfs_inode_table *)get_meta( + * sbi, INODE_TABLE_OFFSET, + * max(sizeof(struct xmfs_inode_table) + + * tbl->count * sizeof(struct xmfs_inode_entry), + * INODE_TABLE_SIZE)); + */ + } + /* + * xmfs_info(sbi->sb, "qyf slave lookup find inode table %d", + * tbl->count); + */ + if (tbl && tbl->magic == XMFS_INODE_TABLE_MAGIC) { + max_entries = (INODE_TABLE_SIZE - 4096) / + sizeof(struct xmfs_inode_entry); + for (i = 0; i < tbl->count && i < max_entries; i++) { + struct xmfs_inode_entry *e = &tbl->entries[i]; + /* + * xmfs_info(sbi->sb, + * "qyf slave lookup iter inode table %d %ld %ld %s %s", + * i, e->p_ino, dir->i_ino, e->name, + * dentry->d_name.name); + */ + if (e->p_ino != dir->i_ino || + strcmp(e->name, dentry->d_name.name) != 0) + continue; + inode = ilookup5(sbi->sb, e->i_ino, xmfs_inode_eq, + &e->i_ino); + /* xmfs_info(sbi->sb, "qyf find inode? %p", inode); */ + if (!inode) + inode = iget5_locked(sbi->sb, e->i_ino, + xmfs_inode_eq, + xmfs_inode_set, &e->i_ino); + if (inode && (inode->i_state & I_NEW)) { + struct xmfs_inode_info *xmfs_inode = + XMFS_I(inode); + xmfs_inode->leaf = (e->i_ino != 0); + inode->i_ino = e->i_ino; + inode->i_mode = e->mode; + if (S_ISREG(e->mode)) { + /* + * xmfs_inode->write_wq = alloc_workqueue( + * "xmfs-prefetch-inode", + * WQ_UNBOUND | WQ_MEM_RECLAIM, 8); + */ + xmfs_init_file_inode(inode); + } else if (S_ISDIR(e->mode)) { + xmfs_init_dir_inode(inode, + sbi->id == 0); + } else if (S_ISLNK(e->mode)) { + xmfs_init_symlink_operations(inode); + } else { + init_special_inode(inode, e->mode, 0); + } + unlock_new_inode(inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, + inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + } + + if (inode) { + inode_lock(inode); + set_nlink(inode, e->nlink); + i_size_write(inode, e->size); + inode->__i_ctime.tv_sec = e->ctime.tv_sec; + inode->__i_ctime.tv_nsec = e->ctime.tv_nsec; + inode->i_mtime.tv_sec = e->mtime.tv_sec; + inode->i_mtime.tv_nsec = e->mtime.tv_nsec; + inode->i_atime.tv_sec = e->atime.tv_sec; + inode->i_atime.tv_nsec = e->atime.tv_nsec; + inode_unlock(inode); + res = d_splice_alias(inode, dentry); + } + break; + } + } + if (tbl && sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + + sbi->stop_cnt--; + return res; +} + +static const struct inode_operations xmfs_dir_inode_operations = { + .create = xmfs_create, + .lookup = xmfs_lookup, + .link = xmfs_link, + .unlink = xmfs_unlink, + .symlink = xmfs_symlink, + .mkdir = xmfs_mkdir, + .rmdir = xmfs_rmdir, + .mknod = xmfs_mknod, + .rename = xmfs_rename, + .setattr = xmfs_slave_setattr, + .getattr = xmfs_slave_getattr, + .update_time = xmfs_update_time, + .listxattr = xmfs_listxattr, +}; + +static int xmfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + struct xmfs_sb_index *xmsi; + int err = 0; + struct xmfs_inode_info *xmfs_inode; + struct dentry *dentry = file->f_path.dentry; + const char *name = dentry->d_name.name; + struct xmfs_log xm_log; + struct inode *inode; + + spin_lock(&sbi->space_lock); + if (sbi->full) { + spin_unlock(&sbi->space_lock); + return -ENOSPC; + } + + xmsi = get_xmsi(sbi); + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int logcount = 1; + + if (logcount + xmsi->used_trunk_count > sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + unsigned long ino = get_next_ino(); + + inode = iget5_locked(dir->i_sb, ino, xmfs_inode_eq, xmfs_inode_set, + &ino); + if (!inode) + return -ENOMEM; + + xmfs_inode = XMFS_I(inode); + xmfs_inode->tmp = true; + + i_size_write(inode, 0); + + inode->i_ino = ino; + inode_init_owner(idmap, inode, dir, mode); + xmfs_init_file_inode(inode); + /* + * xmfs_inode->write_wq = alloc_workqueue("xmfs-prefetch-inode", + * WQ_UNBOUND | WQ_MEM_RECLAIM, + * 8); + */ + err = security_inode_init_security(inode, dir, &dentry->d_name, + xmfs_initxattrs, NULL); + if (err) { + unlock_new_inode(inode); + iput(inode); + return err; + } + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode_set_ctime_current(inode); + dir->i_mtime = inode_set_ctime_current(dir); + unlock_new_inode(inode); + d_tmpfile(file, inode); + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + + err = finish_open_simple(file, 0); + + xm_log.index = 0; + xm_log.end_index = 0; + xm_log.ops = XMFS_CREATE; + xm_log.data_offset = dir->i_mtime.tv_sec; + xm_log.mode = mode; + xm_log.offset = dir->i_mtime.tv_nsec; + memcpy(xm_log.name, name, min(strlen(name), 255UL)); + xm_log.p_ino = parent_ino(dentry); + + unsigned long xmlt_addr = + xmfs_add_log_v3(&xm_log, sbi, + min(strlen(name), 255UL), + inode->i_ino, true); + if (xmlt_addr < 0) { + xmfs_inode->die = true; + iput(inode); + return -EINVAL; + } + + return err; +} + +static const struct inode_operations xmfs_host_dir_inode_operations = { + .create = xmfs_create, + .lookup = xmfs_lookup, + .link = xmfs_link, + .unlink = xmfs_unlink, + .symlink = xmfs_symlink, + .mkdir = xmfs_mkdir, + .rmdir = xmfs_rmdir, + .mknod = xmfs_mknod, + .rename = xmfs_rename, + .setattr = xmfs_setattr, + .tmpfile = xmfs_tmpfile, + .listxattr = xmfs_listxattr, + .getattr = xmfs_getattr, + .update_time = xmfs_update_time, +}; + +struct inode *xmfs_get_root(struct super_block *sb, int host_id) +{ + struct inode *inode; + + inode = new_inode(sb); + if (inode) { + inode->i_ino = 0; + inode->i_mode = S_IFDIR | 0755; + inode->i_uid = make_kuid(current_user_ns(), 0); + inode->i_gid = make_kgid(current_user_ns(), 0); + XMFS_I(inode)->btime = inode->i_atime = inode->i_mtime = + inode_set_ctime_current(inode); + if (host_id == 0) + xmfs_init_dir_inode(inode, true); + else + xmfs_init_dir_inode(inode, false); + + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + lockdep_annotate_inode_mutex_key(inode); + XMFS_I(inode)->leaf = false; + XMFS_I(inode)->i_ino = 0; + XMFS_I(inode)->tmp = false; + XMFS_I(inode)->die = false; + XMFS_I(inode)->log_start = 0; + + xmfs_simple_xattrs_init(&XMFS_I(inode)->xattrs); + + i_size_write(inode, 4096); + } + return inode; +} + +void xmfs_init_dir_inode(struct inode *inode, bool host) +{ + inode->i_op = host ? &xmfs_host_dir_inode_operations : + &xmfs_dir_inode_operations; + xmfs_init_dir_file(inode); +} diff --git a/fs/xmfs/replay.c b/fs/xmfs/replay.c new file mode 100644 index 000000000000..1d75c9780632 --- /dev/null +++ b/fs/xmfs/replay.c @@ -0,0 +1,907 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "xmfs_i.h" + +#include <linux/kernel.h> +#include <linux/namei.h> +#include <linux/pagemap.h> +#include <linux/slab.h> + +static struct inode *xmfs_get_inode_from_table(struct xmfs_sb_info *sbi, + unsigned long ino) +{ + struct xmfs_inode_table *tbl; + struct xmfs_inode_entry *e; + struct inode *inode; + unsigned long max_entries, i; + + inode = ilookup5(sbi->sb, ino, xmfs_inode_eq, &ino); + if (inode) + return inode; + + if (sbi->mode == XMFS_SHARED_MEM_MODE || sbi->mode == XMFS_HYBRID_MODE) + tbl = (struct xmfs_inode_table *)(sbi->kaddr + + INODE_TABLE_OFFSET); + else { + tbl = (struct xmfs_inode_table *)get_meta(sbi, + INODE_TABLE_OFFSET, + INODE_TABLE_SIZE); + /* + * tbl = (struct xmfs_inode_table *)get_meta( + * sbi, INODE_TABLE_OFFSET, + * max(sizeof(struct xmfs_inode_table) + + * tbl->count * sizeof(struct xmfs_inode_entry), + * INODE_TABLE_SIZE)); + */ + } + + if (!tbl || tbl->magic != XMFS_INODE_TABLE_MAGIC) { + if (tbl && sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + return NULL; + } + + max_entries = + (INODE_TABLE_SIZE - 4096) / sizeof(struct xmfs_inode_entry); + e = NULL; + for (i = 0; i < tbl->count && i < max_entries; i++) { + if (tbl->entries[i].i_ino == ino) { + e = &tbl->entries[i]; + break; + } + } + + if (!e) { + if (sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + return NULL; + } + + inode = iget5_locked(sbi->sb, ino, xmfs_inode_eq, xmfs_inode_set, &ino); + if (!inode) { + if (sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + return NULL; + } + + if (inode->i_state & I_NEW) { + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + xmfs_inode->leaf = (e->i_ino != 0); + inode->i_ino = e->i_ino; + inode->i_mode = e->mode; + set_nlink(inode, e->nlink); + i_size_write(inode, e->size); + inode->__i_ctime.tv_sec = e->ctime.tv_sec; + inode->__i_ctime.tv_nsec = e->ctime.tv_nsec; + inode->i_mtime.tv_sec = e->mtime.tv_sec; + inode->i_mtime.tv_nsec = e->mtime.tv_nsec; + inode->i_atime.tv_sec = e->atime.tv_sec; + inode->i_atime.tv_nsec = e->atime.tv_nsec; + if (S_ISREG(e->mode)) { + /* + * xmfs_inode->write_wq = alloc_workqueue( + * "xmfs-prefetch-inode", + * WQ_UNBOUND | WQ_MEM_RECLAIM, 8); + */ + xmfs_init_file_inode(inode); + } else if (S_ISDIR(e->mode)) { + xmfs_init_dir_inode(inode, sbi->id == 0); + } else if (S_ISLNK(e->mode)) { + xmfs_init_symlink_operations(inode); + } else { + init_special_inode(inode, e->mode, 0); + } + unlock_new_inode(inode); + /* iput(inode); */ + ihold(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, GFP_KERNEL); + XMFS_I(inode)->pinned = true; + } + + if (sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + return inode; +} + +static int xmfs_replay_link(struct inode *dir, struct dentry *pdentry, + struct xmfs_log log, unsigned long ino) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + struct inode *inode; + + if (log.writer_id == sbi->id) + return 0; + + inode = ilookup5(dir->i_sb, ino, xmfs_inode_eq, &ino); + if (inode) { + inc_nlink(inode); + iput(inode); + } + + return 0; +} + +static int xmfs_replay_mkdir(struct inode *dir, struct dentry *pdentry, + struct xmfs_log log, unsigned long replay, + unsigned long ino) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + struct inode *inode; + + if (log.writer_id == sbi->id) + return 0; + + inode = ilookup5(dir->i_sb, ino, xmfs_inode_eq, &ino); + if (!inode) { + inode = iget5_locked(dir->i_sb, ino, xmfs_inode_eq, + xmfs_inode_set, &ino); + if (inode && (inode->i_state & I_NEW)) { + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + inode->i_ino = ino; + inode_init_owner(&nop_mnt_idmap, inode, dir, log.mode); + xmfs_init_dir_inode(inode, false); + i_size_write(inode, log.size); + inode->__i_ctime.tv_sec = log.data_offset; + inode->__i_ctime.tv_nsec = log.offset; + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode->__i_ctime; + unlock_new_inode(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, + GFP_KERNEL); + XMFS_I(inode)->pinned = true; + } + } else { + iput(inode); + } + + return 0; +} + +static int xmfs_replay_symlink(struct inode *dir, struct dentry *pdentry, + struct xmfs_log o_log, struct xmfs_log n_log, + unsigned long symname_addr) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + struct inode *inode; + + if (o_log.writer_id == sbi->id) + return 0; + + inode = ilookup5(dir->i_sb, o_log.i_ino, xmfs_inode_eq, &o_log.i_ino); + if (!inode) { + inode = iget5_locked(dir->i_sb, o_log.i_ino, xmfs_inode_eq, + xmfs_inode_set, &o_log.i_ino); + if (inode && (inode->i_state & I_NEW)) { + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + xmfs_inode->symname = symname_addr; + inode->i_ino = o_log.i_ino; + inode_init_owner(&nop_mnt_idmap, inode, dir, + S_IFLNK | 0777); + xmfs_init_symlink_mapping(inode); + i_size_write(inode, o_log.size); + inode->__i_ctime.tv_sec = o_log.data_offset; + inode->__i_ctime.tv_nsec = o_log.offset; + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode->__i_ctime; + unlock_new_inode(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, + GFP_KERNEL); + XMFS_I(inode)->pinned = true; + } + } else { + iput(inode); + } + + return 0; +} + +static int xmfs_replay_rename(struct inode *dir, struct dentry *pdentry, + struct xmfs_log o_log, struct xmfs_log n_log) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (o_log.writer_id == sbi->id) + return 0; + + { + struct inode *old_parent = ilookup5(sbi->sb, o_log.p_ino, + xmfs_inode_eq, + &o_log.p_ino); + if (old_parent) { + struct dentry *pd = d_find_alias(old_parent); + + if (pd) { + struct qstr qname = + QSTR_INIT(o_log.name, + strlen(o_log.name)); + struct dentry *d = d_lookup(pd, &qname); + + if (d) { + d_drop(d); + dput(d); + } + dput(pd); + } + iput(old_parent); + } + } + return 0; +} + +static int xmfs_replay_create(struct inode *dir, struct dentry *pdentry, + struct xmfs_log log, unsigned long replay, + unsigned long ino) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + struct inode *inode; + /* + * xmfs_info(sbi->sb, "qyf replay create before check %ld %d %s", + * log.writer_id, sbi->id, log.name); + */ + if (log.writer_id == sbi->id) + return 0; + /* + * xmfs_info(sbi->sb, "qyf replay create %ld %s size %lld", + * log.i_ino, log.name, log.size); + */ + inode = ilookup5(dir->i_sb, ino, xmfs_inode_eq, &ino); + if (!inode) { + inode = iget5_locked(dir->i_sb, ino, xmfs_inode_eq, + xmfs_inode_set, &ino); + if (inode && (inode->i_state & I_NEW)) { + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + inode->i_ino = ino; + inode_init_owner(&nop_mnt_idmap, inode, dir, log.mode); + if (log.index == 1) + init_special_inode(inode, log.mode, + log.end_index); + else if (S_ISREG(log.mode)) { + /* + * xmfs_inode->write_wq = alloc_workqueue( + * "xmfs-prefetch-inode", + * WQ_UNBOUND | WQ_MEM_RECLAIM, 8); + */ + xmfs_init_file_inode(inode); + } else if (S_ISDIR(log.mode)) { + xmfs_init_dir_inode(inode, false); + } + i_size_write(inode, log.size); + inode->__i_ctime.tv_sec = log.data_offset; + inode->__i_ctime.tv_nsec = log.offset; + xmfs_inode->btime = inode->i_mtime = inode->i_atime = + inode->__i_ctime; + unlock_new_inode(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, + GFP_KERNEL); + XMFS_I(inode)->pinned = true; + } + } else { + iput(inode); + } + + return 0; +} + +static int xmfs_replay_delete(struct inode *dir, struct dentry *pdentry, + struct xmfs_log log) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (log.writer_id == sbi->id) + return 0; + + { + struct inode *parent = + ilookup5(sbi->sb, log.p_ino, xmfs_inode_eq, &log.p_ino); + if (parent) { + struct dentry *pd = d_find_alias(parent); + + if (pd) { + struct qstr qname = + QSTR_INIT(log.name, strlen(log.name)); + struct dentry *d = d_lookup(pd, &qname); + + if (d) { + d_drop(d); + dput(d); + } + dput(pd); + } + iput(parent); + } + } + struct inode *inode; + + inode = ilookup5(sbi->sb, log.i_ino, xmfs_inode_eq, &log.i_ino); + if (!inode) + return 0; + if (inode->i_nlink > 0) + drop_nlink(inode); + if (!inode->i_nlink && XMFS_I(inode)->pinned) { + XMFS_I(inode)->pinned = false; + unsigned long _idx; + void *_entry; + + xa_for_each(&XMFS_I(inode)->shared_pages, _idx, _entry) { + xmfs_mark_page_dead(sbi, (unsigned long)_entry); + } + xa_erase(&sbi->pinned_inodes, inode->i_ino); + inode->i_state &= ~I_DIRTY_TIME; + /* Release pin reference. */ + iput(inode); + } + iput(inode); + return 0; +} + +static int xmfs_replay_update(struct inode *dir, struct dentry *pdentry, + struct xmfs_log log, unsigned long *data_addr, + unsigned long ino) +{ + void *ret; + unsigned long index = log.index; + unsigned long end_index = log.end_index; + unsigned long offset = log.data_offset; + + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (log.writer_id == sbi->id) + return 0; + + struct inode *inode = xmfs_get_inode_from_table(sbi, ino); + + if (!inode) { + xmfs_info(sbi->sb, + "cannot find inode in icache and inode table %ld", + ino); + return -ENOENT; + } + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + int err = 0; + int chunk_header_cnt = 0; + unsigned long i = 0; + unsigned long page_off; + /* + * if (!sbi->always_hot) { + * invalidate_inode_pages2_range(inode->i_mapping, index, + * end_index - 1); + * truncate_pagecache_range(inode, index << PAGE_SHIFT, + * (end_index - 1) << PAGE_SHIFT); + * } + */ + /* + * if (sbi->mode == 0) + * arch_invalidate_pmem(sbi->kaddr + offset, + * (end_index - index + 1) * PAGE_SIZE); + */ + /* + * xmfs_info(sbi->sb, "replay update index %ld to %ld", + * index, end_index); + */ + for (i = index; i < end_index; i++) { + page_off = offset + (i - index) * PAGE_SIZE + + chunk_header_cnt * PAGE_SIZE; + if (page_off % XMFS_DEFAULT_CHUNK_SIZE == 0) { + chunk_header_cnt++; + page_off = offset + (i - index) * PAGE_SIZE + + chunk_header_cnt * PAGE_SIZE; + } + /* + * xmfs_info(sbi->sb, "replay update index %ld offset %ld", + * i, page_off); + */ + ret = xa_store(&xmfs_inode->shared_pages, i, (void *)page_off, + GFP_KERNEL); + if (!xa_err(ret)) { + inode->i_blocks++; + if (sbi->id == 0) { + xmfs_mark_page_dead(sbi, (unsigned long)ret); + xmfs_mark_page_live(sbi, page_off); + unsigned long trunk_off = + log.data_offset & + ~(sbi->trunk_size - 1); + struct xmfs_data_trunk_header *hdr = + (void *)(sbi->kaddr + trunk_off); + if (hdr->magic == XMFS_DATA_TRUNK_MAGIC && + log.version > hdr->last_modify_version) + hdr->last_modify_version = log.version; + } + } else { + xmfs_info(sbi->sb, "replay cannot store %lu %lu", i, + page_off); + } + if (sbi->always_hot) { + struct folio *folio = + filemap_grab_folio(inode->i_mapping, i); + if (IS_ERR(folio)) { + err = -ENOMEM; + goto done; + } + char *kaddr = kmap_local_page(folio_page(folio, 0)); + + get_data_and_copy(sbi, offset + (i - index) * PAGE_SIZE, + kaddr, PAGE_SIZE, NULL, 0); + kunmap_local(kaddr); + folio_mark_uptodate(folio); + folio_unlock(folio); + folio_put(folio); + } + } + +done: + + if (log.size > i_size_read(inode)) + i_size_write(inode, log.size); + + /* return start of data trunk */ + *data_addr = (offset + (end_index - index) * PAGE_SIZE + + (sbi->trunk_size - 1)) & + ~(sbi->trunk_size - 1); + + iput(inode); + + return err; +} + +static int xmfs_replay_xattr(struct inode *dir, struct dentry *pdentry, + struct xmfs_log xm_log, unsigned long ino) +{ + struct inode *inode; + struct xmfs_sb_info *sbi = XMFS_SB(dir->i_sb); + + if (xm_log.writer_id == sbi->id) + return 0; + + if (ino != 0) + inode = xmfs_get_inode_from_table(XMFS_SB(dir->i_sb), ino); + else + inode = dir; + if (!inode) { + xmfs_info(sbi->sb, "xattr cannot find inode xattr %ld", ino); + return -ENOENT; + } + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + size_t size = xm_log.index; + size_t len = xm_log.end_index; + int flags = xm_log.size; + unsigned long offset1 = xm_log.data_offset; + + char *name = kmalloc(len + 1, GFP_KERNEL); + + int err = copy_mc_to_kernel(name, xm_log.name, len); + + if (unlikely(err)) { + xmfs_error(sbi->sb, + "hardware memory error when reading superblock during slave xattr copy name."); + kfree(name); + if (ino != 0) + iput(inode); + goto set_replay; + } + name[len] = '\0'; + if (flags != 0) { + struct simple_xattr *old_xattr = + xmfs_simple_xattr_set(&xmfs_inode->xattrs, name, NULL, + size, flags); + if (!IS_ERR(old_xattr)) { + xmfs_simple_xattr_free(old_xattr); + old_xattr = NULL; + inode->__i_ctime.tv_sec = xm_log.p_ino; + inode->__i_ctime.tv_nsec = xm_log.offset; + } + kfree(name); + if (ino != 0) + iput(inode); + goto set_replay; + } + + char *value = kmalloc(size, GFP_KERNEL); + + err = get_data_and_copy(sbi, offset1, value, size, NULL, 0); + if (unlikely(err)) { + xmfs_error(sbi->sb, + "hardware memory error when reading superblock during slave xattr copy value."); + kfree(name); + kfree(value); + if (ino != 0) + iput(inode); + goto set_replay; + } + + struct simple_xattr *old_xattr = + xmfs_simple_xattr_set(&xmfs_inode->xattrs, name, (void *)value, + size, flags); + if (!IS_ERR(old_xattr)) { + xmfs_simple_xattr_free(old_xattr); + old_xattr = NULL; + inode->__i_ctime.tv_sec = xm_log.data_offset; + inode->__i_ctime.tv_nsec = xm_log.offset; + } + kfree(name); + kfree(value); + if (ino != 0) + iput(inode); + +set_replay: + if (sbi->id == 0) { + xmfs_inode->log_start = sbi->log_start & ~(sbi->log_size - 1); + sbi->log_current = sbi->log_start; + } + return 0; +} + +static int xmfs_replay_root(struct inode *dir, struct dentry *pdentry, + struct xmfs_log xm_log, unsigned long ino) +{ + return 0; +} + +int xmfs_replay_new_entries(struct xmfs_sb_info *sbi, struct dentry *dentry, + unsigned long last_update) +{ + struct super_block *sb = sbi->sb; + struct xmfs_sb_index *xmsi; + struct xmfs_log_trunk *xmlt; + struct inode *dir; + unsigned long data_addr; + unsigned long trunk_offset; + unsigned long start_slot; + unsigned long target; + int err = 0; + struct xmfs_log *tmp_log; + + if (!mutex_trylock(&sbi->replay_lock)) + return 0; + + xmsi = kmalloc(sizeof(*xmsi), GFP_KERNEL); + err = get_xmsi_and_copy(sbi, xmsi); + if (unlikely(err)) { + kfree(xmsi); + mutex_unlock(&sbi->replay_lock); + return err; + } + + if (!dentry) + dentry = sb->s_root; + dir = dentry->d_inode; + data_addr = sbi->addr; + tmp_log = kmalloc(sizeof(struct xmfs_log), GFP_KERNEL); + + if (sbi->last_update >= xmsi->last_update) { + kfree(xmsi); + kfree(tmp_log); + mutex_unlock(&sbi->replay_lock); + return 0; + } + + target = (last_update != -1) ? last_update : xmsi->last_update; + if (target > xmsi->last_update) + target = xmsi->last_update; + if (target < sbi->last_update) { + sbi->last_update = target; + kfree(tmp_log); + mutex_unlock(&sbi->replay_lock); + return 0; + } + + if (sbi->last_replayed_slot == (unsigned long)-1) { + trunk_offset = xmsi->log_start; + start_slot = 0; + sbi->log_start = xmsi->log_start; + } else if (sbi->last_replayed_slot < MAX_LOG_PER_TRUNK) { + trunk_offset = sbi->log_start; + start_slot = sbi->last_replayed_slot + 1; + } else { + struct xmfs_log_trunk *cur = + get_log_trunk_meta(sbi, sbi->log_start); + trunk_offset = (cur && cur->next_trunk) ? cur->next_trunk : 0; + if (cur) + put_log_trunk(sbi, cur); + sbi->log_start = trunk_offset; + start_slot = 0; + sbi->last_replayed_slot = (unsigned long)-1; + } + kfree(xmsi); + + down_read(&sbi->chain_rwsem); + + while (trunk_offset != 0) { + unsigned long next_trunk; + unsigned long i; + int no_ops = 0; + + xmlt = get_log_trunk(sbi, trunk_offset); + if (!xmlt) + break; + + next_trunk = xmlt->next_trunk; + if (xmlt->version_base > target) { + put_log_trunk(sbi, xmlt); + break; + } + /* put_log_trunk(sbi, xmlt); */ + /* xmlt = get_log_trunk_data(sbi, trunk_offset, xmlt->used); */ + + for (i = start_slot; i < xmlt->used; i++) { + struct xmfs_log *replay_log = &xmlt->log[i]; + + switch (replay_log->ops) { + case XMFS_CREATE: + err = xmfs_replay_create(dir, dentry, + *replay_log, 0, + replay_log->i_ino); + break; + case XMFS_DELETE: + err = xmfs_replay_delete(dir, dentry, + *replay_log); + break; + case XMFS_UPDATE: + err = xmfs_replay_update(dir, dentry, + *replay_log, + &data_addr, + replay_log->i_ino); + break; + case XMFS_LINK: + err = xmfs_replay_link(dir, dentry, *replay_log, + replay_log->i_ino); + break; + case XMFS_MKDIR: + err = xmfs_replay_mkdir(dir, dentry, + *replay_log, 0, + replay_log->i_ino); + break; + case XMFS_RENAME: { + if (i + 1 < xmlt->used) { + *tmp_log = xmlt->log[i + 1]; + } else if (xmlt->next_trunk != 0) { + struct xmfs_log_trunk *next_xmlt = + get_log_trunk_meta(sbi, + xmlt->next_trunk); + if (next_xmlt && next_xmlt->used > 0) { + *tmp_log = next_xmlt->log[0]; + put_log_trunk(sbi, next_xmlt); + } else if (next_xmlt) { + put_log_trunk(sbi, next_xmlt); + break; + } else { + break; + } + } else { + break; + } + err = xmfs_replay_rename(dir, dentry, + *replay_log, *tmp_log); + i++; + break; + } + case XMFS_SYMLINK: { + if (i + 1 < xmlt->used) { + *tmp_log = xmlt->log[i + 1]; + } else if (xmlt->next_trunk != 0) { + struct xmfs_log_trunk *next_xmlt = + get_log_trunk_meta(sbi, + xmlt->next_trunk); + if (next_xmlt && next_xmlt->used > 0) { + *tmp_log = next_xmlt->log[0]; + put_log_trunk(sbi, next_xmlt); + } else if (next_xmlt) { + put_log_trunk(sbi, next_xmlt); + break; + } else { + break; + } + } else { + break; + } + { + unsigned long symname_addr; + + if (i + 1 < xmlt->used) + symname_addr = + trunk_offset + + offsetof(struct xmfs_log_trunk, + log[0]) + + (i + + 1) * sizeof(struct xmfs_log); + else + symname_addr = + xmlt->next_trunk + + offsetof(struct xmfs_log_trunk, + log[0]); + err = xmfs_replay_symlink(dir, dentry, + *replay_log, + *tmp_log, + symname_addr); + } + i++; + break; + } + case XMFS_SETATTR: + break; + case XMFS_XATTR: + err = xmfs_replay_xattr(dir, dentry, + *replay_log, + replay_log->i_ino); + break; + case XMFS_ROOT: + err = xmfs_replay_root(dir, dentry, *replay_log, + replay_log->i_ino); + break; + default: + no_ops = 1; + sbi->skip_no_ops++; + /* + * xmfs_error(sbi->sb, + * "qyf xmfs v2: unknown ops %d index %d %d", + * replay_log->ops, i, + * sbi->last_replayed_slot); + */ + if (sbi->skip_no_ops >= 50) { + sbi->skip_no_ops = 0; + xmfs_error(sbi->sb, + "replay fault: unknown ops %d, tried 50 times. This may happened when writing logs is too slow", + replay_log->ops); + } else { + target = replay_log->version; + goto done; + } + } + + if (err != 0) { + xmfs_error(sbi->sb, + "slave %d v2 play log ops %d err %d", + sbi->id, replay_log->ops, -err); + target = replay_log->version; + goto done; + } + err = 0; + sbi->last_replayed_slot = i; + if (replay_log->version > target) { + put_log_trunk(sbi, xmlt); + goto done; + } + /* + * xmfs_info(sbi->sb, + * "qyf replay sbi %d trunk %d replay ops %d " + * "name %s slot %d version %d owner %d " + * "total used %d ino %d index %d " + * "endindex %d size %d", + * sbi->id, xmlt->version_base, replay_log->ops, + * replay_log->name, sbi->last_replayed_slot, + * replay_log->version, replay_log->writer_id, + * xmlt->used, replay_log->i_ino, + * replay_log->index, replay_log->end_index, + * replay_log->size); + */ + } + + /* xmfs_info(sbi->sb, "qyf need to go to next trunk"); */ + start_slot = 0; + if (next_trunk != 0) { + sbi->log_start = next_trunk; + sbi->last_replayed_slot = (unsigned long)-1; + } + put_log_trunk(sbi, xmlt); + trunk_offset = next_trunk; + } + +done: + up_read(&sbi->chain_rwsem); + sbi->last_update = target; + if (sbi->id != 0) { + if (sbi->mode != XMFS_URMA_MODE) { + struct mount_msg_info *mm_info = + sbi->kaddr + MOUNT_MSG_OFFSET; + mm_info->slave_versions[sbi->id] = sbi->last_update; + } else { + unsigned long ver = sbi->last_update; + + xmfs_pmem_write(sbi, + MOUNT_MSG_OFFSET + + offsetof(struct mount_msg_info, + slave_versions) + + sbi->id * sizeof(unsigned long), + &ver, sizeof(ver)); + } + } + + kfree(tmp_log); + mutex_unlock(&sbi->replay_lock); + return 0; +} + +void xmfs_populate_from_inode_table(struct xmfs_sb_info *sbi) +{ + struct xmfs_inode_table *tbl; + struct inode *inode; + unsigned long i; + unsigned long max_entries; + + xmfs_info(sbi->sb, "xmfs_populate_from_inode_table"); + if (sbi->mode == XMFS_SHARED_MEM_MODE || sbi->mode == XMFS_HYBRID_MODE) + tbl = (struct xmfs_inode_table *)(sbi->kaddr + + INODE_TABLE_OFFSET); + else + tbl = (struct xmfs_inode_table *)get_meta(sbi, + INODE_TABLE_OFFSET, + INODE_TABLE_SIZE); + + if (!tbl || tbl->magic != XMFS_INODE_TABLE_MAGIC) { + xmfs_info(sbi->sb, "no valid inode table, skipping populate"); + if (tbl && sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); + return; + } + + max_entries = + (INODE_TABLE_SIZE - 4096) / sizeof(struct xmfs_inode_entry); + xmfs_info(sbi->sb, "populating from inode table, %lu entries", + tbl->count); + + for (i = 0; i < tbl->count && i < max_entries; i++) { + struct xmfs_inode_entry *e = &tbl->entries[i]; + + if (e->i_ino == 0) + continue; + + inode = ilookup5(sbi->sb, e->i_ino, xmfs_inode_eq, &e->i_ino); + if (inode) { + iput(inode); + continue; + } + + inode = iget5_locked(sbi->sb, e->i_ino, xmfs_inode_eq, + xmfs_inode_set, &e->i_ino); + if (!inode) + continue; + + if (inode->i_state & I_NEW) { + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + + xmfs_inode->leaf = (e->i_ino != 0); + inode->i_ino = e->i_ino; + inode->i_mode = e->mode; + set_nlink(inode, e->nlink); + i_size_write(inode, e->size); + if (S_ISREG(e->mode)) { + /* + * xmfs_inode->write_wq = alloc_workqueue( + * "xmfs-prefetch-inode", + * WQ_UNBOUND | WQ_MEM_RECLAIM, 8); + */ + xmfs_init_file_inode(inode); + } else if (S_ISDIR(e->mode)) { + xmfs_init_dir_inode(inode, sbi->id == 0); + } else if (S_ISLNK(e->mode)) { + xmfs_init_symlink_operations(inode); + } else { + init_special_inode(inode, e->mode, 0); + } + unlock_new_inode(inode); + xa_store(&sbi->pinned_inodes, inode->i_ino, inode, + GFP_KERNEL); + XMFS_I(inode)->pinned = true; + } + } + + if (sbi->mode == XMFS_URMA_MODE) + put_meta(sbi, tbl); +} diff --git a/fs/xmfs/super.c b/fs/xmfs/super.c new file mode 100644 index 000000000000..f048fa8844ab --- /dev/null +++ b/fs/xmfs/super.c @@ -0,0 +1,1677 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2026. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#include "xmfs_i.h" + +#include <linux/blkdev.h> +#include <linux/dax.h> +#include <linux/file.h> +#include <linux/fs_context.h> +#include <linux/fs_parser.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +/* open CONFIG_ZONE_DEVICE */ +#include <linux/memremap.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/mount.h> +#include <linux/namei.h> +#include <linux/pagemap.h> +#include <linux/ratelimit.h> +#include <linux/sched.h> +#include <linux/seq_file.h> +#include <linux/slab.h> +#include <linux/statfs.h> +#include <linux/swap.h> +#include <linux/uio.h> + +#include "../../drivers/dax/dax-private.h" +#include "../../drivers/nvdimm/pmem.h" + +#include <linux/cacheflush.h> +#include <asm/set_memory.h> +#include <asm/tlbflush.h> + +static struct kmem_cache *xmfs_inode_cachep; + +#define XMFS_RATELIMIT_INTERVAL (5 * HZ) +#define XMFS_RATELIMIT_BURST 10 + +void xmfs_init_log_ratelimits(struct xmfs_sb_info *sbi) +{ + ratelimit_state_init(&sbi->s_error_ratelimit_state, + XMFS_RATELIMIT_INTERVAL, XMFS_RATELIMIT_BURST); + ratelimit_state_init(&sbi->s_warning_ratelimit_state, + XMFS_RATELIMIT_INTERVAL, XMFS_RATELIMIT_BURST); + ratelimit_state_init(&sbi->s_msg_ratelimit_state, + XMFS_RATELIMIT_INTERVAL, XMFS_RATELIMIT_BURST); + atomic_set(&sbi->s_error_count, 0); + atomic_set(&sbi->s_warning_count, 0); + atomic_set(&sbi->s_msg_count, 0); +} + +static bool xmfs_ratelimit(struct super_block *sb, + struct ratelimit_state *state, atomic_t *count, + const char *name) +{ + if (!sb || !XMFS_SB(sb)) + return true; + + atomic_inc(count); + return ___ratelimit(state, name); +} + +void __xmfs_msg(struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int level; + + if (sb && XMFS_SB(sb) && + !xmfs_ratelimit(sb, &XMFS_SB(sb)->s_msg_ratelimit_state, + &XMFS_SB(sb)->s_msg_count, "XMFS-fs")) + return; + + level = printk_get_level(fmt); + + va_start(args, fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + if (sb) + printk("%c%cXMFS-fs (%s): %pV\n", KERN_SOH_ASCII, level, sb->s_id, &vaf); + else + printk("%c%cXMFS-fs: %pV\n", KERN_SOH_ASCII, level, &vaf); + va_end(args); +} + +void __xmfs_error(struct super_block *sb, const char *function, + unsigned int line, int error, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (sb && XMFS_SB(sb) && + !xmfs_ratelimit(sb, &XMFS_SB(sb)->s_error_ratelimit_state, + &XMFS_SB(sb)->s_error_count, "XMFS-fs error")) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (sb) { + if (error) + printk(KERN_ERR "XMFS-fs error (%s): %s:%u: comm %s: error %d: %pV\n", + sb->s_id, function, line, current->comm, error, &vaf); + else + printk(KERN_ERR "XMFS-fs error (%s): %s:%u: comm %s: %pV\n", + sb->s_id, function, line, current->comm, &vaf); + } else if (error) { + printk(KERN_ERR "XMFS-fs error: %s:%u: comm %s: error %d: %pV\n", + function, line, current->comm, error, &vaf); + } else { + printk(KERN_ERR "XMFS-fs error: %s:%u: comm %s: %pV\n", + function, line, current->comm, &vaf); + } + va_end(args); +} + +void __xmfs_error_inode(struct inode *inode, const char *function, + unsigned int line, int error, const char *fmt, ...) +{ + struct super_block *sb = inode->i_sb; + struct va_format vaf; + va_list args; + + if (XMFS_SB(sb) && + !xmfs_ratelimit(sb, &XMFS_SB(sb)->s_error_ratelimit_state, + &XMFS_SB(sb)->s_error_count, "XMFS-fs error")) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (error) + printk(KERN_ERR "XMFS-fs error (%s): %s:%u: inode #%lu: comm %s: error %d: %pV\n", + sb->s_id, function, line, inode->i_ino, current->comm, error, &vaf); + else + printk(KERN_ERR "XMFS-fs error (%s): %s:%u: inode #%lu: comm %s: %pV\n", + sb->s_id, function, line, inode->i_ino, current->comm, &vaf); + va_end(args); +} + +void __xmfs_error_file(struct file *file, const char *function, + unsigned int line, int error, const char *fmt, ...) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + char pathname[80]; + struct va_format vaf; + const char *path; + va_list args; + + if (XMFS_SB(sb) && + !xmfs_ratelimit(sb, &XMFS_SB(sb)->s_error_ratelimit_state, + &XMFS_SB(sb)->s_error_count, "XMFS-fs error")) + return; + + path = file_path(file, pathname, sizeof(pathname)); + if (IS_ERR(path)) + path = "(unknown)"; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (error) + printk(KERN_ERR "XMFS-fs error (%s): %s:%u: inode #%lu: comm %s: path %s: error %d: %pV\n", + sb->s_id, function, line, inode->i_ino, current->comm, path, + error, &vaf); + else + printk(KERN_ERR "XMFS-fs error (%s): %s:%u: inode #%lu: comm %s: path %s: %pV\n", + sb->s_id, function, line, inode->i_ino, current->comm, path, &vaf); + va_end(args); +} + +void __xmfs_warning(struct super_block *sb, const char *function, + unsigned int line, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (sb && XMFS_SB(sb) && + !xmfs_ratelimit(sb, &XMFS_SB(sb)->s_warning_ratelimit_state, + &XMFS_SB(sb)->s_warning_count, "XMFS-fs warning")) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (sb) + printk(KERN_WARNING "XMFS-fs warning (%s): %s:%u: %pV\n", + sb->s_id, function, line, &vaf); + else + printk(KERN_WARNING "XMFS-fs warning: %s:%u: %pV\n", + function, line, &vaf); + va_end(args); +} + +void __xmfs_warning_inode(const struct inode *inode, const char *function, + unsigned int line, const char *fmt, ...) +{ + struct super_block *sb = inode->i_sb; + struct va_format vaf; + va_list args; + + if (XMFS_SB(sb) && + !xmfs_ratelimit(sb, &XMFS_SB(sb)->s_warning_ratelimit_state, + &XMFS_SB(sb)->s_warning_count, "XMFS-fs warning")) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk(KERN_WARNING "XMFS-fs warning (%s): %s:%u: inode #%lu: comm %s: %pV\n", + sb->s_id, function, line, inode->i_ino, current->comm, &vaf); + va_end(args); +} + +#if defined(__arm__) || defined(__aarch64__) +static int change_memory_cache(pte_t *ptep, unsigned long addr, void *data) +{ + pte_t pte = READ_ONCE(*ptep); + pte_t newpte; + bool cacheable = *(bool *)data; + + if (cacheable) + pte_val(newpte) = (pte_val(pte) & ~PTE_ATTRINDX_MASK) | + PTE_ATTRINDX(MT_NORMAL); + else + pte_val(newpte) = (pte_val(pte) & ~PTE_ATTRINDX_MASK) | + PTE_ATTRINDX(MT_NORMAL_NC); + + if (pte_val(pte) == pte_val(newpte)) + return 0; + + pte_clear(current->mm, addr, ptep); + flush_tlb_kernel_range(addr, addr + 4096); + set_pte(ptep, newpte); + + return 0; +} + +static void set_memory_noncacheable(unsigned long addr, size_t size) +{ + bool cache = false; + int ret = apply_to_page_range(current->mm, addr, size, + change_memory_cache, &cache); + if (ret) + xmfs_error(NULL, "apply noncache for write err %d", ret); +} +#endif + +static struct inode *xmfs_alloc_inode(struct super_block *sb) +{ + struct xmfs_inode_info *p; + + p = alloc_inode_sb(sb, xmfs_inode_cachep, GFP_KERNEL); + if (unlikely(!p)) + return NULL; + return &p->vfs_inode; +} + +static void xmfs_destroy_inode(struct inode *inode) +{ + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (xmfs_inode == NULL) + return; + + xmfs_simple_xattrs_free(&xmfs_inode->xattrs, NULL); + + if (!(xmfs_inode->leaf)) + return; + + sbi->used_ino--; + if (xmfs_inode->write_wq) { + flush_workqueue(xmfs_inode->write_wq); + destroy_workqueue(xmfs_inode->write_wq); + xmfs_inode->write_wq = NULL; + } + /* + * if (sbi->id == 0) { + * unsigned long _idx; + * void *_entry; + * + * xa_for_each(&xmfs_inode->shared_pages, _idx, _entry) { + * xmfs_info(sbi->sb, "destroy inode mark dead %d", _idx); + * xmfs_mark_page_dead(sbi, (unsigned long)_entry); + * } + * } + */ + xa_destroy(&xmfs_inode->shared_pages); + + if (sbi->always_hot || sbi->freq == 0) + return; + + unsigned long i; + void *ptr; + struct page_freq *page_freq; + + xa_for_each(&xmfs_inode->page_freq, i, ptr) { + page_freq = (struct page_freq *)ptr; + xa_erase(&xmfs_inode->page_freq, i); + kfree(page_freq); + } + xa_destroy(&xmfs_inode->page_freq); +} + +static void xmfs_free_inode(struct inode *inode) +{ + kmem_cache_free(xmfs_inode_cachep, XMFS_I(inode)); +} + +static int xmfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct xmfs_sb_info *sbi = XMFS_SB(dentry->d_sb); + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + + if (!xmsi) { + xmfs_error(NULL, + "hardware memory error when reading superblock during statfs."); + return 0; + } + sbi->stop_cnt++; + if (xmsi->last_update != sbi->last_update) { + if (!sbi->debug) + xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + else + xmfs_replay_new_entries(sbi, NULL, xmsi->last_update); + } + sbi->stop_cnt--; + + buf->f_type = XMFS_MAGIC; + buf->f_bsize = sbi->trunk_size; + buf->f_namelen = NAME_MAX; + buf->f_blocks = sbi->max_chunk + DATA_AREA_OFFSET / sbi->trunk_size + 1; + buf->f_bfree = sbi->max_chunk - xmsi->used_trunk_count; + buf->f_bavail = sbi->max_chunk - xmsi->used_trunk_count; + if (xmsi->used_trunk_count > sbi->max_chunk) { + buf->f_bfree = 0; + buf->f_bavail = 0; + } + buf->f_files = sbi->max_chunk * (sbi->trunk_size / sbi->log_size); + buf->f_ffree = sbi->max_chunk * (sbi->trunk_size / sbi->log_size) - + sbi->used_ino; + if (sbi->id == 0) + buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b); + else + buf->f_fsid = xmsi->fsid; + put_xmsi(sbi, xmsi); + return 0; +} + +static int xmfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct xmfs_sb_info *sbi = XMFS_SB(root->d_sb); + +#ifdef CONFIG_XMFS_FS_URMA + if (!sbi->id) + seq_printf(seq, + ",id=%d,log_size=%d,size=%ld,addr=%lld,mode=%d,urma_dev_name=%s,urma_va=%llu,urma_eid_subnet_prefix=%llu, urma_eid_interface_id=%llu,urma_token_id=%u,urma_jetty_id=%u", + sbi->id, sbi->log_size, sbi->len, sbi->start, + sbi->mode, sbi->dev_name, + sbi->urma_cfg.urma_remote_va, + sbi->urma_cfg.urma_eid_subnet_prefix, + sbi->urma_cfg.urma_eid_interface_id, + sbi->urma_cfg.urma_token_id, + sbi->urma_cfg.urma_jetty_id); + else + seq_printf(seq, + ",id=%d,log_size=%d,size=%ld,addr=%lld,mode=%d,urma_dev_name=%s,urma_va=%llu,urma_eid_subnet_prefix=%llu, urma_eid_interface_id=%llu,urma_token_id=%u,urma_jetty_id=%u", + sbi->id, sbi->log_size, sbi->len, sbi->start, + sbi->mode, sbi->dev_name, + sbi->urma_cfg.own_info.urma_remote_va, + sbi->urma_cfg.own_info.urma_eid_subnet_prefix, + sbi->urma_cfg.own_info.urma_eid_interface_id, + sbi->urma_cfg.own_info.urma_token_id, + sbi->urma_cfg.own_info.urma_jetty_id); +#else + if (!sbi->id) + seq_printf(seq, ",id=%d,log_size=%d,size=%ld,addr=%lld,mode=%d", + sbi->id, sbi->log_size, sbi->len, sbi->start, + sbi->mode); + else + seq_printf(seq, ",id=%d,size=%ld,addr=%lld,mode=%d", sbi->id, + sbi->len, sbi->start, sbi->mode); +#endif + return 0; +} + +static int xmfs_syncfs(struct super_block *sb, int wait) +{ + return 0; +} + +static int xmfs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + return 0; +} + +static void xmfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); +} + +static const struct super_operations xmfs_ops = { + .alloc_inode = xmfs_alloc_inode, + .free_inode = xmfs_free_inode, + .destroy_inode = xmfs_destroy_inode, + .statfs = xmfs_statfs, + .show_options = xmfs_show_options, + .sync_fs = xmfs_syncfs, + .write_inode = xmfs_write_inode, + .drop_inode = generic_drop_inode, + .evict_inode = xmfs_evict_inode, +}; + +static int xmfs_find_index(struct xmfs_sb_info *sbi, + struct mount_msg_info *mm_info, bool mount_fn) +{ + int i; + + for (i = 1; i < 256; i++) { + if (mount_fn && mm_info->msg[i] == 'T') { + return i; + } else if (!mount_fn && mm_info->msg[i] != 'D' && + mm_info->msg[i] != 'T') { + /* mm_info_real->msg[i] = 'T'; */ + xmfs_info(NULL, + "------> %s write index %d", __func__, i); + mnt_msg_write(sbi, i, 'T'); + return i; + } + } + /* + * if (mount_fn && mm_info->msg[i - 1] != 'D') + * return -2; + */ + return -1; +} + +static int xmfs_host_mount_fn(void *data) +{ + struct xmfs_sb_info *sbi = data; + struct mount_msg_info *mm_info = + kmalloc(sizeof(struct mount_msg_info), GFP_KERNEL); + struct mount_msg_info *mm_info_real = get_mnt_msg(sbi); + + int id = 0; + int new, i, index; + int err; + + atomic_set(&mm_info_real->id, 0); + while (!kthread_should_stop()) { + err = copy_mc_to_kernel(mm_info, get_mnt_msg(sbi), + sizeof(struct mount_msg_info)); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when host checking mount msgs"); + kfree(mm_info); + return err; + } + if (mm_info_real->msg[0] == 'T') + mm_info_real->msg[0] = 'D'; + + new = atomic_read(&mm_info->id); + if (new == id) { + schedule_timeout_interruptible(msecs_to_jiffies(100)); + continue; + } + xmfs_info(NULL, + "-------------------->got work to do %c %c %c %d %d", + mm_info_real->msg[1], mm_info_real->msg[2], + mm_info_real->msg[3], id, new); + /* + * print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_ADDRESS, 16, 1, + * mm_info_real, sizeof(struct mount_msg_info), true); + */ + for (i = id + 1; i <= new; i++) { + index = xmfs_find_index(sbi, mm_info, true); + xmfs_info(NULL, "-------------------->index :%d", + index); + if (index == -1) { + xmfs_info(NULL, + "There is a race, which means there alreayd have 255 read nodes or there are more than one write nodes mounted parallely."); + break; + } else if (index == -2) { + id++; + break; + } + mm_info_real->msg[index] = 'D'; + id++; + break; + } + + while (index > 0 && + (mm_info_real->urma_infos[index].urma_jetty_id == 0 || + mm_info_real->urma_infos[index].urma_remote_va == 0 || + mm_info_real->urma_infos[index].urma_token_id == 0 || + mm_info_real->send_infos[index].urma_jetty_id == 0 || + mm_info_real->send_infos[index].urma_send_token_id == + 0 || + mm_info_real->send_infos[index].urma_recv_token_id == + 0)) { + schedule_timeout_interruptible(msecs_to_jiffies(10)); + } + if (index > 0) { + sbi->mount_slave++; + xmfs_import_jetty(sbi, mm_info_real->urma_infos[index], + index, false); + xmfs_import_jetty(sbi, mm_info_real->send_infos[index], + index, true); + } + } + kfree(mm_info); + + return 0; +} + +static int xmfs_slave_monitor_fn(void *data) +{ + struct xmfs_sb_info *sbi = data; + int threshold = sbi->freq; + + xmfs_info(NULL, "threshold is %d", threshold); + struct inode *inode; + unsigned long index; + struct page_freq *page_freq; + int cnt; + + while (!kthread_should_stop()) { + spin_lock(&sbi->sb->s_inode_list_lock); + list_for_each_entry(inode, &sbi->sb->s_inodes, i_sb_list) { + if (S_ISDIR(inode->i_mode)) + continue; + + xa_for_each(&(XMFS_I(inode)->page_freq), index, + page_freq) { + if (page_freq->in_cache || + page_freq->never_in_cache || + page_freq->already_in_cache) { + continue; + } + cnt = atomic_read(&page_freq->read_count); + if (cnt >= threshold) + page_freq->in_cache = true; + } + } + spin_unlock(&sbi->sb->s_inode_list_lock); + + schedule_timeout_interruptible(msecs_to_jiffies(10000)); + } + + return 0; +} + +static int xmfs_init_msg_queue(struct xmfs_sb_info *sbi) +{ + struct task_struct *thread; + + if (sbi->id == 0) + thread = kthread_run(xmfs_host_mount_fn, sbi, "msg"); + else if (!sbi->always_hot && sbi->freq) + thread = kthread_run(xmfs_slave_monitor_fn, sbi, + "slave_monitor"); + else + return 0; + + if (IS_ERR(thread)) + return PTR_ERR(thread); + sbi->thread = thread; + + return 0; +} + +static int xmfs_blk_dax_notify_failure(struct dax_device *dax_devp, u64 offset, + u64 len, int mf_flags) +{ + xmfs_error(NULL, "%s: dax_devp %llx offset %llx len %lld mf_flags %x", + __func__, (u64)dax_devp, (u64)offset, (u64)len, mf_flags); + return -EOPNOTSUPP; +} + +static const struct dax_holder_operations xmfs_blk_dax_holder_ops = { + .notify_failure = xmfs_blk_dax_notify_failure, +}; + +static void xmfs_stop_threads(struct xmfs_sb_info *sbi) +{ + if (sbi->thread) { + kthread_stop(sbi->thread); + sbi->thread = NULL; + } + if (sbi->host_worker) { + kthread_stop(sbi->host_worker); + sbi->host_worker = NULL; + } + if (sbi->gc_thread) { + kthread_stop(sbi->gc_thread); + sbi->gc_thread = NULL; + } +} + +static void xmfs_destroy_workqueue(struct workqueue_struct **workqueue) +{ + if (!*workqueue) + return; + + flush_workqueue(*workqueue); + destroy_workqueue(*workqueue); + *workqueue = NULL; +} + +static void xmfs_release_backing_resources(struct xmfs_sb_info *sbi) +{ + if (sbi->dax_filp) { + filp_close(sbi->dax_filp, NULL); + sbi->dax_filp = NULL; + } + if (sbi->dax_dev) { + fs_put_dax(sbi->dax_dev, sbi); + sbi->dax_dev = NULL; + } + if (sbi->handlep) { + bdev_release(sbi->handlep); + sbi->handlep = NULL; + } +} + +static void xmfs_restore_anon_dev(struct super_block *sb, + struct xmfs_sb_info *sbi) +{ + if (sbi->dev) + sb->s_dev = sbi->dev; +} + +static int xmfs_ctx_validate(struct fs_context *fc) +{ + struct xmfs_fs_context *ctx = fc->fs_private; + const char *source = fc->source; + + if (!source) + return invalf(fc, "xmfs: source is required"); + if (!strstr(source, "/dev/pmem") && + !strstr(source, "/dev/obmm") && + !strstr(source, "/dev/loop")) + return invalf(fc, "xmfs: unsupported source %s", source); + + if (ctx->mode < XMFS_SHARED_MEM_MODE || + ctx->mode > XMFS_HYBRID_MODE) + return invalf(fc, "xmfs: invalid mode %d", ctx->mode); + if (ctx->host && ctx->mode != XMFS_SHARED_MEM_MODE) { + errorf(fc, "xmfs: host requires shared-memory mode"); + return -EOPNOTSUPP; + } + if (!IS_ENABLED(CONFIG_XMFS_FS_URMA) && + ctx->mode != XMFS_SHARED_MEM_MODE) { + errorf(fc, "xmfs: mode %d requires URMA support", ctx->mode); + return -EOPNOTSUPP; + } + return 0; +} + +static int xmfs_prepare_slave_transport(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx) +{ + int err; + + if (ctx->host || ctx->mode == XMFS_SHARED_MEM_MODE) + return 0; + + err = xmfs_urma_register(sbi, ctx); + if (err) + goto unregister; + + err = xmfs_urma_import(sbi); + if (!err) + return 0; + + sbi->magic = 0; + xmfs_urma_unimport(sbi); +unregister: + xmfs_urma_unregister(sbi); + return err; +} + +static int xmfs_prepare_host_transport(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx) +{ + if (!ctx->host) + return 0; + +#ifdef CONFIG_XMFS_FS_URMA + return xmfs_urma_register(sbi, ctx); +#else + return 0; +#endif +} + +static int xmfs_map_pmem_device(struct super_block *sb, struct fs_context *fc) +{ + struct xmfs_fs_context *ctx = fc->fs_private; + struct xmfs_sb_info *sbi = XMFS_SB(sb); + struct bdev_handle *handlep; + struct dax_device *dax_dev; + struct pmem_device *pmem; + u64 start_off = 0; + int err; + + handlep = bdev_open_by_path(fc->source, FMODE_READ | FMODE_WRITE, + sbi, NULL); + if (!handlep || IS_ERR(handlep)) + return handlep ? PTR_ERR(handlep) : -ENODEV; + if (!handlep->bdev || IS_ERR(handlep->bdev)) { + xmfs_error(NULL, "%s: failed blkdev_get_by_path(%s)", + __func__, fc->source); + err = handlep->bdev ? PTR_ERR(handlep->bdev) : -EINVAL; + bdev_release(handlep); + return err; + } + + sbi->dev = sb->s_dev; + sb->s_dev = handlep->bdev->bd_dev; + dax_dev = fs_dax_get_by_bdev(handlep->bdev, &start_off, + sbi /* holder */, + &xmfs_blk_dax_holder_ops); + if (IS_ERR(dax_dev) || !dax_dev) { + xmfs_error(NULL, + "%s: unable to get pmem dax dev from handlep->bdev", + __func__); + bdev_release(handlep); + return -ENODEV; + } + + pmem = dax_get_private(dax_dev); + if (IS_ERR(pmem) || !pmem) { + xmfs_error(NULL, + "%s: unable to get pmem dev from handlep->bdev", + __func__); + fs_put_dax(dax_dev, sbi); + bdev_release(handlep); + return -ENODEV; + } + + xmfs_info(NULL, "pmem %ld %lld", pmem->size, pmem->data_offset); + if (!ctx->size || ctx->size > pmem->size - pmem->data_offset) + ctx->size = pmem->size - pmem->data_offset; + + sbi->start = pmem->phys_addr + pmem->data_offset; + sbi->len = ctx->size; + sbi->kaddr = (void *)ioremap_cache(sbi->start, ctx->size); + xmfs_info(NULL, "pmem %p", sbi->kaddr); + /* sbi->dax_dev = dax_dev; */ + sbi->handlep = handlep; + fs_put_dax(dax_dev, sbi); + /* bdev_release(handlep); */ + + return 0; +} + +static int xmfs_probe_slave_backing(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx, + struct xmfs_sb_index *xmsi) +{ + int err; + + err = get_xmsi_and_copy(sbi, xmsi); + if (unlikely(err)) + return err; + if (xmsi->magic != XMFS_MAGIC) { + xmfs_info(NULL, "no host mount yet"); + return -EINVAL; + } + if (xmsi->len != ctx->size) { + xmfs_info(NULL, + "slave mount uses size %ld should use same size %ld as host", + ctx->size, xmsi->len); + ctx->size = xmsi->len; + } + + return 0; +} + +static int xmfs_map_pmem_address(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx, + struct xmfs_sb_index *xmsi) +{ + int err; + + if (!ctx->host) { + sbi->kaddr = (void *)ioremap_cache(ctx->addr, SB_SIZE); + err = xmfs_probe_slave_backing(sbi, ctx, xmsi); + if (err) { + xmfs_error(NULL, + "hardware memory error when reading xmsi during slave mount 1."); + return err; + } + iounmap(sbi->kaddr); + sbi->kaddr = NULL; + } + + sbi->kaddr = (void *)ioremap_cache(ctx->addr, ctx->size); + sbi->start = ctx->addr; + sbi->len = ctx->size; + xmfs_info(NULL, "pmem addr %p", sbi->kaddr); + + return 0; +} + +static int xmfs_map_obmm(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx, + struct xmfs_sb_index *xmsi) +{ + int err; + + if (!ctx->host) { + sbi->kaddr = (void *)memremap(ctx->addr, SB_SIZE, MEMREMAP_WB); + sbi->debug = ctx->debug; + err = xmfs_probe_slave_backing(sbi, ctx, xmsi); + if (err) { + xmfs_error(NULL, + "hardware memory error when reading xmsi during slave mount 2."); + return err; + } + iounmap(sbi->kaddr); + sbi->kaddr = NULL; + } + + sbi->kaddr = memremap(ctx->addr, ctx->size, MEMREMAP_WB); + sbi->start = ctx->addr; + sbi->len = ctx->size; + xmfs_info(NULL, "obmm addr %p", sbi->kaddr); + + return 0; +} + +static int xmfs_map_loop(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx, + struct xmfs_sb_index *xmsi) +{ + int err; + + if (ctx->host) { + sbi->kaddr = vzalloc(ctx->size); + xmfs_info(NULL, "mem addr %p", sbi->kaddr); + sbi->len = ctx->size; + return 0; + } + + sbi->debug = ctx->debug; + err = get_xmsi_and_copy(sbi, xmsi); + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_ADDRESS, 16, 1, + xmsi, sizeof(struct xmfs_sb_index), true); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading xmsi during slave mount 1."); + return err; + } + if (xmsi->magic != XMFS_MAGIC) { + xmfs_info(NULL, "no host mount yet"); + return -EINVAL; + } + if (xmsi->len != ctx->size) + ctx->size = xmsi->len; + + sbi->kaddr = sbi; + return 0; +} + +static int xmfs_map_backing(struct super_block *sb, struct fs_context *fc, + struct xmfs_sb_index *xmsi) +{ + struct xmfs_fs_context *ctx = fc->fs_private; + struct xmfs_sb_info *sbi = XMFS_SB(sb); + const char *source = fc->source; + int err; + + if (!source) + return -EINVAL; + + if (strstr(source, "/dev/pmem")) { + if (ctx->addr) + err = xmfs_map_pmem_address(sbi, ctx, xmsi); + else + err = xmfs_map_pmem_device(sb, fc); + } else if (strstr(source, "/dev/obmm")) { + err = xmfs_map_obmm(sbi, ctx, xmsi); + } else if (strstr(source, "/dev/loop")) { + err = xmfs_map_loop(sbi, ctx, xmsi); + } else { + xmfs_info(NULL, "other"); + return -EINVAL; + } + if (err) + return err; + + if (IS_ERR_OR_NULL(sbi->kaddr)) { + xmfs_error(NULL, "map error %ld", PTR_ERR(sbi->kaddr)); + return sbi->kaddr ? PTR_ERR(sbi->kaddr) : -EINVAL; + } + + if (ctx->host) + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_ADDRESS, 16, 1, + sbi->kaddr, 512, true); + + return 0; +} + +static int xmfs_prepare_host_mount(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx) +{ + struct mount_msg_info *mm_info_real = get_mnt_msg(sbi); + struct mount_msg_info *mm_info = kmalloc(sizeof(*mm_info), GFP_KERNEL); + int loop_count = 0; + int old, tmp; + int err; + +host_get_id: + err = copy_mc_to_kernel(mm_info, get_mnt_msg(sbi), sizeof(*mm_info)); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading superblock at beginning of mounting host and getting id"); + goto free_msg; + } + + old = atomic_read(&mm_info->host_mount); + if (old < 0 || old > 1) { + kfree(mm_info); + goto host_continue; + } else if (old == 1) { + schedule_timeout_interruptible(msecs_to_jiffies(10)); + goto host_get_id; + } + + tmp = atomic_cmpxchg(&mm_info_real->host_mount, old, 1); + if (tmp == old) { + mm_info_real->msg[0] = 'T'; + } else { + xmfs_info(NULL, + "exchange failed, someone else mounting parallel, skip this mount"); + err = -EINVAL; + goto free_msg; + } + + do { + err = copy_mc_to_kernel(mm_info, get_mnt_msg(sbi), + sizeof(*mm_info)); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading superblock at beginning of mounting"); + goto free_msg; + } + loop_count++; + schedule_timeout_interruptible(msecs_to_jiffies(10)); + } while (mm_info->msg[0] != 'D' && loop_count <= 10); + + if (mm_info->msg[0] == 'D') { + mm_info_real->msg[0] = 'C'; + tmp = atomic_cmpxchg(&mm_info_real->host_mount, 1, old); + xmfs_info(NULL, + "host msg returns meaning host already mounted %d %d", + tmp, old); + err = -EINVAL; + goto free_msg; + } + + tmp = atomic_cmpxchg(&mm_info_real->host_mount, 1, old); + if (tmp != 1) + xmfs_info(NULL, + "when checking mounting host done, there is a bug when exchange host mount value to %d", + old); + kfree(mm_info); + + if (ctx->replay) { + sbi->id = 0; + /* + * memset(get_mnt_msg(sbi), 0, + * sizeof(struct mount_msg_info)); + */ + return 0; + } + +host_continue: + memset(sbi->kaddr, 0, DATA_AREA_OFFSET); + sbi->id = 0; + memcpy(sbi->kaddr + MOUNT_MSG_OFFSET + + offsetof(struct mount_msg_info, send_infos[0]), + &sbi->urma_cfg.send_info, sizeof(struct urma_info)); +#ifdef CONFIG_X86 + xmfs_info(NULL, "x86 mount"); + /* + * err = set_memory_uc((unsigned long)sbi->kaddr, + * DATA_AREA_OFFSET >> PAGE_SHIFT); + */ + /* xmfs_info(sbi->sb, "qyf set memory return err %d", err); */ +#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) + xmfs_info(NULL, "arm mount host"); + /* + * set_memory_noncacheable((unsigned long)sbi->kaddr, + * DATA_AREA_OFFSET); + */ +#endif + return 0; + +free_msg: + kfree(mm_info); + return err; +} + +static int xmfs_prepare_slave_mount(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx, + struct xmfs_sb_index *xmsi) +{ + struct mount_msg_info *mm_info = kmalloc(sizeof(*mm_info), GFP_KERNEL); + int loop_count = 0; + int old, new; + int index; + int err; + + err = get_mnt_msg_and_copy(sbi, mm_info); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading superblock at beginning of mounting slave"); + goto free_msg; + } + +slave_get_id: + old = xmfs_atomic_read(sbi, mm_info); + new = old + 1; + xmfs_info(NULL, "------> cas target %d", new); + if (xmfs_cas(sbi, old, new) == old) { + index = xmfs_find_index(sbi, mm_info, false); + xmfs_info(NULL, "------> slave index %d", index); + if (index == -1) { + xmfs_info(NULL, + "Already have 255 mount nodes, cannot mount more"); + err = -EINVAL; + goto free_msg; + } + sbi->id = index; + xmfs_info(NULL, "------> slave id is %d", index); + } else { + err = get_mnt_msg_and_copy(sbi, mm_info); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading superblock during mount slave"); + goto free_msg; + } + goto slave_get_id; + } + + while (mm_info->msg[sbi->id] != 'D' && loop_count < 10) { + loop_count++; + schedule_timeout_interruptible(msecs_to_jiffies(1000)); + err = get_mnt_msg_and_copy(sbi, mm_info); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading superblock during mount slave %d", + index); + goto free_msg; + } + } + if (loop_count == 10) { + xmfs_info(NULL, "try 10 times to check msg mount but failed"); + err = -EINVAL; + goto free_msg; + } + + err = get_xmsi_and_copy(sbi, xmsi); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading xmsi during slave mount."); + goto free_msg; + } + if (xmsi->magic != XMFS_MAGIC || xmsi->len != ctx->size) { + xmfs_info(NULL, + "using the wrong size %ld to mount slave, should use %ld", + ctx->size, xmsi->len); + err = -EINVAL; + goto free_msg; + } + + xmfs_info(NULL, "mount slave id %d", sbi->id); + sbi->log_start = xmsi->log_start; + sbi->last_replayed_slot = (unsigned long)-1; + get_mnt_msg_and_copy(sbi, mm_info); + xmfs_info(NULL, "host send info %lld %lld %lld %lld %d %d %d", + mm_info->send_infos[0].urma_send_va, + mm_info->send_infos[0].urma_recv_va, + mm_info->send_infos[0].urma_eid_subnet_prefix, + mm_info->send_infos[0].urma_eid_interface_id, + mm_info->send_infos[0].urma_jetty_id, + mm_info->send_infos[0].urma_send_token_id, + mm_info->send_infos[0].urma_recv_token_id); + xmfs_import_jetty(sbi, mm_info->send_infos[0], 0, true); + xmfs_pmem_write(sbi, + MOUNT_MSG_OFFSET + + offsetof(struct mount_msg_info, urma_infos[index]), + &sbi->urma_cfg.own_info, sizeof(struct urma_info)); + xmfs_pmem_write(sbi, + MOUNT_MSG_OFFSET + + offsetof(struct mount_msg_info, send_infos[index]), + &sbi->urma_cfg.send_info, sizeof(struct urma_info)); + +free_msg: + kfree(mm_info); + return err; +} + +static void xmfs_init_runtime_state(struct xmfs_sb_info *sbi, + const struct xmfs_fs_context *ctx) +{ + sbi->last_update = 0; + sbi->last_replayed_slot = (unsigned long)-1; + sbi->debug = ctx->debug; + sbi->fallocate_size = ctx->fallocate_size; + spin_lock_init(&sbi->space_lock); + init_rwsem(&sbi->chain_rwsem); + spin_lock_init(&sbi->version_lock); + mutex_init(&sbi->replay_lock); + spin_lock_init(&sbi->local_ring_lock); + xa_init(&sbi->pinned_inodes); +#ifdef CONFIG_XMFS_FS_URMA + memcpy(sbi->dev_name, ctx->dev_name, sizeof(ctx->dev_name)); +#endif + sbi->host_worker = NULL; + sbi->max_slave_id = 0; + + spin_lock_init(&sbi->gc_ctl.gc_lock); + init_waitqueue_head(&sbi->gc_ctl.wait); + sbi->gc_ctl.gc_running = false; + sbi->gc_ctl.gc_batch = XMFS_GC_BATCH; + sbi->gc_ctl.gc_cursor = 0; + sbi->gc_interval = XMFS_GC_INTERVAL_DEFAULT; + sbi->mount_slave = 0; +} + +static void xmfs_init_log_geometry(struct xmfs_sb_info *sbi, int log_size) +{ + if (!log_size) { + sbi->log_size = XMFS_DEFAULT_CHUNK_SIZE; + sbi->trunk_size = XMFS_DEFAULT_CHUNK_SIZE; + } else if (log_size > XMFS_DEFAULT_CHUNK_SIZE) { + sbi->trunk_size = log_size; + sbi->log_size = log_size; + } else { + sbi->trunk_size = XMFS_DEFAULT_CHUNK_SIZE; + sbi->log_size = log_size; + } +} + +static int xmfs_configure_super(struct super_block *sb, + struct xmfs_fs_context *ctx, + struct xmfs_sb_index *xmsi) +{ + struct xmfs_sb_info *sbi = XMFS_SB(sb); + int err; + + xmfs_init_runtime_state(sbi, ctx); + if (ctx->host) + xmfs_init_log_geometry(sbi, ctx->log_size); + + err = get_xmsi_and_copy(sbi, xmsi); + if (unlikely(err)) { + xmfs_error(NULL, + "hardware memory error when reading xmsi during host mount 2."); + return err; + } + + if (ctx->host && ctx->replay && xmsi->magic == XMFS_MAGIC) { + xmfs_info(NULL, "replay"); + if (sbi->log_size != xmsi->log_size || ctx->size != xmsi->len) { + memset(sbi->kaddr, 0, DATA_AREA_OFFSET); + sbi->id = 0; + ctx->replay = false; + } + } + if (ctx->host && ctx->replay && xmsi->magic != XMFS_MAGIC) { + memset(sbi->kaddr, 0, DATA_AREA_OFFSET); + sbi->id = 0; + ctx->replay = false; + } + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = XMFS_MAGIC; + sb->s_op = &xmfs_ops; + xmfs_init_dentry_operations(sb); + sb->s_time_gran = 1; + if (sbi->id == 0) + uuid_gen(&sb->s_uuid); + if (ctx->host) { + sb->s_xattr = xmfs_host_xattr_handlers; + } else { + sb->s_xattr = xmfs_slave_xattr_handlers; + sbi->log_size = xmsi->log_size; + ctx->log_size = sbi->log_size; + xmfs_init_log_geometry(sbi, ctx->log_size); + } + + sbi->max_chunk = + (ctx->size - DATA_AREA_OFFSET) / sbi->trunk_size - 1; + xmfs_info(NULL, "max chunk %ld %ld %ld %ld", sbi->max_chunk, + sizeof(struct xmfs_log_trunk), sizeof(struct mount_msg_info), + DATA_AREA_OFFSET); + + return 0; +} + +static int xmfs_create_root(struct super_block *sb) +{ + struct xmfs_sb_info *sbi = XMFS_SB(sb); + + sb->s_root = d_make_root(xmfs_get_root(sb, sbi->id)); + if (!sb->s_root) + return -ENOMEM; + + return 0; +} + +static int xmfs_alloc_workqueues(struct xmfs_sb_info *sbi) +{ + sbi->prefetch_wq = alloc_workqueue("xmfs-prefetch", + WQ_UNBOUND | WQ_MEM_RECLAIM, 16); + if (!sbi->prefetch_wq) + return -ENOMEM; + + sbi->send_wq = alloc_workqueue("xmfs-send-req", + WQ_UNBOUND | WQ_MEM_RECLAIM, 128); + if (!sbi->send_wq) + return -ENOMEM; + + return 0; +} + +static int xmfs_start_workers(struct xmfs_sb_info *sbi) +{ + struct xmfs_inode_table *tbl; + int err; + + err = xmfs_init_msg_queue(sbi); + if (sbi->id == 0) { + sbi->gc_thread = kthread_run(xmfs_gc_thread_fn, sbi, "xmfs_gc"); + if (IS_ERR(sbi->gc_thread)) + sbi->gc_thread = NULL; + sbi->max_slave_id = 63; + + tbl = (struct xmfs_inode_table *)get_meta(sbi, + INODE_TABLE_OFFSET, + INODE_TABLE_SIZE); + tbl->magic = XMFS_INODE_TABLE_MAGIC; + } else { + sbi->gc_thread = kthread_run(xmfs_gc_thread_slave_fn, sbi, + "xmfs_slave_gc"); + if (IS_ERR(sbi->gc_thread)) + sbi->gc_thread = NULL; + } + + return err; +} + +static int xmfs_finalize_host_mount(struct super_block *sb, + struct xmfs_fs_context *ctx, + struct xmfs_sb_index *xmsi) +{ + struct xmfs_sb_info *sbi = XMFS_SB(sb); + struct xmfs_sb_index *xmsi_real; + + if (ctx->replay && xmsi->magic == XMFS_MAGIC) { + xmfs_info(NULL, "replay"); + if (sbi->log_size != xmsi->log_size) { + xmfs_info(NULL, + "mount host again with wrong log size %d should be %d", + sbi->log_size, xmsi->log_size); + return -EINVAL; + } + if (ctx->size != xmsi->len) { + xmfs_info(NULL, + "mount host again with wrong size %ld should be %ld", + ctx->size, xmsi->len); + return -EINVAL; + } + sbi->during_replay = true; + xmfs_replay_new_entries(sbi, NULL, -1); + sbi->during_replay = false; + } + + if (ctx->replay) { + struct xmfs_sb_index *xmsi_r = get_xmsi(sbi); + + if (xmsi_r) { + unsigned long used_trunks = 0; + unsigned long i; + + for (i = 0; + i < xmsi_r->count && i < XMFS_BITMAP_CAPACITY; + i++) { + if (xmsi_r->bitmap[i] != 'u') + used_trunks++; + } + xmsi_r->used_trunk_count = used_trunks; + put_xmsi(sbi, xmsi_r); + } + } + + xmsi_real = get_xmsi(sbi); + xmsi_real->magic = XMFS_MAGIC; + xmsi_real->log_size = sbi->log_size; + xmsi_real->len = ctx->size; + xmsi_real->fsid = uuid_to_fsid(sb->s_uuid.b); + if (!ctx->replay) { + memset(xmsi_real->bitmap, 'u', XMFS_BITMAP_CAPACITY); + xmsi_real->bitmap_size = 0; + xmsi_real->used_trunk_count = 0; + } + + /* xmfs_populate_from_inode_table(sbi); */ + if (!ctx->replay && XMFS_I(sb->s_root->d_inode)->log_start == 0) { + struct xmfs_log xm_log; + unsigned long xmlt_addr; + + xm_log.ops = XMFS_ROOT; + xmfs_info(NULL, "root 1 %p", XMFS_I(sb->s_root->d_inode)); + xmlt_addr = xmfs_add_log_v3(&xm_log, sbi, 0, + sb->s_root->d_inode->i_ino, true); + if (xmlt_addr < 0) + return -EINVAL; + sbi->last_update = 0; + xmfs_info(NULL, "log start %ld", + XMFS_I(sb->s_root->d_inode)->log_start); + } + /* + * sbi->xmfs_log_wq = alloc_ordered_workqueue("xmfs_log_wq", + * WQ_MEM_RECLAIM); + */ + + return 0; +} + +static int xmfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct xmfs_fs_context *ctx = fc->fs_private; + struct xmfs_sb_index *xmsi; + struct xmfs_sb_info *sbi; + int err; + + sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); + if (!sbi) + return -ENOMEM; + sb->s_fs_info = sbi; + xmfs_init_log_ratelimits(sbi); + sbi->mode = ctx->mode; + sbi->magic = XMFS_MAGIC; + xmsi = kmalloc(sizeof(*xmsi), GFP_KERNEL); + + err = xmfs_prepare_slave_transport(sbi, ctx); + if (err) + goto free_sbi; + + err = xmfs_map_backing(sb, fc, xmsi); + if (err) + goto free_kaddr; + + err = xmfs_prepare_host_transport(sbi, ctx); + if (err) + goto release_dev; + + if (ctx->host) + err = xmfs_prepare_host_mount(sbi, ctx); + else + err = xmfs_prepare_slave_mount(sbi, ctx, xmsi); + if (err) + goto release_dev; + + err = xmfs_configure_super(sb, ctx, xmsi); + if (err) + goto stop_thread; + + err = xmfs_create_root(sb); + if (err) + goto release_dev; + + err = xmfs_alloc_workqueues(sbi); + if (err) + goto release_root; + + err = security_inode_init_security(sb->s_root->d_inode, + sb->s_root->d_parent->d_inode, + &sb->s_root->d_name, xmfs_initxattrs, + NULL); + if (err) + goto release_root; + + sbi->sb = sb; + sbi->used_ino = 0; + sbi->freq = ctx->freq; + sbi->always_hot = ctx->always_hot; + err = xmfs_start_workers(sbi); + if (err) + goto stop_thread; + + if (ctx->host) { + err = super_setup_bdi(sb); + if (err) { + xmfs_info(NULL, "setup bdi failed"); + goto stop_thread; + } + + err = xmfs_finalize_host_mount(sb, ctx, xmsi); + if (err) + goto stop_thread; + } + + kfree(xmsi); + if (ctx->host) + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_ADDRESS, 16, 1, + sbi->kaddr, 400, true); + return 0; + +stop_thread: + xmfs_stop_threads(sbi); +release_root: + dput(sb->s_root); + sb->s_root = NULL; + xmfs_destroy_workqueue(&sbi->prefetch_wq); + xmfs_destroy_workqueue(&sbi->send_wq); +release_dev: + xmfs_release_backing_resources(sbi); +free_kaddr: + if (ctx->host) + vfree(sbi->kaddr); + sbi->magic = 0; + xmfs_urma_unimport(sbi); + xmfs_urma_unregister(sbi); +free_sbi: + xmfs_restore_anon_dev(sb, sbi); + if (sb->s_fs_info == sbi) + sb->s_fs_info = NULL; + kfree(sbi); + kfree(xmsi); + return err; +} + +static int xmfs_get_tree(struct fs_context *fc) +{ + int err; + + err = xmfs_ctx_validate(fc); + if (err) + return err; + + return get_tree_nodev(fc, xmfs_fill_super); +} + +static const struct fs_parameter_spec xmfs_param_specs[] = { + fsparam_bool("host", Opt_host), + fsparam_bool("replay", Opt_replay), + fsparam_bool("debug", Opt_debug), + fsparam_bool("hot", Opt_always_hot), + fsparam_u64("addr", Opt_addr), + fsparam_u64("size", Opt_size), + fsparam_u64("fallocate", Opt_fallocate), + fsparam_s32("freq", Opt_freq), + fsparam_s32("log", Opt_log), + fsparam_s32("mode", Opt_mode), +#ifdef CONFIG_XMFS_FS_URMA + fsparam_string("urma_dev_name", Opt_urma_dev), + fsparam_u64("urma_va", Opt_urma_va), + fsparam_u64("urma_eid_subnet_prefix", Opt_urma_eid_subnet_prefix), + fsparam_u64("urma_eid_interface_id", Opt_urma_eid_interface_id), + fsparam_u32("urma_jetty_id", Opt_urma_jetty_id), + fsparam_u32("urma_token_id", Opt_urma_token_id), +#endif + {} +}; + +static int xmfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct xmfs_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, xmfs_param_specs, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_host: + ctx->host = result.boolean; + break; + case Opt_replay: + ctx->replay = result.boolean; + break; + case Opt_addr: + ctx->addr = result.uint_64; + break; + case Opt_size: + ctx->size = result.uint_64; + break; + case Opt_freq: + ctx->freq = result.int_32; + break; + case Opt_log: + ctx->log_size = result.int_32; + break; + case Opt_debug: + ctx->debug = result.boolean; + break; + case Opt_always_hot: + ctx->always_hot = result.boolean; + break; + case Opt_fallocate: + ctx->fallocate_size = result.uint_64; + break; + case Opt_mode: + ctx->mode = result.int_32; + break; +#ifdef CONFIG_XMFS_FS_URMA + case Opt_urma_va: + ctx->urma_va = result.uint_64; + break; + case Opt_urma_eid_subnet_prefix: + ctx->urma_eid_subnet_prefix = result.uint_64; + break; + case Opt_urma_eid_interface_id: + ctx->urma_eid_interface_id = result.uint_64; + break; + case Opt_urma_jetty_id: + ctx->urma_jetty_id = result.uint_32; + break; + case Opt_urma_token_id: + ctx->urma_token_id = result.uint_32; + break; + case Opt_urma_dev: + memcpy(ctx->dev_name, param->string, strlen(param->string)); + break; +#endif + default: + return -ENOPARAM; + } + + return 0; +} + +static void xmfs_fs_context_free(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations xmfs_fs_context_ops = { + .get_tree = xmfs_get_tree, + .parse_param = xmfs_parse_param, + .free = xmfs_fs_context_free, +}; + +static int xmfs_init_fs_context(struct fs_context *fc) +{ + struct xmfs_fs_context *ctx; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + fc->fs_private = ctx; + fc->ops = &xmfs_fs_context_ops; + + return 0; +} + +static void xmfs_unpin_all_inodes(struct xmfs_sb_info *sbi) +{ + struct inode *inode; + unsigned long index; + + /* Each xarray entry retains one inode reference until it is erased here. */ + xa_for_each(&sbi->pinned_inodes, index, inode) { + inode = xa_erase(&sbi->pinned_inodes, index); + if (!inode) + continue; + + iput(inode); + cond_resched(); + } + + xa_destroy(&sbi->pinned_inodes); +} + +static void xmfs_shutdown_sb(struct super_block *sb, + struct xmfs_sb_info *sbi) +{ + mnt_msg_write(sbi, sbi->id, 'C'); + sbi->magic = 0; + + xmfs_stop_threads(sbi); + xmfs_destroy_workqueue(&sbi->prefetch_wq); + xmfs_destroy_workqueue(&sbi->send_wq); + + xmfs_restore_anon_dev(sb, sbi); + xmfs_unpin_all_inodes(sbi); +} + +static void xmfs_release_sb_info(struct xmfs_sb_info *sbi) +{ + xmfs_release_backing_resources(sbi); + if (sbi->kaddr != sbi) + vfree(sbi->kaddr); + sbi->kaddr = NULL; + xmfs_urma_unimport(sbi); + xmfs_urma_unregister(sbi); + kfree(sbi); +} + +static void xmfs_kill_sb(struct super_block *sb) +{ + struct xmfs_sb_info *sbi = XMFS_SB(sb); + + if (sbi) + xmfs_shutdown_sb(sb, sbi); + + kill_litter_super(sb); + + if (sbi) + xmfs_release_sb_info(sbi); +} + +static struct file_system_type xmfs_fs_type = { + .name = "xmfs", + .owner = THIS_MODULE, + .init_fs_context = xmfs_init_fs_context, + .parameters = xmfs_param_specs, + .kill_sb = xmfs_kill_sb, +}; + +static void xmfs_init_once(void *data) +{ + struct xmfs_inode_info *xmfs_inode = data; + + inode_init_once(&xmfs_inode->vfs_inode); +} + +static int __init init_xmfs(void) +{ + int err; + + xmfs_inode_cachep = kmem_cache_create("xmfs_inode_cache", + sizeof(struct xmfs_inode_info), 0, + SLAB_ACCOUNT, xmfs_init_once); + if (!xmfs_inode_cachep) + return -ENOMEM; + + err = register_filesystem(&xmfs_fs_type); + if (err) + kmem_cache_destroy(xmfs_inode_cachep); + + return err; +} + +static void __exit exit_xmfs(void) +{ + unregister_filesystem(&xmfs_fs_type); + kmem_cache_destroy(xmfs_inode_cachep); +} + +module_init(init_xmfs); +module_exit(exit_xmfs); + +MODULE_AUTHOR("Yifan Qiao <qiaoyifan4(a)huawei.com>"); +MODULE_DESCRIPTION("Log Structured Filesystem for UB"); +MODULE_LICENSE("GPL"); diff --git a/fs/xmfs/symlink.c b/fs/xmfs/symlink.c new file mode 100644 index 000000000000..20870b18f325 --- /dev/null +++ b/fs/xmfs/symlink.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#include "xmfs_i.h" + +#include <linux/pagemap.h> + +static void xmfs_put_link(void *arg) +{ + /* xmfs_info(NULL, "qyf put folio %p", arg); */ + folio_put(arg); +} + +static void xmfs_put_page(void *arg) +{ + /* xmfs_info(NULL, "qyf put page %p", page_address(arg)); */ + put_page(arg); +} + +static const char *xmfs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (!dentry) { + struct folio *folio = filemap_get_folio(inode->i_mapping, 0); + + if (IS_ERR(folio)) + return ERR_PTR(-ECHILD); + set_delayed_call(done, xmfs_put_link, folio); + return folio_address(folio); + } + struct folio *folio = filemap_get_folio(inode->i_mapping, 0); + + if (IS_ERR(folio)) { + struct xmfs_log *ulog = get_log(sbi, XMFS_I(inode)->symname); + struct page *page = alloc_page(GFP_KERNEL); + char *ret = kmap_local_page(page); + + if (ulog->end_index > sbi->len) { + xmfs_info(inode->i_sb, "get link wrong %ld %ld", + inode->i_ino, XMFS_I(inode)->symname); + } + if (!ulog->end_index) { + memcpy(ret, ulog->name, strlen(ulog->name) + 1); + } else { + memcpy(ret, get_data(sbi, ulog->end_index, ulog->index), + ulog->index); + } + kunmap_local(ret); + set_page_dirty(page); + + set_delayed_call(done, xmfs_put_page, page); + if (!ret || !page_address(page)) { + xmfs_info(inode->i_sb, "get link 1 return null %s", + dentry->d_name.name); + } + return page_address(page); + } + + set_delayed_call(done, xmfs_put_link, folio); + return folio_address(folio); +} + +static const struct inode_operations xmfs_symlink_inode_operations = { + .setattr = xmfs_setattr, + .getattr = xmfs_getattr, + .get_link = xmfs_get_link, + .listxattr = xmfs_listxattr, + .update_time = xmfs_update_time, +}; + +void xmfs_init_symlink_mapping(struct inode *inode) +{ + xmfs_init_file_mapping(inode); +} + +void xmfs_init_symlink_operations(struct inode *inode) +{ + inode->i_op = &xmfs_symlink_inode_operations; +} diff --git a/fs/xmfs/urma.c b/fs/xmfs/urma.c new file mode 100644 index 000000000000..d73b825bdbfe --- /dev/null +++ b/fs/xmfs/urma.c @@ -0,0 +1,2482 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "xmfs_i.h" + +#ifdef CONFIG_XMFS_FS_URMA + +#include <ub/urma/ubcore_types.h> +#include <ub/urma/ubcore_uapi.h> +#include <ub/urma/ubcore_api.h> +#include <linux/hashtable.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/completion.h> +#include <linux/atomic.h> +#include <linux/kthread.h> +#include <linux/delay.h> +#include <linux/jiffies.h> +#include <linux/interrupt.h> +#include <linux/cacheflush.h> + +static struct ubcore_device *g_urma_dev[10]; +static char g_dev_name[100]; + +enum xmfs_urma_status { + URMA_ISSUING = 1, + URMA_COMPLETE, + URMA_ERROR, +}; + +struct wr_status { + int magic; + refcount_t refs; + int ret; + int ops; + int slot; + enum xmfs_urma_status status; + struct completion done; +}; + +#define XMFS_ACCESS_LOCAL_ONLY 0x1 +#define XMFS_ACCESS_READ (0x1 << 1) +#define XMFS_ACCESS_WRITE (0x1 << 2) +#define XMFS_ACCESS_ATOMIC (0x1 << 3) + +#define XMFS_DEPTH 1024 +#define XMFS_POST_RETRY_CNT 100 + +static void xmfs_wrs_get(struct wr_status *wrs) +{ + refcount_inc(&wrs->refs); +} + +static void xmfs_wrs_put(struct wr_status **wrs) +{ + if (refcount_dec_and_test(&(*wrs)->refs)) { + /* xmfs_info(NULL, "qyf FREE 1 wrs=%px", *wrs); */ + kfree(*wrs); + *wrs = NULL; + } +} + +static int xmfs_add_udma_device(struct ubcore_device *ubc_dev) +{ + xmfs_info(NULL, "add device %p %s %p %p %d", ubc_dev, ubc_dev->dev_name, + ubc_dev->ops->register_seg, ubc_dev->ops->unregister_seg, + IS_ERR_OR_NULL(ubc_dev->eid_table.eid_entries)); + xmfs_info(NULL, "hooks check %p %p %p", ubc_dev->ops->import_jetty, + ubc_dev->ops->import_jetty_ex, ubc_dev->ops->import_seg); + if (ubc_dev == NULL) { + xmfs_error(NULL, + "Param ubc_dev is null, add udma device fail."); + return -EINVAL; + } + + if (strcmp(ubc_dev->dev_name, g_dev_name) == 0) { + xmfs_info(NULL, "find udma dev %s", ubc_dev->dev_name); + g_urma_dev[0] = ubc_dev; + xmfs_error(NULL, "--------->g_urma_dev set"); + } + return 0; +} + +static void xmfs_remove_udma_device(struct ubcore_device *ubc_dev, + void *client_ctx) +{ + xmfs_error(NULL, "ubma device removed %s, all in-flight ops will fail", + ubc_dev->dev_name); +} + +static int xmfs_urma_pool_init(struct xmfs_urma_pool *pool, void *base, + size_t total_size, size_t slot_size) +{ + pool->base = base; + pool->total_size = total_size; + pool->slot_size = slot_size; + + if (slot_size == PAGE_SIZE) + pool->nr_slots = total_size / slot_size; + else + pool->nr_slots = XMFS_DEPTH; + pool->free_slots = pool->nr_slots; + pool->bitmap = bitmap_zalloc(pool->nr_slots, GFP_KERNEL); + if (!pool->bitmap) + return -ENOMEM; + xmfs_info(NULL, "pool init nr %d slot_size %ld total %ld", + pool->nr_slots, pool->slot_size, pool->total_size); + spin_lock_init(&pool->lock); + init_waitqueue_head(&pool->waitq); + + return 0; +} + +static void xmfs_urma_pool_destroy(struct xmfs_urma_pool *pool) +{ + bitmap_free(pool->bitmap); +} + +void xmfs_urma_unregister(struct xmfs_sb_info *sbi) +{ + if (!sbi) + return; + for (int i = 0; i < 5; i++) + if (sbi->urma_cfg.jetty[i]) + ubcore_delete_jetty(sbi->urma_cfg.jetty[i]); + if (sbi->urma_cfg.jfr) + ubcore_delete_jfr(sbi->urma_cfg.jfr); + if (sbi->urma_cfg.jfc) + ubcore_delete_jfc(sbi->urma_cfg.jfc); + if (sbi->urma_cfg.local_seg) + ubcore_unregister_seg(sbi->urma_cfg.local_seg); + if (sbi->urma_cfg.urma_local_va) + vfree(sbi->urma_cfg.urma_local_va); + if (sbi->urma_cfg.ubc_dev) + ubcore_unregister_client(&sbi->urma_cfg.ubc_client); + xmfs_urma_pool_destroy(&sbi->urma_cfg.pool); + xmfs_urma_pool_destroy(&sbi->urma_cfg.send_pool); + if (sbi->urma_cfg.send_jetty) + ubcore_delete_jetty(sbi->urma_cfg.send_jetty); + if (sbi->urma_cfg.send_jfr) + ubcore_delete_jfr(sbi->urma_cfg.send_jfr); + if (sbi->urma_cfg.send_jfc) + ubcore_delete_jfc(sbi->urma_cfg.send_jfc); + if (sbi->urma_cfg.recv_jfc) + ubcore_delete_jfc(sbi->urma_cfg.recv_jfc); + if (sbi->urma_cfg.send_seg) + ubcore_unregister_seg(sbi->urma_cfg.send_seg); + if (sbi->urma_cfg.recv_seg) + ubcore_unregister_seg(sbi->urma_cfg.recv_seg); + if (sbi->urma_cfg.recv_buf) + vfree(sbi->urma_cfg.recv_buf); + if (sbi->urma_cfg.send_buf) + vfree(sbi->urma_cfg.send_buf); + xa_destroy(&sbi->urma_cfg.grant_waits); + /* g_urma_client = NULL; */ +} + +static void *urma_pool_acquire(struct xmfs_urma_pool *pool, u32 *slot) +{ + unsigned long flags; + u32 idx; + int ret; + + for (;;) { + spin_lock_irqsave(&pool->lock, flags); + + idx = find_first_zero_bit(pool->bitmap, pool->nr_slots); + + if (idx < pool->nr_slots) { + __set_bit(idx, pool->bitmap); + pool->free_slots--; + spin_unlock_irqrestore(&pool->lock, flags); + + if (slot) + *slot = idx; + return pool->base + idx * pool->slot_size; + } + + spin_unlock_irqrestore(&pool->lock, flags); + + ret = wait_event_interruptible_timeout(pool->waitq, + READ_ONCE(pool->free_slots) >= + 1, + msecs_to_jiffies(25000)); + if (ret <= 0) + return NULL; + } + + return NULL; +} + +void *urma_pool_acquire_nslots(struct xmfs_urma_pool *pool, int nr, u32 *slots) +{ + unsigned long flags; + u32 idx, pos; + int found; + int ret; + + if (unlikely(nr <= 0 || nr > pool->nr_slots)) + return NULL; + + for (;;) { + found = 0; + pos = 0; + + spin_lock_irqsave(&pool->lock, flags); + + /* 空闲slot数量都不够，直接等待 */ + if (pool->free_slots < nr) + goto retry; + + /* 找到 nr 个空闲slot */ + while (found < nr) { + idx = find_next_zero_bit(pool->bitmap, pool->nr_slots, + pos); + + if (idx >= pool->nr_slots) + break; + + __set_bit(idx, pool->bitmap); + slots[found++] = idx; + pos = idx + 1; + } + + /* 成功拿到全部slot */ + if (likely(found == nr)) { + void *addr; + + pool->free_slots -= nr; + + addr = pool->base + slots[0] * pool->slot_size; + + spin_unlock_irqrestore(&pool->lock, flags); + return addr; + } + + /* + * 理论上不会进入这里： + * free_slots >= nr，但扫描过程中没找到足够slot。 + * 为了健壮性仍然回滚。 + */ + while (found > 0) + __clear_bit(slots[--found], pool->bitmap); + +retry: + spin_unlock_irqrestore(&pool->lock, flags); + + ret = wait_event_interruptible_timeout(pool->waitq, + READ_ONCE(pool->free_slots) >= + nr, + msecs_to_jiffies(25000)); + if (ret <= 0) + return NULL; + } +} + +void urma_pool_release_nslots(struct xmfs_urma_pool *pool, int nr, + const u32 *slots) +{ + unsigned long flags; + int i; + + spin_lock_irqsave(&pool->lock, flags); + + for (i = 0; i < nr; i++) { + WARN_ON(!test_bit(slots[i], pool->bitmap)); + __clear_bit(slots[i], pool->bitmap); + } + + pool->free_slots += nr; + + spin_unlock_irqrestore(&pool->lock, flags); + + wake_up_all(&pool->waitq); +} + +void urma_pool_release(struct xmfs_urma_pool *pool, u32 slot) +{ + unsigned long flags; + + if (WARN_ON(slot >= pool->nr_slots || slot < 0)) + return; + + spin_lock_irqsave(&pool->lock, flags); + + if (!test_bit(slot, pool->bitmap)) + xmfs_info(NULL, "err release find bit %d empty", slot); + + __clear_bit(slot, pool->bitmap); + pool->free_slots++; + spin_unlock_irqrestore(&pool->lock, flags); + + wake_up(&pool->waitq); +} + +static void *xmfs_get_recv_buf(struct xmfs_sb_info *sbi, int idx) +{ + return (char *)sbi->urma_cfg.recv_buf + + idx * sizeof(struct xmfs_rw_msg); +} + +static int xmfs_post_recv(struct xmfs_sb_info *sbi, int idx) +{ + struct ubcore_jfr_wr *bad_wr = NULL; + struct ubcore_jfr_wr wr = {}; + struct ubcore_sge sge = {}; + int ret; + + sge.addr = (uint64_t)xmfs_get_recv_buf(sbi, idx); + sge.len = sizeof(struct xmfs_rw_msg); + sge.tseg = sbi->urma_cfg.recv_seg; + + wr.src.sge = &sge; + wr.src.num_sge = 1; + wr.user_ctx = idx; + wr.next = NULL; + + ret = ubcore_post_jetty_recv_wr(sbi->urma_cfg.send_jetty, &wr, &bad_wr); + if (ret) + xmfs_info(NULL, "err xmfs post recv buffer failed ret %d", ret); + + return ret; +} + +static void xmfs_process_recv(struct xmfs_sb_info *sbi, struct ubcore_cr *cr) +{ + struct wr_status *wrs; + + if (cr->status != UBCORE_CR_SUCCESS) { + xmfs_warning(sbi->sb, + "JFC receive completion failed: status %d opcode %d", + cr->status, cr->opcode); + if (cr->status == UBCORE_CR_WR_FLUSH_ERR_DONE) + return; + } + wrs = (struct wr_status *)(uintptr_t)cr->user_ctx; + if (!virt_addr_valid(wrs)) { + xmfs_info(NULL, "jfc callback got an invalid wrs %p", wrs); + return; + } + /* xmfs_info(sbi->sb, "qyf COMP user_ctx=%llx", cr->user_ctx); */ + if (!wrs || wrs->magic != XMFS_MAGIC) { + xmfs_info(NULL, + "err cannot get a correct wrs during process recv %p", + wrs); + return; + } + if (cr->status == UBCORE_CR_SUCCESS) + wrs->status = URMA_COMPLETE; + else { + wrs->status = URMA_ERROR; + wrs->ret = cr->status; + } + if (wrs->ops == XMFS_URMA_SEND) + xmfs_info(NULL, "err slave send jfc callback"); + + if (wrs->ops == XMFS_URMA_WRITE && wrs->slot != -1) { + if (wrs->slot >= sbi->urma_cfg.pool.nr_slots || wrs->slot < 0) + xmfs_info(NULL, + "err jfc process write got wrong slot %d %d", + wrs->slot, sbi->urma_cfg.pool.nr_slots); + else + urma_pool_release(&sbi->urma_cfg.pool, wrs->slot); + } + complete(&wrs->done); + xmfs_wrs_put(&wrs); +} + +static void xmfs_jfc_callback(struct ubcore_jfc *jfc) +{ + struct ubcore_cr cr[8]; + + if (!jfc) + return; + struct xmfs_sb_info *sbi = jfc->jfc_cfg.jfc_context; + int ret, i; + + if (!jfc || !sbi || sbi->magic != XMFS_MAGIC) + return; + + if (jfc != sbi->urma_cfg.jfc) { + xmfs_error(NULL, "err foreign jfc cr, drop it"); + ubcore_rearm_jfc(jfc, false); + return; + } + + /* xmfs_info(sbi->sb, "qyf jfc callback"); */ + + do { + ret = ubcore_poll_jfc(jfc, 1, cr); + if (ret < 0) { + xmfs_info(NULL, "err jfc poll failed %d", ret); + break; + } + + for (i = 0; i < ret; i++) + xmfs_process_recv(sbi, &cr[i]); + } while (ret > 0 && jfc && sbi && sbi->magic == XMFS_MAGIC); + + ret = ubcore_rearm_jfc(jfc, false); + if (ret < 0) + xmfs_info(NULL, "err jfc rearm failed %d", ret); +} + +static int xmfs_host_send(struct xmfs_sb_info *sbi, struct ubcore_cr *cr) +{ + u32 idx; + + idx = (u32)cr->user_ctx; + if (cr->status != UBCORE_CR_SUCCESS) + /* if (cr->status != 9 && cr->status != 10) */ + xmfs_warning(sbi->sb, + "send completion failed: node %d status %d index %d", + sbi->id, cr->status, idx); + if (cr->status == UBCORE_CR_WR_FLUSH_ERR_DONE) + return 0; + /* + * if (sbi->id != 0) + * xmfs_info(sbi->sb, + * "qyf send request callback send pool slot %d release", + * idx); + */ + urma_pool_release(&sbi->urma_cfg.send_pool, idx); + return 0; +} + +static void xmfs_host_reply_worker(struct work_struct *work) +{ + struct xmfs_send_work *sw = + container_of(work, struct xmfs_send_work, work); + struct xmfs_rw_msg grant = sw->grant; + struct xmfs_rw_msg *buf; + struct xmfs_sb_info *sbi = sw->sbi; + struct ubcore_jfs_wr *bad_send_wr = NULL; + struct ubcore_jfs_wr send_wr = {}; + struct ubcore_sge sge = {}; + int slot, ret; + + buf = (struct xmfs_rw_msg *)urma_pool_acquire(&sbi->urma_cfg.send_pool, + &slot); + if (!buf) { + xmfs_info(NULL, "cannot get the buf during host process recv"); + kfree(sw); + return; + } + memcpy(buf, &grant, sizeof(struct xmfs_rw_msg)); + + sge.addr = (uint64_t)buf; + sge.len = sizeof(grant); + sge.tseg = sbi->urma_cfg.send_seg; + + send_wr.opcode = UBCORE_OPC_SEND; + send_wr.tjetty = sbi->urma_cfg.send_tjettys[grant.slave_id]; + send_wr.send.src.sge = &sge; + send_wr.send.src.num_sge = 1; + send_wr.user_ctx = slot; + send_wr.flag.value = 0; + send_wr.flag.bs.complete_enable = 1; + send_wr.next = NULL; + + ret = ubcore_post_jetty_send_wr(sbi->urma_cfg.send_jetty, &send_wr, + &bad_send_wr); + if (ret) { + urma_pool_release(&sbi->urma_cfg.send_pool, slot); + xmfs_info(NULL, "err host send post err %d", ret); + } + kfree(sw); +} + +static int xmfs_host_recv(struct xmfs_sb_info *sbi, struct ubcore_cr *cr) +{ + struct xmfs_rw_msg *req; + struct xmfs_rw_msg grant; + u32 idx; + + if (cr->status != UBCORE_CR_SUCCESS) { + idx = (u32)cr->user_ctx; + /* if (cr->status != 9 && cr->status != 10) */ + xmfs_warning(sbi->sb, + "receive completion failed: node %d status %d opcode %d index %d", + sbi->id, cr->status, cr->opcode, idx); + xmfs_post_recv(sbi, idx); + return -EINVAL; + } + + idx = (u32)cr->user_ctx; + + if (sbi->id == 0) { + if (idx >= XMFS_DEPTH || idx < 0) { + xmfs_info(NULL, "err host recv idx %d is not correct", + idx); + return -EINVAL; + } + req = (struct xmfs_rw_msg *)xmfs_get_recv_buf(sbi, idx); + int slave_id = req->slave_id; + struct xmfs_send_work *sw = kmalloc(sizeof(*sw), GFP_ATOMIC); + + if (!sw) { + xmfs_post_recv(sbi, idx); + return 0; + } + /* + * xmfs_info(sbi->sb, + * "qyf host recv callback recv pool slot %d req id %d req ops %d", + * idx, req->request_id, req->ops); + */ + xmfs_host_handle_urma_request(sbi, slave_id, req, &sw->grant); + sw->sbi = sbi; + xmfs_post_recv(sbi, idx); + /* + * xmfs_info(sbi->sb, "host process req %d %d %d", + * req->request_id, req->ops, grant.log_slot_start); + */ + if (req->ops == XMFS_SETATTR) { + kfree(sw); + return 0; + } + + INIT_WORK(&sw->work, xmfs_host_reply_worker); + queue_work(sbi->send_wq, &sw->work); + /* + * buf = (struct xmfs_rw_msg *)urma_pool_acquire( + * &sbi->urma_cfg.send_pool, &slot); + * if (!buf) { + * xmfs_info(sbi->sb, + * "cannot get the buf during host process recv"); + * xmfs_post_recv(sbi, idx); + * return 0; + * } + * memcpy(buf, &grant, sizeof(struct xmfs_rw_msg)); + * xmfs_info(sbi->sb, + * "qyf host recv callback recv pool slot %d " + * "req id %d req ops %d send pool slot %d", + * idx, req->request_id, req->ops, slot); + * xmfs_post_recv(sbi, idx); + * + * sge.addr = (uint64_t)buf; + * sge.len = sizeof(grant); + * sge.tseg = sbi->urma_cfg.send_seg; + * + * send_wr.opcode = UBCORE_OPC_SEND; + * send_wr.tjetty = sbi->urma_cfg.send_tjettys[slave_id]; + * send_wr.send.src.sge = &sge; + * send_wr.send.src.num_sge = 1; + * send_wr.user_ctx = slot; + * send_wr.flag.value = 0; + * send_wr.flag.bs.complete_enable = 1; + * send_wr.next = NULL; + * + * ret = ubcore_post_jetty_send_wr(sbi->urma_cfg.send_jetty, + * &send_wr, &bad_send_wr); + * if (ret) { + * urma_pool_release(&sbi->urma_cfg.send_pool, slot); + * xmfs_info(sbi->sb, "err host send post err %d", ret); + * } + */ + } else { + if (idx >= XMFS_DEPTH || idx < 0) { + xmfs_info(NULL, "err slave recv idx %d is not correct", + idx); + return -EINVAL; + } + req = (struct xmfs_rw_msg *)xmfs_get_recv_buf(sbi, idx); + /* + * xmfs_info(sbi->sb, + * "qyf slave recv callback recv pool slot %d " + * "req id %d req ops %d type %d log addr %ld " + * "data addr %ld %ld data num %d %d version %ld", + * idx, req->request_id, req->ops, req->type, + * req->log_trunk_addr, req->data_addr, + * req->second_data_addr, req->data_page_count, + * req->second_data_pages, req->version); + */ + if (req->ops == XMFS_GC) { + xmfs_info(NULL, "slave receive gc req %d", req->status); + if (req->status == 1) + sbi->gc_ctl.gc_running = true; + else if (req->status == 0) { + sbi->gc_ctl.gc_running = false; + wake_up_all(&sbi->gc_ctl.wait); + } else if (req->status == 2) + sbi->gc_ctl.need_replay = true; + xmfs_post_recv(sbi, idx); + return 0; + } + + memcpy(&grant, req, sizeof(struct xmfs_rw_msg)); + xmfs_post_recv(sbi, idx); + /* xmfs_info(sbi->sb, "qyf slave post recv %d", idx); */ + struct xmfs_grant_wait *gw = + xa_load(&sbi->urma_cfg.grant_waits, grant.request_id); + if (gw) { + memcpy(&gw->grant, &grant, sizeof(struct xmfs_rw_msg)); + complete(&gw->done); + } else { + xmfs_error(NULL, "err cannot get the grant wait %ld", + grant.request_id); + } + } + + return 0; +} + +static void xmfs_send_jfc_callback(struct ubcore_jfc *jfc) +{ + struct ubcore_cr cr[8]; + + if (!jfc) + return; + struct xmfs_sb_info *sbi = jfc->jfc_cfg.jfc_context; + int ret, i; + + if (!jfc || !sbi || sbi->magic != XMFS_MAGIC) + return; + + if (jfc != sbi->urma_cfg.recv_jfc && jfc != sbi->urma_cfg.send_jfc) { + xmfs_error(NULL, "err foreign send jfc cr, drop it"); + ubcore_rearm_jfc(jfc, false); + return; + } + + /* xmfs_info(sbi->sb, "send callback %d", sbi->id); */ + do { + ret = ubcore_poll_jfc(jfc, 1, cr); + if (ret < 0) { + xmfs_info(NULL, "err jfc send poll failed %d", ret); + break; + } + + for (i = 0; i < ret; i++) + xmfs_host_send(sbi, &cr[i]); + } while (ret > 0 && jfc && sbi && sbi->magic == XMFS_MAGIC); + + ret = ubcore_rearm_jfc(jfc, false); + if (ret < 0) + xmfs_info(NULL, "err jfc recv rearm failed %d", ret); +} + +static void xmfs_recv_jfc_callback(struct ubcore_jfc *jfc) +{ + struct ubcore_cr cr[8]; + + if (!jfc) + return; + struct xmfs_sb_info *sbi = jfc->jfc_cfg.jfc_context; + int ret, i; + + if (!jfc || !sbi || sbi->magic != XMFS_MAGIC) + return; + + if (jfc != sbi->urma_cfg.recv_jfc && jfc != sbi->urma_cfg.send_jfc) { + xmfs_error(NULL, "err foreign recv jfc cr, drop it"); + ubcore_rearm_jfc(jfc, false); + return; + } + /* xmfs_info(sbi->sb, "recv callback %d", sbi->id); */ + do { + ret = ubcore_poll_jfc(jfc, 1, cr); + if (ret < 0) { + xmfs_info(NULL, "err jfc recv poll failed %d", ret); + break; + } + + for (i = 0; i < ret; i++) + xmfs_host_recv(sbi, &cr[i]); + } while (ret > 0 && jfc && sbi && sbi->magic == XMFS_MAGIC); + + ret = ubcore_rearm_jfc(jfc, false); + if (ret < 0) + xmfs_info(NULL, "err jfc recv rearm failed %d", ret); +} + +static int xmfs_register_send_recv(struct xmfs_sb_info *sbi, + struct xmfs_fs_context *ctx) +{ + struct ubcore_seg_cfg seg_cfg = { 0 }; + union ubcore_reg_seg_flag seg_flag = { 0 }; + struct ubcore_target_seg *seg; + struct ubcore_jfc_cfg jfc_cfg = { 0 }; + struct ubcore_jfc *jfc; + struct ubcore_jfr_cfg jfr_cfg = { 0 }; + struct ubcore_jfr *jfr; + struct ubcore_jetty *jetty; + struct ubcore_jetty_cfg jetty_cfg = { 0 }; + struct ubcore_eid_info *eid_info; + int err, cnt = 0; + + seg_flag.bs.token_policy = 0; + if (ctx->debug) + seg_flag.bs.access = XMFS_ACCESS_READ | XMFS_ACCESS_WRITE | + XMFS_ACCESS_ATOMIC | + XMFS_ACCESS_LOCAL_ONLY; + else + seg_flag.bs.access = XMFS_ACCESS_READ | XMFS_ACCESS_WRITE | + XMFS_ACCESS_ATOMIC; + seg_flag.bs.cacheable = UBCORE_NON_CACHEABLE; + + sbi->urma_cfg.recv_buf = + vzalloc(PAGE_ALIGN(XMFS_DEPTH * sizeof(struct xmfs_rw_msg))); + if (!sbi->urma_cfg.recv_buf) + return -ENOMEM; + + seg_cfg.va = (uint64_t)sbi->urma_cfg.recv_buf; + seg_cfg.len = PAGE_ALIGN(XMFS_DEPTH * sizeof(struct xmfs_rw_msg)); + seg_cfg.flag = seg_flag; + + seg = ubcore_register_seg(sbi->urma_cfg.ubc_dev, &seg_cfg, NULL); + if (IS_ERR_OR_NULL(seg)) { + xmfs_error(NULL, + "ubcore_register_seg failed during register recv"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.recv_seg = seg; + + seg_flag.bs.token_policy = 0; + seg_flag.bs.access = XMFS_ACCESS_LOCAL_ONLY; + seg_flag.bs.cacheable = UBCORE_NON_CACHEABLE; + + sbi->urma_cfg.send_buf = + vzalloc(PAGE_ALIGN(XMFS_DEPTH * sizeof(struct xmfs_rw_msg))); + if (!sbi->urma_cfg.send_buf) + return -ENOMEM; + + seg_cfg.va = (uint64_t)sbi->urma_cfg.send_buf; + seg_cfg.len = PAGE_ALIGN(XMFS_DEPTH * sizeof(struct xmfs_rw_msg)); + seg_cfg.flag = seg_flag; + xmfs_urma_pool_init(&sbi->urma_cfg.send_pool, (void *)seg_cfg.va, + seg_cfg.len, sizeof(struct xmfs_rw_msg)); + + seg = ubcore_register_seg(sbi->urma_cfg.ubc_dev, &seg_cfg, NULL); + if (IS_ERR_OR_NULL(seg)) { + xmfs_error(NULL, + "ubcore_register_seg failed during register send"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.send_seg = seg; + + jfc_cfg.depth = XMFS_DEPTH; + jfc_cfg.jfc_context = sbi; + jfc = ubcore_create_jfc(sbi->urma_cfg.ubc_dev, &jfc_cfg, + xmfs_recv_jfc_callback, NULL, NULL); + if (IS_ERR_OR_NULL(jfc)) { + xmfs_error(NULL, + "ubcore_create_jfc failed during register recv"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.recv_jfc = jfc; + + jfc = ubcore_create_jfc(sbi->urma_cfg.ubc_dev, &jfc_cfg, + xmfs_send_jfc_callback, NULL, NULL); + if (IS_ERR_OR_NULL(jfc)) { + xmfs_error(NULL, + "ubcore_create_jfc failed during register send"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.send_jfc = jfc; + + err = ubcore_rearm_jfc(sbi->urma_cfg.send_jfc, false); + if (err) + return -EOPNOTSUPP; + err = ubcore_rearm_jfc(sbi->urma_cfg.recv_jfc, false); + if (err) + return -EOPNOTSUPP; + + jfr_cfg.depth = XMFS_DEPTH; + jfr_cfg.trans_mode = UBCORE_TP_RM; + jfr_cfg.min_rnr_timer = (u8)19U; + jfr_cfg.max_sge = 1; + jfr_cfg.jfc = sbi->urma_cfg.recv_jfc; + jfr = ubcore_create_jfr(sbi->urma_cfg.ubc_dev, &jfr_cfg, NULL, NULL); + if (IS_ERR_OR_NULL(jfr)) { + xmfs_error(NULL, + "ubcore_create_jfr failed during register send"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.send_jfr = jfr; + + jetty_cfg.trans_mode = UBCORE_TP_RM; + jetty_cfg.eid_index = ctx->host ? 0 : 1; + jetty_cfg.max_send_sge = 1; + jetty_cfg.max_recv_sge = 1; + jetty_cfg.rnr_retry = 7; + jetty_cfg.min_rnr_timer = 20U; + jetty_cfg.err_timeout = 20; + jetty_cfg.jfs_depth = XMFS_DEPTH; + jetty_cfg.jfr_depth = XMFS_DEPTH; + jetty_cfg.send_jfc = sbi->urma_cfg.send_jfc; + jetty_cfg.recv_jfc = sbi->urma_cfg.recv_jfc; + jetty_cfg.jfr = jfr; + jetty_cfg.flag.bs.share_jfr = 1; + jetty = ubcore_create_jetty(sbi->urma_cfg.ubc_dev, &jetty_cfg, NULL, + NULL); + if (IS_ERR_OR_NULL(jetty)) { + xmfs_error(NULL, + "ubcore_create_jetty failed during register send"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.send_jetty = jetty; + + eid_info = ubcore_get_eid_list(sbi->urma_cfg.ubc_dev, &cnt); + if (!eid_info) { + xmfs_error(NULL, + "ubcore_get_eid_list failed during register send"); + return -EOPNOTSUPP; + } + + sbi->urma_cfg.send_info.urma_send_va = + (u64)(uintptr_t)sbi->urma_cfg.send_buf; + sbi->urma_cfg.send_info.urma_recv_va = + (u64)(uintptr_t)sbi->urma_cfg.recv_buf; + sbi->urma_cfg.send_info.urma_eid_subnet_prefix = + eid_info->eid.in6.subnet_prefix; + sbi->urma_cfg.send_info.urma_eid_interface_id = + eid_info->eid.in6.interface_id; + sbi->urma_cfg.send_info.urma_jetty_id = jetty->jetty_id.id; + sbi->urma_cfg.send_info.urma_send_token_id = + sbi->urma_cfg.send_seg->seg.token_id; + sbi->urma_cfg.send_info.urma_recv_token_id = + sbi->urma_cfg.recv_seg->seg.token_id; + xmfs_info(NULL, "send info %lld %lld %lld %lld %d %d %d", + sbi->urma_cfg.send_info.urma_send_va, + sbi->urma_cfg.send_info.urma_recv_va, + sbi->urma_cfg.send_info.urma_eid_subnet_prefix, + sbi->urma_cfg.send_info.urma_eid_interface_id, + sbi->urma_cfg.send_info.urma_jetty_id, + sbi->urma_cfg.send_info.urma_send_token_id, + sbi->urma_cfg.send_info.urma_recv_token_id); + + ubcore_free_eid_list(eid_info); + + for (int i = 0; i < XMFS_DEPTH; i++) + xmfs_post_recv(sbi, i); + + return 0; +} + +int xmfs_urma_register(struct xmfs_sb_info *sbi, struct xmfs_fs_context *ctx) +{ + struct ubcore_seg_cfg seg_cfg = { 0 }; + union ubcore_reg_seg_flag seg_flag = { 0 }; + struct ubcore_target_seg *seg; + struct ubcore_jfc_cfg jfc_cfg = { 0 }; + struct ubcore_jfc *jfc; + struct ubcore_jfr_cfg jfr_cfg = { 0 }; + struct ubcore_jfr *jfr; + struct ubcore_jetty *jetty; + struct ubcore_jetty_cfg jetty_cfg = { 0 }; + struct ubcore_eid_info *eid_info; + int err, cnt = 0; + + if (!ctx->host) { + sbi->urma_cfg.urma_remote_va = ctx->urma_va; + sbi->urma_cfg.urma_eid_subnet_prefix = + ctx->urma_eid_subnet_prefix; + sbi->urma_cfg.urma_eid_interface_id = + ctx->urma_eid_interface_id; + sbi->urma_cfg.urma_jetty_id = ctx->urma_jetty_id; + sbi->urma_cfg.urma_token_id = ctx->urma_token_id; + sbi->urma_cfg.last_log_start = XMFS_DEFAULT_CHUNK_SIZE * 3; + sbi->urma_cfg.last_data_start = XMFS_DEFAULT_CHUNK_SIZE * 3; + } + + xmfs_info(NULL, "ctx name %ld %s", strlen(ctx->dev_name), + ctx->dev_name); + memcpy(g_dev_name, ctx->dev_name, strlen(ctx->dev_name)); + xmfs_info(NULL, "g dev name %s", g_dev_name); + + INIT_LIST_HEAD(&sbi->urma_cfg.ubc_client.list_node); + sbi->urma_cfg.ubc_client.client_name = "xmfs_urma_client"; + sbi->urma_cfg.ubc_client.add = xmfs_add_udma_device; + sbi->urma_cfg.ubc_client.remove = xmfs_remove_udma_device; + xa_init(&sbi->urma_cfg.grant_waits); + + err = ubcore_register_client(&sbi->urma_cfg.ubc_client); + if (err) { + xmfs_error(NULL, "ubcore_register_client failed"); + return -EOPNOTSUPP; + } + xmfs_info(NULL, "register client done %d %d %d %d", XMFS_ACCESS_READ, + XMFS_ACCESS_WRITE, XMFS_ACCESS_ATOMIC, + XMFS_ACCESS_LOCAL_ONLY); + + seg_flag.bs.token_policy = 0; + if (ctx->debug) + seg_flag.bs.access = XMFS_ACCESS_READ | XMFS_ACCESS_WRITE | + XMFS_ACCESS_ATOMIC | + XMFS_ACCESS_LOCAL_ONLY; + else + seg_flag.bs.access = XMFS_ACCESS_READ | XMFS_ACCESS_WRITE | + XMFS_ACCESS_ATOMIC; + seg_flag.bs.non_pin = 1; + + if (ctx->host) { + seg_cfg.va = (u64)(uintptr_t)sbi->kaddr; + seg_cfg.len = sbi->len; + } else { + sbi->urma_cfg.urma_local_va = vzalloc(21474836480); + if (!sbi->urma_cfg.urma_local_va) { + xmfs_error(NULL, "vzalloc local va failed"); + return -ENOMEM; + } + seg_cfg.va = (u64)(uintptr_t)sbi->urma_cfg.urma_local_va; + seg_cfg.len = 21474836480; + } + xmfs_urma_pool_init(&sbi->urma_cfg.pool, (void *)seg_cfg.va, + seg_cfg.len, PAGE_SIZE); + seg_cfg.flag = seg_flag; + sbi->urma_cfg.ubc_dev = g_urma_dev[0]; + seg = ubcore_register_seg(sbi->urma_cfg.ubc_dev, &seg_cfg, NULL); + if (IS_ERR_OR_NULL(seg)) { + xmfs_error(NULL, "ubcore_register_seg failed"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.local_seg = seg; + + if (ctx->debug) + jfc_cfg.depth = 2048; + else + jfc_cfg.depth = 524288; + jfc_cfg.jfc_context = sbi; + jfc = ubcore_create_jfc(sbi->urma_cfg.ubc_dev, &jfc_cfg, + xmfs_jfc_callback, NULL, NULL); + if (IS_ERR_OR_NULL(jfc)) { + xmfs_error(NULL, "ubcore_create_jfc failed"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.jfc = jfc; + + err = ubcore_rearm_jfc(jfc, false); + if (err) { + xmfs_error(NULL, "ubcore_rearm_jfc failed"); + return -EOPNOTSUPP; + } + + if (ctx->debug) + jfr_cfg.depth = 2048; + else + jfr_cfg.depth = 16384; + jfr_cfg.trans_mode = UBCORE_TP_RM; + jfr_cfg.min_rnr_timer = (u8)19U; + jfr_cfg.max_sge = 1; + jfr_cfg.jfc = jfc; + jfr = ubcore_create_jfr(sbi->urma_cfg.ubc_dev, &jfr_cfg, NULL, NULL); + if (IS_ERR_OR_NULL(jfr)) { + xmfs_error(NULL, "ubcore_create_jfr failed"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.jfr = jfr; + + jetty_cfg.trans_mode = UBCORE_TP_RM; + jetty_cfg.eid_index = ctx->host ? 0 : 1; + jetty_cfg.max_send_sge = 1; + jetty_cfg.max_recv_sge = 1; + jetty_cfg.rnr_retry = 7; + jetty_cfg.min_rnr_timer = 20U; + jetty_cfg.err_timeout = 20; + if (ctx->debug) + jetty_cfg.jfs_depth = 8192; + else + jetty_cfg.jfs_depth = 2048; + jetty_cfg.jfr_depth = jfr_cfg.depth; + jetty_cfg.send_jfc = jfc; + jetty_cfg.recv_jfc = jfc; + jetty_cfg.jfr = jfr; + jetty_cfg.flag.bs.share_jfr = 1; + for (int i = 0; i < 5; i++) { + jetty = ubcore_create_jetty(sbi->urma_cfg.ubc_dev, &jetty_cfg, + NULL, NULL); + if (IS_ERR_OR_NULL(jetty)) { + xmfs_error(NULL, "ubcore_create_jetty failed"); + return -EOPNOTSUPP; + } + sbi->urma_cfg.jetty[i] = jetty; + } + + eid_info = ubcore_get_eid_list(sbi->urma_cfg.ubc_dev, &cnt); + if (!eid_info) { + xmfs_error(NULL, "ubcore_get_eid_list failed"); + return -EOPNOTSUPP; + } + if (ctx->host) { + sbi->urma_cfg.urma_remote_va = (u64)(uintptr_t)sbi->kaddr; + sbi->urma_cfg.urma_eid_subnet_prefix = + eid_info->eid.in6.subnet_prefix; + sbi->urma_cfg.urma_eid_interface_id = + eid_info->eid.in6.interface_id; + sbi->urma_cfg.urma_jetty_id = jetty->jetty_id.id; + sbi->urma_cfg.urma_token_id = seg->seg.token_id; + } else { + sbi->urma_cfg.own_info.urma_remote_va = + (u64)(uintptr_t)sbi->urma_cfg.urma_local_va; + sbi->urma_cfg.own_info.urma_eid_subnet_prefix = + eid_info->eid.in6.subnet_prefix; + sbi->urma_cfg.own_info.urma_eid_interface_id = + eid_info->eid.in6.interface_id; + sbi->urma_cfg.own_info.urma_jetty_id = jetty->jetty_id.id; + sbi->urma_cfg.own_info.urma_token_id = seg->seg.token_id; + } + + ubcore_free_eid_list(eid_info); + + xmfs_info(NULL, "register done"); + return xmfs_register_send_recv(sbi, ctx); +} + +void xmfs_urma_unimport(struct xmfs_sb_info *sbi) +{ + if (!sbi) + return; + if (sbi->urma_cfg.remote_seg) + ubcore_unimport_seg(sbi->urma_cfg.remote_seg); + if (sbi->urma_cfg.tjetty) { + /* ubcore_unbind_jetty(sbi->urma_cfg.jetty); */ + ubcore_unimport_jetty(sbi->urma_cfg.tjetty); + } + for (int i = 0; i < 64; i++) { + if (sbi->urma_cfg.tjettys[i]) + ubcore_unimport_jetty(sbi->urma_cfg.tjettys[i]); + if (sbi->urma_cfg.send_tjettys[i]) + ubcore_unimport_jetty(sbi->urma_cfg.send_tjettys[i]); + } +} + +int xmfs_import_jetty(struct xmfs_sb_info *sbi, struct urma_info info, + int index, bool is_send) +{ + struct ubcore_tjetty_cfg cfg = { 0 }; + struct ubcore_get_tp_cfg tp_flag = { 0 }; + struct ubcore_active_tp_cfg active_tp_cfg = { 0 }; + struct ubcore_tp_info tpid_info = { 0 }; + struct ubcore_tjetty *tjetty; + struct ubcore_eid_info *eid_info; + uint32_t tp_cnt = 1; + int err, cnt = 0; + + eid_info = ubcore_get_eid_list(sbi->urma_cfg.ubc_dev, &cnt); + if (!eid_info) { + xmfs_error(NULL, "ubcore_get_eid_list failed"); + return -EOPNOTSUPP; + } + xmfs_info(NULL, "get eid list done"); + for (int i = 0; i < cnt; i++) { + xmfs_info(NULL, "eid get during import %lld %lld", + eid_info->eid.in6.subnet_prefix, + eid_info->eid.in6.interface_id); + } + memcpy(&cfg.id.eid, &eid_info->eid, sizeof(union ubcore_eid)); + cfg.id.eid.in6.subnet_prefix = info.urma_eid_subnet_prefix; + cfg.id.eid.in6.interface_id = info.urma_eid_interface_id; + cfg.id.id = info.urma_jetty_id; + cfg.trans_mode = UBCORE_TP_RM; + cfg.type = UBCORE_JETTY; + cfg.eid_index = 1; + + tp_flag.flag.bs.ctp = 1; + tp_flag.trans_mode = UBCORE_TP_RM; + memcpy(&tp_flag.peer_eid, &eid_info->eid, sizeof(union ubcore_eid)); + tp_flag.peer_eid.in6.subnet_prefix = info.urma_eid_subnet_prefix; + tp_flag.peer_eid.in6.interface_id = info.urma_eid_interface_id; + memcpy(&tp_flag.local_eid, &eid_info->eid, sizeof(union ubcore_eid)); + xmfs_info(NULL, "node %d import jetty %lld %lld %d", sbi->id, + info.urma_eid_interface_id, info.urma_eid_subnet_prefix, + info.urma_jetty_id); + err = ubcore_get_tp_list(sbi->urma_cfg.ubc_dev, &tp_flag, &tp_cnt, + &tpid_info, NULL); + if (err) { + xmfs_error(NULL, "ubcore_get_tp_list failed."); + return -EOPNOTSUPP; + } + xmfs_info(NULL, "get tp list done"); + active_tp_cfg.tp_handle = tpid_info.tp_handle; + if (sbi->urma_cfg.ubc_dev->ops->import_jetty) + tjetty = ubcore_import_jetty(sbi->urma_cfg.ubc_dev, &cfg, NULL); + else + tjetty = ubcore_import_jetty_ex(sbi->urma_cfg.ubc_dev, &cfg, + &active_tp_cfg, NULL); + if (IS_ERR_OR_NULL(tjetty)) { + xmfs_error(NULL, "ubcore_import_jetty_ex failed."); + return -EOPNOTSUPP; + } + if (!is_send) { + if (sbi->urma_cfg.tjettys[index]) { + xmfs_info(NULL, "unimport tjetty %d", index); + err = ubcore_unimport_jetty(sbi->urma_cfg.tjettys[index]); + xmfs_info(NULL, "unimport tjetty %d ret %d", index, + err); + } + sbi->urma_cfg.tjettys[index] = tjetty; + } else { + if (sbi->urma_cfg.send_tjettys[index]) { + xmfs_info(NULL, "unimport send tjetty %d", index); + err = ubcore_unimport_jetty(sbi->urma_cfg.send_tjettys[index]); + xmfs_info(NULL, "unimport send tjetty %d ret %d", index, + err); + } + sbi->urma_cfg.send_tjettys[index] = tjetty; + } + + ubcore_free_eid_list(eid_info); + xmfs_info(NULL, "import jetty done"); + + return 0; +} + +int xmfs_urma_import(struct xmfs_sb_info *sbi) +{ + struct ubcore_tjetty_cfg cfg = { 0 }; + struct ubcore_get_tp_cfg tp_flag = { 0 }; + struct ubcore_active_tp_cfg active_tp_cfg = { 0 }; + struct ubcore_tp_info tpid_info = { 0 }; + struct ubcore_target_seg_cfg seg_cfg = { 0 }; + struct ubcore_tjetty *tjetty; + struct ubcore_target_seg *tseg; + struct ubcore_eid_info *eid_info; + uint32_t tp_cnt = 1; + int err, cnt = 0; + + xmfs_info(NULL, "start import"); + eid_info = ubcore_get_eid_list(sbi->urma_cfg.ubc_dev, &cnt); + if (!eid_info) { + xmfs_error(NULL, "ubcore_get_eid_list failed"); + return -EOPNOTSUPP; + } + for (int i = 0; i < cnt; i++) { + xmfs_info(NULL, "eid get during import %lld %lld", + eid_info->eid.in6.subnet_prefix, + eid_info->eid.in6.interface_id); + } + xmfs_info(NULL, "get eid list done %lld %lld", + sbi->urma_cfg.urma_eid_subnet_prefix, + sbi->urma_cfg.urma_eid_interface_id); + memcpy(&cfg.id.eid, &eid_info->eid, sizeof(union ubcore_eid)); + cfg.id.eid.in6.subnet_prefix = sbi->urma_cfg.urma_eid_subnet_prefix; + cfg.id.eid.in6.interface_id = sbi->urma_cfg.urma_eid_interface_id; + cfg.id.id = sbi->urma_cfg.urma_jetty_id; + cfg.trans_mode = UBCORE_TP_RM; + cfg.type = UBCORE_JETTY; + cfg.eid_index = 1; + + tp_flag.flag.bs.ctp = 1; + tp_flag.trans_mode = UBCORE_TP_RM; + memcpy(&tp_flag.peer_eid, &eid_info->eid, sizeof(union ubcore_eid)); + tp_flag.peer_eid.in6.subnet_prefix = + sbi->urma_cfg.urma_eid_subnet_prefix; + tp_flag.peer_eid.in6.interface_id = sbi->urma_cfg.urma_eid_interface_id; + memcpy(&tp_flag.local_eid, &eid_info->eid, sizeof(union ubcore_eid)); + + err = ubcore_get_tp_list(sbi->urma_cfg.ubc_dev, &tp_flag, &tp_cnt, + &tpid_info, NULL); + if (err) { + xmfs_error(NULL, "ubcore_get_tp_list failed."); + return -EOPNOTSUPP; + } + xmfs_info(NULL, "get tp list done"); + active_tp_cfg.tp_handle = tpid_info.tp_handle; + if (sbi->urma_cfg.ubc_dev->ops->import_jetty) + tjetty = ubcore_import_jetty(sbi->urma_cfg.ubc_dev, &cfg, NULL); + else + tjetty = ubcore_import_jetty_ex(sbi->urma_cfg.ubc_dev, &cfg, + &active_tp_cfg, NULL); + if (IS_ERR_OR_NULL(tjetty)) { + xmfs_error(NULL, "ubcore_import_jetty_ex failed."); + return -EOPNOTSUPP; + } + sbi->urma_cfg.tjetty = tjetty; + xmfs_info(NULL, "import jetty done"); + /* + * err = ubcore_bind_jetty_ex(sbi->urma_cfg.jetty, tjetty, + * &active_tp_cfg, NULL); + * if (err) { + * xmfs_error(sbi->sb, "ubcore_bind_jetty_ex failed."); + * return -EOPNOTSUPP; + * } + */ + + memcpy(&seg_cfg.seg.ubva.eid, &eid_info->eid, sizeof(union ubcore_eid)); + seg_cfg.seg.ubva.eid.in6.subnet_prefix = + sbi->urma_cfg.urma_eid_subnet_prefix; + seg_cfg.seg.ubva.eid.in6.interface_id = + sbi->urma_cfg.urma_eid_interface_id; + ubcore_free_eid_list(eid_info); + seg_cfg.seg.ubva.va = sbi->urma_cfg.urma_remote_va; + seg_cfg.seg.len = sbi->len; + seg_cfg.seg.token_id = sbi->urma_cfg.urma_token_id; + tseg = ubcore_import_seg(sbi->urma_cfg.ubc_dev, &seg_cfg, NULL); + if (IS_ERR_OR_NULL(tseg)) { + xmfs_error(NULL, "ubcore_import_seg failed."); + return -EOPNOTSUPP; + } + xmfs_info(NULL, "import seg done"); + sbi->urma_cfg.remote_seg = tseg; + + return 0; +} + +static int xmfs_urma_send_and_wait(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr *wr, + struct xmfs_urma_msg *msg) +{ + int err = 0; + struct ubcore_jfs_wr *bad_wr = NULL; + struct ubcore_jfs_wr *tmp; + struct ubcore_jfs_wr *retry_wr; + struct ubcore_jfs_wr *failed_head = NULL; + int ret = 0; + int retry; + int timeout; + unsigned int delay = 50; + unsigned int multi = 1; + + if (sbi->debug) + multi = 1000; + + if (sbi->debug) + timeout = msecs_to_jiffies(20000); + else + timeout = msecs_to_jiffies(20000); + + retry_wr = wr; + for (retry = 0; retry < XMFS_POST_RETRY_CNT && retry_wr; retry++) { + bad_wr = NULL; + err = ubcore_post_jetty_send_wr(sbi->urma_cfg.jetty[jiffies % 5], + retry_wr, &bad_wr); + if (!err) { + retry_wr = NULL; + break; + } + + /* + * xmfs_error(sbi->sb, + * "qyf post wr failed ret=%d retry=%d bad_wr=%px", + * err, retry, bad_wr); + */ + + if (bad_wr) + retry_wr = bad_wr; + usleep_range(delay * multi, delay * multi + 50 * multi); + delay = min(delay * 2, 5000U); + } + failed_head = retry_wr; + if (failed_head) { + xmfs_error(NULL, "still have unposted wr after retry=%d", + XMFS_POST_RETRY_CNT); + + for (tmp = failed_head; tmp; tmp = tmp->next) { + struct wr_status *wrs = + (struct wr_status *)(uintptr_t)tmp->user_ctx; + + kfree(wrs); + wrs = NULL; + } + + ret = -EAGAIN; + } + + for (tmp = wr; tmp != NULL && tmp != failed_head; tmp = tmp->next) { + struct wr_status *wrs = + (struct wr_status *)(uintptr_t)tmp->user_ctx; + err = wait_for_completion_interruptible_timeout(&wrs->done, + timeout); + if (err <= 0) { + xmfs_warning(sbi->sb, + "request wait failed: %d, op %d, status %d, offset %lld, length %lld", + err, wrs->ops, wrs->ret, msg->offset, + msg->len); + ret = -EAGAIN; + } + + if (wrs->status == URMA_ERROR) { + xmfs_error(NULL, + "urma send get err %d ret %d ops %d len %d %d", + wrs->status, wrs->ret, wrs->ops, + wr->rw.src.sge->len, wr->rw.dst.sge->len); + ret = -EAGAIN; + } + xmfs_wrs_put(&wrs); + } + + return ret; +} + +static int xmfs_urma_get_read_wr(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr **wr, + struct xmfs_urma_msg *msg) +{ + int trunk_size; + + if (sbi->debug) + trunk_size = 4096; + else + trunk_size = XMFS_URMA_DEFAULT_WR_SIZE; + + uint64_t wr_count = DIV_ROUND_UP(msg->len, trunk_size); + struct ubcore_jfs_wr *wr_ptr, *wr_pre = NULL; + *wr = vzalloc(wr_count * sizeof(struct ubcore_jfs_wr)); + if (!*wr) + return -ENOMEM; + + wr_ptr = *wr; + for (uint64_t read_count = 0; read_count < msg->len; + read_count += trunk_size) { + struct wr_status *wrs = + kzalloc(sizeof(struct wr_status), GFP_KERNEL); + if (!wrs) + return -ENOMEM; + + wr_ptr->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + wr_ptr->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + if (!wr_ptr->rw.src.sge || !wr_ptr->rw.dst.sge) + return -ENOMEM; + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 1; + wrs->ops = XMFS_URMA_READ; + wrs->magic = XMFS_MAGIC; + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + uint64_t read_len = read_count + trunk_size < msg->len ? + trunk_size : + msg->len - read_count; + if (wr_pre) + wr_pre->next = wr_ptr; + wr_ptr->opcode = UBCORE_OPC_READ; + wr_ptr->flag.bs.complete_enable = 1; + wr_ptr->tjetty = sbi->urma_cfg.tjetty; + wr_ptr->rw.src.sge->addr = + sbi->urma_cfg.urma_remote_va + msg->offset + read_count; + wr_ptr->rw.src.sge->len = read_len; + wr_ptr->rw.src.sge->tseg = sbi->urma_cfg.remote_seg; + wr_ptr->rw.src.num_sge = 1; + wr_ptr->rw.dst.sge->addr = + (uint64_t)msg->local_addr + read_count; + wr_ptr->rw.dst.sge->len = read_len; + wr_ptr->rw.dst.sge->tseg = sbi->urma_cfg.local_seg; + wr_ptr->rw.dst.num_sge = 1; + wr_ptr->user_ctx = (uint64_t)(uintptr_t)wrs; + wr_ptr->next = NULL; + wr_pre = wr_ptr++; + } + msg->wr_cnt = wr_count; + return 0; +} + +static int xmfs_urma_get_write_wr(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr **wr, + struct xmfs_urma_msg *msg) +{ + *wr = vzalloc(sizeof(struct ubcore_jfs_wr)); + if (!*wr) + return -ENOMEM; + + struct wr_status *wrs = kzalloc(sizeof(struct wr_status), GFP_KERNEL); + + if (!wrs) + return -ENOMEM; + + (*wr)->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + (*wr)->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + if (!(*wr)->rw.src.sge || !(*wr)->rw.dst.sge) + return -ENOMEM; + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 2; + wrs->ops = XMFS_URMA_WRITE; + wrs->magic = XMFS_MAGIC; + wrs->slot = -1; /* msg.slot */ + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + memcpy(msg->write_buffer, msg->local_addr, msg->len); + (*wr)->opcode = UBCORE_OPC_WRITE; + (*wr)->flag.bs.complete_enable = 1; + (*wr)->tjetty = sbi->urma_cfg.tjetty; + (*wr)->rw.src.sge->addr = (uint64_t)msg->write_buffer; + (*wr)->rw.src.sge->len = msg->len; + (*wr)->rw.src.sge->tseg = sbi->urma_cfg.local_seg; + (*wr)->rw.src.num_sge = 1; + (*wr)->rw.dst.sge->addr = sbi->urma_cfg.urma_remote_va + msg->offset; + (*wr)->rw.dst.sge->len = msg->len; + (*wr)->rw.dst.sge->tseg = sbi->urma_cfg.remote_seg; + (*wr)->rw.dst.num_sge = 1; + (*wr)->user_ctx = (uint64_t)(uintptr_t)wrs; + (*wr)->next = NULL; + msg->wr_cnt = 1; + return 0; +} + +static int xmfs_urma_get_cas_wr(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr **wr, + struct xmfs_urma_msg *msg) +{ + *wr = vzalloc(sizeof(struct ubcore_jfs_wr)); + if (!*wr) + return -ENOMEM; + + struct wr_status *wrs = kzalloc(sizeof(struct wr_status), GFP_KERNEL); + + if (!wrs) + return -ENOMEM; + + (*wr)->cas.src = vzalloc(sizeof(struct ubcore_sge)); + (*wr)->cas.dst = vzalloc(sizeof(struct ubcore_sge)); + if (!(*wr)->cas.src || !(*wr)->cas.dst) + return -ENOMEM; + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 3; + wrs->ops = XMFS_URMA_CAS; + wrs->magic = XMFS_MAGIC; + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + /* msg->local_addr = sbi->urma_cfg.urma_local_va; */ + (*wr)->opcode = UBCORE_OPC_CAS; + (*wr)->flag.bs.complete_enable = 1; + (*wr)->tjetty = sbi->urma_cfg.tjetty; + (*wr)->cas.src->addr = (uint64_t)msg->local_addr; + (*wr)->cas.src->len = msg->len; + (*wr)->cas.src->tseg = sbi->urma_cfg.local_seg; + (*wr)->cas.dst->addr = sbi->urma_cfg.urma_remote_va + msg->offset; + (*wr)->cas.dst->len = msg->len; + (*wr)->cas.dst->tseg = sbi->urma_cfg.remote_seg; + (*wr)->cas.cmp_data = msg->cas_cmp_data; + (*wr)->cas.swap_data = msg->cas_swap_data; + (*wr)->user_ctx = (uint64_t)(uintptr_t)wrs; + (*wr)->next = NULL; + msg->wr_cnt = 1; + return 0; +} + +static void xmfs_urma_free_wr(struct xmfs_urma_msg *msg, + struct ubcore_jfs_wr *wr, int err) +{ + struct ubcore_jfs_wr *first_wr = wr; + struct wr_status *wrs; + + if (!wr) + return; + while (wr) { + switch (msg->ops) { + case XMFS_URMA_READ: + case XMFS_URMA_WRITE: + vfree(wr->rw.src.sge); + vfree(wr->rw.dst.sge); + break; + case XMFS_URMA_CAS: + vfree(wr->cas.src); + vfree(wr->cas.dst); + break; + default: + break; + } + wrs = (struct wr_status *)wr->user_ctx; + if (wrs != NULL && err != -EAGAIN && err != 0) + kfree(wrs); + + wr = wr->next; + } + vfree(first_wr); +} + +int xmfs_pmem_write(struct xmfs_sb_info *sbi, unsigned long offset, + const void *buf, unsigned long size) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + memcpy(sbi->kaddr + offset, buf, size); + return 0; +#ifdef CONFIG_XMFS_FS_URMA + case XMFS_URMA_MODE: { + unsigned long written = 0; + u32 slot; + + while (written < size) { + unsigned long chunk = size - written; + struct xmfs_urma_msg msg; + int err; + + if (chunk > PAGE_SIZE) + chunk = PAGE_SIZE; + memset(&msg, 0, sizeof(msg)); + msg.ops = XMFS_URMA_WRITE; + msg.offset = offset + written; + msg.len = chunk; + msg.local_addr = (void *)(buf + written); + msg.write_buffer = + urma_pool_acquire(&sbi->urma_cfg.pool, &slot); + if (!msg.write_buffer) { + xmfs_info(NULL, + "pmem write cannot get a buffer to write"); + return -EINVAL; + } + msg.slot = slot; + err = xmfs_urma_send(sbi, &msg); + urma_pool_release(&sbi->urma_cfg.pool, slot); + if (err) + return err; + + written += chunk; + } + return 0; + } +#endif + default: + return -ENODEV; + } +} +EXPORT_SYMBOL(xmfs_pmem_write); + +static int +xmfs_urma_get_read_wr_pages(struct xmfs_sb_info *sbi, struct ubcore_jfs_wr **wr, + struct xmfs_urma_msg *msg, unsigned long *offsets, + struct page **pages, unsigned long page_num, + void **kaddrs, int *cnt) +{ + uint64_t wr_count = DIV_ROUND_UP(msg->len, PAGE_SIZE); + struct ubcore_jfs_wr *wr_ptr, *wr_pre = NULL; + *wr = vzalloc(wr_count * sizeof(struct ubcore_jfs_wr)); + if (!*wr) + return -ENOMEM; + + wr_ptr = *wr; + for (uint64_t read_count = 0; read_count < page_num; read_count++) { + struct wr_status *wrs = + kzalloc(sizeof(struct wr_status), GFP_KERNEL); + if (!wrs) + return -ENOMEM; + + wr_ptr->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + wr_ptr->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + if (!wr_ptr->rw.src.sge || !wr_ptr->rw.dst.sge) + return -ENOMEM; + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 4; + wrs->ops = XMFS_URMA_READ; + wrs->magic = XMFS_MAGIC; + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + uint64_t read_len = PAGE_SIZE; + + if (wr_pre) + wr_pre->next = wr_ptr; + + kaddrs[read_count] = kmap_local_page(pages[read_count]); + *cnt = read_count + 1; + wr_ptr->opcode = UBCORE_OPC_READ; + wr_ptr->flag.bs.complete_enable = 1; + wr_ptr->tjetty = sbi->urma_cfg.tjetty; + wr_ptr->rw.src.sge->addr = + sbi->urma_cfg.urma_remote_va + offsets[read_count]; + wr_ptr->rw.src.sge->len = read_len; + wr_ptr->rw.src.sge->tseg = sbi->urma_cfg.remote_seg; + wr_ptr->rw.src.num_sge = 1; + wr_ptr->rw.dst.sge->addr = (uint64_t)kaddrs[read_count]; + wr_ptr->rw.dst.sge->len = read_len; + wr_ptr->rw.dst.sge->tseg = sbi->urma_cfg.local_seg; + wr_ptr->rw.dst.num_sge = 1; + wr_ptr->user_ctx = (uint64_t)(uintptr_t)wrs; + wr_ptr->next = NULL; + wr_pre = wr_ptr++; + } + return 0; +} + +static int xmfs_urma_get_write_wr_pages(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr **wr, + struct xmfs_urma_msg *msg, int *slots, + struct page **pages, + unsigned long page_num) +{ + uint64_t wr_count = DIV_ROUND_UP(msg->len, PAGE_SIZE); + struct ubcore_jfs_wr *wr_ptr, *wr_pre = NULL; + *wr = vzalloc(wr_count * sizeof(struct ubcore_jfs_wr)); + if (!*wr) + return -ENOMEM; + + wr_ptr = *wr; + void *write_buffer = + urma_pool_acquire_nslots(&sbi->urma_cfg.pool, page_num, slots); + if (!write_buffer) { + xmfs_info(NULL, "%s get write buffer failed", __func__); + return -EAGAIN; + } + for (uint64_t read_count = 0; read_count < page_num; read_count++) { + struct wr_status *wrs = + kzalloc(sizeof(struct wr_status), GFP_KERNEL); + if (!wrs) + return -ENOMEM; + + wr_ptr->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + wr_ptr->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + if (!wr_ptr->rw.src.sge || !wr_ptr->rw.dst.sge) + return -ENOMEM; + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 5; + wrs->ops = XMFS_URMA_WRITE; + wrs->magic = XMFS_MAGIC; + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + wrs->slot = slots[read_count]; + uint64_t read_len = PAGE_SIZE; + + if (wr_pre) + wr_pre->next = wr_ptr; + + void *kaddr = kmap_local_page(pages[read_count]); + + memcpy(write_buffer + read_count * PAGE_SIZE, kaddr, PAGE_SIZE); + kunmap_local(kaddr); + + wr_ptr->opcode = UBCORE_OPC_WRITE; + wr_ptr->flag.bs.complete_enable = 1; + wr_ptr->tjetty = sbi->urma_cfg.tjetty; + wr_ptr->rw.dst.sge->addr = sbi->urma_cfg.urma_remote_va + + msg->offset + read_count * PAGE_SIZE; + wr_ptr->rw.dst.sge->len = read_len; + wr_ptr->rw.dst.sge->tseg = sbi->urma_cfg.remote_seg; + wr_ptr->rw.dst.num_sge = 1; + wr_ptr->rw.src.sge->addr = + (uint64_t)(write_buffer + read_count * PAGE_SIZE); + wr_ptr->rw.src.sge->len = read_len; + wr_ptr->rw.src.sge->tseg = sbi->urma_cfg.local_seg; + wr_ptr->rw.src.num_sge = 1; + wr_ptr->user_ctx = (uint64_t)(uintptr_t)wrs; + wr_ptr->next = NULL; + wr_pre = wr_ptr++; + } + return 0; +} + +int xmfs_urma_send_pages(struct xmfs_sb_info *sbi, struct xmfs_urma_msg *msg, + unsigned long *offsets, struct page **pages, + unsigned long page_num) +{ + struct ubcore_jfs_wr *wr = NULL; + int *slots = vzalloc(sizeof(int) * page_num); + + if (!slots) + return -ENOMEM; + void **kaddrs = vzalloc(sizeof(void *) * page_num); + + if (!kaddrs) { + vfree(slots); + return -ENOMEM; + } + int err = 0; + int cnt = 0; + + switch (msg->ops) { + case XMFS_URMA_READ: + err = xmfs_urma_get_read_wr_pages(sbi, &wr, msg, offsets, pages, + page_num, kaddrs, &cnt); + break; + case XMFS_URMA_WRITE: + err = xmfs_urma_get_write_wr_pages(sbi, &wr, msg, slots, pages, + page_num); + break; + default: + xmfs_error(NULL, "What exactly are you going to do?"); + err = -EOPNOTSUPP; + } + + if (err) { + xmfs_error(NULL, "failed to alloc wr or local buffer %ld %d", + page_num, err); + goto free_ret; + } + + err = xmfs_urma_send_and_wait(sbi, wr, msg); + +free_ret: + if (msg->ops == XMFS_URMA_READ) { + for (int i = 0; i < cnt; i++) + kunmap_local(kaddrs[i]); + } + vfree(kaddrs); + xmfs_urma_free_wr(msg, wr, err); + vfree(slots); + return err; +} + +int xmfs_pmem_write_multi_pages(struct xmfs_sb_info *sbi, unsigned long offset, + struct page **pages, unsigned long page_num) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + /* memcpy(sbi->kaddr + offset, buf, size); */ + return 0; +#ifdef CONFIG_XMFS_FS_URMA + case XMFS_URMA_MODE: { + if (page_num == 0) + return 0; + unsigned long chunk = PAGE_SIZE * page_num; + struct xmfs_urma_msg msg; + int err; + + memset(&msg, 0, sizeof(msg)); + msg.ops = XMFS_URMA_WRITE; + msg.offset = offset; + msg.len = chunk; + err = xmfs_urma_send_pages(sbi, &msg, &offset, pages, page_num); + if (err) + return err; + + return 0; + } +#endif + default: + return -ENODEV; + } +} +EXPORT_SYMBOL(xmfs_pmem_write_multi_pages); + +int xmfs_pmem_read_multi_pages(struct xmfs_sb_info *sbi, unsigned long *offsets, + struct page **pages, unsigned long page_num) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + /* memcpy(sbi->kaddr + offset, buf, size); */ + return 0; +#ifdef CONFIG_XMFS_FS_URMA + case XMFS_URMA_MODE: { + if (page_num == 0) + return 0; + unsigned long chunk = PAGE_SIZE * page_num; + struct xmfs_urma_msg msg; + int err; + + memset(&msg, 0, sizeof(msg)); + msg.ops = XMFS_URMA_READ; + msg.len = chunk; + err = xmfs_urma_send_pages(sbi, &msg, offsets, pages, page_num); + if (err) + return err; + + return 0; + } +#endif + default: + return -ENODEV; + } +} +EXPORT_SYMBOL(xmfs_pmem_read_multi_pages); + +int xmfs_send_request(struct xmfs_sb_info *sbi, struct xmfs_rw_msg *req, + struct xmfs_grant_wait *gw, int id) +{ + struct ubcore_jfs_wr *bad_send_wr = NULL; + struct ubcore_jfs_wr send_wr = {}; + struct ubcore_sge sge = {}; + int ret, slot; + struct xmfs_rw_msg *buf; + + int timeout; + + if (sbi->debug) + timeout = msecs_to_jiffies(20000); + else + timeout = msecs_to_jiffies(20000); + + buf = urma_pool_acquire(&sbi->urma_cfg.send_pool, &slot); + if (!buf) { + xmfs_info(NULL, "cannot get buf to send request"); + return -ENOMEM; + } + /* xmfs_info(sbi->sb, "qyf acquire slot %d", slot); */ + memcpy(buf, req, sizeof(struct xmfs_rw_msg)); + sge.addr = (uint64_t)buf; + sge.len = sizeof(*req); + sge.tseg = sbi->urma_cfg.send_seg; + + send_wr.opcode = UBCORE_OPC_SEND; + send_wr.tjetty = sbi->urma_cfg.send_tjettys[id]; + send_wr.send.src.sge = &sge; + send_wr.send.src.num_sge = 1; + send_wr.flag.value = 0; + send_wr.flag.bs.complete_enable = 1; + send_wr.user_ctx = slot; + send_wr.next = NULL; + /* + * xmfs_info(sbi->sb, "qyf send request send pool slot %d id %d", + * slot, req->request_id); + */ + if (req->ops == XMFS_GC) + xmfs_info(NULL, "gc req send %d", req->status); + ret = ubcore_post_jetty_send_wr(sbi->urma_cfg.send_jetty, &send_wr, + &bad_send_wr); + if (ret) { + xmfs_info(NULL, "err send request failed %d, release %d", ret, + slot); + urma_pool_release(&sbi->urma_cfg.send_pool, slot); + if (req->ops != XMFS_SETATTR && req->ops != XMFS_GC) + xa_erase(&sbi->urma_cfg.grant_waits, req->request_id); + return ret; + } + + if (req->ops == XMFS_SETATTR || req->ops == XMFS_GC) + return 0; + + ret = wait_for_completion_interruptible_timeout(&gw->done, timeout); + xa_erase(&sbi->urma_cfg.grant_waits, req->request_id); + /* + * xmfs_info(sbi->sb, "qyf send request done send pool slot %d id %d", + * slot, req->request_id); + */ + if (ret <= 0) { + if (ret == 0) + ret = -ETIMEDOUT; + xmfs_warning(sbi->sb, + "slave request wait failed: %d, request ID %ld", + ret, req->request_id); + return ret; + } + if (gw->grant.type != MSG_RW_GRANT) { + xmfs_warning(sbi->sb, + "unexpected grant: type %d, request ID %ld, op %d", + gw->grant.type, gw->grant.request_id, + gw->grant.ops); + return -EINVAL; + } + + return 0; +} + +/* + * static int xmfs_urma_post_wr(struct xmfs_sb_info *sbi, + * struct ubcore_jfs_wr *wr) + * { + * struct ubcore_jfs_wr *bad_wr = NULL; + * int err = ubcore_post_jetty_send_wr(sbi->urma_cfg.jetty, wr, + * &bad_wr); + * + * if (err) + * xmfs_error(sbi->sb, + * "ubcore_post_jetty_send_wr failed %d", err); + * return err; + * } + * + * static int xmfs_urma_wait_chain(struct ubcore_jfs_wr *wr) + * { + * struct ubcore_jfs_wr *tmp; + * int ret = 0; + * + * for (tmp = wr; tmp != NULL; tmp = tmp->next) { + * struct wr_status *wrs = + * (struct wr_status *)(uintptr_t)tmp->user_ctx; + * int err; + * + * if (wrs->status != URMA_ISSUING) + * goto skip_wait; + * err = wait_for_completion_interruptible_timeout( + * &wrs->done, msecs_to_jiffies(20000)); + * if (err <= 0) { + * xmfs_info(NULL, + * "wait for wr timeout or interrupted %d ops %d wrs->ret %d", + * err, wrs->ops, wrs->ret); + * ret = -EAGAIN; + * } + * skip_wait: + * if (wrs->status == URMA_ERROR) { + * xmfs_error(NULL, + * "urma send get err status=%d ret=%d ops=%d len src=%d dst=%d", + * wrs->status, wrs->ret, wrs->ops, + * tmp->rw.src.sge->len, + * tmp->rw.dst.sge->len); + * ret = -EAGAIN; + * } + * xmfs_wrs_put(&wrs); + * } + * return ret; + * } + * + * static int xmfs_urma_send_and_wait_new(struct xmfs_sb_info *sbi, + * struct ubcore_jfs_wr *wr, + * struct xmfs_urma_msg *msg) + * { + * int err = xmfs_urma_post_wr(sbi, wr); + * + * if (err) + * return err; + * return xmfs_urma_wait_chain(wr); + * } + * + * struct xmfs_urma_pending { + * struct ubcore_jfs_wr *wr; + * void *write_buffer; + * int *slots; + * int nr_slots; + * struct xmfs_urma_msg msg; + * }; + * + * static void wr_chain_append(struct ubcore_jfs_wr **chain_head, + * struct ubcore_jfs_wr **tail, + * struct ubcore_jfs_wr *wr) + * { + * wr->next = NULL; + * if (*chain_head == NULL) { + * *chain_head = wr; + * *tail = wr; + * } else { + * (*tail)->next = wr; + * *tail = wr; + * } + * } + */ + +static int xmfs_urma_get_write_wr_ubuf(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr **wr, + struct xmfs_urma_msg *msg) +{ + *wr = vzalloc(sizeof(struct ubcore_jfs_wr)); + if (!*wr) + return -ENOMEM; + + struct wr_status *wrs = kzalloc(sizeof(struct wr_status), GFP_KERNEL); + + if (!wrs) + return -ENOMEM; + + (*wr)->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + (*wr)->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + if (!(*wr)->rw.src.sge || !(*wr)->rw.dst.sge) + return -ENOMEM; + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 7; + wrs->ops = XMFS_URMA_WRITE; + wrs->magic = XMFS_MAGIC; + wrs->slot = -1; /* msg.slot */ + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + (*wr)->opcode = UBCORE_OPC_WRITE; + (*wr)->flag.bs.complete_enable = 1; + (*wr)->tjetty = sbi->urma_cfg.tjetty; + (*wr)->rw.src.sge->addr = (uint64_t)msg->write_buffer; + (*wr)->rw.src.sge->len = msg->len; + (*wr)->rw.src.sge->tseg = sbi->urma_cfg.local_seg; + (*wr)->rw.src.num_sge = 1; + (*wr)->rw.dst.sge->addr = sbi->urma_cfg.urma_remote_va + msg->offset; + (*wr)->rw.dst.sge->len = msg->len; + (*wr)->rw.dst.sge->tseg = sbi->urma_cfg.remote_seg; + (*wr)->rw.dst.num_sge = 1; + (*wr)->user_ctx = (uint64_t)(uintptr_t)wrs; + (*wr)->next = NULL; + msg->wr_cnt = 1; + return 0; +} + +static int xmfs_urma_get_read_wr_ubuf(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr **wr, + struct xmfs_urma_msg *msg) +{ + int trunk_size; + + if (sbi->debug) + trunk_size = 4096; + else + trunk_size = msg->len; + + uint64_t wr_count = DIV_ROUND_UP(msg->len, trunk_size); + struct ubcore_jfs_wr *wr_ptr, *wr_pre = NULL; + *wr = vzalloc(wr_count * sizeof(struct ubcore_jfs_wr)); + if (!*wr) + return -ENOMEM; + + wr_ptr = *wr; + for (uint64_t read_count = 0; read_count < msg->len; + read_count += trunk_size) { + struct wr_status *wrs = + kzalloc(sizeof(struct wr_status), GFP_KERNEL); + if (!wrs) + return -ENOMEM; + + wr_ptr->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + wr_ptr->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + if (!wr_ptr->rw.src.sge || !wr_ptr->rw.dst.sge) + return -ENOMEM; + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 1; + wrs->ops = XMFS_URMA_READ; + wrs->magic = XMFS_MAGIC; + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + uint64_t read_len = read_count + trunk_size < msg->len ? + trunk_size : + msg->len - read_count; + if (wr_pre) + wr_pre->next = wr_ptr; + wr_ptr->opcode = UBCORE_OPC_READ; + wr_ptr->flag.bs.complete_enable = 1; + wr_ptr->tjetty = sbi->urma_cfg.tjetty; + wr_ptr->rw.src.sge->addr = + sbi->urma_cfg.urma_remote_va + msg->offset + read_count; + wr_ptr->rw.src.sge->len = read_len; + wr_ptr->rw.src.sge->tseg = sbi->urma_cfg.remote_seg; + wr_ptr->rw.src.num_sge = 1; + wr_ptr->rw.dst.sge->addr = + (uint64_t)msg->local_addr + read_count; + wr_ptr->rw.dst.sge->len = read_len; + wr_ptr->rw.dst.sge->tseg = sbi->urma_cfg.local_seg; + wr_ptr->rw.dst.num_sge = 1; + wr_ptr->user_ctx = (uint64_t)(uintptr_t)wrs; + wr_ptr->next = NULL; + wr_pre = wr_ptr++; + } + msg->wr_cnt = wr_count; + + /* + * *wr = (struct ubcore_jfs_wr *)vzalloc( + * sizeof(struct ubcore_jfs_wr)); + * if (!*wr) + * return -ENOMEM; + * struct wr_status *wrs = vzalloc(sizeof(struct wr_status)); + * if (!wrs) + * return -ENOMEM; + * (*wr)->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + * (*wr)->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + * if (!(*wr)->rw.src.sge || !(*wr)->rw.dst.sge) + * return -ENOMEM; + * wrs->status = URMA_ISSUING; + * init_completion(&wrs->done); + * wrs->ret = 8; + * wrs->ops = XMFS_URMA_READ; + * wrs->magic = XMFS_MAGIC; + * wrs->slot = -1; + * refcount_set(&wrs->refs, 1); + * xmfs_wrs_get(wrs); + * (*wr)->opcode = UBCORE_OPC_READ; + * (*wr)->flag.bs.complete_enable = 1; + * (*wr)->tjetty = sbi->urma_cfg.tjetty; + * (*wr)->rw.dst.sge->addr = (uint64_t)msg->local_addr; + * (*wr)->rw.dst.sge->len = msg->len; + * (*wr)->rw.dst.sge->tseg = sbi->urma_cfg.local_seg; + * (*wr)->rw.dst.num_sge = 1; + * (*wr)->rw.src.sge->addr = + * sbi->urma_cfg.urma_remote_va + msg->offset; + * (*wr)->rw.src.sge->len = msg->len; + * (*wr)->rw.src.sge->tseg = sbi->urma_cfg.remote_seg; + * (*wr)->rw.src.num_sge = 1; + * (*wr)->user_ctx = (uint64_t)(uintptr_t)wrs; + * (*wr)->next = NULL; + * msg->wr_cnt = 1; + */ + return 0; +} + +static int xmfs_urma_send_ubuf(struct xmfs_sb_info *sbi, + struct xmfs_urma_msg *msg) +{ + struct ubcore_jfs_wr *wr = NULL; + int err = 0; + + switch (msg->ops) { + case XMFS_URMA_READ: + err = xmfs_urma_get_read_wr_ubuf(sbi, &wr, msg); + break; + case XMFS_URMA_WRITE: + err = xmfs_urma_get_write_wr_ubuf(sbi, &wr, msg); + break; + default: + xmfs_error(NULL, "What exactly are you going to do?"); + return -EOPNOTSUPP; + } + + if (err) { + xmfs_error(NULL, "failed to alloc wr or local buffer"); + goto free_ret; + } + + err = xmfs_urma_send_and_wait(sbi, wr, msg); + +free_ret: + xmfs_urma_free_wr(msg, wr, err); + return err; +} + +int xmfs_urma_write_ubuf(struct xmfs_sb_info *sbi, void __user *buf, + size_t size, unsigned long offset) +{ + int chunk_size; + + if (sbi->debug) + chunk_size = PAGE_SIZE; + else + chunk_size = XMFS_DEFAULT_CHUNK_SIZE - PAGE_SIZE; + int *slots = vzalloc(sizeof(int) * (size + PAGE_SIZE - 1) / PAGE_SIZE); + + if (!slots) + return -ENOMEM; + int loop_cnt = 0; + int chunk_header_cnt = 0; + size_t written = 0; + int err; + /* + * xmfs_info(sbi->sb, "qyf before acquire nslots %d", + * (size + PAGE_SIZE * 16 - 1) / (PAGE_SIZE * 16)); + */ + void *write_buffer = + urma_pool_acquire_nslots(&sbi->urma_cfg.pool, + (size + PAGE_SIZE - 1) / PAGE_SIZE, + slots); + if (!write_buffer) { + vfree(slots); + xmfs_info(NULL, "write ubuf cannot get a buffer to write"); + return -EINVAL; + } + /* xmfs_info(sbi->sb, "qyf before copy from user"); */ + err = copy_from_user(write_buffer, buf, size); + if (err) { + xmfs_error(NULL, "write ubuf cannot copy from, size is %ld", size); + urma_pool_release_nslots(&sbi->urma_cfg.pool, + (size + PAGE_SIZE - 1) / PAGE_SIZE, slots); + vfree(slots); + return err; + } + /* xmfs_info(sbi->sb, "qyf end copy from user"); */ + while (written < size) { + unsigned long chunk = size - written; + struct xmfs_urma_msg msg; + + if (chunk > chunk_size) + chunk = chunk_size; + msg.ops = XMFS_URMA_WRITE; + msg.len = chunk_size; + msg.offset = offset + loop_cnt * chunk_size + + chunk_header_cnt * PAGE_SIZE; + if (msg.offset % XMFS_DEFAULT_CHUNK_SIZE == 0) { + chunk_header_cnt++; + msg.offset = offset + loop_cnt * chunk_size + + chunk_header_cnt * PAGE_SIZE; + } + /* + * xmfs_info(sbi->sb, + * "write ubuf msg len %d loopcnt %d offset %ld header cnt %d", + * chunk_size, loop_cnt, msg.offset, chunk_header_cnt); + */ + msg.write_buffer = write_buffer + written; + /* TODO */ + err = xmfs_urma_send_ubuf(sbi, &msg); + if (err) + break; + + written += chunk; + loop_cnt++; + /* xmfs_info(sbi->sb, "qyf end send and wait"); */ + } + urma_pool_release_nslots(&sbi->urma_cfg.pool, + (size + PAGE_SIZE - 1) / PAGE_SIZE, slots); + /* xmfs_info(sbi->sb, "qyf end release"); */ + vfree(slots); + return err; +} + +int xmfs_urma_read_ubuf(struct xmfs_sb_info *sbi, void __user *buf, + struct xmfs_read_extent *exts, int ext_cnt, size_t size) +{ + pgoff_t start_index; + unsigned long start_offset; + unsigned int nr_pages; + unsigned long offset = 0; + struct xmfs_urma_msg msg; + void *local_buf = vzalloc(size); + int err; + + msg.ops = XMFS_URMA_READ; + + for (int i = 0; i < ext_cnt; i++) { + start_index = exts[i].start_index; + start_offset = exts[i].start_offset; + nr_pages = exts[i].nr_pages; + msg.local_addr = local_buf + offset; + msg.len = min(nr_pages * PAGE_SIZE, size - offset); + msg.offset = start_offset; + /* + * xmfs_info(sbi->sb, + * "qyf urma read ubuf from %ld read %ld " + * "real len %ld offset %ld size %ld", + * start_offset, nr_pages * PAGE_SIZE, msg.len, offset, + * size); + */ + err = xmfs_urma_send_ubuf(sbi, &msg); + /* xmfs_info(sbi->sb, "qyf urma send ubuf done %d", err); */ + offset += msg.len; + } + err = copy_to_user(buf, local_buf, size); + if (err) + xmfs_error(NULL, "read ubuf cannot copy to, size is %ld", size); + + vfree(local_buf); + return err; +} + +int xmfs_urma_fill_write_page(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr *wr, + struct xmfs_urma_msg *msg, int slot) +{ + struct wr_status *wrs = kzalloc(sizeof(struct wr_status), GFP_KERNEL); + + if (!wrs) + return -ENOMEM; + wr->rw.src.sge = vzalloc(sizeof(struct ubcore_sge)); + wr->rw.dst.sge = vzalloc(sizeof(struct ubcore_sge)); + if (!wr->rw.src.sge || !wr->rw.dst.sge) { + kfree(wrs); + if (wr->rw.src.sge) + vfree(wr->rw.src.sge); + if (wr->rw.dst.sge) + vfree(wr->rw.dst.sge); + return -ENOMEM; + } + + wrs->status = URMA_ISSUING; + init_completion(&wrs->done); + wrs->ret = 6; + wrs->ops = XMFS_URMA_WRITE; + wrs->magic = XMFS_MAGIC; + wrs->slot = -1; /* slot */ + refcount_set(&wrs->refs, 1); + xmfs_wrs_get(wrs); + memcpy(msg->write_buffer, msg->local_addr, msg->len); + + wr->opcode = UBCORE_OPC_WRITE; + wr->flag.bs.complete_enable = 1; + wr->tjetty = sbi->urma_cfg.tjetty; + wr->rw.src.sge->addr = (uint64_t)msg->write_buffer; + wr->rw.src.sge->len = msg->len; + wr->rw.src.sge->tseg = sbi->urma_cfg.local_seg; + wr->rw.src.num_sge = 1; + wr->rw.dst.sge->addr = sbi->urma_cfg.urma_remote_va + msg->offset; + wr->rw.dst.sge->len = msg->len; + wr->rw.dst.sge->tseg = sbi->urma_cfg.remote_seg; + wr->rw.dst.num_sge = 1; + wr->user_ctx = (uint64_t)(uintptr_t)wrs; + wr->next = NULL; + msg->wr_cnt = 1; + return 0; +} + +int xmfs_urma_send_write_pages(struct xmfs_sb_info *sbi, + struct ubcore_jfs_wr *wr) +{ + struct ubcore_jfs_wr *cur; + struct wr_status *wrs; + int err; + + if (!wr) + return -EINVAL; + struct xmfs_urma_msg msg; + + msg.offset = 0; + msg.len = 0; + err = xmfs_urma_send_and_wait(sbi, &wr[0], &msg); + + cur = &wr[0]; + while (cur) { + vfree(cur->rw.src.sge); + vfree(cur->rw.dst.sge); + wrs = (struct wr_status *)cur->user_ctx; + if (wrs != NULL && err != -EAGAIN && err != 0) { + /* xmfs_info(sbi->sb, "qyf FREE 4 wrs=%px", wrs); */ + kfree(wrs); + } + cur = cur->next; + } + + vfree(wr); + + return err; +} + +int xmfs_pmem_read(struct xmfs_sb_info *sbi, unsigned long offset, void *buf, + unsigned long size) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + memcpy(buf, sbi->kaddr + offset, size); + return 0; +#ifdef CONFIG_XMFS_FS_URMA + case XMFS_URMA_MODE: + return get_meta_and_copy(sbi, offset, buf, size, NULL, 0); +#endif + default: + return -ENODEV; + } +} +EXPORT_SYMBOL(xmfs_pmem_read); + +int xmfs_pmem_cas(struct xmfs_sb_info *sbi, unsigned long offset, + uint64_t old_val, uint64_t new_val) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: { + uint64_t *p = (uint64_t *)(sbi->kaddr + offset); + + return atomic64_cmpxchg((atomic64_t *)p, old_val, new_val) == + old_val ? + 0 : + -EAGAIN; + } +#ifdef CONFIG_XMFS_FS_URMA + case XMFS_URMA_MODE: { + struct xmfs_urma_msg msg; + uint64_t result; + int err; + + memset(&msg, 0, sizeof(msg)); + msg.ops = XMFS_URMA_CAS; + msg.offset = offset; + msg.len = 8; + msg.local_addr = &result; + msg.cas_cmp_data = old_val; + msg.cas_swap_data = new_val; + err = xmfs_urma_send(sbi, &msg); + if (err) + return err; + return result == old_val ? 0 : -EAGAIN; + } +#endif + default: + return -ENODEV; + } +} +EXPORT_SYMBOL(xmfs_pmem_cas); + +int xmfs_urma_send(struct xmfs_sb_info *sbi, struct xmfs_urma_msg *msg) +{ + struct ubcore_jfs_wr *wr = NULL; + int err = 0; + + if (msg->ops == XMFS_URMA_WRITE && msg->len > PAGE_SIZE) { + xmfs_error(NULL, + "write more than PAGE_SIZE is now unsupported"); + return -EOPNOTSUPP; + } + + switch (msg->ops) { + case XMFS_URMA_READ: + err = xmfs_urma_get_read_wr(sbi, &wr, msg); + break; + case XMFS_URMA_WRITE: + err = xmfs_urma_get_write_wr(sbi, &wr, msg); + break; + case XMFS_URMA_CAS: + err = xmfs_urma_get_cas_wr(sbi, &wr, msg); + break; + default: + xmfs_error(NULL, "What exactly are you going to do?"); + return -EOPNOTSUPP; + } + + if (err) { + xmfs_error(NULL, "failed to alloc wr or local buffer"); + goto free_ret; + } + + err = xmfs_urma_send_and_wait(sbi, wr, msg); + +free_ret: + xmfs_urma_free_wr(msg, wr, err); + return err; +} + +#else + +void xmfs_urma_unregister(struct xmfs_sb_info *sbi) +{ +} + +int xmfs_urma_register(struct xmfs_sb_info *sbi, struct xmfs_fs_context *ctx) +{ + return -EOPNOTSUPP; +} + +void xmfs_urma_unimport(struct xmfs_sb_info *sbi) +{ +} + +int xmfs_urma_import(struct xmfs_sb_info *sbi) +{ + return -EOPNOTSUPP; +} + +int xmfs_import_jetty(struct xmfs_sb_info *sbi, struct urma_info info, + int index, bool is_send) +{ + return -EOPNOTSUPP; +} + +int xmfs_send_request(struct xmfs_sb_info *sbi, struct xmfs_rw_msg *req, + struct xmfs_grant_wait *gw, int id) +{ + return -EOPNOTSUPP; +} + +int xmfs_urma_send(struct xmfs_sb_info *sbi, struct xmfs_urma_msg *msg) +{ + return -EOPNOTSUPP; +} + +int xmfs_pmem_write_multi_pages(struct xmfs_sb_info *sbi, unsigned long offset, + struct page **pages, unsigned long page_num) +{ + return -EOPNOTSUPP; +} + +int xmfs_pmem_read_multi_pages(struct xmfs_sb_info *sbi, unsigned long *offsets, + struct page **pages, unsigned long page_num) +{ + return -EOPNOTSUPP; +} + +int xmfs_urma_write_ubuf(struct xmfs_sb_info *sbi, void __user *buf, + size_t size, unsigned long offset) +{ + return -EOPNOTSUPP; +} + +int xmfs_urma_read_ubuf(struct xmfs_sb_info *sbi, void __user *buf, + struct xmfs_read_extent *exts, int ext_cnt, size_t size) +{ + return -EOPNOTSUPP; +} + +int xmfs_pmem_write(struct xmfs_sb_info *sbi, unsigned long offset, + const void *buf, unsigned long size) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + memcpy(sbi->kaddr + offset, buf, size); + return 0; + default: + return -EOPNOTSUPP; + } +} +EXPORT_SYMBOL(xmfs_pmem_write); + +int xmfs_pmem_read(struct xmfs_sb_info *sbi, unsigned long offset, void *buf, + unsigned long size) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: + memcpy(buf, sbi->kaddr + offset, size); + return 0; + default: + return -EOPNOTSUPP; + } +} +EXPORT_SYMBOL(xmfs_pmem_read); + +int xmfs_pmem_cas(struct xmfs_sb_info *sbi, unsigned long offset, + uint64_t old_val, uint64_t new_val) +{ + switch (sbi->mode) { + case XMFS_SHARED_MEM_MODE: + case XMFS_HYBRID_MODE: { + uint64_t *p = (uint64_t *)(sbi->kaddr + offset); + + return atomic64_cmpxchg((atomic64_t *)p, old_val, new_val) == + old_val ? + 0 : + -EAGAIN; + } + default: + return -EOPNOTSUPP; + } +} +EXPORT_SYMBOL(xmfs_pmem_cas); + +#endif diff --git a/fs/xmfs/xattr.c b/fs/xmfs/xattr.c new file mode 100644 index 000000000000..96e1325f63f0 --- /dev/null +++ b/fs/xmfs/xattr.c @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * XMFS: Log Structured Filesystem for UB + * Copyright (C) 2024. Huawei Technologies Co., Ltd + * + * This program can be distributed under the terms of the GNU GPL. + * See the file COPYING. + */ + +#include "xmfs_i.h" + +void xmfs_simple_xattr_free(struct simple_xattr *xattr) +{ + if (xattr) + kfree(xattr->name); + kvfree(xattr); +} + +size_t xmfs_simple_xattr_space(const char *name, size_t size) +{ + /* + * Use "40" instead of sizeof(struct simple_xattr), to return the + * same result on 32-bit and 64-bit, and even if simple_xattr grows. + */ + return 40 + size + strlen(name); +} + +void xmfs_simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space) +{ + struct rb_node *rbp; + + if (freed_space) + *freed_space = 0; + rbp = rb_first(&xattrs->rb_root); + while (rbp) { + struct simple_xattr *xattr; + struct rb_node *rbp_next; + + rbp_next = rb_next(rbp); + xattr = rb_entry(rbp, struct simple_xattr, rb_node); + rb_erase(&xattr->rb_node, &xattrs->rb_root); + if (freed_space) + *freed_space += xmfs_simple_xattr_space(xattr->name, + xattr->size); + xmfs_simple_xattr_free(xattr); + rbp = rbp_next; + } +} + +void xmfs_simple_xattrs_init(struct simple_xattrs *xattrs) +{ + xattrs->rb_root = RB_ROOT; + rwlock_init(&xattrs->lock); +} + +struct simple_xattr *xmfs_simple_xattr_alloc(const void *value, size_t size) +{ + struct simple_xattr *new_xattr; + size_t len; + + /* wrap around? */ + len = sizeof(*new_xattr) + size; + if (len < sizeof(*new_xattr)) + return NULL; + + new_xattr = kvmalloc(len, GFP_KERNEL_ACCOUNT); + if (!new_xattr) + return NULL; + + new_xattr->size = size; + memcpy(new_xattr->value, value, size); + return new_xattr; +} + +static int rbtree_simple_xattr_cmp(const void *key, const struct rb_node *node) +{ + const char *xattr_name = key; + const struct simple_xattr *xattr; + + xattr = rb_entry(node, struct simple_xattr, rb_node); + return strcmp(xattr->name, xattr_name); +} + +struct simple_xattr *xmfs_simple_xattr_set(struct simple_xattrs *xattrs, + const char *name, const void *value, + size_t size, int flags) +{ + struct simple_xattr *old_xattr = NULL, *new_xattr = NULL; + struct rb_node *parent = NULL, **rbp; + int err = 0, ret; + + /* value == NULL means remove */ + if (value) { + new_xattr = xmfs_simple_xattr_alloc(value, size); + if (!new_xattr) + return ERR_PTR(-ENOMEM); + + new_xattr->name = kstrdup(name, GFP_KERNEL_ACCOUNT); + if (!new_xattr->name) { + xmfs_simple_xattr_free(new_xattr); + return ERR_PTR(-ENOMEM); + } + } + + write_lock(&xattrs->lock); + rbp = &xattrs->rb_root.rb_node; + while (*rbp) { + parent = *rbp; + ret = rbtree_simple_xattr_cmp(name, *rbp); + if (ret < 0) + rbp = &(*rbp)->rb_left; + else if (ret > 0) + rbp = &(*rbp)->rb_right; + else + old_xattr = + rb_entry(*rbp, struct simple_xattr, rb_node); + if (old_xattr) + break; + } + + if (old_xattr) { + /* Fail if XATTR_CREATE is requested and the xattr exists. */ + if (flags & XATTR_CREATE) { + err = -EEXIST; + goto out_unlock; + } + + if (new_xattr) + rb_replace_node(&old_xattr->rb_node, + &new_xattr->rb_node, &xattrs->rb_root); + else + rb_erase(&old_xattr->rb_node, &xattrs->rb_root); + } else { + /* Fail if XATTR_REPLACE is requested but no xattr is found. */ + if (flags & XATTR_REPLACE) { + err = -ENODATA; + goto out_unlock; + } + + /* + * If XATTR_CREATE or no flags are specified together with a + * new value simply insert it. + */ + if (new_xattr) { + rb_link_node(&new_xattr->rb_node, parent, rbp); + rb_insert_color(&new_xattr->rb_node, &xattrs->rb_root); + } + + /* + * If XATTR_CREATE or no flags are specified and neither an + * old or new xattr exist then we don't need to do anything. + */ + } + +out_unlock: + write_unlock(&xattrs->lock); + if (!err) + return old_xattr; + xmfs_simple_xattr_free(new_xattr); + return ERR_PTR(err); +} + +int xmfs_simple_xattr_get(struct simple_xattrs *xattrs, const char *name, + void *buffer, size_t size) +{ + struct simple_xattr *xattr = NULL; + struct rb_node *rbp; + int ret = -ENODATA; + + read_lock(&xattrs->lock); + rbp = rb_find(name, &xattrs->rb_root, rbtree_simple_xattr_cmp); + if (rbp) { + xattr = rb_entry(rbp, struct simple_xattr, rb_node); + ret = xattr->size; + if (buffer) { + if (size < xattr->size) + ret = -ERANGE; + else + memcpy(buffer, xattr->value, xattr->size); + } + } + read_unlock(&xattrs->lock); + return ret; +} + +static bool xattr_is_trusted(const char *name) +{ + return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN); +} + +int xmfs_xattr_list_one(char **buffer, ssize_t *remaining_size, + const char *name) +{ + size_t len; + + len = strlen(name) + 1; + if (*buffer) { + if (*remaining_size < len) + return -ERANGE; + memcpy(*buffer, name, len); + *buffer += len; + } + *remaining_size -= len; + return 0; +} + +ssize_t xmfs_simple_xattr_list(struct inode *inode, + struct simple_xattrs *xattrs, char *buffer, + size_t size) +{ + bool trusted = ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN); + struct simple_xattr *xattr; + struct rb_node *rbp; + ssize_t remaining_size = size; + int err = 0; + + read_lock(&xattrs->lock); + for (rbp = rb_first(&xattrs->rb_root); rbp; rbp = rb_next(rbp)) { + xattr = rb_entry(rbp, struct simple_xattr, rb_node); + + /* skip "trusted." attributes for unprivileged callers */ + if (!trusted && xattr_is_trusted(xattr->name)) + continue; + + err = xmfs_xattr_list_one(&buffer, &remaining_size, + xattr->name); + if (err) + break; + } + read_unlock(&xattrs->lock); + + return err ? err : size - remaining_size; +} + +static unsigned long xmfs_xattr_write_value(struct inode *inode, + const char *name, const void *value, + size_t size, + struct xmfs_log *xm_log, + struct xmfs_sb_index *xmsi) +{ + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return 0; + unsigned long data_current = xmsi->data_current; + unsigned long data_count = xmsi->data_count; + long slot; + + WARN_ON(xmsi->count == 0); + + if (data_current == 0 || data_count == 511) { + slot = xmfs_alloc_and_mark_slot(xmsi, sbi, 'd'); + if (slot < 0) + return -ENOSPC; + xmsi->data_current = + DATA_AREA_OFFSET + slot * sbi->trunk_size + PAGE_SIZE; + xmsi->data_count = 0; + } else { + xmsi->data_current = data_current + PAGE_SIZE; + } + xm_log->data_offset = xmsi->data_current; + xmsi->data_count = xmsi->data_count + 1; + + xmfs_data_write(sbi, xmsi->data_current, value, size); + xmfs_mark_page_live(sbi, xmsi->data_current); + + return 0; +} + +static int xmfs_xattr_set_log(struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + + if (xmfs_quiesce_wait(sbi)) + return -EINTR; + struct xmfs_inode_info *xmfs_inode = XMFS_I(inode); + struct xmfs_log xm_log = { 0 }; + + xm_log.index = size; + xm_log.end_index = strlen(name); + xm_log.ops = XMFS_XATTR; + xm_log.size = flags; + xm_log.p_ino = inode->__i_ctime.tv_sec; + xm_log.offset = inode->__i_ctime.tv_nsec; + memcpy(xm_log.name, name, strlen(name)); + + spin_lock(&sbi->space_lock); + if (xmfs_inode->die) { + xmfs_info(inode->i_sb, + "inode logs have been interrupted before set xattr"); + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + + struct xmfs_sb_index *xmsi = get_xmsi(sbi); + + if (!xmsi) { + spin_unlock(&sbi->space_lock); + return -EINVAL; + } + int logcount = 1; + int need_data_trunk = + (xmsi->data_count == 511 || xmsi->data_current == 0) ? 1 : 0; + if (xmsi->used_trunk_count + logcount + need_data_trunk > + sbi->max_chunk) { + sbi->full = true; + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + return -ENOSPC; + } + + xmfs_xattr_write_value(inode, name, value, size, &xm_log, xmsi); + spin_unlock(&sbi->space_lock); + put_xmsi(sbi, xmsi); + unsigned long xmlt_addr = + xmfs_add_log_v3(&xm_log, sbi, strlen(name), inode->i_ino, true); + if (xmlt_addr < 0) { + xmfs_inode->die = true; + return -EINVAL; + } + + return 0; +} + +int xmfs_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info) +{ + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + const struct xattr *xattr; + struct simple_xattr *new_xattr; + struct xmfs_inode_info *info = XMFS_I(inode); + size_t len; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + new_xattr = + xmfs_simple_xattr_alloc(xattr->value, xattr->value_len); + if (!new_xattr) + break; + + len = strlen(xattr->name) + 1; + if ((len + XATTR_SECURITY_PREFIX_LEN) > 255 || + new_xattr->size > PAGE_SIZE) { + continue; + } + new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, + GFP_KERNEL_ACCOUNT); + if (!new_xattr->name) { + kvfree(new_xattr); + break; + } + + memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN); + memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, xattr->name, + len); + + new_xattr = xmfs_simple_xattr_set(&info->xattrs, + new_xattr->name, + new_xattr->value, + new_xattr->size, 0); + if (!IS_ERR(new_xattr)) { + inode_set_ctime_current(inode); + if (sbi->id == 0) { + err = xmfs_xattr_set_log(inode, new_xattr->name, + new_xattr->value, + new_xattr->size, 0); + if (err < 0) { + xmfs_simple_xattr_free(new_xattr); + break; + } + } + xmfs_simple_xattr_free(new_xattr); + } else { + break; + } + } + + if (xattr->name != NULL) { + xmfs_simple_xattrs_free(&info->xattrs, NULL); + return -ENOMEM; + } + + return err; +} + +static int xmfs_xattr_slave_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct xmfs_inode_info *info = XMFS_I(inode); + + if (inode->i_ino == 0) { + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_sb_index *xmsi = + kmalloc(sizeof(struct xmfs_sb_index), GFP_KERNEL); + int err = get_xmsi_and_copy(sbi, xmsi); + + if (unlikely(err)) { + xmfs_error(inode->i_sb, + "hardware memory error when reading superblock during slave dir iterate"); + kfree(xmsi); + return err; + } + + sbi->stop_cnt++; + if (xmsi->last_update != sbi->last_update) { + if (!sbi->debug) + xmfs_replay_new_entries(sbi, NULL, + xmsi->last_update); + else + xmfs_replay_new_entries(sbi, NULL, + xmsi->last_update); + } + kfree(xmsi); + } + name = xattr_full_name(handler, name); + return xmfs_simple_xattr_get(&info->xattrs, name, buffer, size); +} + +static int xmfs_xattr_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct xmfs_inode_info *info = XMFS_I(inode); + + name = xattr_full_name(handler, name); + return xmfs_simple_xattr_get(&info->xattrs, name, buffer, size); +} + +static int xmfs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, struct dentry *unused, + struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct simple_xattr *old_xattr; + struct xmfs_inode_info *info = XMFS_I(inode); + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + int err = 0; + + name = xattr_full_name(handler, name); + if (strlen(name) > 255 || size > PAGE_SIZE) + return -EINVAL; + + old_xattr = + xmfs_simple_xattr_set(&info->xattrs, name, value, size, flags); + if (!IS_ERR(old_xattr)) { + inode_set_ctime_current(inode); + if (sbi->id == 0) + err = xmfs_xattr_set_log(inode, name, value, size, + flags); + xmfs_simple_xattr_free(old_xattr); + old_xattr = NULL; + } + + return err == 0 ? PTR_ERR(old_xattr) : err; +} + +ssize_t xmfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct xmfs_inode_info *info = XMFS_I(d_inode(dentry)); + + return xmfs_simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, + size); +} + +static const struct xattr_handler xmfs_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = xmfs_xattr_get, + .set = xmfs_xattr_set, +}; + +static const struct xattr_handler xmfs_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = xmfs_xattr_get, + .set = xmfs_xattr_set, +}; + +static const struct xattr_handler xmfs_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .get = xmfs_xattr_get, + .set = xmfs_xattr_set, +}; + +static int xmfs_xattr_set_slave(const struct xattr_handler *handler, + struct mnt_idmap *idmap, struct dentry *dentry, + struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct xmfs_sb_info *sbi = XMFS_SB(inode->i_sb); + struct xmfs_rw_msg req = {}, grant = {}; + struct xmfs_log xm_log; + unsigned long log_off; + unsigned long pages; + int err; + + if (!sbi || sbi->id == 0) + return -EINVAL; + + if (value && size > 0) + pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; + else + pages = 0; + + req.ops = XMFS_XATTR; + req.i_ino = inode->i_ino; + req.data_pages = pages; + req.log_entries = 1; + req.size = size; + req.mode = flags; + snprintf(req.name, 256, "%s%s", handler->prefix, name); + + req.type = MSG_RW_REQUEST; + req.slave_id = sbi->id; + spin_lock(&sbi->local_ring_lock); + req.request_id = ++sbi->rw_request_id; + spin_unlock(&sbi->local_ring_lock); + struct xmfs_grant_wait gw; + + init_completion(&gw.done); + xa_store(&sbi->urma_cfg.grant_waits, req.request_id, &gw, GFP_KERNEL); + err = xmfs_send_request(sbi, &req, &gw, 0); + if (err) + return err; + grant = gw.grant; + + if (value && size > 0 && grant.data_addr != 0) + xmfs_data_write(sbi, grant.data_addr, (void *)value, size); + + log_off = grant.log_trunk_addr + + offsetof(struct xmfs_log_trunk, log[0]) + + grant.log_slot_start * sizeof(struct xmfs_log); + memset(&xm_log, 0, sizeof(xm_log)); + xm_log.ops = XMFS_XATTR; + xm_log.i_ino = inode->i_ino; + xm_log.version = grant.version; + xm_log.data_offset = grant.data_addr; + xm_log.size = size; + xm_log.mode = flags; + xm_log.writer_id = sbi->id; + snprintf(xm_log.name, 256, "%s%s", handler->prefix, name); + xmfs_data_write(sbi, log_off, &xm_log, sizeof(xm_log)); + + { + const char *full_name = xattr_full_name(handler, name); + struct simple_xattr *old_xattr; + + old_xattr = xmfs_simple_xattr_set(&XMFS_I(inode)->xattrs, + full_name, value, size, + flags); + if (IS_ERR(old_xattr)) + err = PTR_ERR(old_xattr); + else + xmfs_simple_xattr_free(old_xattr); + } + + return err; +} + +static const struct xattr_handler xmfs_slave_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = xmfs_xattr_slave_get, + .set = xmfs_xattr_set_slave, +}; + +static const struct xattr_handler xmfs_slave_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = xmfs_xattr_slave_get, + .set = xmfs_xattr_set_slave, +}; + +static const struct xattr_handler xmfs_slave_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .get = xmfs_xattr_slave_get, + .set = xmfs_xattr_set_slave, +}; + +const struct xattr_handler *xmfs_host_xattr_handlers[] = { + &xmfs_security_xattr_handler, + &xmfs_trusted_xattr_handler, + &xmfs_user_xattr_handler, + NULL +}; + +const struct xattr_handler *xmfs_slave_xattr_handlers[] = { + &xmfs_slave_security_xattr_handler, + &xmfs_slave_trusted_xattr_handler, + &xmfs_slave_user_xattr_handler, + NULL +}; diff --git a/fs/Kconfig b/fs/Kconfig index d80f39d6632f..9a6fd57368a6 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -139,6 +139,7 @@ source "fs/quota/Kconfig" source "fs/autofs/Kconfig" source "fs/fuse/Kconfig" source "fs/overlayfs/Kconfig" +source "fs/xmfs/Kconfig" menu "Caches" diff --git a/fs/Makefile b/fs/Makefile index 49a03a43d7db..c908f403d73e 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -111,6 +111,7 @@ obj-$(CONFIG_AUTOFS_FS) += autofs/ obj-$(CONFIG_ADFS_FS) += adfs/ obj-$(CONFIG_FUSE_FS) += fuse/ obj-$(CONFIG_OVERLAY_FS) += overlayfs/ +obj-$(CONFIG_XMFS_FS) += xmfs/ obj-$(CONFIG_ORANGEFS_FS) += orangefs/ obj-$(CONFIG_UDF_FS) += udf/ obj-$(CONFIG_SUN_OPENPROMFS) += openpromfs/ diff --git a/fs/xmfs/Kconfig b/fs/xmfs/Kconfig new file mode 100644 index 000000000000..2fb10c42b9c5 --- /dev/null +++ b/fs/xmfs/Kconfig @@ -0,0 +1,21 @@ +config XMFS_FS + tristate "XMFS (Log Structured Filesystem for UB) support" + help + XMFS is a high-performance log-structured filesystem specifically + designed for distributed storage environments. It leverages advanced + hardware and transport technologies such as UB, URMA, CXL, PMEM, + and HCCS to achieve ultra-low latency and high throughput. + + It provides robust crash consistency, efficient log management, and + optimized data placement strategies for modern storage architectures. + + If unsure, say N. + +config XMFS_FS_URMA + bool "XMFS URMA support" + depends on XMFS_FS && CONFIG_UB_URMA && CONFIG_UB_UBASE + help + Say Y here if you want to enable URMA (Unified Remote Memory Access) + transport support for XMFS, allowing high-speed network communication + and remote memory operations. + default n diff --git a/fs/xmfs/Makefile b/fs/xmfs/Makefile new file mode 100644 index 000000000000..09544186e5c4 --- /dev/null +++ b/fs/xmfs/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux xmfs-filesystem routines. +# + +obj-$(CONFIG_XMFS_FS) += xmfs.o +xmfs-y += super.o +xmfs-y += inode.o +xmfs-y += replay.o +xmfs-y += file.o +xmfs-y += dir.o +xmfs-y += namei.o +xmfs-y += symlink.o +xmfs-y += xattr.o +xmfs-y += urma.o +xmfs-y += gc.o -- 2.52.0

2 1

[PATCH OLK-6.6 v2 0/6] bonding: Fix CVE-2026-52975
by superdcc97＠163.com 11 Aug '26

11 Aug '26

From: Dong Chenchen <dongchenchen2(a)huawei.com> Fix CVE-2026-52975 Eric Dumazet (2): bonding: 3ad: implement proper RCU rules for port->aggregator bonding: annotate data-races arcound churn variables Hangbin Liu (3): bonding: add support for per-port LACP actor priority bonding: print churn state via netlink bonding: fix NULL pointer dereference in actor_port_prio setting Tonghao Zhang (1): net: bonding: add broadcast_neighbor option for 802.3ad Documentation/networking/bonding.rst | 15 +++ drivers/net/bonding/bond_3ad.c | 131 ++++++++++++++----------- drivers/net/bonding/bond_main.c | 72 ++++++++++++-- drivers/net/bonding/bond_netlink.c | 37 ++++++- drivers/net/bonding/bond_options.c | 71 ++++++++++++++ drivers/net/bonding/bond_procfs.c | 11 ++- drivers/net/bonding/bond_sysfs_slave.c | 17 ++-- include/net/bond_3ad.h | 3 +- include/net/bond_options.h | 2 + include/net/bonding.h | 3 + include/uapi/linux/if_link.h | 3 + 11 files changed, 283 insertions(+), 82 deletions(-) -- 2.43.0

2 7

[PATCH OLK-6.6 0/7] backport minios part2 patches from linux mainline
by Qinxin Xia 11 Aug '26

11 Aug '26

From: Hongye Lin <linhongye(a)h-partners.com> driver inclusion category: feature bugzilla: https://atomgit.com/openeuler/kernel/issues/8904 ---------------------------------------------------------------------- backport minios part2 patches from linux mainline Marc Zyngier (1): arm64: sysreg: Add missing ID_AA64ISAR[13]_EL1 fields and variants Mark Brown (1): arm64/sysreg: Update ID_AA64ISAR3_EL1 to DDI0601 2024-09 Yeoreum Yun (5): arm64: cpufeature: Add FEAT_LSUI arm64: futex: Refactor futex atomic operation arm64: futex: Support futex with FEAT_LSUI arm64: armv8_deprecated: Disable swp emulation when FEAT_LSUI present arm64: Kconfig: Add support for LSUI arch/arm64/Kconfig | 20 ++ arch/arm64/include/asm/futex.h | 311 ++++++++++++++++++++++----- arch/arm64/include/asm/lsui.h | 27 +++ arch/arm64/kernel/armv8_deprecated.c | 14 ++ arch/arm64/kernel/cpufeature.c | 10 + arch/arm64/tools/cpucaps | 1 + arch/arm64/tools/sysreg | 24 ++- 7 files changed, 348 insertions(+), 59 deletions(-) create mode 100644 arch/arm64/include/asm/lsui.h -- 2.33.0

2 8

[PATCH OLK-6.6] netfilter: nf_tables: add hook transactions for device deletions
by Dong Chenchen 11 Aug '26

11 Aug '26

From: Pablo Neira Ayuso <pablo(a)netfilter.org> mainline inclusion from mainline-v7.1-rc2 commit 10f79dbd7719d1da9f5884d13060322d8729f091 category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16187 CVE: CVE-2026-63858 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?… -------------------------------- Restore the flag that indicates that the hook is going away, ie. NFT_HOOK_REMOVE, but add a new transaction object to track deletion of hooks without altering the basechain/flowtable hook_list during the preparation phase. The existing approach that moves the hook from the basechain/flowtable hook_list to transaction hook_list breaks netlink dump path readers of this RCU-protected list. It should be possible use an array for nft_trans_hook to store the deleted hooks to compact the representation but I am not expecting many hook object, specially now that wildcard support for devices is in place. Note that the nft_trans_chain_hooks() list contains a list of struct nft_trans_hook objects for DELCHAIN and DELFLOWTABLE commands, while this list stores struct nft_hook objects for NEWCHAIN and NEWFLOWTABLE. Note that new commands can be updated to use nft_trans_hook for consistency. This patch also adapts the event notification path to deal with the list of hook transactions. Fixes: 7d937b107108 ("netfilter: nf_tables: support for deleting devices in an existing netdev chain") Fixes: b6d9014a3335 ("netfilter: nf_tables: delete flowtable hooks via transaction list") Reported-by: Xiang Mei <xmei5(a)asu.edu> Signed-off-by: Pablo Neira Ayuso <pablo(a)netfilter.org> Conflicts: include/net/netfilter/nf_tables.h net/netfilter/nf_tables_api.c [-commit b7c2d793c28cd, 880ccec0d02e and 4039ce7ef404 that introduce nft_nla_put_hook_dev() are not backport. use nla_put_string to adapt. -commit 3567146b94af add net param and lockdep to nft_dump_basechain_hook. -commit a1050dd07168 add event check in nf_tables_fill_chain_info. Its no need to add redundant hook list check for nft_dump_basechain_hook. -commit 2932ba8d9c99 that introduce kmalloc_obj is not backport. we still use kmalloc for hook -73319a8ee18b9 introduce ops_list to nft_hook, which is not backport -91a089d0569d Pass nf_hook_ops to nft_unregister_flowtable_hook(), which is not backport. -e169285f8c56b that move net,seq from nft_ctx to trans is not backport] Signed-off-by: Dong Chenchen <dongchenchen2(a)huawei.com> --- include/net/netfilter/nf_tables.h | 13 ++ net/netfilter/nf_tables_api.c | 252 +++++++++++++++++++++++------- 2 files changed, 209 insertions(+), 56 deletions(-) diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index df8af10136da..6de207b6d6e6 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -1207,10 +1207,13 @@ struct nft_stats { struct u64_stats_sync syncp; }; +#define NFT_HOOK_REMOVE (1 << 0) + struct nft_hook { struct list_head list; struct nf_hook_ops ops; struct rcu_head rcu; + u8 flags; }; /** @@ -1645,6 +1648,16 @@ struct nft_trans { char data[]; }; +/** + * struct nft_trans_hook - nf_tables hook update in transaction + * @list: used internally + * @hook: struct nft_hook with the device hook + */ +struct nft_trans_hook { + struct list_head list; + struct nft_hook *hook; +}; + struct nft_trans_rule { struct nft_rule *rule; struct nft_flow_rule *flow; diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 3c98d088857a..696b053522f0 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -339,6 +339,30 @@ static void nft_netdev_hook_unlink_free_rcu(struct nft_hook *hook) nft_netdev_hook_free_rcu(hook); } +static void nft_trans_hook_destroy(struct nft_trans_hook *trans_hook) +{ + list_del(&trans_hook->list); + kfree(trans_hook); +} + +static void nft_netdev_unregister_trans_hook(struct net *net, + const struct nft_table *table, + struct list_head *hook_list) +{ + struct nft_trans_hook *trans_hook, *next; + struct nft_hook *hook; + + list_for_each_entry_safe(trans_hook, next, hook_list, list) { + hook = trans_hook->hook; + + if (!(table->flags & NFT_TABLE_F_DORMANT)) + nf_unregister_net_hook(net, &hook->ops); + + nft_netdev_hook_unlink_free_rcu(hook); + nft_trans_hook_destroy(trans_hook); + } +} + static void nft_netdev_unregister_hooks(struct net *net, struct list_head *hook_list, bool release_netdev) @@ -1791,14 +1815,66 @@ static int nft_dump_stats(struct sk_buff *skb, struct nft_stats __percpu *stats) return -ENOSPC; } +struct nft_hook_dump_ctx { + struct nft_hook *first; + int n; +}; + +static int nft_dump_basechain_hook_one(struct sk_buff *skb, + struct nft_hook *hook, + struct nft_hook_dump_ctx *dump_ctx) +{ + if (!dump_ctx->first) + dump_ctx->first = hook; + + if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name)) + return -1; + + dump_ctx->n++; + + return 0; +} + +static int nft_dump_basechain_hook_list(struct sk_buff *skb, + const struct list_head *hook_list, + struct nft_hook_dump_ctx *dump_ctx) +{ + struct nft_hook *hook; + int err; + + list_for_each_entry_rcu(hook, hook_list, list) { + err = nft_dump_basechain_hook_one(skb, hook, dump_ctx); + if (err < 0) + return err; + } + + return 0; +} + +static int nft_dump_basechain_trans_hook_list(struct sk_buff *skb, + const struct list_head *trans_hook_list, + struct nft_hook_dump_ctx *dump_ctx) +{ + struct nft_trans_hook *trans_hook; + int err; + + list_for_each_entry(trans_hook, trans_hook_list, list) { + err = nft_dump_basechain_hook_one(skb, trans_hook->hook, dump_ctx); + if (err < 0) + return err; + } + + return 0; +} + static int nft_dump_basechain_hook(struct sk_buff *skb, int family, const struct nft_base_chain *basechain, - const struct list_head *hook_list) + const struct list_head *hook_list, + const struct list_head *trans_hook_list) { const struct nf_hook_ops *ops = &basechain->ops; - struct nft_hook *hook, *first = NULL; + struct nft_hook_dump_ctx dump_hook_ctx = {}; struct nlattr *nest, *nest_devs; - int n = 0; nest = nla_nest_start_noflag(skb, NFTA_CHAIN_HOOK); if (nest == NULL) @@ -1813,22 +1889,22 @@ static int nft_dump_basechain_hook(struct sk_buff *skb, int family, if (!nest_devs) goto nla_put_failure; - if (!hook_list) + if (!hook_list && !trans_hook_list) hook_list = &basechain->hook_list; - list_for_each_entry_rcu(hook, hook_list, list) { - if (!first) - first = hook; - - if (nla_put_string(skb, NFTA_DEVICE_NAME, - hook->ops.dev->name)) - goto nla_put_failure; - n++; + if (hook_list && + nft_dump_basechain_hook_list(skb, hook_list, &dump_hook_ctx)) { + goto nla_put_failure; + } else if (trans_hook_list && + nft_dump_basechain_trans_hook_list(skb, trans_hook_list, + &dump_hook_ctx)) { + goto nla_put_failure; } + nla_nest_end(skb, nest_devs); - if (n == 1 && - nla_put_string(skb, NFTA_HOOK_DEV, first->ops.dev->name)) + if (dump_hook_ctx.n == 1 && + nla_put_string(skb, NFTA_HOOK_DEV, dump_hook_ctx.first->ops.dev->name)) goto nla_put_failure; } nla_nest_end(skb, nest); @@ -1842,7 +1918,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, u32 portid, u32 seq, int event, u32 flags, int family, const struct nft_table *table, const struct nft_chain *chain, - const struct list_head *hook_list) + const struct list_head *hook_list, + const struct list_head *trans_hook_list) { struct nlmsghdr *nlh; @@ -1862,7 +1939,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, const struct nft_base_chain *basechain = nft_base_chain(chain); struct nft_stats __percpu *stats; - if (nft_dump_basechain_hook(skb, family, basechain, hook_list)) + if (nft_dump_basechain_hook(skb, family, basechain, + hook_list, trans_hook_list)) goto nla_put_failure; if (nla_put_be32(skb, NFTA_CHAIN_POLICY, @@ -1898,7 +1976,8 @@ static int nf_tables_fill_chain_info(struct sk_buff *skb, struct net *net, } static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event, - const struct list_head *hook_list) + const struct list_head *hook_list, + const struct list_head *trans_hook_list) { struct nftables_pernet *nft_net; struct sk_buff *skb; @@ -1918,7 +1997,7 @@ static void nf_tables_chain_notify(const struct nft_ctx *ctx, int event, err = nf_tables_fill_chain_info(skb, ctx->net, ctx->portid, ctx->seq, event, flags, ctx->family, ctx->table, - ctx->chain, hook_list); + ctx->chain, hook_list, trans_hook_list); if (err < 0) { kfree_skb(skb); goto err; @@ -1964,7 +2043,7 @@ static int nf_tables_dump_chains(struct sk_buff *skb, NFT_MSG_NEWCHAIN, NLM_F_MULTI, table->family, table, - chain, NULL) < 0) + chain, NULL, NULL) < 0) goto done; nl_dump_check_consistent(cb, nlmsg_hdr(skb)); @@ -2018,7 +2097,7 @@ static int nf_tables_getchain(struct sk_buff *skb, const struct nfnl_info *info, err = nf_tables_fill_chain_info(skb2, net, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, NFT_MSG_NEWCHAIN, - 0, family, table, chain, NULL); + 0, family, table, chain, NULL, NULL); if (err < 0) goto err_fill_chain_info; @@ -2168,8 +2247,12 @@ static struct nft_hook *nft_hook_list_find(struct list_head *hook_list, struct nft_hook *hook; list_for_each_entry(hook, hook_list, list) { - if (this->ops.dev == hook->ops.dev) + if (this->ops.dev == hook->ops.dev) { + if (hook->flags & NFT_HOOK_REMOVE) + continue; + return hook; + } } return NULL; @@ -2917,6 +3000,32 @@ static int nf_tables_newchain(struct sk_buff *skb, const struct nfnl_info *info, return nf_tables_addchain(&ctx, family, genmask, policy, flags, extack); } +static int nft_trans_delhook(struct nft_hook *hook, + struct list_head *del_list) +{ + struct nft_trans_hook *trans_hook; + + trans_hook = kmalloc(sizeof(*trans_hook), GFP_KERNEL); + if (!trans_hook) + return -ENOMEM; + + trans_hook->hook = hook; + list_add_tail(&trans_hook->list, del_list); + hook->flags |= NFT_HOOK_REMOVE; + + return 0; +} + +static void nft_trans_delhook_abort(struct list_head *del_list) +{ + struct nft_trans_hook *trans_hook, *next; + + list_for_each_entry_safe(trans_hook, next, del_list, list) { + trans_hook->hook->flags &= ~NFT_HOOK_REMOVE; + nft_trans_hook_destroy(trans_hook); + } +} + static int nft_delchain_hook(struct nft_ctx *ctx, struct nft_base_chain *basechain, struct netlink_ext_ack *extack) @@ -2943,7 +3052,10 @@ static int nft_delchain_hook(struct nft_ctx *ctx, err = -ENOENT; goto err_chain_del_hook; } - list_move(&hook->list, &chain_del_list); + if (nft_trans_delhook(hook, &chain_del_list) < 0) { + err = -ENOMEM; + goto err_chain_del_hook; + } } trans = nft_trans_alloc(ctx, NFT_MSG_DELCHAIN, @@ -2964,7 +3076,7 @@ static int nft_delchain_hook(struct nft_ctx *ctx, return 0; err_chain_del_hook: - list_splice(&chain_del_list, &basechain->hook_list); + nft_trans_delhook_abort(&chain_del_list); nft_chain_release_hook(&chain_hook); return err; @@ -8572,6 +8684,22 @@ static void nft_hooks_destroy(struct list_head *hook_list) nft_netdev_hook_unlink_free_rcu(hook); } +static void nft_flowtable_unregister_trans_hook(struct net *net, + struct nft_flowtable *flowtable, + struct list_head *hook_list) +{ + struct nft_trans_hook *trans_hook, *next; + struct nft_hook *hook; + + list_for_each_entry_safe(trans_hook, next, hook_list, list) { + hook = trans_hook->hook; + + nft_unregister_flowtable_hook(net, flowtable, hook); + nft_netdev_hook_unlink_free_rcu(hook); + nft_trans_hook_destroy(trans_hook); + } +} + static int nft_flowtable_update(struct nft_ctx *ctx, const struct nlmsghdr *nlh, struct nft_flowtable *flowtable, struct netlink_ext_ack *extack) @@ -8826,7 +8954,10 @@ static int nft_delflowtable_hook(struct nft_ctx *ctx, err = -ENOENT; goto err_flowtable_del_hook; } - list_move(&hook->list, &flowtable_del_list); + if (nft_trans_delhook(hook, &flowtable_del_list) < 0) { + err = -ENOMEM; + goto err_flowtable_del_hook; + } } trans = nft_trans_alloc(ctx, NFT_MSG_DELFLOWTABLE, @@ -8847,7 +8978,7 @@ static int nft_delflowtable_hook(struct nft_ctx *ctx, return 0; err_flowtable_del_hook: - list_splice(&flowtable_del_list, &flowtable->hook_list); + nft_trans_delhook_abort(&flowtable_del_list); nft_flowtable_hook_release(&flowtable_hook); return err; @@ -8912,8 +9043,10 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net, u32 portid, u32 seq, int event, u32 flags, int family, struct nft_flowtable *flowtable, - struct list_head *hook_list) + struct list_head *hook_list, + struct list_head *trans_hook_list) { + struct nft_trans_hook *trans_hook; struct nlattr *nest, *nest_devs; struct nft_hook *hook; struct nlmsghdr *nlh; @@ -8945,12 +9078,21 @@ static int nf_tables_fill_flowtable_info(struct sk_buff *skb, struct net *net, if (!nest_devs) goto nla_put_failure; - if (!hook_list) + if (!hook_list && !trans_hook_list) hook_list = &flowtable->hook_list; - list_for_each_entry_rcu(hook, hook_list, list) { - if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name)) - goto nla_put_failure; + if (hook_list) { + list_for_each_entry_rcu(hook, hook_list, list) { + if (nla_put_string(skb, NFTA_DEVICE_NAME, hook->ops.dev->name)) + goto nla_put_failure; + } + } else if (trans_hook_list) { + list_for_each_entry(trans_hook, trans_hook_list, list) { + if (nla_put_string(skb, NFTA_DEVICE_NAME, + trans_hook->hook->ops.dev->name)) + goto nla_put_failure; + } + } nla_nest_end(skb, nest_devs); nla_nest_end(skb, nest); @@ -9004,7 +9146,7 @@ static int nf_tables_dump_flowtable(struct sk_buff *skb, NFT_MSG_NEWFLOWTABLE, NLM_F_MULTI | NLM_F_APPEND, table->family, - flowtable, NULL) < 0) + flowtable, NULL, NULL) < 0) goto done; nl_dump_check_consistent(cb, nlmsg_hdr(skb)); @@ -9099,7 +9241,7 @@ static int nf_tables_getflowtable(struct sk_buff *skb, err = nf_tables_fill_flowtable_info(skb2, net, NETLINK_CB(skb).portid, info->nlh->nlmsg_seq, NFT_MSG_NEWFLOWTABLE, 0, family, - flowtable, NULL); + flowtable, NULL, NULL); if (err < 0) goto err_fill_flowtable_info; @@ -9112,7 +9254,9 @@ static int nf_tables_getflowtable(struct sk_buff *skb, static void nf_tables_flowtable_notify(struct nft_ctx *ctx, struct nft_flowtable *flowtable, - struct list_head *hook_list, int event) + struct list_head *hook_list, + struct list_head *trans_hook_list, + int event) { struct nftables_pernet *nft_net = nft_pernet(ctx->net); struct sk_buff *skb; @@ -9132,7 +9276,8 @@ static void nf_tables_flowtable_notify(struct nft_ctx *ctx, err = nf_tables_fill_flowtable_info(skb, ctx->net, ctx->portid, ctx->seq, event, flags, - ctx->family, flowtable, hook_list); + ctx->family, flowtable, + hook_list, trans_hook_list); if (err < 0) { kfree_skb(skb); goto err; @@ -9573,9 +9718,7 @@ static void nft_commit_release(struct nft_trans *trans) break; case NFT_MSG_DELCHAIN: case NFT_MSG_DESTROYCHAIN: - if (nft_trans_chain_update(trans)) - nft_hooks_destroy(&nft_trans_chain_hooks(trans)); - else + if (!nft_trans_chain_update(trans)) nf_tables_chain_destroy(nft_trans_chain(trans)); break; case NFT_MSG_DELRULE: @@ -9598,9 +9741,7 @@ static void nft_commit_release(struct nft_trans *trans) break; case NFT_MSG_DELFLOWTABLE: case NFT_MSG_DESTROYFLOWTABLE: - if (nft_trans_flowtable_update(trans)) - nft_hooks_destroy(&nft_trans_flowtable_hooks(trans)); - else + if (!nft_trans_flowtable_update(trans)) nf_tables_flowtable_destroy(nft_trans_flowtable(trans)); break; } @@ -10349,31 +10490,28 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) if (nft_trans_chain_update(trans)) { nft_chain_commit_update(trans); nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, - &nft_trans_chain_hooks(trans)); + &nft_trans_chain_hooks(trans), NULL); list_splice_rcu(&nft_trans_chain_hooks(trans), &nft_trans_basechain(trans)->hook_list); /* trans destroyed after rcu grace period */ } else { nft_chain_commit_drop_policy(trans); nft_clear(net, trans->ctx.chain); - nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL); + nf_tables_chain_notify(&trans->ctx, NFT_MSG_NEWCHAIN, NULL, NULL); nft_trans_destroy(trans); } break; case NFT_MSG_DELCHAIN: case NFT_MSG_DESTROYCHAIN: if (nft_trans_chain_update(trans)) { - nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN, + nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN, NULL, &nft_trans_chain_hooks(trans)); - if (!(trans->ctx.table->flags & NFT_TABLE_F_DORMANT)) { - nft_netdev_unregister_hooks(net, - &nft_trans_chain_hooks(trans), - true); - } + nft_netdev_unregister_trans_hook(net, trans->ctx.table, + &nft_trans_chain_hooks(trans)); } else { nft_chain_del(trans->ctx.chain); nf_tables_chain_notify(&trans->ctx, NFT_MSG_DELCHAIN, - NULL); + NULL, NULL); nf_tables_unregister_hook(trans->ctx.net, trans->ctx.table, trans->ctx.chain); @@ -10490,6 +10628,7 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) nf_tables_flowtable_notify(&trans->ctx, nft_trans_flowtable(trans), &nft_trans_flowtable_hooks(trans), + NULL, NFT_MSG_NEWFLOWTABLE); list_splice_rcu(&nft_trans_flowtable_hooks(trans), &nft_trans_flowtable(trans)->hook_list); @@ -10498,6 +10637,7 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) nf_tables_flowtable_notify(&trans->ctx, nft_trans_flowtable(trans), NULL, + NULL, NFT_MSG_NEWFLOWTABLE); } nft_trans_destroy(trans); @@ -10507,16 +10647,18 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) if (nft_trans_flowtable_update(trans)) { nf_tables_flowtable_notify(&trans->ctx, nft_trans_flowtable(trans), + NULL, &nft_trans_flowtable_hooks(trans), trans->msg_type); - nft_unregister_flowtable_net_hooks(net, - nft_trans_flowtable(trans), - &nft_trans_flowtable_hooks(trans)); + nft_flowtable_unregister_trans_hook(net, + nft_trans_flowtable(trans), + &nft_trans_flowtable_hooks(trans)); } else { list_del_rcu(&nft_trans_flowtable(trans)->list); nf_tables_flowtable_notify(&trans->ctx, nft_trans_flowtable(trans), NULL, + NULL, trans->msg_type); nft_unregister_flowtable_net_hooks(net, nft_trans_flowtable(trans), @@ -10667,8 +10809,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) case NFT_MSG_DELCHAIN: case NFT_MSG_DESTROYCHAIN: if (nft_trans_chain_update(trans)) { - list_splice(&nft_trans_chain_hooks(trans), - &nft_trans_basechain(trans)->hook_list); + nft_trans_delhook_abort(&nft_trans_chain_hooks(trans)); } else { nft_use_inc_restore(&trans->ctx.table->use); nft_clear(trans->ctx.net, trans->ctx.chain); @@ -10785,8 +10926,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) case NFT_MSG_DELFLOWTABLE: case NFT_MSG_DESTROYFLOWTABLE: if (nft_trans_flowtable_update(trans)) { - list_splice(&nft_trans_flowtable_hooks(trans), - &nft_trans_flowtable(trans)->hook_list); + nft_trans_delhook_abort(&nft_trans_flowtable_hooks(trans)); } else { nft_use_inc_restore(&trans->ctx.table->use); nft_clear(trans->ctx.net, nft_trans_flowtable(trans)); -- 2.25.1

2 1

[PATCH OLK-6.6 0/4] netfilter: Fix CVE-2026-64078
by Dong Chenchen 11 Aug '26

11 Aug '26

Fix CVE-2026-64078 Florian Westphal (4): netfilter: x_tables: allow initial table replace without emitting audit log message netfilter: x_tables: allocate hook ops while under mutex netfilter: x_tables: add and use xt_unregister_table_pre_exit netfilter: x_tables: add and use xtables_unregister_table_exit include/linux/netfilter/x_tables.h | 4 +- include/linux/netfilter_arp/arp_tables.h | 1 - include/linux/netfilter_ipv4/ip_tables.h | 1 - include/linux/netfilter_ipv6/ip6_tables.h | 1 - net/ipv4/netfilter/arp_tables.c | 53 +------ net/ipv4/netfilter/arptable_filter.c | 2 +- net/ipv4/netfilter/ip_tables.c | 59 +------- net/ipv4/netfilter/iptable_filter.c | 2 +- net/ipv4/netfilter/iptable_mangle.c | 2 +- net/ipv4/netfilter/iptable_nat.c | 6 +- net/ipv4/netfilter/iptable_raw.c | 2 +- net/ipv4/netfilter/iptable_security.c | 2 +- net/ipv6/netfilter/ip6_tables.c | 56 +------ net/ipv6/netfilter/ip6table_filter.c | 2 +- net/ipv6/netfilter/ip6table_mangle.c | 2 +- net/ipv6/netfilter/ip6table_nat.c | 6 +- net/ipv6/netfilter/ip6table_raw.c | 2 +- net/ipv6/netfilter/ip6table_security.c | 2 +- net/netfilter/x_tables.c | 177 ++++++++++++++++++---- 19 files changed, 193 insertions(+), 189 deletions(-) -- 2.25.1

2 5

[PATCH OLK-6.6] x86,fs/resctrl: Prevent out-of-bounds access while offlining CPU when SNC enabled
by Zeng Heng 11 Aug '26

11 Aug '26

From: Reinette Chatre <reinette.chatre(a)intel.com> mainline inclusion from mainline-v7.2-rc2 commit fc16126cc11d9f507130bf84ab137ee0938c900e category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16766 CVE: CVE-2026-64477 Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id… -------------------------------- The architecture updates the cpu_mask in a domain's header to track which online CPUs are associated with the domain. When this mask becomes empty the architecture initiates offline of the domain that includes calling on resctrl fs to offline the domain. If it is a monitoring domain in which LLC occupancy is tracked resctrl fs forces the limbo handler to clear all busy RMID state associated with the domain. The limbo handler always reads the current event value associated with a busy RMID irrespective of it being checked as part of regular "is it still busy" check or whether it will be forced released anyway. When reading an RMID on a system with SNC enabled the "logical RMID" is converted to the "physical RMID" and this conversion requires the NUMA node ID of the resctrl monitoring domain that is in turn determined by querying the NUMA node ID of any CPU belonging to the monitoring domain. When the monitoring domain is going offline its cpu_mask is empty causing the NUMA node ID query via cpu_to_node() to be done with "nr_cpu_ids" as argument resulting in an out-of-bounds access. Refactor the limbo handler to skip reading the RMID when the RMID will just be forced to no longer be dirty in the domain anyway. Add a safety check to the architecture's RMID reader to protect against this scenario. Fixes: e13db55b5a0d ("x86/resctrl: Introduce snc_nodes_per_l3_cache") Closes: https://sashiko.dev/#/patchset/cover.1780456704.git.reinette.chatre%40intel… Reported-by: Sashiko <sashiko-bot(a)kernel.org> Signed-off-by: Reinette Chatre <reinette.chatre(a)intel.com> Signed-off-by: Borislav Petkov (AMD) <bp(a)alien8.de> Cc: <stable(a)kernel.org> Link: https://patch.msgid.link/16137433df42f85013b2f7a53626795cbd6637b9.178102912… Conflicts: fs/resctrl/monitor.c fs/resctrl_x86/monitor.c [Redirect file paths without conflicts.] Signed-off-by: Zeng Heng <zengheng4(a)huawei.com> --- arch/x86/kernel/cpu/resctrl/monitor.c | 5 ++++ fs/resctrl_x86/monitor.c | 39 +++++++++++++++------------ 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 53d6f642f486..6ad14c9b39a5 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -257,6 +257,11 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) return -EINVAL; + if (cpumask_empty(&hdr->cpu_mask)) { + pr_warn_once("Domain %d has no CPUs\n", hdr->id); + return -EINVAL; + } + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); hw_dom = resctrl_to_arch_mon_dom(d); cpu = cpumask_any(&hdr->cpu_mask); diff --git a/fs/resctrl_x86/monitor.c b/fs/resctrl_x86/monitor.c index 0cd5476a483a..0b52db0d0f19 100644 --- a/fs/resctrl_x86/monitor.c +++ b/fs/resctrl_x86/monitor.c @@ -135,10 +135,10 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); struct rmid_entry *entry; + bool rmid_dirty = true; u32 idx, cur_idx = 1; void *arch_mon_ctx; void *arch_priv; - bool rmid_dirty; u64 val = 0; arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv; @@ -161,22 +161,27 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) break; entry = __rmid_entry(idx); - if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, entry->rmid, - QOS_L3_OCCUP_EVENT_ID, arch_priv, &val, - arch_mon_ctx)) { - rmid_dirty = true; - } else { - rmid_dirty = (val >= resctrl_rmid_realloc_threshold); - - /* - * x86's CLOSID and RMID are independent numbers, so the entry's - * CLOSID is an empty CLOSID (X86_RESCTRL_EMPTY_CLOSID). On Arm the - * RMID (PMG) extends the CLOSID (PARTID) space with bits that aren't - * used to select the configuration. It is thus necessary to track both - * CLOSID and RMID because there may be dependencies between them - * on some architectures. - */ - trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, d->hdr.id, val); + if (!force_free) { + if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, + entry->rmid, QOS_L3_OCCUP_EVENT_ID, + arch_priv, &val, arch_mon_ctx)) { + rmid_dirty = true; + } else { + rmid_dirty = (val >= resctrl_rmid_realloc_threshold); + + /* + * x86's CLOSID and RMID are independent numbers, + * so the entry's CLOSID is an empty CLOSID + * (X86_RESCTRL_EMPTY_CLOSID). On Arm the RMID + * (PMG) extends the CLOSID (PARTID) space with + * bits that aren't used to select the configuration. + * It is thus necessary to track both CLOSID and + * RMID because there may be dependencies between + * them on some architectures. + */ + trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, + d->hdr.id, val); + } } if (force_free || !rmid_dirty) { -- 2.43.0

2 1

[PATCH] erofs: fix stack-out-of-bounds in erofs_put_metabuf
by Liu Mingrui 10 Aug '26

10 Aug '26

From: Mingrui Liu <liumingrui(a)huawei.com> hulk inclusion category: bugfix bugzilla: https://atomgit.com/src-openeuler/kernel/issues/16872 -------------------------------- Commit 1d4b040dd8ee moved erofs_put_metabuf() to out_unlock, but some error paths jump there before buf is initialized, unlike mainline where buf is always set. This triggers : BUG: KASAN: stack-out-of-bounds in erofs_put_metabuf+0x54/0x158 [erofs] Call Trace: dump_backtrace+0xa0/0x128 show_stack+0x20/0x38 dump_stack_lvl+0x78/0xc8 print_address_description.constprop.0+0x84/0x3c8 print_report+0xb0/0x280 kasan_report+0x84/0xd0 __asan_load8+0x9c/0xc0 erofs_put_metabuf+0x54/0x158 [erofs] erofs_init_inode_xattrs+0x194/0x560 [erofs] erofs_getxattr+0xc8/0x238 [erofs] erofs_get_acl+0x6c/0x150 [erofs] __get_acl.part.0+0x218/0x390 vfs_get_acl+0xd0/0x150 do_get_acl+0x38/0x2a0 do_getxattr+0x9c/0x250 getxattr+0xe0/0x148 path_getxattr+0xdc/0x150 __arm64_sys_getxattr+0x60/0x80 invoke_syscall+0x64/0x178 el0_svc_common.constprop.0+0x11c/0x150 do_el0_svc+0x38/0x50 el0_svc+0x44/0x228 el0t_64_sync_handler+0x100/0x130 el0t_64_sync+0x3c8/0x3d0 Fix by zero-initializing buf at declaration so erofs_put_metabuf() can safely handle it. Fixes: 6a89c63750cb ("erofs: fix metabuf leak in inode xattr initialization") Signed-off-by: Mingrui Liu <liumingrui(a)huawei.com> --- fs/erofs/xattr.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index b270226be379..abd23e990a3a 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -34,6 +34,8 @@ static int erofs_init_inode_xattrs(struct inode *inode) struct super_block *sb = inode->i_sb; int ret = 0; + it.buf = __EROFS_BUF_INITIALIZER; + /* the most case is that xattrs of this inode are initialized. */ if (test_bit(EROFS_I_EA_INITED_BIT, &vi->flags)) { /* @@ -76,7 +78,6 @@ static int erofs_init_inode_xattrs(struct inode *inode) goto out_unlock; } - it.buf = __EROFS_BUF_INITIALIZER; erofs_init_metabuf(&it.buf, sb); it.pos = erofs_iloc(inode) + vi->inode_isize; -- 2.34.1

1 0

[PATCH OLK-6.6 0/6] bonding: Fix CVE-2026-52975
by superdcc97＠163.com 10 Aug '26

10 Aug '26

From: Dong Chenchen <dongchenchen2(a)huawei.com> Fix CVE-2026-52975 Eric Dumazet (2): bonding: 3ad: implement proper RCU rules for port->aggregator bonding: annotate data-races arcound churn variables Hangbin Liu (3): bonding: add support for per-port LACP actor priority bonding: print churn state via netlink bonding: fix NULL pointer dereference in actor_port_prio setting Tonghao Zhang (1): net: bonding: add broadcast_neighbor option for 802.3ad Documentation/networking/bonding.rst | 15 +++ drivers/net/bonding/bond_3ad.c | 131 ++++++++++++++----------- drivers/net/bonding/bond_main.c | 67 +++++++++++-- drivers/net/bonding/bond_netlink.c | 37 ++++++- drivers/net/bonding/bond_options.c | 71 ++++++++++++++ drivers/net/bonding/bond_procfs.c | 11 ++- drivers/net/bonding/bond_sysfs_slave.c | 17 ++-- include/net/bond_3ad.h | 3 +- include/net/bond_options.h | 2 + include/net/bonding.h | 3 + include/uapi/linux/if_link.h | 3 + 11 files changed, 280 insertions(+), 80 deletions(-) -- 2.43.0

2 7

[PATCH OLK-6.6 0/8] net Fix CVE-2026-45905
by superdcc97＠163.com 10 Aug '26

10 Aug '26

From: Dong Chenchen <dongchenchen2(a)huawei.com> fix CVE-2026-45905 Dong Chenchen (1): net: Use __GENKSYMS__ to fix kabi breakage of icmp Guillaume Nault (2): ipv4: Convert icmp_route_lookup() to dscp_t. ipv4: Convert ip_route_input() to dscp_t. Ido Schimmel (2): ipv4: icmp: Pass full DS field to ip_route_input() ipv4: icmp: Unmask upper DSCP bits in icmp_route_lookup() Jiayuan Chen (1): xfrm: fix ip_rt_bug race in icmp_route_lookup reverse path Peilin He (1): net/ipv4: add tracepoint for icmp_send Stanislav Fomichev (1): net: Switch to skb_dstref_steal/skb_dstref_restore for ip_route_input callers include/net/ip.h | 5 +++ include/net/route.h | 5 ++- include/trace/events/icmp.h | 67 +++++++++++++++++++++++++++++++++ net/bridge/br_netfilter_hooks.c | 8 ++-- net/ipv4/icmp.c | 50 +++++++++++++++++------- net/ipv4/ip_options.c | 8 ++-- net/ipv6/ip6_tunnel.c | 4 +- 7 files changed, 123 insertions(+), 24 deletions(-) create mode 100644 include/trace/events/icmp.h -- 2.43.0

2 9

[PATCH OLK-6.6 0/2] net: core: synchronize link-watch when carrier is queried
by superdcc97＠163.com 10 Aug '26

10 Aug '26

From: Dong Chenchen <dongchenchen2(a)huawei.com> synchronize link-watch when carrier is queried Johannes Berg (2): net: core: synchronize link-watch when carrier is queried net: sysfs: fix locking in carrier read include/linux/netdevice.h | 9 +++++++++ net/core/dev.c | 2 +- net/core/dev.h | 1 - net/core/link_watch.c | 2 +- net/core/net-sysfs.c | 17 ++++++++++++++--- net/core/rtnetlink.c | 8 ++++++++ net/ethtool/ioctl.c | 3 +++ 7 files changed, 36 insertions(+), 6 deletions(-) -- 2.43.0

2 3

Jump to page: